diff options
| -rw-r--r-- | eo_tokenizer.rl | 19 |
1 files changed, 16 insertions, 3 deletions
diff --git a/eo_tokenizer.rl b/eo_tokenizer.rl index 6f45f05..a01ed32 100644 --- a/eo_tokenizer.rl +++ b/eo_tokenizer.rl @@ -124,6 +124,7 @@ eo_tokenizer_walk(Eo_Tokenizer *toknz, const char *source) int done = 0; int have = 0; + int offset = 0; stream = fopen(toknz->source, "r"); if (!stream) @@ -170,12 +171,24 @@ eo_tokenizer_walk(Eo_Tokenizer *toknz, const char *source) have = 0; else { - /* There is a prefix to preserve, shift it over. */ + DBG("move data and pointers before buffer feed"); have = toknz->pe - toknz->ts; - memmove( toknz->buf, toknz->ts, have); - toknz->te = toknz->buf + (toknz->te - toknz->ts); + offset = toknz->ts - toknz->buf; + memmove(toknz->buf, toknz->ts, have); + toknz->te -= offset; toknz->ts = toknz->buf; } + + if (toknz->saved.tok != NULL) + { + if ((have == 0) || ((toknz->saved.tok - offset) < toknz->buf)) + { + WRN("reset lost saved token %p", toknz->saved.tok); + toknz->saved.tok = NULL; + } + else + toknz->saved.tok -= offset; + } } fclose(stream); |