take care of saved token before buffer feed

1 files changed, 16 insertions, 3 deletions

diff --git a/eo_tokenizer.rl b/eo_tokenizer.rl
index 6f45f05..a01ed32 100644
--- a/eo_tokenizer.rl
+++ b/eo_tokenizer.rl
@@ -124,6 +124,7 @@ eo_tokenizer_walk(Eo_Tokenizer *toknz, const char *source)
 
    int done = 0;
    int have = 0;
+   int offset = 0;
 
    stream = fopen(toknz->source, "r");
    if (!stream)
@@ -170,12 +171,24 @@ eo_tokenizer_walk(Eo_Tokenizer *toknz, const char *source)
           have = 0;
         else
           {
-             /* There is a prefix to preserve, shift it over. */
+             DBG("move data and pointers before buffer feed");
              have = toknz->pe - toknz->ts;
-             memmove( toknz->buf, toknz->ts, have);
-             toknz->te = toknz->buf + (toknz->te - toknz->ts);
+             offset = toknz->ts - toknz->buf;
+             memmove(toknz->buf, toknz->ts, have);
+             toknz->te -= offset;
              toknz->ts = toknz->buf;
           }
+
+        if (toknz->saved.tok != NULL)
+          {
+             if ((have == 0) || ((toknz->saved.tok - offset) < toknz->buf))
+               {
+                  WRN("reset lost saved token %p", toknz->saved.tok);
+                  toknz->saved.tok = NULL;
+               }
+             else
+               toknz->saved.tok -= offset;
+          }
      }
 
    fclose(stream);