From 77694e40515b98b3d231dd4e0e19e80575539c58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Zurcher?= Date: Sun, 24 Nov 2013 16:30:40 +0000 Subject: take care of saved token before buffer feed --- eo_tokenizer.rl | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/eo_tokenizer.rl b/eo_tokenizer.rl index 6f45f05..a01ed32 100644 --- a/eo_tokenizer.rl +++ b/eo_tokenizer.rl @@ -124,6 +124,7 @@ eo_tokenizer_walk(Eo_Tokenizer *toknz, const char *source) int done = 0; int have = 0; + int offset = 0; stream = fopen(toknz->source, "r"); if (!stream) @@ -170,12 +171,24 @@ eo_tokenizer_walk(Eo_Tokenizer *toknz, const char *source) have = 0; else { - /* There is a prefix to preserve, shift it over. */ + DBG("move data and pointers before buffer feed"); have = toknz->pe - toknz->ts; - memmove( toknz->buf, toknz->ts, have); - toknz->te = toknz->buf + (toknz->te - toknz->ts); + offset = toknz->ts - toknz->buf; + memmove(toknz->buf, toknz->ts, have); + toknz->te -= offset; toknz->ts = toknz->buf; } + + if (toknz->saved.tok != NULL) + { + if ((have == 0) || ((toknz->saved.tok - offset) < toknz->buf)) + { + WRN("reset lost saved token %p", toknz->saved.tok); + toknz->saved.tok = NULL; + } + else + toknz->saved.tok -= offset; + } } fclose(stream); -- cgit v1.1-2-g2b99