1 files changed, 13 insertions, 0 deletions
diff --git a/eo_tokenizer.rl b/eo_tokenizer.rl
index 8a1f57a..8b2118d 100644
--- a/eo_tokenizer.rl
+++ b/eo_tokenizer.rl
@@ -14,6 +14,19 @@ _eo_tokenizer_abort(Eo_Tokenizer *toknz,
 }
 #define ABORT(toknz, msg) _eo_tokenizer_abort(toknz, __FILE__, __FUNCTION__, __LINE__, msg);
 
+static const char*
+_eo_tokenizer_token_get(Eo_Tokenizer *toknz, char *p)
+{
+   if (toknz->saved.tok == NULL) ABORT(toknz, "toknz->saved.tok is NULL");
+   int l = (p - toknz->saved.tok);
+   char d[BUFSIZE];
+   memcpy(d, toknz->saved.tok, l);
+   d[l] = '\0';
+   toknz->saved.tok = NULL;
+   ERR("token : >%s<", d);
+   return eina_stringshare_add(d);
+}
+
 %%{
    machine common;