add _eo_tokenizer_normalize_buf(), call it in _eo_tokenizer_token_get()

2 files changed, 50 insertions, 2 deletions

diff --git a/data/02_classes.c b/data/02_classes.c
index 04cfd29..4a66c5c 100644
--- a/data/02_classes.c
+++ b/data/02_classes.c
@@ -16,7 +16,11 @@
 };

 

 C_class_D  { 

-   /*@ This is a comment for class C_class_D. */

+   /*@ This is a    comment for class C_class_D.

+    * check

+   *check

+* check

+*check      */

 

    // haha

    inherits { Eo_Another, Evas_Common_Interface };

diff --git a/eo_tokenizer.rl b/eo_tokenizer.rl
index daba5bb..5eeb151 100644
--- a/eo_tokenizer.rl
+++ b/eo_tokenizer.rl
@@ -22,14 +22,58 @@ _eo_tokenizer_abort(Eo_Tokenizer *toknz,
 #define ABORT(toknz, ...) \
    _eo_tokenizer_abort(toknz, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__);
 
+static void _eo_tokenizer_normalize_buf(char *buf)
+{
+   int c;
+   char *s, *d;
+   Eina_Bool in_space = EINA_TRUE;
+   Eina_Bool in_newline = EINA_FALSE;
+
+   /* ' '+ -> ' '
+    * '\n' ' '* '*' ' '* -> '\n'
+    */
+   for (s = buf, d = buf; *s != '\0'; s++)
+     {
+        c = *s;
+        *d = c;
+
+        if (!in_space || (c != ' '))
+          d++;
+
+        if (c == ' ')
+          in_space = EINA_TRUE;
+        else
+          in_space = EINA_FALSE;
+
+        if (c == '\n')
+          {
+             in_newline = EINA_TRUE;
+             in_space = EINA_TRUE;
+          }
+        else if (in_newline && c == '*' )
+          {
+             in_space = EINA_TRUE;
+             in_newline = EINA_FALSE;
+             d--;
+          }
+     }
+   /* ' '+$ -> $ */
+   d--;
+   while (*d == ' ') d--;
+   d++;
+   if (d < buf) return;
+   *d = '\0';
+}
+
 static const char*
 _eo_tokenizer_token_get(Eo_Tokenizer *toknz, char *p)
 {
    if (toknz->saved.tok == NULL) ABORT(toknz, "toknz->saved.tok is NULL");
-   int l = (p - toknz->saved.tok);
    char d[BUFSIZE];
+   int l = (p - toknz->saved.tok);
    memcpy(d, toknz->saved.tok, l);
    d[l] = '\0';
+   _eo_tokenizer_normalize_buf(d);
    toknz->saved.tok = NULL;
    DBG("token : >%s<", d);
    return eina_stringshare_add(d);