From a543aa04e995a5a1708490f4b4329da014a11921 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Zurcher?= <jeremy@asynk.ch>
Date: Thu, 28 Nov 2013 11:15:42 +0100
Subject: add _eo_tokenizer_normalize_buf(), call it in
 _eo_tokenizer_token_get()

---
 data/02_classes.c |  6 +++++-
 eo_tokenizer.rl   | 46 +++++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 50 insertions(+), 2 deletions(-)

diff --git a/data/02_classes.c b/data/02_classes.c
index 04cfd29..4a66c5c 100644
--- a/data/02_classes.c
+++ b/data/02_classes.c
@@ -16,7 +16,11 @@
 };
 
 C_class_D  { 
-   /*@ This is a comment for class C_class_D. */
+   /*@ This is a    comment for class C_class_D.
+    * check
+   *check
+* check
+*check      */
 
    // haha
    inherits { Eo_Another, Evas_Common_Interface };
diff --git a/eo_tokenizer.rl b/eo_tokenizer.rl
index daba5bb..5eeb151 100644
--- a/eo_tokenizer.rl
+++ b/eo_tokenizer.rl
@@ -22,14 +22,58 @@ _eo_tokenizer_abort(Eo_Tokenizer *toknz,
 #define ABORT(toknz, ...) \
    _eo_tokenizer_abort(toknz, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__);
 
+static void _eo_tokenizer_normalize_buf(char *buf)
+{
+   int c;
+   char *s, *d;
+   Eina_Bool in_space = EINA_TRUE;
+   Eina_Bool in_newline = EINA_FALSE;
+
+   /* ' '+ -> ' '
+    * '\n' ' '* '*' ' '* -> '\n'
+    */
+   for (s = buf, d = buf; *s != '\0'; s++)
+     {
+        c = *s;
+        *d = c;
+
+        if (!in_space || (c != ' '))
+          d++;
+
+        if (c == ' ')
+          in_space = EINA_TRUE;
+        else
+          in_space = EINA_FALSE;
+
+        if (c == '\n')
+          {
+             in_newline = EINA_TRUE;
+             in_space = EINA_TRUE;
+          }
+        else if (in_newline && c == '*' )
+          {
+             in_space = EINA_TRUE;
+             in_newline = EINA_FALSE;
+             d--;
+          }
+     }
+   /* ' '+$ -> $ */
+   d--;
+   while (*d == ' ') d--;
+   d++;
+   if (d < buf) return;
+   *d = '\0';
+}
+
 static const char*
 _eo_tokenizer_token_get(Eo_Tokenizer *toknz, char *p)
 {
    if (toknz->saved.tok == NULL) ABORT(toknz, "toknz->saved.tok is NULL");
-   int l = (p - toknz->saved.tok);
    char d[BUFSIZE];
+   int l = (p - toknz->saved.tok);
    memcpy(d, toknz->saved.tok, l);
    d[l] = '\0';
+   _eo_tokenizer_normalize_buf(d);
    toknz->saved.tok = NULL;
    DBG("token : >%s<", d);
    return eina_stringshare_add(d);
-- 
cgit v1.1-2-g2b99