diff options
Diffstat (limited to 'eo_tokenizer.rl')
| -rw-r--r-- | eo_tokenizer.rl | 46 |
1 files changed, 45 insertions, 1 deletions
diff --git a/eo_tokenizer.rl b/eo_tokenizer.rl index daba5bb..5eeb151 100644 --- a/eo_tokenizer.rl +++ b/eo_tokenizer.rl @@ -22,14 +22,58 @@ _eo_tokenizer_abort(Eo_Tokenizer *toknz, #define ABORT(toknz, ...) \ _eo_tokenizer_abort(toknz, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__); +static void _eo_tokenizer_normalize_buf(char *buf) +{ + int c; + char *s, *d; + Eina_Bool in_space = EINA_TRUE; + Eina_Bool in_newline = EINA_FALSE; + + /* ' '+ -> ' ' + * '\n' ' '* '*' ' '* -> '\n' + */ + for (s = buf, d = buf; *s != '\0'; s++) + { + c = *s; + *d = c; + + if (!in_space || (c != ' ')) + d++; + + if (c == ' ') + in_space = EINA_TRUE; + else + in_space = EINA_FALSE; + + if (c == '\n') + { + in_newline = EINA_TRUE; + in_space = EINA_TRUE; + } + else if (in_newline && c == '*' ) + { + in_space = EINA_TRUE; + in_newline = EINA_FALSE; + d--; + } + } + /* ' '+$ -> $ */ + d--; + while (*d == ' ') d--; + d++; + if (d < buf) return; + *d = '\0'; +} + static const char* _eo_tokenizer_token_get(Eo_Tokenizer *toknz, char *p) { if (toknz->saved.tok == NULL) ABORT(toknz, "toknz->saved.tok is NULL"); - int l = (p - toknz->saved.tok); char d[BUFSIZE]; + int l = (p - toknz->saved.tok); memcpy(d, toknz->saved.tok, l); d[l] = '\0'; + _eo_tokenizer_normalize_buf(d); toknz->saved.tok = NULL; DBG("token : >%s<", d); return eina_stringshare_add(d); |