diff options
| -rw-r--r-- | eo_tokenizer.rl | 10 |
1 files changed, 10 insertions, 0 deletions
diff --git a/eo_tokenizer.rl b/eo_tokenizer.rl index 8b2118d..94e8a50 100644 --- a/eo_tokenizer.rl +++ b/eo_tokenizer.rl @@ -83,6 +83,16 @@ _eo_tokenizer_token_get(Eo_Tokenizer *toknz, char *p) cpp_comment = "//" (any - cr)* newline; comment = ( c_comment | cpp_comment ) > save_line; + alnum_u = alnum | '_'; + alpha_u = alpha | '_'; + + end_statement = ';'; + begin_def = '{'; + end_def = '}'; + begin_list = '{'; + end_list = '}'; + list_separator = ','; + }%% %%{ |