diff options
| -rw-r--r-- | eo_tokenizer.rl | 11 |
1 files changed, 6 insertions, 5 deletions
diff --git a/eo_tokenizer.rl b/eo_tokenizer.rl index ee66d05..6303cd9 100644 --- a/eo_tokenizer.rl +++ b/eo_tokenizer.rl @@ -101,17 +101,18 @@ _eo_tokenizer_class_get(Eo_Tokenizer *toknz, char *p) newline = cr @inc_line; ignore = (0x00..0x20 - cr)+ newline?; + alnum_u = alnum | '_'; + alpha_u = alpha | '_'; + ident = alpha+ >save_fpc (alnum | '_' )+; + eo_comment = "/*@" ignore* %save_fpc ( any | '\n' @inc_line )* :>> "*/"; c_comment = "/*" ( any | '\n' @inc_line )* :>> "*/"; cpp_comment = "//" (any - cr)* newline; comment = ( c_comment | cpp_comment ) > save_line; - alnum_u = alnum | '_'; - alpha_u = alpha | '_'; - ident = alpha+ >save_fpc (alnum | '_' )+; - + end_statement = ';'; begin_def = '{'; - end_def = '};'; + end_def = '}' end_statement; begin_list = '('; end_list = ')'; list_separator = ','; |