diff options
| author | Jérémy Zurcher <jeremy@asynk.ch> | 2013-11-27 17:10:14 +0100 |
|---|---|---|
| committer | Jérémy Zurcher <jeremy@asynk.ch> | 2013-11-27 17:10:14 +0100 |
| commit | 867d4408bd8cd8e3a752ab88c4157e5a0a8f72dc (patch) | |
| tree | a3a1413a7e785ca84f5a09d01b6ddf304ce544d0 | |
| parent | d9e669ac06ecb55948bc6fa7a5ed568f746be4e0 (diff) | |
| download | eo_tokenizer-867d4408bd8cd8e3a752ab88c4157e5a0a8f72dc.zip eo_tokenizer-867d4408bd8cd8e3a752ab88c4157e5a0a8f72dc.tar.gz | |
clean up ignore token, use cr token
| -rw-r--r-- | eo_tokenizer.rl | 8 |
1 files changed, 4 insertions, 4 deletions
diff --git a/eo_tokenizer.rl b/eo_tokenizer.rl index 3a10e8d..b995b13 100644 --- a/eo_tokenizer.rl +++ b/eo_tokenizer.rl @@ -146,15 +146,15 @@ _eo_tokenizer_accessor_get(Eo_Tokenizer *toknz, Eo_Accessor_Type type) cr_neg = [^\n]; ws = [ \t\r]; newline = cr @inc_line; - ignore = (0x00..0x20 - cr)+ newline?; + ignore = (0x00..0x20 - cr) | newline; alnum_u = alnum | '_'; alpha_u = alpha | '_'; ident = alpha+ >save_fpc (alnum | '_' )+; - eo_comment = "/*@" ignore* alnum_u >save_fpc ( any | '\n' @inc_line )* :>> "*/"; - c_comment = "/*" ( any | '\n' @inc_line )* :>> "*/"; - cpp_comment = "//" (any - cr)* newline; + eo_comment = "/*@" ignore* alnum_u >save_fpc ( any | cr @inc_line )* :>> "*/"; + c_comment = "/*" ( any | cr @inc_line )* :>> "*/"; + cpp_comment = "//" (any - cr )* newline; comment = ( c_comment | cpp_comment ) > save_line; end_statement = ';'; |