diff options
| author | Jérémy Zurcher <jeremy@asynk.ch> | 2013-11-23 18:15:19 +0100 |
|---|---|---|
| committer | Jérémy Zurcher <jeremy@asynk.ch> | 2013-11-23 18:15:19 +0100 |
| commit | be2ece6ff7fbd90ddc42c38cab46250a31ad20e4 (patch) | |
| tree | 04575473cc327a04f97a98a0bc9e3d5588787b79 | |
| parent | c7a85982b468d326d05eeaa21669e019803942a6 (diff) | |
| download | eo_tokenizer-be2ece6ff7fbd90ddc42c38cab46250a31ad20e4.zip eo_tokenizer-be2ece6ff7fbd90ddc42c38cab46250a31ad20e4.tar.gz | |
add some tokens in common
| -rw-r--r-- | eo_tokenizer.rl | 10 |
1 files changed, 10 insertions, 0 deletions
diff --git a/eo_tokenizer.rl b/eo_tokenizer.rl index 8b2118d..94e8a50 100644 --- a/eo_tokenizer.rl +++ b/eo_tokenizer.rl @@ -83,6 +83,16 @@ _eo_tokenizer_token_get(Eo_Tokenizer *toknz, char *p) cpp_comment = "//" (any - cr)* newline; comment = ( c_comment | cpp_comment ) > save_line; + alnum_u = alnum | '_'; + alpha_u = alpha | '_'; + + end_statement = ';'; + begin_def = '{'; + end_def = '}'; + begin_list = '{'; + end_list = '}'; + list_separator = ','; + }%% %%{ |