diff options
| -rw-r--r-- | eo_tokenizer.rl | 71 |
1 files changed, 36 insertions, 35 deletions
diff --git a/eo_tokenizer.rl b/eo_tokenizer.rl index 1db19ee..b96a2b1 100644 --- a/eo_tokenizer.rl +++ b/eo_tokenizer.rl @@ -6,50 +6,33 @@ %%{ machine common; + access toknz->; + variable p toknz->p; + variable pe toknz->pe; + variable eof toknz->eof; + action inc_line { + /* DBG("inc[%d]", toknz->cs); */ toknz->current_line += 1; } action save_start_line { + /* DBG("save[%d] %d", toknz->cs, toknz->current_line); */ toknz->token_start_line = toknz->current_line; } - cr = '\n'; - cr_neg = "[^\n]"; - ws = "[ \t\r]"; - newline = cr @inc_line; - ignore = (0x00..0x20 - cr)* newline?; - - c_comment = "/*" ( any | '\n' @inc_line )* :>> "*/"; - cpp_comment = "//" (any - cr)* newline; - comment = ( c_comment | cpp_comment ) > save_start_line; - -}%% - -%%{ - machine eo_tokenizer; - include common; - - access toknz->; - variable p toknz->p; - variable pe toknz->pe; - variable eof toknz->eof; - action show_comment { - DBG("comment %03d:%03d", toknz->token_start_line, toknz->current_line); + DBG("comment[%d] %03d:%03d", toknz->cs, toknz->token_start_line, toknz->current_line); } action show_ignore { - DBG("ignore %d (%d)", toknz->current_line, (toknz->te - toknz->ts)); + DBG("ignore[%d] %d", toknz->cs, toknz->current_line); } - action error { - /* TODO find a more elegant way, - * -> cant't use $err() or $!error on scanner - * -> fgoto another machine to eat the line, ts is set to NULL - */ - char buf[BUFSIZE]; + action show_error { + DBG("error[%d]", toknz->cs); char *s, *d; + char buf[BUFSIZE]; for (s = fpc, d = buf; (s <= toknz->pe); s++) { if ( (int)*s == 13 || (int)*s == 10) @@ -58,14 +41,29 @@ } *d = '\0'; ERR("error line %d : %s...", toknz->current_line, buf); - toknz->cs = eo_tokenizer_error; - fbreak; + fbreak; /* necessary to stop scanners */ } + cr = '\n'; + cr_neg = [^\n]; + ws = [ \t\r]; + newline = cr @inc_line; + ignore = (0x00..0x20 - cr)+ newline?; + + c_comment = "/*" ( any | '\n' @inc_line )* :>> "*/"; + cpp_comment = "//" (any - cr)* newline; + comment = ( c_comment | cpp_comment ) > save_start_line; + +}%% + +%%{ + machine eo_tokenizer; + include common; + main := |* - ignore => show_ignore; - comment => show_comment; - any => error; + ignore+; #=> show_ignore; + comment => show_comment; + any => show_error; *|; }%% @@ -120,7 +118,10 @@ eo_tokenizer_walk(Eo_Tokenizer *toknz, const char *source) %% write exec; if ( toknz->cs == %%{ write error; }%% ) - break; + { + ERR("wrong terminatison"); + break; + } if ( toknz->ts == 0 ) have = 0; |