1 files changed, 36 insertions, 35 deletions

diff --git a/eo_tokenizer.rl b/eo_tokenizer.rl
index 1db19ee..b96a2b1 100644
--- a/eo_tokenizer.rl
+++ b/eo_tokenizer.rl
@@ -6,50 +6,33 @@
 %%{
    machine common;
 
+   access toknz->;
+   variable p toknz->p;
+   variable pe toknz->pe;
+   variable eof toknz->eof;
+
    action inc_line {
+      /* DBG("inc[%d]", toknz->cs); */
       toknz->current_line += 1;
    }
 
    action save_start_line {
+      /* DBG("save[%d] %d", toknz->cs, toknz->current_line); */
       toknz->token_start_line = toknz->current_line;
    }
 
-   cr                = '\n';
-   cr_neg            = "[^\n]";
-   ws                = "[ \t\r]";
-   newline           = cr @inc_line;
-   ignore            = (0x00..0x20 - cr)* newline?;
-
-   c_comment         = "/*" ( any | '\n' @inc_line )* :>> "*/";
-   cpp_comment       = "//" (any - cr)* newline;
-   comment           = ( c_comment | cpp_comment ) > save_start_line;
-
-}%%
-
-%%{
-   machine eo_tokenizer;
-   include common;
-
-   access toknz->;
-   variable p toknz->p;
-   variable pe toknz->pe;
-   variable eof toknz->eof;
-
    action show_comment {
-      DBG("comment %03d:%03d", toknz->token_start_line, toknz->current_line);
+      DBG("comment[%d] %03d:%03d", toknz->cs, toknz->token_start_line, toknz->current_line);
    }
 
    action show_ignore {
-      DBG("ignore %d (%d)", toknz->current_line, (toknz->te - toknz->ts));
+      DBG("ignore[%d] %d", toknz->cs, toknz->current_line);
    }
 
-   action error {
-      /* TODO find a more elegant way,
-       *  -> cant't use $err() or $!error on scanner
-       *  -> fgoto another machine to eat the line, ts is set to NULL
-       */
-      char buf[BUFSIZE];
+   action show_error {
+      DBG("error[%d]", toknz->cs);
       char *s, *d;
+      char buf[BUFSIZE];
       for (s = fpc, d = buf; (s <= toknz->pe); s++)
         {
            if ( (int)*s == 13 || (int)*s == 10)
@@ -58,14 +41,29 @@
         }
       *d = '\0';
       ERR("error line %d : %s...", toknz->current_line, buf);
-      toknz->cs = eo_tokenizer_error;
-      fbreak;
+      fbreak;  /* necessary to stop scanners */
    }
 
+   cr                = '\n';
+   cr_neg            = [^\n];
+   ws                = [ \t\r];
+   newline           = cr @inc_line;
+   ignore            = (0x00..0x20 - cr)+ newline?;
+
+   c_comment         = "/*" ( any | '\n' @inc_line )* :>> "*/";
+   cpp_comment       = "//" (any - cr)* newline;
+   comment           = ( c_comment | cpp_comment ) > save_start_line;
+
+}%%
+
+%%{
+   machine eo_tokenizer;
+   include common;
+
    main := |*
-      ignore => show_ignore;
-      comment => show_comment;
-      any => error;
+      ignore+; #=> show_ignore;
+      comment  => show_comment;
+      any      => show_error;
    *|;
 
 }%%
@@ -120,7 +118,10 @@ eo_tokenizer_walk(Eo_Tokenizer *toknz, const char *source)
         %% write exec;
 
         if ( toknz->cs == %%{ write error; }%% )
-          break;
+          {
+             ERR("wrong terminatison");
+             break;
+          }
 
         if ( toknz->ts == 0 )
           have = 0;