#include #include #include "eo_tokenizer.h" %%{ machine common; access toknz->; variable p toknz->p; variable pe toknz->pe; variable eof toknz->eof; action inc_line { /* DBG("inc[%d]", toknz->cs); */ toknz->current_line += 1; } action save_start_line { /* DBG("save[%d] %d", toknz->cs, toknz->current_line); */ toknz->token_start_line = toknz->current_line; } action show_comment { DBG("comment[%d] %03d:%03d", toknz->cs, toknz->token_start_line, toknz->current_line); } action show_ignore { DBG("ignore[%d] %d", toknz->cs, toknz->current_line); } action show_error { DBG("error[%d]", toknz->cs); char *s, *d; char buf[BUFSIZE]; for (s = fpc, d = buf; (s <= toknz->pe); s++) { if ( (int)*s == 13 || (int)*s == 10) break; *d++ = *s; } *d = '\0'; ERR("error line %d : %s...", toknz->current_line, buf); fbreak; /* necessary to stop scanners */ } cr = '\n'; cr_neg = [^\n]; ws = [ \t\r]; newline = cr @inc_line; ignore = (0x00..0x20 - cr)+ newline?; c_comment = "/*" ( any | '\n' @inc_line )* :>> "*/"; cpp_comment = "//" (any - cr)* newline; comment = ( c_comment | cpp_comment ) > save_start_line; }%% %%{ machine eo_tokenizer; include common; main := |* ignore+; #=> show_ignore; comment => show_comment; any => show_error; *|; }%% %% write data; Eina_Bool eo_tokenizer_walk(Eo_Tokenizer *toknz, const char *source) { INF("tokenize %s...", source); toknz->source = eina_stringshare_add(source); FILE *stream; int done = 0; int have = 0; stream = fopen(toknz->source, "r"); if (!stream) { ERR("unable to read in %s", toknz->source); return EINA_FALSE; } %% write init; while (!done) { int len; int space; toknz->p = toknz->buf + have; space = BUFSIZE - have; if (space == 0) { ERR("out of buffer space"); fclose(stream); exit(EXIT_FAILURE); } len = fread(toknz->p, 1, space, stream); if (len == 0) break; toknz->pe = toknz->p + len; if (len < space) { toknz->eof = toknz->pe; done = 1; } %% write exec; if ( toknz->cs == %%{ write error; }%% ) { ERR("wrong terminatison"); break; } if ( toknz->ts == 0 ) have = 0; else { /* There is a prefix to preserve, shift it over. */ have = toknz->pe - toknz->ts; memmove( toknz->buf, toknz->ts, have); toknz->te = toknz->buf + (toknz->te - toknz->ts); toknz->ts = toknz->buf; } } fclose(stream); return EINA_TRUE; } Eo_Tokenizer* eo_tokenizer_get() { Eo_Tokenizer *toknz = calloc(1, sizeof(Eo_Tokenizer)); if (!toknz) return NULL; toknz->ts = NULL; toknz->te = NULL; /* toknz->top = 0; */ toknz->source = NULL; toknz->max_nesting = 10; toknz->current_line = 1; toknz->current_nesting = 0; toknz->token_start_line = 0; return toknz; } void eo_tokenizer_dump(Eo_Tokenizer *toknz) { fprintf(stderr, " toknz[%d] n:%d l:%d p:%d pe:%d ts:%d te:%d act:%d\n", toknz->cs, toknz->current_nesting, toknz->current_line, (toknz->p - toknz->buf), (toknz->pe - toknz->buf), toknz->ts, toknz->te, toknz->act); } void eo_tokenizer_free(Eo_Tokenizer *toknz) { if (toknz->source) eina_stringshare_del(toknz->source); free(toknz); }