#include #include #include "eo_tokenizer.h" static void _eo_tokenizer_abort(Eo_Tokenizer *toknz, const char *file, const char* fct, int line, const char *fmt, ...) { va_list ap; va_start (ap, fmt); eina_log_vprint(_eo_tokenizer_log_dom, EINA_LOG_LEVEL_ERR, file, fct, line, fmt, ap); va_end(ap); eo_tokenizer_dump(toknz); exit(EXIT_FAILURE); } #define ABORT(toknz, ...) \ _eo_tokenizer_abort(toknz, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__); static const char* _eo_tokenizer_token_get(Eo_Tokenizer *toknz, char *p) { if (toknz->saved.tok == NULL) ABORT(toknz, "toknz->saved.tok is NULL"); int l = (p - toknz->saved.tok); char d[BUFSIZE]; memcpy(d, toknz->saved.tok, l); d[l] = '\0'; toknz->saved.tok = NULL; DBG("token : >%s<", d); return eina_stringshare_add(d); } %%{ machine common; access toknz->; variable p toknz->p; variable pe toknz->pe; variable eof toknz->eof; action inc_line { toknz->current_line += 1; DBG("inc[%d] %d", toknz->cs, toknz->current_line); } action save_line { toknz->saved.line = toknz->current_line; DBG("save[%d] %d", toknz->cs, toknz->current_line); } action save_fpc { toknz->saved.tok = fpc; DBG("save[%d] %d", toknz->cs, fpc); } action show_comment { DBG("comment[%d] %03d:%03d", toknz->cs, toknz->saved.line, toknz->current_line); } action show_ignore { DBG("ignore[%d] %d", toknz->cs, toknz->current_line); } action show_error { DBG("error[%d]", toknz->cs); char *s, *d; char buf[BUFSIZE]; for (s = fpc, d = buf; (s <= toknz->pe); s++) { if ( (int)*s == 13 || (int)*s == 10) break; *d++ = *s; } *d = '\0'; ERR("error line %d : %s...", toknz->current_line, buf); fbreak; /* necessary to stop scanners */ } cr = '\n'; cr_neg = [^\n]; ws = [ \t\r]; newline = cr @inc_line; ignore = (0x00..0x20 - cr)+ newline?; c_comment = "/*" ( any | '\n' @inc_line )* :>> "*/"; cpp_comment = "//" (any - cr)* newline; comment = ( c_comment | cpp_comment ) > save_line; alnum_u = alnum | '_'; alpha_u = alpha | '_'; end_statement = ';'; begin_def = '{'; end_def = '}'; begin_list = '{'; end_list = '}'; list_separator = ','; }%% %%{ machine eo_tokenizer; include common; main := |* ignore+; #=> show_ignore; comment => show_comment; any => show_error; *|; }%% %% write data; Eina_Bool eo_tokenizer_walk(Eo_Tokenizer *toknz, const char *source) { INF("tokenize %s...", source); toknz->source = eina_stringshare_add(source); FILE *stream; int done = 0; int have = 0; int offset = 0; stream = fopen(toknz->source, "r"); if (!stream) { ERR("unable to read in %s", toknz->source); return EINA_FALSE; } %% write init; while (!done) { int len; int space; toknz->p = toknz->buf + have; space = BUFSIZE - have; if (space == 0) { fclose(stream); ABORT(toknz, "out of buffer space"); } len = fread(toknz->p, 1, space, stream); if (len == 0) break; toknz->pe = toknz->p + len; if (len < space) { toknz->eof = toknz->pe; done = 1; } %% write exec; if ( toknz->cs == %%{ write error; }%% ) { ERR("wrong terminatison"); break; } if ( toknz->ts == 0 ) have = 0; else { DBG("move data and pointers before buffer feed"); have = toknz->pe - toknz->ts; offset = toknz->ts - toknz->buf; memmove(toknz->buf, toknz->ts, have); toknz->te -= offset; toknz->ts = toknz->buf; } if (toknz->saved.tok != NULL) { if ((have == 0) || ((toknz->saved.tok - offset) < toknz->buf)) { WRN("reset lost saved token %p", toknz->saved.tok); toknz->saved.tok = NULL; } else toknz->saved.tok -= offset; } } fclose(stream); return EINA_TRUE; } Eo_Tokenizer* eo_tokenizer_get() { Eo_Tokenizer *toknz = calloc(1, sizeof(Eo_Tokenizer)); if (!toknz) return NULL; toknz->ts = NULL; toknz->te = NULL; /* toknz->top = 0; */ toknz->source = NULL; toknz->max_nesting = 10; toknz->current_line = 1; toknz->current_nesting = 0; toknz->saved.tok = NULL; toknz->saved.line = 0; toknz->classes = NULL; return toknz; } void eo_tokenizer_dump(Eo_Tokenizer *toknz) { fprintf(stderr, " toknz[%d] n:%d l:%d p:%d pe:%d ts:%d te:%d act:%d\n", toknz->cs, toknz->current_nesting, toknz->current_line, (toknz->p - toknz->buf), (toknz->pe - toknz->buf), toknz->ts, toknz->te, toknz->act); } void eo_tokenizer_free(Eo_Tokenizer *toknz) { if (toknz->source) eina_stringshare_del(toknz->source); free(toknz); }