From be2ece6ff7fbd90ddc42c38cab46250a31ad20e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Zurcher?= <jeremy@asynk.ch>
Date: Sat, 23 Nov 2013 17:15:19 +0000
Subject: add some tokens in common

---
 eo_tokenizer.rl | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/eo_tokenizer.rl b/eo_tokenizer.rl
index 8b2118d..94e8a50 100644
--- a/eo_tokenizer.rl
+++ b/eo_tokenizer.rl
@@ -83,6 +83,16 @@ _eo_tokenizer_token_get(Eo_Tokenizer *toknz, char *p)
    cpp_comment       = "//" (any - cr)* newline;
    comment           = ( c_comment | cpp_comment ) > save_line;
 
+   alnum_u           = alnum | '_';
+   alpha_u           = alpha | '_';
+
+   end_statement     = ';';
+   begin_def         = '{';
+   end_def           = '}';
+   begin_list        = '{';
+   end_list          = '}';
+   list_separator    = ',';
+
 }%%
 
 %%{
-- 
cgit v1.1-2-g2b99