diff options
Diffstat (limited to 'mammoth/styles/parser')
| -rw-r--r-- | mammoth/styles/parser/tokeniser.py | 17 |
1 files changed, 4 insertions, 13 deletions
diff --git a/mammoth/styles/parser/tokeniser.py b/mammoth/styles/parser/tokeniser.py index 0f2bbe9..2e8c2c2 100644 --- a/mammoth/styles/parser/tokeniser.py +++ b/mammoth/styles/parser/tokeniser.py @@ -38,22 +38,13 @@ def _to_regex(value): _string_prefix = r"'(?:\\.|[^'])*" +_identifier_character = r"(?:[a-zA-Z\-_]|\\.)" tokenise = regex_tokeniser([ - ("identifier", r"[a-zA-Z][a-zA-Z0-9\-]*"), - ("class name", r"\.(?:[a-zA-Z0-9\-]|\\.)+"), - ("colon", r":"), - ("greater than", r">"), + ("identifier", _identifier_character + "(?:" + _identifier_character + "|[0-9])*"), + ("symbol", r":|>|=>|\^=|=|\(|\)|\[|\]|\||!|\."), ("whitespace", r"\s+"), - ("arrow", r"=>"), - ("equals", r"="), - ("open paren", r"\("), - ("close paren", r"\)"), - ("open square bracket", r"\["), - ("close square bracket", r"\]"), ("string", _string_prefix + "'"), ("unterminated string", _string_prefix), - ("integer", r"([0-9]+)"), - ("choice", r"\|"), - ("bang", r"(!)"), + ("integer", "([0-9]+)"), ]) |