From 1b3d07bca1a8237052c0a81ff78b100662c0c19d Mon Sep 17 00:00:00 2001 From: Michael Williamson Date: Thu, 24 Nov 2016 20:58:49 +0000 Subject: Update tokeniser --- mammoth/styles/parser/tokeniser.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) (limited to 'mammoth/styles') diff --git a/mammoth/styles/parser/tokeniser.py b/mammoth/styles/parser/tokeniser.py index 0f2bbe9..2e8c2c2 100644 --- a/mammoth/styles/parser/tokeniser.py +++ b/mammoth/styles/parser/tokeniser.py @@ -38,22 +38,13 @@ def _to_regex(value): _string_prefix = r"'(?:\\.|[^'])*" +_identifier_character = r"(?:[a-zA-Z\-_]|\\.)" tokenise = regex_tokeniser([ - ("identifier", r"[a-zA-Z][a-zA-Z0-9\-]*"), - ("class name", r"\.(?:[a-zA-Z0-9\-]|\\.)+"), - ("colon", r":"), - ("greater than", r">"), + ("identifier", _identifier_character + "(?:" + _identifier_character + "|[0-9])*"), + ("symbol", r":|>|=>|\^=|=|\(|\)|\[|\]|\||!|\."), ("whitespace", r"\s+"), - ("arrow", r"=>"), - ("equals", r"="), - ("open paren", r"\("), - ("close paren", r"\)"), - ("open square bracket", r"\["), - ("close square bracket", r"\]"), ("string", _string_prefix + "'"), ("unterminated string", _string_prefix), - ("integer", r"([0-9]+)"), - ("choice", r"\|"), - ("bang", r"(!)"), + ("integer", "([0-9]+)"), ]) -- cgit v1.2.3