diff options
Diffstat (limited to 'mammoth')
| -rw-r--r-- | mammoth/styles/parser/tokeniser.py | 22 |
1 files changed, 16 insertions, 6 deletions
diff --git a/mammoth/styles/parser/tokeniser.py b/mammoth/styles/parser/tokeniser.py index 2e8c2c2..c9eb4cf 100644 --- a/mammoth/styles/parser/tokeniser.py +++ b/mammoth/styles/parser/tokeniser.py @@ -5,6 +5,16 @@ import re Token = collections.namedtuple("Token", ["character_index", "type", "value"]) +class TokenType(object): + IDENTIFIER = "identifier" + SYMBOL = "symbol" + WHITESPACE = "whitespace" + STRING = "string" + UNTERMINATED_STRING = "unterminated string" + INTEGER = "integer" + + + def regex_tokeniser(rules): rules = [(token_type, _to_regex(regex)) for token_type, regex in rules] rules.append(("unknown", re.compile("."))) @@ -41,10 +51,10 @@ _string_prefix = r"'(?:\\.|[^'])*" _identifier_character = r"(?:[a-zA-Z\-_]|\\.)" tokenise = regex_tokeniser([ - ("identifier", _identifier_character + "(?:" + _identifier_character + "|[0-9])*"), - ("symbol", r":|>|=>|\^=|=|\(|\)|\[|\]|\||!|\."), - ("whitespace", r"\s+"), - ("string", _string_prefix + "'"), - ("unterminated string", _string_prefix), - ("integer", "([0-9]+)"), + (TokenType.IDENTIFIER, _identifier_character + "(?:" + _identifier_character + "|[0-9])*"), + (TokenType.SYMBOL, r":|>|=>|\^=|=|\(|\)|\[|\]|\||!|\."), + (TokenType.WHITESPACE, r"\s+"), + (TokenType.STRING, _string_prefix + "'"), + (TokenType.UNTERMINATED_STRING, _string_prefix), + (TokenType.INTEGER, "([0-9]+)"), ]) |