From 310a5e5ceb5169a1c6fd67e0b22a08b6aa6a1328 Mon Sep 17 00:00:00 2001 From: Michael Williamson Date: Wed, 16 Nov 2016 22:15:05 +0000 Subject: Add tokeniser --- mammoth/styles.py | 8 --- mammoth/styles/__init__.py | 8 +++ mammoth/styles/parser/__init__.py | 0 mammoth/styles/parser/tokeniser.py | 59 +++++++++++++++++++++ setup.py | 2 +- test-requirements.txt | 1 + tests/styles/__init__.py | 0 tests/styles/parser/__init__.py | 0 tests/styles/parser/tokeniser_tests.py | 97 ++++++++++++++++++++++++++++++++++ 9 files changed, 166 insertions(+), 9 deletions(-) delete mode 100644 mammoth/styles.py create mode 100644 mammoth/styles/__init__.py create mode 100644 mammoth/styles/parser/__init__.py create mode 100644 mammoth/styles/parser/tokeniser.py create mode 100644 tests/styles/__init__.py create mode 100644 tests/styles/parser/__init__.py create mode 100644 tests/styles/parser/tokeniser_tests.py diff --git a/mammoth/styles.py b/mammoth/styles.py deleted file mode 100644 index a90012b..0000000 --- a/mammoth/styles.py +++ /dev/null @@ -1,8 +0,0 @@ -import collections - - -def style(document_matcher, html_path): - return Style(document_matcher, html_path) - - -Style = collections.namedtuple("Style", ["document_matcher", "html_path"]) diff --git a/mammoth/styles/__init__.py b/mammoth/styles/__init__.py new file mode 100644 index 0000000..a90012b --- /dev/null +++ b/mammoth/styles/__init__.py @@ -0,0 +1,8 @@ +import collections + + +def style(document_matcher, html_path): + return Style(document_matcher, html_path) + + +Style = collections.namedtuple("Style", ["document_matcher", "html_path"]) diff --git a/mammoth/styles/parser/__init__.py b/mammoth/styles/parser/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mammoth/styles/parser/tokeniser.py b/mammoth/styles/parser/tokeniser.py new file mode 100644 index 0000000..0f2bbe9 --- /dev/null +++ b/mammoth/styles/parser/tokeniser.py @@ -0,0 +1,59 @@ +import collections +import re + + +Token = collections.namedtuple("Token", ["character_index", "type", "value"]) + + +def regex_tokeniser(rules): + rules = [(token_type, _to_regex(regex)) for token_type, regex in rules] + rules.append(("unknown", re.compile("."))) + + def tokenise(value): + tokens = [] + index = 0 + while index < len(value): + for token_type, regex in rules: + match = regex.match(value, index) + if match is not None: + tokens.append(Token(index, token_type, match.group(0))) + index = match.end() + break + else: + # Should be impossible + raise Exception("Remaining: " + value[index:]) + + tokens.append(Token(index, "end", "")) + + return tokens + + return tokenise + + +def _to_regex(value): + if hasattr(value, "match"): + return value + else: + return re.compile(value) + + +_string_prefix = r"'(?:\\.|[^'])*" + +tokenise = regex_tokeniser([ + ("identifier", r"[a-zA-Z][a-zA-Z0-9\-]*"), + ("class name", r"\.(?:[a-zA-Z0-9\-]|\\.)+"), + ("colon", r":"), + ("greater than", r">"), + ("whitespace", r"\s+"), + ("arrow", r"=>"), + ("equals", r"="), + ("open paren", r"\("), + ("close paren", r"\)"), + ("open square bracket", r"\["), + ("close square bracket", r"\]"), + ("string", _string_prefix + "'"), + ("unterminated string", _string_prefix), + ("integer", r"([0-9]+)"), + ("choice", r"\|"), + ("bang", r"(!)"), +]) diff --git a/setup.py b/setup.py index f593b55..46817da 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ setup( author='Michael Williamson', author_email='mike@zwobble.org', url='http://github.com/mwilliamson/python-mammoth', - packages=['mammoth', 'mammoth.docx', 'mammoth.html', 'mammoth.style_reader', 'mammoth.writers'], + packages=['mammoth', 'mammoth.docx', 'mammoth.html', 'mammoth.style_reader', 'mammoth.styles', 'mammoth.styles.parser', 'mammoth.writers'], entry_points={ "console_scripts": [ "mammoth=mammoth.cli:main" diff --git a/test-requirements.txt b/test-requirements.txt index 7eb8e83..c4516c7 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,3 +4,4 @@ spur.local>=0.3.7,<0.4 tempman>=0.1.2,<0.2 pyflakes>=0.8.1,<0.9 nose-parameterized==0.5.0 +PyHamcrest==1.9.0 diff --git a/tests/styles/__init__.py b/tests/styles/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/styles/parser/__init__.py b/tests/styles/parser/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/styles/parser/tokeniser_tests.py b/tests/styles/parser/tokeniser_tests.py new file mode 100644 index 0000000..06fbdc3 --- /dev/null +++ b/tests/styles/parser/tokeniser_tests.py @@ -0,0 +1,97 @@ +from hamcrest import assert_that, contains, has_properties + +from mammoth.styles.parser.tokeniser import tokenise + + +def test_unknown_tokens_are_tokenised(): + assert_tokens("~", is_token("unknown", "~")) + + +def test_empty_string_is_tokenised_to_end_of_file_token(): + assert_tokens("") + + +def test_whitespace_is_tokenised(): + assert_tokens(" \t\t ", is_token("whitespace", " \t\t ")) + + +def test_identifiers_are_tokenised(): + assert_tokens("Overture", is_token("identifier", "Overture")) + + +def test_integers_are_tokenised(): + assert_tokens("123", is_token("integer", "123")) + + +def test_strings_are_tokenised(): + assert_tokens("'Tristan'", is_token("string", "'Tristan'")) + + +def test_unterminated_strings_are_tokenised(): + assert_tokens("'Tristan", is_token("unterminated string", "'Tristan")) + + +def test_arrows_are_tokenised(): + assert_tokens("=>=>", is_token("arrow", "=>"), is_token("arrow", "=>")) + + +def test_classes_are_tokenised(): + assert_tokens(".overture", is_token("class name", ".overture")) + + +def test_colons_are_tokenised(): + assert_tokens("::", is_token("colon", ":"), is_token("colon", ":")) + + +def test_greater_thans_are_tokenised(): + assert_tokens(">>", is_token("greater than", ">"), is_token("greater than", ">")) + + +def test_equals_are_tokenised(): + assert_tokens("==", is_token("equals", "="), is_token("equals", "=")) + + +def test_open_parens_are_tokenised(): + assert_tokens("((", is_token("open paren", "("), is_token("open paren", "(")) + + +def test_close_parens_are_tokenised(): + assert_tokens("))", is_token("close paren", ")"), is_token("close paren", ")")) + + +def test_open_square_brackets_are_tokenised(): + assert_tokens("[[", is_token("open square bracket", "["), is_token("open square bracket", "[")) + + +def test_close_square_brackets_are_tokenised(): + assert_tokens("]]", is_token("close square bracket", "]"), is_token("close square bracket", "]")) + + +def test_choices_are_tokenised(): + assert_tokens("||", is_token("choice", "|"), is_token("choice", "|")) + + +def test_can_tokenise_multiple_tokens(): + assert_tokens("The Magic Position", + is_token("identifier", "The"), + is_token("whitespace", " "), + is_token("identifier", "Magic"), + is_token("whitespace", " "), + is_token("identifier", "Position"), + ) + + +def assert_tokens(string, *expected): + expected = list(expected) + expected.append(is_token("end", "")) + assert_that( + tokenise(string), + contains(*expected), + ) + + +def is_token(token_type, value): + return has_properties( + type=token_type, + value=value, + ) -- cgit v1.2.3