8 files changed, 158 insertions, 1 deletions
diff --git a/mammoth/styles.py b/mammoth/styles/__init__.py
index a90012b..a90012b 100644
--- a/mammoth/styles.py
+++ b/mammoth/styles/__init__.py
diff --git a/mammoth/styles/parser/__init__.py b/mammoth/styles/parser/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/mammoth/styles/parser/__init__.py
diff --git a/mammoth/styles/parser/tokeniser.py b/mammoth/styles/parser/tokeniser.py
new file mode 100644
index 0000000..0f2bbe9
--- /dev/null
+++ b/mammoth/styles/parser/tokeniser.py
@@ -0,0 +1,59 @@
+import collections
+import re
+
+
+Token = collections.namedtuple("Token", ["character_index", "type", "value"])
+
+
+def regex_tokeniser(rules):
+    rules = [(token_type, _to_regex(regex)) for token_type, regex in rules]
+    rules.append(("unknown", re.compile(".")))
+    
+    def tokenise(value):
+        tokens = []
+        index = 0
+        while index < len(value):
+            for token_type, regex in rules:
+                match = regex.match(value, index)
+                if match is not None:
+                    tokens.append(Token(index, token_type, match.group(0)))
+                    index = match.end()
+                    break
+            else:
+                # Should be impossible
+                raise Exception("Remaining: " + value[index:])
+
+        tokens.append(Token(index, "end", ""))
+
+        return tokens
+
+    return tokenise
+    
+
+def _to_regex(value):
+    if hasattr(value, "match"):
+        return value
+    else:
+        return re.compile(value)
+
+
+_string_prefix = r"'(?:\\.|[^'])*"
+
+tokenise = regex_tokeniser([
+    ("identifier", r"[a-zA-Z][a-zA-Z0-9\-]*"),
+    ("class name", r"\.(?:[a-zA-Z0-9\-]|\\.)+"),
+    ("colon", r":"),
+    ("greater than", r">"),
+    ("whitespace", r"\s+"),
+    ("arrow", r"=>"),
+    ("equals", r"="),
+    ("open paren", r"\("),
+    ("close paren", r"\)"),
+    ("open square bracket", r"\["),
+    ("close square bracket", r"\]"),
+    ("string", _string_prefix + "'"),
+    ("unterminated string", _string_prefix),
+    ("integer", r"([0-9]+)"),
+    ("choice", r"\|"),
+    ("bang", r"(!)"),
+])
diff --git a/setup.py b/setup.py
index f593b55..46817da 100644
--- a/setup.py
+++ b/setup.py
@@ -24,7 +24,7 @@ setup(
     author='Michael Williamson',
     author_email='mike@zwobble.org',
     url='http://github.com/mwilliamson/python-mammoth',
-    packages=['mammoth', 'mammoth.docx', 'mammoth.html', 'mammoth.style_reader', 'mammoth.writers'],
+    packages=['mammoth', 'mammoth.docx', 'mammoth.html', 'mammoth.style_reader', 'mammoth.styles', 'mammoth.styles.parser', 'mammoth.writers'],
     entry_points={
         "console_scripts": [
             "mammoth=mammoth.cli:main"
diff --git a/test-requirements.txt b/test-requirements.txt
index 7eb8e83..c4516c7 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -4,3 +4,4 @@ spur.local>=0.3.7,<0.4
 tempman>=0.1.2,<0.2
 pyflakes>=0.8.1,<0.9
 nose-parameterized==0.5.0
+PyHamcrest==1.9.0
diff --git a/tests/styles/__init__.py b/tests/styles/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tests/styles/__init__.py
diff --git a/tests/styles/parser/__init__.py b/tests/styles/parser/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tests/styles/parser/__init__.py
diff --git a/tests/styles/parser/tokeniser_tests.py b/tests/styles/parser/tokeniser_tests.py
new file mode 100644
index 0000000..06fbdc3
--- /dev/null
+++ b/tests/styles/parser/tokeniser_tests.py
@@ -0,0 +1,97 @@
+from hamcrest import assert_that, contains, has_properties
+
+from mammoth.styles.parser.tokeniser import tokenise
+
+
+def test_unknown_tokens_are_tokenised():
+    assert_tokens("~", is_token("unknown", "~"))
+
+
+def test_empty_string_is_tokenised_to_end_of_file_token():
+    assert_tokens("")
+
+
+def test_whitespace_is_tokenised():
+    assert_tokens(" \t\t  ", is_token("whitespace", " \t\t  "))
+
+
+def test_identifiers_are_tokenised():
+    assert_tokens("Overture", is_token("identifier", "Overture"))
+
+
+def test_integers_are_tokenised():
+    assert_tokens("123", is_token("integer", "123"))
+
+
+def test_strings_are_tokenised():
+    assert_tokens("'Tristan'", is_token("string", "'Tristan'"))
+
+
+def test_unterminated_strings_are_tokenised():
+    assert_tokens("'Tristan", is_token("unterminated string", "'Tristan"))
+
+
+def test_arrows_are_tokenised():
+    assert_tokens("=>=>", is_token("arrow", "=>"), is_token("arrow", "=>"))
+
+
+def test_classes_are_tokenised():
+    assert_tokens(".overture", is_token("class name", ".overture"))
+
+
+def test_colons_are_tokenised():
+    assert_tokens("::", is_token("colon", ":"), is_token("colon", ":"))
+
+
+def test_greater_thans_are_tokenised():
+    assert_tokens(">>", is_token("greater than", ">"), is_token("greater than", ">"))
+
+
+def test_equals_are_tokenised():
+    assert_tokens("==", is_token("equals", "="), is_token("equals", "="))
+
+
+def test_open_parens_are_tokenised():
+    assert_tokens("((", is_token("open paren", "("), is_token("open paren", "("))
+
+
+def test_close_parens_are_tokenised():
+    assert_tokens("))", is_token("close paren", ")"), is_token("close paren", ")"))
+
+
+def test_open_square_brackets_are_tokenised():
+    assert_tokens("[[", is_token("open square bracket", "["), is_token("open square bracket", "["))
+
+
+def test_close_square_brackets_are_tokenised():
+    assert_tokens("]]", is_token("close square bracket", "]"), is_token("close square bracket", "]"))
+
+
+def test_choices_are_tokenised():
+    assert_tokens("||", is_token("choice", "|"), is_token("choice", "|"))
+
+
+def test_can_tokenise_multiple_tokens():
+    assert_tokens("The Magic Position",
+        is_token("identifier", "The"),
+        is_token("whitespace", " "),
+        is_token("identifier", "Magic"),
+        is_token("whitespace", " "),
+        is_token("identifier", "Position"),
+    )
+
+
+def assert_tokens(string, *expected):
+    expected = list(expected)
+    expected.append(is_token("end", ""))
+    assert_that(
+        tokenise(string),
+        contains(*expected),
+    )
+
+
+def is_token(token_type, value):
+    return has_properties(
+        type=token_type,
+        value=value,
+    )