Add tokeniser

3 files changed, 97 insertions, 0 deletions

diff --git a/tests/styles/__init__.py b/tests/styles/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tests/styles/__init__.py
diff --git a/tests/styles/parser/__init__.py b/tests/styles/parser/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tests/styles/parser/__init__.py
diff --git a/tests/styles/parser/tokeniser_tests.py b/tests/styles/parser/tokeniser_tests.py
new file mode 100644
index 0000000..06fbdc3
--- /dev/null
+++ b/tests/styles/parser/tokeniser_tests.py
@@ -0,0 +1,97 @@
+from hamcrest import assert_that, contains, has_properties
+
+from mammoth.styles.parser.tokeniser import tokenise
+
+
+def test_unknown_tokens_are_tokenised():
+    assert_tokens("~", is_token("unknown", "~"))
+
+
+def test_empty_string_is_tokenised_to_end_of_file_token():
+    assert_tokens("")
+
+
+def test_whitespace_is_tokenised():
+    assert_tokens(" \t\t  ", is_token("whitespace", " \t\t  "))
+
+
+def test_identifiers_are_tokenised():
+    assert_tokens("Overture", is_token("identifier", "Overture"))
+
+
+def test_integers_are_tokenised():
+    assert_tokens("123", is_token("integer", "123"))
+
+
+def test_strings_are_tokenised():
+    assert_tokens("'Tristan'", is_token("string", "'Tristan'"))
+
+
+def test_unterminated_strings_are_tokenised():
+    assert_tokens("'Tristan", is_token("unterminated string", "'Tristan"))
+
+
+def test_arrows_are_tokenised():
+    assert_tokens("=>=>", is_token("arrow", "=>"), is_token("arrow", "=>"))
+
+
+def test_classes_are_tokenised():
+    assert_tokens(".overture", is_token("class name", ".overture"))
+
+
+def test_colons_are_tokenised():
+    assert_tokens("::", is_token("colon", ":"), is_token("colon", ":"))
+
+
+def test_greater_thans_are_tokenised():
+    assert_tokens(">>", is_token("greater than", ">"), is_token("greater than", ">"))
+
+
+def test_equals_are_tokenised():
+    assert_tokens("==", is_token("equals", "="), is_token("equals", "="))
+
+
+def test_open_parens_are_tokenised():
+    assert_tokens("((", is_token("open paren", "("), is_token("open paren", "("))
+
+
+def test_close_parens_are_tokenised():
+    assert_tokens("))", is_token("close paren", ")"), is_token("close paren", ")"))
+
+
+def test_open_square_brackets_are_tokenised():
+    assert_tokens("[[", is_token("open square bracket", "["), is_token("open square bracket", "["))
+
+
+def test_close_square_brackets_are_tokenised():
+    assert_tokens("]]", is_token("close square bracket", "]"), is_token("close square bracket", "]"))
+
+
+def test_choices_are_tokenised():
+    assert_tokens("||", is_token("choice", "|"), is_token("choice", "|"))
+
+
+def test_can_tokenise_multiple_tokens():
+    assert_tokens("The Magic Position",
+        is_token("identifier", "The"),
+        is_token("whitespace", " "),
+        is_token("identifier", "Magic"),
+        is_token("whitespace", " "),
+        is_token("identifier", "Position"),
+    )
+
+
+def assert_tokens(string, *expected):
+    expected = list(expected)
+    expected.append(is_token("end", ""))
+    assert_that(
+        tokenise(string),
+        contains(*expected),
+    )
+
+
+def is_token(token_type, value):
+    return has_properties(
+        type=token_type,
+        value=value,
+    )