2 files changed, 27 insertions, 24 deletions
diff --git a/mammoth/styles/parser/tokeniser.py b/mammoth/styles/parser/tokeniser.py
index 0f2bbe9..2e8c2c2 100644
--- a/mammoth/styles/parser/tokeniser.py
+++ b/mammoth/styles/parser/tokeniser.py
@@ -38,22 +38,13 @@ def _to_regex(value):
 
 
 _string_prefix = r"'(?:\\.|[^'])*"
+_identifier_character = r"(?:[a-zA-Z\-_]|\\.)"
 
 tokenise = regex_tokeniser([
-    ("identifier", r"[a-zA-Z][a-zA-Z0-9\-]*"),
-    ("class name", r"\.(?:[a-zA-Z0-9\-]|\\.)+"),
-    ("colon", r":"),
-    ("greater than", r">"),
+    ("identifier", _identifier_character + "(?:" + _identifier_character + "|[0-9])*"),
+    ("symbol", r":|>|=>|\^=|=|\(|\)|\[|\]|\||!|\."),
     ("whitespace", r"\s+"),
-    ("arrow", r"=>"),
-    ("equals", r"="),
-    ("open paren", r"\("),
-    ("close paren", r"\)"),
-    ("open square bracket", r"\["),
-    ("close square bracket", r"\]"),
     ("string", _string_prefix + "'"),
     ("unterminated string", _string_prefix),
-    ("integer", r"([0-9]+)"),
-    ("choice", r"\|"),
-    ("bang", r"(!)"),
+    ("integer", "([0-9]+)"),
 ])
diff --git a/tests/styles/parser/tokeniser_tests.py b/tests/styles/parser/tokeniser_tests.py
index 06fbdc3..e8c3c7b 100644
--- a/tests/styles/parser/tokeniser_tests.py
+++ b/tests/styles/parser/tokeniser_tests.py
@@ -19,6 +19,10 @@ def test_identifiers_are_tokenised():
     assert_tokens("Overture", is_token("identifier", "Overture"))
 
 
+def test_escape_sequences_in_identifiers_are_tokenised():
+    assert_tokens(r"\:", is_token("identifier", r"\:"))
+
+
 def test_integers_are_tokenised():
     assert_tokens("123", is_token("integer", "123"))
 
@@ -27,48 +31,56 @@ def test_strings_are_tokenised():
     assert_tokens("'Tristan'", is_token("string", "'Tristan'"))
 
 
+def test_escape_sequences_in_strings_are_tokenised():
+    assert_tokens(r"'Tristan\''", is_token("string", r"'Tristan\''"))
+
+
 def test_unterminated_strings_are_tokenised():
     assert_tokens("'Tristan", is_token("unterminated string", "'Tristan"))
 
 
 def test_arrows_are_tokenised():
-    assert_tokens("=>=>", is_token("arrow", "=>"), is_token("arrow", "=>"))
+    assert_tokens("=>=>", is_token("symbol", "=>"), is_token("symbol", "=>"))
 
 
-def test_classes_are_tokenised():
-    assert_tokens(".overture", is_token("class name", ".overture"))
+def test_dots_are_tokenised():
+    assert_tokens(".", is_token("symbol", "."))
 
 
 def test_colons_are_tokenised():
-    assert_tokens("::", is_token("colon", ":"), is_token("colon", ":"))
+    assert_tokens("::", is_token("symbol", ":"), is_token("symbol", ":"))
 
 
 def test_greater_thans_are_tokenised():
-    assert_tokens(">>", is_token("greater than", ">"), is_token("greater than", ">"))
+    assert_tokens(">>", is_token("symbol", ">"), is_token("symbol", ">"))
 
 
 def test_equals_are_tokenised():
-    assert_tokens("==", is_token("equals", "="), is_token("equals", "="))
+    assert_tokens("==", is_token("symbol", "="), is_token("symbol", "="))
 
 
 def test_open_parens_are_tokenised():
-    assert_tokens("((", is_token("open paren", "("), is_token("open paren", "("))
+    assert_tokens("((", is_token("symbol", "("), is_token("symbol", "("))
 
 
 def test_close_parens_are_tokenised():
-    assert_tokens("))", is_token("close paren", ")"), is_token("close paren", ")"))
+    assert_tokens("))", is_token("symbol", ")"), is_token("symbol", ")"))
 
 
 def test_open_square_brackets_are_tokenised():
-    assert_tokens("[[", is_token("open square bracket", "["), is_token("open square bracket", "["))
+    assert_tokens("[[", is_token("symbol", "["), is_token("symbol", "["))
 
 
 def test_close_square_brackets_are_tokenised():
-    assert_tokens("]]", is_token("close square bracket", "]"), is_token("close square bracket", "]"))
+    assert_tokens("]]", is_token("symbol", "]"), is_token("symbol", "]"))
 
 
 def test_choices_are_tokenised():
-    assert_tokens("||", is_token("choice", "|"), is_token("choice", "|"))
+    assert_tokens("||", is_token("symbol", "|"), is_token("symbol", "|"))
+
+
+def test_bangs_are_tokenised():
+    assert_tokens("!!", is_token("symbol", "!"), is_token("symbol", "!"))
 
 
 def test_can_tokenise_multiple_tokens():