Add tokeniser

3 files changed, 59 insertions, 0 deletions

diff --git a/mammoth/styles.py b/mammoth/styles/__init__.py
index a90012b..a90012b 100644
--- a/mammoth/styles.py
+++ b/mammoth/styles/__init__.py
diff --git a/mammoth/styles/parser/__init__.py b/mammoth/styles/parser/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/mammoth/styles/parser/__init__.py
diff --git a/mammoth/styles/parser/tokeniser.py b/mammoth/styles/parser/tokeniser.py
new file mode 100644
index 0000000..0f2bbe9
--- /dev/null
+++ b/mammoth/styles/parser/tokeniser.py
@@ -0,0 +1,59 @@
+import collections
+import re
+
+
+Token = collections.namedtuple("Token", ["character_index", "type", "value"])
+
+
+def regex_tokeniser(rules):
+    rules = [(token_type, _to_regex(regex)) for token_type, regex in rules]
+    rules.append(("unknown", re.compile(".")))
+    
+    def tokenise(value):
+        tokens = []
+        index = 0
+        while index < len(value):
+            for token_type, regex in rules:
+                match = regex.match(value, index)
+                if match is not None:
+                    tokens.append(Token(index, token_type, match.group(0)))
+                    index = match.end()
+                    break
+            else:
+                # Should be impossible
+                raise Exception("Remaining: " + value[index:])
+
+        tokens.append(Token(index, "end", ""))
+
+        return tokens
+
+    return tokenise
+    
+
+def _to_regex(value):
+    if hasattr(value, "match"):
+        return value
+    else:
+        return re.compile(value)
+
+
+_string_prefix = r"'(?:\\.|[^'])*"
+
+tokenise = regex_tokeniser([
+    ("identifier", r"[a-zA-Z][a-zA-Z0-9\-]*"),
+    ("class name", r"\.(?:[a-zA-Z0-9\-]|\\.)+"),
+    ("colon", r":"),
+    ("greater than", r">"),
+    ("whitespace", r"\s+"),
+    ("arrow", r"=>"),
+    ("equals", r"="),
+    ("open paren", r"\("),
+    ("close paren", r"\)"),
+    ("open square bracket", r"\["),
+    ("close square bracket", r"\]"),
+    ("string", _string_prefix + "'"),
+    ("unterminated string", _string_prefix),
+    ("integer", r"([0-9]+)"),
+    ("choice", r"\|"),
+    ("bang", r"(!)"),
+])