4 files changed, 134 insertions, 19 deletions
diff --git a/src/test/java/org/zwobble/mammoth/tests/styles/DocumentMatcherParsingTests.java b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/DocumentMatcherParsingTests.java
index 0a8cf0d..3c00909 100644
--- a/src/test/java/org/zwobble/mammoth/tests/styles/DocumentMatcherParsingTests.java
+++ b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/DocumentMatcherParsingTests.java
@@ -1,11 +1,12 @@
-package org.zwobble.mammoth.tests.styles;
+package org.zwobble.mammoth.tests.styles.parsing;
 
 import org.junit.Test;
-import org.parboiled.support.Var;
 import org.zwobble.mammoth.internal.styles.ParagraphMatcher;
 import org.zwobble.mammoth.internal.styles.RunMatcher;
-import org.zwobble.mammoth.internal.styles.parsing.Parsing;
-import org.zwobble.mammoth.internal.styles.parsing.StyleMappingParser;
+import org.zwobble.mammoth.internal.styles.parsing.DocumentMatcherParser;
+import org.zwobble.mammoth.internal.styles.parsing.StyleMappingTokeniser;
+import org.zwobble.mammoth.internal.styles.parsing.TokenIterator;
+import org.zwobble.mammoth.internal.styles.parsing.TokenType;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.zwobble.mammoth.tests.DeepReflectionMatcher.deepEquals;
@@ -67,14 +68,14 @@ public class DocumentMatcherParsingTests {
     }
 
     private ParagraphMatcher parseParagraphMatcher(String input) {
-        Var<ParagraphMatcher> matcher = new Var<>();
-        Parsing.parse(StyleMappingParser.class, parser -> parser.ParagraphMatcher(matcher), input);
-        return matcher.get();
+        TokenIterator tokens = new TokenIterator(StyleMappingTokeniser.tokenise(input));
+        tokens.skip(TokenType.IDENTIFIER, "p");
+        return DocumentMatcherParser.parseParagraphMatcher(tokens);
     }
 
     private RunMatcher parseRunMatcher(String input) {
-        Var<RunMatcher> matcher = new Var<>();
-        Parsing.parse(StyleMappingParser.class, parser -> parser.RunMatcher(matcher), input);
-        return matcher.get();
+        TokenIterator tokens = new TokenIterator(StyleMappingTokeniser.tokenise(input));
+        tokens.skip(TokenType.IDENTIFIER, "r");
+        return DocumentMatcherParser.parseRunMatcher(tokens);
     }
 }
diff --git a/src/test/java/org/zwobble/mammoth/tests/styles/HtmlPathParsingTests.java b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/HtmlPathParsingTests.java
index 3fe1278..1df4b0b 100644
--- a/src/test/java/org/zwobble/mammoth/tests/styles/HtmlPathParsingTests.java
+++ b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/HtmlPathParsingTests.java
@@ -1,16 +1,16 @@
-package org.zwobble.mammoth.tests.styles;
+package org.zwobble.mammoth.tests.styles.parsing;
 
 import org.junit.Test;
-import org.parboiled.support.Var;
 import org.zwobble.mammoth.internal.styles.HtmlPath;
 import org.zwobble.mammoth.internal.styles.HtmlPathElement;
-import org.zwobble.mammoth.internal.styles.parsing.Parsing;
-import org.zwobble.mammoth.internal.styles.parsing.StyleMappingParser;
+import org.zwobble.mammoth.internal.styles.parsing.HtmlPathParser;
+import org.zwobble.mammoth.internal.styles.parsing.StyleMappingTokeniser;
+import org.zwobble.mammoth.internal.styles.parsing.TokenIterator;
 
 import static org.hamcrest.MatcherAssert.assertThat;
-import static org.zwobble.mammoth.tests.DeepReflectionMatcher.deepEquals;
 import static org.zwobble.mammoth.internal.util.Lists.list;
 import static org.zwobble.mammoth.internal.util.Maps.map;
+import static org.zwobble.mammoth.tests.DeepReflectionMatcher.deepEquals;
 
 public class HtmlPathParsingTests {
     @Test
@@ -67,8 +67,6 @@ public class HtmlPathParsingTests {
     }
 
     private HtmlPath parseHtmlPath(String input) {
-        Var<HtmlPath> path = new Var<>();
-        Parsing.parse(StyleMappingParser.class, parser -> parser.HtmlPath(path), input);
-        return path.get();
+        return HtmlPathParser.parse(new TokenIterator(StyleMappingTokeniser.tokenise(input)));
     }
 }
diff --git a/src/test/java/org/zwobble/mammoth/tests/styles/StyleMapParserTests.java b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMapParserTests.java
index a89b192..e76a632 100644
--- a/src/test/java/org/zwobble/mammoth/tests/styles/StyleMapParserTests.java
+++ b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMapParserTests.java
@@ -1,4 +1,4 @@
-package org.zwobble.mammoth.tests.styles;
+package org.zwobble.mammoth.tests.styles.parsing;
 
 import org.junit.Test;
 import org.zwobble.mammoth.internal.styles.HtmlPath;
diff --git a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java
new file mode 100644
index 0000000..de60249
--- /dev/null
+++ b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java
@@ -0,0 +1,116 @@
+package org.zwobble.mammoth.tests.styles.parsing;
+
+import org.hamcrest.Matcher;
+import org.junit.Test;
+import org.zwobble.mammoth.internal.styles.parsing.StyleMappingTokeniser;
+import org.zwobble.mammoth.internal.styles.parsing.Token;
+import org.zwobble.mammoth.internal.styles.parsing.TokenType;
+
+import java.util.List;
+
+import static java.util.Arrays.asList;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.*;
+import static org.zwobble.mammoth.internal.util.Lists.eagerConcat;
+import static org.zwobble.mammoth.internal.util.Lists.list;
+
+public class StyleMappingTokeniserTests {
+    @Test
+    public void emptyStringIsTokenisedToEndOfFileToken() {
+        assertTokens("");
+    }
+
+    @Test
+    public void whitespaceIsTokenised() {
+        assertTokens(" \t\t  ", isToken(TokenType.WHITESPACE, " \t\t  "));
+    }
+
+    @Test
+    public void identifiersAreTokenised() {
+        assertTokens("Overture", isToken(TokenType.IDENTIFIER, "Overture"));
+    }
+
+    @Test
+    public void integersAreTokenised() {
+        assertTokens("123", isToken(TokenType.INTEGER, "123"));
+    }
+
+    @Test
+    public void stringsAreTokenised() {
+        assertTokens("'Tristan'", isToken(TokenType.STRING, "'Tristan'"));
+    }
+
+    @Test
+    public void arrowsAreTokenised() {
+        assertTokens("=>=>", isToken(TokenType.ARROW, "=>"), isToken(TokenType.ARROW, "=>"));
+    }
+
+    @Test
+    public void dotsAreTokenised() {
+        assertTokens("..", isToken(TokenType.DOT, "."), isToken(TokenType.DOT, "."));
+    }
+
+    @Test
+    public void colonsAreTokenised() {
+        assertTokens("::", isToken(TokenType.COLON, ":"), isToken(TokenType.COLON, ":"));
+    }
+
+    @Test
+    public void greaterThansAreTokenised() {
+        assertTokens(">>", isToken(TokenType.GREATER_THAN, ">"), isToken(TokenType.GREATER_THAN, ">"));
+    }
+
+    @Test
+    public void equalsAreTokenised() {
+        assertTokens("==", isToken(TokenType.EQUALS, "="), isToken(TokenType.EQUALS, "="));
+    }
+
+    @Test
+    public void openParensAreTokenised() {
+        assertTokens("((", isToken(TokenType.OPEN_PAREN, "("), isToken(TokenType.OPEN_PAREN, "("));
+    }
+
+    @Test
+    public void closeParensAreTokenised() {
+        assertTokens("))", isToken(TokenType.CLOSE_PAREN, ")"), isToken(TokenType.CLOSE_PAREN, ")"));
+    }
+
+    @Test
+    public void openSquareBracketsAreTokenised() {
+        assertTokens("[[", isToken(TokenType.OPEN_SQUARE_BRACKET, "["), isToken(TokenType.OPEN_SQUARE_BRACKET, "["));
+    }
+
+    @Test
+    public void closeSquareBracketsAreTokenised() {
+        assertTokens("]]", isToken(TokenType.CLOSE_SQUARE_BRACKET, "]"), isToken(TokenType.CLOSE_SQUARE_BRACKET, "]"));
+    }
+
+    @Test
+    public void choicesAreTokenised() {
+        assertTokens("||", isToken(TokenType.CHOICE, "|"), isToken(TokenType.CHOICE, "|"));
+    }
+
+    @Test
+    public void canTokeniseMultipleTokens() {
+        assertTokens("The Magic Position",
+            isToken(TokenType.IDENTIFIER, "The"),
+            isToken(TokenType.WHITESPACE, " "),
+            isToken(TokenType.IDENTIFIER, "Magic"),
+            isToken(TokenType.WHITESPACE, " "),
+            isToken(TokenType.IDENTIFIER, "Position")
+        );
+    }
+
+    @SafeVarargs
+    private final void assertTokens(String input, Matcher<Token>... token) {
+        List<Matcher<? super Token>> tokens = eagerConcat(asList(token), list(isToken(TokenType.EOF, "")));
+        assertThat(StyleMappingTokeniser.tokenise(input), contains(tokens));
+    }
+
+    private Matcher<Token> isToken(TokenType tokenType, String value) {
+        return allOf(
+            hasProperty("tokenType", equalTo(tokenType)),
+            hasProperty("value", equalTo(value))
+        );
+    }
+}