Simplify tokenising

8 files changed, 49 insertions, 56 deletions

diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/DocumentMatcherParser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/DocumentMatcherParser.java
index ce14630..ab9ff8d 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/DocumentMatcherParser.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/DocumentMatcherParser.java
@@ -57,12 +57,11 @@ public class DocumentMatcherParser {
     }
 
     private static Optional<String> parseStyleName(TokenIterator<TokenType> tokens) {
-        if (tokens.peekTokenType() == TokenType.OPEN_SQUARE_BRACKET) {
-            tokens.skip();
+        if (tokens.trySkip(TokenType.SYMBOL, "[")) {
             tokens.skip(TokenType.IDENTIFIER, "style-name");
-            tokens.skip(TokenType.EQUALS);
+            tokens.skip(TokenType.SYMBOL, "=");
             String value = TokenParser.parseString(tokens);
-            tokens.skip(TokenType.CLOSE_SQUARE_BRACKET);
+            tokens.skip(TokenType.SYMBOL, "]");
             return Optional.of(value);
         } else {
             return Optional.empty();
@@ -70,12 +69,11 @@ public class DocumentMatcherParser {
     }
 
     private static Optional<NumberingLevel> parseNumbering(TokenIterator<TokenType> tokens) {
-        if (tokens.peekTokenType() == TokenType.COLON) {
-            tokens.skip();
+        if (tokens.trySkip(TokenType.SYMBOL, ":")) {
             boolean isOrdered = parseListType(tokens);
-            tokens.skip(TokenType.OPEN_PAREN);
+            tokens.skip(TokenType.SYMBOL, "(");
             String level = new BigInteger(tokens.nextValue(TokenType.INTEGER)).subtract(BigInteger.ONE).toString();
-            tokens.skip(TokenType.CLOSE_PAREN);
+            tokens.skip(TokenType.SYMBOL, ")");
             return Optional.of(new NumberingLevel(level, isOrdered));
         } else {
             return Optional.empty();
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/HtmlPathParser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/HtmlPathParser.java
index ef975f6..b384ef7 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/HtmlPathParser.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/HtmlPathParser.java
@@ -13,8 +13,7 @@ import static org.zwobble.mammoth.internal.util.Maps.map;
 
 public class HtmlPathParser {
     public static HtmlPath parse(TokenIterator<TokenType> tokens) {
-        if (tokens.peekTokenType() == TokenType.BANG) {
-            tokens.skip(TokenType.BANG);
+        if (tokens.trySkip(TokenType.SYMBOL, "!")) {
             return HtmlPath.IGNORE;
         } else {
             return parseHtmlPathElements(tokens);
@@ -27,9 +26,9 @@ public class HtmlPathParser {
         if (tokens.peekTokenType() == TokenType.IDENTIFIER) {
             HtmlPathElement element = parseElement(tokens);
             elements.add(element);
-            while (tokens.peekTokenType() == TokenType.WHITESPACE && tokens.peekTokenType(1) == TokenType.GREATER_THAN) {
+            while (tokens.peekTokenType() == TokenType.WHITESPACE && tokens.isNext(1, TokenType.SYMBOL, ">")) {
                 tokens.skip(TokenType.WHITESPACE);
-                tokens.skip(TokenType.GREATER_THAN);
+                tokens.skip(TokenType.SYMBOL, ">");
                 tokens.skip(TokenType.WHITESPACE);
                 elements.add(parseElement(tokens));
             }
@@ -52,8 +51,7 @@ public class HtmlPathParser {
     private static List<String> parseTagNames(TokenIterator<TokenType> tokens) {
         List<String> tagNames = new ArrayList<>();
         tagNames.add(tokens.nextValue(TokenType.IDENTIFIER));
-        while (tokens.peekTokenType() == TokenType.CHOICE) {
-            tokens.skip();
+        while (tokens.trySkip(TokenType.SYMBOL, "|")) {
             tagNames.add(tokens.nextValue(TokenType.IDENTIFIER));
         }
         return tagNames;
@@ -69,20 +67,20 @@ public class HtmlPathParser {
 
     private static boolean parseIsFresh(TokenIterator<TokenType> tokens) {
         return tokens.tryParse(() -> {
-            tokens.skip(TokenType.COLON);
+            tokens.skip(TokenType.SYMBOL, ":");
             tokens.skip(TokenType.IDENTIFIER, "fresh");
         });
     }
 
     private static String parseSeparator(TokenIterator<TokenType> tokens) {
         boolean isSeparator = tokens.tryParse(() -> {
-            tokens.skip(TokenType.COLON);
+            tokens.skip(TokenType.SYMBOL, ":");
             tokens.skip(TokenType.IDENTIFIER, "separator");
         });
         if (isSeparator) {
-            tokens.skip(TokenType.OPEN_PAREN);
+            tokens.skip(TokenType.SYMBOL, "(");
             String value = TokenParser.parseString(tokens);
-            tokens.skip(TokenType.CLOSE_PAREN);
+            tokens.skip(TokenType.SYMBOL, ")");
             return value;
         } else {
             return "";
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java
index c8dc221..002a5aa 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java
@@ -46,7 +46,7 @@ public class StyleMapParser {
         BiConsumer<StyleMapBuilder, HtmlPath> documentMatcher = DocumentMatcherParser.parse(tokens);
 
         tokens.skip(TokenType.WHITESPACE);
-        tokens.skip(TokenType.ARROW);
+        tokens.skip(TokenType.SYMBOL, "=>");
 
         HtmlPath htmlPath = parseHtmlPath(tokens);
 
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java
index 1eec870..704ba86 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java
@@ -13,20 +13,11 @@ public class StyleMappingTokeniser {
             list(
                 RegexTokeniser.rule(TokenType.IDENTIFIER, "([a-zA-Z][a-zA-Z0-9\\-]*)"),
                 RegexTokeniser.rule(TokenType.CLASS_NAME, "(\\.(?:[a-zA-Z0-9\\-]|\\\\.)+)"),
-                RegexTokeniser.rule(TokenType.COLON, ":"),
-                RegexTokeniser.rule(TokenType.GREATER_THAN, ">"),
+                RegexTokeniser.rule(TokenType.SYMBOL, ":|>|=>|=|\\(|\\)|\\[|\\]|\\||!"),
                 RegexTokeniser.rule(TokenType.WHITESPACE, "\\s+"),
-                RegexTokeniser.rule(TokenType.ARROW, "=>"),
-                RegexTokeniser.rule(TokenType.EQUALS, "="),
-                RegexTokeniser.rule(TokenType.OPEN_PAREN, "\\("),
-                RegexTokeniser.rule(TokenType.CLOSE_PAREN, "\\)"),
-                RegexTokeniser.rule(TokenType.OPEN_SQUARE_BRACKET, "\\["),
-                RegexTokeniser.rule(TokenType.CLOSE_SQUARE_BRACKET, "\\]"),
                 RegexTokeniser.rule(TokenType.STRING, stringPrefix + "'"),
                 RegexTokeniser.rule(TokenType.UNTERMINATED_STRING, stringPrefix),
-                RegexTokeniser.rule(TokenType.INTEGER, "([0-9]+)"),
-                RegexTokeniser.rule(TokenType.CHOICE, "\\|"),
-                RegexTokeniser.rule(TokenType.BANG, "!")
+                RegexTokeniser.rule(TokenType.INTEGER, "([0-9]+)")
             )
         );
         List<Token<TokenType>> tokens = tokeniser.tokenise(line);
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java
index f79974d..7ac7643 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java
@@ -11,16 +11,31 @@ public class TokenIterator<T> {
         this.index = 0;
     }
 
-    public T peekTokenType() {
-        return tokens.get(index).getTokenType();
+    public boolean isNext(int offset, T tokenType, String value) {
+        int tokenIndex = index + offset;
+        if (tokenIndex < tokens.size()) {
+            Token<T> token = tokens.get(tokenIndex);
+            return token.getTokenType().equals(tokenType) && token.getValue().equals(value);
+        } else {
+            return false;
+        }
     }
 
-    public T peekTokenType(int count) {
-        return peekToken(count).getTokenType();
+    public boolean isNext(T tokenType, String value) {
+        return isNext(0, tokenType, value);
     }
 
-    public Token<T> peekToken(int count) {
-        return tokens.get(index + count);
+    public boolean trySkip(T tokenType, String value) {
+        if (isNext(tokenType, value)) {
+            skip();
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    public T peekTokenType() {
+        return tokens.get(index).getTokenType();
     }
 
     public Token<T> next(T type) {
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenType.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenType.java
index f17b85e..17442b5 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenType.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenType.java
@@ -4,19 +4,10 @@ public enum TokenType {
     WHITESPACE,
     IDENTIFIER,
     CLASS_NAME,
-    COLON,
-    GREATER_THAN,
-    ARROW,
-    EQUALS,
-    OPEN_PAREN,
-    CLOSE_PAREN,
-    OPEN_SQUARE_BRACKET,
-    CLOSE_SQUARE_BRACKET,
+    SYMBOL,
     STRING,
     UNTERMINATED_STRING,
     INTEGER,
-    CHOICE,
-    BANG,
     EOF,
     UNKNOWN
 }
diff --git a/src/test/java/org/zwobble/mammoth/tests/MammothTests.java b/src/test/java/org/zwobble/mammoth/tests/MammothTests.java
index 3bab856..8454a3d 100644
--- a/src/test/java/org/zwobble/mammoth/tests/MammothTests.java
+++ b/src/test/java/org/zwobble/mammoth/tests/MammothTests.java
@@ -213,7 +213,7 @@ public class MammothTests {
         assertThat(
             exception.getMessage(),
             equalTo(
-                "error reading style map at line 2, character 14: expected token of type STRING but was of type CLOSE_SQUARE_BRACKET\n\n" +
+                "error reading style map at line 2, character 14: expected token of type STRING but was of type SYMBOL\n\n" +
                 "p[style-name=] =>\n" +
                 "             ^"));
     }
diff --git a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java
index a7edf9f..0e0c920 100644
--- a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java
+++ b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java
@@ -52,7 +52,7 @@ public class StyleMappingTokeniserTests {
 
     @Test
     public void arrowsAreTokenised() {
-        assertTokens("=>=>", isToken(TokenType.ARROW, "=>"), isToken(TokenType.ARROW, "=>"));
+        assertTokens("=>=>", isToken(TokenType.SYMBOL, "=>"), isToken(TokenType.SYMBOL, "=>"));
     }
 
     @Test
@@ -62,42 +62,42 @@ public class StyleMappingTokeniserTests {
 
     @Test
     public void colonsAreTokenised() {
-        assertTokens("::", isToken(TokenType.COLON, ":"), isToken(TokenType.COLON, ":"));
+        assertTokens("::", isToken(TokenType.SYMBOL, ":"), isToken(TokenType.SYMBOL, ":"));
     }
 
     @Test
     public void greaterThansAreTokenised() {
-        assertTokens(">>", isToken(TokenType.GREATER_THAN, ">"), isToken(TokenType.GREATER_THAN, ">"));
+        assertTokens(">>", isToken(TokenType.SYMBOL, ">"), isToken(TokenType.SYMBOL, ">"));
     }
 
     @Test
     public void equalsAreTokenised() {
-        assertTokens("==", isToken(TokenType.EQUALS, "="), isToken(TokenType.EQUALS, "="));
+        assertTokens("==", isToken(TokenType.SYMBOL, "="), isToken(TokenType.SYMBOL, "="));
     }
 
     @Test
     public void openParensAreTokenised() {
-        assertTokens("((", isToken(TokenType.OPEN_PAREN, "("), isToken(TokenType.OPEN_PAREN, "("));
+        assertTokens("((", isToken(TokenType.SYMBOL, "("), isToken(TokenType.SYMBOL, "("));
     }
 
     @Test
     public void closeParensAreTokenised() {
-        assertTokens("))", isToken(TokenType.CLOSE_PAREN, ")"), isToken(TokenType.CLOSE_PAREN, ")"));
+        assertTokens("))", isToken(TokenType.SYMBOL, ")"), isToken(TokenType.SYMBOL, ")"));
     }
 
     @Test
     public void openSquareBracketsAreTokenised() {
-        assertTokens("[[", isToken(TokenType.OPEN_SQUARE_BRACKET, "["), isToken(TokenType.OPEN_SQUARE_BRACKET, "["));
+        assertTokens("[[", isToken(TokenType.SYMBOL, "["), isToken(TokenType.SYMBOL, "["));
     }
 
     @Test
     public void closeSquareBracketsAreTokenised() {
-        assertTokens("]]", isToken(TokenType.CLOSE_SQUARE_BRACKET, "]"), isToken(TokenType.CLOSE_SQUARE_BRACKET, "]"));
+        assertTokens("]]", isToken(TokenType.SYMBOL, "]"), isToken(TokenType.SYMBOL, "]"));
     }
 
     @Test
     public void choicesAreTokenised() {
-        assertTokens("||", isToken(TokenType.CHOICE, "|"), isToken(TokenType.CHOICE, "|"));
+        assertTokens("||", isToken(TokenType.SYMBOL, "|"), isToken(TokenType.SYMBOL, "|"));
     }
 
     @Test