Simplify tokenising

6 files changed, 39 insertions, 46 deletions

diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/DocumentMatcherParser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/DocumentMatcherParser.java
index ce14630..ab9ff8d 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/DocumentMatcherParser.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/DocumentMatcherParser.java
@@ -57,12 +57,11 @@ public class DocumentMatcherParser {
     }
 
     private static Optional<String> parseStyleName(TokenIterator<TokenType> tokens) {
-        if (tokens.peekTokenType() == TokenType.OPEN_SQUARE_BRACKET) {
-            tokens.skip();
+        if (tokens.trySkip(TokenType.SYMBOL, "[")) {
             tokens.skip(TokenType.IDENTIFIER, "style-name");
-            tokens.skip(TokenType.EQUALS);
+            tokens.skip(TokenType.SYMBOL, "=");
             String value = TokenParser.parseString(tokens);
-            tokens.skip(TokenType.CLOSE_SQUARE_BRACKET);
+            tokens.skip(TokenType.SYMBOL, "]");
             return Optional.of(value);
         } else {
             return Optional.empty();
@@ -70,12 +69,11 @@ public class DocumentMatcherParser {
     }
 
     private static Optional<NumberingLevel> parseNumbering(TokenIterator<TokenType> tokens) {
-        if (tokens.peekTokenType() == TokenType.COLON) {
-            tokens.skip();
+        if (tokens.trySkip(TokenType.SYMBOL, ":")) {
             boolean isOrdered = parseListType(tokens);
-            tokens.skip(TokenType.OPEN_PAREN);
+            tokens.skip(TokenType.SYMBOL, "(");
             String level = new BigInteger(tokens.nextValue(TokenType.INTEGER)).subtract(BigInteger.ONE).toString();
-            tokens.skip(TokenType.CLOSE_PAREN);
+            tokens.skip(TokenType.SYMBOL, ")");
             return Optional.of(new NumberingLevel(level, isOrdered));
         } else {
             return Optional.empty();
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/HtmlPathParser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/HtmlPathParser.java
index ef975f6..b384ef7 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/HtmlPathParser.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/HtmlPathParser.java
@@ -13,8 +13,7 @@ import static org.zwobble.mammoth.internal.util.Maps.map;
 
 public class HtmlPathParser {
     public static HtmlPath parse(TokenIterator<TokenType> tokens) {
-        if (tokens.peekTokenType() == TokenType.BANG) {
-            tokens.skip(TokenType.BANG);
+        if (tokens.trySkip(TokenType.SYMBOL, "!")) {
             return HtmlPath.IGNORE;
         } else {
             return parseHtmlPathElements(tokens);
@@ -27,9 +26,9 @@ public class HtmlPathParser {
         if (tokens.peekTokenType() == TokenType.IDENTIFIER) {
             HtmlPathElement element = parseElement(tokens);
             elements.add(element);
-            while (tokens.peekTokenType() == TokenType.WHITESPACE && tokens.peekTokenType(1) == TokenType.GREATER_THAN) {
+            while (tokens.peekTokenType() == TokenType.WHITESPACE && tokens.isNext(1, TokenType.SYMBOL, ">")) {
                 tokens.skip(TokenType.WHITESPACE);
-                tokens.skip(TokenType.GREATER_THAN);
+                tokens.skip(TokenType.SYMBOL, ">");
                 tokens.skip(TokenType.WHITESPACE);
                 elements.add(parseElement(tokens));
             }
@@ -52,8 +51,7 @@ public class HtmlPathParser {
     private static List<String> parseTagNames(TokenIterator<TokenType> tokens) {
         List<String> tagNames = new ArrayList<>();
         tagNames.add(tokens.nextValue(TokenType.IDENTIFIER));
-        while (tokens.peekTokenType() == TokenType.CHOICE) {
-            tokens.skip();
+        while (tokens.trySkip(TokenType.SYMBOL, "|")) {
             tagNames.add(tokens.nextValue(TokenType.IDENTIFIER));
         }
         return tagNames;
@@ -69,20 +67,20 @@ public class HtmlPathParser {
 
     private static boolean parseIsFresh(TokenIterator<TokenType> tokens) {
         return tokens.tryParse(() -> {
-            tokens.skip(TokenType.COLON);
+            tokens.skip(TokenType.SYMBOL, ":");
             tokens.skip(TokenType.IDENTIFIER, "fresh");
         });
     }
 
     private static String parseSeparator(TokenIterator<TokenType> tokens) {
         boolean isSeparator = tokens.tryParse(() -> {
-            tokens.skip(TokenType.COLON);
+            tokens.skip(TokenType.SYMBOL, ":");
             tokens.skip(TokenType.IDENTIFIER, "separator");
         });
         if (isSeparator) {
-            tokens.skip(TokenType.OPEN_PAREN);
+            tokens.skip(TokenType.SYMBOL, "(");
             String value = TokenParser.parseString(tokens);
-            tokens.skip(TokenType.CLOSE_PAREN);
+            tokens.skip(TokenType.SYMBOL, ")");
             return value;
         } else {
             return "";
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java
index c8dc221..002a5aa 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java
@@ -46,7 +46,7 @@ public class StyleMapParser {
         BiConsumer<StyleMapBuilder, HtmlPath> documentMatcher = DocumentMatcherParser.parse(tokens);
 
         tokens.skip(TokenType.WHITESPACE);
-        tokens.skip(TokenType.ARROW);
+        tokens.skip(TokenType.SYMBOL, "=>");
 
         HtmlPath htmlPath = parseHtmlPath(tokens);
 
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java
index 1eec870..704ba86 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java
@@ -13,20 +13,11 @@ public class StyleMappingTokeniser {
             list(
                 RegexTokeniser.rule(TokenType.IDENTIFIER, "([a-zA-Z][a-zA-Z0-9\\-]*)"),
                 RegexTokeniser.rule(TokenType.CLASS_NAME, "(\\.(?:[a-zA-Z0-9\\-]|\\\\.)+)"),
-                RegexTokeniser.rule(TokenType.COLON, ":"),
-                RegexTokeniser.rule(TokenType.GREATER_THAN, ">"),
+                RegexTokeniser.rule(TokenType.SYMBOL, ":|>|=>|=|\\(|\\)|\\[|\\]|\\||!"),
                 RegexTokeniser.rule(TokenType.WHITESPACE, "\\s+"),
-                RegexTokeniser.rule(TokenType.ARROW, "=>"),
-                RegexTokeniser.rule(TokenType.EQUALS, "="),
-                RegexTokeniser.rule(TokenType.OPEN_PAREN, "\\("),
-                RegexTokeniser.rule(TokenType.CLOSE_PAREN, "\\)"),
-                RegexTokeniser.rule(TokenType.OPEN_SQUARE_BRACKET, "\\["),
-                RegexTokeniser.rule(TokenType.CLOSE_SQUARE_BRACKET, "\\]"),
                 RegexTokeniser.rule(TokenType.STRING, stringPrefix + "'"),
                 RegexTokeniser.rule(TokenType.UNTERMINATED_STRING, stringPrefix),
-                RegexTokeniser.rule(TokenType.INTEGER, "([0-9]+)"),
-                RegexTokeniser.rule(TokenType.CHOICE, "\\|"),
-                RegexTokeniser.rule(TokenType.BANG, "!")
+                RegexTokeniser.rule(TokenType.INTEGER, "([0-9]+)")
             )
         );
         List<Token<TokenType>> tokens = tokeniser.tokenise(line);
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java
index f79974d..7ac7643 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java
@@ -11,16 +11,31 @@ public class TokenIterator<T> {
         this.index = 0;
     }
 
-    public T peekTokenType() {
-        return tokens.get(index).getTokenType();
+    public boolean isNext(int offset, T tokenType, String value) {
+        int tokenIndex = index + offset;
+        if (tokenIndex < tokens.size()) {
+            Token<T> token = tokens.get(tokenIndex);
+            return token.getTokenType().equals(tokenType) && token.getValue().equals(value);
+        } else {
+            return false;
+        }
     }
 
-    public T peekTokenType(int count) {
-        return peekToken(count).getTokenType();
+    public boolean isNext(T tokenType, String value) {
+        return isNext(0, tokenType, value);
     }
 
-    public Token<T> peekToken(int count) {
-        return tokens.get(index + count);
+    public boolean trySkip(T tokenType, String value) {
+        if (isNext(tokenType, value)) {
+            skip();
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    public T peekTokenType() {
+        return tokens.get(index).getTokenType();
     }
 
     public Token<T> next(T type) {
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenType.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenType.java
index f17b85e..17442b5 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenType.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenType.java
@@ -4,19 +4,10 @@ public enum TokenType {
     WHITESPACE,
     IDENTIFIER,
     CLASS_NAME,
-    COLON,
-    GREATER_THAN,
-    ARROW,
-    EQUALS,
-    OPEN_PAREN,
-    CLOSE_PAREN,
-    OPEN_SQUARE_BRACKET,
-    CLOSE_SQUARE_BRACKET,
+    SYMBOL,
     STRING,
     UNTERMINATED_STRING,
     INTEGER,
-    CHOICE,
-    BANG,
     EOF,
     UNKNOWN
 }