Always return end tokens from end of token iterator

3 files changed, 28 insertions, 17 deletions

diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java
index bb8c0cc..5113a4f 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java
@@ -41,7 +41,7 @@ public class StyleMapParser {
     }
 
     private static Consumer<StyleMapBuilder> parseStyleMapping(String line) {
-        TokenIterator<TokenType> tokens = new TokenIterator<>(StyleMappingTokeniser.tokenise(line));
+        TokenIterator<TokenType> tokens = StyleMappingTokeniser.tokenise(line);
 
         BiConsumer<StyleMapBuilder, HtmlPath> documentMatcher = DocumentMatcherParser.parse(tokens);
 
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java
index b0b44c3..9625d24 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java
@@ -5,7 +5,14 @@ import java.util.List;
 import static org.zwobble.mammoth.internal.util.Lists.list;
 
 public class StyleMappingTokeniser {
-    public static List<Token<TokenType>> tokenise(String line) {
+    public static TokenIterator<TokenType> tokenise(String line) {
+        return new TokenIterator<>(
+            tokeniseToList(line),
+            new Token<>(line.length(), TokenType.EOF, "")
+        );
+    }
+
+    public static List<Token<TokenType>> tokeniseToList(String line) {
         String stringPrefix = "'(?:(?:\\\\.|[^'])*)";
         String identifierCharacter = "(?:[a-zA-Z\\-_]|\\\\.)";
 
@@ -20,8 +27,6 @@ public class StyleMappingTokeniser {
                 RegexTokeniser.rule(TokenType.INTEGER, "[0-9]+")
             )
         );
-        List<Token<TokenType>> tokens = tokeniser.tokenise(line);
-        tokens.add(new Token<>(line.length(), TokenType.EOF, ""));
-        return tokens;
+        return tokeniser.tokenise(line);
     }
 }
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java
index d75e896..69ad7a7 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java
@@ -6,21 +6,19 @@ import static org.zwobble.mammoth.internal.styles.parsing.LineParseException.lin
 
 public class TokenIterator<T> {
     private final List<Token<T>> tokens;
+    private final Token<T> end;
     private int index;
 
-    public TokenIterator(List<Token<T>> tokens) {
+    public TokenIterator(List<Token<T>> tokens, Token<T> end) {
         this.tokens = tokens;
+        this.end = end;
         this.index = 0;
     }
 
     public boolean isNext(int offset, T tokenType, String value) {
         int tokenIndex = index + offset;
-        if (tokenIndex < tokens.size()) {
-            Token<T> token = tokens.get(tokenIndex);
-            return token.getTokenType().equals(tokenType) && token.getValue().equals(value);
-        } else {
-            return false;
-        }
+        Token<T> token = getToken(tokenIndex);
+        return token.getTokenType().equals(tokenType) && token.getValue().equals(value);
     }
 
     public boolean isNext(T tokenType, String value) {
@@ -37,17 +35,17 @@ public class TokenIterator<T> {
     }
 
     public T peekTokenType() {
-        return tokens.get(index).getTokenType();
+        return getToken(index).getTokenType();
     }
 
     public Token<T> next() {
-        Token<T> token = tokens.get(index);
+        Token<T> token = getToken(index);
         index += 1;
         return token;
     }
 
     public Token<T> next(T type) {
-        Token<T> token = tokens.get(index);
+        Token<T> token = getToken(index);
         if (token.getTokenType().equals(type)) {
             index += 1;
             return token;
@@ -65,7 +63,7 @@ public class TokenIterator<T> {
     }
 
     public void skip(T tokenType) {
-        Token<T> token = tokens.get(index);
+        Token<T> token = getToken(index);
         if (!token.getTokenType().equals(tokenType)) {
             throw unexpectedTokenType(tokenType, token);
         }
@@ -73,7 +71,7 @@ public class TokenIterator<T> {
     }
 
     public void skip(T tokenType, String tokenValue) {
-        Token<T> token = tokens.get(index);
+        Token<T> token = getToken(index);
         if (!token.getTokenType().equals(tokenType)) {
             throw unexpectedTokenType(tokenType, token);
         }
@@ -100,6 +98,14 @@ public class TokenIterator<T> {
         }
     }
 
+    private Token<T> getToken(int index) {
+        if (index < tokens.size()) {
+            return tokens.get(index);
+        } else {
+            return end;
+        }
+    }
+
     public interface Action {
         void run();
     }