Always return end tokens from end of token iterator

8 files changed, 58 insertions, 32 deletions

diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java
index bb8c0cc..5113a4f 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java
@@ -41,7 +41,7 @@ public class StyleMapParser {
     }
 
     private static Consumer<StyleMapBuilder> parseStyleMapping(String line) {
-        TokenIterator<TokenType> tokens = new TokenIterator<>(StyleMappingTokeniser.tokenise(line));
+        TokenIterator<TokenType> tokens = StyleMappingTokeniser.tokenise(line);
 
         BiConsumer<StyleMapBuilder, HtmlPath> documentMatcher = DocumentMatcherParser.parse(tokens);
 
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java
index b0b44c3..9625d24 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java
@@ -5,7 +5,14 @@ import java.util.List;
 import static org.zwobble.mammoth.internal.util.Lists.list;
 
 public class StyleMappingTokeniser {
-    public static List<Token<TokenType>> tokenise(String line) {
+    public static TokenIterator<TokenType> tokenise(String line) {
+        return new TokenIterator<>(
+            tokeniseToList(line),
+            new Token<>(line.length(), TokenType.EOF, "")
+        );
+    }
+
+    public static List<Token<TokenType>> tokeniseToList(String line) {
         String stringPrefix = "'(?:(?:\\\\.|[^'])*)";
         String identifierCharacter = "(?:[a-zA-Z\\-_]|\\\\.)";
 
@@ -20,8 +27,6 @@ public class StyleMappingTokeniser {
                 RegexTokeniser.rule(TokenType.INTEGER, "[0-9]+")
             )
         );
-        List<Token<TokenType>> tokens = tokeniser.tokenise(line);
-        tokens.add(new Token<>(line.length(), TokenType.EOF, ""));
-        return tokens;
+        return tokeniser.tokenise(line);
     }
 }
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java
index d75e896..69ad7a7 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java
@@ -6,21 +6,19 @@ import static org.zwobble.mammoth.internal.styles.parsing.LineParseException.lin
 
 public class TokenIterator<T> {
     private final List<Token<T>> tokens;
+    private final Token<T> end;
     private int index;
 
-    public TokenIterator(List<Token<T>> tokens) {
+    public TokenIterator(List<Token<T>> tokens, Token<T> end) {
         this.tokens = tokens;
+        this.end = end;
         this.index = 0;
     }
 
     public boolean isNext(int offset, T tokenType, String value) {
         int tokenIndex = index + offset;
-        if (tokenIndex < tokens.size()) {
-            Token<T> token = tokens.get(tokenIndex);
-            return token.getTokenType().equals(tokenType) && token.getValue().equals(value);
-        } else {
-            return false;
-        }
+        Token<T> token = getToken(tokenIndex);
+        return token.getTokenType().equals(tokenType) && token.getValue().equals(value);
     }
 
     public boolean isNext(T tokenType, String value) {
@@ -37,17 +35,17 @@ public class TokenIterator<T> {
     }
 
     public T peekTokenType() {
-        return tokens.get(index).getTokenType();
+        return getToken(index).getTokenType();
     }
 
     public Token<T> next() {
-        Token<T> token = tokens.get(index);
+        Token<T> token = getToken(index);
         index += 1;
         return token;
     }
 
     public Token<T> next(T type) {
-        Token<T> token = tokens.get(index);
+        Token<T> token = getToken(index);
         if (token.getTokenType().equals(type)) {
             index += 1;
             return token;
@@ -65,7 +63,7 @@ public class TokenIterator<T> {
     }
 
     public void skip(T tokenType) {
-        Token<T> token = tokens.get(index);
+        Token<T> token = getToken(index);
         if (!token.getTokenType().equals(tokenType)) {
             throw unexpectedTokenType(tokenType, token);
         }
@@ -73,7 +71,7 @@ public class TokenIterator<T> {
     }
 
     public void skip(T tokenType, String tokenValue) {
-        Token<T> token = tokens.get(index);
+        Token<T> token = getToken(index);
         if (!token.getTokenType().equals(tokenType)) {
             throw unexpectedTokenType(tokenType, token);
         }
@@ -100,6 +98,14 @@ public class TokenIterator<T> {
         }
     }
 
+    private Token<T> getToken(int index) {
+        if (index < tokens.size()) {
+            return tokens.get(index);
+        } else {
+            return end;
+        }
+    }
+
     public interface Action {
         void run();
     }
diff --git a/src/test/java/org/zwobble/mammoth/tests/Matchers.java b/src/test/java/org/zwobble/mammoth/tests/Matchers.java
new file mode 100644
index 0000000..de582c1
--- /dev/null
+++ b/src/test/java/org/zwobble/mammoth/tests/Matchers.java
@@ -0,0 +1,21 @@
+package org.zwobble.mammoth.tests;
+
+import org.hamcrest.Matcher;
+
+import java.util.List;
+
+import static java.util.Arrays.asList;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.emptyIterable;
+
+public class Matchers {
+    private Matchers() {}
+
+    public static <T> Matcher<Iterable<? extends T>> isSameSequence(Matcher<? super T>... matchers) {
+        return isSameSequence(asList(matchers));
+    }
+
+    public static <T> Matcher<Iterable<? extends T>> isSameSequence(List<Matcher<? super T>> matchers) {
+        return matchers.size() == 0 ? emptyIterable() : contains(matchers);
+    }
+}
diff --git a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/DocumentMatcherParsingTests.java b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/DocumentMatcherParsingTests.java
index 0ee863a..c85bbe0 100644
--- a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/DocumentMatcherParsingTests.java
+++ b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/DocumentMatcherParsingTests.java
@@ -79,13 +79,13 @@ public class DocumentMatcherParsingTests {
     }
 
     private ParagraphMatcher parseParagraphMatcher(String input) {
-        TokenIterator tokens = new TokenIterator(StyleMappingTokeniser.tokenise(input));
+        TokenIterator tokens = StyleMappingTokeniser.tokenise(input);
         tokens.skip(TokenType.IDENTIFIER, "p");
         return DocumentMatcherParser.parseParagraphMatcher(tokens);
     }
 
     private RunMatcher parseRunMatcher(String input) {
-        TokenIterator tokens = new TokenIterator(StyleMappingTokeniser.tokenise(input));
+        TokenIterator tokens = StyleMappingTokeniser.tokenise(input);
         tokens.skip(TokenType.IDENTIFIER, "r");
         return DocumentMatcherParser.parseRunMatcher(tokens);
     }
diff --git a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/HtmlPathParsingTests.java b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/HtmlPathParsingTests.java
index 0b54ad8..a9b19da 100644
--- a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/HtmlPathParsingTests.java
+++ b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/HtmlPathParsingTests.java
@@ -4,7 +4,6 @@ import org.junit.Test;
 import org.zwobble.mammoth.internal.styles.HtmlPath;
 import org.zwobble.mammoth.internal.styles.parsing.HtmlPathParser;
 import org.zwobble.mammoth.internal.styles.parsing.StyleMappingTokeniser;
-import org.zwobble.mammoth.internal.styles.parsing.TokenIterator;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.zwobble.mammoth.internal.util.Lists.list;
@@ -85,6 +84,6 @@ public class HtmlPathParsingTests {
     }
 
     private HtmlPath parseHtmlPath(String input) {
-        return HtmlPathParser.parse(new TokenIterator(StyleMappingTokeniser.tokenise(input)));
+        return HtmlPathParser.parse(StyleMappingTokeniser.tokenise(input));
     }
 }
diff --git a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java
index 85fb6d4..c767957 100644
--- a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java
+++ b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java
@@ -6,13 +6,9 @@ import org.zwobble.mammoth.internal.styles.parsing.StyleMappingTokeniser;
 import org.zwobble.mammoth.internal.styles.parsing.Token;
 import org.zwobble.mammoth.internal.styles.parsing.TokenType;
 
-import java.util.List;
-
-import static java.util.Arrays.asList;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.*;
-import static org.zwobble.mammoth.internal.util.Lists.eagerConcat;
-import static org.zwobble.mammoth.internal.util.Lists.list;
+import static org.zwobble.mammoth.tests.Matchers.isSameSequence;
 
 public class StyleMappingTokeniserTests {
     @Test
@@ -21,7 +17,7 @@ public class StyleMappingTokeniserTests {
     }
 
     @Test
-    public void emptyStringIsTokenisedToEndOfFileToken() {
+    public void emptyStringIsTokenisedToNoTokens() {
         assertTokens("");
     }
 
@@ -132,9 +128,8 @@ public class StyleMappingTokeniserTests {
     }
 
     @SafeVarargs
-    private final void assertTokens(String input, Matcher<Token>... token) {
-        List<Matcher<? super Token>> tokens = eagerConcat(asList(token), list(isToken(TokenType.EOF, "")));
-        assertThat(StyleMappingTokeniser.tokenise(input), contains(tokens));
+    private final void assertTokens(String input, Matcher<Token>... tokens) {
+        assertThat(StyleMappingTokeniser.tokeniseToList(input), isSameSequence(tokens));
     }
 
     private Matcher<Token> isToken(TokenType tokenType, String value) {
diff --git a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/TokenParserTests.java b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/TokenParserTests.java
index ba0de10..aec1f17 100644
--- a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/TokenParserTests.java
+++ b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/TokenParserTests.java
@@ -17,7 +17,7 @@ public class TokenParserTests {
         Optional<String> value = TokenParser.parseClassName(new TokenIterator<>(list(
             new Token<>(0, TokenType.SYMBOL, "."),
             new Token<>(1, TokenType.IDENTIFIER, "\\:")
-        )));
+        ), new Token<>(2, TokenType.EOF, "")));
         assertEquals(Optional.of(":"), value);
     }
 
@@ -25,7 +25,7 @@ public class TokenParserTests {
     public void escapeSequencesInStringAreDecoded() {
         String value = TokenParser.parseString(new TokenIterator<>(list(
             new Token<>(0, TokenType.STRING, "'\\n'")
-        )));
+        ), new Token<>(1, TokenType.EOF, "")));
         assertEquals("\n", value);
     }
 }