Always return end tokens from end of token iterator

5 files changed, 30 insertions, 15 deletions

diff --git a/src/test/java/org/zwobble/mammoth/tests/Matchers.java b/src/test/java/org/zwobble/mammoth/tests/Matchers.java
new file mode 100644
index 0000000..de582c1
--- /dev/null
+++ b/src/test/java/org/zwobble/mammoth/tests/Matchers.java
@@ -0,0 +1,21 @@
+package org.zwobble.mammoth.tests;
+
+import org.hamcrest.Matcher;
+
+import java.util.List;
+
+import static java.util.Arrays.asList;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.emptyIterable;
+
+public class Matchers {
+    private Matchers() {}
+
+    public static <T> Matcher<Iterable<? extends T>> isSameSequence(Matcher<? super T>... matchers) {
+        return isSameSequence(asList(matchers));
+    }
+
+    public static <T> Matcher<Iterable<? extends T>> isSameSequence(List<Matcher<? super T>> matchers) {
+        return matchers.size() == 0 ? emptyIterable() : contains(matchers);
+    }
+}
diff --git a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/DocumentMatcherParsingTests.java b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/DocumentMatcherParsingTests.java
index 0ee863a..c85bbe0 100644
--- a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/DocumentMatcherParsingTests.java
+++ b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/DocumentMatcherParsingTests.java
@@ -79,13 +79,13 @@ public class DocumentMatcherParsingTests {
     }
 
     private ParagraphMatcher parseParagraphMatcher(String input) {
-        TokenIterator tokens = new TokenIterator(StyleMappingTokeniser.tokenise(input));
+        TokenIterator tokens = StyleMappingTokeniser.tokenise(input);
         tokens.skip(TokenType.IDENTIFIER, "p");
         return DocumentMatcherParser.parseParagraphMatcher(tokens);
     }
 
     private RunMatcher parseRunMatcher(String input) {
-        TokenIterator tokens = new TokenIterator(StyleMappingTokeniser.tokenise(input));
+        TokenIterator tokens = StyleMappingTokeniser.tokenise(input);
         tokens.skip(TokenType.IDENTIFIER, "r");
         return DocumentMatcherParser.parseRunMatcher(tokens);
     }
diff --git a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/HtmlPathParsingTests.java b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/HtmlPathParsingTests.java
index 0b54ad8..a9b19da 100644
--- a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/HtmlPathParsingTests.java
+++ b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/HtmlPathParsingTests.java
@@ -4,7 +4,6 @@ import org.junit.Test;
 import org.zwobble.mammoth.internal.styles.HtmlPath;
 import org.zwobble.mammoth.internal.styles.parsing.HtmlPathParser;
 import org.zwobble.mammoth.internal.styles.parsing.StyleMappingTokeniser;
-import org.zwobble.mammoth.internal.styles.parsing.TokenIterator;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.zwobble.mammoth.internal.util.Lists.list;
@@ -85,6 +84,6 @@ public class HtmlPathParsingTests {
     }
 
     private HtmlPath parseHtmlPath(String input) {
-        return HtmlPathParser.parse(new TokenIterator(StyleMappingTokeniser.tokenise(input)));
+        return HtmlPathParser.parse(StyleMappingTokeniser.tokenise(input));
     }
 }
diff --git a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java
index 85fb6d4..c767957 100644
--- a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java
+++ b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java
@@ -6,13 +6,9 @@ import org.zwobble.mammoth.internal.styles.parsing.StyleMappingTokeniser;
 import org.zwobble.mammoth.internal.styles.parsing.Token;
 import org.zwobble.mammoth.internal.styles.parsing.TokenType;
 
-import java.util.List;
-
-import static java.util.Arrays.asList;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.*;
-import static org.zwobble.mammoth.internal.util.Lists.eagerConcat;
-import static org.zwobble.mammoth.internal.util.Lists.list;
+import static org.zwobble.mammoth.tests.Matchers.isSameSequence;
 
 public class StyleMappingTokeniserTests {
     @Test
@@ -21,7 +17,7 @@ public class StyleMappingTokeniserTests {
     }
 
     @Test
-    public void emptyStringIsTokenisedToEndOfFileToken() {
+    public void emptyStringIsTokenisedToNoTokens() {
         assertTokens("");
     }
 
@@ -132,9 +128,8 @@ public class StyleMappingTokeniserTests {
     }
 
     @SafeVarargs
-    private final void assertTokens(String input, Matcher<Token>... token) {
-        List<Matcher<? super Token>> tokens = eagerConcat(asList(token), list(isToken(TokenType.EOF, "")));
-        assertThat(StyleMappingTokeniser.tokenise(input), contains(tokens));
+    private final void assertTokens(String input, Matcher<Token>... tokens) {
+        assertThat(StyleMappingTokeniser.tokeniseToList(input), isSameSequence(tokens));
     }
 
     private Matcher<Token> isToken(TokenType tokenType, String value) {
diff --git a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/TokenParserTests.java b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/TokenParserTests.java
index ba0de10..aec1f17 100644
--- a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/TokenParserTests.java
+++ b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/TokenParserTests.java
@@ -17,7 +17,7 @@ public class TokenParserTests {
         Optional<String> value = TokenParser.parseClassName(new TokenIterator<>(list(
             new Token<>(0, TokenType.SYMBOL, "."),
             new Token<>(1, TokenType.IDENTIFIER, "\\:")
-        )));
+        ), new Token<>(2, TokenType.EOF, "")));
         assertEquals(Optional.of(":"), value);
     }
 
@@ -25,7 +25,7 @@ public class TokenParserTests {
     public void escapeSequencesInStringAreDecoded() {
         String value = TokenParser.parseString(new TokenIterator<>(list(
             new Token<>(0, TokenType.STRING, "'\\n'")
-        )));
+        ), new Token<>(1, TokenType.EOF, "")));
         assertEquals("\n", value);
     }
 }