From d078ad2ad09fb504b46f07b75df4fe365a59c6a4 Mon Sep 17 00:00:00 2001 From: Michael Williamson Date: Fri, 14 Apr 2017 17:41:53 +0100 Subject: Always return end tokens from end of token iterator --- .../internal/styles/parsing/StyleMapParser.java | 2 +- .../styles/parsing/StyleMappingTokeniser.java | 13 +++++++--- .../internal/styles/parsing/TokenIterator.java | 30 +++++++++++++--------- .../java/org/zwobble/mammoth/tests/Matchers.java | 21 +++++++++++++++ .../parsing/DocumentMatcherParsingTests.java | 4 +-- .../tests/styles/parsing/HtmlPathParsingTests.java | 3 +-- .../styles/parsing/StyleMappingTokeniserTests.java | 13 +++------- .../tests/styles/parsing/TokenParserTests.java | 4 +-- 8 files changed, 58 insertions(+), 32 deletions(-) create mode 100644 src/test/java/org/zwobble/mammoth/tests/Matchers.java (limited to 'src') diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java index bb8c0cc..5113a4f 100644 --- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java +++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java @@ -41,7 +41,7 @@ public class StyleMapParser { } private static Consumer parseStyleMapping(String line) { - TokenIterator tokens = new TokenIterator<>(StyleMappingTokeniser.tokenise(line)); + TokenIterator tokens = StyleMappingTokeniser.tokenise(line); BiConsumer documentMatcher = DocumentMatcherParser.parse(tokens); diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java index b0b44c3..9625d24 100644 --- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java +++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java @@ -5,7 +5,14 @@ import java.util.List; import static org.zwobble.mammoth.internal.util.Lists.list; public class StyleMappingTokeniser { - public static List> tokenise(String line) { + public static TokenIterator tokenise(String line) { + return new TokenIterator<>( + tokeniseToList(line), + new Token<>(line.length(), TokenType.EOF, "") + ); + } + + public static List> tokeniseToList(String line) { String stringPrefix = "'(?:(?:\\\\.|[^'])*)"; String identifierCharacter = "(?:[a-zA-Z\\-_]|\\\\.)"; @@ -20,8 +27,6 @@ public class StyleMappingTokeniser { RegexTokeniser.rule(TokenType.INTEGER, "[0-9]+") ) ); - List> tokens = tokeniser.tokenise(line); - tokens.add(new Token<>(line.length(), TokenType.EOF, "")); - return tokens; + return tokeniser.tokenise(line); } } diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java index d75e896..69ad7a7 100644 --- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java +++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java @@ -6,21 +6,19 @@ import static org.zwobble.mammoth.internal.styles.parsing.LineParseException.lin public class TokenIterator { private final List> tokens; + private final Token end; private int index; - public TokenIterator(List> tokens) { + public TokenIterator(List> tokens, Token end) { this.tokens = tokens; + this.end = end; this.index = 0; } public boolean isNext(int offset, T tokenType, String value) { int tokenIndex = index + offset; - if (tokenIndex < tokens.size()) { - Token token = tokens.get(tokenIndex); - return token.getTokenType().equals(tokenType) && token.getValue().equals(value); - } else { - return false; - } + Token token = getToken(tokenIndex); + return token.getTokenType().equals(tokenType) && token.getValue().equals(value); } public boolean isNext(T tokenType, String value) { @@ -37,17 +35,17 @@ public class TokenIterator { } public T peekTokenType() { - return tokens.get(index).getTokenType(); + return getToken(index).getTokenType(); } public Token next() { - Token token = tokens.get(index); + Token token = getToken(index); index += 1; return token; } public Token next(T type) { - Token token = tokens.get(index); + Token token = getToken(index); if (token.getTokenType().equals(type)) { index += 1; return token; @@ -65,7 +63,7 @@ public class TokenIterator { } public void skip(T tokenType) { - Token token = tokens.get(index); + Token token = getToken(index); if (!token.getTokenType().equals(tokenType)) { throw unexpectedTokenType(tokenType, token); } @@ -73,7 +71,7 @@ public class TokenIterator { } public void skip(T tokenType, String tokenValue) { - Token token = tokens.get(index); + Token token = getToken(index); if (!token.getTokenType().equals(tokenType)) { throw unexpectedTokenType(tokenType, token); } @@ -100,6 +98,14 @@ public class TokenIterator { } } + private Token getToken(int index) { + if (index < tokens.size()) { + return tokens.get(index); + } else { + return end; + } + } + public interface Action { void run(); } diff --git a/src/test/java/org/zwobble/mammoth/tests/Matchers.java b/src/test/java/org/zwobble/mammoth/tests/Matchers.java new file mode 100644 index 0000000..de582c1 --- /dev/null +++ b/src/test/java/org/zwobble/mammoth/tests/Matchers.java @@ -0,0 +1,21 @@ +package org.zwobble.mammoth.tests; + +import org.hamcrest.Matcher; + +import java.util.List; + +import static java.util.Arrays.asList; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.emptyIterable; + +public class Matchers { + private Matchers() {} + + public static Matcher> isSameSequence(Matcher... matchers) { + return isSameSequence(asList(matchers)); + } + + public static Matcher> isSameSequence(List> matchers) { + return matchers.size() == 0 ? emptyIterable() : contains(matchers); + } +} diff --git a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/DocumentMatcherParsingTests.java b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/DocumentMatcherParsingTests.java index 0ee863a..c85bbe0 100644 --- a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/DocumentMatcherParsingTests.java +++ b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/DocumentMatcherParsingTests.java @@ -79,13 +79,13 @@ public class DocumentMatcherParsingTests { } private ParagraphMatcher parseParagraphMatcher(String input) { - TokenIterator tokens = new TokenIterator(StyleMappingTokeniser.tokenise(input)); + TokenIterator tokens = StyleMappingTokeniser.tokenise(input); tokens.skip(TokenType.IDENTIFIER, "p"); return DocumentMatcherParser.parseParagraphMatcher(tokens); } private RunMatcher parseRunMatcher(String input) { - TokenIterator tokens = new TokenIterator(StyleMappingTokeniser.tokenise(input)); + TokenIterator tokens = StyleMappingTokeniser.tokenise(input); tokens.skip(TokenType.IDENTIFIER, "r"); return DocumentMatcherParser.parseRunMatcher(tokens); } diff --git a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/HtmlPathParsingTests.java b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/HtmlPathParsingTests.java index 0b54ad8..a9b19da 100644 --- a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/HtmlPathParsingTests.java +++ b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/HtmlPathParsingTests.java @@ -4,7 +4,6 @@ import org.junit.Test; import org.zwobble.mammoth.internal.styles.HtmlPath; import org.zwobble.mammoth.internal.styles.parsing.HtmlPathParser; import org.zwobble.mammoth.internal.styles.parsing.StyleMappingTokeniser; -import org.zwobble.mammoth.internal.styles.parsing.TokenIterator; import static org.hamcrest.MatcherAssert.assertThat; import static org.zwobble.mammoth.internal.util.Lists.list; @@ -85,6 +84,6 @@ public class HtmlPathParsingTests { } private HtmlPath parseHtmlPath(String input) { - return HtmlPathParser.parse(new TokenIterator(StyleMappingTokeniser.tokenise(input))); + return HtmlPathParser.parse(StyleMappingTokeniser.tokenise(input)); } } diff --git a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java index 85fb6d4..c767957 100644 --- a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java +++ b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java @@ -6,13 +6,9 @@ import org.zwobble.mammoth.internal.styles.parsing.StyleMappingTokeniser; import org.zwobble.mammoth.internal.styles.parsing.Token; import org.zwobble.mammoth.internal.styles.parsing.TokenType; -import java.util.List; - -import static java.util.Arrays.asList; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.*; -import static org.zwobble.mammoth.internal.util.Lists.eagerConcat; -import static org.zwobble.mammoth.internal.util.Lists.list; +import static org.zwobble.mammoth.tests.Matchers.isSameSequence; public class StyleMappingTokeniserTests { @Test @@ -21,7 +17,7 @@ public class StyleMappingTokeniserTests { } @Test - public void emptyStringIsTokenisedToEndOfFileToken() { + public void emptyStringIsTokenisedToNoTokens() { assertTokens(""); } @@ -132,9 +128,8 @@ public class StyleMappingTokeniserTests { } @SafeVarargs - private final void assertTokens(String input, Matcher... token) { - List> tokens = eagerConcat(asList(token), list(isToken(TokenType.EOF, ""))); - assertThat(StyleMappingTokeniser.tokenise(input), contains(tokens)); + private final void assertTokens(String input, Matcher... tokens) { + assertThat(StyleMappingTokeniser.tokeniseToList(input), isSameSequence(tokens)); } private Matcher isToken(TokenType tokenType, String value) { diff --git a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/TokenParserTests.java b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/TokenParserTests.java index ba0de10..aec1f17 100644 --- a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/TokenParserTests.java +++ b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/TokenParserTests.java @@ -17,7 +17,7 @@ public class TokenParserTests { Optional value = TokenParser.parseClassName(new TokenIterator<>(list( new Token<>(0, TokenType.SYMBOL, "."), new Token<>(1, TokenType.IDENTIFIER, "\\:") - ))); + ), new Token<>(2, TokenType.EOF, ""))); assertEquals(Optional.of(":"), value); } @@ -25,7 +25,7 @@ public class TokenParserTests { public void escapeSequencesInStringAreDecoded() { String value = TokenParser.parseString(new TokenIterator<>(list( new Token<>(0, TokenType.STRING, "'\\n'") - ))); + ), new Token<>(1, TokenType.EOF, ""))); assertEquals("\n", value); } } -- cgit v1.2.3