diff options
8 files changed, 49 insertions, 56 deletions
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/DocumentMatcherParser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/DocumentMatcherParser.java index ce14630..ab9ff8d 100644 --- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/DocumentMatcherParser.java +++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/DocumentMatcherParser.java @@ -57,12 +57,11 @@ public class DocumentMatcherParser { } private static Optional<String> parseStyleName(TokenIterator<TokenType> tokens) { - if (tokens.peekTokenType() == TokenType.OPEN_SQUARE_BRACKET) { - tokens.skip(); + if (tokens.trySkip(TokenType.SYMBOL, "[")) { tokens.skip(TokenType.IDENTIFIER, "style-name"); - tokens.skip(TokenType.EQUALS); + tokens.skip(TokenType.SYMBOL, "="); String value = TokenParser.parseString(tokens); - tokens.skip(TokenType.CLOSE_SQUARE_BRACKET); + tokens.skip(TokenType.SYMBOL, "]"); return Optional.of(value); } else { return Optional.empty(); @@ -70,12 +69,11 @@ public class DocumentMatcherParser { } private static Optional<NumberingLevel> parseNumbering(TokenIterator<TokenType> tokens) { - if (tokens.peekTokenType() == TokenType.COLON) { - tokens.skip(); + if (tokens.trySkip(TokenType.SYMBOL, ":")) { boolean isOrdered = parseListType(tokens); - tokens.skip(TokenType.OPEN_PAREN); + tokens.skip(TokenType.SYMBOL, "("); String level = new BigInteger(tokens.nextValue(TokenType.INTEGER)).subtract(BigInteger.ONE).toString(); - tokens.skip(TokenType.CLOSE_PAREN); + tokens.skip(TokenType.SYMBOL, ")"); return Optional.of(new NumberingLevel(level, isOrdered)); } else { return Optional.empty(); diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/HtmlPathParser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/HtmlPathParser.java index ef975f6..b384ef7 100644 --- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/HtmlPathParser.java +++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/HtmlPathParser.java @@ -13,8 +13,7 @@ import static org.zwobble.mammoth.internal.util.Maps.map; public class HtmlPathParser { public static HtmlPath parse(TokenIterator<TokenType> tokens) { - if (tokens.peekTokenType() == TokenType.BANG) { - tokens.skip(TokenType.BANG); + if (tokens.trySkip(TokenType.SYMBOL, "!")) { return HtmlPath.IGNORE; } else { return parseHtmlPathElements(tokens); @@ -27,9 +26,9 @@ public class HtmlPathParser { if (tokens.peekTokenType() == TokenType.IDENTIFIER) { HtmlPathElement element = parseElement(tokens); elements.add(element); - while (tokens.peekTokenType() == TokenType.WHITESPACE && tokens.peekTokenType(1) == TokenType.GREATER_THAN) { + while (tokens.peekTokenType() == TokenType.WHITESPACE && tokens.isNext(1, TokenType.SYMBOL, ">")) { tokens.skip(TokenType.WHITESPACE); - tokens.skip(TokenType.GREATER_THAN); + tokens.skip(TokenType.SYMBOL, ">"); tokens.skip(TokenType.WHITESPACE); elements.add(parseElement(tokens)); } @@ -52,8 +51,7 @@ public class HtmlPathParser { private static List<String> parseTagNames(TokenIterator<TokenType> tokens) { List<String> tagNames = new ArrayList<>(); tagNames.add(tokens.nextValue(TokenType.IDENTIFIER)); - while (tokens.peekTokenType() == TokenType.CHOICE) { - tokens.skip(); + while (tokens.trySkip(TokenType.SYMBOL, "|")) { tagNames.add(tokens.nextValue(TokenType.IDENTIFIER)); } return tagNames; @@ -69,20 +67,20 @@ public class HtmlPathParser { private static boolean parseIsFresh(TokenIterator<TokenType> tokens) { return tokens.tryParse(() -> { - tokens.skip(TokenType.COLON); + tokens.skip(TokenType.SYMBOL, ":"); tokens.skip(TokenType.IDENTIFIER, "fresh"); }); } private static String parseSeparator(TokenIterator<TokenType> tokens) { boolean isSeparator = tokens.tryParse(() -> { - tokens.skip(TokenType.COLON); + tokens.skip(TokenType.SYMBOL, ":"); tokens.skip(TokenType.IDENTIFIER, "separator"); }); if (isSeparator) { - tokens.skip(TokenType.OPEN_PAREN); + tokens.skip(TokenType.SYMBOL, "("); String value = TokenParser.parseString(tokens); - tokens.skip(TokenType.CLOSE_PAREN); + tokens.skip(TokenType.SYMBOL, ")"); return value; } else { return ""; diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java index c8dc221..002a5aa 100644 --- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java +++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java @@ -46,7 +46,7 @@ public class StyleMapParser { BiConsumer<StyleMapBuilder, HtmlPath> documentMatcher = DocumentMatcherParser.parse(tokens); tokens.skip(TokenType.WHITESPACE); - tokens.skip(TokenType.ARROW); + tokens.skip(TokenType.SYMBOL, "=>"); HtmlPath htmlPath = parseHtmlPath(tokens); diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java index 1eec870..704ba86 100644 --- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java +++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java @@ -13,20 +13,11 @@ public class StyleMappingTokeniser { list( RegexTokeniser.rule(TokenType.IDENTIFIER, "([a-zA-Z][a-zA-Z0-9\\-]*)"), RegexTokeniser.rule(TokenType.CLASS_NAME, "(\\.(?:[a-zA-Z0-9\\-]|\\\\.)+)"), - RegexTokeniser.rule(TokenType.COLON, ":"), - RegexTokeniser.rule(TokenType.GREATER_THAN, ">"), + RegexTokeniser.rule(TokenType.SYMBOL, ":|>|=>|=|\\(|\\)|\\[|\\]|\\||!"), RegexTokeniser.rule(TokenType.WHITESPACE, "\\s+"), - RegexTokeniser.rule(TokenType.ARROW, "=>"), - RegexTokeniser.rule(TokenType.EQUALS, "="), - RegexTokeniser.rule(TokenType.OPEN_PAREN, "\\("), - RegexTokeniser.rule(TokenType.CLOSE_PAREN, "\\)"), - RegexTokeniser.rule(TokenType.OPEN_SQUARE_BRACKET, "\\["), - RegexTokeniser.rule(TokenType.CLOSE_SQUARE_BRACKET, "\\]"), RegexTokeniser.rule(TokenType.STRING, stringPrefix + "'"), RegexTokeniser.rule(TokenType.UNTERMINATED_STRING, stringPrefix), - RegexTokeniser.rule(TokenType.INTEGER, "([0-9]+)"), - RegexTokeniser.rule(TokenType.CHOICE, "\\|"), - RegexTokeniser.rule(TokenType.BANG, "!") + RegexTokeniser.rule(TokenType.INTEGER, "([0-9]+)") ) ); List<Token<TokenType>> tokens = tokeniser.tokenise(line); diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java index f79974d..7ac7643 100644 --- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java +++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java @@ -11,16 +11,31 @@ public class TokenIterator<T> { this.index = 0; } - public T peekTokenType() { - return tokens.get(index).getTokenType(); + public boolean isNext(int offset, T tokenType, String value) { + int tokenIndex = index + offset; + if (tokenIndex < tokens.size()) { + Token<T> token = tokens.get(tokenIndex); + return token.getTokenType().equals(tokenType) && token.getValue().equals(value); + } else { + return false; + } } - public T peekTokenType(int count) { - return peekToken(count).getTokenType(); + public boolean isNext(T tokenType, String value) { + return isNext(0, tokenType, value); } - public Token<T> peekToken(int count) { - return tokens.get(index + count); + public boolean trySkip(T tokenType, String value) { + if (isNext(tokenType, value)) { + skip(); + return true; + } else { + return false; + } + } + + public T peekTokenType() { + return tokens.get(index).getTokenType(); } public Token<T> next(T type) { diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenType.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenType.java index f17b85e..17442b5 100644 --- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenType.java +++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenType.java @@ -4,19 +4,10 @@ public enum TokenType { WHITESPACE, IDENTIFIER, CLASS_NAME, - COLON, - GREATER_THAN, - ARROW, - EQUALS, - OPEN_PAREN, - CLOSE_PAREN, - OPEN_SQUARE_BRACKET, - CLOSE_SQUARE_BRACKET, + SYMBOL, STRING, UNTERMINATED_STRING, INTEGER, - CHOICE, - BANG, EOF, UNKNOWN } diff --git a/src/test/java/org/zwobble/mammoth/tests/MammothTests.java b/src/test/java/org/zwobble/mammoth/tests/MammothTests.java index 3bab856..8454a3d 100644 --- a/src/test/java/org/zwobble/mammoth/tests/MammothTests.java +++ b/src/test/java/org/zwobble/mammoth/tests/MammothTests.java @@ -213,7 +213,7 @@ public class MammothTests { assertThat( exception.getMessage(), equalTo( - "error reading style map at line 2, character 14: expected token of type STRING but was of type CLOSE_SQUARE_BRACKET\n\n" + + "error reading style map at line 2, character 14: expected token of type STRING but was of type SYMBOL\n\n" + "p[style-name=] =>\n" + " ^")); } diff --git a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java index a7edf9f..0e0c920 100644 --- a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java +++ b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java @@ -52,7 +52,7 @@ public class StyleMappingTokeniserTests { @Test public void arrowsAreTokenised() { - assertTokens("=>=>", isToken(TokenType.ARROW, "=>"), isToken(TokenType.ARROW, "=>")); + assertTokens("=>=>", isToken(TokenType.SYMBOL, "=>"), isToken(TokenType.SYMBOL, "=>")); } @Test @@ -62,42 +62,42 @@ public class StyleMappingTokeniserTests { @Test public void colonsAreTokenised() { - assertTokens("::", isToken(TokenType.COLON, ":"), isToken(TokenType.COLON, ":")); + assertTokens("::", isToken(TokenType.SYMBOL, ":"), isToken(TokenType.SYMBOL, ":")); } @Test public void greaterThansAreTokenised() { - assertTokens(">>", isToken(TokenType.GREATER_THAN, ">"), isToken(TokenType.GREATER_THAN, ">")); + assertTokens(">>", isToken(TokenType.SYMBOL, ">"), isToken(TokenType.SYMBOL, ">")); } @Test public void equalsAreTokenised() { - assertTokens("==", isToken(TokenType.EQUALS, "="), isToken(TokenType.EQUALS, "=")); + assertTokens("==", isToken(TokenType.SYMBOL, "="), isToken(TokenType.SYMBOL, "=")); } @Test public void openParensAreTokenised() { - assertTokens("((", isToken(TokenType.OPEN_PAREN, "("), isToken(TokenType.OPEN_PAREN, "(")); + assertTokens("((", isToken(TokenType.SYMBOL, "("), isToken(TokenType.SYMBOL, "(")); } @Test public void closeParensAreTokenised() { - assertTokens("))", isToken(TokenType.CLOSE_PAREN, ")"), isToken(TokenType.CLOSE_PAREN, ")")); + assertTokens("))", isToken(TokenType.SYMBOL, ")"), isToken(TokenType.SYMBOL, ")")); } @Test public void openSquareBracketsAreTokenised() { - assertTokens("[[", isToken(TokenType.OPEN_SQUARE_BRACKET, "["), isToken(TokenType.OPEN_SQUARE_BRACKET, "[")); + assertTokens("[[", isToken(TokenType.SYMBOL, "["), isToken(TokenType.SYMBOL, "[")); } @Test public void closeSquareBracketsAreTokenised() { - assertTokens("]]", isToken(TokenType.CLOSE_SQUARE_BRACKET, "]"), isToken(TokenType.CLOSE_SQUARE_BRACKET, "]")); + assertTokens("]]", isToken(TokenType.SYMBOL, "]"), isToken(TokenType.SYMBOL, "]")); } @Test public void choicesAreTokenised() { - assertTokens("||", isToken(TokenType.CHOICE, "|"), isToken(TokenType.CHOICE, "|")); + assertTokens("||", isToken(TokenType.SYMBOL, "|"), isToken(TokenType.SYMBOL, "|")); } @Test |