diff options
Diffstat (limited to 'src/main/java/org/zwobble')
7 files changed, 112 insertions, 143 deletions
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/DocumentMatcherParser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/DocumentMatcherParser.java index e7de55c..e019cff 100644 --- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/DocumentMatcherParser.java +++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/DocumentMatcherParser.java @@ -11,8 +11,8 @@ import java.util.Optional; import java.util.function.BiConsumer; public class DocumentMatcherParser { - public static BiConsumer<StyleMapBuilder, HtmlPath> parse(TokenIterator tokens) { - Token identifier = tokens.next(TokenType.IDENTIFIER); + public static BiConsumer<StyleMapBuilder, HtmlPath> parse(TokenIterator<TokenType> tokens) { + Token<TokenType> identifier = tokens.next(TokenType.IDENTIFIER); switch (identifier.getValue()) { case "p": ParagraphMatcher paragraph = parseParagraphMatcher(tokens); @@ -35,20 +35,20 @@ public class DocumentMatcherParser { } } - public static ParagraphMatcher parseParagraphMatcher(TokenIterator tokens) { + public static ParagraphMatcher parseParagraphMatcher(TokenIterator<TokenType> tokens) { Optional<String> styleId = parseStyleId(tokens); Optional<String> styleName = parseStyleName(tokens); Optional<NumberingLevel> numbering = parseNumbering(tokens); return new ParagraphMatcher(styleId, styleName, numbering); } - public static RunMatcher parseRunMatcher(TokenIterator tokens) { + public static RunMatcher parseRunMatcher(TokenIterator<TokenType> tokens) { Optional<String> styleId = parseStyleId(tokens); Optional<String> styleName = parseStyleName(tokens); return new RunMatcher(styleId, styleName); } - private static Optional<String> parseStyleId(TokenIterator tokens) { + private static Optional<String> parseStyleId(TokenIterator<TokenType> tokens) { if (tokens.peekTokenType() == TokenType.DOT) { tokens.skip(); return Optional.of(tokens.nextValue(TokenType.IDENTIFIER)); @@ -57,7 +57,7 @@ public class DocumentMatcherParser { } } - private static Optional<String> parseStyleName(TokenIterator tokens) { + private static Optional<String> parseStyleName(TokenIterator<TokenType> tokens) { if (tokens.peekTokenType() == TokenType.OPEN_SQUARE_BRACKET) { tokens.skip(); tokens.skip(TokenType.IDENTIFIER, "style-name"); @@ -70,7 +70,7 @@ public class DocumentMatcherParser { } } - private static Optional<NumberingLevel> parseNumbering(TokenIterator tokens) { + private static Optional<NumberingLevel> parseNumbering(TokenIterator<TokenType> tokens) { if (tokens.peekTokenType() == TokenType.COLON) { tokens.skip(); boolean isOrdered = parseListType(tokens); @@ -83,8 +83,8 @@ public class DocumentMatcherParser { } } - private static boolean parseListType(TokenIterator tokens) { - Token listType = tokens.next(TokenType.IDENTIFIER); + private static boolean parseListType(TokenIterator<TokenType> tokens) { + Token<TokenType> listType = tokens.next(TokenType.IDENTIFIER); switch (listType.getValue()) { case "ordered-list": return true; diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/HtmlPathParser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/HtmlPathParser.java index e7eefe2..2db2553 100644 --- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/HtmlPathParser.java +++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/HtmlPathParser.java @@ -11,7 +11,7 @@ import java.util.Map; import static org.zwobble.mammoth.internal.util.Maps.map; public class HtmlPathParser { - public static HtmlPath parse(TokenIterator tokens) { + public static HtmlPath parse(TokenIterator<TokenType> tokens) { if (tokens.peekTokenType() == TokenType.BANG) { tokens.skip(TokenType.BANG); return HtmlPath.IGNORE; @@ -20,7 +20,7 @@ public class HtmlPathParser { } } - private static HtmlPath parseHtmlPathElements(TokenIterator tokens) { + private static HtmlPath parseHtmlPathElements(TokenIterator<TokenType> tokens) { List<HtmlPathElement> elements = new ArrayList<>(); if (tokens.peekTokenType() == TokenType.IDENTIFIER) { @@ -37,7 +37,7 @@ public class HtmlPathParser { return new HtmlPathElements(elements); } - private static HtmlPathElement parseElement(TokenIterator tokens) { + private static HtmlPathElement parseElement(TokenIterator<TokenType> tokens) { List<String> tagNames = parseTagNames(tokens); List<String> classNames = parseClassNames(tokens); Map<String, String> attributes = classNames.isEmpty() @@ -47,7 +47,7 @@ public class HtmlPathParser { return new HtmlPathElement(tagNames, attributes, !isFresh); } - private static List<String> parseTagNames(TokenIterator tokens) { + private static List<String> parseTagNames(TokenIterator<TokenType> tokens) { List<String> tagNames = new ArrayList<>(); tagNames.add(tokens.nextValue(TokenType.IDENTIFIER)); while (tokens.peekTokenType() == TokenType.CHOICE) { @@ -57,7 +57,7 @@ public class HtmlPathParser { return tagNames; } - private static List<String> parseClassNames(TokenIterator tokens) { + private static List<String> parseClassNames(TokenIterator<TokenType> tokens) { List<String> classNames = new ArrayList<>(); while (tokens.peekTokenType() == TokenType.DOT) { tokens.skip(); @@ -66,7 +66,7 @@ public class HtmlPathParser { return classNames; } - private static boolean parseIsFresh(TokenIterator tokens) { + private static boolean parseIsFresh(TokenIterator<TokenType> tokens) { if (tokens.peekTokenType() == TokenType.COLON) { tokens.skip(); tokens.skip(TokenType.IDENTIFIER, "fresh"); diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/RegexTokeniser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/RegexTokeniser.java new file mode 100644 index 0000000..06ecbd3 --- /dev/null +++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/RegexTokeniser.java @@ -0,0 +1,52 @@ +package org.zwobble.mammoth.internal.styles.parsing; + +import java.nio.CharBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class RegexTokeniser<T> { + public static <T> TokenRule<T> rule(T type, String regex) { + return new TokenRule<>(type, Pattern.compile(regex)); + } + + public static class TokenRule<T> { + private final T type; + private final Pattern regex; + + public TokenRule(T type, Pattern regex) { + this.type = type; + this.regex = regex; + } + } + + private final List<TokenRule<T>> rules; + + public RegexTokeniser(List<TokenRule<T>> rules) { + this.rules = rules; + } + + public List<Token<T>> tokenise(String value) { + List<Token<T>> tokens = new ArrayList<>(); + CharBuffer remaining = CharBuffer.wrap(value); + while (remaining.hasRemaining()) { + boolean matched = false; + for (TokenRule<T> rule : rules) { + Matcher matcher = rule.regex.matcher(remaining); + if (matcher.lookingAt()) { + tokens.add(new Token<>(remaining.position(), rule.type, matcher.group())); + remaining.position(remaining.position() + matcher.end()); + matched = true; + break; + } + } + if (!matched) { + throw new RuntimeException("Remaining: " + remaining); + } + } + return tokens; + } + + +} diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java index 8dd4099..c8dc221 100644 --- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java +++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java @@ -41,7 +41,7 @@ public class StyleMapParser { } private static Consumer<StyleMapBuilder> parseStyleMapping(String line) { - TokenIterator tokens = new TokenIterator(StyleMappingTokeniser.tokenise(line)); + TokenIterator<TokenType> tokens = new TokenIterator<>(StyleMappingTokeniser.tokenise(line)); BiConsumer<StyleMapBuilder, HtmlPath> documentMatcher = DocumentMatcherParser.parse(tokens); @@ -55,7 +55,7 @@ public class StyleMapParser { return styleMap -> documentMatcher.accept(styleMap, htmlPath); } - private static HtmlPath parseHtmlPath(TokenIterator tokens) { + private static HtmlPath parseHtmlPath(TokenIterator<TokenType> tokens) { if (tokens.peekTokenType() == TokenType.EOF) { return HtmlPath.EMPTY; } else { diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java index 28400ff..bfef347 100644 --- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java +++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java @@ -1,118 +1,35 @@ package org.zwobble.mammoth.internal.styles.parsing; -import java.util.ArrayList; import java.util.List; -public class StyleMappingTokeniser { - public static List<Token> tokenise(String line) { - List<Token> tokens = new ArrayList<>(); - - int index = 0; - while (index < line.length()) { - Token token = nextToken(line, index); - tokens.add(token); - index += token.getValue().length(); - } +import static org.zwobble.mammoth.internal.util.Lists.list; - tokens.add(new Token(index, TokenType.EOF, "")); +public class StyleMappingTokeniser { + public static List<Token<TokenType>> tokenise(String line) { + String stringPrefix = "'((?:\\.|[^'])*)"; + + RegexTokeniser<TokenType> tokeniser = new RegexTokeniser<>( + list( + RegexTokeniser.rule(TokenType.IDENTIFIER, "([a-zA-Z][a-zA-Z0-9\\-]*)"), + RegexTokeniser.rule(TokenType.DOT, "\\."), + RegexTokeniser.rule(TokenType.COLON, ":"), + RegexTokeniser.rule(TokenType.GREATER_THAN, ">"), + RegexTokeniser.rule(TokenType.WHITESPACE, "\\s+"), + RegexTokeniser.rule(TokenType.ARROW, "=>"), + RegexTokeniser.rule(TokenType.EQUALS, "="), + RegexTokeniser.rule(TokenType.OPEN_PAREN, "\\("), + RegexTokeniser.rule(TokenType.CLOSE_PAREN, "\\)"), + RegexTokeniser.rule(TokenType.OPEN_SQUARE_BRACKET, "\\["), + RegexTokeniser.rule(TokenType.CLOSE_SQUARE_BRACKET, "\\]"), + RegexTokeniser.rule(TokenType.STRING, stringPrefix + "'"), + RegexTokeniser.rule(TokenType.UNTERMINATED_STRING, stringPrefix), + RegexTokeniser.rule(TokenType.INTEGER, "([0-9]+)"), + RegexTokeniser.rule(TokenType.CHOICE, "\\|"), + RegexTokeniser.rule(TokenType.BANG, "!") + ) + ); + List<Token<TokenType>> tokens = tokeniser.tokenise(line); + tokens.add(new Token<>(line.length(), TokenType.EOF, "")); return tokens; } - - private static Token nextToken(String line, int startIndex) { - char nextChar = line.charAt(startIndex); - TokenType tokenType = tokenType(nextChar); - int index = 0; - if (nextChar == '=' && startIndex + 1 < line.length() && line.charAt(startIndex + 1) == '>') { - tokenType = TokenType.ARROW; - index = startIndex + 2; - } else { - index = seekEndOfToken(line, startIndex + 1, tokenType); - if (tokenType == TokenType.STRING && index == line.length() + 1) { - tokenType = TokenType.UNTERMINATED_STRING; - } - } - index = Math.min(index, line.length()); - return new Token(startIndex, tokenType, line.substring(startIndex, index)); - } - - private static TokenType tokenType(char nextChar) { - if (isAlphabetic(nextChar)) { - return TokenType.IDENTIFIER; - } else if (isWhitespace(nextChar)) { - return TokenType.WHITESPACE; - } else if (isNumeric(nextChar)) { - return TokenType.INTEGER; - } else { - switch (nextChar) { - case '\'': - return TokenType.STRING; - case '.': - return TokenType.DOT; - case ':': - return TokenType.COLON; - case '>': - return TokenType.GREATER_THAN; - case '=': - return TokenType.EQUALS; - case '(': - return TokenType.OPEN_PAREN; - case ')': - return TokenType.CLOSE_PAREN; - case '[': - return TokenType.OPEN_SQUARE_BRACKET; - case ']': - return TokenType.CLOSE_SQUARE_BRACKET; - case '|': - return TokenType.CHOICE; - case '!': - return TokenType.BANG; - default: - return TokenType.UNKNOWN; - } - } - } - - private static int seekEndOfToken(String line, int index, TokenType tokenType) { - switch (tokenType) { - case WHITESPACE: - while (index < line.length() && isWhitespace(line.charAt(index))) { - index += 1; - } - return index; - case IDENTIFIER: - while (index < line.length() && isIdentifierCharacter(line.charAt(index))) { - index += 1; - } - return index; - case INTEGER: - while (index < line.length() && isNumeric(line.charAt(index))) { - index += 1; - } - return index; - case STRING: - while (index < line.length() && line.charAt(index) != '\'') { - index += 1; - } - index += 1; - return index; - default: - return index; - } - } - - private static boolean isWhitespace(char c) { - return c == ' ' || c == '\t'; - } - - private static boolean isIdentifierCharacter(char c) { - return isAlphabetic(c) || isNumeric(c) || c == '-' || c == '_'; - } - - private static boolean isAlphabetic(char nextChar) { - return (nextChar >= 'a' && nextChar <= 'z') || (nextChar >= 'A' && nextChar <= 'Z'); - } - - private static boolean isNumeric(char c) { - return c >= '0' && c <= '9'; - } } diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/Token.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/Token.java index 3059e5b..bf009c0 100644 --- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/Token.java +++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/Token.java @@ -1,11 +1,11 @@ package org.zwobble.mammoth.internal.styles.parsing; -public class Token { +public class Token<T> { private final int characterIndex; - private final TokenType tokenType; + private final T tokenType; private final String value; - public Token(int characterIndex, TokenType tokenType, String value) { + public Token(int characterIndex, T tokenType, String value) { this.characterIndex = characterIndex; this.tokenType = tokenType; this.value = value; @@ -15,7 +15,7 @@ public class Token { return characterIndex; } - public TokenType getTokenType() { + public T getTokenType() { return tokenType; } diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java index 805e5f0..0bb3f46 100644 --- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java +++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java @@ -2,25 +2,25 @@ package org.zwobble.mammoth.internal.styles.parsing; import java.util.List; -public class TokenIterator { - private final List<Token> tokens; +public class TokenIterator<T> { + private final List<Token<T>> tokens; private int index; - public TokenIterator(List<Token> tokens) { + public TokenIterator(List<Token<T>> tokens) { this.tokens = tokens; this.index = 0; } - public TokenType peekTokenType() { + public T peekTokenType() { return tokens.get(index).getTokenType(); } - public TokenType peekTokenType(int count) { + public T peekTokenType(int count) { return tokens.get(index + count).getTokenType(); } - public Token next(TokenType type) { - Token token = tokens.get(index); + public Token<T> next(T type) { + Token<T> token = tokens.get(index); if (token.getTokenType() == type) { index += 1; return token; @@ -29,7 +29,7 @@ public class TokenIterator { } } - public String nextValue(TokenType type) { + public String nextValue(T type) { return next(type).getValue(); } @@ -37,7 +37,7 @@ public class TokenIterator { index += 1; } - public void skip(TokenType tokenType) { + public void skip(T tokenType) { Token token = tokens.get(index); if (token.getTokenType() != tokenType) { throw unexpectedTokenType(tokenType, token); @@ -45,8 +45,8 @@ public class TokenIterator { index += 1; } - public void skip(TokenType tokenType, String tokenValue) { - Token token = tokens.get(index); + public void skip(T tokenType, String tokenValue) { + Token<T> token = tokens.get(index); if (token.getTokenType() != tokenType) { throw unexpectedTokenType(tokenType, token); } @@ -57,7 +57,7 @@ public class TokenIterator { index += 1; } - private LineParseException unexpectedTokenType(TokenType expected, Token actual) { + private LineParseException unexpectedTokenType(T expected, Token<T> actual) { return new LineParseException(actual, "expected token of type " + expected + " but was of type " + actual.getTokenType()); } } |