diff options
Diffstat (limited to 'src')
14 files changed, 541 insertions, 271 deletions
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/DocumentMatcherParser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/DocumentMatcherParser.java new file mode 100644 index 0000000..0998f8b --- /dev/null +++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/DocumentMatcherParser.java @@ -0,0 +1,94 @@ +package org.zwobble.mammoth.internal.styles.parsing; + +import org.zwobble.mammoth.internal.documents.NumberingLevel; +import org.zwobble.mammoth.internal.styles.HtmlPath; +import org.zwobble.mammoth.internal.styles.ParagraphMatcher; +import org.zwobble.mammoth.internal.styles.RunMatcher; +import org.zwobble.mammoth.internal.styles.StyleMapBuilder; + +import java.math.BigInteger; +import java.util.Optional; +import java.util.function.BiConsumer; + +public class DocumentMatcherParser { + public static BiConsumer<StyleMapBuilder, HtmlPath> parse(TokenIterator tokens) { + String identifier = tokens.nextValue(TokenType.IDENTIFIER); + switch (identifier) { + case "p": + ParagraphMatcher paragraph = parseParagraphMatcher(tokens); + return (builder, path) -> builder.mapParagraph(paragraph, path); + case "r": + RunMatcher run = parseRunMatcher(tokens); + return (builder, path) -> builder.mapRun(run, path); + case "b": + return StyleMapBuilder::bold; + case "i": + return StyleMapBuilder::italic; + case "u": + return StyleMapBuilder::underline; + case "strike": + return StyleMapBuilder::strikethrough; + default: + throw new IllegalArgumentException("Unrecognised document element: " + identifier); + } + } + + public static ParagraphMatcher parseParagraphMatcher(TokenIterator tokens) { + Optional<String> styleId = parseStyleId(tokens); + Optional<String> styleName = parseStyleName(tokens); + Optional<NumberingLevel> numbering = parseNumbering(tokens); + return new ParagraphMatcher(styleId, styleName, numbering); + } + + public static RunMatcher parseRunMatcher(TokenIterator tokens) { + Optional<String> styleId = parseStyleId(tokens); + Optional<String> styleName = parseStyleName(tokens); + return new RunMatcher(styleId, styleName); + } + + private static Optional<String> parseStyleId(TokenIterator tokens) { + if (tokens.peekTokenType() == TokenType.DOT) { + tokens.skip(); + return Optional.of(tokens.nextValue(TokenType.IDENTIFIER)); + } else { + return Optional.empty(); + } + } + + private static Optional<String> parseStyleName(TokenIterator tokens) { + if (tokens.peekTokenType() == TokenType.OPEN_SQUARE_BRACKET) { + tokens.skip(); + tokens.skip(TokenType.IDENTIFIER, "style-name"); + tokens.skip(TokenType.EQUALS); + String value = tokens.nextValue(TokenType.STRING); + tokens.skip(TokenType.CLOSE_SQUARE_BRACKET); + return Optional.of(value.substring(1, value.length() - 1)); + } else { + return Optional.empty(); + } + } + + private static Optional<NumberingLevel> parseNumbering(TokenIterator tokens) { + if (tokens.peekTokenType() == TokenType.COLON) { + tokens.skip(); + boolean isOrdered = parseListType(tokens); + tokens.skip(TokenType.OPEN_PAREN); + String level = new BigInteger(tokens.nextValue(TokenType.INTEGER)).subtract(BigInteger.ONE).toString(); + tokens.skip(TokenType.CLOSE_PAREN); + return Optional.of(new NumberingLevel(level, isOrdered)); + } else { + return Optional.empty(); + } + } + + private static boolean parseListType(TokenIterator tokens) { + String listType = tokens.nextValue(TokenType.IDENTIFIER); + if (listType.equals("ordered-list")) { + return true; + } else if (listType.equals("unordered-list")) { + return false; + } else { + throw new IllegalArgumentException("Unrecognised list type: " + listType); + } + } +} diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/HtmlPathParser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/HtmlPathParser.java new file mode 100644 index 0000000..105504c --- /dev/null +++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/HtmlPathParser.java @@ -0,0 +1,68 @@ +package org.zwobble.mammoth.internal.styles.parsing; + +import org.zwobble.mammoth.internal.styles.HtmlPath; +import org.zwobble.mammoth.internal.styles.HtmlPathElement; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.zwobble.mammoth.internal.util.Maps.map; + +public class HtmlPathParser { + public static HtmlPath parse(TokenIterator tokens) { + List<HtmlPathElement> elements = new ArrayList<>(); + + if (tokens.peekTokenType() == TokenType.IDENTIFIER) { + HtmlPathElement element = parseElement(tokens); + elements.add(element); + while (tokens.peekTokenType() == TokenType.WHITESPACE && tokens.peekTokenType(1) == TokenType.GREATER_THAN) { + tokens.skip(TokenType.WHITESPACE); + tokens.skip(TokenType.GREATER_THAN); + tokens.skip(TokenType.WHITESPACE); + elements.add(parseElement(tokens)); + } + } + + return new HtmlPath(elements); + } + + private static HtmlPathElement parseElement(TokenIterator tokens) { + List<String> tagNames = parseTagNames(tokens); + List<String> classNames = parseClassNames(tokens); + Map<String, String> attributes = classNames.isEmpty() + ? map() + : map("class", String.join(" ", classNames)); + boolean isFresh = parseIsFresh(tokens); + return new HtmlPathElement(tagNames, attributes, !isFresh); + } + + private static List<String> parseTagNames(TokenIterator tokens) { + List<String> tagNames = new ArrayList<>(); + tagNames.add(tokens.nextValue(TokenType.IDENTIFIER)); + while (tokens.peekTokenType() == TokenType.CHOICE) { + tokens.skip(); + tagNames.add(tokens.nextValue(TokenType.IDENTIFIER)); + } + return tagNames; + } + + private static List<String> parseClassNames(TokenIterator tokens) { + List<String> classNames = new ArrayList<>(); + while (tokens.peekTokenType() == TokenType.DOT) { + tokens.skip(); + classNames.add(tokens.nextValue(TokenType.IDENTIFIER)); + } + return classNames; + } + + private static boolean parseIsFresh(TokenIterator tokens) { + if (tokens.peekTokenType() == TokenType.COLON) { + tokens.skip(); + tokens.skip(TokenType.IDENTIFIER, "fresh"); + return true; + } else { + return false; + } + } +} diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/ListVar.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/ListVar.java deleted file mode 100644 index 0404a3a..0000000 --- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/ListVar.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.zwobble.mammoth.internal.styles.parsing; - - -import org.parboiled.support.Var; - -import java.util.ArrayList; -import java.util.List; - -class ListVar<T> extends Var<List<T>> { - boolean append(Var<T> element) { - return append(element.get()); - } - - boolean append(T element) { - get().add(element); - return set(get()); - } - - @Override - public List<T> get() { - if (isNotSet()) { - set(new ArrayList<T>()); - } - return super.get(); - } - - public boolean isEmpty() { - return get().isEmpty(); - } -} diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/Parsing.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/Parsing.java deleted file mode 100644 index bef3c0b..0000000 --- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/Parsing.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.zwobble.mammoth.internal.styles.parsing; - -import org.parboiled.BaseParser; -import org.parboiled.Parboiled; -import org.parboiled.Rule; -import org.parboiled.parserunners.ReportingParseRunner; -import org.parboiled.support.ParsingResult; - -import java.util.function.Function; - -public class Parsing { - public static <T, U extends BaseParser<T>, V> void parse(Class<U> parserType, Function<U, Rule> rule, String value) { - U parser = Parboiled.createParser(parserType); - ReportingParseRunner<T> runner = new ReportingParseRunner<>(rule.apply(parser)); - ParsingResult<T> result = runner.run(value); - if (result.hasErrors()) { - // TODO: wrap in result - throw new RuntimeException("Parse error"); - } - } -} diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java index d0bf2c7..9e8e8c9 100644 --- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java +++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java @@ -1,10 +1,12 @@ package org.zwobble.mammoth.internal.styles.parsing; -import org.parboiled.support.Var; +import org.zwobble.mammoth.internal.styles.HtmlPath; import org.zwobble.mammoth.internal.styles.StyleMap; import org.zwobble.mammoth.internal.styles.StyleMapBuilder; import java.util.List; +import java.util.function.BiConsumer; +import java.util.function.Consumer; import static java.util.Arrays.asList; @@ -14,14 +16,14 @@ public class StyleMapParser { } public static StyleMap parseStyleMappings(List<String> lines) { - Var<StyleMapBuilder> styleMap = new Var<>(StyleMap.builder()); + StyleMapBuilder styleMap = StyleMap.builder(); for (String line : lines) { handleLine(styleMap, line); } - return styleMap.get().build(); + return styleMap.build(); } - private static void handleLine(Var<StyleMapBuilder> styleMap, String line) { + private static void handleLine(StyleMapBuilder styleMap, String line) { if (line.startsWith("#")) { return; } @@ -29,6 +31,31 @@ public class StyleMapParser { if (line.isEmpty()) { return; } - Parsing.parse(StyleMappingParser.class, parser -> parser.StyleMapping(styleMap), line); + + parseStyleMapping(line).accept(styleMap); + } + + private static Consumer<StyleMapBuilder> parseStyleMapping(String line) { + TokenIterator tokens = new TokenIterator(StyleMappingTokeniser.tokenise(line)); + + BiConsumer<StyleMapBuilder, HtmlPath> documentMatcher = DocumentMatcherParser.parse(tokens); + + tokens.skip(TokenType.WHITESPACE); + tokens.skip(TokenType.ARROW); + + HtmlPath htmlPath = parseHtmlPath(tokens); + + tokens.skip(TokenType.EOF); + + return styleMap -> documentMatcher.accept(styleMap, htmlPath); + } + + private static HtmlPath parseHtmlPath(TokenIterator tokens) { + if (tokens.peekTokenType() == TokenType.EOF) { + return HtmlPath.EMPTY; + } else { + tokens.skip(TokenType.WHITESPACE); + return HtmlPathParser.parse(tokens); + } } } diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingParser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingParser.java deleted file mode 100644 index c006319..0000000 --- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingParser.java +++ /dev/null @@ -1,196 +0,0 @@ -package org.zwobble.mammoth.internal.styles.parsing; - -import org.parboiled.Action; -import org.parboiled.BaseParser; -import org.parboiled.Context; -import org.parboiled.Rule; -import org.parboiled.support.Var; -import org.zwobble.mammoth.internal.documents.NumberingLevel; -import org.zwobble.mammoth.internal.styles.*; - -import java.math.BigInteger; -import java.util.Optional; -import java.util.function.BiFunction; - -import static org.zwobble.mammoth.internal.util.Maps.map; - -public class StyleMappingParser extends BaseParser<StyleMapBuilder> { - Rule StyleMapping(Var<StyleMapBuilder> styleMap) { - Var<BiFunction<StyleMapBuilder, HtmlPath, StyleMapBuilder>> documentElementMatcher = new Var<>(); - Var<HtmlPath> path = new Var<>(); - return Sequence( - DocumentElementMatcher(documentElementMatcher), Whitespace(), "=>", Whitespace(), HtmlPath(path), EOI, - styleMap.set(documentElementMatcher.get().apply(styleMap.get(), path.get()))); - } - - Rule DocumentElementMatcher(Var<BiFunction<StyleMapBuilder, HtmlPath, StyleMapBuilder>> updateBuilder) { - Var<ParagraphMatcher> paragraphMatcher = new Var<>(); - Var<RunMatcher> runMatcher = new Var<>(); - return FirstOf( - Sequence( - ParagraphMatcher(paragraphMatcher), - new Action() { - @Override - public boolean run(Context context) { - updateBuilder.set((styleMap, htmlPath) -> styleMap.mapParagraph(paragraphMatcher.get(), htmlPath)); - return true; - } - }), - Sequence( - RunMatcher(runMatcher), - new Action() { - @Override - public boolean run(Context context) { - updateBuilder.set((styleMap, htmlPath) -> styleMap.mapRun(runMatcher.get(), htmlPath)); - return true; - } - }), - Sequence( - "b", - new Action() { - @Override - public boolean run(Context context) { - updateBuilder.set(StyleMapBuilder::bold); - return true; - } - }), - Sequence( - "i", - new Action() { - @Override - public boolean run(Context context) { - updateBuilder.set(StyleMapBuilder::italic); - return true; - } - }), - Sequence( - "u", - new Action() { - @Override - public boolean run(Context context) { - updateBuilder.set(StyleMapBuilder::underline); - return true; - } - }), - Sequence( - "strike", - new Action() { - @Override - public boolean run(Context context) { - updateBuilder.set(StyleMapBuilder::strikethrough); - return true; - } - })); - } - - public Rule ParagraphMatcher(Var<ParagraphMatcher> matcher) { - Var<Optional<String>> styleId = new Var<>(); - Var<Optional<String>> styleName = new Var<>(); - Var<Optional<NumberingLevel>> numbering = new Var<>(); - return Sequence( - "p", StyleId(styleId), StyleName(styleName), Numbering(numbering), - matcher.set(new ParagraphMatcher(styleId.get(), styleName.get(), numbering.get()))); - } - - public Rule RunMatcher(Var<RunMatcher> matcher) { - Var<Optional<String>> styleId = new Var<>(); - Var<Optional<String>> styleName = new Var<>(); - return Sequence( - "r", StyleId(styleId), StyleName(styleName), - matcher.set(new RunMatcher(styleId.get(), styleName.get()))); - } - - Rule StyleId(Var<Optional<String>> styleId) { - return Sequence(styleId.set(Optional.empty()), Optional(".", Identifier(), styleId.set(Optional.of(match())))); - } - - Rule StyleName(Var<Optional<String>> styleName) { - Var<String> value = new Var<>(); - return Sequence( - styleName.set(Optional.empty()), - Optional("[style-name=", SingleQuotedString(value), "]", styleName.set(Optional.of(value.get())))); - } - - Rule Numbering(Var<Optional<NumberingLevel>> numbering) { - Var<Boolean> isOrdered = new Var<>(); - Var<BigInteger> level = new Var<>(); - return Sequence( - numbering.set(Optional.empty()), - Optional(Sequence( - ":", NumberingType(isOrdered), "(", Number(level), ")", - numbering.set(Optional.of(new NumberingLevel(level.get().subtract(BigInteger.ONE).toString(), isOrdered.get())))))); - } - - Rule NumberingType(Var<Boolean> isOrdered) { - return FirstOf( - Sequence("ordered-list", isOrdered.set(true)), - Sequence("unordered-list", isOrdered.set(false))); - } - - Rule Number(Var<BigInteger> level) { - return Sequence( - OneOrMore(CharRange('0', '9')), - level.set(new BigInteger(match()))); - } - - public Rule HtmlPath(Var<HtmlPath> htmlPath) { - ListVar<HtmlPathElement> elements = new ListVar<>(); - return Sequence(Optional(HtmlPathElements(elements)), htmlPath.set(new HtmlPath(elements.get()))); - } - - Rule HtmlPathElements(ListVar<HtmlPathElement> elements) { - Var<HtmlPathElement> element = new Var<>(); - return Sequence( - HtmlPathElement(element), - elements.append(element), - ZeroOrMore(Sequence(Whitespace(), Ch('>'), Whitespace(), HtmlPathElement(element), elements.append(element)))); - } - - Rule HtmlPathElement(Var<HtmlPathElement> element) { - ListVar<String> tagNames = new ListVar<>(); - ListVar<String> classNames = new ListVar<>(); - Var<Boolean> fresh = new Var<>(); - return Sequence( - TagNames(tagNames), - ClassNames(classNames), - Fresh(fresh), - element.set(new HtmlPathElement( - tagNames.get(), - classNames.isEmpty() ? map() : map("class", String.join(" ", classNames.get())), - !fresh.get()))); - } - - Rule TagNames(ListVar<String> tagNames) { - return Sequence( - Identifier(), - tagNames.append(match()), - ZeroOrMore(Sequence("|", Identifier(), tagNames.append(match())))); - } - - Rule ClassNames(ListVar<String> classNames) { - Var<String> className = new Var<>(); - return ZeroOrMore(Sequence(ClassName(className), classNames.append(className))); - } - - Rule ClassName(Var<String> className) { - return Sequence(".", Identifier(), className.set(match())); - } - - Rule Fresh(Var<Boolean> fresh) { - return Sequence(fresh.set(false), Optional(":fresh", fresh.set(true))); - } - - Rule SingleQuotedString(Var<String> value) { - return Sequence("'", ZeroOrMore(NoneOf("\'")), value.set(match()), "'"); - } - - Rule Identifier() { - return Sequence( - FirstOf(CharRange('a', 'z'), CharRange('A', 'Z')), - ZeroOrMore(FirstOf(CharRange('a', 'z'), CharRange('A', 'Z'), CharRange('0', '9'), Ch('-'), Ch('_')))); - } - - Rule Whitespace() { - return ZeroOrMore(AnyOf(" \t")); - } -} diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java new file mode 100644 index 0000000..fe36242 --- /dev/null +++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java @@ -0,0 +1,112 @@ +package org.zwobble.mammoth.internal.styles.parsing; + +import java.util.ArrayList; +import java.util.List; + +public class StyleMappingTokeniser { + public static List<Token> tokenise(String line) { + List<Token> tokens = new ArrayList<>(); + + int index = 0; + while (index < line.length()) { + Token token = nextToken(line, index); + tokens.add(token); + index += token.getValue().length(); + } + + tokens.add(new Token(TokenType.EOF, "")); + return tokens; + } + + private static Token nextToken(String line, int startIndex) { + char nextChar = line.charAt(startIndex); + TokenType tokenType = tokenType(nextChar); + int index = 0; + if (nextChar == '=' && startIndex + 1 < line.length() && line.charAt(startIndex + 1) == '>') { + tokenType = TokenType.ARROW; + index = startIndex + 2; + } else { + index = seekEndOfToken(line, startIndex + 1, tokenType); + } + return new Token(tokenType, line.substring(startIndex, index)); + } + + private static TokenType tokenType(char nextChar) { + if (isAlphabetic(nextChar)) { + return TokenType.IDENTIFIER; + } else if (isWhitespace(nextChar)) { + return TokenType.WHITESPACE; + } else if (isNumeric(nextChar)) { + return TokenType.INTEGER; + } else { + switch (nextChar) { + case '\'': + return TokenType.STRING; + case '.': + return TokenType.DOT; + case ':': + return TokenType.COLON; + case '>': + return TokenType.GREATER_THAN; + case '=': + return TokenType.EQUALS; + case '(': + return TokenType.OPEN_PAREN; + case ')': + return TokenType.CLOSE_PAREN; + case '[': + return TokenType.OPEN_SQUARE_BRACKET; + case ']': + return TokenType.CLOSE_SQUARE_BRACKET; + case '|': + return TokenType.CHOICE; + default: + return TokenType.UNKNOWN; + } + } + } + + private static int seekEndOfToken(String line, int index, TokenType tokenType) { + switch (tokenType) { + case WHITESPACE: + while (index < line.length() && isWhitespace(line.charAt(index))) { + index += 1; + } + return index; + case IDENTIFIER: + while (index < line.length() && isIdentifierCharacter(line.charAt(index))) { + index += 1; + } + return index; + case INTEGER: + while (index < line.length() && isNumeric(line.charAt(index))) { + index += 1; + } + return index; + case STRING: + while (index < line.length() && line.charAt(index) != '\'') { + index += 1; + } + index += 1; + return index; + default: + return index; + } + } + + private static boolean isWhitespace(char c) { + return c == ' ' || c == '\t'; + } + + private static boolean isIdentifierCharacter(char c) { + return isAlphabetic(c) || isNumeric(c) || c == '-' || c == '_'; + } + + private static boolean isAlphabetic(char nextChar) { + return (nextChar >= 'a' && nextChar <= 'z') || (nextChar >= 'A' && nextChar <= 'Z'); + } + + private static boolean isNumeric(char c) { + return c >= '0' && c <= '9'; + } +} diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/Token.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/Token.java new file mode 100644 index 0000000..f6b771e --- /dev/null +++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/Token.java @@ -0,0 +1,24 @@ +package org.zwobble.mammoth.internal.styles.parsing; + +public class Token { + private final TokenType tokenType; + private final String value; + + public Token(TokenType tokenType, String value) { + this.tokenType = tokenType; + this.value = value; + } + + public TokenType getTokenType() { + return tokenType; + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "Token(tokenType=" + tokenType + ", value=" + value + ")"; + } +} diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java new file mode 100644 index 0000000..df22b5d --- /dev/null +++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java @@ -0,0 +1,57 @@ +package org.zwobble.mammoth.internal.styles.parsing; + +import java.util.List; + +public class TokenIterator { + private final List<Token> tokens; + private int index; + + public TokenIterator(List<Token> tokens) { + this.tokens = tokens; + this.index = 0; + } + + public TokenType peekTokenType() { + return tokens.get(index).getTokenType(); + } + + public TokenType peekTokenType(int count) { + return tokens.get(index + count).getTokenType(); + } + + public String nextValue(TokenType type) { + Token token = tokens.get(index); + if (token.getTokenType() == type) { + index += 1; + return token.getValue(); + } else { + throw unexpectedTokenType(type, token.getTokenType()); + } + } + + public void skip() { + index += 1; + } + + public void skip(TokenType tokenType) { + if (peekTokenType() != tokenType) { + throw unexpectedTokenType(tokenType, peekTokenType()); + } + index += 1; + } + + public void skip(TokenType tokenType, String tokenValue) { + if (peekTokenType() != tokenType) { + throw unexpectedTokenType(tokenType, peekTokenType()); + } + String actualValue = tokens.get(index).getValue(); + if (!actualValue.equals(tokenValue)) { + throw new IllegalArgumentException("expected " + tokenType + " token with value " + tokenValue + " but value was " + actualValue); + } + index += 1; + } + + private IllegalArgumentException unexpectedTokenType(TokenType expected, TokenType actual) { + return new IllegalArgumentException("expected token of type " + expected + " but was of type " + actual); + } +} diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenType.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenType.java new file mode 100644 index 0000000..c9cc9a4 --- /dev/null +++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenType.java @@ -0,0 +1,20 @@ +package org.zwobble.mammoth.internal.styles.parsing; + +public enum TokenType { + WHITESPACE, + IDENTIFIER, + DOT, + COLON, + GREATER_THAN, + ARROW, + EQUALS, + OPEN_PAREN, + CLOSE_PAREN, + OPEN_SQUARE_BRACKET, + CLOSE_SQUARE_BRACKET, + STRING, + INTEGER, + CHOICE, + EOF, + UNKNOWN +} diff --git a/src/test/java/org/zwobble/mammoth/tests/styles/DocumentMatcherParsingTests.java b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/DocumentMatcherParsingTests.java index 0a8cf0d..3c00909 100644 --- a/src/test/java/org/zwobble/mammoth/tests/styles/DocumentMatcherParsingTests.java +++ b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/DocumentMatcherParsingTests.java @@ -1,11 +1,12 @@ -package org.zwobble.mammoth.tests.styles; +package org.zwobble.mammoth.tests.styles.parsing; import org.junit.Test; -import org.parboiled.support.Var; import org.zwobble.mammoth.internal.styles.ParagraphMatcher; import org.zwobble.mammoth.internal.styles.RunMatcher; -import org.zwobble.mammoth.internal.styles.parsing.Parsing; -import org.zwobble.mammoth.internal.styles.parsing.StyleMappingParser; +import org.zwobble.mammoth.internal.styles.parsing.DocumentMatcherParser; +import org.zwobble.mammoth.internal.styles.parsing.StyleMappingTokeniser; +import org.zwobble.mammoth.internal.styles.parsing.TokenIterator; +import org.zwobble.mammoth.internal.styles.parsing.TokenType; import static org.hamcrest.MatcherAssert.assertThat; import static org.zwobble.mammoth.tests.DeepReflectionMatcher.deepEquals; @@ -67,14 +68,14 @@ public class DocumentMatcherParsingTests { } private ParagraphMatcher parseParagraphMatcher(String input) { - Var<ParagraphMatcher> matcher = new Var<>(); - Parsing.parse(StyleMappingParser.class, parser -> parser.ParagraphMatcher(matcher), input); - return matcher.get(); + TokenIterator tokens = new TokenIterator(StyleMappingTokeniser.tokenise(input)); + tokens.skip(TokenType.IDENTIFIER, "p"); + return DocumentMatcherParser.parseParagraphMatcher(tokens); } private RunMatcher parseRunMatcher(String input) { - Var<RunMatcher> matcher = new Var<>(); - Parsing.parse(StyleMappingParser.class, parser -> parser.RunMatcher(matcher), input); - return matcher.get(); + TokenIterator tokens = new TokenIterator(StyleMappingTokeniser.tokenise(input)); + tokens.skip(TokenType.IDENTIFIER, "r"); + return DocumentMatcherParser.parseRunMatcher(tokens); } } diff --git a/src/test/java/org/zwobble/mammoth/tests/styles/HtmlPathParsingTests.java b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/HtmlPathParsingTests.java index 3fe1278..1df4b0b 100644 --- a/src/test/java/org/zwobble/mammoth/tests/styles/HtmlPathParsingTests.java +++ b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/HtmlPathParsingTests.java @@ -1,16 +1,16 @@ -package org.zwobble.mammoth.tests.styles; +package org.zwobble.mammoth.tests.styles.parsing; import org.junit.Test; -import org.parboiled.support.Var; import org.zwobble.mammoth.internal.styles.HtmlPath; import org.zwobble.mammoth.internal.styles.HtmlPathElement; -import org.zwobble.mammoth.internal.styles.parsing.Parsing; -import org.zwobble.mammoth.internal.styles.parsing.StyleMappingParser; +import org.zwobble.mammoth.internal.styles.parsing.HtmlPathParser; +import org.zwobble.mammoth.internal.styles.parsing.StyleMappingTokeniser; +import org.zwobble.mammoth.internal.styles.parsing.TokenIterator; import static org.hamcrest.MatcherAssert.assertThat; -import static org.zwobble.mammoth.tests.DeepReflectionMatcher.deepEquals; import static org.zwobble.mammoth.internal.util.Lists.list; import static org.zwobble.mammoth.internal.util.Maps.map; +import static org.zwobble.mammoth.tests.DeepReflectionMatcher.deepEquals; public class HtmlPathParsingTests { @Test @@ -67,8 +67,6 @@ public class HtmlPathParsingTests { } private HtmlPath parseHtmlPath(String input) { - Var<HtmlPath> path = new Var<>(); - Parsing.parse(StyleMappingParser.class, parser -> parser.HtmlPath(path), input); - return path.get(); + return HtmlPathParser.parse(new TokenIterator(StyleMappingTokeniser.tokenise(input))); } } diff --git a/src/test/java/org/zwobble/mammoth/tests/styles/StyleMapParserTests.java b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMapParserTests.java index a89b192..e76a632 100644 --- a/src/test/java/org/zwobble/mammoth/tests/styles/StyleMapParserTests.java +++ b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMapParserTests.java @@ -1,4 +1,4 @@ -package org.zwobble.mammoth.tests.styles; +package org.zwobble.mammoth.tests.styles.parsing; import org.junit.Test; import org.zwobble.mammoth.internal.styles.HtmlPath; diff --git a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java new file mode 100644 index 0000000..de60249 --- /dev/null +++ b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java @@ -0,0 +1,116 @@ +package org.zwobble.mammoth.tests.styles.parsing; + +import org.hamcrest.Matcher; +import org.junit.Test; +import org.zwobble.mammoth.internal.styles.parsing.StyleMappingTokeniser; +import org.zwobble.mammoth.internal.styles.parsing.Token; +import org.zwobble.mammoth.internal.styles.parsing.TokenType; + +import java.util.List; + +import static java.util.Arrays.asList; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; +import static org.zwobble.mammoth.internal.util.Lists.eagerConcat; +import static org.zwobble.mammoth.internal.util.Lists.list; + +public class StyleMappingTokeniserTests { + @Test + public void emptyStringIsTokenisedToEndOfFileToken() { + assertTokens(""); + } + + @Test + public void whitespaceIsTokenised() { + assertTokens(" \t\t ", isToken(TokenType.WHITESPACE, " \t\t ")); + } + + @Test + public void identifiersAreTokenised() { + assertTokens("Overture", isToken(TokenType.IDENTIFIER, "Overture")); + } + + @Test + public void integersAreTokenised() { + assertTokens("123", isToken(TokenType.INTEGER, "123")); + } + + @Test + public void stringsAreTokenised() { + assertTokens("'Tristan'", isToken(TokenType.STRING, "'Tristan'")); + } + + @Test + public void arrowsAreTokenised() { + assertTokens("=>=>", isToken(TokenType.ARROW, "=>"), isToken(TokenType.ARROW, "=>")); + } + + @Test + public void dotsAreTokenised() { + assertTokens("..", isToken(TokenType.DOT, "."), isToken(TokenType.DOT, ".")); + } + + @Test + public void colonsAreTokenised() { + assertTokens("::", isToken(TokenType.COLON, ":"), isToken(TokenType.COLON, ":")); + } + + @Test + public void greaterThansAreTokenised() { + assertTokens(">>", isToken(TokenType.GREATER_THAN, ">"), isToken(TokenType.GREATER_THAN, ">")); + } + + @Test + public void equalsAreTokenised() { + assertTokens("==", isToken(TokenType.EQUALS, "="), isToken(TokenType.EQUALS, "=")); + } + + @Test + public void openParensAreTokenised() { + assertTokens("((", isToken(TokenType.OPEN_PAREN, "("), isToken(TokenType.OPEN_PAREN, "(")); + } + + @Test + public void closeParensAreTokenised() { + assertTokens("))", isToken(TokenType.CLOSE_PAREN, ")"), isToken(TokenType.CLOSE_PAREN, ")")); + } + + @Test + public void openSquareBracketsAreTokenised() { + assertTokens("[[", isToken(TokenType.OPEN_SQUARE_BRACKET, "["), isToken(TokenType.OPEN_SQUARE_BRACKET, "[")); + } + + @Test + public void closeSquareBracketsAreTokenised() { + assertTokens("]]", isToken(TokenType.CLOSE_SQUARE_BRACKET, "]"), isToken(TokenType.CLOSE_SQUARE_BRACKET, "]")); + } + + @Test + public void choicesAreTokenised() { + assertTokens("||", isToken(TokenType.CHOICE, "|"), isToken(TokenType.CHOICE, "|")); + } + + @Test + public void canTokeniseMultipleTokens() { + assertTokens("The Magic Position", + isToken(TokenType.IDENTIFIER, "The"), + isToken(TokenType.WHITESPACE, " "), + isToken(TokenType.IDENTIFIER, "Magic"), + isToken(TokenType.WHITESPACE, " "), + isToken(TokenType.IDENTIFIER, "Position") + ); + } + + @SafeVarargs + private final void assertTokens(String input, Matcher<Token>... token) { + List<Matcher<? super Token>> tokens = eagerConcat(asList(token), list(isToken(TokenType.EOF, ""))); + assertThat(StyleMappingTokeniser.tokenise(input), contains(tokens)); + } + + private Matcher<Token> isToken(TokenType tokenType, String value) { + return allOf( + hasProperty("tokenType", equalTo(tokenType)), + hasProperty("value", equalTo(value)) + ); + } +} |