Copy regex from mammoth.js for consistency

7 files changed, 112 insertions, 143 deletions

diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/DocumentMatcherParser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/DocumentMatcherParser.java
index e7de55c..e019cff 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/DocumentMatcherParser.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/DocumentMatcherParser.java
@@ -11,8 +11,8 @@ import java.util.Optional;
 import java.util.function.BiConsumer;
 
 public class DocumentMatcherParser {
-    public static BiConsumer<StyleMapBuilder, HtmlPath> parse(TokenIterator tokens) {
-        Token identifier = tokens.next(TokenType.IDENTIFIER);
+    public static BiConsumer<StyleMapBuilder, HtmlPath> parse(TokenIterator<TokenType> tokens) {
+        Token<TokenType> identifier = tokens.next(TokenType.IDENTIFIER);
         switch (identifier.getValue()) {
             case "p":
                 ParagraphMatcher paragraph = parseParagraphMatcher(tokens);
@@ -35,20 +35,20 @@ public class DocumentMatcherParser {
         }
     }
 
-    public static ParagraphMatcher parseParagraphMatcher(TokenIterator tokens) {
+    public static ParagraphMatcher parseParagraphMatcher(TokenIterator<TokenType> tokens) {
         Optional<String> styleId = parseStyleId(tokens);
         Optional<String> styleName = parseStyleName(tokens);
         Optional<NumberingLevel> numbering = parseNumbering(tokens);
         return new ParagraphMatcher(styleId, styleName, numbering);
     }
 
-    public static RunMatcher parseRunMatcher(TokenIterator tokens) {
+    public static RunMatcher parseRunMatcher(TokenIterator<TokenType> tokens) {
         Optional<String> styleId = parseStyleId(tokens);
         Optional<String> styleName = parseStyleName(tokens);
         return new RunMatcher(styleId, styleName);
     }
 
-    private static Optional<String> parseStyleId(TokenIterator tokens) {
+    private static Optional<String> parseStyleId(TokenIterator<TokenType> tokens) {
         if (tokens.peekTokenType() == TokenType.DOT) {
             tokens.skip();
             return Optional.of(tokens.nextValue(TokenType.IDENTIFIER));
@@ -57,7 +57,7 @@ public class DocumentMatcherParser {
         }
     }
 
-    private static Optional<String> parseStyleName(TokenIterator tokens) {
+    private static Optional<String> parseStyleName(TokenIterator<TokenType> tokens) {
         if (tokens.peekTokenType() == TokenType.OPEN_SQUARE_BRACKET) {
             tokens.skip();
             tokens.skip(TokenType.IDENTIFIER, "style-name");
@@ -70,7 +70,7 @@ public class DocumentMatcherParser {
         }
     }
 
-    private static Optional<NumberingLevel> parseNumbering(TokenIterator tokens) {
+    private static Optional<NumberingLevel> parseNumbering(TokenIterator<TokenType> tokens) {
         if (tokens.peekTokenType() == TokenType.COLON) {
             tokens.skip();
             boolean isOrdered = parseListType(tokens);
@@ -83,8 +83,8 @@ public class DocumentMatcherParser {
         }
     }
 
-    private static boolean parseListType(TokenIterator tokens) {
-        Token listType = tokens.next(TokenType.IDENTIFIER);
+    private static boolean parseListType(TokenIterator<TokenType> tokens) {
+        Token<TokenType> listType = tokens.next(TokenType.IDENTIFIER);
         switch (listType.getValue()) {
             case "ordered-list":
                 return true;
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/HtmlPathParser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/HtmlPathParser.java
index e7eefe2..2db2553 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/HtmlPathParser.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/HtmlPathParser.java
@@ -11,7 +11,7 @@ import java.util.Map;
 import static org.zwobble.mammoth.internal.util.Maps.map;
 
 public class HtmlPathParser {
-    public static HtmlPath parse(TokenIterator tokens) {
+    public static HtmlPath parse(TokenIterator<TokenType> tokens) {
         if (tokens.peekTokenType() == TokenType.BANG) {
             tokens.skip(TokenType.BANG);
             return HtmlPath.IGNORE;
@@ -20,7 +20,7 @@ public class HtmlPathParser {
         }
     }
 
-    private static HtmlPath parseHtmlPathElements(TokenIterator tokens) {
+    private static HtmlPath parseHtmlPathElements(TokenIterator<TokenType> tokens) {
         List<HtmlPathElement> elements = new ArrayList<>();
 
         if (tokens.peekTokenType() == TokenType.IDENTIFIER) {
@@ -37,7 +37,7 @@ public class HtmlPathParser {
         return new HtmlPathElements(elements);
     }
 
-    private static HtmlPathElement parseElement(TokenIterator tokens) {
+    private static HtmlPathElement parseElement(TokenIterator<TokenType> tokens) {
         List<String> tagNames = parseTagNames(tokens);
         List<String> classNames = parseClassNames(tokens);
         Map<String, String> attributes = classNames.isEmpty()
@@ -47,7 +47,7 @@ public class HtmlPathParser {
         return new HtmlPathElement(tagNames, attributes, !isFresh);
     }
 
-    private static List<String> parseTagNames(TokenIterator tokens) {
+    private static List<String> parseTagNames(TokenIterator<TokenType> tokens) {
         List<String> tagNames = new ArrayList<>();
         tagNames.add(tokens.nextValue(TokenType.IDENTIFIER));
         while (tokens.peekTokenType() == TokenType.CHOICE) {
@@ -57,7 +57,7 @@ public class HtmlPathParser {
         return tagNames;
     }
 
-    private static List<String> parseClassNames(TokenIterator tokens) {
+    private static List<String> parseClassNames(TokenIterator<TokenType> tokens) {
         List<String> classNames = new ArrayList<>();
         while (tokens.peekTokenType() == TokenType.DOT) {
             tokens.skip();
@@ -66,7 +66,7 @@ public class HtmlPathParser {
         return classNames;
     }
 
-    private static boolean parseIsFresh(TokenIterator tokens) {
+    private static boolean parseIsFresh(TokenIterator<TokenType> tokens) {
         if (tokens.peekTokenType() == TokenType.COLON) {
             tokens.skip();
             tokens.skip(TokenType.IDENTIFIER, "fresh");
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/RegexTokeniser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/RegexTokeniser.java
new file mode 100644
index 0000000..06ecbd3
--- /dev/null
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/RegexTokeniser.java
@@ -0,0 +1,52 @@
+package org.zwobble.mammoth.internal.styles.parsing;
+
+import java.nio.CharBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class RegexTokeniser<T> {
+    public static <T> TokenRule<T> rule(T type, String regex) {
+        return new TokenRule<>(type, Pattern.compile(regex));
+    }
+
+    public static class TokenRule<T> {
+        private final T type;
+        private final Pattern regex;
+
+        public TokenRule(T type, Pattern regex) {
+            this.type = type;
+            this.regex = regex;
+        }
+    }
+
+    private final List<TokenRule<T>> rules;
+
+    public RegexTokeniser(List<TokenRule<T>> rules) {
+        this.rules = rules;
+    }
+
+    public List<Token<T>> tokenise(String value) {
+        List<Token<T>> tokens = new ArrayList<>();
+        CharBuffer remaining = CharBuffer.wrap(value);
+        while (remaining.hasRemaining()) {
+            boolean matched = false;
+            for (TokenRule<T> rule : rules) {
+                Matcher matcher = rule.regex.matcher(remaining);
+                if (matcher.lookingAt()) {
+                    tokens.add(new Token<>(remaining.position(), rule.type, matcher.group()));
+                    remaining.position(remaining.position() + matcher.end());
+                    matched = true;
+                    break;
+                }
+            }
+            if (!matched) {
+                throw new RuntimeException("Remaining: " + remaining);
+            }
+        }
+        return tokens;
+    }
+
+
+}
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java
index 8dd4099..c8dc221 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMapParser.java
@@ -41,7 +41,7 @@ public class StyleMapParser {
     }
 
     private static Consumer<StyleMapBuilder> parseStyleMapping(String line) {
-        TokenIterator tokens = new TokenIterator(StyleMappingTokeniser.tokenise(line));
+        TokenIterator<TokenType> tokens = new TokenIterator<>(StyleMappingTokeniser.tokenise(line));
 
         BiConsumer<StyleMapBuilder, HtmlPath> documentMatcher = DocumentMatcherParser.parse(tokens);
 
@@ -55,7 +55,7 @@ public class StyleMapParser {
         return styleMap -> documentMatcher.accept(styleMap, htmlPath);
     }
 
-    private static HtmlPath parseHtmlPath(TokenIterator tokens) {
+    private static HtmlPath parseHtmlPath(TokenIterator<TokenType> tokens) {
         if (tokens.peekTokenType() == TokenType.EOF) {
             return HtmlPath.EMPTY;
         } else {
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java
index 28400ff..bfef347 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java
@@ -1,118 +1,35 @@
 package org.zwobble.mammoth.internal.styles.parsing;
 
-import java.util.ArrayList;
 import java.util.List;
 
-public class StyleMappingTokeniser {
-    public static List<Token> tokenise(String line) {
-        List<Token> tokens = new ArrayList<>();
-
-        int index = 0;
-        while (index < line.length()) {
-            Token token = nextToken(line, index);
-            tokens.add(token);
-            index += token.getValue().length();
-        }
+import static org.zwobble.mammoth.internal.util.Lists.list;
 
-        tokens.add(new Token(index, TokenType.EOF, ""));
+public class StyleMappingTokeniser {
+    public static List<Token<TokenType>> tokenise(String line) {
+        String stringPrefix = "'((?:\\.|[^'])*)";
+
+        RegexTokeniser<TokenType> tokeniser = new RegexTokeniser<>(
+            list(
+                RegexTokeniser.rule(TokenType.IDENTIFIER, "([a-zA-Z][a-zA-Z0-9\\-]*)"),
+                RegexTokeniser.rule(TokenType.DOT, "\\."),
+                RegexTokeniser.rule(TokenType.COLON, ":"),
+                RegexTokeniser.rule(TokenType.GREATER_THAN, ">"),
+                RegexTokeniser.rule(TokenType.WHITESPACE, "\\s+"),
+                RegexTokeniser.rule(TokenType.ARROW, "=>"),
+                RegexTokeniser.rule(TokenType.EQUALS, "="),
+                RegexTokeniser.rule(TokenType.OPEN_PAREN, "\\("),
+                RegexTokeniser.rule(TokenType.CLOSE_PAREN, "\\)"),
+                RegexTokeniser.rule(TokenType.OPEN_SQUARE_BRACKET, "\\["),
+                RegexTokeniser.rule(TokenType.CLOSE_SQUARE_BRACKET, "\\]"),
+                RegexTokeniser.rule(TokenType.STRING, stringPrefix + "'"),
+                RegexTokeniser.rule(TokenType.UNTERMINATED_STRING, stringPrefix),
+                RegexTokeniser.rule(TokenType.INTEGER, "([0-9]+)"),
+                RegexTokeniser.rule(TokenType.CHOICE, "\\|"),
+                RegexTokeniser.rule(TokenType.BANG, "!")
+            )
+        );
+        List<Token<TokenType>> tokens = tokeniser.tokenise(line);
+        tokens.add(new Token<>(line.length(), TokenType.EOF, ""));
         return tokens;
     }
-
-    private static Token nextToken(String line, int startIndex) {
-        char nextChar = line.charAt(startIndex);
-        TokenType tokenType = tokenType(nextChar);
-        int index = 0;
-        if (nextChar == '=' && startIndex + 1 < line.length() && line.charAt(startIndex + 1) == '>') {
-            tokenType = TokenType.ARROW;
-            index = startIndex + 2;
-        } else {
-            index = seekEndOfToken(line, startIndex + 1, tokenType);
-            if (tokenType == TokenType.STRING && index == line.length() + 1) {
-                tokenType = TokenType.UNTERMINATED_STRING;
-            }
-        }
-        index = Math.min(index, line.length());
-        return new Token(startIndex, tokenType, line.substring(startIndex, index));
-    }
-
-    private static TokenType tokenType(char nextChar) {
-        if (isAlphabetic(nextChar)) {
-            return TokenType.IDENTIFIER;
-        } else if (isWhitespace(nextChar)) {
-            return TokenType.WHITESPACE;
-        } else if (isNumeric(nextChar)) {
-            return TokenType.INTEGER;
-        } else {
-            switch (nextChar) {
-                case '\'':
-                    return TokenType.STRING;
-                case '.':
-                    return TokenType.DOT;
-                case ':':
-                    return TokenType.COLON;
-                case '>':
-                    return TokenType.GREATER_THAN;
-                case '=':
-                    return TokenType.EQUALS;
-                case '(':
-                    return TokenType.OPEN_PAREN;
-                case ')':
-                    return TokenType.CLOSE_PAREN;
-                case '[':
-                    return TokenType.OPEN_SQUARE_BRACKET;
-                case ']':
-                    return TokenType.CLOSE_SQUARE_BRACKET;
-                case '|':
-                    return TokenType.CHOICE;
-                case '!':
-                    return TokenType.BANG;
-                default:
-                    return TokenType.UNKNOWN;
-            }
-        }
-    }
-
-    private static int seekEndOfToken(String line, int index, TokenType tokenType) {
-        switch (tokenType) {
-            case WHITESPACE:
-                while (index < line.length() && isWhitespace(line.charAt(index))) {
-                    index += 1;
-                }
-                return index;
-            case IDENTIFIER:
-                while (index < line.length() && isIdentifierCharacter(line.charAt(index))) {
-                    index += 1;
-                }
-                return index;
-            case INTEGER:
-                while (index < line.length() && isNumeric(line.charAt(index))) {
-                    index += 1;
-                }
-                return index;
-            case STRING:
-                while (index < line.length() && line.charAt(index) != '\'') {
-                    index += 1;
-                }
-                index += 1;
-                return index;
-            default:
-                return index;
-        }
-    }
-
-    private static boolean isWhitespace(char c) {
-        return c == ' ' || c == '\t';
-    }
-
-    private static boolean isIdentifierCharacter(char c) {
-        return isAlphabetic(c) || isNumeric(c) || c == '-' || c == '_';
-    }
-
-    private static boolean isAlphabetic(char nextChar) {
-        return (nextChar >= 'a' && nextChar <= 'z') || (nextChar >= 'A' && nextChar <= 'Z');
-    }
-
-    private static boolean isNumeric(char c) {
-        return c >= '0' && c <= '9';
-    }
 }
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/Token.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/Token.java
index 3059e5b..bf009c0 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/Token.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/Token.java
@@ -1,11 +1,11 @@
 package org.zwobble.mammoth.internal.styles.parsing;
 
-public class Token {
+public class Token<T> {
     private final int characterIndex;
-    private final TokenType tokenType;
+    private final T tokenType;
     private final String value;
 
-    public Token(int characterIndex, TokenType tokenType, String value) {
+    public Token(int characterIndex, T tokenType, String value) {
         this.characterIndex = characterIndex;
         this.tokenType = tokenType;
         this.value = value;
@@ -15,7 +15,7 @@ public class Token {
         return characterIndex;
     }
 
-    public TokenType getTokenType() {
+    public T getTokenType() {
         return tokenType;
     }
 
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java
index 805e5f0..0bb3f46 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/TokenIterator.java
@@ -2,25 +2,25 @@ package org.zwobble.mammoth.internal.styles.parsing;
 
 import java.util.List;
 
-public class TokenIterator {
-    private final List<Token> tokens;
+public class TokenIterator<T> {
+    private final List<Token<T>> tokens;
     private int index;
 
-    public TokenIterator(List<Token> tokens) {
+    public TokenIterator(List<Token<T>> tokens) {
         this.tokens = tokens;
         this.index = 0;
     }
 
-    public TokenType peekTokenType() {
+    public T peekTokenType() {
         return tokens.get(index).getTokenType();
     }
 
-    public TokenType peekTokenType(int count) {
+    public T peekTokenType(int count) {
         return tokens.get(index + count).getTokenType();
     }
 
-    public Token next(TokenType type) {
-        Token token = tokens.get(index);
+    public Token<T> next(T type) {
+        Token<T> token = tokens.get(index);
         if (token.getTokenType() == type) {
             index += 1;
             return token;
@@ -29,7 +29,7 @@ public class TokenIterator {
         }
     }
 
-    public String nextValue(TokenType type) {
+    public String nextValue(T type) {
         return next(type).getValue();
     }
 
@@ -37,7 +37,7 @@ public class TokenIterator {
         index += 1;
     }
 
-    public void skip(TokenType tokenType) {
+    public void skip(T tokenType) {
         Token token = tokens.get(index);
         if (token.getTokenType() != tokenType) {
             throw unexpectedTokenType(tokenType, token);
@@ -45,8 +45,8 @@ public class TokenIterator {
         index += 1;
     }
 
-    public void skip(TokenType tokenType, String tokenValue) {
-        Token token = tokens.get(index);
+    public void skip(T tokenType, String tokenValue) {
+        Token<T> token = tokens.get(index);
         if (token.getTokenType() != tokenType) {
             throw unexpectedTokenType(tokenType, token);
         }
@@ -57,7 +57,7 @@ public class TokenIterator {
         index += 1;
     }
 
-    private LineParseException unexpectedTokenType(TokenType expected, Token actual) {
+    private LineParseException unexpectedTokenType(T expected, Token<T> actual) {
         return new LineParseException(actual, "expected token of type " + expected + " but was of type " + actual.getTokenType());
     }
 }