Use single pattern in parsing

3 files changed, 46 insertions, 20 deletions

diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/RegexTokeniser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/RegexTokeniser.java
index 8ff4392..03eaf98 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/RegexTokeniser.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/RegexTokeniser.java
@@ -1,11 +1,14 @@
 package org.zwobble.mammoth.internal.styles.parsing;
 
-import java.nio.CharBuffer;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import static org.zwobble.mammoth.internal.util.Iterables.*;
+import static org.zwobble.mammoth.internal.util.Lists.eagerMap;
+
 public class RegexTokeniser<T> {
     public static <T> TokenRule<T> rule(T type, String regex) {
         return new TokenRule<>(type, Pattern.compile(regex));
@@ -16,35 +19,37 @@ public class RegexTokeniser<T> {
         private final Pattern regex;
 
         public TokenRule(T type, Pattern regex) {
+            if (regex.matcher("").groupCount() != 0) {
+                throw new RuntimeException("regex cannot contain any groups");
+            }
+
             this.type = type;
             this.regex = regex;
         }
     }
 
-    private final List<TokenRule<T>> rules;
+    private final Pattern pattern;
+    private final List<T> rules;
 
     public RegexTokeniser(T unknown, List<TokenRule<T>> rules) {
-        this.rules = new ArrayList<>(rules);
-        this.rules.add(rule(unknown, "."));
+        List<TokenRule<T>> allRules = new ArrayList<>(rules);
+        allRules.add(rule(unknown, "."));
+        this.pattern = Pattern.compile(String.join("|", lazyMap(allRules, rule -> "(" + rule.regex.pattern() + ")")));
+        this.rules = eagerMap(allRules, rule -> rule.type);
     }
 
     public List<Token<T>> tokenise(String value) {
+        Matcher matcher = pattern.matcher(value);
         List<Token<T>> tokens = new ArrayList<>();
-        CharBuffer remaining = CharBuffer.wrap(value);
-        while (remaining.hasRemaining()) {
-            boolean matched = false;
-            for (TokenRule<T> rule : rules) {
-                Matcher matcher = rule.regex.matcher(remaining);
-                if (matcher.lookingAt()) {
-                    tokens.add(new Token<>(remaining.position(), rule.type, matcher.group()));
-                    remaining.position(remaining.position() + matcher.end());
-                    matched = true;
-                    break;
-                }
-            }
-            if (!matched) {
+        while (matcher.lookingAt()) {
+            Optional<Integer> groupIndex = tryFind(intRange(0, this.rules.size()), index -> matcher.group(index + 1) != null);
+            if (groupIndex.isPresent()) {
+                T tokenType = this.rules.get(groupIndex.get());
+                tokens.add(new Token(matcher.regionStart(), tokenType, matcher.group()));
+                matcher.region(matcher.end(), value.length());
+            } else {
                 // Should be impossible
-                throw new RuntimeException("Remaining: " + remaining);
+                throw new RuntimeException("Could not find group");
             }
         }
         return tokens;
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java
index 36a43f3..b0b44c3 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java
@@ -6,7 +6,7 @@ import static org.zwobble.mammoth.internal.util.Lists.list;
 
 public class StyleMappingTokeniser {
     public static List<Token<TokenType>> tokenise(String line) {
-        String stringPrefix = "'((?:\\\\.|[^'])*)";
+        String stringPrefix = "'(?:(?:\\\\.|[^'])*)";
         String identifierCharacter = "(?:[a-zA-Z\\-_]|\\\\.)";
 
         RegexTokeniser<TokenType> tokeniser = new RegexTokeniser<>(
@@ -17,7 +17,7 @@ public class StyleMappingTokeniser {
                 RegexTokeniser.rule(TokenType.WHITESPACE, "\\s+"),
                 RegexTokeniser.rule(TokenType.STRING, stringPrefix + "'"),
                 RegexTokeniser.rule(TokenType.UNTERMINATED_STRING, stringPrefix),
-                RegexTokeniser.rule(TokenType.INTEGER, "([0-9]+)")
+                RegexTokeniser.rule(TokenType.INTEGER, "[0-9]+")
             )
         );
         List<Token<TokenType>> tokens = tokeniser.tokenise(line);
diff --git a/src/main/java/org/zwobble/mammoth/internal/util/Iterables.java b/src/main/java/org/zwobble/mammoth/internal/util/Iterables.java
index 205f6a2..2856f66 100644
--- a/src/main/java/org/zwobble/mammoth/internal/util/Iterables.java
+++ b/src/main/java/org/zwobble/mammoth/internal/util/Iterables.java
@@ -164,4 +164,25 @@ public class Iterables {
         }
         return OptionalInt.empty();
     }
+
+    public static Iterable<Integer> intRange(int start, int end) {
+        return new Iterable<Integer>() {
+            @Override
+            public Iterator<Integer> iterator() {
+                return new Iterator<Integer>() {
+                    private int next = start;
+
+                    @Override
+                    public boolean hasNext() {
+                        return next < end;
+                    }
+
+                    @Override
+                    public Integer next() {
+                        return next++;
+                    }
+                };
+            }
+        };
+    }
 }