diff options
Diffstat (limited to 'src/main/java/org/zwobble')
3 files changed, 46 insertions, 20 deletions
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/RegexTokeniser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/RegexTokeniser.java index 8ff4392..03eaf98 100644 --- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/RegexTokeniser.java +++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/RegexTokeniser.java @@ -1,11 +1,14 @@ package org.zwobble.mammoth.internal.styles.parsing; -import java.nio.CharBuffer; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; +import static org.zwobble.mammoth.internal.util.Iterables.*; +import static org.zwobble.mammoth.internal.util.Lists.eagerMap; + public class RegexTokeniser<T> { public static <T> TokenRule<T> rule(T type, String regex) { return new TokenRule<>(type, Pattern.compile(regex)); @@ -16,35 +19,37 @@ public class RegexTokeniser<T> { private final Pattern regex; public TokenRule(T type, Pattern regex) { + if (regex.matcher("").groupCount() != 0) { + throw new RuntimeException("regex cannot contain any groups"); + } + this.type = type; this.regex = regex; } } - private final List<TokenRule<T>> rules; + private final Pattern pattern; + private final List<T> rules; public RegexTokeniser(T unknown, List<TokenRule<T>> rules) { - this.rules = new ArrayList<>(rules); - this.rules.add(rule(unknown, ".")); + List<TokenRule<T>> allRules = new ArrayList<>(rules); + allRules.add(rule(unknown, ".")); + this.pattern = Pattern.compile(String.join("|", lazyMap(allRules, rule -> "(" + rule.regex.pattern() + ")"))); + this.rules = eagerMap(allRules, rule -> rule.type); } public List<Token<T>> tokenise(String value) { + Matcher matcher = pattern.matcher(value); List<Token<T>> tokens = new ArrayList<>(); - CharBuffer remaining = CharBuffer.wrap(value); - while (remaining.hasRemaining()) { - boolean matched = false; - for (TokenRule<T> rule : rules) { - Matcher matcher = rule.regex.matcher(remaining); - if (matcher.lookingAt()) { - tokens.add(new Token<>(remaining.position(), rule.type, matcher.group())); - remaining.position(remaining.position() + matcher.end()); - matched = true; - break; - } - } - if (!matched) { + while (matcher.lookingAt()) { + Optional<Integer> groupIndex = tryFind(intRange(0, this.rules.size()), index -> matcher.group(index + 1) != null); + if (groupIndex.isPresent()) { + T tokenType = this.rules.get(groupIndex.get()); + tokens.add(new Token(matcher.regionStart(), tokenType, matcher.group())); + matcher.region(matcher.end(), value.length()); + } else { // Should be impossible - throw new RuntimeException("Remaining: " + remaining); + throw new RuntimeException("Could not find group"); } } return tokens; diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java index 36a43f3..b0b44c3 100644 --- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java +++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java @@ -6,7 +6,7 @@ import static org.zwobble.mammoth.internal.util.Lists.list; public class StyleMappingTokeniser { public static List<Token<TokenType>> tokenise(String line) { - String stringPrefix = "'((?:\\\\.|[^'])*)"; + String stringPrefix = "'(?:(?:\\\\.|[^'])*)"; String identifierCharacter = "(?:[a-zA-Z\\-_]|\\\\.)"; RegexTokeniser<TokenType> tokeniser = new RegexTokeniser<>( @@ -17,7 +17,7 @@ public class StyleMappingTokeniser { RegexTokeniser.rule(TokenType.WHITESPACE, "\\s+"), RegexTokeniser.rule(TokenType.STRING, stringPrefix + "'"), RegexTokeniser.rule(TokenType.UNTERMINATED_STRING, stringPrefix), - RegexTokeniser.rule(TokenType.INTEGER, "([0-9]+)") + RegexTokeniser.rule(TokenType.INTEGER, "[0-9]+") ) ); List<Token<TokenType>> tokens = tokeniser.tokenise(line); diff --git a/src/main/java/org/zwobble/mammoth/internal/util/Iterables.java b/src/main/java/org/zwobble/mammoth/internal/util/Iterables.java index 205f6a2..2856f66 100644 --- a/src/main/java/org/zwobble/mammoth/internal/util/Iterables.java +++ b/src/main/java/org/zwobble/mammoth/internal/util/Iterables.java @@ -164,4 +164,25 @@ public class Iterables { } return OptionalInt.empty(); } + + public static Iterable<Integer> intRange(int start, int end) { + return new Iterable<Integer>() { + @Override + public Iterator<Integer> iterator() { + return new Iterator<Integer>() { + private int next = start; + + @Override + public boolean hasNext() { + return next < end; + } + + @Override + public Integer next() { + return next++; + } + }; + } + }; + } } |