package org.shedlang.compiler.tokeniser; import org.zwobble.sourcetext.SourceText; import java.util.ArrayList; import java.util.List; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.IntStream; public class RegexTokeniser { public static TokenRule rule(T type, String regex) { var pattern = Pattern.compile(regex); if (pattern.matcher("").groupCount() != 0) { throw new RuntimeException("regex cannot contain any groups"); } return new TokenRule(type, pattern); } public static class TokenRule { private final T type; private final Pattern regex; private TokenRule(T type, Pattern regex) { this.type = type; this.regex = regex; } } private final Pattern pattern; private final List rules; public RegexTokeniser(T unknown, List> rules) { var allRules = new ArrayList<>(rules); allRules.add(rule(unknown, ".")); this.pattern = Pattern.compile( allRules.stream() .map(rule -> "(" + rule.regex.pattern() + ")") .collect(Collectors.joining("|")) ); this.rules = allRules.stream() .map(rule -> rule.type) .collect(Collectors.toList()); } public List> tokenise(SourceText sourceText) { var matcher = pattern.matcher(sourceText.charSequence()); var tokens = new ArrayList>(); while (matcher.lookingAt()) { var groupIndex = IntStream.rangeClosed(1, this.rules.size()) .filter(index -> matcher.group(index) != null) .findFirst() .getAsInt(); var tokenType = this.rules.get(groupIndex - 1); var startPosition = sourceText.characterPosition(matcher.regionStart()); var endPosition = sourceText.characterPosition(matcher.end()); tokens.add(new Token<>( startPosition.to(endPosition), tokenType )); matcher.region(matcher.end(), sourceText.characterLength()); } return tokens; } }