blob: 85818dbdbdbd0f6abe932def62321c1b89f4aca0 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
|
package org.shedlang.compiler.tokeniser;
import org.zwobble.sourcetext.SourceText;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
public class RegexTokeniser<T> {
public static <T> TokenRule<T> rule(T type, String regex) {
var pattern = Pattern.compile(regex);
if (pattern.matcher("").groupCount() != 0) {
throw new RuntimeException("regex cannot contain any groups");
}
return new TokenRule<T>(type, pattern);
}
public static class TokenRule<T> {
private final T type;
private final Pattern regex;
private TokenRule(T type, Pattern regex) {
this.type = type;
this.regex = regex;
}
}
private final Pattern pattern;
private final List<T> rules;
public RegexTokeniser(T unknown, List<TokenRule<T>> rules) {
var allRules = new ArrayList<>(rules);
allRules.add(rule(unknown, "."));
this.pattern = Pattern.compile(
allRules.stream()
.map(rule -> "(" + rule.regex.pattern() + ")")
.collect(Collectors.joining("|"))
);
this.rules = allRules.stream()
.map(rule -> rule.type)
.collect(Collectors.toList());
}
public List<Token<T>> tokenise(SourceText sourceText) {
var matcher = pattern.matcher(sourceText.charSequence());
var tokens = new ArrayList<Token<T>>();
while (matcher.lookingAt()) {
var groupIndex = IntStream.rangeClosed(1, this.rules.size())
.filter(index -> matcher.group(index) != null)
.findFirst()
.getAsInt();
var tokenType = this.rules.get(groupIndex - 1);
var startPosition = sourceText.characterPosition(matcher.regionStart());
var endPosition = sourceText.characterPosition(matcher.end());
tokens.add(new Token<>(
startPosition.to(endPosition),
tokenType
));
matcher.region(matcher.end(), sourceText.characterLength());
}
return tokens;
}
}
|
