1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
|
import collections
import re
Token = collections.namedtuple("Token", ["character_index", "type", "value"])
class TokenType(object):
IDENTIFIER = "identifier"
SYMBOL = "symbol"
WHITESPACE = "whitespace"
STRING = "string"
UNTERMINATED_STRING = "unterminated string"
INTEGER = "integer"
END = "end"
def regex_tokeniser(rules):
rules = [(token_type, _to_regex(regex)) for token_type, regex in rules]
rules.append(("unknown", re.compile(".")))
def tokenise(value):
tokens = []
index = 0
while index < len(value):
for token_type, regex in rules:
match = regex.match(value, index)
if match is not None:
tokens.append(Token(index, token_type, match.group(0)))
index = match.end()
break
else:
# Should be impossible
raise Exception("Remaining: " + value[index:])
tokens.append(Token(index, TokenType.END, ""))
return tokens
return tokenise
def _to_regex(value):
if hasattr(value, "match"):
return value
else:
return re.compile(value)
_string_prefix = r"'(?:\\.|[^'])*"
_identifier_character = r"(?:[a-zA-Z\-_]|\\.)"
tokenise = regex_tokeniser([
(TokenType.IDENTIFIER, _identifier_character + "(?:" + _identifier_character + "|[0-9])*"),
(TokenType.SYMBOL, r":|>|=>|\^=|=|\(|\)|\[|\]|\||!|\."),
(TokenType.WHITESPACE, r"\s+"),
(TokenType.STRING, _string_prefix + "'"),
(TokenType.UNTERMINATED_STRING, _string_prefix),
(TokenType.INTEGER, "([0-9]+)"),
])
|
