1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
|
var lop = require("lop");
var RegexTokeniser = lop.RegexTokeniser;
exports.tokenise = tokenise;
var stringPrefix = "'((?:\\\\(?:.|$)|[^'\\\\])*)";
function tokenise(string) {
var identifierCharacter = "(?:[a-zA-Z\\-_]|\\\\.)";
var tokeniser = new RegexTokeniser([
{name: "identifier", regex: new RegExp("(" + identifierCharacter + "(?:" + identifierCharacter + "|[0-9])*)")},
{name: "dot", regex: /\./},
{name: "colon", regex: /:/},
{name: "gt", regex: />/},
{name: "whitespace", regex: /\s+/},
{name: "arrow", regex: /=>/},
{name: "equals", regex: /=/},
{name: "startsWith", regex: /\^=/},
{name: "open-paren", regex: /\(/},
{name: "close-paren", regex: /\)/},
{name: "open-square-bracket", regex: /\[/},
{name: "close-square-bracket", regex: /\]/},
{name: "string", regex: new RegExp(stringPrefix + "'")},
{name: "unterminated-string", regex: new RegExp(stringPrefix)},
{name: "integer", regex: /([0-9]+)/},
{name: "choice", regex: /\|/},
{name: "bang", regex: /(!)/}
]);
return tokeniser.tokenise(string);
}
|
