1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
|
var hamjest = require("hamjest");
var assertThat = hamjest.assertThat;
var contains = hamjest.contains;
var hasProperties = hamjest.hasProperties;
var tokenise = require("../../../lib/styles/parser/tokeniser").tokenise;
var test = require("../../test")(module);
test("unknown tokens are tokenised", function() {
assertTokens("~", [isToken("unrecognisedCharacter", "~")]);
});
test("empty string is tokenised to end of file token", function() {
assertTokens("", []);
});
test("whitespace is tokenised", function() {
assertTokens(" \t\t ", [isToken("whitespace")]);
});
test("identifiers are tokenised", function() {
assertTokens("Overture", [isToken("identifier", "Overture")]);
});
test("integers are tokenised", function() {
assertTokens("123", [isToken("integer", "123")]);
});
test("strings are tokenised", function() {
assertTokens("'Tristan'", [isToken("string", "Tristan")]);
});
test("unterminated strings are tokenised", function() {
assertTokens("'Tristan", [isToken("unterminated-string", "Tristan")]);
});
test("arrows are tokenised", function() {
assertTokens("=>", [isToken("arrow")]);
});
test("classes are tokenised", function() {
assertTokens(".overture", [isToken("dot"), isToken("identifier", "overture")]);
});
test("colons are tokenised", function() {
assertTokens("::", [isToken("colon"), isToken("colon")]);
});
test("greater thans are tokenised", function() {
assertTokens(">>", [isToken("gt"), isToken("gt")]);
});
test("equals are tokenised", function() {
assertTokens("==", [isToken("equals"), isToken("equals")]);
});
test("startsWith symbols are tokenised", function() {
assertTokens("^=^=", [isToken("startsWith"), isToken("startsWith")]);
});
test("open parens are tokenised", function() {
assertTokens("((", [isToken("open-paren"), isToken("open-paren")]);
});
test("close parens are tokenised", function() {
assertTokens("))", [isToken("close-paren"), isToken("close-paren")]);
});
test("open square brackets are tokenised", function() {
assertTokens("[[", [isToken("open-square-bracket"), isToken("open-square-bracket")]);
});
test("close square brackets are tokenised", function() {
assertTokens("]]", [isToken("close-square-bracket"), isToken("close-square-bracket")]);
});
test("choices are tokenised", function() {
assertTokens("||", [isToken("choice"), isToken("choice")]);
});
test("can tokenise multiple tokens", function() {
assertTokens("The Magic Position", [
isToken("identifier", "The"),
isToken("whitespace"),
isToken("identifier", "Magic"),
isToken("whitespace"),
isToken("identifier", "Position")
]);
});
function assertTokens(input, expectedTokens) {
assertThat(
tokenise(input),
contains.apply(null, expectedTokens.concat([isToken("end", null)]))
);
}
function isToken(tokenType, value) {
return hasProperties({
name: tokenType,
value: value
});
}
|
