Use source range to get value of token

4 files changed, 78 insertions, 86 deletions

diff --git a/src/test/java/org/shedlang/compiler/parser/ParserFunctionDefinitionTests.java b/src/test/java/org/shedlang/compiler/parser/ParserFunctionDefinitionTests.java
index 9c3148a3..c5c2932f 100644
--- a/src/test/java/org/shedlang/compiler/parser/ParserFunctionDefinitionTests.java
+++ b/src/test/java/org/shedlang/compiler/parser/ParserFunctionDefinitionTests.java
@@ -119,7 +119,7 @@ public class ParserFunctionDefinitionTests {
             () -> parseString(source, Parser::parseNamespaceStatement)
         );
 
-        assertThat(error.getActual().value(), equalTo("c"));
+        assertThat(error.getActual().value().toString(), equalTo("c"));
         assertThat(error.getExpected(), equalTo(TokenType.SYMBOL_BRACE_CLOSE.toString()));
     }
 }
diff --git a/src/test/java/org/shedlang/compiler/parser/ParserFunctionExpressionTests.java b/src/test/java/org/shedlang/compiler/parser/ParserFunctionExpressionTests.java
index 8b4c72a6..d5db8035 100644
--- a/src/test/java/org/shedlang/compiler/parser/ParserFunctionExpressionTests.java
+++ b/src/test/java/org/shedlang/compiler/parser/ParserFunctionExpressionTests.java
@@ -89,7 +89,7 @@ public class ParserFunctionExpressionTests {
             () -> parseString(source, Parser::parseTopLevelExpression)
         );
 
-        assertThat(error.getActual().value(), equalTo("c"));
+        assertThat(error.getActual().value().toString(), equalTo("c"));
         assertThat(error.getExpected(), equalTo(TokenType.SYMBOL_BRACE_CLOSE.toString()));
     }
 }
diff --git a/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java b/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java
index be47dda5..dd64735a 100644
--- a/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java
+++ b/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java
@@ -39,8 +39,7 @@ public class RegexTokeniserTests {
         assertThat(result, isSequence(
             equalTo(new Token<>(
                 sourceText.characterPosition(0).to(sourceText.characterPosition(5)),
-                TokenType.IDENTIFIER,
-                "count"
+                TokenType.IDENTIFIER
             ))
         ));
     }
@@ -58,8 +57,7 @@ public class RegexTokeniserTests {
         assertThat(result, isSequence(
             equalTo(new Token<>(
                 sourceText.characterPosition(0).to(sourceText.characterPosition(2)),
-                TokenType.KEYWORD,
-                "if"
+                TokenType.KEYWORD
             ))
         ));
     }
@@ -77,18 +75,15 @@ public class RegexTokeniserTests {
         assertThat(result, isSequence(
             equalTo(new Token<>(
                 sourceText.characterPosition(0).to(sourceText.characterPosition(3)),
-                TokenType.IDENTIFIER,
-                "one"
+                TokenType.IDENTIFIER
             )),
             equalTo(new Token<>(
                 sourceText.characterPosition(3).to(sourceText.characterPosition(4)),
-                TokenType.WHITESPACE,
-                " "
+                TokenType.WHITESPACE
             )),
             equalTo(new Token<>(
                 sourceText.characterPosition(4).to(sourceText.characterPosition(7)),
-                TokenType.IDENTIFIER,
-                "two"
+                TokenType.IDENTIFIER
             ))
         ));
     }
@@ -105,23 +100,19 @@ public class RegexTokeniserTests {
         assertThat(result, isSequence(
             equalTo(new Token<>(
                 sourceText.characterPosition(0).to(sourceText.characterPosition(1)),
-                TokenType.UNKNOWN,
-                "!"
+                TokenType.UNKNOWN
             )),
             equalTo(new Token<>(
                 sourceText.characterPosition(1).to(sourceText.characterPosition(2)),
-                TokenType.UNKNOWN,
-                "!"
+                TokenType.UNKNOWN
             )),
             equalTo(new Token<>(
                 sourceText.characterPosition(2).to(sourceText.characterPosition(3)),
-                TokenType.IDENTIFIER,
-                "a"
+                TokenType.IDENTIFIER
             )),
             equalTo(new Token<>(
                 sourceText.characterPosition(3).to(sourceText.characterPosition(4)),
-                TokenType.UNKNOWN,
-                "!"
+                TokenType.UNKNOWN
             ))
         ));
     }
diff --git a/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java b/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java
index dd22550f..80bafd0b 100644
--- a/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java
+++ b/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java
@@ -22,11 +22,11 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -42,11 +42,11 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -65,11 +65,11 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
         assertThat(tokens.trySkip(TokenType.SYMBOL), equalTo(true));
@@ -85,11 +85,11 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -104,11 +104,11 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
         assertThat(tokens.nextValue(TokenType.SYMBOL), equalTo("!"));
@@ -121,11 +121,11 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -144,11 +144,11 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -162,11 +162,11 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -180,11 +180,11 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -198,11 +198,11 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -216,11 +216,11 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -234,17 +234,17 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
-        assertThat(tokens.peek(), equalTo(token(sourceText, 0, TokenType.SYMBOL, "!")));
-        assertThat(tokens.peek(), equalTo(token(sourceText, 0, TokenType.SYMBOL, "!")));
+        assertThat(tokens.peek(), equalTo(token(sourceText, 0, 1, TokenType.SYMBOL)));
+        assertThat(tokens.peek(), equalTo(token(sourceText, 0, 1, TokenType.SYMBOL)));
         tokens.skip(TokenType.SYMBOL);
-        assertThat(tokens.peek(), equalTo(token(sourceText, 1, TokenType.IDENTIFIER, "x")));
+        assertThat(tokens.peek(), equalTo(token(sourceText, 1, 2, TokenType.IDENTIFIER)));
     }
 
     @Test
@@ -253,29 +253,30 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
-        assertThat(tokens.peek(0), equalTo(token(sourceText, 0, TokenType.SYMBOL, "!")));
-        assertThat(tokens.peek(1), equalTo(token(sourceText, 1, TokenType.IDENTIFIER, "x")));
-        assertThat(tokens.peek(2), equalTo(token(sourceText, 2, TokenType.SYMBOL, "!")));
+        assertThat(tokens.peek(0), equalTo(token(sourceText, 0, 1, TokenType.SYMBOL)));
+        assertThat(tokens.peek(1), equalTo(token(sourceText, 1, 2, TokenType.IDENTIFIER)));
+        assertThat(tokens.peek(2), equalTo(token(sourceText, 2, 3, TokenType.SYMBOL)));
         tokens.skip(TokenType.SYMBOL);
-        assertThat(tokens.peek(0), equalTo(token(sourceText, 1, TokenType.IDENTIFIER, "x")));
-        assertThat(tokens.peek(1), equalTo(token(sourceText, 2, TokenType.SYMBOL, "!")));
-        assertThat(tokens.peek(2), equalTo(token(sourceText, 3, TokenType.END, "")));
+        assertThat(tokens.peek(0), equalTo(token(sourceText, 1, 2, TokenType.IDENTIFIER)));
+        assertThat(tokens.peek(1), equalTo(token(sourceText, 2, 3, TokenType.SYMBOL)));
+        assertThat(tokens.peek(2), equalTo(token(sourceText, 3, 3, TokenType.END)));
     }
 
     private Token<TokenType> token(
         SourceText sourceText,
-        int characterIndex,
-        TokenType tokenType,
-        String value
+        int startCharacterIndex,
+        int endCharacterIndex,
+        TokenType tokenType
     ) {
-        var sourcePosition = sourceText.characterPosition(characterIndex);
-        return new Token<>(sourcePosition.to(sourcePosition), tokenType, value);
+        var start = sourceText.characterPosition(startCharacterIndex);
+        var end = sourceText.characterPosition(endCharacterIndex);
+        return new Token<>(start.to(end), tokenType);
     }
 }