Use source range to get value of token

8 files changed, 90 insertions, 96 deletions

diff --git a/src/main/java/org/shedlang/compiler/parser/Tokeniser.java b/src/main/java/org/shedlang/compiler/parser/Tokeniser.java
index 85172578..8c1aefa3 100644
--- a/src/main/java/org/shedlang/compiler/parser/Tokeniser.java
+++ b/src/main/java/org/shedlang/compiler/parser/Tokeniser.java
@@ -82,14 +82,14 @@ public class Tokeniser {
             )
             .map(token ->
                 token.tokenType() == TokenType.IDENTIFIER
-                    ? new Token<>(token.sourceRange(), KEYWORDS.getOrDefault(token.value(), token.tokenType()), token.value())
+                    ? new Token<>(token.sourceRange(), KEYWORDS.getOrDefault(token.value().toString(), token.tokenType()))
                     : token
             )
             .collect(Collectors.toList());
 
         var endPosition = sourceText.characterPosition(sourceText.characterLength());
         var endSourceRange = endPosition.to(endPosition);
-        var endToken = token(endSourceRange, TokenType.END, "");
+        var endToken = token(endSourceRange, TokenType.END);
 
         return new TokenIterator<>(tokens, endToken);
     }
diff --git a/src/main/java/org/shedlang/compiler/tokeniser/RegexTokeniser.java b/src/main/java/org/shedlang/compiler/tokeniser/RegexTokeniser.java
index a34ae3d5..85818dbd 100644
--- a/src/main/java/org/shedlang/compiler/tokeniser/RegexTokeniser.java
+++ b/src/main/java/org/shedlang/compiler/tokeniser/RegexTokeniser.java
@@ -62,9 +62,7 @@ public class RegexTokeniser<T> {
             var endPosition = sourceText.characterPosition(matcher.end());
             tokens.add(new Token<>(
                 startPosition.to(endPosition),
-                tokenType,
-                // TODO: the text value is redundant, can be gotten from the source range?
-                matcher.group()
+                tokenType
             ));
             matcher.region(matcher.end(), sourceText.characterLength());
         }
diff --git a/src/main/java/org/shedlang/compiler/tokeniser/Token.java b/src/main/java/org/shedlang/compiler/tokeniser/Token.java
index 7e131304..eb2d6ba9 100644
--- a/src/main/java/org/shedlang/compiler/tokeniser/Token.java
+++ b/src/main/java/org/shedlang/compiler/tokeniser/Token.java
@@ -2,12 +2,16 @@ package org.shedlang.compiler.tokeniser;
 
 import org.zwobble.sourcetext.SourceRange;
 
-public record Token<T>(SourceRange sourceRange, T tokenType, String value) {
-    public static <T> Token<T> token(SourceRange sourceRange, T tokenType, String value) {
-        return new Token<T>(sourceRange, tokenType, value);
+public record Token<T>(SourceRange sourceRange, T tokenType) {
+    public static <T> Token<T> token(SourceRange sourceRange, T tokenType) {
+        return new Token<T>(sourceRange, tokenType);
     }
 
     public String describe() {
-        return tokenType + ": " + value;
+        return tokenType + ": " + sourceRange.charSequence();
+    }
+
+    public CharSequence value() {
+        return this.sourceRange.charSequence();
     }
 }
diff --git a/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java b/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java
index 9b3467a8..4d1cd5df 100644
--- a/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java
+++ b/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java
@@ -29,7 +29,7 @@ public class TokenIterator<TTokenType> {
     }
 
     public String nextValue(TTokenType tokenType) {
-        return this.next(tokenType).value();
+        return this.next(tokenType).value().toString();
     }
 
     public boolean isNext(TTokenType tokenType) {
diff --git a/src/test/java/org/shedlang/compiler/parser/ParserFunctionDefinitionTests.java b/src/test/java/org/shedlang/compiler/parser/ParserFunctionDefinitionTests.java
index 9c3148a3..c5c2932f 100644
--- a/src/test/java/org/shedlang/compiler/parser/ParserFunctionDefinitionTests.java
+++ b/src/test/java/org/shedlang/compiler/parser/ParserFunctionDefinitionTests.java
@@ -119,7 +119,7 @@ public class ParserFunctionDefinitionTests {
             () -> parseString(source, Parser::parseNamespaceStatement)
         );
 
-        assertThat(error.getActual().value(), equalTo("c"));
+        assertThat(error.getActual().value().toString(), equalTo("c"));
         assertThat(error.getExpected(), equalTo(TokenType.SYMBOL_BRACE_CLOSE.toString()));
     }
 }
diff --git a/src/test/java/org/shedlang/compiler/parser/ParserFunctionExpressionTests.java b/src/test/java/org/shedlang/compiler/parser/ParserFunctionExpressionTests.java
index 8b4c72a6..d5db8035 100644
--- a/src/test/java/org/shedlang/compiler/parser/ParserFunctionExpressionTests.java
+++ b/src/test/java/org/shedlang/compiler/parser/ParserFunctionExpressionTests.java
@@ -89,7 +89,7 @@ public class ParserFunctionExpressionTests {
             () -> parseString(source, Parser::parseTopLevelExpression)
         );
 
-        assertThat(error.getActual().value(), equalTo("c"));
+        assertThat(error.getActual().value().toString(), equalTo("c"));
         assertThat(error.getExpected(), equalTo(TokenType.SYMBOL_BRACE_CLOSE.toString()));
     }
 }
diff --git a/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java b/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java
index be47dda5..dd64735a 100644
--- a/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java
+++ b/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java
@@ -39,8 +39,7 @@ public class RegexTokeniserTests {
         assertThat(result, isSequence(
             equalTo(new Token<>(
                 sourceText.characterPosition(0).to(sourceText.characterPosition(5)),
-                TokenType.IDENTIFIER,
-                "count"
+                TokenType.IDENTIFIER
             ))
         ));
     }
@@ -58,8 +57,7 @@ public class RegexTokeniserTests {
         assertThat(result, isSequence(
             equalTo(new Token<>(
                 sourceText.characterPosition(0).to(sourceText.characterPosition(2)),
-                TokenType.KEYWORD,
-                "if"
+                TokenType.KEYWORD
             ))
         ));
     }
@@ -77,18 +75,15 @@ public class RegexTokeniserTests {
         assertThat(result, isSequence(
             equalTo(new Token<>(
                 sourceText.characterPosition(0).to(sourceText.characterPosition(3)),
-                TokenType.IDENTIFIER,
-                "one"
+                TokenType.IDENTIFIER
             )),
             equalTo(new Token<>(
                 sourceText.characterPosition(3).to(sourceText.characterPosition(4)),
-                TokenType.WHITESPACE,
-                " "
+                TokenType.WHITESPACE
             )),
             equalTo(new Token<>(
                 sourceText.characterPosition(4).to(sourceText.characterPosition(7)),
-                TokenType.IDENTIFIER,
-                "two"
+                TokenType.IDENTIFIER
             ))
         ));
     }
@@ -105,23 +100,19 @@ public class RegexTokeniserTests {
         assertThat(result, isSequence(
             equalTo(new Token<>(
                 sourceText.characterPosition(0).to(sourceText.characterPosition(1)),
-                TokenType.UNKNOWN,
-                "!"
+                TokenType.UNKNOWN
             )),
             equalTo(new Token<>(
                 sourceText.characterPosition(1).to(sourceText.characterPosition(2)),
-                TokenType.UNKNOWN,
-                "!"
+                TokenType.UNKNOWN
             )),
             equalTo(new Token<>(
                 sourceText.characterPosition(2).to(sourceText.characterPosition(3)),
-                TokenType.IDENTIFIER,
-                "a"
+                TokenType.IDENTIFIER
             )),
             equalTo(new Token<>(
                 sourceText.characterPosition(3).to(sourceText.characterPosition(4)),
-                TokenType.UNKNOWN,
-                "!"
+                TokenType.UNKNOWN
             ))
         ));
     }
diff --git a/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java b/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java
index dd22550f..80bafd0b 100644
--- a/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java
+++ b/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java
@@ -22,11 +22,11 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -42,11 +42,11 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -65,11 +65,11 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
         assertThat(tokens.trySkip(TokenType.SYMBOL), equalTo(true));
@@ -85,11 +85,11 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -104,11 +104,11 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
         assertThat(tokens.nextValue(TokenType.SYMBOL), equalTo("!"));
@@ -121,11 +121,11 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -144,11 +144,11 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -162,11 +162,11 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -180,11 +180,11 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -198,11 +198,11 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -216,11 +216,11 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -234,17 +234,17 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
-        assertThat(tokens.peek(), equalTo(token(sourceText, 0, TokenType.SYMBOL, "!")));
-        assertThat(tokens.peek(), equalTo(token(sourceText, 0, TokenType.SYMBOL, "!")));
+        assertThat(tokens.peek(), equalTo(token(sourceText, 0, 1, TokenType.SYMBOL)));
+        assertThat(tokens.peek(), equalTo(token(sourceText, 0, 1, TokenType.SYMBOL)));
         tokens.skip(TokenType.SYMBOL);
-        assertThat(tokens.peek(), equalTo(token(sourceText, 1, TokenType.IDENTIFIER, "x")));
+        assertThat(tokens.peek(), equalTo(token(sourceText, 1, 2, TokenType.IDENTIFIER)));
     }
 
     @Test
@@ -253,29 +253,30 @@ public class TokenIteratorTests {
 
         var tokens = new TokenIterator<>(
             List.of(
-                token(sourceText, 0, TokenType.SYMBOL, "!"),
-                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
-                token(sourceText, 2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, 1, TokenType.SYMBOL),
+                token(sourceText, 1, 2, TokenType.IDENTIFIER),
+                token(sourceText, 2, 3, TokenType.SYMBOL)
             ),
-            token(sourceText, 3, TokenType.END, "")
+            token(sourceText, 3, 3, TokenType.END)
         );
 
-        assertThat(tokens.peek(0), equalTo(token(sourceText, 0, TokenType.SYMBOL, "!")));
-        assertThat(tokens.peek(1), equalTo(token(sourceText, 1, TokenType.IDENTIFIER, "x")));
-        assertThat(tokens.peek(2), equalTo(token(sourceText, 2, TokenType.SYMBOL, "!")));
+        assertThat(tokens.peek(0), equalTo(token(sourceText, 0, 1, TokenType.SYMBOL)));
+        assertThat(tokens.peek(1), equalTo(token(sourceText, 1, 2, TokenType.IDENTIFIER)));
+        assertThat(tokens.peek(2), equalTo(token(sourceText, 2, 3, TokenType.SYMBOL)));
         tokens.skip(TokenType.SYMBOL);
-        assertThat(tokens.peek(0), equalTo(token(sourceText, 1, TokenType.IDENTIFIER, "x")));
-        assertThat(tokens.peek(1), equalTo(token(sourceText, 2, TokenType.SYMBOL, "!")));
-        assertThat(tokens.peek(2), equalTo(token(sourceText, 3, TokenType.END, "")));
+        assertThat(tokens.peek(0), equalTo(token(sourceText, 1, 2, TokenType.IDENTIFIER)));
+        assertThat(tokens.peek(1), equalTo(token(sourceText, 2, 3, TokenType.SYMBOL)));
+        assertThat(tokens.peek(2), equalTo(token(sourceText, 3, 3, TokenType.END)));
     }
 
     private Token<TokenType> token(
         SourceText sourceText,
-        int characterIndex,
-        TokenType tokenType,
-        String value
+        int startCharacterIndex,
+        int endCharacterIndex,
+        TokenType tokenType
     ) {
-        var sourcePosition = sourceText.characterPosition(characterIndex);
-        return new Token<>(sourcePosition.to(sourcePosition), tokenType, value);
+        var start = sourceText.characterPosition(startCharacterIndex);
+        var end = sourceText.characterPosition(endCharacterIndex);
+        return new Token<>(start.to(end), tokenType);
     }
 }