Use source range to get value of token

4 files changed, 12 insertions, 10 deletions

diff --git a/src/main/java/org/shedlang/compiler/parser/Tokeniser.java b/src/main/java/org/shedlang/compiler/parser/Tokeniser.java
index 85172578..8c1aefa3 100644
--- a/src/main/java/org/shedlang/compiler/parser/Tokeniser.java
+++ b/src/main/java/org/shedlang/compiler/parser/Tokeniser.java
@@ -82,14 +82,14 @@ public class Tokeniser {
             )
             .map(token ->
                 token.tokenType() == TokenType.IDENTIFIER
-                    ? new Token<>(token.sourceRange(), KEYWORDS.getOrDefault(token.value(), token.tokenType()), token.value())
+                    ? new Token<>(token.sourceRange(), KEYWORDS.getOrDefault(token.value().toString(), token.tokenType()))
                     : token
             )
             .collect(Collectors.toList());
 
         var endPosition = sourceText.characterPosition(sourceText.characterLength());
         var endSourceRange = endPosition.to(endPosition);
-        var endToken = token(endSourceRange, TokenType.END, "");
+        var endToken = token(endSourceRange, TokenType.END);
 
         return new TokenIterator<>(tokens, endToken);
     }
diff --git a/src/main/java/org/shedlang/compiler/tokeniser/RegexTokeniser.java b/src/main/java/org/shedlang/compiler/tokeniser/RegexTokeniser.java
index a34ae3d5..85818dbd 100644
--- a/src/main/java/org/shedlang/compiler/tokeniser/RegexTokeniser.java
+++ b/src/main/java/org/shedlang/compiler/tokeniser/RegexTokeniser.java
@@ -62,9 +62,7 @@ public class RegexTokeniser<T> {
             var endPosition = sourceText.characterPosition(matcher.end());
             tokens.add(new Token<>(
                 startPosition.to(endPosition),
-                tokenType,
-                // TODO: the text value is redundant, can be gotten from the source range?
-                matcher.group()
+                tokenType
             ));
             matcher.region(matcher.end(), sourceText.characterLength());
         }
diff --git a/src/main/java/org/shedlang/compiler/tokeniser/Token.java b/src/main/java/org/shedlang/compiler/tokeniser/Token.java
index 7e131304..eb2d6ba9 100644
--- a/src/main/java/org/shedlang/compiler/tokeniser/Token.java
+++ b/src/main/java/org/shedlang/compiler/tokeniser/Token.java
@@ -2,12 +2,16 @@ package org.shedlang.compiler.tokeniser;
 
 import org.zwobble.sourcetext.SourceRange;
 
-public record Token<T>(SourceRange sourceRange, T tokenType, String value) {
-    public static <T> Token<T> token(SourceRange sourceRange, T tokenType, String value) {
-        return new Token<T>(sourceRange, tokenType, value);
+public record Token<T>(SourceRange sourceRange, T tokenType) {
+    public static <T> Token<T> token(SourceRange sourceRange, T tokenType) {
+        return new Token<T>(sourceRange, tokenType);
     }
 
     public String describe() {
-        return tokenType + ": " + value;
+        return tokenType + ": " + sourceRange.charSequence();
+    }
+
+    public CharSequence value() {
+        return this.sourceRange.charSequence();
     }
 }
diff --git a/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java b/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java
index 9b3467a8..4d1cd5df 100644
--- a/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java
+++ b/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java
@@ -29,7 +29,7 @@ public class TokenIterator<TTokenType> {
     }
 
     public String nextValue(TTokenType tokenType) {
-        return this.next(tokenType).value();
+        return this.next(tokenType).value().toString();
     }
 
     public boolean isNext(TTokenType tokenType) {