From acb019f53eefaee7dd7ab6e808cab0a0daa0b91f Mon Sep 17 00:00:00 2001
From: Michael Williamson <mike@zwobble.org>
Date: Thu, 9 Jan 2025 20:43:55 +0000
Subject: Use source range to get value of token

---
 src/main/java/org/shedlang/compiler/parser/Tokeniser.java    |  4 ++--
 .../java/org/shedlang/compiler/tokeniser/RegexTokeniser.java |  4 +---
 src/main/java/org/shedlang/compiler/tokeniser/Token.java     | 12 ++++++++----
 .../java/org/shedlang/compiler/tokeniser/TokenIterator.java  |  2 +-
 4 files changed, 12 insertions(+), 10 deletions(-)

(limited to 'src/main/java')

diff --git a/src/main/java/org/shedlang/compiler/parser/Tokeniser.java b/src/main/java/org/shedlang/compiler/parser/Tokeniser.java
index 85172578..8c1aefa3 100644
--- a/src/main/java/org/shedlang/compiler/parser/Tokeniser.java
+++ b/src/main/java/org/shedlang/compiler/parser/Tokeniser.java
@@ -82,14 +82,14 @@ public class Tokeniser {
             )
             .map(token ->
                 token.tokenType() == TokenType.IDENTIFIER
-                    ? new Token<>(token.sourceRange(), KEYWORDS.getOrDefault(token.value(), token.tokenType()), token.value())
+                    ? new Token<>(token.sourceRange(), KEYWORDS.getOrDefault(token.value().toString(), token.tokenType()))
                     : token
             )
             .collect(Collectors.toList());
 
         var endPosition = sourceText.characterPosition(sourceText.characterLength());
         var endSourceRange = endPosition.to(endPosition);
-        var endToken = token(endSourceRange, TokenType.END, "");
+        var endToken = token(endSourceRange, TokenType.END);
 
         return new TokenIterator<>(tokens, endToken);
     }
diff --git a/src/main/java/org/shedlang/compiler/tokeniser/RegexTokeniser.java b/src/main/java/org/shedlang/compiler/tokeniser/RegexTokeniser.java
index a34ae3d5..85818dbd 100644
--- a/src/main/java/org/shedlang/compiler/tokeniser/RegexTokeniser.java
+++ b/src/main/java/org/shedlang/compiler/tokeniser/RegexTokeniser.java
@@ -62,9 +62,7 @@ public class RegexTokeniser<T> {
             var endPosition = sourceText.characterPosition(matcher.end());
             tokens.add(new Token<>(
                 startPosition.to(endPosition),
-                tokenType,
-                // TODO: the text value is redundant, can be gotten from the source range?
-                matcher.group()
+                tokenType
             ));
             matcher.region(matcher.end(), sourceText.characterLength());
         }
diff --git a/src/main/java/org/shedlang/compiler/tokeniser/Token.java b/src/main/java/org/shedlang/compiler/tokeniser/Token.java
index 7e131304..eb2d6ba9 100644
--- a/src/main/java/org/shedlang/compiler/tokeniser/Token.java
+++ b/src/main/java/org/shedlang/compiler/tokeniser/Token.java
@@ -2,12 +2,16 @@ package org.shedlang.compiler.tokeniser;
 
 import org.zwobble.sourcetext.SourceRange;
 
-public record Token<T>(SourceRange sourceRange, T tokenType, String value) {
-    public static <T> Token<T> token(SourceRange sourceRange, T tokenType, String value) {
-        return new Token<T>(sourceRange, tokenType, value);
+public record Token<T>(SourceRange sourceRange, T tokenType) {
+    public static <T> Token<T> token(SourceRange sourceRange, T tokenType) {
+        return new Token<T>(sourceRange, tokenType);
     }
 
     public String describe() {
-        return tokenType + ": " + value;
+        return tokenType + ": " + sourceRange.charSequence();
+    }
+
+    public CharSequence value() {
+        return this.sourceRange.charSequence();
     }
 }
diff --git a/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java b/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java
index 9b3467a8..4d1cd5df 100644
--- a/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java
+++ b/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java
@@ -29,7 +29,7 @@ public class TokenIterator<TTokenType> {
     }
 
     public String nextValue(TTokenType tokenType) {
-        return this.next(tokenType).value();
+        return this.next(tokenType).value().toString();
     }
 
     public boolean isNext(TTokenType tokenType) {
-- 
cgit v1.2.3