From acb019f53eefaee7dd7ab6e808cab0a0daa0b91f Mon Sep 17 00:00:00 2001 From: Michael Williamson Date: Thu, 9 Jan 2025 20:43:55 +0000 Subject: Use source range to get value of token --- .../java/org/shedlang/compiler/tokeniser/RegexTokeniser.java | 4 +--- src/main/java/org/shedlang/compiler/tokeniser/Token.java | 12 ++++++++---- .../java/org/shedlang/compiler/tokeniser/TokenIterator.java | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) (limited to 'src/main/java/org/shedlang/compiler/tokeniser') diff --git a/src/main/java/org/shedlang/compiler/tokeniser/RegexTokeniser.java b/src/main/java/org/shedlang/compiler/tokeniser/RegexTokeniser.java index a34ae3d5..85818dbd 100644 --- a/src/main/java/org/shedlang/compiler/tokeniser/RegexTokeniser.java +++ b/src/main/java/org/shedlang/compiler/tokeniser/RegexTokeniser.java @@ -62,9 +62,7 @@ public class RegexTokeniser { var endPosition = sourceText.characterPosition(matcher.end()); tokens.add(new Token<>( startPosition.to(endPosition), - tokenType, - // TODO: the text value is redundant, can be gotten from the source range? - matcher.group() + tokenType )); matcher.region(matcher.end(), sourceText.characterLength()); } diff --git a/src/main/java/org/shedlang/compiler/tokeniser/Token.java b/src/main/java/org/shedlang/compiler/tokeniser/Token.java index 7e131304..eb2d6ba9 100644 --- a/src/main/java/org/shedlang/compiler/tokeniser/Token.java +++ b/src/main/java/org/shedlang/compiler/tokeniser/Token.java @@ -2,12 +2,16 @@ package org.shedlang.compiler.tokeniser; import org.zwobble.sourcetext.SourceRange; -public record Token(SourceRange sourceRange, T tokenType, String value) { - public static Token token(SourceRange sourceRange, T tokenType, String value) { - return new Token(sourceRange, tokenType, value); +public record Token(SourceRange sourceRange, T tokenType) { + public static Token token(SourceRange sourceRange, T tokenType) { + return new Token(sourceRange, tokenType); } public String describe() { - return tokenType + ": " + value; + return tokenType + ": " + sourceRange.charSequence(); + } + + public CharSequence value() { + return this.sourceRange.charSequence(); } } diff --git a/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java b/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java index 9b3467a8..4d1cd5df 100644 --- a/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java +++ b/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java @@ -29,7 +29,7 @@ public class TokenIterator { } public String nextValue(TTokenType tokenType) { - return this.next(tokenType).value(); + return this.next(tokenType).value().toString(); } public boolean isNext(TTokenType tokenType) { -- cgit v1.2.3