From 3666a4af2eb35177eecd557d2a75f4bac9da75b9 Mon Sep 17 00:00:00 2001 From: Michael Williamson Date: Wed, 8 Jan 2025 19:14:03 +0000 Subject: Use source text to implement FileFragmentSource --- .../compiler/tokeniser/RegexTokeniserTests.java | 81 ++++++++--- .../compiler/tokeniser/TokenIteratorTests.java | 160 ++++++++++++--------- 2 files changed, 154 insertions(+), 87 deletions(-) (limited to 'src/test/java/org/shedlang/compiler/tokeniser') diff --git a/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java b/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java index 06989d4d..be47dda5 100644 --- a/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java +++ b/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java @@ -1,8 +1,7 @@ package org.shedlang.compiler.tokeniser; import org.junit.jupiter.api.Test; -import org.shedlang.compiler.sources.Source; -import org.shedlang.compiler.sources.Sources; +import org.zwobble.sourcetext.SourceText; import java.util.List; @@ -21,9 +20,9 @@ public class RegexTokeniserTests { @Test public void emptyStringIsTokenisedToNoTokens() { var tokeniser = new RegexTokeniser<>(TokenType.UNKNOWN, List.of()); - var source = source(""); + var sourceText = sourceText(""); - var result = tokeniser.tokenise(source); + var result = tokeniser.tokenise(sourceText); assertThat(result, isSequence()); } @@ -33,12 +32,16 @@ public class RegexTokeniserTests { var tokeniser = new RegexTokeniser<>(TokenType.UNKNOWN, List.of( RegexTokeniser.rule(TokenType.IDENTIFIER, "[A-Za-z0-9]+") )); - var source = source("count"); + var sourceText = sourceText("count"); - var result = tokeniser.tokenise(source); + var result = tokeniser.tokenise(sourceText); assertThat(result, isSequence( - equalTo(new Token<>(source.at(0, 5), TokenType.IDENTIFIER, "count")) + equalTo(new Token<>( + sourceText.characterPosition(0).to(sourceText.characterPosition(5)), + TokenType.IDENTIFIER, + "count" + )) )); } @@ -48,12 +51,16 @@ public class RegexTokeniserTests { RegexTokeniser.rule(TokenType.KEYWORD, "if"), RegexTokeniser.rule(TokenType.IDENTIFIER, "[A-Za-z0-9]+") )); - var source = source("if"); + var sourceText = sourceText("if"); - var result = tokeniser.tokenise(source); + var result = tokeniser.tokenise(sourceText); assertThat(result, isSequence( - equalTo(new Token<>(source.at(0, 2), TokenType.KEYWORD, "if")) + equalTo(new Token<>( + sourceText.characterPosition(0).to(sourceText.characterPosition(2)), + TokenType.KEYWORD, + "if" + )) )); } @@ -63,14 +70,26 @@ public class RegexTokeniserTests { RegexTokeniser.rule(TokenType.IDENTIFIER, "[A-Za-z0-9]+"), RegexTokeniser.rule(TokenType.WHITESPACE, "\\s") )); - var source = source("one two"); + var sourceText = sourceText("one two"); - var result = tokeniser.tokenise(source); + var result = tokeniser.tokenise(sourceText); assertThat(result, isSequence( - equalTo(new Token<>(source.at(0, 3), TokenType.IDENTIFIER, "one")), - equalTo(new Token<>(source.at(3, 4), TokenType.WHITESPACE, " ")), - equalTo(new Token<>(source.at(4, 7), TokenType.IDENTIFIER, "two")) + equalTo(new Token<>( + sourceText.characterPosition(0).to(sourceText.characterPosition(3)), + TokenType.IDENTIFIER, + "one" + )), + equalTo(new Token<>( + sourceText.characterPosition(3).to(sourceText.characterPosition(4)), + TokenType.WHITESPACE, + " " + )), + equalTo(new Token<>( + sourceText.characterPosition(4).to(sourceText.characterPosition(7)), + TokenType.IDENTIFIER, + "two" + )) )); } @@ -79,19 +98,35 @@ public class RegexTokeniserTests { var tokeniser = new RegexTokeniser<>(TokenType.UNKNOWN, List.of( RegexTokeniser.rule(TokenType.IDENTIFIER, "[A-Za-z0-9]+") )); - var source = source("!!a!"); + var sourceText = sourceText("!!a!"); - var result = tokeniser.tokenise(source); + var result = tokeniser.tokenise(sourceText); assertThat(result, isSequence( - equalTo(new Token<>(source.at(0, 1), TokenType.UNKNOWN, "!")), - equalTo(new Token<>(source.at(1, 2), TokenType.UNKNOWN, "!")), - equalTo(new Token<>(source.at(2, 3), TokenType.IDENTIFIER, "a")), - equalTo(new Token<>(source.at(3, 4), TokenType.UNKNOWN, "!")) + equalTo(new Token<>( + sourceText.characterPosition(0).to(sourceText.characterPosition(1)), + TokenType.UNKNOWN, + "!" + )), + equalTo(new Token<>( + sourceText.characterPosition(1).to(sourceText.characterPosition(2)), + TokenType.UNKNOWN, + "!" + )), + equalTo(new Token<>( + sourceText.characterPosition(2).to(sourceText.characterPosition(3)), + TokenType.IDENTIFIER, + "a" + )), + equalTo(new Token<>( + sourceText.characterPosition(3).to(sourceText.characterPosition(4)), + TokenType.UNKNOWN, + "!" + )) )); } - private Source source(String sourceContents) { - return Sources.file("", sourceContents); + private SourceText sourceText(String sourceContents) { + return SourceText.fromString("", sourceContents); } } diff --git a/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java b/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java index 52d036a8..dd22550f 100644 --- a/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java +++ b/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java @@ -1,7 +1,7 @@ package org.shedlang.compiler.tokeniser; import org.junit.jupiter.api.Test; -import org.shedlang.compiler.sources.Sources; +import org.zwobble.sourcetext.SourceText; import java.util.List; @@ -18,13 +18,15 @@ public class TokenIteratorTests { @Test public void skipSucceedsWhenNextTokenHasMatchingTokenType() { + var sourceText = SourceText.fromString("", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); tokens.skip(TokenType.SYMBOL); @@ -36,13 +38,15 @@ public class TokenIteratorTests { @Test public void skipFailsWhenNextTokenDoesNotHaveMatchingTokenType() { + var sourceText = SourceText.fromString("", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); tokens.skip(TokenType.SYMBOL); @@ -57,13 +61,15 @@ public class TokenIteratorTests { @Test public void trySkipAdvancesIteratorAndReturnsTrueWhenNextTokenHasMatchingTokenType() { + var sourceText = SourceText.fromString("", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); assertThat(tokens.trySkip(TokenType.SYMBOL), equalTo(true)); @@ -75,13 +81,15 @@ public class TokenIteratorTests { @Test public void trySkipDoesNotAdvanceIteratorAndReturnsFalseWhenNextTokenDoesNotHaveMatchingTokenType() { + var sourceText = SourceText.fromString("", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); tokens.skip(TokenType.SYMBOL); @@ -92,13 +100,15 @@ public class TokenIteratorTests { @Test public void nextValueReturnsValueWhenCurrentTokenHasMatchingTokenType() { + var sourceText = SourceText.fromString("", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); assertThat(tokens.nextValue(TokenType.SYMBOL), equalTo("!")); @@ -107,13 +117,15 @@ public class TokenIteratorTests { @Test public void nextValueFailsWhenCurrentTokenDoesNotHaveMatchingTokenType() { + var sourceText = SourceText.fromString("", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); tokens.skip(TokenType.SYMBOL); @@ -128,13 +140,15 @@ public class TokenIteratorTests { @Test public void isNextWithOneArgReturnsTrueIfNextTokenIsOfMatchingType() { + var sourceText = SourceText.fromString("", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); tokens.skip(TokenType.SYMBOL); @@ -144,13 +158,15 @@ public class TokenIteratorTests { @Test public void isNextWithOneArgReturnsFalseIfNextTokenIsNotOfMatchingType() { + var sourceText = SourceText.fromString("", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); tokens.skip(TokenType.SYMBOL); @@ -160,13 +176,15 @@ public class TokenIteratorTests { @Test public void isNextWithTwoArgReturnsTrueIfBothNextTokensMatchRespectiveTypes() { + var sourceText = SourceText.fromString("", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); tokens.skip(TokenType.SYMBOL); @@ -176,13 +194,15 @@ public class TokenIteratorTests { @Test public void isNextWithTwoArgReturnsFalseIfFirstTokenDoesNotMatchRespectiveType() { + var sourceText = SourceText.fromString("", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); tokens.skip(TokenType.SYMBOL); @@ -192,13 +212,15 @@ public class TokenIteratorTests { @Test public void isNextWithTwoArgReturnsFalseIfSecondTokenDoesNotMatchRespectiveType() { + var sourceText = SourceText.fromString("", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); tokens.skip(TokenType.SYMBOL); @@ -208,42 +230,52 @@ public class TokenIteratorTests { @Test public void peekReturnsNextTokenWithoutAdvancing() { + var sourceText = SourceText.fromString("", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); - assertThat(tokens.peek(), equalTo(token(0, TokenType.SYMBOL, "!"))); - assertThat(tokens.peek(), equalTo(token(0, TokenType.SYMBOL, "!"))); + assertThat(tokens.peek(), equalTo(token(sourceText, 0, TokenType.SYMBOL, "!"))); + assertThat(tokens.peek(), equalTo(token(sourceText, 0, TokenType.SYMBOL, "!"))); tokens.skip(TokenType.SYMBOL); - assertThat(tokens.peek(), equalTo(token(1, TokenType.IDENTIFIER, "x"))); + assertThat(tokens.peek(), equalTo(token(sourceText, 1, TokenType.IDENTIFIER, "x"))); } @Test public void canPeekMultipleTokensAhead() { + var sourceText = SourceText.fromString("", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); - assertThat(tokens.peek(0), equalTo(token(0, TokenType.SYMBOL, "!"))); - assertThat(tokens.peek(1), equalTo(token(1, TokenType.IDENTIFIER, "x"))); - assertThat(tokens.peek(2), equalTo(token(2, TokenType.SYMBOL, "!"))); + assertThat(tokens.peek(0), equalTo(token(sourceText, 0, TokenType.SYMBOL, "!"))); + assertThat(tokens.peek(1), equalTo(token(sourceText, 1, TokenType.IDENTIFIER, "x"))); + assertThat(tokens.peek(2), equalTo(token(sourceText, 2, TokenType.SYMBOL, "!"))); tokens.skip(TokenType.SYMBOL); - assertThat(tokens.peek(0), equalTo(token(1, TokenType.IDENTIFIER, "x"))); - assertThat(tokens.peek(1), equalTo(token(2, TokenType.SYMBOL, "!"))); - assertThat(tokens.peek(2), equalTo(token(3, TokenType.END, ""))); + assertThat(tokens.peek(0), equalTo(token(sourceText, 1, TokenType.IDENTIFIER, "x"))); + assertThat(tokens.peek(1), equalTo(token(sourceText, 2, TokenType.SYMBOL, "!"))); + assertThat(tokens.peek(2), equalTo(token(sourceText, 3, TokenType.END, ""))); } - private Token token(int characterIndex, TokenType tokenType, String value) { - return new Token<>(Sources.NULL, tokenType, value); + private Token token( + SourceText sourceText, + int characterIndex, + TokenType tokenType, + String value + ) { + var sourcePosition = sourceText.characterPosition(characterIndex); + return new Token<>(sourcePosition.to(sourcePosition), tokenType, value); } } -- cgit v1.2.3