diff options
| author | Michael Williamson <mike@zwobble.org> | 2025-01-08 19:14:03 +0000 |
|---|---|---|
| committer | Michael Williamson <mike@zwobble.org> | 2025-01-09 20:36:42 +0000 |
| commit | 3666a4af2eb35177eecd557d2a75f4bac9da75b9 (patch) | |
| tree | a85230de832bc7f970dd89aaa6bb2da6ea5875f4 /src/test/java/org/shedlang/compiler/tokeniser | |
| parent | 1e6e1f7bb8776fc013f76828ed8df3f9297d095b (diff) | |
Use source text to implement FileFragmentSource
Diffstat (limited to 'src/test/java/org/shedlang/compiler/tokeniser')
| -rw-r--r-- | src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java | 81 | ||||
| -rw-r--r-- | src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java | 160 |
2 files changed, 154 insertions, 87 deletions
diff --git a/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java b/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java index 06989d4d..be47dda5 100644 --- a/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java +++ b/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java @@ -1,8 +1,7 @@ package org.shedlang.compiler.tokeniser; import org.junit.jupiter.api.Test; -import org.shedlang.compiler.sources.Source; -import org.shedlang.compiler.sources.Sources; +import org.zwobble.sourcetext.SourceText; import java.util.List; @@ -21,9 +20,9 @@ public class RegexTokeniserTests { @Test public void emptyStringIsTokenisedToNoTokens() { var tokeniser = new RegexTokeniser<>(TokenType.UNKNOWN, List.of()); - var source = source(""); + var sourceText = sourceText(""); - var result = tokeniser.tokenise(source); + var result = tokeniser.tokenise(sourceText); assertThat(result, isSequence()); } @@ -33,12 +32,16 @@ public class RegexTokeniserTests { var tokeniser = new RegexTokeniser<>(TokenType.UNKNOWN, List.of( RegexTokeniser.rule(TokenType.IDENTIFIER, "[A-Za-z0-9]+") )); - var source = source("count"); + var sourceText = sourceText("count"); - var result = tokeniser.tokenise(source); + var result = tokeniser.tokenise(sourceText); assertThat(result, isSequence( - equalTo(new Token<>(source.at(0, 5), TokenType.IDENTIFIER, "count")) + equalTo(new Token<>( + sourceText.characterPosition(0).to(sourceText.characterPosition(5)), + TokenType.IDENTIFIER, + "count" + )) )); } @@ -48,12 +51,16 @@ public class RegexTokeniserTests { RegexTokeniser.rule(TokenType.KEYWORD, "if"), RegexTokeniser.rule(TokenType.IDENTIFIER, "[A-Za-z0-9]+") )); - var source = source("if"); + var sourceText = sourceText("if"); - var result = tokeniser.tokenise(source); + var result = tokeniser.tokenise(sourceText); assertThat(result, isSequence( - equalTo(new Token<>(source.at(0, 2), TokenType.KEYWORD, "if")) + equalTo(new Token<>( + sourceText.characterPosition(0).to(sourceText.characterPosition(2)), + TokenType.KEYWORD, + "if" + )) )); } @@ -63,14 +70,26 @@ public class RegexTokeniserTests { RegexTokeniser.rule(TokenType.IDENTIFIER, "[A-Za-z0-9]+"), RegexTokeniser.rule(TokenType.WHITESPACE, "\\s") )); - var source = source("one two"); + var sourceText = sourceText("one two"); - var result = tokeniser.tokenise(source); + var result = tokeniser.tokenise(sourceText); assertThat(result, isSequence( - equalTo(new Token<>(source.at(0, 3), TokenType.IDENTIFIER, "one")), - equalTo(new Token<>(source.at(3, 4), TokenType.WHITESPACE, " ")), - equalTo(new Token<>(source.at(4, 7), TokenType.IDENTIFIER, "two")) + equalTo(new Token<>( + sourceText.characterPosition(0).to(sourceText.characterPosition(3)), + TokenType.IDENTIFIER, + "one" + )), + equalTo(new Token<>( + sourceText.characterPosition(3).to(sourceText.characterPosition(4)), + TokenType.WHITESPACE, + " " + )), + equalTo(new Token<>( + sourceText.characterPosition(4).to(sourceText.characterPosition(7)), + TokenType.IDENTIFIER, + "two" + )) )); } @@ -79,19 +98,35 @@ public class RegexTokeniserTests { var tokeniser = new RegexTokeniser<>(TokenType.UNKNOWN, List.of( RegexTokeniser.rule(TokenType.IDENTIFIER, "[A-Za-z0-9]+") )); - var source = source("!!a!"); + var sourceText = sourceText("!!a!"); - var result = tokeniser.tokenise(source); + var result = tokeniser.tokenise(sourceText); assertThat(result, isSequence( - equalTo(new Token<>(source.at(0, 1), TokenType.UNKNOWN, "!")), - equalTo(new Token<>(source.at(1, 2), TokenType.UNKNOWN, "!")), - equalTo(new Token<>(source.at(2, 3), TokenType.IDENTIFIER, "a")), - equalTo(new Token<>(source.at(3, 4), TokenType.UNKNOWN, "!")) + equalTo(new Token<>( + sourceText.characterPosition(0).to(sourceText.characterPosition(1)), + TokenType.UNKNOWN, + "!" + )), + equalTo(new Token<>( + sourceText.characterPosition(1).to(sourceText.characterPosition(2)), + TokenType.UNKNOWN, + "!" + )), + equalTo(new Token<>( + sourceText.characterPosition(2).to(sourceText.characterPosition(3)), + TokenType.IDENTIFIER, + "a" + )), + equalTo(new Token<>( + sourceText.characterPosition(3).to(sourceText.characterPosition(4)), + TokenType.UNKNOWN, + "!" + )) )); } - private Source source(String sourceContents) { - return Sources.file("<filename>", sourceContents); + private SourceText sourceText(String sourceContents) { + return SourceText.fromString("<filename>", sourceContents); } } diff --git a/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java b/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java index 52d036a8..dd22550f 100644 --- a/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java +++ b/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java @@ -1,7 +1,7 @@ package org.shedlang.compiler.tokeniser; import org.junit.jupiter.api.Test; -import org.shedlang.compiler.sources.Sources; +import org.zwobble.sourcetext.SourceText; import java.util.List; @@ -18,13 +18,15 @@ public class TokenIteratorTests { @Test public void skipSucceedsWhenNextTokenHasMatchingTokenType() { + var sourceText = SourceText.fromString("<filename>", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); tokens.skip(TokenType.SYMBOL); @@ -36,13 +38,15 @@ public class TokenIteratorTests { @Test public void skipFailsWhenNextTokenDoesNotHaveMatchingTokenType() { + var sourceText = SourceText.fromString("<filename>", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); tokens.skip(TokenType.SYMBOL); @@ -57,13 +61,15 @@ public class TokenIteratorTests { @Test public void trySkipAdvancesIteratorAndReturnsTrueWhenNextTokenHasMatchingTokenType() { + var sourceText = SourceText.fromString("<filename>", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); assertThat(tokens.trySkip(TokenType.SYMBOL), equalTo(true)); @@ -75,13 +81,15 @@ public class TokenIteratorTests { @Test public void trySkipDoesNotAdvanceIteratorAndReturnsFalseWhenNextTokenDoesNotHaveMatchingTokenType() { + var sourceText = SourceText.fromString("<filename>", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); tokens.skip(TokenType.SYMBOL); @@ -92,13 +100,15 @@ public class TokenIteratorTests { @Test public void nextValueReturnsValueWhenCurrentTokenHasMatchingTokenType() { + var sourceText = SourceText.fromString("<filename>", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); assertThat(tokens.nextValue(TokenType.SYMBOL), equalTo("!")); @@ -107,13 +117,15 @@ public class TokenIteratorTests { @Test public void nextValueFailsWhenCurrentTokenDoesNotHaveMatchingTokenType() { + var sourceText = SourceText.fromString("<filename>", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); tokens.skip(TokenType.SYMBOL); @@ -128,13 +140,15 @@ public class TokenIteratorTests { @Test public void isNextWithOneArgReturnsTrueIfNextTokenIsOfMatchingType() { + var sourceText = SourceText.fromString("<filename>", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); tokens.skip(TokenType.SYMBOL); @@ -144,13 +158,15 @@ public class TokenIteratorTests { @Test public void isNextWithOneArgReturnsFalseIfNextTokenIsNotOfMatchingType() { + var sourceText = SourceText.fromString("<filename>", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); tokens.skip(TokenType.SYMBOL); @@ -160,13 +176,15 @@ public class TokenIteratorTests { @Test public void isNextWithTwoArgReturnsTrueIfBothNextTokensMatchRespectiveTypes() { + var sourceText = SourceText.fromString("<filename>", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); tokens.skip(TokenType.SYMBOL); @@ -176,13 +194,15 @@ public class TokenIteratorTests { @Test public void isNextWithTwoArgReturnsFalseIfFirstTokenDoesNotMatchRespectiveType() { + var sourceText = SourceText.fromString("<filename>", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); tokens.skip(TokenType.SYMBOL); @@ -192,13 +212,15 @@ public class TokenIteratorTests { @Test public void isNextWithTwoArgReturnsFalseIfSecondTokenDoesNotMatchRespectiveType() { + var sourceText = SourceText.fromString("<filename>", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); tokens.skip(TokenType.SYMBOL); @@ -208,42 +230,52 @@ public class TokenIteratorTests { @Test public void peekReturnsNextTokenWithoutAdvancing() { + var sourceText = SourceText.fromString("<filename>", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); - assertThat(tokens.peek(), equalTo(token(0, TokenType.SYMBOL, "!"))); - assertThat(tokens.peek(), equalTo(token(0, TokenType.SYMBOL, "!"))); + assertThat(tokens.peek(), equalTo(token(sourceText, 0, TokenType.SYMBOL, "!"))); + assertThat(tokens.peek(), equalTo(token(sourceText, 0, TokenType.SYMBOL, "!"))); tokens.skip(TokenType.SYMBOL); - assertThat(tokens.peek(), equalTo(token(1, TokenType.IDENTIFIER, "x"))); + assertThat(tokens.peek(), equalTo(token(sourceText, 1, TokenType.IDENTIFIER, "x"))); } @Test public void canPeekMultipleTokensAhead() { + var sourceText = SourceText.fromString("<filename>", "!x!"); + var tokens = new TokenIterator<>( List.of( - token(0, TokenType.SYMBOL, "!"), - token(1, TokenType.IDENTIFIER, "x"), - token(2, TokenType.SYMBOL, "!") + token(sourceText, 0, TokenType.SYMBOL, "!"), + token(sourceText, 1, TokenType.IDENTIFIER, "x"), + token(sourceText, 2, TokenType.SYMBOL, "!") ), - token(3, TokenType.END, "") + token(sourceText, 3, TokenType.END, "") ); - assertThat(tokens.peek(0), equalTo(token(0, TokenType.SYMBOL, "!"))); - assertThat(tokens.peek(1), equalTo(token(1, TokenType.IDENTIFIER, "x"))); - assertThat(tokens.peek(2), equalTo(token(2, TokenType.SYMBOL, "!"))); + assertThat(tokens.peek(0), equalTo(token(sourceText, 0, TokenType.SYMBOL, "!"))); + assertThat(tokens.peek(1), equalTo(token(sourceText, 1, TokenType.IDENTIFIER, "x"))); + assertThat(tokens.peek(2), equalTo(token(sourceText, 2, TokenType.SYMBOL, "!"))); tokens.skip(TokenType.SYMBOL); - assertThat(tokens.peek(0), equalTo(token(1, TokenType.IDENTIFIER, "x"))); - assertThat(tokens.peek(1), equalTo(token(2, TokenType.SYMBOL, "!"))); - assertThat(tokens.peek(2), equalTo(token(3, TokenType.END, ""))); + assertThat(tokens.peek(0), equalTo(token(sourceText, 1, TokenType.IDENTIFIER, "x"))); + assertThat(tokens.peek(1), equalTo(token(sourceText, 2, TokenType.SYMBOL, "!"))); + assertThat(tokens.peek(2), equalTo(token(sourceText, 3, TokenType.END, ""))); } - private Token<TokenType> token(int characterIndex, TokenType tokenType, String value) { - return new Token<>(Sources.NULL, tokenType, value); + private Token<TokenType> token( + SourceText sourceText, + int characterIndex, + TokenType tokenType, + String value + ) { + var sourcePosition = sourceText.characterPosition(characterIndex); + return new Token<>(sourcePosition.to(sourcePosition), tokenType, value); } } |
