aboutsummaryrefslogtreecommitdiff
path: root/src/test/java/org/shedlang/compiler/tokeniser
diff options
context:
space:
mode:
authorMichael Williamson <mike@zwobble.org>2025-01-08 19:14:03 +0000
committerMichael Williamson <mike@zwobble.org>2025-01-09 20:36:42 +0000
commit3666a4af2eb35177eecd557d2a75f4bac9da75b9 (patch)
treea85230de832bc7f970dd89aaa6bb2da6ea5875f4 /src/test/java/org/shedlang/compiler/tokeniser
parent1e6e1f7bb8776fc013f76828ed8df3f9297d095b (diff)
Use source text to implement FileFragmentSource
Diffstat (limited to 'src/test/java/org/shedlang/compiler/tokeniser')
-rw-r--r--src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java81
-rw-r--r--src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java160
2 files changed, 154 insertions, 87 deletions
diff --git a/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java b/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java
index 06989d4d..be47dda5 100644
--- a/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java
+++ b/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java
@@ -1,8 +1,7 @@
package org.shedlang.compiler.tokeniser;
import org.junit.jupiter.api.Test;
-import org.shedlang.compiler.sources.Source;
-import org.shedlang.compiler.sources.Sources;
+import org.zwobble.sourcetext.SourceText;
import java.util.List;
@@ -21,9 +20,9 @@ public class RegexTokeniserTests {
@Test
public void emptyStringIsTokenisedToNoTokens() {
var tokeniser = new RegexTokeniser<>(TokenType.UNKNOWN, List.of());
- var source = source("");
+ var sourceText = sourceText("");
- var result = tokeniser.tokenise(source);
+ var result = tokeniser.tokenise(sourceText);
assertThat(result, isSequence());
}
@@ -33,12 +32,16 @@ public class RegexTokeniserTests {
var tokeniser = new RegexTokeniser<>(TokenType.UNKNOWN, List.of(
RegexTokeniser.rule(TokenType.IDENTIFIER, "[A-Za-z0-9]+")
));
- var source = source("count");
+ var sourceText = sourceText("count");
- var result = tokeniser.tokenise(source);
+ var result = tokeniser.tokenise(sourceText);
assertThat(result, isSequence(
- equalTo(new Token<>(source.at(0, 5), TokenType.IDENTIFIER, "count"))
+ equalTo(new Token<>(
+ sourceText.characterPosition(0).to(sourceText.characterPosition(5)),
+ TokenType.IDENTIFIER,
+ "count"
+ ))
));
}
@@ -48,12 +51,16 @@ public class RegexTokeniserTests {
RegexTokeniser.rule(TokenType.KEYWORD, "if"),
RegexTokeniser.rule(TokenType.IDENTIFIER, "[A-Za-z0-9]+")
));
- var source = source("if");
+ var sourceText = sourceText("if");
- var result = tokeniser.tokenise(source);
+ var result = tokeniser.tokenise(sourceText);
assertThat(result, isSequence(
- equalTo(new Token<>(source.at(0, 2), TokenType.KEYWORD, "if"))
+ equalTo(new Token<>(
+ sourceText.characterPosition(0).to(sourceText.characterPosition(2)),
+ TokenType.KEYWORD,
+ "if"
+ ))
));
}
@@ -63,14 +70,26 @@ public class RegexTokeniserTests {
RegexTokeniser.rule(TokenType.IDENTIFIER, "[A-Za-z0-9]+"),
RegexTokeniser.rule(TokenType.WHITESPACE, "\\s")
));
- var source = source("one two");
+ var sourceText = sourceText("one two");
- var result = tokeniser.tokenise(source);
+ var result = tokeniser.tokenise(sourceText);
assertThat(result, isSequence(
- equalTo(new Token<>(source.at(0, 3), TokenType.IDENTIFIER, "one")),
- equalTo(new Token<>(source.at(3, 4), TokenType.WHITESPACE, " ")),
- equalTo(new Token<>(source.at(4, 7), TokenType.IDENTIFIER, "two"))
+ equalTo(new Token<>(
+ sourceText.characterPosition(0).to(sourceText.characterPosition(3)),
+ TokenType.IDENTIFIER,
+ "one"
+ )),
+ equalTo(new Token<>(
+ sourceText.characterPosition(3).to(sourceText.characterPosition(4)),
+ TokenType.WHITESPACE,
+ " "
+ )),
+ equalTo(new Token<>(
+ sourceText.characterPosition(4).to(sourceText.characterPosition(7)),
+ TokenType.IDENTIFIER,
+ "two"
+ ))
));
}
@@ -79,19 +98,35 @@ public class RegexTokeniserTests {
var tokeniser = new RegexTokeniser<>(TokenType.UNKNOWN, List.of(
RegexTokeniser.rule(TokenType.IDENTIFIER, "[A-Za-z0-9]+")
));
- var source = source("!!a!");
+ var sourceText = sourceText("!!a!");
- var result = tokeniser.tokenise(source);
+ var result = tokeniser.tokenise(sourceText);
assertThat(result, isSequence(
- equalTo(new Token<>(source.at(0, 1), TokenType.UNKNOWN, "!")),
- equalTo(new Token<>(source.at(1, 2), TokenType.UNKNOWN, "!")),
- equalTo(new Token<>(source.at(2, 3), TokenType.IDENTIFIER, "a")),
- equalTo(new Token<>(source.at(3, 4), TokenType.UNKNOWN, "!"))
+ equalTo(new Token<>(
+ sourceText.characterPosition(0).to(sourceText.characterPosition(1)),
+ TokenType.UNKNOWN,
+ "!"
+ )),
+ equalTo(new Token<>(
+ sourceText.characterPosition(1).to(sourceText.characterPosition(2)),
+ TokenType.UNKNOWN,
+ "!"
+ )),
+ equalTo(new Token<>(
+ sourceText.characterPosition(2).to(sourceText.characterPosition(3)),
+ TokenType.IDENTIFIER,
+ "a"
+ )),
+ equalTo(new Token<>(
+ sourceText.characterPosition(3).to(sourceText.characterPosition(4)),
+ TokenType.UNKNOWN,
+ "!"
+ ))
));
}
- private Source source(String sourceContents) {
- return Sources.file("<filename>", sourceContents);
+ private SourceText sourceText(String sourceContents) {
+ return SourceText.fromString("<filename>", sourceContents);
}
}
diff --git a/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java b/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java
index 52d036a8..dd22550f 100644
--- a/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java
+++ b/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java
@@ -1,7 +1,7 @@
package org.shedlang.compiler.tokeniser;
import org.junit.jupiter.api.Test;
-import org.shedlang.compiler.sources.Sources;
+import org.zwobble.sourcetext.SourceText;
import java.util.List;
@@ -18,13 +18,15 @@ public class TokenIteratorTests {
@Test
public void skipSucceedsWhenNextTokenHasMatchingTokenType() {
+ var sourceText = SourceText.fromString("<filename>", "!x!");
+
var tokens = new TokenIterator<>(
List.of(
- token(0, TokenType.SYMBOL, "!"),
- token(1, TokenType.IDENTIFIER, "x"),
- token(2, TokenType.SYMBOL, "!")
+ token(sourceText, 0, TokenType.SYMBOL, "!"),
+ token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+ token(sourceText, 2, TokenType.SYMBOL, "!")
),
- token(3, TokenType.END, "")
+ token(sourceText, 3, TokenType.END, "")
);
tokens.skip(TokenType.SYMBOL);
@@ -36,13 +38,15 @@ public class TokenIteratorTests {
@Test
public void skipFailsWhenNextTokenDoesNotHaveMatchingTokenType() {
+ var sourceText = SourceText.fromString("<filename>", "!x!");
+
var tokens = new TokenIterator<>(
List.of(
- token(0, TokenType.SYMBOL, "!"),
- token(1, TokenType.IDENTIFIER, "x"),
- token(2, TokenType.SYMBOL, "!")
+ token(sourceText, 0, TokenType.SYMBOL, "!"),
+ token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+ token(sourceText, 2, TokenType.SYMBOL, "!")
),
- token(3, TokenType.END, "")
+ token(sourceText, 3, TokenType.END, "")
);
tokens.skip(TokenType.SYMBOL);
@@ -57,13 +61,15 @@ public class TokenIteratorTests {
@Test
public void trySkipAdvancesIteratorAndReturnsTrueWhenNextTokenHasMatchingTokenType() {
+ var sourceText = SourceText.fromString("<filename>", "!x!");
+
var tokens = new TokenIterator<>(
List.of(
- token(0, TokenType.SYMBOL, "!"),
- token(1, TokenType.IDENTIFIER, "x"),
- token(2, TokenType.SYMBOL, "!")
+ token(sourceText, 0, TokenType.SYMBOL, "!"),
+ token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+ token(sourceText, 2, TokenType.SYMBOL, "!")
),
- token(3, TokenType.END, "")
+ token(sourceText, 3, TokenType.END, "")
);
assertThat(tokens.trySkip(TokenType.SYMBOL), equalTo(true));
@@ -75,13 +81,15 @@ public class TokenIteratorTests {
@Test
public void trySkipDoesNotAdvanceIteratorAndReturnsFalseWhenNextTokenDoesNotHaveMatchingTokenType() {
+ var sourceText = SourceText.fromString("<filename>", "!x!");
+
var tokens = new TokenIterator<>(
List.of(
- token(0, TokenType.SYMBOL, "!"),
- token(1, TokenType.IDENTIFIER, "x"),
- token(2, TokenType.SYMBOL, "!")
+ token(sourceText, 0, TokenType.SYMBOL, "!"),
+ token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+ token(sourceText, 2, TokenType.SYMBOL, "!")
),
- token(3, TokenType.END, "")
+ token(sourceText, 3, TokenType.END, "")
);
tokens.skip(TokenType.SYMBOL);
@@ -92,13 +100,15 @@ public class TokenIteratorTests {
@Test
public void nextValueReturnsValueWhenCurrentTokenHasMatchingTokenType() {
+ var sourceText = SourceText.fromString("<filename>", "!x!");
+
var tokens = new TokenIterator<>(
List.of(
- token(0, TokenType.SYMBOL, "!"),
- token(1, TokenType.IDENTIFIER, "x"),
- token(2, TokenType.SYMBOL, "!")
+ token(sourceText, 0, TokenType.SYMBOL, "!"),
+ token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+ token(sourceText, 2, TokenType.SYMBOL, "!")
),
- token(3, TokenType.END, "")
+ token(sourceText, 3, TokenType.END, "")
);
assertThat(tokens.nextValue(TokenType.SYMBOL), equalTo("!"));
@@ -107,13 +117,15 @@ public class TokenIteratorTests {
@Test
public void nextValueFailsWhenCurrentTokenDoesNotHaveMatchingTokenType() {
+ var sourceText = SourceText.fromString("<filename>", "!x!");
+
var tokens = new TokenIterator<>(
List.of(
- token(0, TokenType.SYMBOL, "!"),
- token(1, TokenType.IDENTIFIER, "x"),
- token(2, TokenType.SYMBOL, "!")
+ token(sourceText, 0, TokenType.SYMBOL, "!"),
+ token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+ token(sourceText, 2, TokenType.SYMBOL, "!")
),
- token(3, TokenType.END, "")
+ token(sourceText, 3, TokenType.END, "")
);
tokens.skip(TokenType.SYMBOL);
@@ -128,13 +140,15 @@ public class TokenIteratorTests {
@Test
public void isNextWithOneArgReturnsTrueIfNextTokenIsOfMatchingType() {
+ var sourceText = SourceText.fromString("<filename>", "!x!");
+
var tokens = new TokenIterator<>(
List.of(
- token(0, TokenType.SYMBOL, "!"),
- token(1, TokenType.IDENTIFIER, "x"),
- token(2, TokenType.SYMBOL, "!")
+ token(sourceText, 0, TokenType.SYMBOL, "!"),
+ token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+ token(sourceText, 2, TokenType.SYMBOL, "!")
),
- token(3, TokenType.END, "")
+ token(sourceText, 3, TokenType.END, "")
);
tokens.skip(TokenType.SYMBOL);
@@ -144,13 +158,15 @@ public class TokenIteratorTests {
@Test
public void isNextWithOneArgReturnsFalseIfNextTokenIsNotOfMatchingType() {
+ var sourceText = SourceText.fromString("<filename>", "!x!");
+
var tokens = new TokenIterator<>(
List.of(
- token(0, TokenType.SYMBOL, "!"),
- token(1, TokenType.IDENTIFIER, "x"),
- token(2, TokenType.SYMBOL, "!")
+ token(sourceText, 0, TokenType.SYMBOL, "!"),
+ token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+ token(sourceText, 2, TokenType.SYMBOL, "!")
),
- token(3, TokenType.END, "")
+ token(sourceText, 3, TokenType.END, "")
);
tokens.skip(TokenType.SYMBOL);
@@ -160,13 +176,15 @@ public class TokenIteratorTests {
@Test
public void isNextWithTwoArgReturnsTrueIfBothNextTokensMatchRespectiveTypes() {
+ var sourceText = SourceText.fromString("<filename>", "!x!");
+
var tokens = new TokenIterator<>(
List.of(
- token(0, TokenType.SYMBOL, "!"),
- token(1, TokenType.IDENTIFIER, "x"),
- token(2, TokenType.SYMBOL, "!")
+ token(sourceText, 0, TokenType.SYMBOL, "!"),
+ token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+ token(sourceText, 2, TokenType.SYMBOL, "!")
),
- token(3, TokenType.END, "")
+ token(sourceText, 3, TokenType.END, "")
);
tokens.skip(TokenType.SYMBOL);
@@ -176,13 +194,15 @@ public class TokenIteratorTests {
@Test
public void isNextWithTwoArgReturnsFalseIfFirstTokenDoesNotMatchRespectiveType() {
+ var sourceText = SourceText.fromString("<filename>", "!x!");
+
var tokens = new TokenIterator<>(
List.of(
- token(0, TokenType.SYMBOL, "!"),
- token(1, TokenType.IDENTIFIER, "x"),
- token(2, TokenType.SYMBOL, "!")
+ token(sourceText, 0, TokenType.SYMBOL, "!"),
+ token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+ token(sourceText, 2, TokenType.SYMBOL, "!")
),
- token(3, TokenType.END, "")
+ token(sourceText, 3, TokenType.END, "")
);
tokens.skip(TokenType.SYMBOL);
@@ -192,13 +212,15 @@ public class TokenIteratorTests {
@Test
public void isNextWithTwoArgReturnsFalseIfSecondTokenDoesNotMatchRespectiveType() {
+ var sourceText = SourceText.fromString("<filename>", "!x!");
+
var tokens = new TokenIterator<>(
List.of(
- token(0, TokenType.SYMBOL, "!"),
- token(1, TokenType.IDENTIFIER, "x"),
- token(2, TokenType.SYMBOL, "!")
+ token(sourceText, 0, TokenType.SYMBOL, "!"),
+ token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+ token(sourceText, 2, TokenType.SYMBOL, "!")
),
- token(3, TokenType.END, "")
+ token(sourceText, 3, TokenType.END, "")
);
tokens.skip(TokenType.SYMBOL);
@@ -208,42 +230,52 @@ public class TokenIteratorTests {
@Test
public void peekReturnsNextTokenWithoutAdvancing() {
+ var sourceText = SourceText.fromString("<filename>", "!x!");
+
var tokens = new TokenIterator<>(
List.of(
- token(0, TokenType.SYMBOL, "!"),
- token(1, TokenType.IDENTIFIER, "x"),
- token(2, TokenType.SYMBOL, "!")
+ token(sourceText, 0, TokenType.SYMBOL, "!"),
+ token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+ token(sourceText, 2, TokenType.SYMBOL, "!")
),
- token(3, TokenType.END, "")
+ token(sourceText, 3, TokenType.END, "")
);
- assertThat(tokens.peek(), equalTo(token(0, TokenType.SYMBOL, "!")));
- assertThat(tokens.peek(), equalTo(token(0, TokenType.SYMBOL, "!")));
+ assertThat(tokens.peek(), equalTo(token(sourceText, 0, TokenType.SYMBOL, "!")));
+ assertThat(tokens.peek(), equalTo(token(sourceText, 0, TokenType.SYMBOL, "!")));
tokens.skip(TokenType.SYMBOL);
- assertThat(tokens.peek(), equalTo(token(1, TokenType.IDENTIFIER, "x")));
+ assertThat(tokens.peek(), equalTo(token(sourceText, 1, TokenType.IDENTIFIER, "x")));
}
@Test
public void canPeekMultipleTokensAhead() {
+ var sourceText = SourceText.fromString("<filename>", "!x!");
+
var tokens = new TokenIterator<>(
List.of(
- token(0, TokenType.SYMBOL, "!"),
- token(1, TokenType.IDENTIFIER, "x"),
- token(2, TokenType.SYMBOL, "!")
+ token(sourceText, 0, TokenType.SYMBOL, "!"),
+ token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+ token(sourceText, 2, TokenType.SYMBOL, "!")
),
- token(3, TokenType.END, "")
+ token(sourceText, 3, TokenType.END, "")
);
- assertThat(tokens.peek(0), equalTo(token(0, TokenType.SYMBOL, "!")));
- assertThat(tokens.peek(1), equalTo(token(1, TokenType.IDENTIFIER, "x")));
- assertThat(tokens.peek(2), equalTo(token(2, TokenType.SYMBOL, "!")));
+ assertThat(tokens.peek(0), equalTo(token(sourceText, 0, TokenType.SYMBOL, "!")));
+ assertThat(tokens.peek(1), equalTo(token(sourceText, 1, TokenType.IDENTIFIER, "x")));
+ assertThat(tokens.peek(2), equalTo(token(sourceText, 2, TokenType.SYMBOL, "!")));
tokens.skip(TokenType.SYMBOL);
- assertThat(tokens.peek(0), equalTo(token(1, TokenType.IDENTIFIER, "x")));
- assertThat(tokens.peek(1), equalTo(token(2, TokenType.SYMBOL, "!")));
- assertThat(tokens.peek(2), equalTo(token(3, TokenType.END, "")));
+ assertThat(tokens.peek(0), equalTo(token(sourceText, 1, TokenType.IDENTIFIER, "x")));
+ assertThat(tokens.peek(1), equalTo(token(sourceText, 2, TokenType.SYMBOL, "!")));
+ assertThat(tokens.peek(2), equalTo(token(sourceText, 3, TokenType.END, "")));
}
- private Token<TokenType> token(int characterIndex, TokenType tokenType, String value) {
- return new Token<>(Sources.NULL, tokenType, value);
+ private Token<TokenType> token(
+ SourceText sourceText,
+ int characterIndex,
+ TokenType tokenType,
+ String value
+ ) {
+ var sourcePosition = sourceText.characterPosition(characterIndex);
+ return new Token<>(sourcePosition.to(sourcePosition), tokenType, value);
}
}