Use source text to implement FileFragmentSource

12 files changed, 177 insertions, 325 deletions

diff --git a/src/test/java/org/shedlang/compiler/config/BinConfigTests.java b/src/test/java/org/shedlang/compiler/config/BinConfigTests.java
index b11562d5..fac051a5 100644
--- a/src/test/java/org/shedlang/compiler/config/BinConfigTests.java
+++ b/src/test/java/org/shedlang/compiler/config/BinConfigTests.java
@@ -3,8 +3,8 @@ package org.shedlang.compiler.config;
 import org.junit.jupiter.api.Test;
 import org.shedlang.compiler.config.errors.ConfigError;
 import org.shedlang.compiler.config.json.Json5ObjectWithSource;
+import org.shedlang.compiler.sources.SourceTexts;
 import org.shedlang.compiler.types.NamespaceName;
-import org.zwobble.sourcetext.SourceText;
 
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.zwobble.precisely.AssertThat.assertThat;
@@ -56,7 +56,7 @@ public class BinConfigTests {
     }
 
     private BinConfig parseBinConfig(String text) {
-        var sourceText = SourceText.fromString("<string>", text);
+        var sourceText = SourceTexts.string(text);
         return BinConfig.fromJson5(Json5ObjectWithSource.parse(sourceText));
     }
 }
diff --git a/src/test/java/org/shedlang/compiler/config/GcConfigTests.java b/src/test/java/org/shedlang/compiler/config/GcConfigTests.java
index 1dfa27a4..064306c9 100644
--- a/src/test/java/org/shedlang/compiler/config/GcConfigTests.java
+++ b/src/test/java/org/shedlang/compiler/config/GcConfigTests.java
@@ -2,7 +2,7 @@ package org.shedlang.compiler.config;
 
 import org.junit.jupiter.api.Test;
 import org.shedlang.compiler.config.json.Json5ObjectWithSource;
-import org.zwobble.sourcetext.SourceText;
+import org.shedlang.compiler.sources.SourceTexts;
 
 import java.util.OptionalLong;
 
@@ -71,7 +71,7 @@ public class GcConfigTests {
     }
 
     private GcConfig parseGcConfig(String text) {
-        var sourceText = SourceText.fromString("<string>", text);
+        var sourceText = SourceTexts.string(text);
         return GcConfig.fromJson5(Json5ObjectWithSource.parse(sourceText));
     }
 }
diff --git a/src/test/java/org/shedlang/compiler/config/GcMarkAndSweepConfigTests.java b/src/test/java/org/shedlang/compiler/config/GcMarkAndSweepConfigTests.java
index d867fd71..ee5dac09 100644
--- a/src/test/java/org/shedlang/compiler/config/GcMarkAndSweepConfigTests.java
+++ b/src/test/java/org/shedlang/compiler/config/GcMarkAndSweepConfigTests.java
@@ -2,7 +2,7 @@ package org.shedlang.compiler.config;
 
 import org.junit.jupiter.api.Test;
 import org.shedlang.compiler.config.json.Json5ObjectWithSource;
-import org.zwobble.sourcetext.SourceText;
+import org.shedlang.compiler.sources.SourceTexts;
 
 import java.util.OptionalLong;
 
@@ -13,7 +13,7 @@ import static org.zwobble.precisely.Matchers.has;
 public class GcMarkAndSweepConfigTests {
     @Test
     public void collectIntervalCanBeMissing() {
-        var sourceText = SourceText.fromString("<string>", "{}");
+        var sourceText = SourceTexts.string("{}");
 
         var config = GcMarkAndSweepConfig.fromJson5(Json5ObjectWithSource.parse(sourceText));
 
@@ -26,7 +26,7 @@ public class GcMarkAndSweepConfigTests {
 
     @Test
     public void additionalFieldsCanBeUsedBySpecificGcConfig() {
-        var source = SourceText.fromString("<string>", """
+        var source = SourceTexts.string("""
             {
                 name: "mark-and-sweep",
                 collectInterval: 10000,
diff --git a/src/test/java/org/shedlang/compiler/config/ProfileConfigTests.java b/src/test/java/org/shedlang/compiler/config/ProfileConfigTests.java
index c923e654..b43d7b34 100644
--- a/src/test/java/org/shedlang/compiler/config/ProfileConfigTests.java
+++ b/src/test/java/org/shedlang/compiler/config/ProfileConfigTests.java
@@ -2,7 +2,7 @@ package org.shedlang.compiler.config;
 
 import org.junit.jupiter.api.Test;
 import org.shedlang.compiler.config.json.Json5ObjectWithSource;
-import org.zwobble.sourcetext.SourceText;
+import org.shedlang.compiler.sources.SourceTexts;
 
 import static org.zwobble.precisely.AssertThat.assertThat;
 import static org.zwobble.precisely.Matchers.*;
@@ -55,7 +55,7 @@ public class ProfileConfigTests {
     }
 
     private ProfileConfig parseProfileConfig(String text) {
-        var sourceText = SourceText.fromString("<string>", text);
+        var sourceText = SourceTexts.string(text);
         return ProfileConfig.fromJson5(Json5ObjectWithSource.parse(sourceText));
     }
 }
diff --git a/src/test/java/org/shedlang/compiler/config/ProjectConfigTests.java b/src/test/java/org/shedlang/compiler/config/ProjectConfigTests.java
index 449dd9ff..e7059c34 100644
--- a/src/test/java/org/shedlang/compiler/config/ProjectConfigTests.java
+++ b/src/test/java/org/shedlang/compiler/config/ProjectConfigTests.java
@@ -3,8 +3,8 @@ package org.shedlang.compiler.config;
 import org.junit.jupiter.api.Test;
 import org.shedlang.compiler.config.errors.ConfigError;
 import org.shedlang.compiler.config.errors.Json5SyntaxConfigError;
+import org.shedlang.compiler.sources.SourceTexts;
 import org.shedlang.compiler.types.NamespaceName;
-import org.zwobble.sourcetext.SourceText;
 
 import java.nio.file.Path;
 
@@ -252,7 +252,7 @@ public class ProjectConfigTests {
     }
 
     private ProjectConfig parseProjectConfig(String text) {
-        var sourceText = SourceText.fromString("<string>", text);
+        var sourceText = SourceTexts.string(text);
         return ProjectConfig.fromSourceText(PROJECT_PATH, sourceText);
     }
 }
diff --git a/src/test/java/org/shedlang/compiler/config/json/Json5ObjectWithSourceTests.java b/src/test/java/org/shedlang/compiler/config/json/Json5ObjectWithSourceTests.java
index 15946c26..35e97a48 100644
--- a/src/test/java/org/shedlang/compiler/config/json/Json5ObjectWithSourceTests.java
+++ b/src/test/java/org/shedlang/compiler/config/json/Json5ObjectWithSourceTests.java
@@ -2,7 +2,7 @@ package org.shedlang.compiler.config.json;
 
 import org.junit.jupiter.api.Test;
 import org.shedlang.compiler.config.errors.ConfigError;
-import org.zwobble.sourcetext.SourceText;
+import org.shedlang.compiler.sources.SourceTexts;
 
 import java.util.Optional;
 import java.util.OptionalLong;
@@ -405,7 +405,7 @@ public class Json5ObjectWithSourceTests {
     }
 
     private Json5ObjectWithSource parseJson5Object(String text) {
-        var sourceText = SourceText.fromString("<string>", text);
+        var sourceText = SourceTexts.string(text);
         return Json5ObjectWithSource.parse(sourceText);
     }
 }
diff --git a/src/test/java/org/shedlang/compiler/parser/ParserStringLiteralTests.java b/src/test/java/org/shedlang/compiler/parser/ParserStringLiteralTests.java
index 9f84c935..21c6a2ca 100644
--- a/src/test/java/org/shedlang/compiler/parser/ParserStringLiteralTests.java
+++ b/src/test/java/org/shedlang/compiler/parser/ParserStringLiteralTests.java
@@ -49,8 +49,8 @@ public class ParserStringLiteralTests {
 
         assertThat(error.getMessage(), equalTo("Unrecognised escape sequence: \\a"));
         assertThat(error.source(), instanceOf(FileFragmentSource.class, allOf(
-            has("characterIndexStart", x -> x.characterIndexStart(), equalTo(1)),
-            has("characterIndexEnd", x -> x.characterIndexEnd(), equalTo(3))
+            has("characterIndexStart", x -> x.sourceRange().start().characterIndex(), equalTo(2)),
+            has("characterIndexEnd", x -> x.sourceRange().end().characterIndex(), equalTo(4))
         )));
     }
 }
diff --git a/src/test/java/org/shedlang/compiler/parser/ParserTesting.java b/src/test/java/org/shedlang/compiler/parser/ParserTesting.java
index fd420f73..1e4f202f 100644
--- a/src/test/java/org/shedlang/compiler/parser/ParserTesting.java
+++ b/src/test/java/org/shedlang/compiler/parser/ParserTesting.java
@@ -1,6 +1,6 @@
 package org.shedlang.compiler.parser;
 
-import org.shedlang.compiler.sources.Sources;
+import org.shedlang.compiler.sources.SourceTexts;
 import org.shedlang.compiler.tokeniser.TokenIterator;
 
 import java.util.function.Function;
@@ -14,8 +14,8 @@ public class ParserTesting {
         String sourceContents,
         Function<TokenIterator<TokenType>, T> parseTokens
     ) {
-        var source = Sources.file("<string>", sourceContents);
-        var tokens = tokenise(source);
+        var sourceText = SourceTexts.string(sourceContents);
+        var tokens = tokenise(sourceText);
         var result = parseTokens.apply(tokens);
         assertThat(tokens.peek().tokenType(), equalTo(TokenType.END));
         return result;
diff --git a/src/test/java/org/shedlang/compiler/parser/TokeniserTests.java b/src/test/java/org/shedlang/compiler/parser/TokeniserTests.java
index 69cfdae7..0dc9a560 100644
--- a/src/test/java/org/shedlang/compiler/parser/TokeniserTests.java
+++ b/src/test/java/org/shedlang/compiler/parser/TokeniserTests.java
@@ -1,7 +1,7 @@
 package org.shedlang.compiler.parser;
 
 import org.junit.jupiter.api.Test;
-import org.shedlang.compiler.sources.Sources;
+import org.shedlang.compiler.sources.SourceTexts;
 
 import static org.zwobble.precisely.AssertThat.assertThat;
 import static org.zwobble.precisely.Matchers.equalTo;
@@ -9,11 +9,11 @@ import static org.zwobble.precisely.Matchers.equalTo;
 public class TokeniserTests {
     @Test
     public void whitespaceTokensAreIgnored() {
-        var source = Sources.file("X.shed", """
+        var sourceText = SourceTexts.string("""
             one two three
             """);
 
-        var result = Tokeniser.tokenise(source);
+        var result = Tokeniser.tokenise(sourceText);
 
         assertThat(result.nextValue(TokenType.IDENTIFIER), equalTo("one"));
         assertThat(result.nextValue(TokenType.IDENTIFIER), equalTo("two"));
@@ -23,14 +23,14 @@ public class TokeniserTests {
 
     @Test
     public void singleLineCommentsAreIgnored() {
-        var source = Sources.file("X.shed", """
+        var sourceText = SourceTexts.string("""
             // a
             one
             // b
             two
             """);
 
-        var result = Tokeniser.tokenise(source);
+        var result = Tokeniser.tokenise(sourceText);
 
         assertThat(result.nextValue(TokenType.IDENTIFIER), equalTo("one"));
         assertThat(result.nextValue(TokenType.IDENTIFIER), equalTo("two"));
diff --git a/src/test/java/org/shedlang/compiler/sources/FileFragmentSourceTests.java b/src/test/java/org/shedlang/compiler/sources/FileFragmentSourceTests.java
deleted file mode 100644
index 0c1df6ac..00000000
--- a/src/test/java/org/shedlang/compiler/sources/FileFragmentSourceTests.java
+++ /dev/null
@@ -1,215 +0,0 @@
-package org.shedlang.compiler.sources;
-
-import org.junit.jupiter.api.DynamicTest;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.TestFactory;
-
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.zwobble.precisely.AssertThat.assertThat;
-import static org.zwobble.precisely.Matchers.equalTo;
-
-public class FileFragmentSourceTests {
-    private record TestCase(
-        String name,
-        String contents,
-        int characterIndexStart,
-        String expectedContext
-    ) {
-        public int characterIndexEnd() {
-            return characterIndexStart;
-        }
-    }
-
-    @TestFactory
-    public List<DynamicTest> sourceContextIsSourceLineWithPointer() {
-        return Stream.of(
-            new TestCase(
-                "one line, first character",
-                "abcd",
-                0,
-                """
-                    <filename>:1:1
-                    abcd
-                    ^"""
-            ),
-            new TestCase(
-                "one line, last character",
-                "abcd",
-                3,
-                """
-                    <filename>:1:4
-                    abcd
-                       ^"""
-            ),
-            new TestCase(
-                "one line, end",
-                "abcd",
-                4,
-                """
-                    <filename>:1:5
-                    abcd
-                        ^"""
-            ),
-            new TestCase(
-                "many lines, end",
-                """
-                    abc
-                    def""",
-                7,
-                """
-                    <filename>:2:4
-                    def
-                       ^"""
-            ),
-            new TestCase(
-                "first line, first character",
-                """
-                    abc
-                    def""",
-                0,
-                """
-                    <filename>:1:1
-                    abc
-                    ^"""
-            ),
-            new TestCase(
-                "first line, last character",
-                """
-                    abc
-                    def""",
-                2,
-                """
-                    <filename>:1:3
-                    abc
-                      ^"""
-            ),
-            new TestCase(
-                "last line, first character",
-                """
-                    abc
-                    def""",
-                4,
-                """
-                    <filename>:2:1
-                    def
-                    ^"""
-            )
-        ).map(testCase -> DynamicTest.dynamicTest(testCase.name, () -> {
-            var source = new FileFragmentSource(
-                "<filename>",
-                testCase.contents,
-                testCase.characterIndexStart,
-                testCase.characterIndexEnd()
-            );
-
-            assertThat(source.describe(), equalTo(testCase.expectedContext));
-        })).collect(Collectors.toList());
-    }
-
-    @Test
-    public void whenSourceIsZeroLengthThenPointerPointsToNextCharacter() {
-        var source = new FileFragmentSource(
-            "<filename>",
-            """
-                abcdef
-                ghijkl
-                mnopqr""",
-            9,
-            9
-        );
-
-        assertThat(source.describe(), equalTo("""
-            <filename>:2:3
-            ghijkl
-              ^"""));
-    }
-
-    @Test
-    public void whenSourceIsOneCharacterThenPointerPointsToCharacter() {
-        var source = new FileFragmentSource(
-            "<filename>",
-            """
-                abcdef
-                ghijkl
-                mnopqr""",
-            9,
-            9
-        );
-
-        assertThat(source.describe(), equalTo("""
-            <filename>:2:3
-            ghijkl
-              ^"""));
-    }
-
-    @Test
-    public void whenSourceIsMultipleCharactersOnSameLineThenPointerPointsToAllCharacters() {
-        var source = new FileFragmentSource(
-            "<filename>",
-            """
-                abcdef
-                ghijkl
-                mnopqr""",
-            9,
-            12
-        );
-
-        assertThat(source.describe(), equalTo("""
-            <filename>:2:3
-            ghijkl
-              ^^^"""));
-    }
-
-    @Test
-    public void whenSourceIsMultipleCharactersOnMultipleLinesThenPointerPointsToFirstCharacter() {
-        // TODO: show multiple lines
-        var source = new FileFragmentSource(
-            "<filename>",
-            """
-                abcdef
-                ghijkl
-                mnopqr""",
-            9,
-            14
-        );
-
-        assertThat(source.describe(), equalTo("""
-            <filename>:2:3
-            ghijkl
-              ^"""));
-    }
-
-    @Test
-    public void atReturnsSubsectionOfCompleteSource() {
-        var source = FileFragmentSource.file("<filename>", "abcdefg");
-
-        var fragment = source.at(2, 5);
-
-        assertThat(fragment, equalTo(new FileFragmentSource("<filename>", "abcdefg", 2, 5)));
-    }
-
-    @Test
-    public void atReturnsSubsectionOfIncompleteSource() {
-        var source = FileFragmentSource.file("<filename>", "abcdefg");
-
-        var fragment = source.at(1, 6).at(1, 4);
-
-        assertThat(fragment, equalTo(new FileFragmentSource("<filename>", "abcdefg", 2, 5)));
-    }
-
-    @Test
-    public void whenAtIsCalledWithEndBeforeStartThenErrorIsThrown() {
-        var source = FileFragmentSource.file("<filename>", "abcdefg");
-
-        var error = assertThrows(
-            IllegalArgumentException.class,
-            () -> source.at(2, 1)
-        );
-
-        assertThat(error.getMessage(), equalTo("characterIndexStart > characterIndexEnd"));
-    }
-}
diff --git a/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java b/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java
index 06989d4d..be47dda5 100644
--- a/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java
+++ b/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java
@@ -1,8 +1,7 @@
 package org.shedlang.compiler.tokeniser;
 
 import org.junit.jupiter.api.Test;
-import org.shedlang.compiler.sources.Source;
-import org.shedlang.compiler.sources.Sources;
+import org.zwobble.sourcetext.SourceText;
 
 import java.util.List;
 
@@ -21,9 +20,9 @@ public class RegexTokeniserTests {
     @Test
     public void emptyStringIsTokenisedToNoTokens() {
         var tokeniser = new RegexTokeniser<>(TokenType.UNKNOWN, List.of());
-        var source = source("");
+        var sourceText = sourceText("");
 
-        var result = tokeniser.tokenise(source);
+        var result = tokeniser.tokenise(sourceText);
 
         assertThat(result, isSequence());
     }
@@ -33,12 +32,16 @@ public class RegexTokeniserTests {
         var tokeniser = new RegexTokeniser<>(TokenType.UNKNOWN, List.of(
             RegexTokeniser.rule(TokenType.IDENTIFIER, "[A-Za-z0-9]+")
         ));
-        var source = source("count");
+        var sourceText = sourceText("count");
 
-        var result = tokeniser.tokenise(source);
+        var result = tokeniser.tokenise(sourceText);
 
         assertThat(result, isSequence(
-            equalTo(new Token<>(source.at(0, 5), TokenType.IDENTIFIER, "count"))
+            equalTo(new Token<>(
+                sourceText.characterPosition(0).to(sourceText.characterPosition(5)),
+                TokenType.IDENTIFIER,
+                "count"
+            ))
         ));
     }
 
@@ -48,12 +51,16 @@ public class RegexTokeniserTests {
             RegexTokeniser.rule(TokenType.KEYWORD, "if"),
             RegexTokeniser.rule(TokenType.IDENTIFIER, "[A-Za-z0-9]+")
         ));
-        var source = source("if");
+        var sourceText = sourceText("if");
 
-        var result = tokeniser.tokenise(source);
+        var result = tokeniser.tokenise(sourceText);
 
         assertThat(result, isSequence(
-            equalTo(new Token<>(source.at(0, 2), TokenType.KEYWORD, "if"))
+            equalTo(new Token<>(
+                sourceText.characterPosition(0).to(sourceText.characterPosition(2)),
+                TokenType.KEYWORD,
+                "if"
+            ))
         ));
     }
 
@@ -63,14 +70,26 @@ public class RegexTokeniserTests {
             RegexTokeniser.rule(TokenType.IDENTIFIER, "[A-Za-z0-9]+"),
             RegexTokeniser.rule(TokenType.WHITESPACE, "\\s")
         ));
-        var source = source("one two");
+        var sourceText = sourceText("one two");
 
-        var result = tokeniser.tokenise(source);
+        var result = tokeniser.tokenise(sourceText);
 
         assertThat(result, isSequence(
-            equalTo(new Token<>(source.at(0, 3), TokenType.IDENTIFIER, "one")),
-            equalTo(new Token<>(source.at(3, 4), TokenType.WHITESPACE, " ")),
-            equalTo(new Token<>(source.at(4, 7), TokenType.IDENTIFIER, "two"))
+            equalTo(new Token<>(
+                sourceText.characterPosition(0).to(sourceText.characterPosition(3)),
+                TokenType.IDENTIFIER,
+                "one"
+            )),
+            equalTo(new Token<>(
+                sourceText.characterPosition(3).to(sourceText.characterPosition(4)),
+                TokenType.WHITESPACE,
+                " "
+            )),
+            equalTo(new Token<>(
+                sourceText.characterPosition(4).to(sourceText.characterPosition(7)),
+                TokenType.IDENTIFIER,
+                "two"
+            ))
         ));
     }
 
@@ -79,19 +98,35 @@ public class RegexTokeniserTests {
         var tokeniser = new RegexTokeniser<>(TokenType.UNKNOWN, List.of(
             RegexTokeniser.rule(TokenType.IDENTIFIER, "[A-Za-z0-9]+")
         ));
-        var source = source("!!a!");
+        var sourceText = sourceText("!!a!");
 
-        var result = tokeniser.tokenise(source);
+        var result = tokeniser.tokenise(sourceText);
 
         assertThat(result, isSequence(
-            equalTo(new Token<>(source.at(0, 1), TokenType.UNKNOWN, "!")),
-            equalTo(new Token<>(source.at(1, 2), TokenType.UNKNOWN, "!")),
-            equalTo(new Token<>(source.at(2, 3), TokenType.IDENTIFIER, "a")),
-            equalTo(new Token<>(source.at(3, 4), TokenType.UNKNOWN, "!"))
+            equalTo(new Token<>(
+                sourceText.characterPosition(0).to(sourceText.characterPosition(1)),
+                TokenType.UNKNOWN,
+                "!"
+            )),
+            equalTo(new Token<>(
+                sourceText.characterPosition(1).to(sourceText.characterPosition(2)),
+                TokenType.UNKNOWN,
+                "!"
+            )),
+            equalTo(new Token<>(
+                sourceText.characterPosition(2).to(sourceText.characterPosition(3)),
+                TokenType.IDENTIFIER,
+                "a"
+            )),
+            equalTo(new Token<>(
+                sourceText.characterPosition(3).to(sourceText.characterPosition(4)),
+                TokenType.UNKNOWN,
+                "!"
+            ))
         ));
     }
 
-    private Source source(String sourceContents) {
-        return Sources.file("<filename>", sourceContents);
+    private SourceText sourceText(String sourceContents) {
+        return SourceText.fromString("<filename>", sourceContents);
     }
 }
diff --git a/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java b/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java
index 52d036a8..dd22550f 100644
--- a/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java
+++ b/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java
@@ -1,7 +1,7 @@
 package org.shedlang.compiler.tokeniser;
 
 import org.junit.jupiter.api.Test;
-import org.shedlang.compiler.sources.Sources;
+import org.zwobble.sourcetext.SourceText;
 
 import java.util.List;
 
@@ -18,13 +18,15 @@ public class TokenIteratorTests {
 
     @Test
     public void skipSucceedsWhenNextTokenHasMatchingTokenType() {
+        var sourceText = SourceText.fromString("<filename>", "!x!");
+
         var tokens = new TokenIterator<>(
             List.of(
-                token(0, TokenType.SYMBOL, "!"),
-                token(1, TokenType.IDENTIFIER, "x"),
-                token(2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, TokenType.SYMBOL, "!"),
+                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+                token(sourceText, 2, TokenType.SYMBOL, "!")
             ),
-            token(3, TokenType.END, "")
+            token(sourceText, 3, TokenType.END, "")
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -36,13 +38,15 @@ public class TokenIteratorTests {
 
     @Test
     public void skipFailsWhenNextTokenDoesNotHaveMatchingTokenType() {
+        var sourceText = SourceText.fromString("<filename>", "!x!");
+
         var tokens = new TokenIterator<>(
             List.of(
-                token(0, TokenType.SYMBOL, "!"),
-                token(1, TokenType.IDENTIFIER, "x"),
-                token(2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, TokenType.SYMBOL, "!"),
+                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+                token(sourceText, 2, TokenType.SYMBOL, "!")
             ),
-            token(3, TokenType.END, "")
+            token(sourceText, 3, TokenType.END, "")
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -57,13 +61,15 @@ public class TokenIteratorTests {
 
     @Test
     public void trySkipAdvancesIteratorAndReturnsTrueWhenNextTokenHasMatchingTokenType() {
+        var sourceText = SourceText.fromString("<filename>", "!x!");
+
         var tokens = new TokenIterator<>(
             List.of(
-                token(0, TokenType.SYMBOL, "!"),
-                token(1, TokenType.IDENTIFIER, "x"),
-                token(2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, TokenType.SYMBOL, "!"),
+                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+                token(sourceText, 2, TokenType.SYMBOL, "!")
             ),
-            token(3, TokenType.END, "")
+            token(sourceText, 3, TokenType.END, "")
         );
 
         assertThat(tokens.trySkip(TokenType.SYMBOL), equalTo(true));
@@ -75,13 +81,15 @@ public class TokenIteratorTests {
 
     @Test
     public void trySkipDoesNotAdvanceIteratorAndReturnsFalseWhenNextTokenDoesNotHaveMatchingTokenType() {
+        var sourceText = SourceText.fromString("<filename>", "!x!");
+
         var tokens = new TokenIterator<>(
             List.of(
-                token(0, TokenType.SYMBOL, "!"),
-                token(1, TokenType.IDENTIFIER, "x"),
-                token(2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, TokenType.SYMBOL, "!"),
+                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+                token(sourceText, 2, TokenType.SYMBOL, "!")
             ),
-            token(3, TokenType.END, "")
+            token(sourceText, 3, TokenType.END, "")
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -92,13 +100,15 @@ public class TokenIteratorTests {
 
     @Test
     public void nextValueReturnsValueWhenCurrentTokenHasMatchingTokenType() {
+        var sourceText = SourceText.fromString("<filename>", "!x!");
+
         var tokens = new TokenIterator<>(
             List.of(
-                token(0, TokenType.SYMBOL, "!"),
-                token(1, TokenType.IDENTIFIER, "x"),
-                token(2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, TokenType.SYMBOL, "!"),
+                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+                token(sourceText, 2, TokenType.SYMBOL, "!")
             ),
-            token(3, TokenType.END, "")
+            token(sourceText, 3, TokenType.END, "")
         );
 
         assertThat(tokens.nextValue(TokenType.SYMBOL), equalTo("!"));
@@ -107,13 +117,15 @@ public class TokenIteratorTests {
 
     @Test
     public void nextValueFailsWhenCurrentTokenDoesNotHaveMatchingTokenType() {
+        var sourceText = SourceText.fromString("<filename>", "!x!");
+
         var tokens = new TokenIterator<>(
             List.of(
-                token(0, TokenType.SYMBOL, "!"),
-                token(1, TokenType.IDENTIFIER, "x"),
-                token(2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, TokenType.SYMBOL, "!"),
+                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+                token(sourceText, 2, TokenType.SYMBOL, "!")
             ),
-            token(3, TokenType.END, "")
+            token(sourceText, 3, TokenType.END, "")
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -128,13 +140,15 @@ public class TokenIteratorTests {
 
     @Test
     public void isNextWithOneArgReturnsTrueIfNextTokenIsOfMatchingType() {
+        var sourceText = SourceText.fromString("<filename>", "!x!");
+
         var tokens = new TokenIterator<>(
             List.of(
-                token(0, TokenType.SYMBOL, "!"),
-                token(1, TokenType.IDENTIFIER, "x"),
-                token(2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, TokenType.SYMBOL, "!"),
+                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+                token(sourceText, 2, TokenType.SYMBOL, "!")
             ),
-            token(3, TokenType.END, "")
+            token(sourceText, 3, TokenType.END, "")
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -144,13 +158,15 @@ public class TokenIteratorTests {
 
     @Test
     public void isNextWithOneArgReturnsFalseIfNextTokenIsNotOfMatchingType() {
+        var sourceText = SourceText.fromString("<filename>", "!x!");
+
         var tokens = new TokenIterator<>(
             List.of(
-                token(0, TokenType.SYMBOL, "!"),
-                token(1, TokenType.IDENTIFIER, "x"),
-                token(2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, TokenType.SYMBOL, "!"),
+                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+                token(sourceText, 2, TokenType.SYMBOL, "!")
             ),
-            token(3, TokenType.END, "")
+            token(sourceText, 3, TokenType.END, "")
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -160,13 +176,15 @@ public class TokenIteratorTests {
 
     @Test
     public void isNextWithTwoArgReturnsTrueIfBothNextTokensMatchRespectiveTypes() {
+        var sourceText = SourceText.fromString("<filename>", "!x!");
+
         var tokens = new TokenIterator<>(
             List.of(
-                token(0, TokenType.SYMBOL, "!"),
-                token(1, TokenType.IDENTIFIER, "x"),
-                token(2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, TokenType.SYMBOL, "!"),
+                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+                token(sourceText, 2, TokenType.SYMBOL, "!")
             ),
-            token(3, TokenType.END, "")
+            token(sourceText, 3, TokenType.END, "")
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -176,13 +194,15 @@ public class TokenIteratorTests {
 
     @Test
     public void isNextWithTwoArgReturnsFalseIfFirstTokenDoesNotMatchRespectiveType() {
+        var sourceText = SourceText.fromString("<filename>", "!x!");
+
         var tokens = new TokenIterator<>(
             List.of(
-                token(0, TokenType.SYMBOL, "!"),
-                token(1, TokenType.IDENTIFIER, "x"),
-                token(2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, TokenType.SYMBOL, "!"),
+                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+                token(sourceText, 2, TokenType.SYMBOL, "!")
             ),
-            token(3, TokenType.END, "")
+            token(sourceText, 3, TokenType.END, "")
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -192,13 +212,15 @@ public class TokenIteratorTests {
 
     @Test
     public void isNextWithTwoArgReturnsFalseIfSecondTokenDoesNotMatchRespectiveType() {
+        var sourceText = SourceText.fromString("<filename>", "!x!");
+
         var tokens = new TokenIterator<>(
             List.of(
-                token(0, TokenType.SYMBOL, "!"),
-                token(1, TokenType.IDENTIFIER, "x"),
-                token(2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, TokenType.SYMBOL, "!"),
+                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+                token(sourceText, 2, TokenType.SYMBOL, "!")
             ),
-            token(3, TokenType.END, "")
+            token(sourceText, 3, TokenType.END, "")
         );
 
         tokens.skip(TokenType.SYMBOL);
@@ -208,42 +230,52 @@ public class TokenIteratorTests {
 
     @Test
     public void peekReturnsNextTokenWithoutAdvancing() {
+        var sourceText = SourceText.fromString("<filename>", "!x!");
+
         var tokens = new TokenIterator<>(
             List.of(
-                token(0, TokenType.SYMBOL, "!"),
-                token(1, TokenType.IDENTIFIER, "x"),
-                token(2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, TokenType.SYMBOL, "!"),
+                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+                token(sourceText, 2, TokenType.SYMBOL, "!")
             ),
-            token(3, TokenType.END, "")
+            token(sourceText, 3, TokenType.END, "")
         );
 
-        assertThat(tokens.peek(), equalTo(token(0, TokenType.SYMBOL, "!")));
-        assertThat(tokens.peek(), equalTo(token(0, TokenType.SYMBOL, "!")));
+        assertThat(tokens.peek(), equalTo(token(sourceText, 0, TokenType.SYMBOL, "!")));
+        assertThat(tokens.peek(), equalTo(token(sourceText, 0, TokenType.SYMBOL, "!")));
         tokens.skip(TokenType.SYMBOL);
-        assertThat(tokens.peek(), equalTo(token(1, TokenType.IDENTIFIER, "x")));
+        assertThat(tokens.peek(), equalTo(token(sourceText, 1, TokenType.IDENTIFIER, "x")));
     }
 
     @Test
     public void canPeekMultipleTokensAhead() {
+        var sourceText = SourceText.fromString("<filename>", "!x!");
+
         var tokens = new TokenIterator<>(
             List.of(
-                token(0, TokenType.SYMBOL, "!"),
-                token(1, TokenType.IDENTIFIER, "x"),
-                token(2, TokenType.SYMBOL, "!")
+                token(sourceText, 0, TokenType.SYMBOL, "!"),
+                token(sourceText, 1, TokenType.IDENTIFIER, "x"),
+                token(sourceText, 2, TokenType.SYMBOL, "!")
             ),
-            token(3, TokenType.END, "")
+            token(sourceText, 3, TokenType.END, "")
         );
 
-        assertThat(tokens.peek(0), equalTo(token(0, TokenType.SYMBOL, "!")));
-        assertThat(tokens.peek(1), equalTo(token(1, TokenType.IDENTIFIER, "x")));
-        assertThat(tokens.peek(2), equalTo(token(2, TokenType.SYMBOL, "!")));
+        assertThat(tokens.peek(0), equalTo(token(sourceText, 0, TokenType.SYMBOL, "!")));
+        assertThat(tokens.peek(1), equalTo(token(sourceText, 1, TokenType.IDENTIFIER, "x")));
+        assertThat(tokens.peek(2), equalTo(token(sourceText, 2, TokenType.SYMBOL, "!")));
         tokens.skip(TokenType.SYMBOL);
-        assertThat(tokens.peek(0), equalTo(token(1, TokenType.IDENTIFIER, "x")));
-        assertThat(tokens.peek(1), equalTo(token(2, TokenType.SYMBOL, "!")));
-        assertThat(tokens.peek(2), equalTo(token(3, TokenType.END, "")));
+        assertThat(tokens.peek(0), equalTo(token(sourceText, 1, TokenType.IDENTIFIER, "x")));
+        assertThat(tokens.peek(1), equalTo(token(sourceText, 2, TokenType.SYMBOL, "!")));
+        assertThat(tokens.peek(2), equalTo(token(sourceText, 3, TokenType.END, "")));
     }
 
-    private Token<TokenType> token(int characterIndex, TokenType tokenType, String value) {
-        return new Token<>(Sources.NULL, tokenType, value);
+    private Token<TokenType> token(
+        SourceText sourceText,
+        int characterIndex,
+        TokenType tokenType,
+        String value
+    ) {
+        var sourcePosition = sourceText.characterPosition(characterIndex);
+        return new Token<>(sourcePosition.to(sourcePosition), tokenType, value);
     }
 }