Add tokeniser

5 files changed, 479 insertions, 0 deletions

diff --git a/src/test/java/org/shedlang/compiler/sources/FileFragmentSourceTests.java b/src/test/java/org/shedlang/compiler/sources/FileFragmentSourceTests.java
new file mode 100644
index 00000000..534f10c5
--- /dev/null
+++ b/src/test/java/org/shedlang/compiler/sources/FileFragmentSourceTests.java
@@ -0,0 +1,110 @@
+package org.shedlang.compiler.sources;
+
+import org.junit.jupiter.api.DynamicTest;
+import org.junit.jupiter.api.TestFactory;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.zwobble.precisely.AssertThat.assertThat;
+import static org.zwobble.precisely.Matchers.equalTo;
+
+public class FileFragmentSourceTests {
+    private record TestCase(
+        String name,
+        String contents,
+        int characterIndexStart,
+        String expectedContext
+    ) {
+        public int characterIndexEnd() {
+            return characterIndexStart;
+        }
+    }
+
+    @TestFactory
+    public List<DynamicTest> sourceContextIsSourceLineWithPointer() {
+        return Stream.of(
+            new TestCase(
+                "one line, first character",
+                "abcd",
+                0,
+                """
+                    <filename>:1:1
+                    abcd
+                    ^"""
+            ),
+            new TestCase(
+                "one line, last character",
+                "abcd",
+                3,
+                """
+                    <filename>:1:4
+                    abcd
+                       ^"""
+            ),
+            new TestCase(
+                "one line, end",
+                "abcd",
+                4,
+                """
+                    <filename>:1:5
+                    abcd
+                        ^"""
+            ),
+            new TestCase(
+                "many lines, end",
+                """
+                    abc
+                    def""",
+                7,
+                """
+                    <filename>:2:4
+                    def
+                       ^"""
+            ),
+            new TestCase(
+                "first line, first character",
+                """
+                    abc
+                    def""",
+                0,
+                """
+                    <filename>:1:1
+                    abc
+                    ^"""
+            ),
+            new TestCase(
+                "first line, last character",
+                """
+                    abc
+                    def""",
+                2,
+                """
+                    <filename>:1:3
+                    abc
+                      ^"""
+            ),
+            new TestCase(
+                "last line, first character",
+                """
+                    abc
+                    def""",
+                4,
+                """
+                    <filename>:2:1
+                    def
+                    ^"""
+            )
+        ).map(testCase -> DynamicTest.dynamicTest(testCase.name, () -> {
+            var source = new FileFragmentSource(
+                "<filename>",
+                testCase.contents,
+                testCase.characterIndexStart,
+                testCase.characterIndexEnd()
+            );
+
+            assertThat(source.describe(), equalTo(testCase.expectedContext));
+        })).collect(Collectors.toList());
+    }
+}
diff --git a/src/test/java/org/shedlang/compiler/sources/SourceMatchers.java b/src/test/java/org/shedlang/compiler/sources/SourceMatchers.java
new file mode 100644
index 00000000..64e70d18
--- /dev/null
+++ b/src/test/java/org/shedlang/compiler/sources/SourceMatchers.java
@@ -0,0 +1,15 @@
+package org.shedlang.compiler.sources;
+
+import org.zwobble.precisely.Matcher;
+
+import static org.zwobble.precisely.Matchers.instanceOf;
+
+public class SourceMatchers {
+    private SourceMatchers() {
+
+    }
+
+    public static Matcher<Source> isFileFragmentSource(Matcher<FileFragmentSource> matcher) {
+        return instanceOf(FileFragmentSource.class, matcher);
+    }
+}
diff --git a/src/test/java/org/shedlang/compiler/tokeniser/InfinitelyPaddedIteratorTests.java b/src/test/java/org/shedlang/compiler/tokeniser/InfinitelyPaddedIteratorTests.java
new file mode 100644
index 00000000..44628533
--- /dev/null
+++ b/src/test/java/org/shedlang/compiler/tokeniser/InfinitelyPaddedIteratorTests.java
@@ -0,0 +1,28 @@
+package org.shedlang.compiler.tokeniser;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.zwobble.precisely.AssertThat.assertThat;
+import static org.zwobble.precisely.Matchers.equalTo;
+
+public class InfinitelyPaddedIteratorTests {
+    @Test
+    public void iteratesThroughListThenAlwaysReturnsEnd() {
+        var tokens = new InfinitelyPaddedIterator<>(
+            List.of("one", "two", "three"),
+            "end"
+        );
+
+        assertThat(tokens.get(), equalTo("one"));
+        tokens.moveNext();
+        assertThat(tokens.get(), equalTo("two"));
+        tokens.moveNext();
+        assertThat(tokens.get(), equalTo("three"));
+        tokens.moveNext();
+        assertThat(tokens.get(), equalTo("end"));
+        tokens.moveNext();
+        assertThat(tokens.get(), equalTo("end"));
+    }
+}
diff --git a/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java b/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java
new file mode 100644
index 00000000..06989d4d
--- /dev/null
+++ b/src/test/java/org/shedlang/compiler/tokeniser/RegexTokeniserTests.java
@@ -0,0 +1,97 @@
+package org.shedlang.compiler.tokeniser;
+
+import org.junit.jupiter.api.Test;
+import org.shedlang.compiler.sources.Source;
+import org.shedlang.compiler.sources.Sources;
+
+import java.util.List;
+
+import static org.zwobble.precisely.AssertThat.assertThat;
+import static org.zwobble.precisely.Matchers.equalTo;
+import static org.zwobble.precisely.Matchers.isSequence;
+
+public class RegexTokeniserTests {
+    enum TokenType {
+        UNKNOWN,
+        KEYWORD,
+        IDENTIFIER,
+        WHITESPACE
+    }
+
+    @Test
+    public void emptyStringIsTokenisedToNoTokens() {
+        var tokeniser = new RegexTokeniser<>(TokenType.UNKNOWN, List.of());
+        var source = source("");
+
+        var result = tokeniser.tokenise(source);
+
+        assertThat(result, isSequence());
+    }
+
+    @Test
+    public void regexIsAppliedToGenerateToken() {
+        var tokeniser = new RegexTokeniser<>(TokenType.UNKNOWN, List.of(
+            RegexTokeniser.rule(TokenType.IDENTIFIER, "[A-Za-z0-9]+")
+        ));
+        var source = source("count");
+
+        var result = tokeniser.tokenise(source);
+
+        assertThat(result, isSequence(
+            equalTo(new Token<>(source.at(0, 5), TokenType.IDENTIFIER, "count"))
+        ));
+    }
+
+    @Test
+    public void firstMatchingRegexIsUsed() {
+        var tokeniser = new RegexTokeniser<>(TokenType.UNKNOWN, List.of(
+            RegexTokeniser.rule(TokenType.KEYWORD, "if"),
+            RegexTokeniser.rule(TokenType.IDENTIFIER, "[A-Za-z0-9]+")
+        ));
+        var source = source("if");
+
+        var result = tokeniser.tokenise(source);
+
+        assertThat(result, isSequence(
+            equalTo(new Token<>(source.at(0, 2), TokenType.KEYWORD, "if"))
+        ));
+    }
+
+    @Test
+    public void sequenceOfTokensCanBeGenerated() {
+        var tokeniser = new RegexTokeniser<>(TokenType.UNKNOWN, List.of(
+            RegexTokeniser.rule(TokenType.IDENTIFIER, "[A-Za-z0-9]+"),
+            RegexTokeniser.rule(TokenType.WHITESPACE, "\\s")
+        ));
+        var source = source("one two");
+
+        var result = tokeniser.tokenise(source);
+
+        assertThat(result, isSequence(
+            equalTo(new Token<>(source.at(0, 3), TokenType.IDENTIFIER, "one")),
+            equalTo(new Token<>(source.at(3, 4), TokenType.WHITESPACE, " ")),
+            equalTo(new Token<>(source.at(4, 7), TokenType.IDENTIFIER, "two"))
+        ));
+    }
+
+    @Test
+    public void whenNoRulesMatchThenUnknownTokensAreGenerated() {
+        var tokeniser = new RegexTokeniser<>(TokenType.UNKNOWN, List.of(
+            RegexTokeniser.rule(TokenType.IDENTIFIER, "[A-Za-z0-9]+")
+        ));
+        var source = source("!!a!");
+
+        var result = tokeniser.tokenise(source);
+
+        assertThat(result, isSequence(
+            equalTo(new Token<>(source.at(0, 1), TokenType.UNKNOWN, "!")),
+            equalTo(new Token<>(source.at(1, 2), TokenType.UNKNOWN, "!")),
+            equalTo(new Token<>(source.at(2, 3), TokenType.IDENTIFIER, "a")),
+            equalTo(new Token<>(source.at(3, 4), TokenType.UNKNOWN, "!"))
+        ));
+    }
+
+    private Source source(String sourceContents) {
+        return Sources.file("<filename>", sourceContents);
+    }
+}
diff --git a/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java b/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java
new file mode 100644
index 00000000..315551ee
--- /dev/null
+++ b/src/test/java/org/shedlang/compiler/tokeniser/TokenIteratorTests.java
@@ -0,0 +1,229 @@
+package org.shedlang.compiler.tokeniser;
+
+import org.junit.jupiter.api.Test;
+import org.shedlang.compiler.sources.Sources;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.zwobble.precisely.AssertThat.assertThat;
+import static org.zwobble.precisely.Matchers.equalTo;
+
+public class TokenIteratorTests {
+    enum TokenType {
+        IDENTIFIER,
+        SYMBOL,
+        END
+    }
+
+    @Test
+    public void skipSucceedsWhenNextTokenHasMatchingTokenType() {
+        var tokens = new TokenIterator<>(
+            List.of(
+                token(0, TokenType.SYMBOL, "!"),
+                token(1, TokenType.IDENTIFIER, "x"),
+                token(2, TokenType.SYMBOL, "!")
+            ),
+            token(3, TokenType.END, "")
+        );
+
+        tokens.skip(TokenType.SYMBOL);
+        tokens.skip(TokenType.IDENTIFIER);
+        tokens.skip(TokenType.SYMBOL);
+        tokens.skip(TokenType.END);
+        tokens.skip(TokenType.END);
+    }
+
+    @Test
+    public void skipFailsWhenNextTokenDoesNotHaveMatchingTokenType() {
+        var tokens = new TokenIterator<>(
+            List.of(
+                token(0, TokenType.SYMBOL, "!"),
+                token(1, TokenType.IDENTIFIER, "x"),
+                token(2, TokenType.SYMBOL, "!")
+            ),
+            token(3, TokenType.END, "")
+        );
+
+        tokens.skip(TokenType.SYMBOL);
+
+        try {
+            tokens.skip(TokenType.SYMBOL);
+            fail("expected exception");
+        } catch (UnexpectedTokenException exception) {
+            assertThat(exception.getMessage(), equalTo("Expected: SYMBOL\nBut got: IDENTIFIER \"x\""));
+        }
+    }
+
+    @Test
+    public void trySkipAdvancesIteratorAndReturnsTrueWhenNextTokenHasMatchingTokenType() {
+        var tokens = new TokenIterator<>(
+            List.of(
+                token(0, TokenType.SYMBOL, "!"),
+                token(1, TokenType.IDENTIFIER, "x"),
+                token(2, TokenType.SYMBOL, "!")
+            ),
+            token(3, TokenType.END, "")
+        );
+
+        assertThat(tokens.trySkip(TokenType.SYMBOL), equalTo(true));
+        assertThat(tokens.trySkip(TokenType.IDENTIFIER), equalTo(true));
+        assertThat(tokens.trySkip(TokenType.SYMBOL), equalTo(true));
+        assertThat(tokens.trySkip(TokenType.END), equalTo(true));
+        assertThat(tokens.trySkip(TokenType.END), equalTo(true));
+    }
+
+    @Test
+    public void trySkipDoesNotAdvanceIteratorAndReturnsFalseWhenNextTokenDoesNotHaveMatchingTokenType() {
+        var tokens = new TokenIterator<>(
+            List.of(
+                token(0, TokenType.SYMBOL, "!"),
+                token(1, TokenType.IDENTIFIER, "x"),
+                token(2, TokenType.SYMBOL, "!")
+            ),
+            token(3, TokenType.END, "")
+        );
+
+        tokens.skip(TokenType.SYMBOL);
+
+        assertThat(tokens.trySkip(TokenType.SYMBOL), equalTo(false));
+        assertThat(tokens.trySkip(TokenType.IDENTIFIER), equalTo(true));
+    }
+
+    @Test
+    public void nextValueReturnsValueWhenCurrentTokenHasMatchingTokenType() {
+        var tokens = new TokenIterator<>(
+            List.of(
+                token(0, TokenType.SYMBOL, "!"),
+                token(1, TokenType.IDENTIFIER, "x"),
+                token(2, TokenType.SYMBOL, "!")
+            ),
+            token(3, TokenType.END, "")
+        );
+
+        assertThat(tokens.nextValue(TokenType.SYMBOL), equalTo("!"));
+        assertThat(tokens.nextValue(TokenType.IDENTIFIER), equalTo("x"));
+    }
+
+    @Test
+    public void nextValueFailsWhenCurrentTokenDoesNotHaveMatchingTokenType() {
+        var tokens = new TokenIterator<>(
+            List.of(
+                token(0, TokenType.SYMBOL, "!"),
+                token(1, TokenType.IDENTIFIER, "x"),
+                token(2, TokenType.SYMBOL, "!")
+            ),
+            token(3, TokenType.END, "")
+        );
+
+        tokens.skip(TokenType.SYMBOL);
+
+        try {
+            tokens.nextValue(TokenType.SYMBOL);
+            fail("expected exception");
+        } catch (UnexpectedTokenException exception) {
+            assertThat(exception.getMessage(), equalTo("Expected: SYMBOL\nBut got: IDENTIFIER \"x\""));
+        }
+    }
+
+    @Test
+    public void isNextWithOneArgReturnsTrueIfNextTokenIsOfMatchingType() {
+        var tokens = new TokenIterator<>(
+            List.of(
+                token(0, TokenType.SYMBOL, "!"),
+                token(1, TokenType.IDENTIFIER, "x"),
+                token(2, TokenType.SYMBOL, "!")
+            ),
+            token(3, TokenType.END, "")
+        );
+
+        tokens.skip(TokenType.SYMBOL);
+
+        assertThat(tokens.isNext(TokenType.IDENTIFIER), equalTo(true));
+    }
+
+    @Test
+    public void isNextWithOneArgReturnsFalseIfNextTokenIsNotOfMatchingType() {
+        var tokens = new TokenIterator<>(
+            List.of(
+                token(0, TokenType.SYMBOL, "!"),
+                token(1, TokenType.IDENTIFIER, "x"),
+                token(2, TokenType.SYMBOL, "!")
+            ),
+            token(3, TokenType.END, "")
+        );
+
+        tokens.skip(TokenType.SYMBOL);
+
+        assertThat(tokens.isNext(TokenType.SYMBOL), equalTo(false));
+    }
+
+    @Test
+    public void isNextWithTwoArgReturnsTrueIfBothNextTokensMatchRespectiveTypes() {
+        var tokens = new TokenIterator<>(
+            List.of(
+                token(0, TokenType.SYMBOL, "!"),
+                token(1, TokenType.IDENTIFIER, "x"),
+                token(2, TokenType.SYMBOL, "!")
+            ),
+            token(3, TokenType.END, "")
+        );
+
+        tokens.skip(TokenType.SYMBOL);
+
+        assertThat(tokens.isNext(TokenType.IDENTIFIER, TokenType.SYMBOL), equalTo(true));
+    }
+
+    @Test
+    public void isNextWithTwoArgReturnsFalseIfFirstTokenDoesNotMatchRespectiveType() {
+        var tokens = new TokenIterator<>(
+            List.of(
+                token(0, TokenType.SYMBOL, "!"),
+                token(1, TokenType.IDENTIFIER, "x"),
+                token(2, TokenType.SYMBOL, "!")
+            ),
+            token(3, TokenType.END, "")
+        );
+
+        tokens.skip(TokenType.SYMBOL);
+
+        assertThat(tokens.isNext(TokenType.SYMBOL, TokenType.SYMBOL), equalTo(false));
+    }
+
+    @Test
+    public void isNextWithTwoArgReturnsFalseIfSecondTokenDoesNotMatchRespectiveType() {
+        var tokens = new TokenIterator<>(
+            List.of(
+                token(0, TokenType.SYMBOL, "!"),
+                token(1, TokenType.IDENTIFIER, "x"),
+                token(2, TokenType.SYMBOL, "!")
+            ),
+            token(3, TokenType.END, "")
+        );
+
+        tokens.skip(TokenType.SYMBOL);
+
+        assertThat(tokens.isNext(TokenType.IDENTIFIER, TokenType.IDENTIFIER), equalTo(false));
+    }
+
+    @Test
+    public void peekReturnsNextTokenWithoutAdvancing() {
+        var tokens = new TokenIterator<>(
+            List.of(
+                token(0, TokenType.SYMBOL, "!"),
+                token(1, TokenType.IDENTIFIER, "x"),
+                token(2, TokenType.SYMBOL, "!")
+            ),
+            token(3, TokenType.END, "")
+        );
+
+        assertThat(tokens.peek(), equalTo(token(0, TokenType.SYMBOL, "!")));
+        assertThat(tokens.peek(), equalTo(token(0, TokenType.SYMBOL, "!")));
+        tokens.skip(TokenType.SYMBOL);
+        assertThat(tokens.peek(), equalTo(token(1, TokenType.IDENTIFIER, "x")));
+    }
+
+    private Token<TokenType> token(int characterIndex, TokenType tokenType, String value) {
+        return new Token<>(Sources.NULL, tokenType, value);
+    }
+}