Add initial tokenizer

1 files changed, 46 insertions, 0 deletions

diff --git a/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java
new file mode 100644
index 0000000..52c5f79
--- /dev/null
+++ b/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java
@@ -0,0 +1,46 @@
+package org.zwobble.hobgoblin.parser;
+
+import org.junit.jupiter.api.Test;
+import org.zwobble.sourcetext.SourceRange;
+import org.zwobble.sourcetext.SourceText;
+
+import static org.zwobble.precisely.AssertThat.assertThat;
+import static org.zwobble.precisely.Matchers.equalTo;
+import static org.zwobble.precisely.Matchers.isSequence;
+
+public class TokenizerTests {
+    @Test
+    public void emptySourceTextIsTokenizedToEmpty() {
+        var sourceText = SourceText.fromString("<string>", "");
+
+        var tokens = Tokenizer.tokenize(sourceText);
+
+        assertThat(tokens, isSequence());
+    }
+
+    @Test
+    public void structKeywordIsTokenized() {
+        var sourceText = SourceText.fromString("<string>", "struct");
+
+        var tokens = Tokenizer.tokenize(sourceText);
+
+        assertThat(tokens, isSequence(
+            equalTo(new Token(TokenType.KEYWORD_STRUCT, sourceRange(sourceText, 0, 6)))
+        ));
+    }
+
+    @Test
+    public void identifierIsTokenized() {
+        var sourceText = SourceText.fromString("<string>", "blah");
+
+        var tokens = Tokenizer.tokenize(sourceText);
+
+        assertThat(tokens, isSequence(
+            equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 0, 4)))
+        ));
+    }
+
+    private static SourceRange sourceRange(SourceText sourceText, int fromCharacterIndex, int toCharacterIndex) {
+        return sourceText.characterPosition(fromCharacterIndex).to(sourceText.characterPosition(toCharacterIndex));
+    }
+}