From 00592db2490ac6274b3926fd007365d2c50327a8 Mon Sep 17 00:00:00 2001
From: Michael Williamson <mike@zwobble.org>
Date: Sat, 18 Apr 2026 13:09:30 +0100
Subject: Add initial tokenizer

---
 .../zwobble/hobgoblin/parser/TokenizerTests.java   | 46 ++++++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java

(limited to 'src/test/java/org')

diff --git a/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java
new file mode 100644
index 0000000..52c5f79
--- /dev/null
+++ b/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java
@@ -0,0 +1,46 @@
+package org.zwobble.hobgoblin.parser;
+
+import org.junit.jupiter.api.Test;
+import org.zwobble.sourcetext.SourceRange;
+import org.zwobble.sourcetext.SourceText;
+
+import static org.zwobble.precisely.AssertThat.assertThat;
+import static org.zwobble.precisely.Matchers.equalTo;
+import static org.zwobble.precisely.Matchers.isSequence;
+
+public class TokenizerTests {
+    @Test
+    public void emptySourceTextIsTokenizedToEmpty() {
+        var sourceText = SourceText.fromString("<string>", "");
+
+        var tokens = Tokenizer.tokenize(sourceText);
+
+        assertThat(tokens, isSequence());
+    }
+
+    @Test
+    public void structKeywordIsTokenized() {
+        var sourceText = SourceText.fromString("<string>", "struct");
+
+        var tokens = Tokenizer.tokenize(sourceText);
+
+        assertThat(tokens, isSequence(
+            equalTo(new Token(TokenType.KEYWORD_STRUCT, sourceRange(sourceText, 0, 6)))
+        ));
+    }
+
+    @Test
+    public void identifierIsTokenized() {
+        var sourceText = SourceText.fromString("<string>", "blah");
+
+        var tokens = Tokenizer.tokenize(sourceText);
+
+        assertThat(tokens, isSequence(
+            equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 0, 4)))
+        ));
+    }
+
+    private static SourceRange sourceRange(SourceText sourceText, int fromCharacterIndex, int toCharacterIndex) {
+        return sourceText.characterPosition(fromCharacterIndex).to(sourceText.characterPosition(toCharacterIndex));
+    }
+}
-- 
cgit v1.2.3