From bd7d0b6335238a465090af626a50df5fdfd425be Mon Sep 17 00:00:00 2001
From: Michael Williamson <mike@zwobble.org>
Date: Sun, 3 May 2026 11:55:15 +0100
Subject: Tokenize comma

---
 .../hobgoblin/compiler/parser/TokenType.java       |  1 +
 .../hobgoblin/compiler/parser/Tokenizer.java       |  5 ++++
 .../hobgoblin/compiler/parser/TokenizerTests.java  | 35 ++++++++++++++--------
 3 files changed, 29 insertions(+), 12 deletions(-)

diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java
index 9c11815..a0b36e7 100644
--- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java
+++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java
@@ -13,6 +13,7 @@ enum TokenType {
     SYMBOL_BRACE_OPEN,
     SYMBOL_BRACE_CLOSE,
     SYMBOL_COLON,
+    SYMBOL_COMMA,
     SYMBOL_SEMICOLON,
     SYMBOL_SQUARE_OPEN,
     SYMBOL_SQUARE_CLOSE,
diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
index 5ec42e6..1a110f7 100644
--- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
+++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
@@ -120,6 +120,11 @@ class Tokenizer {
                 iterator.skip();
                 break;
 
+            case ',':
+                tokenType = TokenType.SYMBOL_COMMA;
+                iterator.skip();
+                break;
+
             case ';':
                 tokenType = TokenType.SYMBOL_SEMICOLON;
                 iterator.skip();
diff --git a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
index 64e64fa..70fba72 100644
--- a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
+++ b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
@@ -107,46 +107,57 @@ public class TokenizerTests {
     }
 
     @Test
-    public void symbolSquareOpenIsTokenized() {
-        var sourceText = SourceText.fromString("<string>", "[");
+    public void symbolColonIsTokenized() {
+        var sourceText = SourceText.fromString("<string>", ":");
 
         var tokens = Tokenizer.tokenize(sourceText);
 
         assertThat(tokens.tokens(), isSequence(
-            equalTo(new Token(TokenType.SYMBOL_SQUARE_OPEN, sourceRange(sourceText, 0, 1)))
+            equalTo(new Token(TokenType.SYMBOL_COLON, sourceRange(sourceText, 0, 1)))
         ));
     }
 
     @Test
-    public void symbolSquareCloseIsTokenized() {
-        var sourceText = SourceText.fromString("<string>", "]");
+    public void symbolCommaIsTokenized() {
+        var sourceText = SourceText.fromString("<string>", ",");
 
         var tokens = Tokenizer.tokenize(sourceText);
 
         assertThat(tokens.tokens(), isSequence(
-            equalTo(new Token(TokenType.SYMBOL_SQUARE_CLOSE, sourceRange(sourceText, 0, 1)))
+            equalTo(new Token(TokenType.SYMBOL_COMMA, sourceRange(sourceText, 0, 1)))
         ));
     }
 
     @Test
-    public void symbolColonIsTokenized() {
-        var sourceText = SourceText.fromString("<string>", ":");
+    public void symbolSemicolonIsTokenized() {
+        var sourceText = SourceText.fromString("<string>", ";");
 
         var tokens = Tokenizer.tokenize(sourceText);
 
         assertThat(tokens.tokens(), isSequence(
-            equalTo(new Token(TokenType.SYMBOL_COLON, sourceRange(sourceText, 0, 1)))
+            equalTo(new Token(TokenType.SYMBOL_SEMICOLON, sourceRange(sourceText, 0, 1)))
         ));
     }
 
     @Test
-    public void symbolSemicolonIsTokenized() {
-        var sourceText = SourceText.fromString("<string>", ";");
+    public void symbolSquareOpenIsTokenized() {
+        var sourceText = SourceText.fromString("<string>", "[");
 
         var tokens = Tokenizer.tokenize(sourceText);
 
         assertThat(tokens.tokens(), isSequence(
-            equalTo(new Token(TokenType.SYMBOL_SEMICOLON, sourceRange(sourceText, 0, 1)))
+            equalTo(new Token(TokenType.SYMBOL_SQUARE_OPEN, sourceRange(sourceText, 0, 1)))
+        ));
+    }
+
+    @Test
+    public void symbolSquareCloseIsTokenized() {
+        var sourceText = SourceText.fromString("<string>", "]");
+
+        var tokens = Tokenizer.tokenize(sourceText);
+
+        assertThat(tokens.tokens(), isSequence(
+            equalTo(new Token(TokenType.SYMBOL_SQUARE_CLOSE, sourceRange(sourceText, 0, 1)))
         ));
     }
 
-- 
cgit v1.2.3