From 12f16e818935114c64e203d43202f5ca043ae098 Mon Sep 17 00:00:00 2001 From: Michael Williamson Date: Thu, 23 Apr 2026 11:32:44 +0100 Subject: Tokenize colon symbol --- .../java/org/zwobble/hobgoblin/compiler/parser/TokenType.java | 1 + .../java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java | 5 +++++ .../org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java | 11 +++++++++++ 3 files changed, 17 insertions(+) (limited to 'src') diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java index 64be2d4..22e7bf0 100644 --- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java @@ -10,4 +10,5 @@ enum TokenType { SYMBOL_BRACE_OPEN, SYMBOL_BRACE_CLOSE, + SYMBOL_COLON, } diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java index 6a43a09..0f5984e 100644 --- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java @@ -111,6 +111,11 @@ class Tokenizer { iterator.skip(); break; + case ':': + tokenType = TokenType.SYMBOL_COLON; + iterator.skip(); + break; + default: return Optional.empty(); } diff --git a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java index 68db18d..affd157 100644 --- a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java +++ b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java @@ -73,6 +73,17 @@ public class TokenizerTests { )); } + @Test + public void symbolColonIsTokenized() { + var sourceText = SourceText.fromString("", ":"); + + var tokens = Tokenizer.tokenize(sourceText); + + assertThat(tokens.tokens(), isSequence( + equalTo(new Token(TokenType.SYMBOL_COLON, sourceRange(sourceText, 0, 1))) + )); + } + @Test public void whitespaceIsIgnored() { var sourceText = SourceText.fromString("", " blah "); -- cgit v1.2.3