diff options
3 files changed, 17 insertions, 0 deletions
diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java index 22e7bf0..bd2ce7b 100644 --- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java @@ -11,4 +11,5 @@ enum TokenType { SYMBOL_BRACE_OPEN, SYMBOL_BRACE_CLOSE, SYMBOL_COLON, + SYMBOL_SEMICOLON, } diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java index 0f5984e..6fc5e20 100644 --- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java @@ -116,6 +116,11 @@ class Tokenizer { iterator.skip(); break; + case ';': + tokenType = TokenType.SYMBOL_SEMICOLON; + iterator.skip(); + break; + default: return Optional.empty(); } diff --git a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java index affd157..7d4ad35 100644 --- a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java +++ b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java @@ -85,6 +85,17 @@ public class TokenizerTests { } @Test + public void symbolSemicolonIsTokenized() { + var sourceText = SourceText.fromString("<string>", ";"); + + var tokens = Tokenizer.tokenize(sourceText); + + assertThat(tokens.tokens(), isSequence( + equalTo(new Token(TokenType.SYMBOL_SEMICOLON, sourceRange(sourceText, 0, 1))) + )); + } + + @Test public void whitespaceIsIgnored() { var sourceText = SourceText.fromString("<string>", " blah "); |