diff options
3 files changed, 29 insertions, 12 deletions
diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java index 9c11815..a0b36e7 100644 --- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java @@ -13,6 +13,7 @@ enum TokenType { SYMBOL_BRACE_OPEN, SYMBOL_BRACE_CLOSE, SYMBOL_COLON, + SYMBOL_COMMA, SYMBOL_SEMICOLON, SYMBOL_SQUARE_OPEN, SYMBOL_SQUARE_CLOSE, diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java index 5ec42e6..1a110f7 100644 --- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java @@ -120,6 +120,11 @@ class Tokenizer { iterator.skip(); break; + case ',': + tokenType = TokenType.SYMBOL_COMMA; + iterator.skip(); + break; + case ';': tokenType = TokenType.SYMBOL_SEMICOLON; iterator.skip(); diff --git a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java index 64e64fa..70fba72 100644 --- a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java +++ b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java @@ -107,46 +107,57 @@ public class TokenizerTests { } @Test - public void symbolSquareOpenIsTokenized() { - var sourceText = SourceText.fromString("<string>", "["); + public void symbolColonIsTokenized() { + var sourceText = SourceText.fromString("<string>", ":"); var tokens = Tokenizer.tokenize(sourceText); assertThat(tokens.tokens(), isSequence( - equalTo(new Token(TokenType.SYMBOL_SQUARE_OPEN, sourceRange(sourceText, 0, 1))) + equalTo(new Token(TokenType.SYMBOL_COLON, sourceRange(sourceText, 0, 1))) )); } @Test - public void symbolSquareCloseIsTokenized() { - var sourceText = SourceText.fromString("<string>", "]"); + public void symbolCommaIsTokenized() { + var sourceText = SourceText.fromString("<string>", ","); var tokens = Tokenizer.tokenize(sourceText); assertThat(tokens.tokens(), isSequence( - equalTo(new Token(TokenType.SYMBOL_SQUARE_CLOSE, sourceRange(sourceText, 0, 1))) + equalTo(new Token(TokenType.SYMBOL_COMMA, sourceRange(sourceText, 0, 1))) )); } @Test - public void symbolColonIsTokenized() { - var sourceText = SourceText.fromString("<string>", ":"); + public void symbolSemicolonIsTokenized() { + var sourceText = SourceText.fromString("<string>", ";"); var tokens = Tokenizer.tokenize(sourceText); assertThat(tokens.tokens(), isSequence( - equalTo(new Token(TokenType.SYMBOL_COLON, sourceRange(sourceText, 0, 1))) + equalTo(new Token(TokenType.SYMBOL_SEMICOLON, sourceRange(sourceText, 0, 1))) )); } @Test - public void symbolSemicolonIsTokenized() { - var sourceText = SourceText.fromString("<string>", ";"); + public void symbolSquareOpenIsTokenized() { + var sourceText = SourceText.fromString("<string>", "["); var tokens = Tokenizer.tokenize(sourceText); assertThat(tokens.tokens(), isSequence( - equalTo(new Token(TokenType.SYMBOL_SEMICOLON, sourceRange(sourceText, 0, 1))) + equalTo(new Token(TokenType.SYMBOL_SQUARE_OPEN, sourceRange(sourceText, 0, 1))) + )); + } + + @Test + public void symbolSquareCloseIsTokenized() { + var sourceText = SourceText.fromString("<string>", "]"); + + var tokens = Tokenizer.tokenize(sourceText); + + assertThat(tokens.tokens(), isSequence( + equalTo(new Token(TokenType.SYMBOL_SQUARE_CLOSE, sourceRange(sourceText, 0, 1))) )); } |