diff options
3 files changed, 17 insertions, 0 deletions
diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java index 63ff5e1..85ac1b9 100644 --- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java @@ -16,6 +16,7 @@ enum TokenType { SYMBOL_BRACE_CLOSE, SYMBOL_COLON, SYMBOL_COMMA, + SYMBOL_DOT, SYMBOL_FORWARD_SLASH, SYMBOL_SEMICOLON, SYMBOL_SQUARE_OPEN, diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java index 35bbeda..929b516 100644 --- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java @@ -150,6 +150,11 @@ class Tokenizer { iterator.skip(); break; + case '.': + tokenType = TokenType.SYMBOL_DOT; + iterator.skip(); + break; + case '/': tokenType = TokenType.SYMBOL_FORWARD_SLASH; iterator.skip(); diff --git a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java index 815dede..68b4a89 100644 --- a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java +++ b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java @@ -140,6 +140,17 @@ public class TokenizerTests { } @Test + public void symbolDotIsTokenized() { + var sourceText = SourceText.fromString("<string>", "."); + + var tokens = Tokenizer.tokenize(sourceText); + + assertThat(tokens.tokens(), isSequence( + equalTo(new Token(TokenType.SYMBOL_DOT, sourceRange(sourceText, 0, 1))) + )); + } + + @Test public void symbolForwardSlashIsTokenized() { var sourceText = SourceText.fromString("<string>", "/"); |