diff options
3 files changed, 34 insertions, 0 deletions
diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java index df86ae6..2a85072 100644 --- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java @@ -22,6 +22,8 @@ enum TokenType { SYMBOL_DOT, SYMBOL_DOT_DOT, SYMBOL_FORWARD_SLASH, + SYMBOL_PAREN_OPEN, + SYMBOL_PAREN_CLOSE, SYMBOL_SEMICOLON, SYMBOL_SQUARE_OPEN, SYMBOL_SQUARE_CLOSE, diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java index c0cf716..dc06612 100644 --- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java @@ -186,6 +186,16 @@ class Tokenizer { iterator.skip(); break; + case '(': + tokenType = TokenType.SYMBOL_PAREN_OPEN; + iterator.skip(); + break; + + case ')': + tokenType = TokenType.SYMBOL_PAREN_CLOSE; + iterator.skip(); + break; + case ';': tokenType = TokenType.SYMBOL_SEMICOLON; iterator.skip(); diff --git a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java index 848738a..39aded5 100644 --- a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java +++ b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java @@ -206,6 +206,28 @@ public class TokenizerTests { } @Test + public void symbolParenOpenIsTokenized() { + var sourceText = SourceText.fromString("<string>", "("); + + var tokens = Tokenizer.tokenize(sourceText); + + assertThat(tokens.tokens(), isSequence( + equalTo(new Token(TokenType.SYMBOL_PAREN_OPEN, sourceRange(sourceText, 0, 1))) + )); + } + + @Test + public void symbolParenCloseIsTokenized() { + var sourceText = SourceText.fromString("<string>", ")"); + + var tokens = Tokenizer.tokenize(sourceText); + + assertThat(tokens.tokens(), isSequence( + equalTo(new Token(TokenType.SYMBOL_PAREN_CLOSE, sourceRange(sourceText, 0, 1))) + )); + } + + @Test public void symbolSemicolonIsTokenized() { var sourceText = SourceText.fromString("<string>", ";"); |