Tokenize braces

2 files changed, 68 insertions, 4 deletions

diff --git a/src/main/java/org/zwobble/hobgoblin/parser/Tokenizer.java b/src/main/java/org/zwobble/hobgoblin/parser/Tokenizer.java
index e616ec0..a642d76 100644
--- a/src/main/java/org/zwobble/hobgoblin/parser/Tokenizer.java
+++ b/src/main/java/org/zwobble/hobgoblin/parser/Tokenizer.java
@@ -21,9 +21,9 @@ class Tokenizer {
             if (trySkipWhitespace(iterator)) {
                 // Skip whitespace.
             } else {
-                var identifierOrKeyword = tokenizeIdentifierOrKeyword(iterator);
-                if (identifierOrKeyword.isPresent()) {
-                    tokens.add(identifierOrKeyword.get());
+                var token = tokenizeToken(iterator);
+                if (token.isPresent()) {
+                    tokens.add(token.get());
                 } else {
                     throw ParseError.unexpectedTextError(
                         "token",
@@ -37,6 +37,24 @@ class Tokenizer {
         return tokens;
     }
 
+    private static Optional<Token> tokenizeToken(CharacterIterator iterator) {
+        var identifierOrKeyword = tokenizeIdentifierOrKeyword(iterator);
+        if (identifierOrKeyword.isPresent()) {
+            return identifierOrKeyword;
+        }
+
+        var symbol = tokenizeSymbol(iterator);
+        if (symbol.isPresent()) {
+            return symbol;
+        }
+
+        throw ParseError.unexpectedTextError(
+            "token",
+            describeCharacter(iterator.peek()),
+            iterator.characterSourceRange()
+        );
+    }
+
     private static boolean trySkipWhitespace(CharacterIterator iterator) {
         var whitespace = false;
 
@@ -75,6 +93,30 @@ class Tokenizer {
         return Optional.of(token);
     }
 
+    private static Optional<Token> tokenizeSymbol(CharacterIterator iterator) {
+        TokenType tokenType;
+        var start = iterator.position();
+
+        switch (iterator.peek()) {
+            case '{':
+                tokenType = TokenType.SYMBOL_BRACE_OPEN;
+                iterator.skip();
+                break;
+
+            case '}':
+                tokenType = TokenType.SYMBOL_BRACE_CLOSE;
+                iterator.skip();
+                break;
+
+            default:
+                return Optional.empty();
+        }
+
+        var end = iterator.position();
+        var token = new Token(tokenType, start.to(end));
+        return Optional.of(token);
+    }
+
     private static String describeCharacter(int character) {
         // TODO: handle characters that should be escaped
         if (character == -1) {
diff --git a/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java
index aa6dbe4..05848a3 100644
--- a/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java
+++ b/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java
@@ -19,7 +19,7 @@ public class TokenizerTests {
     }
 
     @Test
-    public void structKeywordIsTokenized() {
+    public void keywordStructIsTokenized() {
         var sourceText = SourceText.fromString("<string>", "struct");
 
         var tokens = Tokenizer.tokenize(sourceText);
@@ -41,6 +41,28 @@ public class TokenizerTests {
     }
 
     @Test
+    public void symbolBraceOpenIsTokenized() {
+        var sourceText = SourceText.fromString("<string>", "{");
+
+        var tokens = Tokenizer.tokenize(sourceText);
+
+        assertThat(tokens, isSequence(
+            equalTo(new Token(TokenType.SYMBOL_BRACE_OPEN, sourceRange(sourceText, 0, 1)))
+        ));
+    }
+
+    @Test
+    public void symbolBraceCloseIsTokenized() {
+        var sourceText = SourceText.fromString("<string>", "}");
+
+        var tokens = Tokenizer.tokenize(sourceText);
+
+        assertThat(tokens, isSequence(
+            equalTo(new Token(TokenType.SYMBOL_BRACE_CLOSE, sourceRange(sourceText, 0, 1)))
+        ));
+    }
+
+    @Test
     public void whitespaceIsIgnored() {
         var sourceText = SourceText.fromString("<string>", "  blah  ");