Tokenize dot dot symbol

3 files changed, 66 insertions, 48 deletions

diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java
index f7242e2..8277653 100644
--- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java
+++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java
@@ -19,6 +19,7 @@ enum TokenType {
     SYMBOL_COLON,
     SYMBOL_COMMA,
     SYMBOL_DOT,
+    SYMBOL_DOT_DOT,
     SYMBOL_FORWARD_SLASH,
     SYMBOL_SEMICOLON,
     SYMBOL_SQUARE_OPEN,
diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
index 7db0806..8acde49 100644
--- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
+++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
@@ -148,54 +148,60 @@ class Tokenizer {
         TokenType tokenType;
         var start = iterator.position();
 
-        switch (iterator.peek()) {
-            case '{':
-                tokenType = TokenType.SYMBOL_BRACE_OPEN;
-                iterator.skip();
-                break;
-
-            case '}':
-                tokenType = TokenType.SYMBOL_BRACE_CLOSE;
-                iterator.skip();
-                break;
-
-            case ':':
-                tokenType = TokenType.SYMBOL_COLON;
-                iterator.skip();
-                break;
-
-            case ',':
-                tokenType = TokenType.SYMBOL_COMMA;
-                iterator.skip();
-                break;
-
-            case '.':
-                tokenType = TokenType.SYMBOL_DOT;
-                iterator.skip();
-                break;
-
-            case '/':
-                tokenType = TokenType.SYMBOL_FORWARD_SLASH;
-                iterator.skip();
-                break;
-
-            case ';':
-                tokenType = TokenType.SYMBOL_SEMICOLON;
-                iterator.skip();
-                break;
-
-            case '[':
-                tokenType = TokenType.SYMBOL_SQUARE_OPEN;
-                iterator.skip();
-                break;
-
-            case ']':
-                tokenType = TokenType.SYMBOL_SQUARE_CLOSE;
-                iterator.skip();
-                break;
-
-            default:
-                return Optional.empty();
+        if ("..".contentEquals(iterator.peekSequence(2))) {
+            tokenType = TokenType.SYMBOL_DOT_DOT;
+            iterator.skip();
+            iterator.skip();
+        } else {
+            switch (iterator.peek()) {
+                case '{':
+                    tokenType = TokenType.SYMBOL_BRACE_OPEN;
+                    iterator.skip();
+                    break;
+
+                case '}':
+                    tokenType = TokenType.SYMBOL_BRACE_CLOSE;
+                    iterator.skip();
+                    break;
+
+                case ':':
+                    tokenType = TokenType.SYMBOL_COLON;
+                    iterator.skip();
+                    break;
+
+                case ',':
+                    tokenType = TokenType.SYMBOL_COMMA;
+                    iterator.skip();
+                    break;
+
+                case '.':
+                    tokenType = TokenType.SYMBOL_DOT;
+                    iterator.skip();
+                    break;
+
+                case '/':
+                    tokenType = TokenType.SYMBOL_FORWARD_SLASH;
+                    iterator.skip();
+                    break;
+
+                case ';':
+                    tokenType = TokenType.SYMBOL_SEMICOLON;
+                    iterator.skip();
+                    break;
+
+                case '[':
+                    tokenType = TokenType.SYMBOL_SQUARE_OPEN;
+                    iterator.skip();
+                    break;
+
+                case ']':
+                    tokenType = TokenType.SYMBOL_SQUARE_CLOSE;
+                    iterator.skip();
+                    break;
+
+                default:
+                    return Optional.empty();
+            }
         }
 
         var end = iterator.position();
diff --git a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
index 2b624ab..0440719 100644
--- a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
+++ b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
@@ -173,6 +173,17 @@ public class TokenizerTests {
     }
 
     @Test
+    public void symbolDotDotIsTokenized() {
+        var sourceText = SourceText.fromString("<string>", "..");
+
+        var tokens = Tokenizer.tokenize(sourceText);
+
+        assertThat(tokens.tokens(), isSequence(
+            equalTo(new Token(TokenType.SYMBOL_DOT_DOT, sourceRange(sourceText, 0, 2)))
+        ));
+    }
+
+    @Test
     public void symbolForwardSlashIsTokenized() {
         var sourceText = SourceText.fromString("<string>", "/");