Tokenize forward slash

3 files changed, 26 insertions, 5 deletions

diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java
index e25a9f0..63ff5e1 100644
--- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java
+++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java
@@ -16,6 +16,7 @@ enum TokenType {
     SYMBOL_BRACE_CLOSE,
     SYMBOL_COLON,
     SYMBOL_COMMA,
+    SYMBOL_FORWARD_SLASH,
     SYMBOL_SEMICOLON,
     SYMBOL_SQUARE_OPEN,
     SYMBOL_SQUARE_CLOSE,
diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
index 7119cfd..35bbeda 100644
--- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
+++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
@@ -44,16 +44,16 @@ class Tokenizer {
             return identifierOrKeyword;
         }
 
-        var symbol = tokenizeSymbol(iterator);
-        if (symbol.isPresent()) {
-            return symbol;
-        }
-
         var docComment = tokenizeDocComment(iterator);
         if (docComment.isPresent()) {
             return docComment;
         }
 
+        var symbol = tokenizeSymbol(iterator);
+        if (symbol.isPresent()) {
+            return symbol;
+        }
+
         throw ParseError.unexpectedTextError(
             "token",
             describeCharacter(iterator.peek()),
@@ -150,6 +150,11 @@ class Tokenizer {
                 iterator.skip();
                 break;
 
+            case '/':
+                tokenType = TokenType.SYMBOL_FORWARD_SLASH;
+                iterator.skip();
+                break;
+
             case ';':
                 tokenType = TokenType.SYMBOL_SEMICOLON;
                 iterator.skip();
@@ -203,6 +208,10 @@ class Tokenizer {
         }
 
         CharSequence peekSequence(int length) {
+            // TODO: fix this in source-text
+            if (length > this.iterator.remaining()) {
+                length = this.iterator.remaining();
+            }
             return this.iterator.peekSequence(length);
         }
 
diff --git a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
index e8dfd36..815dede 100644
--- a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
+++ b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
@@ -140,6 +140,17 @@ public class TokenizerTests {
     }
 
     @Test
+    public void symbolForwardSlashIsTokenized() {
+        var sourceText = SourceText.fromString("<string>", "/");
+
+        var tokens = Tokenizer.tokenize(sourceText);
+
+        assertThat(tokens.tokens(), isSequence(
+            equalTo(new Token(TokenType.SYMBOL_FORWARD_SLASH, sourceRange(sourceText, 0, 1)))
+        ));
+    }
+
+    @Test
     public void symbolSemicolonIsTokenized() {
         var sourceText = SourceText.fromString("<string>", ";");