Tokenize forward slash

2 files changed, 15 insertions, 5 deletions

diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java
index e25a9f0..63ff5e1 100644
--- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java
+++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java
@@ -16,6 +16,7 @@ enum TokenType {
     SYMBOL_BRACE_CLOSE,
     SYMBOL_COLON,
     SYMBOL_COMMA,
+    SYMBOL_FORWARD_SLASH,
     SYMBOL_SEMICOLON,
     SYMBOL_SQUARE_OPEN,
     SYMBOL_SQUARE_CLOSE,
diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
index 7119cfd..35bbeda 100644
--- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
+++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
@@ -44,16 +44,16 @@ class Tokenizer {
             return identifierOrKeyword;
         }
 
-        var symbol = tokenizeSymbol(iterator);
-        if (symbol.isPresent()) {
-            return symbol;
-        }
-
         var docComment = tokenizeDocComment(iterator);
         if (docComment.isPresent()) {
             return docComment;
         }
 
+        var symbol = tokenizeSymbol(iterator);
+        if (symbol.isPresent()) {
+            return symbol;
+        }
+
         throw ParseError.unexpectedTextError(
             "token",
             describeCharacter(iterator.peek()),
@@ -150,6 +150,11 @@ class Tokenizer {
                 iterator.skip();
                 break;
 
+            case '/':
+                tokenType = TokenType.SYMBOL_FORWARD_SLASH;
+                iterator.skip();
+                break;
+
             case ';':
                 tokenType = TokenType.SYMBOL_SEMICOLON;
                 iterator.skip();
@@ -203,6 +208,10 @@ class Tokenizer {
         }
 
         CharSequence peekSequence(int length) {
+            // TODO: fix this in source-text
+            if (length > this.iterator.remaining()) {
+                length = this.iterator.remaining();
+            }
             return this.iterator.peekSequence(length);
         }