Tokenize doc comments

2 files changed, 28 insertions, 0 deletions

diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java
index a0b36e7..ef1c9dc 100644
--- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java
+++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java
@@ -1,6 +1,7 @@
 package org.zwobble.hobgoblin.compiler.parser;
 
 enum TokenType {
+    DOC_COMMENT,
     END,
     IDENTIFIER,
     WHITESPACE,
diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
index 1a110f7..1311df3 100644
--- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
+++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
@@ -49,6 +49,11 @@ class Tokenizer {
             return symbol;
         }
 
+        var docComment = tokenizeDocComment(iterator);
+        if (docComment.isPresent()) {
+            return docComment;
+        }
+
         throw ParseError.unexpectedTextError(
             "token",
             describeCharacter(iterator.peek()),
@@ -71,6 +76,24 @@ class Tokenizer {
         return Character.isWhitespace(character);
     }
 
+    private static Optional<Token> tokenizeDocComment(CharacterIterator iterator) {
+        if (!"///".contentEquals(iterator.peekSequence(3))) {
+            return Optional.empty();
+        }
+
+        var start = iterator.position();
+        while (!iterator.isEnd() && iterator.peek() != '\n') {
+            iterator.skip();
+        }
+        if (iterator.peek() == '\n') {
+            iterator.skip();
+        }
+        var end = iterator.position();
+
+        var token = new Token(TokenType.DOC_COMMENT, start.to(end));
+        return Optional.of(token);
+    }
+
     private static Optional<Token> tokenizeIdentifierOrKeyword(CharacterIterator iterator) {
         if (!Character.isAlphabetic(iterator.peek())) {
             return Optional.empty();
@@ -177,6 +200,10 @@ class Tokenizer {
             return this.iterator.peek();
         }
 
+        CharSequence peekSequence(int length) {
+            return this.iterator.peekSequence(length);
+        }
+
         void skip() {
             this.iterator.skip();
         }