Tokenize doc comments

1 files changed, 14 insertions, 0 deletions

diff --git a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
index 70fba72..551e8d3 100644
--- a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
+++ b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
@@ -185,6 +185,20 @@ public class TokenizerTests {
         ));
     }
 
+    @Test
+    public void docCommentRunsToEndOfLine() {
+        var sourceText = SourceText.fromString("<string>", "a\n/// One\n    /// Two\nb");
+
+        var tokens = Tokenizer.tokenize(sourceText);
+
+        assertThat(tokens.tokens(), isSequence(
+            equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 0, 1))),
+            equalTo(new Token(TokenType.DOC_COMMENT, sourceRange(sourceText, 2, 10))),
+            equalTo(new Token(TokenType.DOC_COMMENT, sourceRange(sourceText, 14, 22))),
+            equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 22, 23)))
+        ));
+    }
+
     private static SourceRange sourceRange(SourceText sourceText, int fromCharacterIndex, int toCharacterIndex) {
         return sourceText.characterPosition(fromCharacterIndex).to(sourceText.characterPosition(toCharacterIndex));
     }