From 193a62fa52e360ee76cca3d0f6bdbe2f6a69602a Mon Sep 17 00:00:00 2001 From: Michael Williamson Date: Sun, 10 May 2026 15:15:31 +0100 Subject: Tokenize doc comments --- .../zwobble/hobgoblin/compiler/parser/TokenizerTests.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'src/test/java/org') diff --git a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java index 70fba72..551e8d3 100644 --- a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java +++ b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java @@ -185,6 +185,20 @@ public class TokenizerTests { )); } + @Test + public void docCommentRunsToEndOfLine() { + var sourceText = SourceText.fromString("", "a\n/// One\n /// Two\nb"); + + var tokens = Tokenizer.tokenize(sourceText); + + assertThat(tokens.tokens(), isSequence( + equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 0, 1))), + equalTo(new Token(TokenType.DOC_COMMENT, sourceRange(sourceText, 2, 10))), + equalTo(new Token(TokenType.DOC_COMMENT, sourceRange(sourceText, 14, 22))), + equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 22, 23))) + )); + } + private static SourceRange sourceRange(SourceText sourceText, int fromCharacterIndex, int toCharacterIndex) { return sourceText.characterPosition(fromCharacterIndex).to(sourceText.characterPosition(toCharacterIndex)); } -- cgit v1.2.3