diff options
3 files changed, 42 insertions, 0 deletions
diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java index a0b36e7..ef1c9dc 100644 --- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java @@ -1,6 +1,7 @@ package org.zwobble.hobgoblin.compiler.parser; enum TokenType { + DOC_COMMENT, END, IDENTIFIER, WHITESPACE, diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java index 1a110f7..1311df3 100644 --- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java @@ -49,6 +49,11 @@ class Tokenizer { return symbol; } + var docComment = tokenizeDocComment(iterator); + if (docComment.isPresent()) { + return docComment; + } + throw ParseError.unexpectedTextError( "token", describeCharacter(iterator.peek()), @@ -71,6 +76,24 @@ class Tokenizer { return Character.isWhitespace(character); } + private static Optional<Token> tokenizeDocComment(CharacterIterator iterator) { + if (!"///".contentEquals(iterator.peekSequence(3))) { + return Optional.empty(); + } + + var start = iterator.position(); + while (!iterator.isEnd() && iterator.peek() != '\n') { + iterator.skip(); + } + if (iterator.peek() == '\n') { + iterator.skip(); + } + var end = iterator.position(); + + var token = new Token(TokenType.DOC_COMMENT, start.to(end)); + return Optional.of(token); + } + private static Optional<Token> tokenizeIdentifierOrKeyword(CharacterIterator iterator) { if (!Character.isAlphabetic(iterator.peek())) { return Optional.empty(); @@ -177,6 +200,10 @@ class Tokenizer { return this.iterator.peek(); } + CharSequence peekSequence(int length) { + return this.iterator.peekSequence(length); + } + void skip() { this.iterator.skip(); } diff --git a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java index 70fba72..551e8d3 100644 --- a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java +++ b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java @@ -185,6 +185,20 @@ public class TokenizerTests { )); } + @Test + public void docCommentRunsToEndOfLine() { + var sourceText = SourceText.fromString("<string>", "a\n/// One\n /// Two\nb"); + + var tokens = Tokenizer.tokenize(sourceText); + + assertThat(tokens.tokens(), isSequence( + equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 0, 1))), + equalTo(new Token(TokenType.DOC_COMMENT, sourceRange(sourceText, 2, 10))), + equalTo(new Token(TokenType.DOC_COMMENT, sourceRange(sourceText, 14, 22))), + equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 22, 23))) + )); + } + private static SourceRange sourceRange(SourceText sourceText, int fromCharacterIndex, int toCharacterIndex) { return sourceText.characterPosition(fromCharacterIndex).to(sourceText.characterPosition(toCharacterIndex)); } |