Ignore single line comments

2 files changed, 31 insertions, 4 deletions

diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
index 929b516..8e70291 100644
--- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
+++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
@@ -19,6 +19,8 @@ class Tokenizer {
         while (!iterator.isEnd()) {
             if (trySkipWhitespace(iterator)) {
                 // Skip whitespace.
+            } else if (trySkipComment(iterator)) {
+                // Skip comments.
             } else {
                 var token = tokenizeToken(iterator);
                 if (token.isPresent()) {
@@ -38,6 +40,15 @@ class Tokenizer {
         return new Tokens(tokens, tokenEnd);
     }
 
+    private static boolean trySkipComment(CharacterIterator iterator) {
+        if (!"//".contentEquals(iterator.peekSequence(2)) || "///".contentEquals(iterator.peekSequence(3))) {
+            return false;
+        }
+
+        skipToEndOfLine(iterator);
+        return true;
+    }
+
     private static Optional<Token> tokenizeToken(CharacterIterator iterator) {
         var identifierOrKeyword = tokenizeIdentifierOrKeyword(iterator);
         if (identifierOrKeyword.isPresent()) {
@@ -82,16 +93,20 @@ class Tokenizer {
         }
 
         var start = iterator.position();
+        skipToEndOfLine(iterator);
+        var end = iterator.position();
+
+        var token = new Token(TokenType.DOC_COMMENT, start.to(end));
+        return Optional.of(token);
+    }
+
+    private static void skipToEndOfLine(CharacterIterator iterator) {
         while (!iterator.isEnd() && iterator.peek() != '\n') {
             iterator.skip();
         }
         if (iterator.peek() == '\n') {
             iterator.skip();
         }
-        var end = iterator.position();
-
-        var token = new Token(TokenType.DOC_COMMENT, start.to(end));
-        return Optional.of(token);
     }
 
     private static Optional<Token> tokenizeIdentifierOrKeyword(CharacterIterator iterator) {
diff --git a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
index 68b4a89..1764a41 100644
--- a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
+++ b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
@@ -219,6 +219,18 @@ public class TokenizerTests {
     }
 
     @Test
+    public void singleLineCommentsAreIgnored() {
+        var sourceText = SourceText.fromString("<string>", "one // two\nthree\n// four");
+
+        var tokens = Tokenizer.tokenize(sourceText);
+
+        assertThat(tokens.tokens(), isSequence(
+            equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 0, 3))),
+            equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 11, 16)))
+        ));
+    }
+
+    @Test
     public void docCommentRunsToEndOfLine() {
         var sourceText = SourceText.fromString("<string>", "a\n/// One\n    /// Two\nb");