Skip whitespace

1 files changed, 11 insertions, 0 deletions

diff --git a/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java
index 52c5f79..67ecc1e 100644
--- a/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java
+++ b/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java
@@ -40,6 +40,17 @@ public class TokenizerTests {
         ));
     }
 
+    @Test
+    public void whitespaceIsIgnored() {
+        var sourceText = SourceText.fromString("<string>", "  blah  ");
+
+        var tokens = Tokenizer.tokenize(sourceText);
+
+        assertThat(tokens, isSequence(
+            equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 2, 6)))
+        ));
+    }
+
     private static SourceRange sourceRange(SourceText sourceText, int fromCharacterIndex, int toCharacterIndex) {
         return sourceText.characterPosition(fromCharacterIndex).to(sourceText.characterPosition(toCharacterIndex));
     }