Add test for whitespace separation

1 files changed, 13 insertions, 0 deletions

diff --git a/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java
index 67ecc1e..aa6dbe4 100644
--- a/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java
+++ b/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java
@@ -51,6 +51,19 @@ public class TokenizerTests {
         ));
     }
 
+    @Test
+    public void whitespaceSeparatesOtherTokens() {
+        var sourceText = SourceText.fromString("<string>", "one two \t three");
+
+        var tokens = Tokenizer.tokenize(sourceText);
+
+        assertThat(tokens, isSequence(
+            equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 0, 3))),
+            equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 4, 7))),
+            equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 10, 15)))
+        ));
+    }
+
     private static SourceRange sourceRange(SourceText sourceText, int fromCharacterIndex, int toCharacterIndex) {
         return sourceText.characterPosition(fromCharacterIndex).to(sourceText.characterPosition(toCharacterIndex));
     }