From acda00f5659990a23114f9b006f52498f01b25a4 Mon Sep 17 00:00:00 2001 From: Michael Williamson Date: Sat, 18 Apr 2026 13:29:07 +0100 Subject: Add test for whitespace separation --- .../java/org/zwobble/hobgoblin/parser/TokenizerTests.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'src/test/java/org/zwobble') diff --git a/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java index 67ecc1e..aa6dbe4 100644 --- a/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java +++ b/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java @@ -51,6 +51,19 @@ public class TokenizerTests { )); } + @Test + public void whitespaceSeparatesOtherTokens() { + var sourceText = SourceText.fromString("", "one two \t three"); + + var tokens = Tokenizer.tokenize(sourceText); + + assertThat(tokens, isSequence( + equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 0, 3))), + equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 4, 7))), + equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 10, 15))) + )); + } + private static SourceRange sourceRange(SourceText sourceText, int fromCharacterIndex, int toCharacterIndex) { return sourceText.characterPosition(fromCharacterIndex).to(sourceText.characterPosition(toCharacterIndex)); } -- cgit v1.2.3