Skip whitespace

2 files changed, 38 insertions, 8 deletions

diff --git a/src/main/java/org/zwobble/hobgoblin/parser/Tokenizer.java b/src/main/java/org/zwobble/hobgoblin/parser/Tokenizer.java
index f4e0773..e616ec0 100644
--- a/src/main/java/org/zwobble/hobgoblin/parser/Tokenizer.java
+++ b/src/main/java/org/zwobble/hobgoblin/parser/Tokenizer.java
@@ -18,21 +18,40 @@ class Tokenizer {
         var tokens = new ArrayList<Token>();
 
         while (!iterator.isEnd()) {
-            var identifierOrKeyword = tokenizeIdentifierOrKeyword(iterator);
-            if (identifierOrKeyword.isPresent()) {
-                tokens.add(identifierOrKeyword.get());
+            if (trySkipWhitespace(iterator)) {
+                // Skip whitespace.
             } else {
-                throw ParseError.unexpectedTextError(
-                    "token",
-                    describeCharacter(iterator.peek()),
-                    iterator.characterSourceRange()
-                );
+                var identifierOrKeyword = tokenizeIdentifierOrKeyword(iterator);
+                if (identifierOrKeyword.isPresent()) {
+                    tokens.add(identifierOrKeyword.get());
+                } else {
+                    throw ParseError.unexpectedTextError(
+                        "token",
+                        describeCharacter(iterator.peek()),
+                        iterator.characterSourceRange()
+                    );
+                }
             }
         }
 
         return tokens;
     }
 
+    private static boolean trySkipWhitespace(CharacterIterator iterator) {
+        var whitespace = false;
+
+        while (isWhitespace(iterator.peek())) {
+            iterator.skip();
+            whitespace = true;
+        }
+
+        return whitespace;
+    }
+
+    private static boolean isWhitespace(int character) {
+        return Character.isWhitespace(character);
+    }
+
     private static Optional<Token> tokenizeIdentifierOrKeyword(CharacterIterator iterator) {
         if (!Character.isAlphabetic(iterator.peek())) {
             return Optional.empty();
diff --git a/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java
index 52c5f79..67ecc1e 100644
--- a/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java
+++ b/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java
@@ -40,6 +40,17 @@ public class TokenizerTests {
         ));
     }
 
+    @Test
+    public void whitespaceIsIgnored() {
+        var sourceText = SourceText.fromString("<string>", "  blah  ");
+
+        var tokens = Tokenizer.tokenize(sourceText);
+
+        assertThat(tokens, isSequence(
+            equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 2, 6)))
+        ));
+    }
+
     private static SourceRange sourceRange(SourceText sourceText, int fromCharacterIndex, int toCharacterIndex) {
         return sourceText.characterPosition(fromCharacterIndex).to(sourceText.characterPosition(toCharacterIndex));
     }