Skip whitespace

1 files changed, 27 insertions, 8 deletions

diff --git a/src/main/java/org/zwobble/hobgoblin/parser/Tokenizer.java b/src/main/java/org/zwobble/hobgoblin/parser/Tokenizer.java
index f4e0773..e616ec0 100644
--- a/src/main/java/org/zwobble/hobgoblin/parser/Tokenizer.java
+++ b/src/main/java/org/zwobble/hobgoblin/parser/Tokenizer.java
@@ -18,21 +18,40 @@ class Tokenizer {
         var tokens = new ArrayList<Token>();
 
         while (!iterator.isEnd()) {
-            var identifierOrKeyword = tokenizeIdentifierOrKeyword(iterator);
-            if (identifierOrKeyword.isPresent()) {
-                tokens.add(identifierOrKeyword.get());
+            if (trySkipWhitespace(iterator)) {
+                // Skip whitespace.
             } else {
-                throw ParseError.unexpectedTextError(
-                    "token",
-                    describeCharacter(iterator.peek()),
-                    iterator.characterSourceRange()
-                );
+                var identifierOrKeyword = tokenizeIdentifierOrKeyword(iterator);
+                if (identifierOrKeyword.isPresent()) {
+                    tokens.add(identifierOrKeyword.get());
+                } else {
+                    throw ParseError.unexpectedTextError(
+                        "token",
+                        describeCharacter(iterator.peek()),
+                        iterator.characterSourceRange()
+                    );
+                }
             }
         }
 
         return tokens;
     }
 
+    private static boolean trySkipWhitespace(CharacterIterator iterator) {
+        var whitespace = false;
+
+        while (isWhitespace(iterator.peek())) {
+            iterator.skip();
+            whitespace = true;
+        }
+
+        return whitespace;
+    }
+
+    private static boolean isWhitespace(int character) {
+        return Character.isWhitespace(character);
+    }
+
     private static Optional<Token> tokenizeIdentifierOrKeyword(CharacterIterator iterator) {
         if (!Character.isAlphabetic(iterator.peek())) {
             return Optional.empty();