Allow digits in identifiers

2 files changed, 13 insertions, 2 deletions

diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
index 6fc5e20..fff1977 100644
--- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
+++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
@@ -79,7 +79,7 @@ class Tokenizer {
         var start = iterator.position();
         do {
             iterator.skip();
-        } while (Character.isAlphabetic(iterator.peek()));
+        } while (Character.isAlphabetic(iterator.peek()) || Character.isDigit(iterator.peek()));
         var end = iterator.position();
 
         var sourceRange = start.to(end);
diff --git a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
index 7d4ad35..ac10214 100644
--- a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
+++ b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
@@ -41,7 +41,7 @@ public class TokenizerTests {
     }
 
     @Test
-    public void identifierIsTokenized() {
+    public void alphabeticIdentifierIsTokenized() {
         var sourceText = SourceText.fromString("<string>", "blah");
 
         var tokens = Tokenizer.tokenize(sourceText);
@@ -52,6 +52,17 @@ public class TokenizerTests {
     }
 
     @Test
+    public void digitsCanBePartOfIdentifier() {
+        var sourceText = SourceText.fromString("<string>", "blah32");
+
+        var tokens = Tokenizer.tokenize(sourceText);
+
+        assertThat(tokens.tokens(), isSequence(
+            equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 0, 6)))
+        ));
+    }
+
+    @Test
     public void symbolBraceOpenIsTokenized() {
         var sourceText = SourceText.fromString("<string>", "{");