Tokenize numbers

3 files changed, 63 insertions, 0 deletions

diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java
index 12626cc..5896d6f 100644
--- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java
+++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java
@@ -4,6 +4,7 @@ enum TokenType {
     DOC_COMMENT,
     END,
     IDENTIFIER,
+    NUMBER,
     WHITESPACE,
 
     KEYWORD_ENUM,
diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
index 63973a6..b36ca12 100644
--- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
+++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
@@ -55,6 +55,11 @@ class Tokenizer {
             return identifierOrKeyword;
         }
 
+        var number = tokenizeNumber(iterator);
+        if (number.isPresent()) {
+            return number;
+        }
+
         var docComment = tokenizeDocComment(iterator);
         if (docComment.isPresent()) {
             return docComment;
@@ -144,6 +149,30 @@ class Tokenizer {
         return Optional.of(token);
     }
 
+    private static Optional<Token> tokenizeNumber(CharacterIterator iterator) {
+        var firstChar = iterator.peek();
+        if (!(firstChar >= '1' && firstChar <= '9')) {
+            return Optional.empty();
+        }
+
+        var start = iterator.position();
+
+        while (true) {
+            var nextChar = iterator.peek();
+            if (!(nextChar >= '0' && nextChar <= '9')) {
+                if (Character.isAlphabetic(nextChar)) {
+                    throw new ParseError("number must contain only digits", iterator.characterSourceRange());
+                }
+
+                var end = iterator.position();
+                var token = new Token(TokenType.NUMBER, start.to(end));
+                return Optional.of(token);
+            }
+
+            iterator.skip();
+        }
+    }
+
     private static Optional<Token> tokenizeSymbol(CharacterIterator iterator) {
         TokenType tokenType;
         var start = iterator.position();
diff --git a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
index 1e7025d..b434ac0 100644
--- a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
+++ b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
@@ -4,6 +4,7 @@ import org.junit.jupiter.api.Test;
 import org.zwobble.sourcetext.SourceRange;
 import org.zwobble.sourcetext.SourceText;
 
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.zwobble.precisely.AssertThat.assertThat;
 import static org.zwobble.precisely.Matchers.equalTo;
 import static org.zwobble.precisely.Matchers.isSequence;
@@ -118,6 +119,38 @@ public class TokenizerTests {
     }
 
     @Test
+    public void digitsAreTokenizedAsNumber() {
+        var sourceText = SourceText.fromString("<string>", "42");
+
+        var tokens = Tokenizer.tokenize(sourceText);
+
+        assertThat(tokens.tokens(), isSequence(
+            equalTo(new Token(TokenType.NUMBER, sourceRange(sourceText, 0, 2)))
+        ));
+    }
+
+    @Test
+    public void digitsFollowedByNonDigitIdentifierCharacterProducesError() {
+        var sourceText = SourceText.fromString("<string>", "42a");
+
+        var error = assertThrows(ParseError.class, () -> Tokenizer.tokenize(sourceText));
+
+        assertThat(error.getMessage(), equalTo("number must contain only digits"));
+    }
+
+    @Test
+    public void digitsMayBeFollowedByNonDigitNonIdentifierCharacter() {
+        var sourceText = SourceText.fromString("<string>", "42==");
+
+        var tokens = Tokenizer.tokenize(sourceText);
+
+        assertThat(tokens.tokens(), isSequence(
+            equalTo(new Token(TokenType.NUMBER, sourceRange(sourceText, 0, 2))),
+            equalTo(new Token(TokenType.SYMBOL_EQUALS_EQUALS, sourceRange(sourceText, 2, 4)))
+        ));
+    }
+
+    @Test
     public void symbolBraceOpenIsTokenized() {
         var sourceText = SourceText.fromString("<string>", "{");