Tokenize numbers

1 files changed, 33 insertions, 0 deletions

diff --git a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
index 1e7025d..b434ac0 100644
--- a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
+++ b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
@@ -4,6 +4,7 @@ import org.junit.jupiter.api.Test;
 import org.zwobble.sourcetext.SourceRange;
 import org.zwobble.sourcetext.SourceText;
 
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.zwobble.precisely.AssertThat.assertThat;
 import static org.zwobble.precisely.Matchers.equalTo;
 import static org.zwobble.precisely.Matchers.isSequence;
@@ -118,6 +119,38 @@ public class TokenizerTests {
     }
 
     @Test
+    public void digitsAreTokenizedAsNumber() {
+        var sourceText = SourceText.fromString("<string>", "42");
+
+        var tokens = Tokenizer.tokenize(sourceText);
+
+        assertThat(tokens.tokens(), isSequence(
+            equalTo(new Token(TokenType.NUMBER, sourceRange(sourceText, 0, 2)))
+        ));
+    }
+
+    @Test
+    public void digitsFollowedByNonDigitIdentifierCharacterProducesError() {
+        var sourceText = SourceText.fromString("<string>", "42a");
+
+        var error = assertThrows(ParseError.class, () -> Tokenizer.tokenize(sourceText));
+
+        assertThat(error.getMessage(), equalTo("number must contain only digits"));
+    }
+
+    @Test
+    public void digitsMayBeFollowedByNonDigitNonIdentifierCharacter() {
+        var sourceText = SourceText.fromString("<string>", "42==");
+
+        var tokens = Tokenizer.tokenize(sourceText);
+
+        assertThat(tokens.tokens(), isSequence(
+            equalTo(new Token(TokenType.NUMBER, sourceRange(sourceText, 0, 2))),
+            equalTo(new Token(TokenType.SYMBOL_EQUALS_EQUALS, sourceRange(sourceText, 2, 4)))
+        ));
+    }
+
+    @Test
     public void symbolBraceOpenIsTokenized() {
         var sourceText = SourceText.fromString("<string>", "{");