summaryrefslogtreecommitdiff
path: root/src/main/java
diff options
context:
space:
mode:
authorMichael Williamson <mike@zwobble.org>2026-11-06 15:38:59 +0000
committerMichael Williamson <mike@zwobble.org>2026-11-06 15:38:59 +0000
commitf81f0b737c95b9ba97f6307cf4c00a342b6bad02 (patch)
treeefb53fae5172bed7dcebdd074af8b68474c049c2 /src/main/java
parentbd709df8618e7c848879599c07000c36d8134a11 (diff)
Tokenize numbers
Diffstat (limited to 'src/main/java')
-rw-r--r--src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java1
-rw-r--r--src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java29
2 files changed, 30 insertions, 0 deletions
diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java
index 12626cc..5896d6f 100644
--- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java
+++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java
@@ -4,6 +4,7 @@ enum TokenType {
DOC_COMMENT,
END,
IDENTIFIER,
+ NUMBER,
WHITESPACE,
KEYWORD_ENUM,
diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
index 63973a6..b36ca12 100644
--- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
+++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
@@ -55,6 +55,11 @@ class Tokenizer {
return identifierOrKeyword;
}
+ var number = tokenizeNumber(iterator);
+ if (number.isPresent()) {
+ return number;
+ }
+
var docComment = tokenizeDocComment(iterator);
if (docComment.isPresent()) {
return docComment;
@@ -144,6 +149,30 @@ class Tokenizer {
return Optional.of(token);
}
+ private static Optional<Token> tokenizeNumber(CharacterIterator iterator) {
+ var firstChar = iterator.peek();
+ if (!(firstChar >= '1' && firstChar <= '9')) {
+ return Optional.empty();
+ }
+
+ var start = iterator.position();
+
+ while (true) {
+ var nextChar = iterator.peek();
+ if (!(nextChar >= '0' && nextChar <= '9')) {
+ if (Character.isAlphabetic(nextChar)) {
+ throw new ParseError("number must contain only digits", iterator.characterSourceRange());
+ }
+
+ var end = iterator.position();
+ var token = new Token(TokenType.NUMBER, start.to(end));
+ return Optional.of(token);
+ }
+
+ iterator.skip();
+ }
+ }
+
private static Optional<Token> tokenizeSymbol(CharacterIterator iterator) {
TokenType tokenType;
var start = iterator.position();