diff options
| author | Michael Williamson <mike@zwobble.org> | 2026-11-06 15:38:59 +0000 |
|---|---|---|
| committer | Michael Williamson <mike@zwobble.org> | 2026-11-06 15:38:59 +0000 |
| commit | f81f0b737c95b9ba97f6307cf4c00a342b6bad02 (patch) | |
| tree | efb53fae5172bed7dcebdd074af8b68474c049c2 /src/main/java | |
| parent | bd709df8618e7c848879599c07000c36d8134a11 (diff) | |
Tokenize numbers
Diffstat (limited to 'src/main/java')
| -rw-r--r-- | src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java | 1 | ||||
| -rw-r--r-- | src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java | 29 |
2 files changed, 30 insertions, 0 deletions
diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java index 12626cc..5896d6f 100644 --- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java @@ -4,6 +4,7 @@ enum TokenType { DOC_COMMENT, END, IDENTIFIER, + NUMBER, WHITESPACE, KEYWORD_ENUM, diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java index 63973a6..b36ca12 100644 --- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java @@ -55,6 +55,11 @@ class Tokenizer { return identifierOrKeyword; } + var number = tokenizeNumber(iterator); + if (number.isPresent()) { + return number; + } + var docComment = tokenizeDocComment(iterator); if (docComment.isPresent()) { return docComment; @@ -144,6 +149,30 @@ class Tokenizer { return Optional.of(token); } + private static Optional<Token> tokenizeNumber(CharacterIterator iterator) { + var firstChar = iterator.peek(); + if (!(firstChar >= '1' && firstChar <= '9')) { + return Optional.empty(); + } + + var start = iterator.position(); + + while (true) { + var nextChar = iterator.peek(); + if (!(nextChar >= '0' && nextChar <= '9')) { + if (Character.isAlphabetic(nextChar)) { + throw new ParseError("number must contain only digits", iterator.characterSourceRange()); + } + + var end = iterator.position(); + var token = new Token(TokenType.NUMBER, start.to(end)); + return Optional.of(token); + } + + iterator.skip(); + } + } + private static Optional<Token> tokenizeSymbol(CharacterIterator iterator) { TokenType tokenType; var start = iterator.position(); |
