From 9c84e3365661387be283ca528cea1650ea46ae1f Mon Sep 17 00:00:00 2001 From: Michael Williamson Date: Thu, 23 Apr 2026 11:27:29 +0100 Subject: Tokenize field keyword --- .../org/zwobble/hobgoblin/compiler/parser/TokenType.java | 1 + .../org/zwobble/hobgoblin/compiler/parser/Tokenizer.java | 4 +++- .../zwobble/hobgoblin/compiler/parser/TokenizerTests.java | 14 +++++++++++--- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java index 72b1e88..64be2d4 100644 --- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java @@ -5,6 +5,7 @@ enum TokenType { IDENTIFIER, WHITESPACE, + KEYWORD_FIELD, KEYWORD_STRUCT, SYMBOL_BRACE_OPEN, diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java index c628346..6a43a09 100644 --- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java @@ -84,7 +84,9 @@ class Tokenizer { var sourceRange = start.to(end); TokenType tokenType; - if (CharSequence.compare(sourceRange.charSequence(), "struct") == 0) { + if (CharSequence.compare(sourceRange.charSequence(), "field") == 0) { + tokenType = TokenType.KEYWORD_FIELD; + } else if (CharSequence.compare(sourceRange.charSequence(), "struct") == 0) { tokenType = TokenType.KEYWORD_STRUCT; } else { tokenType = TokenType.IDENTIFIER; diff --git a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java index e564e7b..68db18d 100644 --- a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java +++ b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java @@ -1,9 +1,6 @@ package org.zwobble.hobgoblin.compiler.parser; import org.junit.jupiter.api.Test; -import org.zwobble.hobgoblin.compiler.parser.Token; -import org.zwobble.hobgoblin.compiler.parser.TokenType; -import org.zwobble.hobgoblin.compiler.parser.Tokenizer; import org.zwobble.sourcetext.SourceRange; import org.zwobble.sourcetext.SourceText; @@ -21,6 +18,17 @@ public class TokenizerTests { assertThat(tokens.tokens(), isSequence()); } + @Test + public void keywordFieldIsTokenized() { + var sourceText = SourceText.fromString("", "field"); + + var tokens = Tokenizer.tokenize(sourceText); + + assertThat(tokens.tokens(), isSequence( + equalTo(new Token(TokenType.KEYWORD_FIELD, sourceRange(sourceText, 0, 5))) + )); + } + @Test public void keywordStructIsTokenized() { var sourceText = SourceText.fromString("", "struct"); -- cgit v1.2.3