diff options
3 files changed, 28 insertions, 0 deletions
diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java index 85ac1b9..f7242e2 100644 --- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java @@ -7,6 +7,8 @@ enum TokenType { WHITESPACE, KEYWORD_FIELD, + KEYWORD_FROM, + KEYWORD_IMPORT, KEYWORD_NATIVE, KEYWORD_STRUCT, KEYWORD_SUM, diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java index 8e70291..7db0806 100644 --- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java @@ -124,6 +124,10 @@ class Tokenizer { TokenType tokenType; if (CharSequence.compare(sourceRange.charSequence(), "field") == 0) { tokenType = TokenType.KEYWORD_FIELD; + } else if (CharSequence.compare(sourceRange.charSequence(), "from") == 0) { + tokenType = TokenType.KEYWORD_FROM; + } else if (CharSequence.compare(sourceRange.charSequence(), "import") == 0) { + tokenType = TokenType.KEYWORD_IMPORT; } else if (CharSequence.compare(sourceRange.charSequence(), "native") == 0) { tokenType = TokenType.KEYWORD_NATIVE; } else if (CharSequence.compare(sourceRange.charSequence(), "struct") == 0) { diff --git a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java index 1764a41..2b624ab 100644 --- a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java +++ b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java @@ -30,6 +30,28 @@ public class TokenizerTests { } @Test + public void keywordFromIsTokenized() { + var sourceText = SourceText.fromString("<string>", "from"); + + var tokens = Tokenizer.tokenize(sourceText); + + assertThat(tokens.tokens(), isSequence( + equalTo(new Token(TokenType.KEYWORD_FROM, sourceRange(sourceText, 0, 4))) + )); + } + + @Test + public void keywordImportIsTokenized() { + var sourceText = SourceText.fromString("<string>", "import"); + + var tokens = Tokenizer.tokenize(sourceText); + + assertThat(tokens.tokens(), isSequence( + equalTo(new Token(TokenType.KEYWORD_IMPORT, sourceRange(sourceText, 0, 6))) + )); + } + + @Test public void keywordNativeIsTokenized() { var sourceText = SourceText.fromString("<string>", "native"); |