summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java2
-rw-r--r--src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java4
-rw-r--r--src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java22
3 files changed, 28 insertions, 0 deletions
diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java
index 85ac1b9..f7242e2 100644
--- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java
+++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java
@@ -7,6 +7,8 @@ enum TokenType {
WHITESPACE,
KEYWORD_FIELD,
+ KEYWORD_FROM,
+ KEYWORD_IMPORT,
KEYWORD_NATIVE,
KEYWORD_STRUCT,
KEYWORD_SUM,
diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
index 8e70291..7db0806 100644
--- a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
+++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java
@@ -124,6 +124,10 @@ class Tokenizer {
TokenType tokenType;
if (CharSequence.compare(sourceRange.charSequence(), "field") == 0) {
tokenType = TokenType.KEYWORD_FIELD;
+ } else if (CharSequence.compare(sourceRange.charSequence(), "from") == 0) {
+ tokenType = TokenType.KEYWORD_FROM;
+ } else if (CharSequence.compare(sourceRange.charSequence(), "import") == 0) {
+ tokenType = TokenType.KEYWORD_IMPORT;
} else if (CharSequence.compare(sourceRange.charSequence(), "native") == 0) {
tokenType = TokenType.KEYWORD_NATIVE;
} else if (CharSequence.compare(sourceRange.charSequence(), "struct") == 0) {
diff --git a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
index 1764a41..2b624ab 100644
--- a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
+++ b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java
@@ -30,6 +30,28 @@ public class TokenizerTests {
}
@Test
+ public void keywordFromIsTokenized() {
+ var sourceText = SourceText.fromString("<string>", "from");
+
+ var tokens = Tokenizer.tokenize(sourceText);
+
+ assertThat(tokens.tokens(), isSequence(
+ equalTo(new Token(TokenType.KEYWORD_FROM, sourceRange(sourceText, 0, 4)))
+ ));
+ }
+
+ @Test
+ public void keywordImportIsTokenized() {
+ var sourceText = SourceText.fromString("<string>", "import");
+
+ var tokens = Tokenizer.tokenize(sourceText);
+
+ assertThat(tokens.tokens(), isSequence(
+ equalTo(new Token(TokenType.KEYWORD_IMPORT, sourceRange(sourceText, 0, 6)))
+ ));
+ }
+
+ @Test
public void keywordNativeIsTokenized() {
var sourceText = SourceText.fromString("<string>", "native");