Add initial tokenizer

8 files changed, 213 insertions, 0 deletions

diff --git a/src/main/java/org/zwobble/hobgoblin/ast/untyped/UntypedNamespaceNode.java b/src/main/java/org/zwobble/hobgoblin/ast/untyped/UntypedNamespaceNode.java
new file mode 100644
index 0000000..bbbbd8a
--- /dev/null
+++ b/src/main/java/org/zwobble/hobgoblin/ast/untyped/UntypedNamespaceNode.java
@@ -0,0 +1,4 @@
+package org.zwobble.hobgoblin.ast.untyped;
+
+public class UntypedNamespaceNode {
+}
diff --git a/src/main/java/org/zwobble/hobgoblin/parser/ParseError.java b/src/main/java/org/zwobble/hobgoblin/parser/ParseError.java
new file mode 100644
index 0000000..27fa8c5
--- /dev/null
+++ b/src/main/java/org/zwobble/hobgoblin/parser/ParseError.java
@@ -0,0 +1,29 @@
+package org.zwobble.hobgoblin.parser;
+
+import org.zwobble.sourcetext.SourceRange;
+
+public class ParseError extends RuntimeException {
+    public static ParseError unexpectedTextError(
+        String expected,
+        String actual,
+        SourceRange sourceRange
+    ) {
+        var message = String.format(
+            "Expected %s, but was %s",
+            expected,
+            actual
+        );
+        return new ParseError(message, sourceRange);
+    }
+
+    private final SourceRange sourceRange;
+
+    public ParseError(String message, SourceRange sourceRange) {
+        super(message);
+        this.sourceRange = sourceRange;
+    }
+
+    public SourceRange sourceRange() {
+        return sourceRange;
+    }
+}
diff --git a/src/main/java/org/zwobble/hobgoblin/parser/Parser.java b/src/main/java/org/zwobble/hobgoblin/parser/Parser.java
new file mode 100644
index 0000000..baa769e
--- /dev/null
+++ b/src/main/java/org/zwobble/hobgoblin/parser/Parser.java
@@ -0,0 +1,12 @@
+package org.zwobble.hobgoblin.parser;
+
+import org.zwobble.hobgoblin.ast.untyped.UntypedNamespaceNode;
+
+public class Parser {
+    private Parser() {
+    }
+
+    public static UntypedNamespaceNode parseNamespace() {
+        return new UntypedNamespaceNode();
+    }
+}
diff --git a/src/main/java/org/zwobble/hobgoblin/parser/Token.java b/src/main/java/org/zwobble/hobgoblin/parser/Token.java
new file mode 100644
index 0000000..ca25fa7
--- /dev/null
+++ b/src/main/java/org/zwobble/hobgoblin/parser/Token.java
@@ -0,0 +1,6 @@
+package org.zwobble.hobgoblin.parser;
+
+import org.zwobble.sourcetext.SourceRange;
+
+record Token(TokenType tokenType, SourceRange sourceRange) {
+}
diff --git a/src/main/java/org/zwobble/hobgoblin/parser/TokenIterator.java b/src/main/java/org/zwobble/hobgoblin/parser/TokenIterator.java
new file mode 100644
index 0000000..e1f4ae2
--- /dev/null
+++ b/src/main/java/org/zwobble/hobgoblin/parser/TokenIterator.java
@@ -0,0 +1,9 @@
+package org.zwobble.hobgoblin.parser;
+
+import java.util.List;
+
+class TokenIterator {
+    TokenIterator(List<Token> tokens, Token tokenEnd) {
+
+    }
+}
diff --git a/src/main/java/org/zwobble/hobgoblin/parser/TokenType.java b/src/main/java/org/zwobble/hobgoblin/parser/TokenType.java
new file mode 100644
index 0000000..100ee77
--- /dev/null
+++ b/src/main/java/org/zwobble/hobgoblin/parser/TokenType.java
@@ -0,0 +1,12 @@
+package org.zwobble.hobgoblin.parser;
+
+enum TokenType {
+    END,
+    IDENTIFIER,
+    WHITESPACE,
+
+    KEYWORD_STRUCT,
+
+    SYMBOL_BRACE_OPEN,
+    SYMBOL_BRACE_CLOSE,
+}
diff --git a/src/main/java/org/zwobble/hobgoblin/parser/Tokenizer.java b/src/main/java/org/zwobble/hobgoblin/parser/Tokenizer.java
new file mode 100644
index 0000000..f4e0773
--- /dev/null
+++ b/src/main/java/org/zwobble/hobgoblin/parser/Tokenizer.java
@@ -0,0 +1,95 @@
+package org.zwobble.hobgoblin.parser;
+
+import org.zwobble.sourcetext.SourceCharacterIterator;
+import org.zwobble.sourcetext.SourcePosition;
+import org.zwobble.sourcetext.SourceRange;
+import org.zwobble.sourcetext.SourceText;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+class Tokenizer {
+    private Tokenizer() {
+    }
+
+    static List<Token> tokenize(SourceText sourceText) {
+        var iterator = new CharacterIterator(sourceText);
+        var tokens = new ArrayList<Token>();
+
+        while (!iterator.isEnd()) {
+            var identifierOrKeyword = tokenizeIdentifierOrKeyword(iterator);
+            if (identifierOrKeyword.isPresent()) {
+                tokens.add(identifierOrKeyword.get());
+            } else {
+                throw ParseError.unexpectedTextError(
+                    "token",
+                    describeCharacter(iterator.peek()),
+                    iterator.characterSourceRange()
+                );
+            }
+        }
+
+        return tokens;
+    }
+
+    private static Optional<Token> tokenizeIdentifierOrKeyword(CharacterIterator iterator) {
+        if (!Character.isAlphabetic(iterator.peek())) {
+            return Optional.empty();
+        }
+
+        var start = iterator.position();
+        do {
+            iterator.skip();
+        } while (Character.isAlphabetic(iterator.peek()));
+        var end = iterator.position();
+
+        var sourceRange = start.to(end);
+        TokenType tokenType;
+        if (CharSequence.compare(sourceRange.charSequence(), "struct") == 0) {
+            tokenType = TokenType.KEYWORD_STRUCT;
+        } else {
+            tokenType = TokenType.IDENTIFIER;
+        }
+        var token = new Token(tokenType, start.to(end));
+
+        return Optional.of(token);
+    }
+
+    private static String describeCharacter(int character) {
+        // TODO: handle characters that should be escaped
+        if (character == -1) {
+            return "end of document";
+        } else {
+            return String.format("'%s'", new String(new int[]{character}, 0, 1));
+        }
+    }
+
+    private static class CharacterIterator {
+        private final SourceCharacterIterator iterator;
+
+        CharacterIterator(SourceText sourceText) {
+            this.iterator = sourceText.characterIterator();
+        }
+
+        boolean isEnd() {
+            return this.iterator.isEnd();
+        }
+
+        SourceRange characterSourceRange() {
+            return this.iterator.characterSourceRange();
+        }
+
+        int peek() {
+            return this.iterator.peek();
+        }
+
+        void skip() {
+            this.iterator.skip();
+        }
+
+        public SourcePosition position() {
+            return this.iterator.position();
+        }
+    }
+}
diff --git a/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java
new file mode 100644
index 0000000..52c5f79
--- /dev/null
+++ b/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java
@@ -0,0 +1,46 @@
+package org.zwobble.hobgoblin.parser;
+
+import org.junit.jupiter.api.Test;
+import org.zwobble.sourcetext.SourceRange;
+import org.zwobble.sourcetext.SourceText;
+
+import static org.zwobble.precisely.AssertThat.assertThat;
+import static org.zwobble.precisely.Matchers.equalTo;
+import static org.zwobble.precisely.Matchers.isSequence;
+
+public class TokenizerTests {
+    @Test
+    public void emptySourceTextIsTokenizedToEmpty() {
+        var sourceText = SourceText.fromString("<string>", "");
+
+        var tokens = Tokenizer.tokenize(sourceText);
+
+        assertThat(tokens, isSequence());
+    }
+
+    @Test
+    public void structKeywordIsTokenized() {
+        var sourceText = SourceText.fromString("<string>", "struct");
+
+        var tokens = Tokenizer.tokenize(sourceText);
+
+        assertThat(tokens, isSequence(
+            equalTo(new Token(TokenType.KEYWORD_STRUCT, sourceRange(sourceText, 0, 6)))
+        ));
+    }
+
+    @Test
+    public void identifierIsTokenized() {
+        var sourceText = SourceText.fromString("<string>", "blah");
+
+        var tokens = Tokenizer.tokenize(sourceText);
+
+        assertThat(tokens, isSequence(
+            equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 0, 4)))
+        ));
+    }
+
+    private static SourceRange sourceRange(SourceText sourceText, int fromCharacterIndex, int toCharacterIndex) {
+        return sourceText.characterPosition(fromCharacterIndex).to(sourceText.characterPosition(toCharacterIndex));
+    }
+}