From fe5c62948545b21ca4af1089a1eb33a70899086c Mon Sep 17 00:00:00 2001
From: Michael Williamson <mike@zwobble.org>
Date: Tue, 6 Jun 2023 18:52:39 +0100
Subject: Add stubbed parser

---
 .../compiler/ast/UntypedNamespaceNode.java         |  8 +++++
 .../org/shedlang/compiler/ast/UntypedNode.java     |  7 ++++
 .../java/org/shedlang/compiler/parser/Parser.java  | 17 ++++++++++
 .../org/shedlang/compiler/parser/TokenType.java    | 10 ++++++
 .../org/shedlang/compiler/parser/Tokeniser.java    | 39 ++++++++++++++++++++++
 .../compiler/sources/FileFragmentSource.java       | 18 +++++++++-
 .../org/shedlang/compiler/sources/NullSource.java  | 15 +++++++++
 .../java/org/shedlang/compiler/sources/Source.java |  3 ++
 .../shedlang/compiler/tokeniser/TokenIterator.java |  6 ++++
 9 files changed, 122 insertions(+), 1 deletion(-)
 create mode 100644 src/main/java/org/shedlang/compiler/ast/UntypedNamespaceNode.java
 create mode 100644 src/main/java/org/shedlang/compiler/ast/UntypedNode.java
 create mode 100644 src/main/java/org/shedlang/compiler/parser/Parser.java
 create mode 100644 src/main/java/org/shedlang/compiler/parser/TokenType.java
 create mode 100644 src/main/java/org/shedlang/compiler/parser/Tokeniser.java

(limited to 'src/main/java')

diff --git a/src/main/java/org/shedlang/compiler/ast/UntypedNamespaceNode.java b/src/main/java/org/shedlang/compiler/ast/UntypedNamespaceNode.java
new file mode 100644
index 00000000..055e1a9e
--- /dev/null
+++ b/src/main/java/org/shedlang/compiler/ast/UntypedNamespaceNode.java
@@ -0,0 +1,8 @@
+package org.shedlang.compiler.ast;
+
+import org.shedlang.compiler.sources.Source;
+
+public record UntypedNamespaceNode(
+    Source source
+) implements UntypedNode {
+}
diff --git a/src/main/java/org/shedlang/compiler/ast/UntypedNode.java b/src/main/java/org/shedlang/compiler/ast/UntypedNode.java
new file mode 100644
index 00000000..4b6e3f2f
--- /dev/null
+++ b/src/main/java/org/shedlang/compiler/ast/UntypedNode.java
@@ -0,0 +1,7 @@
+package org.shedlang.compiler.ast;
+
+import org.shedlang.compiler.sources.Source;
+
+public interface UntypedNode {
+    Source source();
+}
diff --git a/src/main/java/org/shedlang/compiler/parser/Parser.java b/src/main/java/org/shedlang/compiler/parser/Parser.java
new file mode 100644
index 00000000..79a81576
--- /dev/null
+++ b/src/main/java/org/shedlang/compiler/parser/Parser.java
@@ -0,0 +1,17 @@
+package org.shedlang.compiler.parser;
+
+import org.shedlang.compiler.ast.UntypedNamespaceNode;
+import org.shedlang.compiler.tokeniser.TokenIterator;
+
+public class Parser {
+    private Parser() {
+    }
+
+    static UntypedNamespaceNode parseNamespace(TokenIterator<TokenType> tokens) {
+        var start = tokens.position();
+        var end = tokens.position();
+
+        var source = start.to(end);
+        return new UntypedNamespaceNode(source);
+    }
+}
diff --git a/src/main/java/org/shedlang/compiler/parser/TokenType.java b/src/main/java/org/shedlang/compiler/parser/TokenType.java
new file mode 100644
index 00000000..7d438ef1
--- /dev/null
+++ b/src/main/java/org/shedlang/compiler/parser/TokenType.java
@@ -0,0 +1,10 @@
+package org.shedlang.compiler.parser;
+
+public enum TokenType {
+    END,
+    IDENTIFIER,
+    UNKNOWN,
+    WHITESPACE,
+
+    KEYWORD_FUN,
+}
diff --git a/src/main/java/org/shedlang/compiler/parser/Tokeniser.java b/src/main/java/org/shedlang/compiler/parser/Tokeniser.java
new file mode 100644
index 00000000..78216a7c
--- /dev/null
+++ b/src/main/java/org/shedlang/compiler/parser/Tokeniser.java
@@ -0,0 +1,39 @@
+package org.shedlang.compiler.parser;
+
+import org.shedlang.compiler.sources.Source;
+import org.shedlang.compiler.tokeniser.RegexTokeniser;
+import org.shedlang.compiler.tokeniser.Token;
+import org.shedlang.compiler.tokeniser.TokenIterator;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.shedlang.compiler.tokeniser.Token.token;
+
+public class Tokeniser {
+    private static final Map<String, TokenType> KEYWORDS = Map.ofEntries(
+        Map.entry("fun", TokenType.KEYWORD_FUN)
+    );
+
+    private static final RegexTokeniser<TokenType> tokeniser = new RegexTokeniser<>(
+        TokenType.UNKNOWN,
+        List.of(
+            RegexTokeniser.rule(TokenType.IDENTIFIER, "[A-Za-z][A-Za-z0-9]*"),
+            RegexTokeniser.rule(TokenType.WHITESPACE, "[\r\n\t ]+")
+        )
+    );
+
+    public static TokenIterator<TokenType> tokenise(Source source) {
+        var tokens = tokeniser.tokenise(source).stream()
+            .filter(token -> token.tokenType() != TokenType.WHITESPACE)
+            .map(token ->
+                token.tokenType() == TokenType.IDENTIFIER
+                    ? new Token<>(token.source(), KEYWORDS.getOrDefault(token.value(), token.tokenType()), token.value())
+                    : token
+            )
+            .collect(Collectors.toList());
+
+        return new TokenIterator<>(tokens, token(source.end(), TokenType.END, ""));
+    }
+}
diff --git a/src/main/java/org/shedlang/compiler/sources/FileFragmentSource.java b/src/main/java/org/shedlang/compiler/sources/FileFragmentSource.java
index 4fc8889a..6cf3a42e 100644
--- a/src/main/java/org/shedlang/compiler/sources/FileFragmentSource.java
+++ b/src/main/java/org/shedlang/compiler/sources/FileFragmentSource.java
@@ -41,8 +41,24 @@ record FileFragmentSource(
         );
     }
 
+    @Override
+    public Source to(Source end) {
+        if (end instanceof FileFragmentSource endFileFragment) {
+            return new FileFragmentSource(filename, contents, this.characterIndexStart, endFileFragment.characterIndexEnd);
+        } else {
+            throw new IllegalArgumentException("end must be an instance of FileFragmentSource");
+        }
+    }
+
+    @Override
+    public Source start() {
+        return at(0, 0);
+    }
+
+    @Override
     public Source end() {
-        return at(contents.length(), contents.length());
+        var length = characterIndexEnd - characterIndexStart;
+        return at(length, length);
     }
 
     private String context(String line, int lineIndex, int columnIndex) {
diff --git a/src/main/java/org/shedlang/compiler/sources/NullSource.java b/src/main/java/org/shedlang/compiler/sources/NullSource.java
index e4f0948f..2f09702c 100644
--- a/src/main/java/org/shedlang/compiler/sources/NullSource.java
+++ b/src/main/java/org/shedlang/compiler/sources/NullSource.java
@@ -17,8 +17,23 @@ class NullSource implements Source {
         return "(null)";
     }
 
+    @Override
+    public Source start() {
+        return this;
+    }
+
+    @Override
+    public Source end() {
+        return this;
+    }
+
     @Override
     public Source at(int characterIndexStart, int characterIndexEnd) {
         return this;
     }
+
+    @Override
+    public Source to(Source end) {
+        return null;
+    }
 }
diff --git a/src/main/java/org/shedlang/compiler/sources/Source.java b/src/main/java/org/shedlang/compiler/sources/Source.java
index b40f3b63..8dc1733c 100644
--- a/src/main/java/org/shedlang/compiler/sources/Source.java
+++ b/src/main/java/org/shedlang/compiler/sources/Source.java
@@ -3,5 +3,8 @@ package org.shedlang.compiler.sources;
 public interface Source {
     String contents();
     String describe();
+    Source start();
+    Source end();
     Source at(int characterIndexStart, int characterIndexEnd);
+    Source to(Source end);
 }
diff --git a/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java b/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java
index 4133cc01..c6181a1f 100644
--- a/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java
+++ b/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java
@@ -1,5 +1,7 @@
 package org.shedlang.compiler.tokeniser;
 
+import org.shedlang.compiler.sources.Source;
+
 import java.util.List;
 
 public class TokenIterator<TTokenType> {
@@ -48,4 +50,8 @@ public class TokenIterator<TTokenType> {
     public Token<TTokenType> peek() {
         return tokens.get();
     }
+
+    public Source position() {
+        return tokens.get().source().start();
+    }
 }
-- 
cgit v1.2.3