aboutsummaryrefslogtreecommitdiff
path: root/src/main/java
diff options
context:
space:
mode:
authorMichael Williamson <mike@zwobble.org>2023-06-06 19:10:26 +0100
committerMichael Williamson <mike@zwobble.org>2023-06-06 19:10:26 +0100
commit24019585814742baa6c8bee85b8373be319fbaa8 (patch)
tree38faac50c0ff6a5d29e9bbb81da98bfaa518539f /src/main/java
parentfe5c62948545b21ca4af1089a1eb33a70899086c (diff)
Parse string literals
Diffstat (limited to 'src/main/java')
-rw-r--r--src/main/java/org/shedlang/compiler/ast/UntypedExpressionNode.java5
-rw-r--r--src/main/java/org/shedlang/compiler/ast/UntypedStringLiteralNode.java6
-rw-r--r--src/main/java/org/shedlang/compiler/parser/Parser.java54
-rw-r--r--src/main/java/org/shedlang/compiler/parser/TokenType.java1
-rw-r--r--src/main/java/org/shedlang/compiler/parser/Tokeniser.java1
-rw-r--r--src/main/java/org/shedlang/compiler/parser/UnrecognisedEscapeSequenceError.java10
-rw-r--r--src/main/java/org/shedlang/compiler/sources/FileFragmentSource.java2
-rw-r--r--src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java3
-rw-r--r--src/main/java/org/shedlang/compiler/tokeniser/UnexpectedTokenException.java13
9 files changed, 86 insertions, 9 deletions
diff --git a/src/main/java/org/shedlang/compiler/ast/UntypedExpressionNode.java b/src/main/java/org/shedlang/compiler/ast/UntypedExpressionNode.java
new file mode 100644
index 00000000..03988b0d
--- /dev/null
+++ b/src/main/java/org/shedlang/compiler/ast/UntypedExpressionNode.java
@@ -0,0 +1,5 @@
+package org.shedlang.compiler.ast;
+
+public sealed interface UntypedExpressionNode extends UntypedNode
+ permits UntypedStringLiteralNode {
+}
diff --git a/src/main/java/org/shedlang/compiler/ast/UntypedStringLiteralNode.java b/src/main/java/org/shedlang/compiler/ast/UntypedStringLiteralNode.java
new file mode 100644
index 00000000..b8fd181b
--- /dev/null
+++ b/src/main/java/org/shedlang/compiler/ast/UntypedStringLiteralNode.java
@@ -0,0 +1,6 @@
+package org.shedlang.compiler.ast;
+
+import org.shedlang.compiler.sources.Source;
+
+public record UntypedStringLiteralNode(String value, Source source) implements UntypedExpressionNode {
+}
diff --git a/src/main/java/org/shedlang/compiler/parser/Parser.java b/src/main/java/org/shedlang/compiler/parser/Parser.java
index 79a81576..0e838b28 100644
--- a/src/main/java/org/shedlang/compiler/parser/Parser.java
+++ b/src/main/java/org/shedlang/compiler/parser/Parser.java
@@ -1,12 +1,66 @@
package org.shedlang.compiler.parser;
+import org.shedlang.compiler.ast.UntypedExpressionNode;
import org.shedlang.compiler.ast.UntypedNamespaceNode;
+import org.shedlang.compiler.ast.UntypedStringLiteralNode;
+import org.shedlang.compiler.sources.Source;
import org.shedlang.compiler.tokeniser.TokenIterator;
+import org.shedlang.compiler.tokeniser.UnexpectedTokenException;
+
+import java.util.regex.Pattern;
public class Parser {
private Parser() {
}
+ private static UntypedExpressionNode parseStringLiteral(TokenIterator<TokenType> tokens) {
+ var start = tokens.position();
+
+ var tokenValue = tokens.nextValue(TokenType.STRING);
+ var escapedValue = tokenValue.substring(1, tokenValue.length() - 1);
+ var unescapedValue = unescape(escapedValue, start);
+
+ var end = tokens.position();
+ var source = start.to(end);
+
+ return new UntypedStringLiteralNode(unescapedValue, source);
+ }
+
+ private static final Pattern STRING_ESCAPE_PATTERN = Pattern.compile("\\\\(.)");
+
+ private static String unescape(String value, Source source) {
+ var matcher = STRING_ESCAPE_PATTERN.matcher(value);
+ var result = new StringBuilder();
+ var lastIndex = 0;
+ while (matcher.find()) {
+ result.append(value.subSequence(lastIndex, matcher.start()));
+ var code = matcher.group(1);
+ result.append(unescapeCharacter(code, source.at(matcher.start(), matcher.end())));
+ lastIndex = matcher.end();
+ }
+ result.append(value.subSequence(lastIndex, value.length()));
+ return result.toString();
+ }
+
+ private static char unescapeCharacter(String code, Source source) {
+ return switch (code) {
+ case "n" -> '\n';
+ case "r" -> '\r';
+ case "t" -> '\t';
+ case "\"" -> '"';
+ case "\\" -> '\\';
+ default -> throw new UnrecognisedEscapeSequenceError("\\" + code, source);
+ };
+ }
+
+ static UntypedExpressionNode parseTopLevelExpression(TokenIterator<TokenType> tokens) {
+ if (tokens.isNext(TokenType.STRING)) {
+ return parseStringLiteral(tokens);
+ } else {
+ throw new UnexpectedTokenException("expression", tokens.peek());
+ }
+ }
+
static UntypedNamespaceNode parseNamespace(TokenIterator<TokenType> tokens) {
var start = tokens.position();
var end = tokens.position();
diff --git a/src/main/java/org/shedlang/compiler/parser/TokenType.java b/src/main/java/org/shedlang/compiler/parser/TokenType.java
index 7d438ef1..a7f56c35 100644
--- a/src/main/java/org/shedlang/compiler/parser/TokenType.java
+++ b/src/main/java/org/shedlang/compiler/parser/TokenType.java
@@ -3,6 +3,7 @@ package org.shedlang.compiler.parser;
public enum TokenType {
END,
IDENTIFIER,
+ STRING,
UNKNOWN,
WHITESPACE,
diff --git a/src/main/java/org/shedlang/compiler/parser/Tokeniser.java b/src/main/java/org/shedlang/compiler/parser/Tokeniser.java
index 78216a7c..091212df 100644
--- a/src/main/java/org/shedlang/compiler/parser/Tokeniser.java
+++ b/src/main/java/org/shedlang/compiler/parser/Tokeniser.java
@@ -20,6 +20,7 @@ public class Tokeniser {
TokenType.UNKNOWN,
List.of(
RegexTokeniser.rule(TokenType.IDENTIFIER, "[A-Za-z][A-Za-z0-9]*"),
+ RegexTokeniser.rule(TokenType.STRING, "\"(?:[^\\\\\"\n\r]|\\\\.)*\""),
RegexTokeniser.rule(TokenType.WHITESPACE, "[\r\n\t ]+")
)
);
diff --git a/src/main/java/org/shedlang/compiler/parser/UnrecognisedEscapeSequenceError.java b/src/main/java/org/shedlang/compiler/parser/UnrecognisedEscapeSequenceError.java
new file mode 100644
index 00000000..f85c77db
--- /dev/null
+++ b/src/main/java/org/shedlang/compiler/parser/UnrecognisedEscapeSequenceError.java
@@ -0,0 +1,10 @@
+package org.shedlang.compiler.parser;
+
+import org.shedlang.compiler.errors.SourceError;
+import org.shedlang.compiler.sources.Source;
+
+public class UnrecognisedEscapeSequenceError extends SourceError {
+ public UnrecognisedEscapeSequenceError(String escapeSequence, Source source) {
+ super("Unrecognised escape sequence: " + escapeSequence, source);
+ }
+}
diff --git a/src/main/java/org/shedlang/compiler/sources/FileFragmentSource.java b/src/main/java/org/shedlang/compiler/sources/FileFragmentSource.java
index 6cf3a42e..2a8e6bdf 100644
--- a/src/main/java/org/shedlang/compiler/sources/FileFragmentSource.java
+++ b/src/main/java/org/shedlang/compiler/sources/FileFragmentSource.java
@@ -1,6 +1,6 @@
package org.shedlang.compiler.sources;
-record FileFragmentSource(
+public record FileFragmentSource(
String filename,
String contents,
int characterIndexStart,
diff --git a/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java b/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java
index c6181a1f..db8d057b 100644
--- a/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java
+++ b/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java
@@ -23,8 +23,7 @@ public class TokenIterator<TTokenType> {
} else {
throw new UnexpectedTokenException(
tokenType.toString(),
- token.tokenType() + " \"" + token.value() + "\"",
- token.source()
+ token
);
}
}
diff --git a/src/main/java/org/shedlang/compiler/tokeniser/UnexpectedTokenException.java b/src/main/java/org/shedlang/compiler/tokeniser/UnexpectedTokenException.java
index 701f38e8..75fa19dc 100644
--- a/src/main/java/org/shedlang/compiler/tokeniser/UnexpectedTokenException.java
+++ b/src/main/java/org/shedlang/compiler/tokeniser/UnexpectedTokenException.java
@@ -1,18 +1,19 @@
package org.shedlang.compiler.tokeniser;
import org.shedlang.compiler.errors.SourceError;
-import org.shedlang.compiler.sources.Source;
public class UnexpectedTokenException extends SourceError {
private final String expected;
- private final String actual;
+ private final Token<?> actual;
public UnexpectedTokenException(
String expected,
- String actual,
- Source source
+ Token<?> actual
) {
- super("Expected: " + expected + "\nBut got: " + actual, source);
+ super(
+ "Expected: " + expected + "\nBut got: " + actual.tokenType() + " \"" + actual.value() + "\"",
+ actual.source()
+ );
this.expected = expected;
this.actual = actual;
}
@@ -21,7 +22,7 @@ public class UnexpectedTokenException extends SourceError {
return expected;
}
- public String getActual() {
+ public Token<?> getActual() {
return actual;
}
}