From 24019585814742baa6c8bee85b8373be319fbaa8 Mon Sep 17 00:00:00 2001 From: Michael Williamson Date: Tue, 6 Jun 2023 19:10:26 +0100 Subject: Parse string literals --- .../compiler/ast/UntypedExpressionNode.java | 5 ++ .../compiler/ast/UntypedStringLiteralNode.java | 6 +++ .../java/org/shedlang/compiler/parser/Parser.java | 54 ++++++++++++++++++++++ .../org/shedlang/compiler/parser/TokenType.java | 1 + .../org/shedlang/compiler/parser/Tokeniser.java | 1 + .../parser/UnrecognisedEscapeSequenceError.java | 10 ++++ .../compiler/sources/FileFragmentSource.java | 2 +- .../shedlang/compiler/tokeniser/TokenIterator.java | 3 +- .../tokeniser/UnexpectedTokenException.java | 13 +++--- 9 files changed, 86 insertions(+), 9 deletions(-) create mode 100644 src/main/java/org/shedlang/compiler/ast/UntypedExpressionNode.java create mode 100644 src/main/java/org/shedlang/compiler/ast/UntypedStringLiteralNode.java create mode 100644 src/main/java/org/shedlang/compiler/parser/UnrecognisedEscapeSequenceError.java (limited to 'src/main/java') diff --git a/src/main/java/org/shedlang/compiler/ast/UntypedExpressionNode.java b/src/main/java/org/shedlang/compiler/ast/UntypedExpressionNode.java new file mode 100644 index 00000000..03988b0d --- /dev/null +++ b/src/main/java/org/shedlang/compiler/ast/UntypedExpressionNode.java @@ -0,0 +1,5 @@ +package org.shedlang.compiler.ast; + +public sealed interface UntypedExpressionNode extends UntypedNode + permits UntypedStringLiteralNode { +} diff --git a/src/main/java/org/shedlang/compiler/ast/UntypedStringLiteralNode.java b/src/main/java/org/shedlang/compiler/ast/UntypedStringLiteralNode.java new file mode 100644 index 00000000..b8fd181b --- /dev/null +++ b/src/main/java/org/shedlang/compiler/ast/UntypedStringLiteralNode.java @@ -0,0 +1,6 @@ +package org.shedlang.compiler.ast; + +import org.shedlang.compiler.sources.Source; + +public record UntypedStringLiteralNode(String value, Source source) implements UntypedExpressionNode { +} diff --git a/src/main/java/org/shedlang/compiler/parser/Parser.java b/src/main/java/org/shedlang/compiler/parser/Parser.java index 79a81576..0e838b28 100644 --- a/src/main/java/org/shedlang/compiler/parser/Parser.java +++ b/src/main/java/org/shedlang/compiler/parser/Parser.java @@ -1,12 +1,66 @@ package org.shedlang.compiler.parser; +import org.shedlang.compiler.ast.UntypedExpressionNode; import org.shedlang.compiler.ast.UntypedNamespaceNode; +import org.shedlang.compiler.ast.UntypedStringLiteralNode; +import org.shedlang.compiler.sources.Source; import org.shedlang.compiler.tokeniser.TokenIterator; +import org.shedlang.compiler.tokeniser.UnexpectedTokenException; + +import java.util.regex.Pattern; public class Parser { private Parser() { } + private static UntypedExpressionNode parseStringLiteral(TokenIterator tokens) { + var start = tokens.position(); + + var tokenValue = tokens.nextValue(TokenType.STRING); + var escapedValue = tokenValue.substring(1, tokenValue.length() - 1); + var unescapedValue = unescape(escapedValue, start); + + var end = tokens.position(); + var source = start.to(end); + + return new UntypedStringLiteralNode(unescapedValue, source); + } + + private static final Pattern STRING_ESCAPE_PATTERN = Pattern.compile("\\\\(.)"); + + private static String unescape(String value, Source source) { + var matcher = STRING_ESCAPE_PATTERN.matcher(value); + var result = new StringBuilder(); + var lastIndex = 0; + while (matcher.find()) { + result.append(value.subSequence(lastIndex, matcher.start())); + var code = matcher.group(1); + result.append(unescapeCharacter(code, source.at(matcher.start(), matcher.end()))); + lastIndex = matcher.end(); + } + result.append(value.subSequence(lastIndex, value.length())); + return result.toString(); + } + + private static char unescapeCharacter(String code, Source source) { + return switch (code) { + case "n" -> '\n'; + case "r" -> '\r'; + case "t" -> '\t'; + case "\"" -> '"'; + case "\\" -> '\\'; + default -> throw new UnrecognisedEscapeSequenceError("\\" + code, source); + }; + } + + static UntypedExpressionNode parseTopLevelExpression(TokenIterator tokens) { + if (tokens.isNext(TokenType.STRING)) { + return parseStringLiteral(tokens); + } else { + throw new UnexpectedTokenException("expression", tokens.peek()); + } + } + static UntypedNamespaceNode parseNamespace(TokenIterator tokens) { var start = tokens.position(); var end = tokens.position(); diff --git a/src/main/java/org/shedlang/compiler/parser/TokenType.java b/src/main/java/org/shedlang/compiler/parser/TokenType.java index 7d438ef1..a7f56c35 100644 --- a/src/main/java/org/shedlang/compiler/parser/TokenType.java +++ b/src/main/java/org/shedlang/compiler/parser/TokenType.java @@ -3,6 +3,7 @@ package org.shedlang.compiler.parser; public enum TokenType { END, IDENTIFIER, + STRING, UNKNOWN, WHITESPACE, diff --git a/src/main/java/org/shedlang/compiler/parser/Tokeniser.java b/src/main/java/org/shedlang/compiler/parser/Tokeniser.java index 78216a7c..091212df 100644 --- a/src/main/java/org/shedlang/compiler/parser/Tokeniser.java +++ b/src/main/java/org/shedlang/compiler/parser/Tokeniser.java @@ -20,6 +20,7 @@ public class Tokeniser { TokenType.UNKNOWN, List.of( RegexTokeniser.rule(TokenType.IDENTIFIER, "[A-Za-z][A-Za-z0-9]*"), + RegexTokeniser.rule(TokenType.STRING, "\"(?:[^\\\\\"\n\r]|\\\\.)*\""), RegexTokeniser.rule(TokenType.WHITESPACE, "[\r\n\t ]+") ) ); diff --git a/src/main/java/org/shedlang/compiler/parser/UnrecognisedEscapeSequenceError.java b/src/main/java/org/shedlang/compiler/parser/UnrecognisedEscapeSequenceError.java new file mode 100644 index 00000000..f85c77db --- /dev/null +++ b/src/main/java/org/shedlang/compiler/parser/UnrecognisedEscapeSequenceError.java @@ -0,0 +1,10 @@ +package org.shedlang.compiler.parser; + +import org.shedlang.compiler.errors.SourceError; +import org.shedlang.compiler.sources.Source; + +public class UnrecognisedEscapeSequenceError extends SourceError { + public UnrecognisedEscapeSequenceError(String escapeSequence, Source source) { + super("Unrecognised escape sequence: " + escapeSequence, source); + } +} diff --git a/src/main/java/org/shedlang/compiler/sources/FileFragmentSource.java b/src/main/java/org/shedlang/compiler/sources/FileFragmentSource.java index 6cf3a42e..2a8e6bdf 100644 --- a/src/main/java/org/shedlang/compiler/sources/FileFragmentSource.java +++ b/src/main/java/org/shedlang/compiler/sources/FileFragmentSource.java @@ -1,6 +1,6 @@ package org.shedlang.compiler.sources; -record FileFragmentSource( +public record FileFragmentSource( String filename, String contents, int characterIndexStart, diff --git a/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java b/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java index c6181a1f..db8d057b 100644 --- a/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java +++ b/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java @@ -23,8 +23,7 @@ public class TokenIterator { } else { throw new UnexpectedTokenException( tokenType.toString(), - token.tokenType() + " \"" + token.value() + "\"", - token.source() + token ); } } diff --git a/src/main/java/org/shedlang/compiler/tokeniser/UnexpectedTokenException.java b/src/main/java/org/shedlang/compiler/tokeniser/UnexpectedTokenException.java index 701f38e8..75fa19dc 100644 --- a/src/main/java/org/shedlang/compiler/tokeniser/UnexpectedTokenException.java +++ b/src/main/java/org/shedlang/compiler/tokeniser/UnexpectedTokenException.java @@ -1,18 +1,19 @@ package org.shedlang.compiler.tokeniser; import org.shedlang.compiler.errors.SourceError; -import org.shedlang.compiler.sources.Source; public class UnexpectedTokenException extends SourceError { private final String expected; - private final String actual; + private final Token actual; public UnexpectedTokenException( String expected, - String actual, - Source source + Token actual ) { - super("Expected: " + expected + "\nBut got: " + actual, source); + super( + "Expected: " + expected + "\nBut got: " + actual.tokenType() + " \"" + actual.value() + "\"", + actual.source() + ); this.expected = expected; this.actual = actual; } @@ -21,7 +22,7 @@ public class UnexpectedTokenException extends SourceError { return expected; } - public String getActual() { + public Token getActual() { return actual; } } -- cgit v1.2.3