aboutsummaryrefslogtreecommitdiff
path: root/src/main/java/org
diff options
context:
space:
mode:
authorMichael Williamson <mike@zwobble.org>2025-01-08 19:14:03 +0000
committerMichael Williamson <mike@zwobble.org>2025-01-09 20:36:42 +0000
commit3666a4af2eb35177eecd557d2a75f4bac9da75b9 (patch)
treea85230de832bc7f970dd89aaa6bb2da6ea5875f4 /src/main/java/org
parent1e6e1f7bb8776fc013f76828ed8df3f9297d095b (diff)
Use source text to implement FileFragmentSource
Diffstat (limited to 'src/main/java/org')
-rw-r--r--src/main/java/org/shedlang/compiler/ShedCompiler.java6
-rw-r--r--src/main/java/org/shedlang/compiler/config/BinConfig.java9
-rw-r--r--src/main/java/org/shedlang/compiler/config/errors/Json5SyntaxConfigError.java4
-rw-r--r--src/main/java/org/shedlang/compiler/config/json/Json5ObjectWithSource.java11
-rw-r--r--src/main/java/org/shedlang/compiler/config/json/Json5Source.java18
-rw-r--r--src/main/java/org/shedlang/compiler/parser/OperationParser.java7
-rw-r--r--src/main/java/org/shedlang/compiler/parser/Parser.java358
-rw-r--r--src/main/java/org/shedlang/compiler/parser/SourceParser.java10
-rw-r--r--src/main/java/org/shedlang/compiler/parser/Tokeniser.java14
-rw-r--r--src/main/java/org/shedlang/compiler/sources/FileFragmentSource.java71
-rw-r--r--src/main/java/org/shedlang/compiler/sources/NullSource.java25
-rw-r--r--src/main/java/org/shedlang/compiler/sources/RuntimeSource.java25
-rw-r--r--src/main/java/org/shedlang/compiler/sources/Source.java5
-rw-r--r--src/main/java/org/shedlang/compiler/sources/SourceTexts.java12
-rw-r--r--src/main/java/org/shedlang/compiler/sources/Sources.java14
-rw-r--r--src/main/java/org/shedlang/compiler/tokeniser/InfinitelyPaddedIterator.java4
-rw-r--r--src/main/java/org/shedlang/compiler/tokeniser/RegexTokeniser.java13
-rw-r--r--src/main/java/org/shedlang/compiler/tokeniser/Token.java8
-rw-r--r--src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java22
-rw-r--r--src/main/java/org/shedlang/compiler/tokeniser/UnexpectedTokenException.java3
20 files changed, 276 insertions, 363 deletions
diff --git a/src/main/java/org/shedlang/compiler/ShedCompiler.java b/src/main/java/org/shedlang/compiler/ShedCompiler.java
index 0efbe1d5..a57a3386 100644
--- a/src/main/java/org/shedlang/compiler/ShedCompiler.java
+++ b/src/main/java/org/shedlang/compiler/ShedCompiler.java
@@ -9,11 +9,11 @@ import org.shedlang.compiler.config.ProjectConfig;
import org.shedlang.compiler.parser.Parser;
import org.shedlang.compiler.shid.ShidCodeGen;
import org.shedlang.compiler.shid.ShidNamespace;
-import org.shedlang.compiler.sources.Sources;
import org.shedlang.compiler.typechecker.TypeChecker;
import org.shedlang.compiler.typechecker.TypeCheckerContext;
import org.shedlang.compiler.types.*;
import org.shedlang.compiler.util.P;
+import org.zwobble.sourcetext.SourceText;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
@@ -131,9 +131,9 @@ public class ShedCompiler {
var namespaceName = new NamespaceName(namespaceNameParts);
var sourceContents = Files.readString(sourceFilePath, StandardCharsets.UTF_8);
- var source = Sources.file(sourceFilePath, sourceContents);
+ var sourceText = SourceText.fromString(sourceFilePath.toString(), sourceContents);
- var tokens = tokenise(source);
+ var tokens = tokenise(sourceText);
var untypedNode = Parser.parseNamespace(tokens, namespaceName);
untypedNamespaceNodes.add(untypedNode);
diff --git a/src/main/java/org/shedlang/compiler/config/BinConfig.java b/src/main/java/org/shedlang/compiler/config/BinConfig.java
index a81b2737..3541c40b 100644
--- a/src/main/java/org/shedlang/compiler/config/BinConfig.java
+++ b/src/main/java/org/shedlang/compiler/config/BinConfig.java
@@ -2,9 +2,9 @@ package org.shedlang.compiler.config;
import org.shedlang.compiler.config.errors.ConfigError;
import org.shedlang.compiler.config.json.Json5ObjectWithSource;
-import org.shedlang.compiler.config.json.Json5Source;
import org.shedlang.compiler.parser.SourceParser;
-import org.shedlang.compiler.sources.Sources;
+import org.shedlang.compiler.sources.FileFragmentSource;
+import org.shedlang.compiler.sources.SourceTexts;
import org.shedlang.compiler.tokeniser.UnexpectedTokenException;
import org.shedlang.compiler.types.NamespaceName;
@@ -17,7 +17,8 @@ public record BinConfig(String name, NamespaceName mainNamespace) {
var mainNamespaceString = object.getString("mainNamespace");
try {
- var mainNamespace = SourceParser.parseNamespaceName(Sources.string(mainNamespaceString.value()));
+ var sourceText = SourceTexts.string(mainNamespaceString.value());
+ var mainNamespace = SourceParser.parseNamespaceName(sourceText);
return new BinConfig(name, mainNamespace);
} catch (UnexpectedTokenException exception) {
throw new ConfigError(
@@ -26,7 +27,7 @@ public record BinConfig(String name, NamespaceName mainNamespace) {
exception.getMessage(),
exception.source().describe()
),
- Json5Source.fromSourceRange(mainNamespaceString.sourceRange())
+ new FileFragmentSource(mainNamespaceString.sourceRange())
);
}
}
diff --git a/src/main/java/org/shedlang/compiler/config/errors/Json5SyntaxConfigError.java b/src/main/java/org/shedlang/compiler/config/errors/Json5SyntaxConfigError.java
index 2009af43..d67b1a77 100644
--- a/src/main/java/org/shedlang/compiler/config/errors/Json5SyntaxConfigError.java
+++ b/src/main/java/org/shedlang/compiler/config/errors/Json5SyntaxConfigError.java
@@ -1,6 +1,6 @@
package org.shedlang.compiler.config.errors;
-import org.shedlang.compiler.config.json.Json5Source;
+import org.shedlang.compiler.sources.FileFragmentSource;
import org.shedlang.compiler.sources.Source;
import org.zwobble.json5.parser.Json5ParseError;
@@ -8,7 +8,7 @@ public class Json5SyntaxConfigError extends ConfigError {
public static Json5SyntaxConfigError fromJson5ParseError(
Json5ParseError error
) {
- var source = Json5Source.fromSourceRange(error.sourceRange());
+ var source = new FileFragmentSource(error.sourceRange());
return new Json5SyntaxConfigError(error.getMessage(), source);
}
diff --git a/src/main/java/org/shedlang/compiler/config/json/Json5ObjectWithSource.java b/src/main/java/org/shedlang/compiler/config/json/Json5ObjectWithSource.java
index cd7d811e..48db3b9b 100644
--- a/src/main/java/org/shedlang/compiler/config/json/Json5ObjectWithSource.java
+++ b/src/main/java/org/shedlang/compiler/config/json/Json5ObjectWithSource.java
@@ -3,6 +3,7 @@ package org.shedlang.compiler.config.json;
import org.shedlang.compiler.config.errors.ConfigError;
import org.shedlang.compiler.config.errors.Json5SyntaxConfigError;
import org.shedlang.compiler.config.errors.UnexpectedTypeConfigError;
+import org.shedlang.compiler.sources.FileFragmentSource;
import org.zwobble.json5.parser.Json5ParseError;
import org.zwobble.json5.parser.Json5Parser;
import org.zwobble.json5.values.*;
@@ -33,7 +34,7 @@ public class Json5ObjectWithSource {
document.path(),
Json5Object.class,
document.getClass(),
- Json5Source.fromSourceRange(document.sourceRange())
+ new FileFragmentSource(document.sourceRange())
);
}
} catch (Json5ParseError error) {
@@ -75,7 +76,7 @@ public class Json5ObjectWithSource {
.orElseThrow(() -> new ConfigError(
// TODO: escape member name
MessageFormat.format("{0} is missing {1}", object.path(), memberName),
- Json5Source.fromSourceRange(object.sourceRange())
+ new FileFragmentSource(object.sourceRange())
));
}
@@ -94,7 +95,7 @@ public class Json5ObjectWithSource {
"%s must be a 64-bit integer",
numberFinite.get().path()
),
- Json5Source.fromSourceRange(numberFinite.get().sourceRange())
+ new FileFragmentSource(numberFinite.get().sourceRange())
);
}
}
@@ -104,7 +105,7 @@ public class Json5ObjectWithSource {
.orElseThrow(() -> new ConfigError(
// TODO: escape member name
MessageFormat.format("{0} is missing {1}", object.path(), memberName),
- Json5Source.fromSourceRange(object.sourceRange())
+ new FileFragmentSource(object.sourceRange())
));
}
@@ -130,7 +131,7 @@ public class Json5ObjectWithSource {
value.path(),
valueType,
value.getClass(),
- Json5Source.fromSourceRange(value.sourceRange())
+ new FileFragmentSource(value.sourceRange())
);
}
}
diff --git a/src/main/java/org/shedlang/compiler/config/json/Json5Source.java b/src/main/java/org/shedlang/compiler/config/json/Json5Source.java
deleted file mode 100644
index e2a556e9..00000000
--- a/src/main/java/org/shedlang/compiler/config/json/Json5Source.java
+++ /dev/null
@@ -1,18 +0,0 @@
-package org.shedlang.compiler.config.json;
-
-import org.shedlang.compiler.sources.FileFragmentSource;
-import org.shedlang.compiler.sources.Source;
-import org.zwobble.sourcetext.SourceRange;
-
-public class Json5Source {
- private Json5Source() {}
-
- public static Source fromSourceRange(SourceRange sourceRange) {
- return new FileFragmentSource(
- sourceRange.sourceText().name(),
- sourceRange.sourceText().charBuffer(0, sourceRange.sourceText().characterLength()).toString(),
- sourceRange.start().characterIndex(),
- sourceRange.end().characterIndex()
- );
- }
-}
diff --git a/src/main/java/org/shedlang/compiler/parser/OperationParser.java b/src/main/java/org/shedlang/compiler/parser/OperationParser.java
index 9a4a25ea..e56b3ffe 100644
--- a/src/main/java/org/shedlang/compiler/parser/OperationParser.java
+++ b/src/main/java/org/shedlang/compiler/parser/OperationParser.java
@@ -1,11 +1,16 @@
package org.shedlang.compiler.parser;
import org.shedlang.compiler.tokeniser.TokenIterator;
+import org.zwobble.sourcetext.SourcePosition;
import java.util.stream.Stream;
public interface OperationParser<TOperatorPrecedence, TUntypedExpressionNode> {
TOperatorPrecedence precedence();
Stream<TokenType> operatorTokenTypes();
- TUntypedExpressionNode parse(TUntypedExpressionNode left, TokenIterator<TokenType> tokens);
+ TUntypedExpressionNode parse(
+ SourcePosition start,
+ TUntypedExpressionNode left,
+ TokenIterator<TokenType> tokens
+ );
}
diff --git a/src/main/java/org/shedlang/compiler/parser/Parser.java b/src/main/java/org/shedlang/compiler/parser/Parser.java
index 680ead15..cda360e3 100644
--- a/src/main/java/org/shedlang/compiler/parser/Parser.java
+++ b/src/main/java/org/shedlang/compiler/parser/Parser.java
@@ -2,12 +2,15 @@ package org.shedlang.compiler.parser;
import org.shedlang.compiler.ast.untyped.*;
import org.shedlang.compiler.parser.errors.*;
+import org.shedlang.compiler.sources.FileFragmentSource;
import org.shedlang.compiler.sources.Source;
import org.shedlang.compiler.tokeniser.TokenIterator;
import org.shedlang.compiler.tokeniser.UnexpectedTokenException;
import org.shedlang.compiler.types.NamespaceName;
import org.shedlang.compiler.types.Visibility;
import org.shedlang.compiler.util.Iterables;
+import org.zwobble.sourcetext.SourcePosition;
+import org.zwobble.sourcetext.SourceRange;
import java.math.BigInteger;
import java.util.ArrayList;
@@ -53,27 +56,29 @@ public class Parser {
}
@Override
- public UntypedExpressionNode parse(UntypedExpressionNode left, TokenIterator<TokenType> tokens) {
- var start = left.source().start();
-
+ public UntypedExpressionNode parse(
+ SourcePosition start,
+ UntypedExpressionNode left,
+ TokenIterator<TokenType> tokens
+ ) {
tokens.skip(operatorTokenType);
var right = parseSubExpression(tokens, precedence);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var sourceRange = new FileFragmentSource(start.to(end));
return new UntypedBinaryOperationNode(
operator,
left,
right,
- source
+ sourceRange
);
}
}
private static UntypedArgsNode parseArgs(TokenIterator<TokenType> tokens) {
- var argsStart = tokens.position();
+ var argsStart = tokens.startPosition();
var positional = new ArrayList<UntypedExpressionNode>();
var named = new ArrayList<UntypedNamedArgNode>();
@@ -82,14 +87,14 @@ public class Parser {
() -> tokens.isNext(TokenType.SYMBOL_PAREN_CLOSE),
() -> {
if (tokens.trySkip(TokenType.SYMBOL_DOT)) {
- var argStart = tokens.position();
+ var argStart = tokens.startPosition();
var argName = tokens.nextValue(TokenType.IDENTIFIER);
tokens.skip(TokenType.SYMBOL_EQUALS);
var expression = parseTopLevelExpression(tokens);
- var argEnd = tokens.position();
- var argSource = argStart.to(argEnd);
+ var argEnd = tokens.endPosition();
+ var argSource = new FileFragmentSource(argStart.to(argEnd));
named.add(new UntypedNamedArgNode(argName, expression, argSource));
} else {
@@ -104,14 +109,14 @@ public class Parser {
() -> tokens.trySkip(TokenType.SYMBOL_COMMA)
);
- var argsEnd = tokens.position();
- var argsSource = argsStart.to(argsEnd);
+ var argsEnd = tokens.endPosition();
+ var argsSource = new FileFragmentSource(argsStart.to(argsEnd));
return new UntypedArgsNode(positional, named, argsSource);
}
private static UntypedAssignNode parseAssign(TokenIterator<TokenType> tokens) {
- var start = tokens.position();
+ var start = tokens.startPosition();
var target = parseTarget(tokens);
@@ -121,20 +126,20 @@ public class Parser {
tokens.skip(TokenType.SYMBOL_SEMICOLON);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedAssignNode(target, expression, source);
}
private static UntypedBlockNode parseBlock(TokenIterator<TokenType> tokens) {
- var start = tokens.position();
+ var start = tokens.startPosition();
tokens.skip(TokenType.SYMBOL_BRACE_OPEN);
if (tokens.trySkip(TokenType.SYMBOL_BRACE_CLOSE)) {
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedBlockNode(List.of(), source);
}
@@ -152,30 +157,30 @@ public class Parser {
) {
tokens.skip(TokenType.SYMBOL_BRACE_CLOSE);
- var end = tokens.position();
+ var end = tokens.endPosition();
var source = start.to(end);
- return new UntypedBlockNode(body, source);
+ return new UntypedBlockNode(body, new FileFragmentSource(source));
}
}
}
private static UntypedExpressionNode parseBoolLiteral(TokenIterator<TokenType> tokens) {
- var start = tokens.position();
+ var start = tokens.startPosition();
var value = tokens.trySkip(TokenType.KEYWORD_TRUE);
if (!value) {
tokens.skip(TokenType.KEYWORD_FALSE);
}
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedBoolLiteralNode(value, source);
}
private static UntypedFunctionStatementNode parseBreak(TokenIterator<TokenType> tokens) {
- var start = tokens.position();
+ var start = tokens.startPosition();
tokens.skip(TokenType.KEYWORD_BREAK);
@@ -185,8 +190,8 @@ public class Parser {
tokens.skip(TokenType.SYMBOL_SEMICOLON);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedBreakNode(expression, source);
}
@@ -210,9 +215,11 @@ public class Parser {
}
@Override
- public UntypedExpressionNode parse(UntypedExpressionNode left, TokenIterator<TokenType> tokens) {
- var start = left.source().start();
-
+ public UntypedExpressionNode parse(
+ SourcePosition start,
+ UntypedExpressionNode left,
+ TokenIterator<TokenType> tokens
+ ) {
var hasEffects = tokens.trySkip(TokenType.SYMBOL_BANG);
var typeLevelArgs = parseTypeLevelArgsOptional(tokens);
@@ -221,8 +228,8 @@ public class Parser {
var args = parseArgs(tokens);
tokens.skip(TokenType.SYMBOL_PAREN_CLOSE);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedCallNode(
left,
@@ -249,9 +256,11 @@ public class Parser {
}
@Override
- public UntypedTypeLevelExpressionNode parse(UntypedTypeLevelExpressionNode left, TokenIterator<TokenType> tokens) {
- var start = left.source().start();
-
+ public UntypedTypeLevelExpressionNode parse(
+ SourcePosition start,
+ UntypedTypeLevelExpressionNode left,
+ TokenIterator<TokenType> tokens
+ ) {
tokens.skip(TokenType.SYMBOL_SQUARE_OPEN);
var args = parseMany(
() -> tokens.isNext(TokenType.SYMBOL_SQUARE_CLOSE),
@@ -260,8 +269,8 @@ public class Parser {
);
tokens.skip(TokenType.SYMBOL_SQUARE_CLOSE);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedConstructedTypeNode(left, args, source);
}
@@ -270,7 +279,7 @@ public class Parser {
private static UntypedEffectDefinitionNode parseEffectDefinition(
TokenIterator<TokenType> tokens
) {
- var start = tokens.position();
+ var start = tokens.startPosition();
var visibility = parseVisibility(tokens);
@@ -288,8 +297,8 @@ public class Parser {
tokens.skip(TokenType.SYMBOL_BRACE_CLOSE);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedEffectDefinitionNode(name, visibility, functions, source);
}
@@ -297,7 +306,7 @@ public class Parser {
private static UntypedEffectDefinitionMethodNode parseEffectDefinitionField(
TokenIterator<TokenType> tokens
) {
- var start = tokens.position();
+ var start = tokens.startPosition();
tokens.skip(TokenType.KEYWORD_FUN);
@@ -307,8 +316,8 @@ public class Parser {
tokens.skip(TokenType.SYMBOL_SEMICOLON);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedEffectDefinitionMethodNode(name, type, source);
}
@@ -316,7 +325,7 @@ public class Parser {
private static UntypedFunctionTypeNode parseEffectDefinitionFieldType(
TokenIterator<TokenType> tokens
) {
- var start = tokens.position();
+ var start = tokens.startPosition();
tokens.skip(TokenType.SYMBOL_PAREN_OPEN);
var params = parseParamTypes(tokens);
@@ -326,8 +335,8 @@ public class Parser {
var returnType = parseTopLevelTypeLevelExpression(tokens);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedFunctionTypeNode(params, Optional.empty(), returnType, source);
}
@@ -352,6 +361,7 @@ public class Parser {
TokenIterator<TokenType> tokens,
Optional<OperatorPrecedence> parentPrecedence
) {
+ var start = tokens.startPosition();
var left = parsePrimaryExpression(tokens);
while (true) {
@@ -364,12 +374,12 @@ public class Parser {
) {
return left;
}
- left = operationParser.parse(left, tokens);
+ left = operationParser.parse(start, left, tokens);
}
}
private static UntypedFunctionStatementNode parseExpressionStatement(TokenIterator<TokenType> tokens) {
- var start = tokens.position();
+ var start = tokens.startPosition();
var expression = parseTopLevelExpression(tokens);
@@ -377,8 +387,8 @@ public class Parser {
? expressionStatementBehaviour(branchingExpression.exhaustiveChildBlocks()) == UntypedFunctionStatementBehaviour.BLOCK_RETURN
: !tokens.trySkip(TokenType.SYMBOL_SEMICOLON);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return isReturn
? new UntypedBlockReturnNode(expression, source)
@@ -406,7 +416,7 @@ public class Parser {
}
private static UntypedFunctionDefinitionNode parseFunctionDefinition(TokenIterator<TokenType> tokens) {
- var start = tokens.position();
+ var start = tokens.startPosition();
var visibility = parseVisibility(tokens);
@@ -417,8 +427,8 @@ public class Parser {
var body = parseBlock(tokens);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedFunctionDefinitionNode(
name,
@@ -432,7 +442,7 @@ public class Parser {
private static UntypedExpressionNode parseFunctionExpression(
TokenIterator<TokenType> tokens
) {
- var start = tokens.position();
+ var start = tokens.startPosition();
tokens.skip(TokenType.KEYWORD_FN);
@@ -440,8 +450,8 @@ public class Parser {
var body = parseBlock(tokens);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedFunctionExpressionNode(
signature,
@@ -451,7 +461,7 @@ public class Parser {
}
private static UntypedFunctionStatementNode parseFunctionReturn(TokenIterator<TokenType> tokens) {
- var start = tokens.position();
+ var start = tokens.startPosition();
tokens.skip(TokenType.KEYWORD_RETURN);
@@ -461,8 +471,8 @@ public class Parser {
tokens.skip(TokenType.SYMBOL_SEMICOLON);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedFunctionReturnNode(expression, source);
}
@@ -470,7 +480,7 @@ public class Parser {
static UntypedFunctionSignatureNode parseFunctionSignature(
TokenIterator<TokenType> tokens
) {
- var start = tokens.position();
+ var start = tokens.startPosition();
var typeLevelParams = parseTypeLevelParamsOptional(tokens);
@@ -482,8 +492,8 @@ public class Parser {
var returnType = parseFunctionSignatureReturnType(tokens);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedFunctionSignatureNode(
typeLevelParams,
@@ -555,7 +565,7 @@ public class Parser {
private static UntypedTypeLevelExpressionNode parseFunctionType(
TokenIterator<TokenType> tokens
) {
- var start = tokens.position();
+ var start = tokens.startPosition();
tokens.skip(TokenType.KEYWORD_FN);
@@ -569,8 +579,8 @@ public class Parser {
var returnType = parseTopLevelTypeLevelExpression(tokens);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedFunctionTypeNode(
params,
@@ -589,7 +599,7 @@ public class Parser {
}
private static UntypedExpressionNode parseIf(TokenIterator<TokenType> tokens) {
- var start = tokens.position();
+ var start = tokens.startPosition();
var conditionalBranches = new ArrayList<UntypedConditionalBranchNode>();
var elseBody = Optional.<UntypedBlockNode>empty();
@@ -609,18 +619,21 @@ public class Parser {
}
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedIfNode(
conditionalBranches,
- elseBody.orElseGet(() -> new UntypedBlockNode(List.of(), end)),
+ elseBody.orElseGet(() -> new UntypedBlockNode(
+ List.of(),
+ new FileFragmentSource(end.to(end))
+ )),
source
);
}
private static UntypedConditionalBranchNode parseConditionalBranch(TokenIterator<TokenType> tokens) {
- var start = tokens.position();
+ var start = tokens.startPosition();
tokens.skip(TokenType.KEYWORD_IF);
@@ -630,14 +643,14 @@ public class Parser {
var body = parseBlock(tokens);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedConditionalBranchNode(condition, body, source);
}
private static UntypedImportNode parseImport(TokenIterator<TokenType> tokens) {
- var start = tokens.position();
+ var start = tokens.startPosition();
tokens.skip(TokenType.KEYWORD_IMPORT);
@@ -645,21 +658,21 @@ public class Parser {
tokens.skip(TokenType.SYMBOL_SEMICOLON);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedImportNode(namespaceName, source);
}
private static UntypedExpressionNode parseInt(TokenIterator<TokenType> tokens) {
- var start = tokens.position();
+ var start = tokens.startPosition();
var string = tokens.nextValue(TokenType.INT);
var value = new BigInteger(string);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
if (string.length() > 1 && string.startsWith("0")) {
throw new IntLiteralCannotHaveLeadingZeroesError(source);
@@ -731,15 +744,17 @@ public class Parser {
}
@Override
- public UntypedExpressionNode parse(UntypedExpressionNode left, TokenIterator<TokenType> tokens) {
- var start = left.source().start();
-
+ public UntypedExpressionNode parse(
+ SourcePosition start,
+ UntypedExpressionNode left,
+ TokenIterator<TokenType> tokens
+ ) {
tokens.skip(TokenType.SYMBOL_DOT);
var memberName = tokens.nextValue(TokenType.IDENTIFIER);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedMemberAccessNode(left, memberName, source);
}
@@ -797,7 +812,7 @@ public class Parser {
}
private static UntypedParamsNode parseParams(TokenIterator<TokenType> tokens) {
- var paramsStart = tokens.position();
+ var paramsStart = tokens.startPosition();
var positional = new ArrayList<UntypedParamNode>();
var named = new ArrayList<UntypedParamNode>();
@@ -805,7 +820,7 @@ public class Parser {
parseMany(
() -> tokens.isNext(TokenType.SYMBOL_PAREN_CLOSE),
() -> {
- var paramStart = tokens.position();
+ var paramStart = tokens.startPosition();
var isNamed = tokens.trySkip(TokenType.SYMBOL_DOT);
@@ -815,8 +830,8 @@ public class Parser {
? Optional.of(parseTopLevelTypeLevelExpression(tokens))
: Optional.<UntypedTypeLevelExpressionNode>empty();
- var paramEnd = tokens.position();
- var paramSource = paramStart.to(paramEnd);
+ var paramEnd = tokens.endPosition();
+ var paramSource = new FileFragmentSource(paramStart.to(paramEnd));
if (!isNamed && !named.isEmpty()) {
throw new PositionalParamAfterNamedParamError(paramSource);
@@ -829,14 +844,14 @@ public class Parser {
() -> tokens.trySkip(TokenType.SYMBOL_COMMA)
);
- var paramsEnd = tokens.position();
- var paramsSource = paramsStart.to(paramsEnd);
+ var paramsEnd = tokens.endPosition();
+ var paramsSource = new FileFragmentSource(paramsStart.to(paramsEnd));
return new UntypedParamsNode(positional, named, paramsSource);
}
private static UntypedFunctionTypeParamsNode parseParamTypes(TokenIterator<TokenType> tokens) {
- var paramsStart = tokens.position();
+ var paramsStart = tokens.startPosition();
var positional = new ArrayList<UntypedTypeLevelExpressionNode>();
var named = new ArrayList<UntypedFunctionTypeNamedParamNode>();
@@ -844,7 +859,7 @@ public class Parser {
parseMany(
() -> tokens.isNext(TokenType.SYMBOL_PAREN_CLOSE),
() -> {
- var paramStart = tokens.position();
+ var paramStart = tokens.startPosition();
var isNamed = tokens.trySkip(TokenType.SYMBOL_DOT);
@@ -853,8 +868,8 @@ public class Parser {
tokens.skip(TokenType.SYMBOL_COLON);
var type = parseTopLevelTypeLevelExpression(tokens);
- var paramEnd = tokens.position();
- var paramSource = paramStart.to(paramEnd);
+ var paramEnd = tokens.endPosition();
+ var paramSource = new FileFragmentSource(paramStart.to(paramEnd));
named.add(new UntypedFunctionTypeNamedParamNode(name, type, paramSource));
} else {
@@ -871,10 +886,10 @@ public class Parser {
() -> tokens.trySkip(TokenType.SYMBOL_COMMA)
);
- var paramsEnd = tokens.position();
+ var paramsEnd = tokens.endPosition();
var paramsSource = paramsStart.to(paramsEnd);
- return new UntypedFunctionTypeParamsNode(positional, named, paramsSource);
+ return new UntypedFunctionTypeParamsNode(positional, named, new FileFragmentSource(paramsSource));
}
private static UntypedExpressionNode parsePrimaryExpression(TokenIterator<TokenType> tokens) {
@@ -922,7 +937,7 @@ public class Parser {
}
private static UntypedFunctionStatementNode parseResume(TokenIterator<TokenType> tokens) {
- var start = tokens.position();
+ var start = tokens.startPosition();
tokens.skip(TokenType.KEYWORD_RESUME);
@@ -932,38 +947,44 @@ public class Parser {
tokens.skip(TokenType.SYMBOL_SEMICOLON);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedResumeNode(expression, source);
}
private static UntypedExpressionNode parseStringLiteral(TokenIterator<TokenType> tokens) {
- var start = tokens.position();
+ var start = tokens.startPosition();
- var tokenValue = tokens.nextValue(TokenType.STRING);
- var escapedValue = tokenValue.substring(1, tokenValue.length() - 1);
- var unescapedValue = unescape(escapedValue, start);
+ var token = tokens.next(TokenType.STRING);
+ var stringContentsStart = token.sourceRange().characterPosition(1);
+ var stringContentsEnd = token.sourceRange()
+ .characterPosition(token.sourceRange().characterLength() - 1);
+ var stringContents = stringContentsStart.to(stringContentsEnd);
+ var unescapedValue = unescape(stringContents);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedStringLiteralNode(unescapedValue, source);
}
private static final Pattern STRING_ESCAPE_PATTERN = Pattern.compile("\\\\(.)");
- private static String unescape(String value, Source source) {
- var matcher = STRING_ESCAPE_PATTERN.matcher(value);
+ private static String unescape(SourceRange sourceRange) {
+ var stringContents = sourceRange.charSequence();
+ var matcher = STRING_ESCAPE_PATTERN.matcher(stringContents);
var result = new StringBuilder();
var lastIndex = 0;
while (matcher.find()) {
- result.append(value.subSequence(lastIndex, matcher.start()));
+ result.append(stringContents.subSequence(lastIndex, matcher.start()));
var code = matcher.group(1);
- result.append(unescapeCharacter(code, source.at(matcher.start(), matcher.end())));
+ var escapeSequenceSourceRange = sourceRange.characterPosition(matcher.start())
+ .to(sourceRange.characterPosition(matcher.end()));
+ result.append(unescapeCharacter(code, new FileFragmentSource(escapeSequenceSourceRange)));
lastIndex = matcher.end();
}
- result.append(value.subSequence(lastIndex, value.length()));
+ result.append(stringContents.subSequence(lastIndex, stringContents.length()));
return result.toString();
}
@@ -979,7 +1000,7 @@ public class Parser {
}
private static UntypedNamespaceStatementNode parseStructDefinition(TokenIterator<TokenType> tokens) {
- var start = tokens.position();
+ var start = tokens.startPosition();
var visibility = parseVisibility(tokens);
@@ -1008,8 +1029,8 @@ public class Parser {
tokens.skip(TokenType.SYMBOL_BRACE_CLOSE);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedStructDefinitionNode(
name,
@@ -1026,7 +1047,7 @@ public class Parser {
TokenIterator<TokenType> tokens,
boolean structHasPositionalFields
) {
- var start = tokens.position();
+ var start = tokens.startPosition();
var fieldIsPositional = !tokens.trySkip(TokenType.SYMBOL_DOT);
var name = tokens.nextValue(TokenType.IDENTIFIER);
@@ -1035,8 +1056,8 @@ public class Parser {
var type = parseTopLevelTypeLevelExpression(tokens);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
if (structHasPositionalFields ^ fieldIsPositional) {
throw new InconsistentStructFieldsError(source);
@@ -1050,7 +1071,7 @@ public class Parser {
}
private static UntypedNamespaceStatementNode parseSumTypeDefinition(TokenIterator<TokenType> tokens) {
- var start = tokens.position();
+ var start = tokens.startPosition();
var visibility = parseVisibility(tokens);
@@ -1069,8 +1090,8 @@ public class Parser {
tokens.skip(TokenType.SYMBOL_SEMICOLON);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedSumTypeDefinitionNode(
name,
@@ -1093,7 +1114,7 @@ public class Parser {
TokenIterator<TokenType> tokens,
NamespaceName namespaceName
) {
- var start = tokens.position();
+ var start = tokens.startPosition();
var imports = parseMany(
() -> !tokens.isNext(TokenType.KEYWORD_IMPORT),
@@ -1107,25 +1128,25 @@ public class Parser {
() -> true
);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedNamespaceNode(namespaceName, imports, body, source);
}
private static UntypedExpressionNode parseReference(TokenIterator<TokenType> tokens) {
- var start = tokens.position();
+ var start = tokens.startPosition();
var name = tokens.nextValue(TokenType.IDENTIFIER);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedReferenceNode(name, source);
}
private static UntypedTargetNode parseTarget(TokenIterator<TokenType> tokens) {
- var start = tokens.position();
+ var start = tokens.startPosition();
return switch (tokens.peek().tokenType()) {
case KEYWORD_ASSIGN -> {
@@ -1133,8 +1154,8 @@ public class Parser {
var name = tokens.nextValue(TokenType.IDENTIFIER);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
yield new UntypedTargetNode.Reassign(name, source);
}
@@ -1146,8 +1167,8 @@ public class Parser {
var name = tokens.nextValue(TokenType.IDENTIFIER);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
yield new UntypedTargetNode.Val(name, visibility, source);
}
@@ -1157,8 +1178,8 @@ public class Parser {
var name = tokens.nextValue(TokenType.IDENTIFIER);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
yield new UntypedTargetNode.Var(name, source);
}
@@ -1174,8 +1195,8 @@ public class Parser {
tokens.skip(TokenType.SYMBOL_PAREN_CLOSE);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
yield new UntypedTargetNode.Tuple(elements, source);
}
@@ -1183,8 +1204,8 @@ public class Parser {
case SYMBOL_UNDERSCORE -> {
tokens.skip(TokenType.SYMBOL_UNDERSCORE);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
yield new UntypedTargetNode.Ignore(source);
}
@@ -1197,7 +1218,7 @@ public class Parser {
private static UntypedExpressionNode parseTry(
TokenIterator<TokenType> tokens
) {
- var start = tokens.position();
+ var start = tokens.startPosition();
tokens.skip(TokenType.KEYWORD_TRY);
@@ -1217,14 +1238,14 @@ public class Parser {
tokens.skip(TokenType.SYMBOL_BRACE_CLOSE);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedTryNode(body, handledEffect, handlerMethods, source);
}
private static UntypedTryHandlerMethodNode parseTryHandlerMethod(TokenIterator<TokenType> tokens) {
- var start = tokens.position();
+ var start = tokens.startPosition();
tokens.skip(TokenType.KEYWORD_FUN);
var name = tokens.nextValue(TokenType.IDENTIFIER);
@@ -1235,8 +1256,8 @@ public class Parser {
var body = parseBlock(tokens);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedTryHandlerMethodNode(
name,
@@ -1251,7 +1272,7 @@ public class Parser {
private static UntypedExpressionNode parseTuple(
TokenIterator<TokenType> tokens
) {
- var start = tokens.position();
+ var start = tokens.startPosition();
tokens.skip(TokenType.SYMBOL_HASH_PAREN_OPEN);
@@ -1263,8 +1284,8 @@ public class Parser {
tokens.skip(TokenType.SYMBOL_PAREN_CLOSE);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedTupleNode(elements, source);
}
@@ -1272,7 +1293,7 @@ public class Parser {
private static UntypedTypeLevelExpressionNode parseTupleType(
TokenIterator<TokenType> tokens
) {
- var start = tokens.position();
+ var start = tokens.startPosition();
tokens.skip(TokenType.SYMBOL_HASH_PAREN_OPEN);
@@ -1284,8 +1305,8 @@ public class Parser {
tokens.skip(TokenType.SYMBOL_PAREN_CLOSE);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedTupleTypeNode(elements, source);
}
@@ -1293,7 +1314,7 @@ public class Parser {
private static Optional<UntypedTypeLevelArgsNode> parseTypeLevelArgsOptional(
TokenIterator<TokenType> tokens
) {
- var start = tokens.position();
+ var start = tokens.startPosition();
if (tokens.trySkip(TokenType.SYMBOL_SQUARE_OPEN)) {
var params = parseMany(
@@ -1304,8 +1325,8 @@ public class Parser {
tokens.skip(TokenType.SYMBOL_SQUARE_CLOSE);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return Optional.of(new UntypedTypeLevelArgsNode(params, source));
} else {
@@ -1328,6 +1349,7 @@ public class Parser {
TokenIterator<TokenType> tokens,
Optional<OperatorPrecedence> parentPrecedence
) {
+ var start = tokens.startPosition();
var left = parsePrimaryTypeLevelExpression(tokens);
while (true) {
@@ -1340,7 +1362,7 @@ public class Parser {
) {
return left;
}
- left = operationParser.parse(left, tokens);
+ left = operationParser.parse(start, left, tokens);
}
}
@@ -1363,14 +1385,14 @@ public class Parser {
}
static UntypedTypeLevelParamNode parseTypeLevelParam(TokenIterator<TokenType> tokens) {
- var start = tokens.position();
+ var start = tokens.startPosition();
var isEffectParam = tokens.trySkip(TokenType.SYMBOL_BANG);
var name = tokens.nextValue(TokenType.IDENTIFIER);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return isEffectParam
? new UntypedEffectParamNode(name, source)
@@ -1378,23 +1400,23 @@ public class Parser {
}
private static UntypedTypeLevelReferenceNode parseTypeLevelReference(TokenIterator<TokenType> tokens) {
- var start = tokens.position();
+ var start = tokens.startPosition();
var name = tokens.nextValue(TokenType.IDENTIFIER);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedTypeLevelReferenceNode(name, source);
}
private static UntypedExpressionNode parseUnitLiteral(TokenIterator<TokenType> tokens) {
- var start = tokens.position();
+ var start = tokens.startPosition();
tokens.skip(TokenType.KEYWORD_UNIT);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedUnitLiteralNode(source);
}
@@ -1405,7 +1427,7 @@ public class Parser {
}
private static UntypedExpressionNode parseWhen(TokenIterator<TokenType> tokens) {
- var start = tokens.position();
+ var start = tokens.startPosition();
tokens.skip(TokenType.KEYWORD_WHEN);
@@ -1421,14 +1443,14 @@ public class Parser {
);
tokens.skip(TokenType.SYMBOL_BRACE_CLOSE);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedWhenNode(expression, cases, source);
}
private static UntypedWhenCaseNode parseWhenCase(TokenIterator<TokenType> tokens) {
- var start = tokens.position();
+ var start = tokens.startPosition();
tokens.skip(TokenType.KEYWORD_IS);
@@ -1436,8 +1458,8 @@ public class Parser {
var body = parseBlock(tokens);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedWhenCaseNode(caseType, body, source);
}
@@ -1445,7 +1467,7 @@ public class Parser {
private static UntypedFunctionStatementNode parseWhile(
TokenIterator<TokenType> tokens
) {
- var start = tokens.position();
+ var start = tokens.startPosition();
tokens.skip(TokenType.KEYWORD_WHILE);
tokens.skip(TokenType.SYMBOL_PAREN_OPEN);
@@ -1454,8 +1476,8 @@ public class Parser {
var body = parseBlock(tokens);
- var end = tokens.position();
- var source = start.to(end);
+ var end = tokens.endPosition();
+ var source = new FileFragmentSource(start.to(end));
return new UntypedWhileNode(condition, body, source);
}
diff --git a/src/main/java/org/shedlang/compiler/parser/SourceParser.java b/src/main/java/org/shedlang/compiler/parser/SourceParser.java
index 19be58d4..8144f11a 100644
--- a/src/main/java/org/shedlang/compiler/parser/SourceParser.java
+++ b/src/main/java/org/shedlang/compiler/parser/SourceParser.java
@@ -1,8 +1,8 @@
package org.shedlang.compiler.parser;
-import org.shedlang.compiler.sources.Source;
import org.shedlang.compiler.tokeniser.TokenIterator;
import org.shedlang.compiler.types.NamespaceName;
+import org.zwobble.sourcetext.SourceText;
import java.util.function.Function;
@@ -12,15 +12,15 @@ public class SourceParser {
private SourceParser() {
}
- public static NamespaceName parseNamespaceName(Source source) {
- return parseSource(source, tokens -> Parser.parseNamespaceName(tokens));
+ public static NamespaceName parseNamespaceName(SourceText sourceText) {
+ return parseSource(sourceText, tokens -> Parser.parseNamespaceName(tokens));
}
private static <T> T parseSource(
- Source source,
+ SourceText sourceText,
Function<TokenIterator<TokenType>, T> parseTokens
) {
- var tokens = tokenise(source);
+ var tokens = tokenise(sourceText);
var result = parseTokens.apply(tokens);
tokens.skip(TokenType.END);
return result;
diff --git a/src/main/java/org/shedlang/compiler/parser/Tokeniser.java b/src/main/java/org/shedlang/compiler/parser/Tokeniser.java
index 1fdd584b..85172578 100644
--- a/src/main/java/org/shedlang/compiler/parser/Tokeniser.java
+++ b/src/main/java/org/shedlang/compiler/parser/Tokeniser.java
@@ -1,9 +1,9 @@
package org.shedlang.compiler.parser;
-import org.shedlang.compiler.sources.Source;
import org.shedlang.compiler.tokeniser.RegexTokeniser;
import org.shedlang.compiler.tokeniser.Token;
import org.shedlang.compiler.tokeniser.TokenIterator;
+import org.zwobble.sourcetext.SourceText;
import java.util.List;
import java.util.Map;
@@ -74,19 +74,23 @@ public class Tokeniser {
)
);
- public static TokenIterator<TokenType> tokenise(Source source) {
- var tokens = tokeniser.tokenise(source).stream()
+ public static TokenIterator<TokenType> tokenise(SourceText sourceText) {
+ var tokens = tokeniser.tokenise(sourceText).stream()
.filter(token ->
token.tokenType() != TokenType.WHITESPACE &&
token.tokenType() != TokenType.COMMENT_SINGLE_LINE
)
.map(token ->
token.tokenType() == TokenType.IDENTIFIER
- ? new Token<>(token.source(), KEYWORDS.getOrDefault(token.value(), token.tokenType()), token.value())
+ ? new Token<>(token.sourceRange(), KEYWORDS.getOrDefault(token.value(), token.tokenType()), token.value())
: token
)
.collect(Collectors.toList());
- return new TokenIterator<>(tokens, token(source.end(), TokenType.END, ""));
+ var endPosition = sourceText.characterPosition(sourceText.characterLength());
+ var endSourceRange = endPosition.to(endPosition);
+ var endToken = token(endSourceRange, TokenType.END, "");
+
+ return new TokenIterator<>(tokens, endToken);
}
}
diff --git a/src/main/java/org/shedlang/compiler/sources/FileFragmentSource.java b/src/main/java/org/shedlang/compiler/sources/FileFragmentSource.java
index 0fc05672..ae027e99 100644
--- a/src/main/java/org/shedlang/compiler/sources/FileFragmentSource.java
+++ b/src/main/java/org/shedlang/compiler/sources/FileFragmentSource.java
@@ -1,75 +1,10 @@
package org.shedlang.compiler.sources;
-public record FileFragmentSource(
- String filename,
- String contents,
- int characterIndexStart,
- int characterIndexEnd
-) implements Source {
- static FileFragmentSource file(String filename, String contents) {
- return new FileFragmentSource(filename, contents, 0, contents.length());
- }
+import org.zwobble.sourcetext.SourceRange;
+public record FileFragmentSource(SourceRange sourceRange) implements Source {
@Override
public String describe() {
- var lines = contents.split("\n");
- var position = 0;
-
- for (var lineIndex = 0; lineIndex < lines.length; lineIndex++) {
- var line = lines[lineIndex];
- var nextLinePosition = position + line.length() + 1;
- if (nextLinePosition > characterIndexStart || nextLinePosition >= contents.length()) {
- return context(
- line,
- lineIndex,
- characterIndexStart - position,
- nextLinePosition <= characterIndexEnd
- ? 1
- : Math.max(characterIndexEnd - characterIndexStart, 1)
- );
- }
- position = nextLinePosition;
- }
- throw new RuntimeException("should be impossible (but evidently isn't)");
- }
-
- @Override
- public Source at(int characterIndexStart, int characterIndexEnd) {
- if (characterIndexStart > characterIndexEnd) {
- throw new IllegalArgumentException("characterIndexStart > characterIndexEnd");
- }
-
- return new FileFragmentSource(
- filename,
- contents,
- this.characterIndexStart + characterIndexStart,
- this.characterIndexStart + characterIndexEnd
- );
- }
-
- @Override
- public Source to(Source end) {
- if (end instanceof FileFragmentSource endFileFragment) {
- return new FileFragmentSource(filename, contents, this.characterIndexStart, endFileFragment.characterIndexEnd);
- } else {
- throw new IllegalArgumentException("end must be an instance of FileFragmentSource");
- }
- }
-
- @Override
- public Source start() {
- return at(0, 0);
- }
-
- @Override
- public Source end() {
- var length = characterIndexEnd - characterIndexStart;
- return at(length, length);
- }
-
- private String context(String line, int lineIndex, int columnIndex, int length) {
- var lineNumber = lineIndex + 1;
- var columnNumber = columnIndex + 1;
- return filename + ":" + lineNumber + ":" + columnNumber + "\n" + line + "\n" + " ".repeat(columnIndex) + "^".repeat(length);
+ return this.sourceRange.describe();
}
}
diff --git a/src/main/java/org/shedlang/compiler/sources/NullSource.java b/src/main/java/org/shedlang/compiler/sources/NullSource.java
index 2f09702c..f9e86295 100644
--- a/src/main/java/org/shedlang/compiler/sources/NullSource.java
+++ b/src/main/java/org/shedlang/compiler/sources/NullSource.java
@@ -8,32 +8,7 @@ class NullSource implements Source {
static final NullSource INSTANCE = new NullSource();
@Override
- public String contents() {
- return "";
- }
-
- @Override
public String describe() {
return "(null)";
}
-
- @Override
- public Source start() {
- return this;
- }
-
- @Override
- public Source end() {
- return this;
- }
-
- @Override
- public Source at(int characterIndexStart, int characterIndexEnd) {
- return this;
- }
-
- @Override
- public Source to(Source end) {
- return null;
- }
}
diff --git a/src/main/java/org/shedlang/compiler/sources/RuntimeSource.java b/src/main/java/org/shedlang/compiler/sources/RuntimeSource.java
index 83937493..06310478 100644
--- a/src/main/java/org/shedlang/compiler/sources/RuntimeSource.java
+++ b/src/main/java/org/shedlang/compiler/sources/RuntimeSource.java
@@ -8,32 +8,7 @@ class RuntimeSource implements Source {
static final RuntimeSource INSTANCE = new RuntimeSource();
@Override
- public String contents() {
- return "";
- }
-
- @Override
public String describe() {
return "(runtime)";
}
-
- @Override
- public Source start() {
- return this;
- }
-
- @Override
- public Source end() {
- return this;
- }
-
- @Override
- public Source at(int characterIndexStart, int characterIndexEnd) {
- return this;
- }
-
- @Override
- public Source to(Source end) {
- return null;
- }
}
diff --git a/src/main/java/org/shedlang/compiler/sources/Source.java b/src/main/java/org/shedlang/compiler/sources/Source.java
index 8dc1733c..edd9ad88 100644
--- a/src/main/java/org/shedlang/compiler/sources/Source.java
+++ b/src/main/java/org/shedlang/compiler/sources/Source.java
@@ -1,10 +1,5 @@
package org.shedlang.compiler.sources;
public interface Source {
- String contents();
String describe();
- Source start();
- Source end();
- Source at(int characterIndexStart, int characterIndexEnd);
- Source to(Source end);
}
diff --git a/src/main/java/org/shedlang/compiler/sources/SourceTexts.java b/src/main/java/org/shedlang/compiler/sources/SourceTexts.java
new file mode 100644
index 00000000..d79408e6
--- /dev/null
+++ b/src/main/java/org/shedlang/compiler/sources/SourceTexts.java
@@ -0,0 +1,12 @@
+package org.shedlang.compiler.sources;
+
+import org.zwobble.sourcetext.SourceText;
+
+public class SourceTexts {
+ public static SourceText string(String text) {
+ return SourceText.fromString("<string>", text);
+ }
+
+ private SourceTexts() {
+ }
+}
diff --git a/src/main/java/org/shedlang/compiler/sources/Sources.java b/src/main/java/org/shedlang/compiler/sources/Sources.java
index ccfadca6..f1e59cfa 100644
--- a/src/main/java/org/shedlang/compiler/sources/Sources.java
+++ b/src/main/java/org/shedlang/compiler/sources/Sources.java
@@ -1,20 +1,6 @@
package org.shedlang.compiler.sources;
-import java.nio.file.Path;
-
public class Sources {
public static Source NULL = NullSource.INSTANCE;
public static Source RUNTIME = RuntimeSource.INSTANCE;
-
- public static FileFragmentSource file(Path path, String contents) {
- return FileFragmentSource.file(path.toString(), contents);
- }
-
- public static FileFragmentSource file(String filename, String contents) {
- return FileFragmentSource.file(filename, contents);
- }
-
- public static FileFragmentSource string(String contents) {
- return FileFragmentSource.file("<string>", contents);
- }
}
diff --git a/src/main/java/org/shedlang/compiler/tokeniser/InfinitelyPaddedIterator.java b/src/main/java/org/shedlang/compiler/tokeniser/InfinitelyPaddedIterator.java
index 534020d9..54548610 100644
--- a/src/main/java/org/shedlang/compiler/tokeniser/InfinitelyPaddedIterator.java
+++ b/src/main/java/org/shedlang/compiler/tokeniser/InfinitelyPaddedIterator.java
@@ -28,4 +28,8 @@ public class InfinitelyPaddedIterator<T> {
public void moveNext() {
index++;
}
+
+ public boolean isAtStart() {
+ return this.index == 0;
+ }
}
diff --git a/src/main/java/org/shedlang/compiler/tokeniser/RegexTokeniser.java b/src/main/java/org/shedlang/compiler/tokeniser/RegexTokeniser.java
index 942960a3..a34ae3d5 100644
--- a/src/main/java/org/shedlang/compiler/tokeniser/RegexTokeniser.java
+++ b/src/main/java/org/shedlang/compiler/tokeniser/RegexTokeniser.java
@@ -1,6 +1,6 @@
package org.shedlang.compiler.tokeniser;
-import org.shedlang.compiler.sources.Source;
+import org.zwobble.sourcetext.SourceText;
import java.util.ArrayList;
import java.util.List;
@@ -47,8 +47,8 @@ public class RegexTokeniser<T> {
.collect(Collectors.toList());
}
- public List<Token<T>> tokenise(Source source) {
- var matcher = pattern.matcher(source.contents());
+ public List<Token<T>> tokenise(SourceText sourceText) {
+ var matcher = pattern.matcher(sourceText.charSequence());
var tokens = new ArrayList<Token<T>>();
while (matcher.lookingAt()) {
@@ -58,12 +58,15 @@ public class RegexTokeniser<T> {
.getAsInt();
var tokenType = this.rules.get(groupIndex - 1);
+ var startPosition = sourceText.characterPosition(matcher.regionStart());
+ var endPosition = sourceText.characterPosition(matcher.end());
tokens.add(new Token<>(
- source.at(matcher.regionStart(), matcher.end()),
+ startPosition.to(endPosition),
tokenType,
+ // TODO: the text value is redundant, can be gotten from the source range?
matcher.group()
));
- matcher.region(matcher.end(), source.contents().length());
+ matcher.region(matcher.end(), sourceText.characterLength());
}
return tokens;
diff --git a/src/main/java/org/shedlang/compiler/tokeniser/Token.java b/src/main/java/org/shedlang/compiler/tokeniser/Token.java
index e79d89eb..7e131304 100644
--- a/src/main/java/org/shedlang/compiler/tokeniser/Token.java
+++ b/src/main/java/org/shedlang/compiler/tokeniser/Token.java
@@ -1,10 +1,10 @@
package org.shedlang.compiler.tokeniser;
-import org.shedlang.compiler.sources.Source;
+import org.zwobble.sourcetext.SourceRange;
-public record Token<T>(Source source, T tokenType, String value) {
- public static <T> Token<T> token(Source source, T tokenType, String value) {
- return new Token<T>(source, tokenType, value);
+public record Token<T>(SourceRange sourceRange, T tokenType, String value) {
+ public static <T> Token<T> token(SourceRange sourceRange, T tokenType, String value) {
+ return new Token<T>(sourceRange, tokenType, value);
}
public String describe() {
diff --git a/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java b/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java
index 3462b4b7..9b3467a8 100644
--- a/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java
+++ b/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java
@@ -1,6 +1,6 @@
package org.shedlang.compiler.tokeniser;
-import org.shedlang.compiler.sources.Source;
+import org.zwobble.sourcetext.SourcePosition;
import java.util.List;
@@ -15,11 +15,11 @@ public class TokenIterator<TTokenType> {
nextValue(tokenType);
}
- public String nextValue(TTokenType tokenType) {
+ public Token<TTokenType> next(TTokenType tokenType) {
var token = tokens.get();
if (token.tokenType().equals(tokenType)) {
tokens.moveNext();
- return token.value();
+ return token;
} else {
throw new UnexpectedTokenException(
tokenType.toString(),
@@ -28,6 +28,10 @@ public class TokenIterator<TTokenType> {
}
}
+ public String nextValue(TTokenType tokenType) {
+ return this.next(tokenType).value();
+ }
+
public boolean isNext(TTokenType tokenType) {
return tokens.get().tokenType().equals(tokenType);
}
@@ -54,7 +58,15 @@ public class TokenIterator<TTokenType> {
return tokens.get(offset);
}
- public Source position() {
- return tokens.get().source().start();
+ public SourcePosition startPosition() {
+ return tokens.get().sourceRange().start();
+ }
+
+ public SourcePosition endPosition() {
+ if (tokens.isAtStart()) {
+ return startPosition();
+ } else {
+ return tokens.get(-1).sourceRange().end();
+ }
}
}
diff --git a/src/main/java/org/shedlang/compiler/tokeniser/UnexpectedTokenException.java b/src/main/java/org/shedlang/compiler/tokeniser/UnexpectedTokenException.java
index 75fa19dc..714edf6b 100644
--- a/src/main/java/org/shedlang/compiler/tokeniser/UnexpectedTokenException.java
+++ b/src/main/java/org/shedlang/compiler/tokeniser/UnexpectedTokenException.java
@@ -1,6 +1,7 @@
package org.shedlang.compiler.tokeniser;
import org.shedlang.compiler.errors.SourceError;
+import org.shedlang.compiler.sources.FileFragmentSource;
public class UnexpectedTokenException extends SourceError {
private final String expected;
@@ -12,7 +13,7 @@ public class UnexpectedTokenException extends SourceError {
) {
super(
"Expected: " + expected + "\nBut got: " + actual.tokenType() + " \"" + actual.value() + "\"",
- actual.source()
+ new FileFragmentSource(actual.sourceRange())
);
this.expected = expected;
this.actual = actual;