diff options
| author | Michael Williamson <mike@zwobble.org> | 2025-01-08 19:14:03 +0000 |
|---|---|---|
| committer | Michael Williamson <mike@zwobble.org> | 2025-01-09 20:36:42 +0000 |
| commit | 3666a4af2eb35177eecd557d2a75f4bac9da75b9 (patch) | |
| tree | a85230de832bc7f970dd89aaa6bb2da6ea5875f4 /src/main/java/org | |
| parent | 1e6e1f7bb8776fc013f76828ed8df3f9297d095b (diff) | |
Use source text to implement FileFragmentSource
Diffstat (limited to 'src/main/java/org')
20 files changed, 276 insertions, 363 deletions
diff --git a/src/main/java/org/shedlang/compiler/ShedCompiler.java b/src/main/java/org/shedlang/compiler/ShedCompiler.java index 0efbe1d5..a57a3386 100644 --- a/src/main/java/org/shedlang/compiler/ShedCompiler.java +++ b/src/main/java/org/shedlang/compiler/ShedCompiler.java @@ -9,11 +9,11 @@ import org.shedlang.compiler.config.ProjectConfig; import org.shedlang.compiler.parser.Parser; import org.shedlang.compiler.shid.ShidCodeGen; import org.shedlang.compiler.shid.ShidNamespace; -import org.shedlang.compiler.sources.Sources; import org.shedlang.compiler.typechecker.TypeChecker; import org.shedlang.compiler.typechecker.TypeCheckerContext; import org.shedlang.compiler.types.*; import org.shedlang.compiler.util.P; +import org.zwobble.sourcetext.SourceText; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -131,9 +131,9 @@ public class ShedCompiler { var namespaceName = new NamespaceName(namespaceNameParts); var sourceContents = Files.readString(sourceFilePath, StandardCharsets.UTF_8); - var source = Sources.file(sourceFilePath, sourceContents); + var sourceText = SourceText.fromString(sourceFilePath.toString(), sourceContents); - var tokens = tokenise(source); + var tokens = tokenise(sourceText); var untypedNode = Parser.parseNamespace(tokens, namespaceName); untypedNamespaceNodes.add(untypedNode); diff --git a/src/main/java/org/shedlang/compiler/config/BinConfig.java b/src/main/java/org/shedlang/compiler/config/BinConfig.java index a81b2737..3541c40b 100644 --- a/src/main/java/org/shedlang/compiler/config/BinConfig.java +++ b/src/main/java/org/shedlang/compiler/config/BinConfig.java @@ -2,9 +2,9 @@ package org.shedlang.compiler.config; import org.shedlang.compiler.config.errors.ConfigError; import org.shedlang.compiler.config.json.Json5ObjectWithSource; -import org.shedlang.compiler.config.json.Json5Source; import org.shedlang.compiler.parser.SourceParser; -import org.shedlang.compiler.sources.Sources; +import org.shedlang.compiler.sources.FileFragmentSource; +import org.shedlang.compiler.sources.SourceTexts; import org.shedlang.compiler.tokeniser.UnexpectedTokenException; import org.shedlang.compiler.types.NamespaceName; @@ -17,7 +17,8 @@ public record BinConfig(String name, NamespaceName mainNamespace) { var mainNamespaceString = object.getString("mainNamespace"); try { - var mainNamespace = SourceParser.parseNamespaceName(Sources.string(mainNamespaceString.value())); + var sourceText = SourceTexts.string(mainNamespaceString.value()); + var mainNamespace = SourceParser.parseNamespaceName(sourceText); return new BinConfig(name, mainNamespace); } catch (UnexpectedTokenException exception) { throw new ConfigError( @@ -26,7 +27,7 @@ public record BinConfig(String name, NamespaceName mainNamespace) { exception.getMessage(), exception.source().describe() ), - Json5Source.fromSourceRange(mainNamespaceString.sourceRange()) + new FileFragmentSource(mainNamespaceString.sourceRange()) ); } } diff --git a/src/main/java/org/shedlang/compiler/config/errors/Json5SyntaxConfigError.java b/src/main/java/org/shedlang/compiler/config/errors/Json5SyntaxConfigError.java index 2009af43..d67b1a77 100644 --- a/src/main/java/org/shedlang/compiler/config/errors/Json5SyntaxConfigError.java +++ b/src/main/java/org/shedlang/compiler/config/errors/Json5SyntaxConfigError.java @@ -1,6 +1,6 @@ package org.shedlang.compiler.config.errors; -import org.shedlang.compiler.config.json.Json5Source; +import org.shedlang.compiler.sources.FileFragmentSource; import org.shedlang.compiler.sources.Source; import org.zwobble.json5.parser.Json5ParseError; @@ -8,7 +8,7 @@ public class Json5SyntaxConfigError extends ConfigError { public static Json5SyntaxConfigError fromJson5ParseError( Json5ParseError error ) { - var source = Json5Source.fromSourceRange(error.sourceRange()); + var source = new FileFragmentSource(error.sourceRange()); return new Json5SyntaxConfigError(error.getMessage(), source); } diff --git a/src/main/java/org/shedlang/compiler/config/json/Json5ObjectWithSource.java b/src/main/java/org/shedlang/compiler/config/json/Json5ObjectWithSource.java index cd7d811e..48db3b9b 100644 --- a/src/main/java/org/shedlang/compiler/config/json/Json5ObjectWithSource.java +++ b/src/main/java/org/shedlang/compiler/config/json/Json5ObjectWithSource.java @@ -3,6 +3,7 @@ package org.shedlang.compiler.config.json; import org.shedlang.compiler.config.errors.ConfigError; import org.shedlang.compiler.config.errors.Json5SyntaxConfigError; import org.shedlang.compiler.config.errors.UnexpectedTypeConfigError; +import org.shedlang.compiler.sources.FileFragmentSource; import org.zwobble.json5.parser.Json5ParseError; import org.zwobble.json5.parser.Json5Parser; import org.zwobble.json5.values.*; @@ -33,7 +34,7 @@ public class Json5ObjectWithSource { document.path(), Json5Object.class, document.getClass(), - Json5Source.fromSourceRange(document.sourceRange()) + new FileFragmentSource(document.sourceRange()) ); } } catch (Json5ParseError error) { @@ -75,7 +76,7 @@ public class Json5ObjectWithSource { .orElseThrow(() -> new ConfigError( // TODO: escape member name MessageFormat.format("{0} is missing {1}", object.path(), memberName), - Json5Source.fromSourceRange(object.sourceRange()) + new FileFragmentSource(object.sourceRange()) )); } @@ -94,7 +95,7 @@ public class Json5ObjectWithSource { "%s must be a 64-bit integer", numberFinite.get().path() ), - Json5Source.fromSourceRange(numberFinite.get().sourceRange()) + new FileFragmentSource(numberFinite.get().sourceRange()) ); } } @@ -104,7 +105,7 @@ public class Json5ObjectWithSource { .orElseThrow(() -> new ConfigError( // TODO: escape member name MessageFormat.format("{0} is missing {1}", object.path(), memberName), - Json5Source.fromSourceRange(object.sourceRange()) + new FileFragmentSource(object.sourceRange()) )); } @@ -130,7 +131,7 @@ public class Json5ObjectWithSource { value.path(), valueType, value.getClass(), - Json5Source.fromSourceRange(value.sourceRange()) + new FileFragmentSource(value.sourceRange()) ); } } diff --git a/src/main/java/org/shedlang/compiler/config/json/Json5Source.java b/src/main/java/org/shedlang/compiler/config/json/Json5Source.java deleted file mode 100644 index e2a556e9..00000000 --- a/src/main/java/org/shedlang/compiler/config/json/Json5Source.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.shedlang.compiler.config.json; - -import org.shedlang.compiler.sources.FileFragmentSource; -import org.shedlang.compiler.sources.Source; -import org.zwobble.sourcetext.SourceRange; - -public class Json5Source { - private Json5Source() {} - - public static Source fromSourceRange(SourceRange sourceRange) { - return new FileFragmentSource( - sourceRange.sourceText().name(), - sourceRange.sourceText().charBuffer(0, sourceRange.sourceText().characterLength()).toString(), - sourceRange.start().characterIndex(), - sourceRange.end().characterIndex() - ); - } -} diff --git a/src/main/java/org/shedlang/compiler/parser/OperationParser.java b/src/main/java/org/shedlang/compiler/parser/OperationParser.java index 9a4a25ea..e56b3ffe 100644 --- a/src/main/java/org/shedlang/compiler/parser/OperationParser.java +++ b/src/main/java/org/shedlang/compiler/parser/OperationParser.java @@ -1,11 +1,16 @@ package org.shedlang.compiler.parser; import org.shedlang.compiler.tokeniser.TokenIterator; +import org.zwobble.sourcetext.SourcePosition; import java.util.stream.Stream; public interface OperationParser<TOperatorPrecedence, TUntypedExpressionNode> { TOperatorPrecedence precedence(); Stream<TokenType> operatorTokenTypes(); - TUntypedExpressionNode parse(TUntypedExpressionNode left, TokenIterator<TokenType> tokens); + TUntypedExpressionNode parse( + SourcePosition start, + TUntypedExpressionNode left, + TokenIterator<TokenType> tokens + ); } diff --git a/src/main/java/org/shedlang/compiler/parser/Parser.java b/src/main/java/org/shedlang/compiler/parser/Parser.java index 680ead15..cda360e3 100644 --- a/src/main/java/org/shedlang/compiler/parser/Parser.java +++ b/src/main/java/org/shedlang/compiler/parser/Parser.java @@ -2,12 +2,15 @@ package org.shedlang.compiler.parser; import org.shedlang.compiler.ast.untyped.*; import org.shedlang.compiler.parser.errors.*; +import org.shedlang.compiler.sources.FileFragmentSource; import org.shedlang.compiler.sources.Source; import org.shedlang.compiler.tokeniser.TokenIterator; import org.shedlang.compiler.tokeniser.UnexpectedTokenException; import org.shedlang.compiler.types.NamespaceName; import org.shedlang.compiler.types.Visibility; import org.shedlang.compiler.util.Iterables; +import org.zwobble.sourcetext.SourcePosition; +import org.zwobble.sourcetext.SourceRange; import java.math.BigInteger; import java.util.ArrayList; @@ -53,27 +56,29 @@ public class Parser { } @Override - public UntypedExpressionNode parse(UntypedExpressionNode left, TokenIterator<TokenType> tokens) { - var start = left.source().start(); - + public UntypedExpressionNode parse( + SourcePosition start, + UntypedExpressionNode left, + TokenIterator<TokenType> tokens + ) { tokens.skip(operatorTokenType); var right = parseSubExpression(tokens, precedence); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var sourceRange = new FileFragmentSource(start.to(end)); return new UntypedBinaryOperationNode( operator, left, right, - source + sourceRange ); } } private static UntypedArgsNode parseArgs(TokenIterator<TokenType> tokens) { - var argsStart = tokens.position(); + var argsStart = tokens.startPosition(); var positional = new ArrayList<UntypedExpressionNode>(); var named = new ArrayList<UntypedNamedArgNode>(); @@ -82,14 +87,14 @@ public class Parser { () -> tokens.isNext(TokenType.SYMBOL_PAREN_CLOSE), () -> { if (tokens.trySkip(TokenType.SYMBOL_DOT)) { - var argStart = tokens.position(); + var argStart = tokens.startPosition(); var argName = tokens.nextValue(TokenType.IDENTIFIER); tokens.skip(TokenType.SYMBOL_EQUALS); var expression = parseTopLevelExpression(tokens); - var argEnd = tokens.position(); - var argSource = argStart.to(argEnd); + var argEnd = tokens.endPosition(); + var argSource = new FileFragmentSource(argStart.to(argEnd)); named.add(new UntypedNamedArgNode(argName, expression, argSource)); } else { @@ -104,14 +109,14 @@ public class Parser { () -> tokens.trySkip(TokenType.SYMBOL_COMMA) ); - var argsEnd = tokens.position(); - var argsSource = argsStart.to(argsEnd); + var argsEnd = tokens.endPosition(); + var argsSource = new FileFragmentSource(argsStart.to(argsEnd)); return new UntypedArgsNode(positional, named, argsSource); } private static UntypedAssignNode parseAssign(TokenIterator<TokenType> tokens) { - var start = tokens.position(); + var start = tokens.startPosition(); var target = parseTarget(tokens); @@ -121,20 +126,20 @@ public class Parser { tokens.skip(TokenType.SYMBOL_SEMICOLON); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedAssignNode(target, expression, source); } private static UntypedBlockNode parseBlock(TokenIterator<TokenType> tokens) { - var start = tokens.position(); + var start = tokens.startPosition(); tokens.skip(TokenType.SYMBOL_BRACE_OPEN); if (tokens.trySkip(TokenType.SYMBOL_BRACE_CLOSE)) { - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedBlockNode(List.of(), source); } @@ -152,30 +157,30 @@ public class Parser { ) { tokens.skip(TokenType.SYMBOL_BRACE_CLOSE); - var end = tokens.position(); + var end = tokens.endPosition(); var source = start.to(end); - return new UntypedBlockNode(body, source); + return new UntypedBlockNode(body, new FileFragmentSource(source)); } } } private static UntypedExpressionNode parseBoolLiteral(TokenIterator<TokenType> tokens) { - var start = tokens.position(); + var start = tokens.startPosition(); var value = tokens.trySkip(TokenType.KEYWORD_TRUE); if (!value) { tokens.skip(TokenType.KEYWORD_FALSE); } - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedBoolLiteralNode(value, source); } private static UntypedFunctionStatementNode parseBreak(TokenIterator<TokenType> tokens) { - var start = tokens.position(); + var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_BREAK); @@ -185,8 +190,8 @@ public class Parser { tokens.skip(TokenType.SYMBOL_SEMICOLON); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedBreakNode(expression, source); } @@ -210,9 +215,11 @@ public class Parser { } @Override - public UntypedExpressionNode parse(UntypedExpressionNode left, TokenIterator<TokenType> tokens) { - var start = left.source().start(); - + public UntypedExpressionNode parse( + SourcePosition start, + UntypedExpressionNode left, + TokenIterator<TokenType> tokens + ) { var hasEffects = tokens.trySkip(TokenType.SYMBOL_BANG); var typeLevelArgs = parseTypeLevelArgsOptional(tokens); @@ -221,8 +228,8 @@ public class Parser { var args = parseArgs(tokens); tokens.skip(TokenType.SYMBOL_PAREN_CLOSE); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedCallNode( left, @@ -249,9 +256,11 @@ public class Parser { } @Override - public UntypedTypeLevelExpressionNode parse(UntypedTypeLevelExpressionNode left, TokenIterator<TokenType> tokens) { - var start = left.source().start(); - + public UntypedTypeLevelExpressionNode parse( + SourcePosition start, + UntypedTypeLevelExpressionNode left, + TokenIterator<TokenType> tokens + ) { tokens.skip(TokenType.SYMBOL_SQUARE_OPEN); var args = parseMany( () -> tokens.isNext(TokenType.SYMBOL_SQUARE_CLOSE), @@ -260,8 +269,8 @@ public class Parser { ); tokens.skip(TokenType.SYMBOL_SQUARE_CLOSE); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedConstructedTypeNode(left, args, source); } @@ -270,7 +279,7 @@ public class Parser { private static UntypedEffectDefinitionNode parseEffectDefinition( TokenIterator<TokenType> tokens ) { - var start = tokens.position(); + var start = tokens.startPosition(); var visibility = parseVisibility(tokens); @@ -288,8 +297,8 @@ public class Parser { tokens.skip(TokenType.SYMBOL_BRACE_CLOSE); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedEffectDefinitionNode(name, visibility, functions, source); } @@ -297,7 +306,7 @@ public class Parser { private static UntypedEffectDefinitionMethodNode parseEffectDefinitionField( TokenIterator<TokenType> tokens ) { - var start = tokens.position(); + var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_FUN); @@ -307,8 +316,8 @@ public class Parser { tokens.skip(TokenType.SYMBOL_SEMICOLON); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedEffectDefinitionMethodNode(name, type, source); } @@ -316,7 +325,7 @@ public class Parser { private static UntypedFunctionTypeNode parseEffectDefinitionFieldType( TokenIterator<TokenType> tokens ) { - var start = tokens.position(); + var start = tokens.startPosition(); tokens.skip(TokenType.SYMBOL_PAREN_OPEN); var params = parseParamTypes(tokens); @@ -326,8 +335,8 @@ public class Parser { var returnType = parseTopLevelTypeLevelExpression(tokens); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedFunctionTypeNode(params, Optional.empty(), returnType, source); } @@ -352,6 +361,7 @@ public class Parser { TokenIterator<TokenType> tokens, Optional<OperatorPrecedence> parentPrecedence ) { + var start = tokens.startPosition(); var left = parsePrimaryExpression(tokens); while (true) { @@ -364,12 +374,12 @@ public class Parser { ) { return left; } - left = operationParser.parse(left, tokens); + left = operationParser.parse(start, left, tokens); } } private static UntypedFunctionStatementNode parseExpressionStatement(TokenIterator<TokenType> tokens) { - var start = tokens.position(); + var start = tokens.startPosition(); var expression = parseTopLevelExpression(tokens); @@ -377,8 +387,8 @@ public class Parser { ? expressionStatementBehaviour(branchingExpression.exhaustiveChildBlocks()) == UntypedFunctionStatementBehaviour.BLOCK_RETURN : !tokens.trySkip(TokenType.SYMBOL_SEMICOLON); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return isReturn ? new UntypedBlockReturnNode(expression, source) @@ -406,7 +416,7 @@ public class Parser { } private static UntypedFunctionDefinitionNode parseFunctionDefinition(TokenIterator<TokenType> tokens) { - var start = tokens.position(); + var start = tokens.startPosition(); var visibility = parseVisibility(tokens); @@ -417,8 +427,8 @@ public class Parser { var body = parseBlock(tokens); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedFunctionDefinitionNode( name, @@ -432,7 +442,7 @@ public class Parser { private static UntypedExpressionNode parseFunctionExpression( TokenIterator<TokenType> tokens ) { - var start = tokens.position(); + var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_FN); @@ -440,8 +450,8 @@ public class Parser { var body = parseBlock(tokens); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedFunctionExpressionNode( signature, @@ -451,7 +461,7 @@ public class Parser { } private static UntypedFunctionStatementNode parseFunctionReturn(TokenIterator<TokenType> tokens) { - var start = tokens.position(); + var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_RETURN); @@ -461,8 +471,8 @@ public class Parser { tokens.skip(TokenType.SYMBOL_SEMICOLON); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedFunctionReturnNode(expression, source); } @@ -470,7 +480,7 @@ public class Parser { static UntypedFunctionSignatureNode parseFunctionSignature( TokenIterator<TokenType> tokens ) { - var start = tokens.position(); + var start = tokens.startPosition(); var typeLevelParams = parseTypeLevelParamsOptional(tokens); @@ -482,8 +492,8 @@ public class Parser { var returnType = parseFunctionSignatureReturnType(tokens); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedFunctionSignatureNode( typeLevelParams, @@ -555,7 +565,7 @@ public class Parser { private static UntypedTypeLevelExpressionNode parseFunctionType( TokenIterator<TokenType> tokens ) { - var start = tokens.position(); + var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_FN); @@ -569,8 +579,8 @@ public class Parser { var returnType = parseTopLevelTypeLevelExpression(tokens); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedFunctionTypeNode( params, @@ -589,7 +599,7 @@ public class Parser { } private static UntypedExpressionNode parseIf(TokenIterator<TokenType> tokens) { - var start = tokens.position(); + var start = tokens.startPosition(); var conditionalBranches = new ArrayList<UntypedConditionalBranchNode>(); var elseBody = Optional.<UntypedBlockNode>empty(); @@ -609,18 +619,21 @@ public class Parser { } - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedIfNode( conditionalBranches, - elseBody.orElseGet(() -> new UntypedBlockNode(List.of(), end)), + elseBody.orElseGet(() -> new UntypedBlockNode( + List.of(), + new FileFragmentSource(end.to(end)) + )), source ); } private static UntypedConditionalBranchNode parseConditionalBranch(TokenIterator<TokenType> tokens) { - var start = tokens.position(); + var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_IF); @@ -630,14 +643,14 @@ public class Parser { var body = parseBlock(tokens); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedConditionalBranchNode(condition, body, source); } private static UntypedImportNode parseImport(TokenIterator<TokenType> tokens) { - var start = tokens.position(); + var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_IMPORT); @@ -645,21 +658,21 @@ public class Parser { tokens.skip(TokenType.SYMBOL_SEMICOLON); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedImportNode(namespaceName, source); } private static UntypedExpressionNode parseInt(TokenIterator<TokenType> tokens) { - var start = tokens.position(); + var start = tokens.startPosition(); var string = tokens.nextValue(TokenType.INT); var value = new BigInteger(string); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); if (string.length() > 1 && string.startsWith("0")) { throw new IntLiteralCannotHaveLeadingZeroesError(source); @@ -731,15 +744,17 @@ public class Parser { } @Override - public UntypedExpressionNode parse(UntypedExpressionNode left, TokenIterator<TokenType> tokens) { - var start = left.source().start(); - + public UntypedExpressionNode parse( + SourcePosition start, + UntypedExpressionNode left, + TokenIterator<TokenType> tokens + ) { tokens.skip(TokenType.SYMBOL_DOT); var memberName = tokens.nextValue(TokenType.IDENTIFIER); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedMemberAccessNode(left, memberName, source); } @@ -797,7 +812,7 @@ public class Parser { } private static UntypedParamsNode parseParams(TokenIterator<TokenType> tokens) { - var paramsStart = tokens.position(); + var paramsStart = tokens.startPosition(); var positional = new ArrayList<UntypedParamNode>(); var named = new ArrayList<UntypedParamNode>(); @@ -805,7 +820,7 @@ public class Parser { parseMany( () -> tokens.isNext(TokenType.SYMBOL_PAREN_CLOSE), () -> { - var paramStart = tokens.position(); + var paramStart = tokens.startPosition(); var isNamed = tokens.trySkip(TokenType.SYMBOL_DOT); @@ -815,8 +830,8 @@ public class Parser { ? Optional.of(parseTopLevelTypeLevelExpression(tokens)) : Optional.<UntypedTypeLevelExpressionNode>empty(); - var paramEnd = tokens.position(); - var paramSource = paramStart.to(paramEnd); + var paramEnd = tokens.endPosition(); + var paramSource = new FileFragmentSource(paramStart.to(paramEnd)); if (!isNamed && !named.isEmpty()) { throw new PositionalParamAfterNamedParamError(paramSource); @@ -829,14 +844,14 @@ public class Parser { () -> tokens.trySkip(TokenType.SYMBOL_COMMA) ); - var paramsEnd = tokens.position(); - var paramsSource = paramsStart.to(paramsEnd); + var paramsEnd = tokens.endPosition(); + var paramsSource = new FileFragmentSource(paramsStart.to(paramsEnd)); return new UntypedParamsNode(positional, named, paramsSource); } private static UntypedFunctionTypeParamsNode parseParamTypes(TokenIterator<TokenType> tokens) { - var paramsStart = tokens.position(); + var paramsStart = tokens.startPosition(); var positional = new ArrayList<UntypedTypeLevelExpressionNode>(); var named = new ArrayList<UntypedFunctionTypeNamedParamNode>(); @@ -844,7 +859,7 @@ public class Parser { parseMany( () -> tokens.isNext(TokenType.SYMBOL_PAREN_CLOSE), () -> { - var paramStart = tokens.position(); + var paramStart = tokens.startPosition(); var isNamed = tokens.trySkip(TokenType.SYMBOL_DOT); @@ -853,8 +868,8 @@ public class Parser { tokens.skip(TokenType.SYMBOL_COLON); var type = parseTopLevelTypeLevelExpression(tokens); - var paramEnd = tokens.position(); - var paramSource = paramStart.to(paramEnd); + var paramEnd = tokens.endPosition(); + var paramSource = new FileFragmentSource(paramStart.to(paramEnd)); named.add(new UntypedFunctionTypeNamedParamNode(name, type, paramSource)); } else { @@ -871,10 +886,10 @@ public class Parser { () -> tokens.trySkip(TokenType.SYMBOL_COMMA) ); - var paramsEnd = tokens.position(); + var paramsEnd = tokens.endPosition(); var paramsSource = paramsStart.to(paramsEnd); - return new UntypedFunctionTypeParamsNode(positional, named, paramsSource); + return new UntypedFunctionTypeParamsNode(positional, named, new FileFragmentSource(paramsSource)); } private static UntypedExpressionNode parsePrimaryExpression(TokenIterator<TokenType> tokens) { @@ -922,7 +937,7 @@ public class Parser { } private static UntypedFunctionStatementNode parseResume(TokenIterator<TokenType> tokens) { - var start = tokens.position(); + var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_RESUME); @@ -932,38 +947,44 @@ public class Parser { tokens.skip(TokenType.SYMBOL_SEMICOLON); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedResumeNode(expression, source); } private static UntypedExpressionNode parseStringLiteral(TokenIterator<TokenType> tokens) { - var start = tokens.position(); + var start = tokens.startPosition(); - var tokenValue = tokens.nextValue(TokenType.STRING); - var escapedValue = tokenValue.substring(1, tokenValue.length() - 1); - var unescapedValue = unescape(escapedValue, start); + var token = tokens.next(TokenType.STRING); + var stringContentsStart = token.sourceRange().characterPosition(1); + var stringContentsEnd = token.sourceRange() + .characterPosition(token.sourceRange().characterLength() - 1); + var stringContents = stringContentsStart.to(stringContentsEnd); + var unescapedValue = unescape(stringContents); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedStringLiteralNode(unescapedValue, source); } private static final Pattern STRING_ESCAPE_PATTERN = Pattern.compile("\\\\(.)"); - private static String unescape(String value, Source source) { - var matcher = STRING_ESCAPE_PATTERN.matcher(value); + private static String unescape(SourceRange sourceRange) { + var stringContents = sourceRange.charSequence(); + var matcher = STRING_ESCAPE_PATTERN.matcher(stringContents); var result = new StringBuilder(); var lastIndex = 0; while (matcher.find()) { - result.append(value.subSequence(lastIndex, matcher.start())); + result.append(stringContents.subSequence(lastIndex, matcher.start())); var code = matcher.group(1); - result.append(unescapeCharacter(code, source.at(matcher.start(), matcher.end()))); + var escapeSequenceSourceRange = sourceRange.characterPosition(matcher.start()) + .to(sourceRange.characterPosition(matcher.end())); + result.append(unescapeCharacter(code, new FileFragmentSource(escapeSequenceSourceRange))); lastIndex = matcher.end(); } - result.append(value.subSequence(lastIndex, value.length())); + result.append(stringContents.subSequence(lastIndex, stringContents.length())); return result.toString(); } @@ -979,7 +1000,7 @@ public class Parser { } private static UntypedNamespaceStatementNode parseStructDefinition(TokenIterator<TokenType> tokens) { - var start = tokens.position(); + var start = tokens.startPosition(); var visibility = parseVisibility(tokens); @@ -1008,8 +1029,8 @@ public class Parser { tokens.skip(TokenType.SYMBOL_BRACE_CLOSE); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedStructDefinitionNode( name, @@ -1026,7 +1047,7 @@ public class Parser { TokenIterator<TokenType> tokens, boolean structHasPositionalFields ) { - var start = tokens.position(); + var start = tokens.startPosition(); var fieldIsPositional = !tokens.trySkip(TokenType.SYMBOL_DOT); var name = tokens.nextValue(TokenType.IDENTIFIER); @@ -1035,8 +1056,8 @@ public class Parser { var type = parseTopLevelTypeLevelExpression(tokens); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); if (structHasPositionalFields ^ fieldIsPositional) { throw new InconsistentStructFieldsError(source); @@ -1050,7 +1071,7 @@ public class Parser { } private static UntypedNamespaceStatementNode parseSumTypeDefinition(TokenIterator<TokenType> tokens) { - var start = tokens.position(); + var start = tokens.startPosition(); var visibility = parseVisibility(tokens); @@ -1069,8 +1090,8 @@ public class Parser { tokens.skip(TokenType.SYMBOL_SEMICOLON); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedSumTypeDefinitionNode( name, @@ -1093,7 +1114,7 @@ public class Parser { TokenIterator<TokenType> tokens, NamespaceName namespaceName ) { - var start = tokens.position(); + var start = tokens.startPosition(); var imports = parseMany( () -> !tokens.isNext(TokenType.KEYWORD_IMPORT), @@ -1107,25 +1128,25 @@ public class Parser { () -> true ); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedNamespaceNode(namespaceName, imports, body, source); } private static UntypedExpressionNode parseReference(TokenIterator<TokenType> tokens) { - var start = tokens.position(); + var start = tokens.startPosition(); var name = tokens.nextValue(TokenType.IDENTIFIER); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedReferenceNode(name, source); } private static UntypedTargetNode parseTarget(TokenIterator<TokenType> tokens) { - var start = tokens.position(); + var start = tokens.startPosition(); return switch (tokens.peek().tokenType()) { case KEYWORD_ASSIGN -> { @@ -1133,8 +1154,8 @@ public class Parser { var name = tokens.nextValue(TokenType.IDENTIFIER); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); yield new UntypedTargetNode.Reassign(name, source); } @@ -1146,8 +1167,8 @@ public class Parser { var name = tokens.nextValue(TokenType.IDENTIFIER); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); yield new UntypedTargetNode.Val(name, visibility, source); } @@ -1157,8 +1178,8 @@ public class Parser { var name = tokens.nextValue(TokenType.IDENTIFIER); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); yield new UntypedTargetNode.Var(name, source); } @@ -1174,8 +1195,8 @@ public class Parser { tokens.skip(TokenType.SYMBOL_PAREN_CLOSE); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); yield new UntypedTargetNode.Tuple(elements, source); } @@ -1183,8 +1204,8 @@ public class Parser { case SYMBOL_UNDERSCORE -> { tokens.skip(TokenType.SYMBOL_UNDERSCORE); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); yield new UntypedTargetNode.Ignore(source); } @@ -1197,7 +1218,7 @@ public class Parser { private static UntypedExpressionNode parseTry( TokenIterator<TokenType> tokens ) { - var start = tokens.position(); + var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_TRY); @@ -1217,14 +1238,14 @@ public class Parser { tokens.skip(TokenType.SYMBOL_BRACE_CLOSE); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedTryNode(body, handledEffect, handlerMethods, source); } private static UntypedTryHandlerMethodNode parseTryHandlerMethod(TokenIterator<TokenType> tokens) { - var start = tokens.position(); + var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_FUN); var name = tokens.nextValue(TokenType.IDENTIFIER); @@ -1235,8 +1256,8 @@ public class Parser { var body = parseBlock(tokens); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedTryHandlerMethodNode( name, @@ -1251,7 +1272,7 @@ public class Parser { private static UntypedExpressionNode parseTuple( TokenIterator<TokenType> tokens ) { - var start = tokens.position(); + var start = tokens.startPosition(); tokens.skip(TokenType.SYMBOL_HASH_PAREN_OPEN); @@ -1263,8 +1284,8 @@ public class Parser { tokens.skip(TokenType.SYMBOL_PAREN_CLOSE); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedTupleNode(elements, source); } @@ -1272,7 +1293,7 @@ public class Parser { private static UntypedTypeLevelExpressionNode parseTupleType( TokenIterator<TokenType> tokens ) { - var start = tokens.position(); + var start = tokens.startPosition(); tokens.skip(TokenType.SYMBOL_HASH_PAREN_OPEN); @@ -1284,8 +1305,8 @@ public class Parser { tokens.skip(TokenType.SYMBOL_PAREN_CLOSE); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedTupleTypeNode(elements, source); } @@ -1293,7 +1314,7 @@ public class Parser { private static Optional<UntypedTypeLevelArgsNode> parseTypeLevelArgsOptional( TokenIterator<TokenType> tokens ) { - var start = tokens.position(); + var start = tokens.startPosition(); if (tokens.trySkip(TokenType.SYMBOL_SQUARE_OPEN)) { var params = parseMany( @@ -1304,8 +1325,8 @@ public class Parser { tokens.skip(TokenType.SYMBOL_SQUARE_CLOSE); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return Optional.of(new UntypedTypeLevelArgsNode(params, source)); } else { @@ -1328,6 +1349,7 @@ public class Parser { TokenIterator<TokenType> tokens, Optional<OperatorPrecedence> parentPrecedence ) { + var start = tokens.startPosition(); var left = parsePrimaryTypeLevelExpression(tokens); while (true) { @@ -1340,7 +1362,7 @@ public class Parser { ) { return left; } - left = operationParser.parse(left, tokens); + left = operationParser.parse(start, left, tokens); } } @@ -1363,14 +1385,14 @@ public class Parser { } static UntypedTypeLevelParamNode parseTypeLevelParam(TokenIterator<TokenType> tokens) { - var start = tokens.position(); + var start = tokens.startPosition(); var isEffectParam = tokens.trySkip(TokenType.SYMBOL_BANG); var name = tokens.nextValue(TokenType.IDENTIFIER); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return isEffectParam ? new UntypedEffectParamNode(name, source) @@ -1378,23 +1400,23 @@ public class Parser { } private static UntypedTypeLevelReferenceNode parseTypeLevelReference(TokenIterator<TokenType> tokens) { - var start = tokens.position(); + var start = tokens.startPosition(); var name = tokens.nextValue(TokenType.IDENTIFIER); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedTypeLevelReferenceNode(name, source); } private static UntypedExpressionNode parseUnitLiteral(TokenIterator<TokenType> tokens) { - var start = tokens.position(); + var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_UNIT); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedUnitLiteralNode(source); } @@ -1405,7 +1427,7 @@ public class Parser { } private static UntypedExpressionNode parseWhen(TokenIterator<TokenType> tokens) { - var start = tokens.position(); + var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_WHEN); @@ -1421,14 +1443,14 @@ public class Parser { ); tokens.skip(TokenType.SYMBOL_BRACE_CLOSE); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedWhenNode(expression, cases, source); } private static UntypedWhenCaseNode parseWhenCase(TokenIterator<TokenType> tokens) { - var start = tokens.position(); + var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_IS); @@ -1436,8 +1458,8 @@ public class Parser { var body = parseBlock(tokens); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedWhenCaseNode(caseType, body, source); } @@ -1445,7 +1467,7 @@ public class Parser { private static UntypedFunctionStatementNode parseWhile( TokenIterator<TokenType> tokens ) { - var start = tokens.position(); + var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_WHILE); tokens.skip(TokenType.SYMBOL_PAREN_OPEN); @@ -1454,8 +1476,8 @@ public class Parser { var body = parseBlock(tokens); - var end = tokens.position(); - var source = start.to(end); + var end = tokens.endPosition(); + var source = new FileFragmentSource(start.to(end)); return new UntypedWhileNode(condition, body, source); } diff --git a/src/main/java/org/shedlang/compiler/parser/SourceParser.java b/src/main/java/org/shedlang/compiler/parser/SourceParser.java index 19be58d4..8144f11a 100644 --- a/src/main/java/org/shedlang/compiler/parser/SourceParser.java +++ b/src/main/java/org/shedlang/compiler/parser/SourceParser.java @@ -1,8 +1,8 @@ package org.shedlang.compiler.parser; -import org.shedlang.compiler.sources.Source; import org.shedlang.compiler.tokeniser.TokenIterator; import org.shedlang.compiler.types.NamespaceName; +import org.zwobble.sourcetext.SourceText; import java.util.function.Function; @@ -12,15 +12,15 @@ public class SourceParser { private SourceParser() { } - public static NamespaceName parseNamespaceName(Source source) { - return parseSource(source, tokens -> Parser.parseNamespaceName(tokens)); + public static NamespaceName parseNamespaceName(SourceText sourceText) { + return parseSource(sourceText, tokens -> Parser.parseNamespaceName(tokens)); } private static <T> T parseSource( - Source source, + SourceText sourceText, Function<TokenIterator<TokenType>, T> parseTokens ) { - var tokens = tokenise(source); + var tokens = tokenise(sourceText); var result = parseTokens.apply(tokens); tokens.skip(TokenType.END); return result; diff --git a/src/main/java/org/shedlang/compiler/parser/Tokeniser.java b/src/main/java/org/shedlang/compiler/parser/Tokeniser.java index 1fdd584b..85172578 100644 --- a/src/main/java/org/shedlang/compiler/parser/Tokeniser.java +++ b/src/main/java/org/shedlang/compiler/parser/Tokeniser.java @@ -1,9 +1,9 @@ package org.shedlang.compiler.parser; -import org.shedlang.compiler.sources.Source; import org.shedlang.compiler.tokeniser.RegexTokeniser; import org.shedlang.compiler.tokeniser.Token; import org.shedlang.compiler.tokeniser.TokenIterator; +import org.zwobble.sourcetext.SourceText; import java.util.List; import java.util.Map; @@ -74,19 +74,23 @@ public class Tokeniser { ) ); - public static TokenIterator<TokenType> tokenise(Source source) { - var tokens = tokeniser.tokenise(source).stream() + public static TokenIterator<TokenType> tokenise(SourceText sourceText) { + var tokens = tokeniser.tokenise(sourceText).stream() .filter(token -> token.tokenType() != TokenType.WHITESPACE && token.tokenType() != TokenType.COMMENT_SINGLE_LINE ) .map(token -> token.tokenType() == TokenType.IDENTIFIER - ? new Token<>(token.source(), KEYWORDS.getOrDefault(token.value(), token.tokenType()), token.value()) + ? new Token<>(token.sourceRange(), KEYWORDS.getOrDefault(token.value(), token.tokenType()), token.value()) : token ) .collect(Collectors.toList()); - return new TokenIterator<>(tokens, token(source.end(), TokenType.END, "")); + var endPosition = sourceText.characterPosition(sourceText.characterLength()); + var endSourceRange = endPosition.to(endPosition); + var endToken = token(endSourceRange, TokenType.END, ""); + + return new TokenIterator<>(tokens, endToken); } } diff --git a/src/main/java/org/shedlang/compiler/sources/FileFragmentSource.java b/src/main/java/org/shedlang/compiler/sources/FileFragmentSource.java index 0fc05672..ae027e99 100644 --- a/src/main/java/org/shedlang/compiler/sources/FileFragmentSource.java +++ b/src/main/java/org/shedlang/compiler/sources/FileFragmentSource.java @@ -1,75 +1,10 @@ package org.shedlang.compiler.sources; -public record FileFragmentSource( - String filename, - String contents, - int characterIndexStart, - int characterIndexEnd -) implements Source { - static FileFragmentSource file(String filename, String contents) { - return new FileFragmentSource(filename, contents, 0, contents.length()); - } +import org.zwobble.sourcetext.SourceRange; +public record FileFragmentSource(SourceRange sourceRange) implements Source { @Override public String describe() { - var lines = contents.split("\n"); - var position = 0; - - for (var lineIndex = 0; lineIndex < lines.length; lineIndex++) { - var line = lines[lineIndex]; - var nextLinePosition = position + line.length() + 1; - if (nextLinePosition > characterIndexStart || nextLinePosition >= contents.length()) { - return context( - line, - lineIndex, - characterIndexStart - position, - nextLinePosition <= characterIndexEnd - ? 1 - : Math.max(characterIndexEnd - characterIndexStart, 1) - ); - } - position = nextLinePosition; - } - throw new RuntimeException("should be impossible (but evidently isn't)"); - } - - @Override - public Source at(int characterIndexStart, int characterIndexEnd) { - if (characterIndexStart > characterIndexEnd) { - throw new IllegalArgumentException("characterIndexStart > characterIndexEnd"); - } - - return new FileFragmentSource( - filename, - contents, - this.characterIndexStart + characterIndexStart, - this.characterIndexStart + characterIndexEnd - ); - } - - @Override - public Source to(Source end) { - if (end instanceof FileFragmentSource endFileFragment) { - return new FileFragmentSource(filename, contents, this.characterIndexStart, endFileFragment.characterIndexEnd); - } else { - throw new IllegalArgumentException("end must be an instance of FileFragmentSource"); - } - } - - @Override - public Source start() { - return at(0, 0); - } - - @Override - public Source end() { - var length = characterIndexEnd - characterIndexStart; - return at(length, length); - } - - private String context(String line, int lineIndex, int columnIndex, int length) { - var lineNumber = lineIndex + 1; - var columnNumber = columnIndex + 1; - return filename + ":" + lineNumber + ":" + columnNumber + "\n" + line + "\n" + " ".repeat(columnIndex) + "^".repeat(length); + return this.sourceRange.describe(); } } diff --git a/src/main/java/org/shedlang/compiler/sources/NullSource.java b/src/main/java/org/shedlang/compiler/sources/NullSource.java index 2f09702c..f9e86295 100644 --- a/src/main/java/org/shedlang/compiler/sources/NullSource.java +++ b/src/main/java/org/shedlang/compiler/sources/NullSource.java @@ -8,32 +8,7 @@ class NullSource implements Source { static final NullSource INSTANCE = new NullSource(); @Override - public String contents() { - return ""; - } - - @Override public String describe() { return "(null)"; } - - @Override - public Source start() { - return this; - } - - @Override - public Source end() { - return this; - } - - @Override - public Source at(int characterIndexStart, int characterIndexEnd) { - return this; - } - - @Override - public Source to(Source end) { - return null; - } } diff --git a/src/main/java/org/shedlang/compiler/sources/RuntimeSource.java b/src/main/java/org/shedlang/compiler/sources/RuntimeSource.java index 83937493..06310478 100644 --- a/src/main/java/org/shedlang/compiler/sources/RuntimeSource.java +++ b/src/main/java/org/shedlang/compiler/sources/RuntimeSource.java @@ -8,32 +8,7 @@ class RuntimeSource implements Source { static final RuntimeSource INSTANCE = new RuntimeSource(); @Override - public String contents() { - return ""; - } - - @Override public String describe() { return "(runtime)"; } - - @Override - public Source start() { - return this; - } - - @Override - public Source end() { - return this; - } - - @Override - public Source at(int characterIndexStart, int characterIndexEnd) { - return this; - } - - @Override - public Source to(Source end) { - return null; - } } diff --git a/src/main/java/org/shedlang/compiler/sources/Source.java b/src/main/java/org/shedlang/compiler/sources/Source.java index 8dc1733c..edd9ad88 100644 --- a/src/main/java/org/shedlang/compiler/sources/Source.java +++ b/src/main/java/org/shedlang/compiler/sources/Source.java @@ -1,10 +1,5 @@ package org.shedlang.compiler.sources; public interface Source { - String contents(); String describe(); - Source start(); - Source end(); - Source at(int characterIndexStart, int characterIndexEnd); - Source to(Source end); } diff --git a/src/main/java/org/shedlang/compiler/sources/SourceTexts.java b/src/main/java/org/shedlang/compiler/sources/SourceTexts.java new file mode 100644 index 00000000..d79408e6 --- /dev/null +++ b/src/main/java/org/shedlang/compiler/sources/SourceTexts.java @@ -0,0 +1,12 @@ +package org.shedlang.compiler.sources; + +import org.zwobble.sourcetext.SourceText; + +public class SourceTexts { + public static SourceText string(String text) { + return SourceText.fromString("<string>", text); + } + + private SourceTexts() { + } +} diff --git a/src/main/java/org/shedlang/compiler/sources/Sources.java b/src/main/java/org/shedlang/compiler/sources/Sources.java index ccfadca6..f1e59cfa 100644 --- a/src/main/java/org/shedlang/compiler/sources/Sources.java +++ b/src/main/java/org/shedlang/compiler/sources/Sources.java @@ -1,20 +1,6 @@ package org.shedlang.compiler.sources; -import java.nio.file.Path; - public class Sources { public static Source NULL = NullSource.INSTANCE; public static Source RUNTIME = RuntimeSource.INSTANCE; - - public static FileFragmentSource file(Path path, String contents) { - return FileFragmentSource.file(path.toString(), contents); - } - - public static FileFragmentSource file(String filename, String contents) { - return FileFragmentSource.file(filename, contents); - } - - public static FileFragmentSource string(String contents) { - return FileFragmentSource.file("<string>", contents); - } } diff --git a/src/main/java/org/shedlang/compiler/tokeniser/InfinitelyPaddedIterator.java b/src/main/java/org/shedlang/compiler/tokeniser/InfinitelyPaddedIterator.java index 534020d9..54548610 100644 --- a/src/main/java/org/shedlang/compiler/tokeniser/InfinitelyPaddedIterator.java +++ b/src/main/java/org/shedlang/compiler/tokeniser/InfinitelyPaddedIterator.java @@ -28,4 +28,8 @@ public class InfinitelyPaddedIterator<T> { public void moveNext() { index++; } + + public boolean isAtStart() { + return this.index == 0; + } } diff --git a/src/main/java/org/shedlang/compiler/tokeniser/RegexTokeniser.java b/src/main/java/org/shedlang/compiler/tokeniser/RegexTokeniser.java index 942960a3..a34ae3d5 100644 --- a/src/main/java/org/shedlang/compiler/tokeniser/RegexTokeniser.java +++ b/src/main/java/org/shedlang/compiler/tokeniser/RegexTokeniser.java @@ -1,6 +1,6 @@ package org.shedlang.compiler.tokeniser; -import org.shedlang.compiler.sources.Source; +import org.zwobble.sourcetext.SourceText; import java.util.ArrayList; import java.util.List; @@ -47,8 +47,8 @@ public class RegexTokeniser<T> { .collect(Collectors.toList()); } - public List<Token<T>> tokenise(Source source) { - var matcher = pattern.matcher(source.contents()); + public List<Token<T>> tokenise(SourceText sourceText) { + var matcher = pattern.matcher(sourceText.charSequence()); var tokens = new ArrayList<Token<T>>(); while (matcher.lookingAt()) { @@ -58,12 +58,15 @@ public class RegexTokeniser<T> { .getAsInt(); var tokenType = this.rules.get(groupIndex - 1); + var startPosition = sourceText.characterPosition(matcher.regionStart()); + var endPosition = sourceText.characterPosition(matcher.end()); tokens.add(new Token<>( - source.at(matcher.regionStart(), matcher.end()), + startPosition.to(endPosition), tokenType, + // TODO: the text value is redundant, can be gotten from the source range? matcher.group() )); - matcher.region(matcher.end(), source.contents().length()); + matcher.region(matcher.end(), sourceText.characterLength()); } return tokens; diff --git a/src/main/java/org/shedlang/compiler/tokeniser/Token.java b/src/main/java/org/shedlang/compiler/tokeniser/Token.java index e79d89eb..7e131304 100644 --- a/src/main/java/org/shedlang/compiler/tokeniser/Token.java +++ b/src/main/java/org/shedlang/compiler/tokeniser/Token.java @@ -1,10 +1,10 @@ package org.shedlang.compiler.tokeniser; -import org.shedlang.compiler.sources.Source; +import org.zwobble.sourcetext.SourceRange; -public record Token<T>(Source source, T tokenType, String value) { - public static <T> Token<T> token(Source source, T tokenType, String value) { - return new Token<T>(source, tokenType, value); +public record Token<T>(SourceRange sourceRange, T tokenType, String value) { + public static <T> Token<T> token(SourceRange sourceRange, T tokenType, String value) { + return new Token<T>(sourceRange, tokenType, value); } public String describe() { diff --git a/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java b/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java index 3462b4b7..9b3467a8 100644 --- a/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java +++ b/src/main/java/org/shedlang/compiler/tokeniser/TokenIterator.java @@ -1,6 +1,6 @@ package org.shedlang.compiler.tokeniser; -import org.shedlang.compiler.sources.Source; +import org.zwobble.sourcetext.SourcePosition; import java.util.List; @@ -15,11 +15,11 @@ public class TokenIterator<TTokenType> { nextValue(tokenType); } - public String nextValue(TTokenType tokenType) { + public Token<TTokenType> next(TTokenType tokenType) { var token = tokens.get(); if (token.tokenType().equals(tokenType)) { tokens.moveNext(); - return token.value(); + return token; } else { throw new UnexpectedTokenException( tokenType.toString(), @@ -28,6 +28,10 @@ public class TokenIterator<TTokenType> { } } + public String nextValue(TTokenType tokenType) { + return this.next(tokenType).value(); + } + public boolean isNext(TTokenType tokenType) { return tokens.get().tokenType().equals(tokenType); } @@ -54,7 +58,15 @@ public class TokenIterator<TTokenType> { return tokens.get(offset); } - public Source position() { - return tokens.get().source().start(); + public SourcePosition startPosition() { + return tokens.get().sourceRange().start(); + } + + public SourcePosition endPosition() { + if (tokens.isAtStart()) { + return startPosition(); + } else { + return tokens.get(-1).sourceRange().end(); + } } } diff --git a/src/main/java/org/shedlang/compiler/tokeniser/UnexpectedTokenException.java b/src/main/java/org/shedlang/compiler/tokeniser/UnexpectedTokenException.java index 75fa19dc..714edf6b 100644 --- a/src/main/java/org/shedlang/compiler/tokeniser/UnexpectedTokenException.java +++ b/src/main/java/org/shedlang/compiler/tokeniser/UnexpectedTokenException.java @@ -1,6 +1,7 @@ package org.shedlang.compiler.tokeniser; import org.shedlang.compiler.errors.SourceError; +import org.shedlang.compiler.sources.FileFragmentSource; public class UnexpectedTokenException extends SourceError { private final String expected; @@ -12,7 +13,7 @@ public class UnexpectedTokenException extends SourceError { ) { super( "Expected: " + expected + "\nBut got: " + actual.tokenType() + " \"" + actual.value() + "\"", - actual.source() + new FileFragmentSource(actual.sourceRange()) ); this.expected = expected; this.actual = actual; |
