package org.shedlang.compiler.parser; import org.shedlang.compiler.ast.untyped.*; import org.shedlang.compiler.parser.errors.*; import org.shedlang.compiler.sources.FileFragmentSource; import org.shedlang.compiler.sources.Source; import org.shedlang.compiler.tokeniser.TokenIterator; import org.shedlang.compiler.tokeniser.UnexpectedTokenException; import org.shedlang.compiler.types.NamespaceName; import org.shedlang.compiler.types.Visibility; import org.shedlang.compiler.util.Iterables; import org.zwobble.sourcetext.SourcePosition; import org.zwobble.sourcetext.SourceRange; import java.math.BigInteger; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.function.BooleanSupplier; import java.util.function.Supplier; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; public class Parser { private Parser() { } private static class BinaryOperationParser implements OperationParser< OperatorPrecedence, UntypedExpressionNode > { private final TokenType operatorTokenType; private final OperatorPrecedence precedence; private final UntypedBinaryOperator operator; private BinaryOperationParser( TokenType operatorTokenType, OperatorPrecedence precedence, UntypedBinaryOperator operator ) { this.operatorTokenType = operatorTokenType; this.precedence = precedence; this.operator = operator; } @Override public OperatorPrecedence precedence() { return precedence; } @Override public Stream operatorTokenTypes() { return Stream.of(operatorTokenType); } @Override public UntypedExpressionNode parse( SourcePosition start, UntypedExpressionNode left, TokenIterator tokens ) { tokens.skip(operatorTokenType); var right = parseSubExpression(tokens, precedence); var end = tokens.endPosition(); var sourceRange = new FileFragmentSource(start.to(end)); return new UntypedBinaryOperationNode( operator, left, right, sourceRange ); } } private static UntypedArgsNode parseArgs(TokenIterator tokens) { var argsStart = tokens.startPosition(); var positional = new ArrayList(); var named = new ArrayList(); parseMany( () -> tokens.isNext(TokenType.SYMBOL_PAREN_CLOSE), () -> { if (tokens.trySkip(TokenType.SYMBOL_DOT)) { var argStart = tokens.startPosition(); var argName = tokens.nextValue(TokenType.IDENTIFIER); tokens.skip(TokenType.SYMBOL_EQUALS); var expression = parseTopLevelExpression(tokens); var argEnd = tokens.endPosition(); var argSource = new FileFragmentSource(argStart.to(argEnd)); named.add(new UntypedNamedArgNode(argName, expression, argSource)); } else { var expression = parseTopLevelExpression(tokens); if (!named.isEmpty()) { throw new PositionalArgAfterNamedArgError(expression.source()); } positional.add(expression); } return null; }, () -> tokens.trySkip(TokenType.SYMBOL_COMMA) ); var argsEnd = tokens.endPosition(); var argsSource = new FileFragmentSource(argsStart.to(argsEnd)); return new UntypedArgsNode(positional, named, argsSource); } private static UntypedAssignNode parseAssign(TokenIterator tokens) { var start = tokens.startPosition(); var target = parseTarget(tokens); tokens.skip(TokenType.SYMBOL_EQUALS); var expression = parseTopLevelExpression(tokens); tokens.skip(TokenType.SYMBOL_SEMICOLON); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedAssignNode(target, expression, source); } private static UntypedBlockNode parseBlock(TokenIterator tokens) { var start = tokens.startPosition(); tokens.skip(TokenType.SYMBOL_BRACE_OPEN); if (tokens.trySkip(TokenType.SYMBOL_BRACE_CLOSE)) { var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedBlockNode(List.of(), source); } var body = new ArrayList(); while (true) { var statement = parseFunctionStatement(tokens); body.add(statement); if ( tokens.isNext(TokenType.SYMBOL_BRACE_CLOSE) || statement.behaviour() == UntypedFunctionStatementBehaviour.BLOCK_RETURN ) { tokens.skip(TokenType.SYMBOL_BRACE_CLOSE); var end = tokens.endPosition(); var source = start.to(end); return new UntypedBlockNode(body, new FileFragmentSource(source)); } } } private static UntypedExpressionNode parseBoolLiteral(TokenIterator tokens) { var start = tokens.startPosition(); var value = tokens.trySkip(TokenType.KEYWORD_TRUE); if (!value) { tokens.skip(TokenType.KEYWORD_FALSE); } var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedBoolLiteralNode(value, source); } private static UntypedFunctionStatementNode parseBreak(TokenIterator tokens) { var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_BREAK); var expression = tokens.isNext(TokenType.SYMBOL_SEMICOLON) ? Optional.empty() : Optional.of(parseTopLevelExpression(tokens)); tokens.skip(TokenType.SYMBOL_SEMICOLON); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedBreakNode(expression, source); } private static class CallParser implements OperationParser< OperatorPrecedence, UntypedExpressionNode > { @Override public OperatorPrecedence precedence() { return OperatorPrecedence.CALL; } @Override public Stream operatorTokenTypes() { return Stream.of( TokenType.SYMBOL_BANG, TokenType.SYMBOL_PAREN_OPEN, TokenType.SYMBOL_SQUARE_OPEN ); } @Override public UntypedExpressionNode parse( SourcePosition start, UntypedExpressionNode left, TokenIterator tokens ) { var hasEffects = tokens.trySkip(TokenType.SYMBOL_BANG); var typeLevelArgs = parseTypeLevelArgsOptional(tokens); tokens.skip(TokenType.SYMBOL_PAREN_OPEN); var args = parseArgs(tokens); tokens.skip(TokenType.SYMBOL_PAREN_CLOSE); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedCallNode( left, typeLevelArgs, args, hasEffects, source ); } } private static class ConstructedTypeParser implements OperationParser< TypeLevelOperatorPrecedence, UntypedTypeLevelExpressionNode > { @Override public TypeLevelOperatorPrecedence precedence() { return TypeLevelOperatorPrecedence.CONSTRUCTED_TYPE; } @Override public Stream operatorTokenTypes() { return Stream.of(TokenType.SYMBOL_SQUARE_OPEN); } @Override public UntypedTypeLevelExpressionNode parse( SourcePosition start, UntypedTypeLevelExpressionNode left, TokenIterator tokens ) { tokens.skip(TokenType.SYMBOL_SQUARE_OPEN); var args = parseMany( () -> tokens.isNext(TokenType.SYMBOL_SQUARE_CLOSE), () -> parseTopLevelTypeLevelExpression(tokens), () -> tokens.trySkip(TokenType.SYMBOL_COMMA) ); tokens.skip(TokenType.SYMBOL_SQUARE_CLOSE); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedConstructedTypeNode(left, args, source); } } private static UntypedEffectDefinitionNode parseEffectDefinition( TokenIterator tokens ) { var start = tokens.startPosition(); var visibility = parseVisibility(tokens); tokens.skip(TokenType.KEYWORD_EFFECT); var name = tokens.nextValue(TokenType.IDENTIFIER); tokens.skip(TokenType.SYMBOL_BRACE_OPEN); var functions = parseMany( () -> tokens.isNext(TokenType.SYMBOL_BRACE_CLOSE), () -> parseEffectDefinitionField(tokens), () -> true ); tokens.skip(TokenType.SYMBOL_BRACE_CLOSE); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedEffectDefinitionNode(name, visibility, functions, source); } private static UntypedEffectDefinitionMethodNode parseEffectDefinitionField( TokenIterator tokens ) { var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_FUN); var name = tokens.nextValue(TokenType.IDENTIFIER); var type = parseEffectDefinitionFieldType(tokens); tokens.skip(TokenType.SYMBOL_SEMICOLON); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedEffectDefinitionMethodNode(name, type, source); } private static UntypedFunctionTypeNode parseEffectDefinitionFieldType( TokenIterator tokens ) { var start = tokens.startPosition(); tokens.skip(TokenType.SYMBOL_PAREN_OPEN); var params = parseParamTypes(tokens); tokens.skip(TokenType.SYMBOL_PAREN_CLOSE); tokens.skip(TokenType.SYMBOL_THIN_ARROW); var returnType = parseTopLevelTypeLevelExpression(tokens); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedFunctionTypeNode( Optional.empty(), params, Optional.empty(), returnType, source ); } private static final Map> OPERATION_PARSERS = Stream.of( new MemberAccessParser(), new BinaryOperationParser(TokenType.SYMBOL_EQUALS_EQUALS, OperatorPrecedence.EQUALS, UntypedBinaryOperator.EQUALS), new BinaryOperationParser(TokenType.SYMBOL_BANG_EQUALS, OperatorPrecedence.EQUALS, UntypedBinaryOperator.NOT_EQUAL), new BinaryOperationParser(TokenType.SYMBOL_LESS_THAN, OperatorPrecedence.RELATIONAL, UntypedBinaryOperator.LESS_THAN), new BinaryOperationParser(TokenType.SYMBOL_LESS_THAN_OR_EQUAL, OperatorPrecedence.RELATIONAL, UntypedBinaryOperator.LESS_THAN_OR_EQUAL), new BinaryOperationParser(TokenType.SYMBOL_GREATER_THAN, OperatorPrecedence.RELATIONAL, UntypedBinaryOperator.GREATER_THAN), new BinaryOperationParser(TokenType.SYMBOL_GREATER_THAN_OR_EQUAL, OperatorPrecedence.RELATIONAL, UntypedBinaryOperator.GREATER_THAN_OR_EQUAL), new BinaryOperationParser(TokenType.SYMBOL_PLUS, OperatorPrecedence.ADD, UntypedBinaryOperator.ADD), new BinaryOperationParser(TokenType.SYMBOL_MINUS, OperatorPrecedence.ADD, UntypedBinaryOperator.SUB), new BinaryOperationParser(TokenType.SYMBOL_ASTERISK, OperatorPrecedence.MULTIPLY, UntypedBinaryOperator.MULTIPLY), new CallParser() ) .flatMap(parser -> parser.operatorTokenTypes().map(tokenType -> Map.entry(tokenType, parser)) ) .collect(Collectors.toMap( x -> x.getKey(), x -> x.getValue() )); static UntypedExpressionNode parseExpression( TokenIterator tokens, Optional parentPrecedence ) { var start = tokens.startPosition(); var left = parsePrimaryExpression(tokens); while (true) { var operationParser = OPERATION_PARSERS.get(tokens.peek().tokenType()); if ( operationParser == null || ( parentPrecedence.isPresent() && operationParser.precedence().precedence() <= parentPrecedence.get().precedence() ) ) { return left; } left = operationParser.parse(start, left, tokens); } } private static UntypedFunctionStatementNode parseExpressionStatement(TokenIterator tokens) { var start = tokens.startPosition(); var expression = parseTopLevelExpression(tokens); var isReturn = expression instanceof UntypedExpressionWithChildBlocksNode branchingExpression ? expressionStatementBehaviour(branchingExpression.exhaustiveChildBlocks()) == UntypedFunctionStatementBehaviour.BLOCK_RETURN : !tokens.trySkip(TokenType.SYMBOL_SEMICOLON); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return isReturn ? new UntypedBlockReturnNode(expression, source) : new UntypedExpressionStatementNode(expression, source); } static UntypedFunctionStatementBehaviour expressionStatementBehaviour( Iterable branchBodies ) { return Iterables.foldLeft( UntypedFunctionStatementBehaviour.TERMINATE, branchBodies, (expressionStatementBehaviour, branchBody) -> { if (expressionStatementBehaviour == UntypedFunctionStatementBehaviour.TERMINATE) { return branchBody.behaviour(); } else if (branchBody.behaviour() == UntypedFunctionStatementBehaviour.TERMINATE) { return expressionStatementBehaviour; } else if (expressionStatementBehaviour == branchBody.behaviour()) { return expressionStatementBehaviour; } else { throw new InconsistentBlockBehaviourError(branchBody.source()); } } ); } private static UntypedFunctionDefinitionNode parseFunctionDefinition(TokenIterator tokens) { var start = tokens.startPosition(); var visibility = parseVisibility(tokens); tokens.skip(TokenType.KEYWORD_FUN); var name = tokens.nextValue(TokenType.IDENTIFIER); var signature = parseFunctionSignature(tokens); var body = parseBlock(tokens); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedFunctionDefinitionNode( name, visibility, signature, body, source ); } private static UntypedExpressionNode parseFunctionExpression( TokenIterator tokens ) { var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_FN); var signature = parseFunctionSignature(tokens); var body = parseBlock(tokens); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedFunctionExpressionNode( signature, body, source ); } private static UntypedFunctionStatementNode parseFunctionReturn(TokenIterator tokens) { var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_RETURN); var expression = tokens.isNext(TokenType.SYMBOL_SEMICOLON) ? Optional.empty() : Optional.of(parseTopLevelExpression(tokens)); tokens.skip(TokenType.SYMBOL_SEMICOLON); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedFunctionReturnNode(expression, source); } static UntypedFunctionSignatureNode parseFunctionSignature( TokenIterator tokens ) { var start = tokens.startPosition(); var typeLevelParams = parseTypeLevelParamsOptional(tokens); tokens.skip(TokenType.SYMBOL_PAREN_OPEN); var params = parseParams(tokens); tokens.skip(TokenType.SYMBOL_PAREN_CLOSE); var effect = parseFunctionTypeEffect(tokens); var returnType = parseFunctionSignatureReturnType(tokens); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedFunctionSignatureNode( typeLevelParams, params, effect, returnType, source ); } private static Optional parseFunctionSignatureReturnType(TokenIterator tokens) { if (tokens.trySkip(TokenType.SYMBOL_THIN_ARROW)) { var expression = parseTopLevelTypeLevelExpression(tokens); return Optional.of(expression); } else { return Optional.empty(); } } static UntypedFunctionStatementNode parseFunctionStatement(TokenIterator tokens) { return switch (tokens.peek().tokenType()) { case KEYWORD_SET, KEYWORD_VAL, KEYWORD_VAR, SYMBOL_UNDERSCORE -> parseAssign(tokens); case KEYWORD_BREAK -> parseBreak(tokens); case KEYWORD_RESUME -> parseResume(tokens); case KEYWORD_RETURN -> parseFunctionReturn(tokens); case KEYWORD_WHILE -> parseWhile(tokens); case SYMBOL_HASH_PAREN_OPEN -> parseFunctionStatementTuple(tokens); default -> parseExpressionStatement(tokens); }; } private static UntypedFunctionStatementNode parseFunctionStatementTuple( TokenIterator tokens ) { // We need to disambiguate destructuring assignments from // tuple expression statements. var offset = 0; while (true) { var token = tokens.peek(offset); switch (token.tokenType()) { case SYMBOL_HASH_PAREN_OPEN -> offset += 1; case KEYWORD_SET, KEYWORD_VAL, KEYWORD_VAR -> { return parseAssign(tokens); } default -> { return parseExpressionStatement(tokens); } } } } private static UntypedTypeLevelExpressionNode parseFunctionType( TokenIterator tokens ) { var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_FN); var signature = parseFunctionTypeSignature(tokens); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedFunctionTypeNode( signature.typeLevelParams(), signature.params(), signature.effect(), signature.returnType(), source ); } private static UntypedFunctionTypeNode parseFunctionTypeSignature( TokenIterator tokens ) { var start = tokens.startPosition(); var typeLevelParams = parseTypeLevelParamsOptional(tokens); tokens.skip(TokenType.SYMBOL_PAREN_OPEN); var params = parseParamTypes(tokens); tokens.skip(TokenType.SYMBOL_PAREN_CLOSE); var effect = parseFunctionTypeEffect(tokens); tokens.skip(TokenType.SYMBOL_THIN_ARROW); var returnType = parseTopLevelTypeLevelExpression(tokens); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedFunctionTypeNode( typeLevelParams, params, effect, returnType, source ); } private static Optional parseFunctionTypeEffect(TokenIterator tokens) { if (tokens.trySkip(TokenType.SYMBOL_BANG)) { return Optional.of(parseTopLevelTypeLevelExpression(tokens)); } else { return Optional.empty(); } } private static UntypedExpressionNode parseIf(TokenIterator tokens) { var start = tokens.startPosition(); var conditionalBranches = new ArrayList(); var elseBody = Optional.empty(); while (true) { var conditionalBranch = parseConditionalBranch(tokens); conditionalBranches.add(conditionalBranch); if (!tokens.trySkip(TokenType.KEYWORD_ELSE)) { break; } if (!tokens.isNext(TokenType.KEYWORD_IF)) { elseBody = Optional.of(parseBlock(tokens)); break; } } var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedIfNode( conditionalBranches, elseBody.orElseGet(() -> new UntypedBlockNode( List.of(), new FileFragmentSource(end.to(end)) )), source ); } private static UntypedConditionalBranchNode parseConditionalBranch(TokenIterator tokens) { var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_IF); tokens.skip(TokenType.SYMBOL_PAREN_OPEN); var condition = parseTopLevelExpression(tokens); tokens.skip(TokenType.SYMBOL_PAREN_CLOSE); var body = parseBlock(tokens); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedConditionalBranchNode(condition, body, source); } private static UntypedImportNode parseImport(TokenIterator tokens) { var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_IMPORT); var namespaceName = parseNamespaceName(tokens); tokens.skip(TokenType.SYMBOL_SEMICOLON); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedImportNode(namespaceName, source); } private static UntypedExpressionNode parseInt(TokenIterator tokens) { var start = tokens.startPosition(); var string = tokens.nextValue(TokenType.INT); var value = new BigInteger(string); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); if (string.length() > 1 && string.startsWith("0")) { throw new IntLiteralCannotHaveLeadingZeroesError(source); } else if (string.length() > 2 && string.startsWith("-0")) { throw new IntLiteralCannotHaveLeadingZeroesError(source); } return new UntypedIntLiteralNode(value, source); } /** * Parse zero, one, or many instances of {@code T} using {@code parseOne}. * Each instance is expected to be separated by tokens consumed by * {@code parseSeparator}. Parsing stops either if {@code parseSeparator} * returns false, or if {@code stop} returns true. Trailing separators * are permitted. */ private static List parseMany( BooleanSupplier stop, Supplier parseOne, BooleanSupplier parseSeparator ) { var values = new ArrayList(); while (!stop.getAsBoolean()) { var value = parseOne.get(); values.add(value); if (!parseSeparator.getAsBoolean()) { return values; } } return values; } /** * Parse one or more instances of {@code T} using {@code parseOne}. Each * instance is expected to be separated by tokens consumed by * {@code parseSeparator}. Parsing stops if {@code parseSeparator} returns * false. Trailing separators are not permitted. */ private static List parseOneOrMoreNoTrailingSeparator( Supplier parseOne, BooleanSupplier parseSeparator ) { var values = new ArrayList(); while (true) { var value = parseOne.get(); values.add(value); if (!parseSeparator.getAsBoolean()) { return values; } } } private static class MemberAccessParser implements OperationParser< OperatorPrecedence, UntypedExpressionNode > { @Override public OperatorPrecedence precedence() { return OperatorPrecedence.CALL; } @Override public Stream operatorTokenTypes() { return Stream.of(TokenType.SYMBOL_DOT); } @Override public UntypedExpressionNode parse( SourcePosition start, UntypedExpressionNode left, TokenIterator tokens ) { tokens.skip(TokenType.SYMBOL_DOT); var memberName = tokens.nextValue(TokenType.IDENTIFIER); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedMemberAccessNode(left, memberName, source); } } public static NamespaceName parseNamespaceName(TokenIterator tokens) { var namespaceNameParts = parseOneOrMoreNoTrailingSeparator( () -> tokens.nextValue(TokenType.IDENTIFIER), () -> tokens.trySkip(TokenType.SYMBOL_FORWARD_SLASH) ); return new NamespaceName(namespaceNameParts); } static UntypedNamespaceStatementNode parseNamespaceStatement(TokenIterator tokens) { return switch (tokens.peek().tokenType()) { case KEYWORD_EFFECT -> parseEffectDefinition(tokens); case KEYWORD_FUN -> parseFunctionDefinition(tokens); case KEYWORD_NATIVE -> switch (tokens.peek(1).tokenType()) { case KEYWORD_FUN -> parseNativeFunctionDeclaration(tokens); case KEYWORD_TYPE -> parseNativeTypeDeclaration(tokens); default -> throw new UnexpectedTokenException("namespace statement", tokens.peek()); }; case KEYWORD_PUB -> switch (tokens.peek(1).tokenType()) { case KEYWORD_EFFECT -> parseEffectDefinition(tokens); case KEYWORD_FUN -> parseFunctionDefinition(tokens); case KEYWORD_NATIVE -> switch (tokens.peek(2).tokenType()) { case KEYWORD_FUN -> parseNativeFunctionDeclaration(tokens); case KEYWORD_TYPE -> parseNativeTypeDeclaration(tokens); default -> throw new UnexpectedTokenException("namespace statement", tokens.peek()); }; case KEYWORD_STRUCT -> parseStructDefinition(tokens); case KEYWORD_SUMTYPE -> parseSumTypeDefinition(tokens); case KEYWORD_VAL -> parseAssign(tokens); default -> throw new UnexpectedTokenException("namespace statement", tokens.peek()); }; case KEYWORD_STRUCT -> parseStructDefinition(tokens); case KEYWORD_SUMTYPE -> parseSumTypeDefinition(tokens); case KEYWORD_VAL -> parseAssign(tokens); default -> throw new UnexpectedTokenException("namespace statement", tokens.peek()); }; } private static UntypedNativeFunctionDeclarationNode parseNativeFunctionDeclaration( TokenIterator tokens ) { var start = tokens.startPosition(); var visibility = parseVisibility(tokens); tokens.skip(TokenType.KEYWORD_NATIVE); tokens.skip(TokenType.KEYWORD_FUN); var name = tokens.nextValue(TokenType.IDENTIFIER); var signature = parseFunctionTypeSignature(tokens); tokens.skip(TokenType.SYMBOL_SEMICOLON); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedNativeFunctionDeclarationNode( name, visibility, signature, source ); } private static UntypedNamespaceStatementNode parseNativeTypeDeclaration( TokenIterator tokens ) { var start = tokens.startPosition(); var visibility = parseVisibility(tokens); tokens.skip(TokenType.KEYWORD_NATIVE); tokens.skip(TokenType.KEYWORD_TYPE); var name = tokens.nextValue(TokenType.IDENTIFIER); var typeLevelParams = parseTypeLevelParamsOptional(tokens); tokens.skip(TokenType.SYMBOL_BRACE_OPEN); var methods = parseMany( () -> tokens.isNext(TokenType.SYMBOL_BRACE_CLOSE), () -> parseNativeFunctionDeclaration(tokens), () -> true ); tokens.skip(TokenType.SYMBOL_BRACE_CLOSE); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedNativeTypeDeclarationNode( name, visibility, typeLevelParams, methods, source ); } private static UntypedParamsNode parseParams(TokenIterator tokens) { var paramsStart = tokens.startPosition(); var positional = new ArrayList(); var named = new ArrayList(); parseMany( () -> tokens.isNext(TokenType.SYMBOL_PAREN_CLOSE), () -> { var paramStart = tokens.startPosition(); var isNamed = tokens.trySkip(TokenType.SYMBOL_DOT); var name = tokens.nextValue(TokenType.IDENTIFIER); var type = tokens.trySkip(TokenType.SYMBOL_COLON) ? Optional.of(parseTopLevelTypeLevelExpression(tokens)) : Optional.empty(); var paramEnd = tokens.endPosition(); var paramSource = new FileFragmentSource(paramStart.to(paramEnd)); if (!isNamed && !named.isEmpty()) { throw new PositionalParamAfterNamedParamError(paramSource); } (isNamed ? named : positional).add(new UntypedParamNode(name, type, paramSource)); return null; }, () -> tokens.trySkip(TokenType.SYMBOL_COMMA) ); var paramsEnd = tokens.endPosition(); var paramsSource = new FileFragmentSource(paramsStart.to(paramsEnd)); return new UntypedParamsNode(positional, named, paramsSource); } private static UntypedFunctionTypeParamsNode parseParamTypes(TokenIterator tokens) { var paramsStart = tokens.startPosition(); var positional = new ArrayList(); var named = new ArrayList(); parseMany( () -> tokens.isNext(TokenType.SYMBOL_PAREN_CLOSE), () -> { var paramStart = tokens.startPosition(); var isNamed = tokens.trySkip(TokenType.SYMBOL_DOT); if (isNamed) { var name = tokens.nextValue(TokenType.IDENTIFIER); tokens.skip(TokenType.SYMBOL_COLON); var type = parseTopLevelTypeLevelExpression(tokens); var paramEnd = tokens.endPosition(); var paramSource = new FileFragmentSource(paramStart.to(paramEnd)); named.add(new UntypedFunctionTypeNamedParamNode(name, type, paramSource)); } else { var type = parseTopLevelTypeLevelExpression(tokens); if (!named.isEmpty()) { throw new PositionalParamAfterNamedParamError(type.source()); } positional.add(type); } return null; }, () -> tokens.trySkip(TokenType.SYMBOL_COMMA) ); var paramsEnd = tokens.endPosition(); var paramsSource = paramsStart.to(paramsEnd); return new UntypedFunctionTypeParamsNode(positional, named, new FileFragmentSource(paramsSource)); } private static UntypedExpressionNode parsePrimaryExpression(TokenIterator tokens) { return switch (tokens.peek().tokenType()) { case IDENTIFIER -> parseReference(tokens); case INT -> parseInt(tokens); case KEYWORD_IF -> parseIf(tokens); case KEYWORD_FALSE, KEYWORD_TRUE -> parseBoolLiteral(tokens); case KEYWORD_FN -> parseFunctionExpression(tokens); case KEYWORD_TRY -> parseTry(tokens); case KEYWORD_WHEN -> parseWhen(tokens); case KEYWORD_UNIT -> parseUnitLiteral(tokens); case STRING -> parseStringLiteral(tokens); case SYMBOL_HASH_PAREN_OPEN -> parseTuple(tokens); default -> throw new UnexpectedTokenException("expression", tokens.peek()); }; } static UntypedTypeLevelExpressionNode parsePrimaryTypeLevelExpression(TokenIterator tokens) { return switch (tokens.peek().tokenType()) { case IDENTIFIER -> parseTypeLevelReference(tokens); case KEYWORD_FN -> parseFunctionType(tokens); case SYMBOL_HASH_PAREN_OPEN -> parseTupleType(tokens); default -> throw new UnexpectedTokenException("type-level expression", tokens.peek()); }; } private static UntypedFunctionStatementNode parseResume(TokenIterator tokens) { var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_RESUME); var expression = tokens.isNext(TokenType.SYMBOL_SEMICOLON) ? Optional.empty() : Optional.of(parseTopLevelExpression(tokens)); tokens.skip(TokenType.SYMBOL_SEMICOLON); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedResumeNode(expression, source); } private static UntypedExpressionNode parseStringLiteral(TokenIterator tokens) { var start = tokens.startPosition(); var token = tokens.next(TokenType.STRING); var stringContentsStart = token.sourceRange().characterPosition(1); var stringContentsEnd = token.sourceRange() .characterPosition(token.sourceRange().characterLength() - 1); var stringContents = stringContentsStart.to(stringContentsEnd); var unescapedValue = unescape(stringContents); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedStringLiteralNode(unescapedValue, source); } private static final Pattern STRING_ESCAPE_PATTERN = Pattern.compile("\\\\(.)"); private static String unescape(SourceRange sourceRange) { var stringContents = sourceRange.charSequence(); var matcher = STRING_ESCAPE_PATTERN.matcher(stringContents); var result = new StringBuilder(); var lastIndex = 0; while (matcher.find()) { result.append(stringContents.subSequence(lastIndex, matcher.start())); var code = matcher.group(1); var escapeSequenceSourceRange = sourceRange.characterPosition(matcher.start()) .to(sourceRange.characterPosition(matcher.end())); result.append(unescapeCharacter(code, new FileFragmentSource(escapeSequenceSourceRange))); lastIndex = matcher.end(); } result.append(stringContents.subSequence(lastIndex, stringContents.length())); return result.toString(); } private static char unescapeCharacter(String code, Source source) { return switch (code) { case "n" -> '\n'; case "r" -> '\r'; case "t" -> '\t'; case "\"" -> '"'; case "\\" -> '\\'; default -> throw new UnrecognisedEscapeSequenceError("\\" + code, source); }; } private static UntypedNamespaceStatementNode parseStructDefinition(TokenIterator tokens) { var start = tokens.startPosition(); var visibility = parseVisibility(tokens); tokens.skip(TokenType.KEYWORD_STRUCT); var name = tokens.nextValue(TokenType.IDENTIFIER); var typeLevelParams = parseTypeLevelParamsOptional(tokens); tokens.skip(TokenType.SYMBOL_PAREN_OPEN); var structHasPositionalFields = !tokens.isNext(TokenType.SYMBOL_DOT); var fields = parseMany( () -> tokens.isNext(TokenType.SYMBOL_PAREN_CLOSE), () -> parseStructFieldDefinition(tokens, structHasPositionalFields), () -> tokens.trySkip(TokenType.SYMBOL_COMMA) ); tokens.skip(TokenType.SYMBOL_PAREN_CLOSE); tokens.skip(TokenType.SYMBOL_BRACE_OPEN); var methods = parseMany( () -> tokens.isNext(TokenType.SYMBOL_BRACE_CLOSE), () -> parseFunctionDefinition(tokens), () -> true ); tokens.skip(TokenType.SYMBOL_BRACE_CLOSE); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedStructDefinitionNode( name, visibility, typeLevelParams, fields, structHasPositionalFields, methods, source ); } private static UntypedStructFieldDefinitionNode parseStructFieldDefinition( TokenIterator tokens, boolean structHasPositionalFields ) { var start = tokens.startPosition(); var fieldIsPositional = !tokens.trySkip(TokenType.SYMBOL_DOT); var name = tokens.nextValue(TokenType.IDENTIFIER); tokens.skip(TokenType.SYMBOL_COLON); var type = parseTopLevelTypeLevelExpression(tokens); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); if (structHasPositionalFields ^ fieldIsPositional) { throw new InconsistentStructFieldsError(source); } return new UntypedStructFieldDefinitionNode(name, type, source); } static UntypedExpressionNode parseSubExpression(TokenIterator tokens, OperatorPrecedence parentPrecedence) { return parseExpression(tokens, Optional.of(parentPrecedence)); } private static UntypedNamespaceStatementNode parseSumTypeDefinition(TokenIterator tokens) { var start = tokens.startPosition(); var visibility = parseVisibility(tokens); tokens.skip(TokenType.KEYWORD_SUMTYPE); var name = tokens.nextValue(TokenType.IDENTIFIER); var typeLevelParams = parseTypeLevelParamsOptional(tokens); tokens.skip(TokenType.SYMBOL_EQUALS); var variants = parseOneOrMoreNoTrailingSeparator( () -> parseTopLevelTypeLevelExpression(tokens), () -> tokens.trySkip(TokenType.SYMBOL_BAR) ); tokens.skip(TokenType.SYMBOL_SEMICOLON); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedSumTypeDefinitionNode( name, visibility, typeLevelParams, variants, source ); } static UntypedExpressionNode parseTopLevelExpression(TokenIterator tokens) { return parseExpression(tokens, Optional.empty()); } static UntypedTypeLevelExpressionNode parseTopLevelTypeLevelExpression(TokenIterator tokens) { return parseTypeLevelExpression(tokens, Optional.empty()); } public static UntypedNamespaceNode parseNamespace( TokenIterator tokens, NamespaceName namespaceName ) { var start = tokens.startPosition(); var imports = parseMany( () -> !tokens.isNext(TokenType.KEYWORD_IMPORT), () -> parseImport(tokens), () -> true ); var body = parseMany( () -> tokens.isNext(TokenType.END), () -> parseNamespaceStatement(tokens), () -> true ); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedNamespaceNode(namespaceName, imports, body, source); } private static UntypedExpressionNode parseReference(TokenIterator tokens) { var start = tokens.startPosition(); var name = tokens.nextValue(TokenType.IDENTIFIER); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedReferenceNode(name, source); } private static UntypedTargetNode parseTarget(TokenIterator tokens) { var start = tokens.startPosition(); return switch (tokens.peek().tokenType()) { case KEYWORD_SET -> { tokens.skip(TokenType.KEYWORD_SET); var name = tokens.nextValue(TokenType.IDENTIFIER); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); yield new UntypedTargetNode.Reassign(name, source); } case KEYWORD_VAL, KEYWORD_PUB -> { var visibility = parseVisibility(tokens); tokens.skip(TokenType.KEYWORD_VAL); var name = tokens.nextValue(TokenType.IDENTIFIER); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); yield new UntypedTargetNode.Val(name, visibility, source); } case KEYWORD_VAR -> { tokens.skip(TokenType.KEYWORD_VAR); var name = tokens.nextValue(TokenType.IDENTIFIER); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); yield new UntypedTargetNode.Var(name, source); } case SYMBOL_HASH_PAREN_OPEN -> { tokens.skip(TokenType.SYMBOL_HASH_PAREN_OPEN); var elements = parseMany( () -> tokens.isNext(TokenType.SYMBOL_PAREN_CLOSE), () -> parseTarget(tokens), () -> tokens.trySkip(TokenType.SYMBOL_COMMA) ); tokens.skip(TokenType.SYMBOL_PAREN_CLOSE); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); yield new UntypedTargetNode.Tuple(elements, source); } case SYMBOL_UNDERSCORE -> { tokens.skip(TokenType.SYMBOL_UNDERSCORE); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); yield new UntypedTargetNode.Ignore(source); } default -> throw new UnexpectedTokenException("target", tokens.peek()); }; } private static UntypedExpressionNode parseTry( TokenIterator tokens ) { var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_TRY); var body = parseBlock(tokens); tokens.skip(TokenType.KEYWORD_HANDLE); var handledEffect = parseTopLevelTypeLevelExpression(tokens); tokens.skip(TokenType.SYMBOL_BRACE_OPEN); var handlerMethods = parseMany( () -> tokens.isNext(TokenType.SYMBOL_BRACE_CLOSE), () -> parseTryHandlerMethod(tokens), () -> true ); tokens.skip(TokenType.SYMBOL_BRACE_CLOSE); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedTryNode(body, handledEffect, handlerMethods, source); } private static UntypedTryHandlerMethodNode parseTryHandlerMethod(TokenIterator tokens) { var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_FUN); var name = tokens.nextValue(TokenType.IDENTIFIER); tokens.skip(TokenType.SYMBOL_PAREN_OPEN); var params = parseParams(tokens); tokens.skip(TokenType.SYMBOL_PAREN_CLOSE); var body = parseBlock(tokens); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedTryHandlerMethodNode( name, params, Optional.empty(), Optional.empty(), body, source ); } private static UntypedExpressionNode parseTuple( TokenIterator tokens ) { var start = tokens.startPosition(); tokens.skip(TokenType.SYMBOL_HASH_PAREN_OPEN); var elements = parseMany( () -> tokens.isNext(TokenType.SYMBOL_PAREN_CLOSE), () -> parseTopLevelExpression(tokens), () -> tokens.trySkip(TokenType.SYMBOL_COMMA) ); tokens.skip(TokenType.SYMBOL_PAREN_CLOSE); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedTupleNode(elements, source); } private static UntypedTypeLevelExpressionNode parseTupleType( TokenIterator tokens ) { var start = tokens.startPosition(); tokens.skip(TokenType.SYMBOL_HASH_PAREN_OPEN); var elements = parseMany( () -> tokens.isNext(TokenType.SYMBOL_PAREN_CLOSE), () -> parseTopLevelTypeLevelExpression(tokens), () -> tokens.trySkip(TokenType.SYMBOL_COMMA) ); tokens.skip(TokenType.SYMBOL_PAREN_CLOSE); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedTupleTypeNode(elements, source); } private static Optional parseTypeLevelArgsOptional( TokenIterator tokens ) { var start = tokens.startPosition(); if (tokens.trySkip(TokenType.SYMBOL_SQUARE_OPEN)) { var params = parseMany( () -> tokens.isNext(TokenType.SYMBOL_SQUARE_CLOSE), () -> parseTopLevelTypeLevelExpression(tokens), () -> tokens.trySkip(TokenType.SYMBOL_COMMA) ); tokens.skip(TokenType.SYMBOL_SQUARE_CLOSE); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return Optional.of(new UntypedTypeLevelArgsNode(params, source)); } else { return Optional.empty(); } } private static final Map> TYPE_LEVEL_OPERATION_PARSERS = Stream.of( new ConstructedTypeParser() ) .flatMap(parser -> parser.operatorTokenTypes().map(tokenType -> Map.entry(tokenType, parser)) ) .collect(Collectors.toMap( x -> x.getKey(), x -> x.getValue() )); private static UntypedTypeLevelExpressionNode parseTypeLevelExpression( TokenIterator tokens, Optional parentPrecedence ) { var start = tokens.startPosition(); var left = parsePrimaryTypeLevelExpression(tokens); while (true) { var operationParser = TYPE_LEVEL_OPERATION_PARSERS.get(tokens.peek().tokenType()); if ( operationParser == null || ( parentPrecedence.isPresent() && operationParser.precedence().precedence() <= parentPrecedence.get().precedence() ) ) { return left; } left = operationParser.parse(start, left, tokens); } } private static Optional> parseTypeLevelParamsOptional( TokenIterator tokens ) { if (tokens.trySkip(TokenType.SYMBOL_SQUARE_OPEN)) { var params = parseMany( () -> tokens.isNext(TokenType.SYMBOL_SQUARE_CLOSE), () -> parseTypeLevelParam(tokens), () -> tokens.trySkip(TokenType.SYMBOL_COMMA) ); tokens.skip(TokenType.SYMBOL_SQUARE_CLOSE); return Optional.of(params); } else { return Optional.empty(); } } static UntypedTypeLevelParamNode parseTypeLevelParam(TokenIterator tokens) { var start = tokens.startPosition(); var isEffectParam = tokens.trySkip(TokenType.SYMBOL_BANG); var name = tokens.nextValue(TokenType.IDENTIFIER); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return isEffectParam ? new UntypedEffectParamNode(name, source) : new UntypedTypeParamNode(name, source); } private static UntypedTypeLevelReferenceNode parseTypeLevelReference(TokenIterator tokens) { var start = tokens.startPosition(); var name = tokens.nextValue(TokenType.IDENTIFIER); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedTypeLevelReferenceNode(name, source); } private static UntypedExpressionNode parseUnitLiteral(TokenIterator tokens) { var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_UNIT); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedUnitLiteralNode(source); } private static Visibility parseVisibility(TokenIterator tokens) { var isPublic = tokens.trySkip(TokenType.KEYWORD_PUB); return isPublic ? Visibility.PUBLIC : Visibility.PRIVATE; } private static UntypedExpressionNode parseWhen(TokenIterator tokens) { var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_WHEN); tokens.skip(TokenType.SYMBOL_PAREN_OPEN); var expression = parseTopLevelExpression(tokens); tokens.skip(TokenType.SYMBOL_PAREN_CLOSE); tokens.skip(TokenType.SYMBOL_BRACE_OPEN); var cases = parseMany( () -> tokens.isNext(TokenType.SYMBOL_BRACE_CLOSE), () -> parseWhenCase(tokens), () -> true ); tokens.skip(TokenType.SYMBOL_BRACE_CLOSE); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedWhenNode(expression, cases, source); } private static UntypedWhenCaseNode parseWhenCase(TokenIterator tokens) { var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_IS); var caseType = parseTopLevelTypeLevelExpression(tokens); var body = parseBlock(tokens); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedWhenCaseNode(caseType, body, source); } private static UntypedFunctionStatementNode parseWhile( TokenIterator tokens ) { var start = tokens.startPosition(); tokens.skip(TokenType.KEYWORD_WHILE); tokens.skip(TokenType.SYMBOL_PAREN_OPEN); var condition = parseTopLevelExpression(tokens); tokens.skip(TokenType.SYMBOL_PAREN_CLOSE); var body = parseBlock(tokens); var end = tokens.endPosition(); var source = new FileFragmentSource(start.to(end)); return new UntypedWhileNode(condition, body, source); } }