From 27af53ebd256d9e593ade3fc84d0af0f8a074f41 Mon Sep 17 00:00:00 2001 From: Michael Williamson Date: Mon, 20 Apr 2026 13:56:36 +0100 Subject: Move code into compiler subpackage --- .../ast/untyped/UntypedNamespaceNode.java | 13 -- .../ast/untyped/UntypedNamespaceStatementNode.java | 4 - .../zwobble/hobgoblin/ast/untyped/UntypedNode.java | 7 - .../ast/untyped/UntypedStructDefinitionNode.java | 9 -- .../compiler/ast/untyped/UntypedNamespaceNode.java | 13 ++ .../ast/untyped/UntypedNamespaceStatementNode.java | 4 + .../compiler/ast/untyped/UntypedNode.java | 7 + .../ast/untyped/UntypedStructDefinitionNode.java | 9 ++ .../hobgoblin/compiler/errors/SourceError.java | 20 +++ .../hobgoblin/compiler/parser/ParseError.java | 23 +++ .../zwobble/hobgoblin/compiler/parser/Parser.java | 34 +++++ .../zwobble/hobgoblin/compiler/parser/Token.java | 9 ++ .../hobgoblin/compiler/parser/TokenIterator.java | 41 ++++++ .../hobgoblin/compiler/parser/TokenType.java | 12 ++ .../hobgoblin/compiler/parser/Tokenizer.java | 157 +++++++++++++++++++++ .../zwobble/hobgoblin/compiler/parser/Tokens.java | 9 ++ .../compiler/parser/UnexpectedTokenError.java | 28 ++++ .../compiler/sources/FileFragmentSource.java | 10 ++ .../hobgoblin/compiler/sources/NullSource.java | 13 ++ .../hobgoblin/compiler/sources/RuntimeSource.java | 13 ++ .../zwobble/hobgoblin/compiler/sources/Source.java | 5 + .../hobgoblin/compiler/types/NamespaceName.java | 15 ++ .../org/zwobble/hobgoblin/errors/SourceError.java | 20 --- .../org/zwobble/hobgoblin/parser/ParseError.java | 23 --- .../java/org/zwobble/hobgoblin/parser/Parser.java | 34 ----- .../java/org/zwobble/hobgoblin/parser/Token.java | 9 -- .../zwobble/hobgoblin/parser/TokenIterator.java | 41 ------ .../org/zwobble/hobgoblin/parser/TokenType.java | 12 -- .../org/zwobble/hobgoblin/parser/Tokenizer.java | 157 --------------------- .../java/org/zwobble/hobgoblin/parser/Tokens.java | 9 -- .../hobgoblin/parser/UnexpectedTokenError.java | 28 ---- .../hobgoblin/sources/FileFragmentSource.java | 10 -- .../org/zwobble/hobgoblin/sources/NullSource.java | 13 -- .../zwobble/hobgoblin/sources/RuntimeSource.java | 13 -- .../java/org/zwobble/hobgoblin/sources/Source.java | 5 - .../org/zwobble/hobgoblin/types/NamespaceName.java | 15 -- .../compiler/parser/ParserNamespaceTests.java | 27 ++++ .../hobgoblin/compiler/parser/ParserTesting.java | 25 ++++ .../hobgoblin/compiler/parser/TokenizerTests.java | 95 +++++++++++++ .../hobgoblin/parser/ParserNamespaceTests.java | 27 ---- .../zwobble/hobgoblin/parser/ParserTesting.java | 22 --- .../zwobble/hobgoblin/parser/TokenizerTests.java | 92 ------------ 42 files changed, 569 insertions(+), 563 deletions(-) delete mode 100644 src/main/java/org/zwobble/hobgoblin/ast/untyped/UntypedNamespaceNode.java delete mode 100644 src/main/java/org/zwobble/hobgoblin/ast/untyped/UntypedNamespaceStatementNode.java delete mode 100644 src/main/java/org/zwobble/hobgoblin/ast/untyped/UntypedNode.java delete mode 100644 src/main/java/org/zwobble/hobgoblin/ast/untyped/UntypedStructDefinitionNode.java create mode 100644 src/main/java/org/zwobble/hobgoblin/compiler/ast/untyped/UntypedNamespaceNode.java create mode 100644 src/main/java/org/zwobble/hobgoblin/compiler/ast/untyped/UntypedNamespaceStatementNode.java create mode 100644 src/main/java/org/zwobble/hobgoblin/compiler/ast/untyped/UntypedNode.java create mode 100644 src/main/java/org/zwobble/hobgoblin/compiler/ast/untyped/UntypedStructDefinitionNode.java create mode 100644 src/main/java/org/zwobble/hobgoblin/compiler/errors/SourceError.java create mode 100644 src/main/java/org/zwobble/hobgoblin/compiler/parser/ParseError.java create mode 100644 src/main/java/org/zwobble/hobgoblin/compiler/parser/Parser.java create mode 100644 src/main/java/org/zwobble/hobgoblin/compiler/parser/Token.java create mode 100644 src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenIterator.java create mode 100644 src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java create mode 100644 src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java create mode 100644 src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokens.java create mode 100644 src/main/java/org/zwobble/hobgoblin/compiler/parser/UnexpectedTokenError.java create mode 100644 src/main/java/org/zwobble/hobgoblin/compiler/sources/FileFragmentSource.java create mode 100644 src/main/java/org/zwobble/hobgoblin/compiler/sources/NullSource.java create mode 100644 src/main/java/org/zwobble/hobgoblin/compiler/sources/RuntimeSource.java create mode 100644 src/main/java/org/zwobble/hobgoblin/compiler/sources/Source.java create mode 100644 src/main/java/org/zwobble/hobgoblin/compiler/types/NamespaceName.java delete mode 100644 src/main/java/org/zwobble/hobgoblin/errors/SourceError.java delete mode 100644 src/main/java/org/zwobble/hobgoblin/parser/ParseError.java delete mode 100644 src/main/java/org/zwobble/hobgoblin/parser/Parser.java delete mode 100644 src/main/java/org/zwobble/hobgoblin/parser/Token.java delete mode 100644 src/main/java/org/zwobble/hobgoblin/parser/TokenIterator.java delete mode 100644 src/main/java/org/zwobble/hobgoblin/parser/TokenType.java delete mode 100644 src/main/java/org/zwobble/hobgoblin/parser/Tokenizer.java delete mode 100644 src/main/java/org/zwobble/hobgoblin/parser/Tokens.java delete mode 100644 src/main/java/org/zwobble/hobgoblin/parser/UnexpectedTokenError.java delete mode 100644 src/main/java/org/zwobble/hobgoblin/sources/FileFragmentSource.java delete mode 100644 src/main/java/org/zwobble/hobgoblin/sources/NullSource.java delete mode 100644 src/main/java/org/zwobble/hobgoblin/sources/RuntimeSource.java delete mode 100644 src/main/java/org/zwobble/hobgoblin/sources/Source.java delete mode 100644 src/main/java/org/zwobble/hobgoblin/types/NamespaceName.java create mode 100644 src/test/java/org/zwobble/hobgoblin/compiler/parser/ParserNamespaceTests.java create mode 100644 src/test/java/org/zwobble/hobgoblin/compiler/parser/ParserTesting.java create mode 100644 src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java delete mode 100644 src/test/java/org/zwobble/hobgoblin/parser/ParserNamespaceTests.java delete mode 100644 src/test/java/org/zwobble/hobgoblin/parser/ParserTesting.java delete mode 100644 src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java diff --git a/src/main/java/org/zwobble/hobgoblin/ast/untyped/UntypedNamespaceNode.java b/src/main/java/org/zwobble/hobgoblin/ast/untyped/UntypedNamespaceNode.java deleted file mode 100644 index 0e74a2d..0000000 --- a/src/main/java/org/zwobble/hobgoblin/ast/untyped/UntypedNamespaceNode.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.zwobble.hobgoblin.ast.untyped; - -import org.zwobble.hobgoblin.sources.Source; -import org.zwobble.hobgoblin.types.NamespaceName; - -import java.util.List; - -public record UntypedNamespaceNode( - NamespaceName namespaceName, - List body, - Source source -) { -} diff --git a/src/main/java/org/zwobble/hobgoblin/ast/untyped/UntypedNamespaceStatementNode.java b/src/main/java/org/zwobble/hobgoblin/ast/untyped/UntypedNamespaceStatementNode.java deleted file mode 100644 index e1cda4c..0000000 --- a/src/main/java/org/zwobble/hobgoblin/ast/untyped/UntypedNamespaceStatementNode.java +++ /dev/null @@ -1,4 +0,0 @@ -package org.zwobble.hobgoblin.ast.untyped; - -public sealed interface UntypedNamespaceStatementNode extends UntypedNode permits UntypedStructDefinitionNode { -} diff --git a/src/main/java/org/zwobble/hobgoblin/ast/untyped/UntypedNode.java b/src/main/java/org/zwobble/hobgoblin/ast/untyped/UntypedNode.java deleted file mode 100644 index cadc7e4..0000000 --- a/src/main/java/org/zwobble/hobgoblin/ast/untyped/UntypedNode.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.zwobble.hobgoblin.ast.untyped; - -import org.zwobble.hobgoblin.sources.Source; - -public interface UntypedNode { - Source source(); -} diff --git a/src/main/java/org/zwobble/hobgoblin/ast/untyped/UntypedStructDefinitionNode.java b/src/main/java/org/zwobble/hobgoblin/ast/untyped/UntypedStructDefinitionNode.java deleted file mode 100644 index 87f1a8a..0000000 --- a/src/main/java/org/zwobble/hobgoblin/ast/untyped/UntypedStructDefinitionNode.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.zwobble.hobgoblin.ast.untyped; - -import org.zwobble.hobgoblin.sources.Source; - -public record UntypedStructDefinitionNode( - String name, - Source source -) implements UntypedNamespaceStatementNode { -} diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/ast/untyped/UntypedNamespaceNode.java b/src/main/java/org/zwobble/hobgoblin/compiler/ast/untyped/UntypedNamespaceNode.java new file mode 100644 index 0000000..d4d98d3 --- /dev/null +++ b/src/main/java/org/zwobble/hobgoblin/compiler/ast/untyped/UntypedNamespaceNode.java @@ -0,0 +1,13 @@ +package org.zwobble.hobgoblin.compiler.ast.untyped; + +import org.zwobble.hobgoblin.compiler.sources.Source; +import org.zwobble.hobgoblin.compiler.types.NamespaceName; + +import java.util.List; + +public record UntypedNamespaceNode( + NamespaceName namespaceName, + List body, + Source source +) { +} diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/ast/untyped/UntypedNamespaceStatementNode.java b/src/main/java/org/zwobble/hobgoblin/compiler/ast/untyped/UntypedNamespaceStatementNode.java new file mode 100644 index 0000000..c7b4cac --- /dev/null +++ b/src/main/java/org/zwobble/hobgoblin/compiler/ast/untyped/UntypedNamespaceStatementNode.java @@ -0,0 +1,4 @@ +package org.zwobble.hobgoblin.compiler.ast.untyped; + +public sealed interface UntypedNamespaceStatementNode extends UntypedNode permits UntypedStructDefinitionNode { +} diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/ast/untyped/UntypedNode.java b/src/main/java/org/zwobble/hobgoblin/compiler/ast/untyped/UntypedNode.java new file mode 100644 index 0000000..c816495 --- /dev/null +++ b/src/main/java/org/zwobble/hobgoblin/compiler/ast/untyped/UntypedNode.java @@ -0,0 +1,7 @@ +package org.zwobble.hobgoblin.compiler.ast.untyped; + +import org.zwobble.hobgoblin.compiler.sources.Source; + +public interface UntypedNode { + Source source(); +} diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/ast/untyped/UntypedStructDefinitionNode.java b/src/main/java/org/zwobble/hobgoblin/compiler/ast/untyped/UntypedStructDefinitionNode.java new file mode 100644 index 0000000..e86b141 --- /dev/null +++ b/src/main/java/org/zwobble/hobgoblin/compiler/ast/untyped/UntypedStructDefinitionNode.java @@ -0,0 +1,9 @@ +package org.zwobble.hobgoblin.compiler.ast.untyped; + +import org.zwobble.hobgoblin.compiler.sources.Source; + +public record UntypedStructDefinitionNode( + String name, + Source source +) implements UntypedNamespaceStatementNode { +} diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/errors/SourceError.java b/src/main/java/org/zwobble/hobgoblin/compiler/errors/SourceError.java new file mode 100644 index 0000000..afeb9b4 --- /dev/null +++ b/src/main/java/org/zwobble/hobgoblin/compiler/errors/SourceError.java @@ -0,0 +1,20 @@ +package org.zwobble.hobgoblin.compiler.errors; + +import org.zwobble.sourcetext.SourceRange; + +/** + * An error in the source code. These errors are expected when attempting to + * compile invalid code. + */ +public class SourceError extends RuntimeException { + private final SourceRange sourceRange; + + public SourceError(String message, SourceRange sourceRange) { + super(message); + this.sourceRange = sourceRange; + } + + public SourceRange sourceRange() { + return sourceRange; + } +} diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/ParseError.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/ParseError.java new file mode 100644 index 0000000..78d063d --- /dev/null +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/ParseError.java @@ -0,0 +1,23 @@ +package org.zwobble.hobgoblin.compiler.parser; + +import org.zwobble.hobgoblin.compiler.errors.SourceError; +import org.zwobble.sourcetext.SourceRange; + +public class ParseError extends SourceError { + public static ParseError unexpectedTextError( + String expected, + String actual, + SourceRange sourceRange + ) { + var message = String.format( + "Expected %s, but was %s", + expected, + actual + ); + return new ParseError(message, sourceRange); + } + + public ParseError(String message, SourceRange sourceRange) { + super(message, sourceRange); + } +} diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Parser.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Parser.java new file mode 100644 index 0000000..704f34f --- /dev/null +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Parser.java @@ -0,0 +1,34 @@ +package org.zwobble.hobgoblin.compiler.parser; + +import org.zwobble.hobgoblin.compiler.ast.untyped.UntypedNamespaceNode; +import org.zwobble.hobgoblin.compiler.sources.FileFragmentSource; +import org.zwobble.hobgoblin.compiler.sources.Source; +import org.zwobble.hobgoblin.compiler.types.NamespaceName; +import org.zwobble.sourcetext.SourcePosition; + +import java.util.List; + +public class Parser { + private Parser() { + } + + public static UntypedNamespaceNode parseNamespace( + TokenIterator tokens, + NamespaceName namespaceName + ) { + var start = tokens.startPosition(); + + var end = tokens.endPosition(); + var source = createSource(start, end); + + return new UntypedNamespaceNode( + namespaceName, + List.of(), + source + ); + } + + private static Source createSource(SourcePosition start, SourcePosition end) { + return new FileFragmentSource(start.to(end)); + } +} diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Token.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Token.java new file mode 100644 index 0000000..2cfa085 --- /dev/null +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Token.java @@ -0,0 +1,9 @@ +package org.zwobble.hobgoblin.compiler.parser; + +import org.zwobble.sourcetext.SourceRange; + +record Token(TokenType tokenType, SourceRange sourceRange) { + CharSequence charSequence() { + return sourceRange.charSequence(); + } +} diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenIterator.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenIterator.java new file mode 100644 index 0000000..9e1dd9c --- /dev/null +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenIterator.java @@ -0,0 +1,41 @@ +package org.zwobble.hobgoblin.compiler.parser; + +import org.zwobble.sourcetext.SourcePosition; + +import java.util.List; + +class TokenIterator { + private final List tokens; + private int tokenIndex; + private final Token tokenEnd; + + TokenIterator(List tokens, Token tokenEnd) { + this.tokens = tokens; + this.tokenEnd = tokenEnd; + this.tokenIndex = 0; + } + + public Token peek() { + return this.getAbsolute(this.tokenIndex); + } + + private Token getAbsolute(int index) { + if (index < this.tokens.size()) { + return this.tokens.get(index); + } else { + return this.tokenEnd; + } + } + + SourcePosition startPosition() { + return this.peek().sourceRange().start(); + } + + SourcePosition endPosition() { + if (this.tokenIndex == 0) { + return startPosition(); + } else { + return getAbsolute(this.tokenIndex - 1).sourceRange().end(); + } + } +} diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java new file mode 100644 index 0000000..72b1e88 --- /dev/null +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/TokenType.java @@ -0,0 +1,12 @@ +package org.zwobble.hobgoblin.compiler.parser; + +enum TokenType { + END, + IDENTIFIER, + WHITESPACE, + + KEYWORD_STRUCT, + + SYMBOL_BRACE_OPEN, + SYMBOL_BRACE_CLOSE, +} diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java new file mode 100644 index 0000000..c628346 --- /dev/null +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokenizer.java @@ -0,0 +1,157 @@ +package org.zwobble.hobgoblin.compiler.parser; + +import org.zwobble.sourcetext.SourceCharacterIterator; +import org.zwobble.sourcetext.SourcePosition; +import org.zwobble.sourcetext.SourceRange; +import org.zwobble.sourcetext.SourceText; + +import java.util.ArrayList; +import java.util.Optional; + +class Tokenizer { + private Tokenizer() { + } + + static Tokens tokenize(SourceText sourceText) { + var iterator = new CharacterIterator(sourceText); + var tokens = new ArrayList(); + + while (!iterator.isEnd()) { + if (trySkipWhitespace(iterator)) { + // Skip whitespace. + } else { + var token = tokenizeToken(iterator); + if (token.isPresent()) { + tokens.add(token.get()); + } else { + throw ParseError.unexpectedTextError( + "token", + describeCharacter(iterator.peek()), + iterator.characterSourceRange() + ); + } + } + } + + var tokenEnd = new Token(TokenType.END, iterator.characterSourceRange()); + + return new Tokens(tokens, tokenEnd); + } + + private static Optional tokenizeToken(CharacterIterator iterator) { + var identifierOrKeyword = tokenizeIdentifierOrKeyword(iterator); + if (identifierOrKeyword.isPresent()) { + return identifierOrKeyword; + } + + var symbol = tokenizeSymbol(iterator); + if (symbol.isPresent()) { + return symbol; + } + + throw ParseError.unexpectedTextError( + "token", + describeCharacter(iterator.peek()), + iterator.characterSourceRange() + ); + } + + private static boolean trySkipWhitespace(CharacterIterator iterator) { + var whitespace = false; + + while (isWhitespace(iterator.peek())) { + iterator.skip(); + whitespace = true; + } + + return whitespace; + } + + private static boolean isWhitespace(int character) { + return Character.isWhitespace(character); + } + + private static Optional tokenizeIdentifierOrKeyword(CharacterIterator iterator) { + if (!Character.isAlphabetic(iterator.peek())) { + return Optional.empty(); + } + + var start = iterator.position(); + do { + iterator.skip(); + } while (Character.isAlphabetic(iterator.peek())); + var end = iterator.position(); + + var sourceRange = start.to(end); + TokenType tokenType; + if (CharSequence.compare(sourceRange.charSequence(), "struct") == 0) { + tokenType = TokenType.KEYWORD_STRUCT; + } else { + tokenType = TokenType.IDENTIFIER; + } + var token = new Token(tokenType, start.to(end)); + + return Optional.of(token); + } + + private static Optional tokenizeSymbol(CharacterIterator iterator) { + TokenType tokenType; + var start = iterator.position(); + + switch (iterator.peek()) { + case '{': + tokenType = TokenType.SYMBOL_BRACE_OPEN; + iterator.skip(); + break; + + case '}': + tokenType = TokenType.SYMBOL_BRACE_CLOSE; + iterator.skip(); + break; + + default: + return Optional.empty(); + } + + var end = iterator.position(); + var token = new Token(tokenType, start.to(end)); + return Optional.of(token); + } + + private static String describeCharacter(int character) { + // TODO: handle characters that should be escaped + if (character == -1) { + return "end of document"; + } else { + return String.format("'%s'", new String(new int[]{character}, 0, 1)); + } + } + + private static class CharacterIterator { + private final SourceCharacterIterator iterator; + + CharacterIterator(SourceText sourceText) { + this.iterator = sourceText.characterIterator(); + } + + boolean isEnd() { + return this.iterator.isEnd(); + } + + SourceRange characterSourceRange() { + return this.iterator.characterSourceRange(); + } + + int peek() { + return this.iterator.peek(); + } + + void skip() { + this.iterator.skip(); + } + + public SourcePosition position() { + return this.iterator.position(); + } + } +} diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokens.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokens.java new file mode 100644 index 0000000..618114e --- /dev/null +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/Tokens.java @@ -0,0 +1,9 @@ +package org.zwobble.hobgoblin.compiler.parser; + +import java.util.List; + +record Tokens(List tokens, Token tokenEnd) { + TokenIterator iterator() { + return new TokenIterator(this.tokens, this.tokenEnd); + } +} diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/parser/UnexpectedTokenError.java b/src/main/java/org/zwobble/hobgoblin/compiler/parser/UnexpectedTokenError.java new file mode 100644 index 0000000..543aeb4 --- /dev/null +++ b/src/main/java/org/zwobble/hobgoblin/compiler/parser/UnexpectedTokenError.java @@ -0,0 +1,28 @@ +package org.zwobble.hobgoblin.compiler.parser; + +import org.zwobble.hobgoblin.compiler.errors.SourceError; + +public class UnexpectedTokenError extends SourceError { + private final String expected; + private final Token actual; + + public UnexpectedTokenError( + String expected, + Token actual + ) { + super( + "Expected: " + expected + "\nBut got: " + actual.tokenType() + " \"" + actual.charSequence() + "\"", + actual.sourceRange() + ); + this.expected = expected; + this.actual = actual; + } + + public String getExpected() { + return expected; + } + + public Token getActual() { + return actual; + } +} diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/sources/FileFragmentSource.java b/src/main/java/org/zwobble/hobgoblin/compiler/sources/FileFragmentSource.java new file mode 100644 index 0000000..016a74c --- /dev/null +++ b/src/main/java/org/zwobble/hobgoblin/compiler/sources/FileFragmentSource.java @@ -0,0 +1,10 @@ +package org.zwobble.hobgoblin.compiler.sources; + +import org.zwobble.sourcetext.SourceRange; + +public record FileFragmentSource(SourceRange sourceRange) implements Source { + @Override + public String describe() { + return this.sourceRange.describe(); + } +} diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/sources/NullSource.java b/src/main/java/org/zwobble/hobgoblin/compiler/sources/NullSource.java new file mode 100644 index 0000000..8937951 --- /dev/null +++ b/src/main/java/org/zwobble/hobgoblin/compiler/sources/NullSource.java @@ -0,0 +1,13 @@ +package org.zwobble.hobgoblin.compiler.sources; + +public class NullSource implements Source { + private NullSource() { + } + + public static final NullSource INSTANCE = new NullSource(); + + @Override + public String describe() { + return "(null)"; + } +} diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/sources/RuntimeSource.java b/src/main/java/org/zwobble/hobgoblin/compiler/sources/RuntimeSource.java new file mode 100644 index 0000000..aec5683 --- /dev/null +++ b/src/main/java/org/zwobble/hobgoblin/compiler/sources/RuntimeSource.java @@ -0,0 +1,13 @@ +package org.zwobble.hobgoblin.compiler.sources; + +public class RuntimeSource implements Source { + private RuntimeSource() { + } + + public static final RuntimeSource INSTANCE = new RuntimeSource(); + + @Override + public String describe() { + return "(runtime)"; + } +} diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/sources/Source.java b/src/main/java/org/zwobble/hobgoblin/compiler/sources/Source.java new file mode 100644 index 0000000..aecf30b --- /dev/null +++ b/src/main/java/org/zwobble/hobgoblin/compiler/sources/Source.java @@ -0,0 +1,5 @@ +package org.zwobble.hobgoblin.compiler.sources; + +public interface Source { + String describe(); +} diff --git a/src/main/java/org/zwobble/hobgoblin/compiler/types/NamespaceName.java b/src/main/java/org/zwobble/hobgoblin/compiler/types/NamespaceName.java new file mode 100644 index 0000000..e0fb2d3 --- /dev/null +++ b/src/main/java/org/zwobble/hobgoblin/compiler/types/NamespaceName.java @@ -0,0 +1,15 @@ +package org.zwobble.hobgoblin.compiler.types; + +import java.util.Arrays; +import java.util.List; + +public record NamespaceName(List parts) { + public static NamespaceName of(String... parts) { + return new NamespaceName(Arrays.asList(parts)); + } + + @Override + public String toString() { + return String.join("/", parts); + } +} diff --git a/src/main/java/org/zwobble/hobgoblin/errors/SourceError.java b/src/main/java/org/zwobble/hobgoblin/errors/SourceError.java deleted file mode 100644 index 1ebb679..0000000 --- a/src/main/java/org/zwobble/hobgoblin/errors/SourceError.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.zwobble.hobgoblin.errors; - -import org.zwobble.sourcetext.SourceRange; - -/** - * An error in the source code. These errors are expected when attempting to - * compile invalid code. - */ -public class SourceError extends RuntimeException { - private final SourceRange sourceRange; - - public SourceError(String message, SourceRange sourceRange) { - super(message); - this.sourceRange = sourceRange; - } - - public SourceRange sourceRange() { - return sourceRange; - } -} diff --git a/src/main/java/org/zwobble/hobgoblin/parser/ParseError.java b/src/main/java/org/zwobble/hobgoblin/parser/ParseError.java deleted file mode 100644 index 16b0c4f..0000000 --- a/src/main/java/org/zwobble/hobgoblin/parser/ParseError.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.zwobble.hobgoblin.parser; - -import org.zwobble.hobgoblin.errors.SourceError; -import org.zwobble.sourcetext.SourceRange; - -public class ParseError extends SourceError { - public static ParseError unexpectedTextError( - String expected, - String actual, - SourceRange sourceRange - ) { - var message = String.format( - "Expected %s, but was %s", - expected, - actual - ); - return new ParseError(message, sourceRange); - } - - public ParseError(String message, SourceRange sourceRange) { - super(message, sourceRange); - } -} diff --git a/src/main/java/org/zwobble/hobgoblin/parser/Parser.java b/src/main/java/org/zwobble/hobgoblin/parser/Parser.java deleted file mode 100644 index 7e6c3fe..0000000 --- a/src/main/java/org/zwobble/hobgoblin/parser/Parser.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.zwobble.hobgoblin.parser; - -import org.zwobble.hobgoblin.ast.untyped.UntypedNamespaceNode; -import org.zwobble.hobgoblin.sources.FileFragmentSource; -import org.zwobble.hobgoblin.sources.Source; -import org.zwobble.hobgoblin.types.NamespaceName; -import org.zwobble.sourcetext.SourcePosition; - -import java.util.List; - -public class Parser { - private Parser() { - } - - public static UntypedNamespaceNode parseNamespace( - TokenIterator tokens, - NamespaceName namespaceName - ) { - var start = tokens.startPosition(); - - var end = tokens.endPosition(); - var source = createSource(start, end); - - return new UntypedNamespaceNode( - namespaceName, - List.of(), - source - ); - } - - private static Source createSource(SourcePosition start, SourcePosition end) { - return new FileFragmentSource(start.to(end)); - } -} diff --git a/src/main/java/org/zwobble/hobgoblin/parser/Token.java b/src/main/java/org/zwobble/hobgoblin/parser/Token.java deleted file mode 100644 index a3edf50..0000000 --- a/src/main/java/org/zwobble/hobgoblin/parser/Token.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.zwobble.hobgoblin.parser; - -import org.zwobble.sourcetext.SourceRange; - -record Token(TokenType tokenType, SourceRange sourceRange) { - CharSequence charSequence() { - return sourceRange.charSequence(); - } -} diff --git a/src/main/java/org/zwobble/hobgoblin/parser/TokenIterator.java b/src/main/java/org/zwobble/hobgoblin/parser/TokenIterator.java deleted file mode 100644 index f64d51f..0000000 --- a/src/main/java/org/zwobble/hobgoblin/parser/TokenIterator.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.zwobble.hobgoblin.parser; - -import org.zwobble.sourcetext.SourcePosition; - -import java.util.List; - -class TokenIterator { - private final List tokens; - private int tokenIndex; - private final Token tokenEnd; - - TokenIterator(List tokens, Token tokenEnd) { - this.tokens = tokens; - this.tokenEnd = tokenEnd; - this.tokenIndex = 0; - } - - public Token peek() { - return this.getAbsolute(this.tokenIndex); - } - - private Token getAbsolute(int index) { - if (index < this.tokens.size()) { - return this.tokens.get(index); - } else { - return this.tokenEnd; - } - } - - SourcePosition startPosition() { - return this.peek().sourceRange().start(); - } - - SourcePosition endPosition() { - if (this.tokenIndex == 0) { - return startPosition(); - } else { - return getAbsolute(this.tokenIndex - 1).sourceRange().end(); - } - } -} diff --git a/src/main/java/org/zwobble/hobgoblin/parser/TokenType.java b/src/main/java/org/zwobble/hobgoblin/parser/TokenType.java deleted file mode 100644 index 100ee77..0000000 --- a/src/main/java/org/zwobble/hobgoblin/parser/TokenType.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.zwobble.hobgoblin.parser; - -enum TokenType { - END, - IDENTIFIER, - WHITESPACE, - - KEYWORD_STRUCT, - - SYMBOL_BRACE_OPEN, - SYMBOL_BRACE_CLOSE, -} diff --git a/src/main/java/org/zwobble/hobgoblin/parser/Tokenizer.java b/src/main/java/org/zwobble/hobgoblin/parser/Tokenizer.java deleted file mode 100644 index 82c843d..0000000 --- a/src/main/java/org/zwobble/hobgoblin/parser/Tokenizer.java +++ /dev/null @@ -1,157 +0,0 @@ -package org.zwobble.hobgoblin.parser; - -import org.zwobble.sourcetext.SourceCharacterIterator; -import org.zwobble.sourcetext.SourcePosition; -import org.zwobble.sourcetext.SourceRange; -import org.zwobble.sourcetext.SourceText; - -import java.util.ArrayList; -import java.util.Optional; - -class Tokenizer { - private Tokenizer() { - } - - static Tokens tokenize(SourceText sourceText) { - var iterator = new CharacterIterator(sourceText); - var tokens = new ArrayList(); - - while (!iterator.isEnd()) { - if (trySkipWhitespace(iterator)) { - // Skip whitespace. - } else { - var token = tokenizeToken(iterator); - if (token.isPresent()) { - tokens.add(token.get()); - } else { - throw ParseError.unexpectedTextError( - "token", - describeCharacter(iterator.peek()), - iterator.characterSourceRange() - ); - } - } - } - - var tokenEnd = new Token(TokenType.END, iterator.characterSourceRange()); - - return new Tokens(tokens, tokenEnd); - } - - private static Optional tokenizeToken(CharacterIterator iterator) { - var identifierOrKeyword = tokenizeIdentifierOrKeyword(iterator); - if (identifierOrKeyword.isPresent()) { - return identifierOrKeyword; - } - - var symbol = tokenizeSymbol(iterator); - if (symbol.isPresent()) { - return symbol; - } - - throw ParseError.unexpectedTextError( - "token", - describeCharacter(iterator.peek()), - iterator.characterSourceRange() - ); - } - - private static boolean trySkipWhitespace(CharacterIterator iterator) { - var whitespace = false; - - while (isWhitespace(iterator.peek())) { - iterator.skip(); - whitespace = true; - } - - return whitespace; - } - - private static boolean isWhitespace(int character) { - return Character.isWhitespace(character); - } - - private static Optional tokenizeIdentifierOrKeyword(CharacterIterator iterator) { - if (!Character.isAlphabetic(iterator.peek())) { - return Optional.empty(); - } - - var start = iterator.position(); - do { - iterator.skip(); - } while (Character.isAlphabetic(iterator.peek())); - var end = iterator.position(); - - var sourceRange = start.to(end); - TokenType tokenType; - if (CharSequence.compare(sourceRange.charSequence(), "struct") == 0) { - tokenType = TokenType.KEYWORD_STRUCT; - } else { - tokenType = TokenType.IDENTIFIER; - } - var token = new Token(tokenType, start.to(end)); - - return Optional.of(token); - } - - private static Optional tokenizeSymbol(CharacterIterator iterator) { - TokenType tokenType; - var start = iterator.position(); - - switch (iterator.peek()) { - case '{': - tokenType = TokenType.SYMBOL_BRACE_OPEN; - iterator.skip(); - break; - - case '}': - tokenType = TokenType.SYMBOL_BRACE_CLOSE; - iterator.skip(); - break; - - default: - return Optional.empty(); - } - - var end = iterator.position(); - var token = new Token(tokenType, start.to(end)); - return Optional.of(token); - } - - private static String describeCharacter(int character) { - // TODO: handle characters that should be escaped - if (character == -1) { - return "end of document"; - } else { - return String.format("'%s'", new String(new int[]{character}, 0, 1)); - } - } - - private static class CharacterIterator { - private final SourceCharacterIterator iterator; - - CharacterIterator(SourceText sourceText) { - this.iterator = sourceText.characterIterator(); - } - - boolean isEnd() { - return this.iterator.isEnd(); - } - - SourceRange characterSourceRange() { - return this.iterator.characterSourceRange(); - } - - int peek() { - return this.iterator.peek(); - } - - void skip() { - this.iterator.skip(); - } - - public SourcePosition position() { - return this.iterator.position(); - } - } -} diff --git a/src/main/java/org/zwobble/hobgoblin/parser/Tokens.java b/src/main/java/org/zwobble/hobgoblin/parser/Tokens.java deleted file mode 100644 index 3e979a3..0000000 --- a/src/main/java/org/zwobble/hobgoblin/parser/Tokens.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.zwobble.hobgoblin.parser; - -import java.util.List; - -record Tokens(List tokens, Token tokenEnd) { - TokenIterator iterator() { - return new TokenIterator(this.tokens, this.tokenEnd); - } -} diff --git a/src/main/java/org/zwobble/hobgoblin/parser/UnexpectedTokenError.java b/src/main/java/org/zwobble/hobgoblin/parser/UnexpectedTokenError.java deleted file mode 100644 index 2ff6e2c..0000000 --- a/src/main/java/org/zwobble/hobgoblin/parser/UnexpectedTokenError.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.zwobble.hobgoblin.parser; - -import org.zwobble.hobgoblin.errors.SourceError; - -public class UnexpectedTokenError extends SourceError { - private final String expected; - private final Token actual; - - public UnexpectedTokenError( - String expected, - Token actual - ) { - super( - "Expected: " + expected + "\nBut got: " + actual.tokenType() + " \"" + actual.charSequence() + "\"", - actual.sourceRange() - ); - this.expected = expected; - this.actual = actual; - } - - public String getExpected() { - return expected; - } - - public Token getActual() { - return actual; - } -} diff --git a/src/main/java/org/zwobble/hobgoblin/sources/FileFragmentSource.java b/src/main/java/org/zwobble/hobgoblin/sources/FileFragmentSource.java deleted file mode 100644 index 25116dc..0000000 --- a/src/main/java/org/zwobble/hobgoblin/sources/FileFragmentSource.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.zwobble.hobgoblin.sources; - -import org.zwobble.sourcetext.SourceRange; - -public record FileFragmentSource(SourceRange sourceRange) implements Source { - @Override - public String describe() { - return this.sourceRange.describe(); - } -} diff --git a/src/main/java/org/zwobble/hobgoblin/sources/NullSource.java b/src/main/java/org/zwobble/hobgoblin/sources/NullSource.java deleted file mode 100644 index 61cc36d..0000000 --- a/src/main/java/org/zwobble/hobgoblin/sources/NullSource.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.zwobble.hobgoblin.sources; - -public class NullSource implements Source { - private NullSource() { - } - - public static final NullSource INSTANCE = new NullSource(); - - @Override - public String describe() { - return "(null)"; - } -} diff --git a/src/main/java/org/zwobble/hobgoblin/sources/RuntimeSource.java b/src/main/java/org/zwobble/hobgoblin/sources/RuntimeSource.java deleted file mode 100644 index 7c23bfa..0000000 --- a/src/main/java/org/zwobble/hobgoblin/sources/RuntimeSource.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.zwobble.hobgoblin.sources; - -public class RuntimeSource implements Source { - private RuntimeSource() { - } - - public static final RuntimeSource INSTANCE = new RuntimeSource(); - - @Override - public String describe() { - return "(runtime)"; - } -} diff --git a/src/main/java/org/zwobble/hobgoblin/sources/Source.java b/src/main/java/org/zwobble/hobgoblin/sources/Source.java deleted file mode 100644 index 2debfa6..0000000 --- a/src/main/java/org/zwobble/hobgoblin/sources/Source.java +++ /dev/null @@ -1,5 +0,0 @@ -package org.zwobble.hobgoblin.sources; - -public interface Source { - String describe(); -} diff --git a/src/main/java/org/zwobble/hobgoblin/types/NamespaceName.java b/src/main/java/org/zwobble/hobgoblin/types/NamespaceName.java deleted file mode 100644 index 143d0c4..0000000 --- a/src/main/java/org/zwobble/hobgoblin/types/NamespaceName.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.zwobble.hobgoblin.types; - -import java.util.Arrays; -import java.util.List; - -public record NamespaceName(List parts) { - public static NamespaceName of(String... parts) { - return new NamespaceName(Arrays.asList(parts)); - } - - @Override - public String toString() { - return String.join("/", parts); - } -} diff --git a/src/test/java/org/zwobble/hobgoblin/compiler/parser/ParserNamespaceTests.java b/src/test/java/org/zwobble/hobgoblin/compiler/parser/ParserNamespaceTests.java new file mode 100644 index 0000000..e5796e4 --- /dev/null +++ b/src/test/java/org/zwobble/hobgoblin/compiler/parser/ParserNamespaceTests.java @@ -0,0 +1,27 @@ +package org.zwobble.hobgoblin.compiler.parser; + +import org.junit.jupiter.api.Test; +import org.zwobble.hobgoblin.compiler.ast.untyped.UntypedNamespaceNode; +import org.zwobble.hobgoblin.compiler.types.NamespaceName; + +import static org.zwobble.hobgoblin.compiler.parser.ParserTesting.parseString; +import static org.zwobble.precisely.AssertThat.assertThat; +import static org.zwobble.precisely.Matchers.*; + +public class ParserNamespaceTests { + @Test + public void emptyNamespaceHasEmptyBody() { + var source = ""; + + var node = parseString( + source, + tokens -> Parser.parseNamespace(tokens, NamespaceName.of("Example", "Project")) + ); + + assertThat(node, instanceOf( + UntypedNamespaceNode.class, + has("namespaceName", x -> x.namespaceName(), equalTo(NamespaceName.of("Example", "Project"))), + has("body", x -> x.body(), isSequence()) + )); + } +} diff --git a/src/test/java/org/zwobble/hobgoblin/compiler/parser/ParserTesting.java b/src/test/java/org/zwobble/hobgoblin/compiler/parser/ParserTesting.java new file mode 100644 index 0000000..6fec759 --- /dev/null +++ b/src/test/java/org/zwobble/hobgoblin/compiler/parser/ParserTesting.java @@ -0,0 +1,25 @@ +package org.zwobble.hobgoblin.compiler.parser; + +import org.zwobble.hobgoblin.compiler.parser.TokenIterator; +import org.zwobble.hobgoblin.compiler.parser.TokenType; +import org.zwobble.hobgoblin.compiler.parser.Tokenizer; +import org.zwobble.sourcetext.SourceText; + +import java.util.function.Function; + +import static org.zwobble.precisely.AssertThat.assertThat; +import static org.zwobble.precisely.Matchers.equalTo; + +public class ParserTesting { + public static T parseString( + String sourceContents, + Function parseTokens + ) { + var sourceText = SourceText.fromString("", sourceContents); + var tokens = Tokenizer.tokenize(sourceText); + var tokenIterator = tokens.iterator(); + var result = parseTokens.apply(tokenIterator); + assertThat(tokenIterator.peek().tokenType(), equalTo(TokenType.END)); + return result; + } +} diff --git a/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java new file mode 100644 index 0000000..e564e7b --- /dev/null +++ b/src/test/java/org/zwobble/hobgoblin/compiler/parser/TokenizerTests.java @@ -0,0 +1,95 @@ +package org.zwobble.hobgoblin.compiler.parser; + +import org.junit.jupiter.api.Test; +import org.zwobble.hobgoblin.compiler.parser.Token; +import org.zwobble.hobgoblin.compiler.parser.TokenType; +import org.zwobble.hobgoblin.compiler.parser.Tokenizer; +import org.zwobble.sourcetext.SourceRange; +import org.zwobble.sourcetext.SourceText; + +import static org.zwobble.precisely.AssertThat.assertThat; +import static org.zwobble.precisely.Matchers.equalTo; +import static org.zwobble.precisely.Matchers.isSequence; + +public class TokenizerTests { + @Test + public void emptySourceTextIsTokenizedToEmpty() { + var sourceText = SourceText.fromString("", ""); + + var tokens = Tokenizer.tokenize(sourceText); + + assertThat(tokens.tokens(), isSequence()); + } + + @Test + public void keywordStructIsTokenized() { + var sourceText = SourceText.fromString("", "struct"); + + var tokens = Tokenizer.tokenize(sourceText); + + assertThat(tokens.tokens(), isSequence( + equalTo(new Token(TokenType.KEYWORD_STRUCT, sourceRange(sourceText, 0, 6))) + )); + } + + @Test + public void identifierIsTokenized() { + var sourceText = SourceText.fromString("", "blah"); + + var tokens = Tokenizer.tokenize(sourceText); + + assertThat(tokens.tokens(), isSequence( + equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 0, 4))) + )); + } + + @Test + public void symbolBraceOpenIsTokenized() { + var sourceText = SourceText.fromString("", "{"); + + var tokens = Tokenizer.tokenize(sourceText); + + assertThat(tokens.tokens(), isSequence( + equalTo(new Token(TokenType.SYMBOL_BRACE_OPEN, sourceRange(sourceText, 0, 1))) + )); + } + + @Test + public void symbolBraceCloseIsTokenized() { + var sourceText = SourceText.fromString("", "}"); + + var tokens = Tokenizer.tokenize(sourceText); + + assertThat(tokens.tokens(), isSequence( + equalTo(new Token(TokenType.SYMBOL_BRACE_CLOSE, sourceRange(sourceText, 0, 1))) + )); + } + + @Test + public void whitespaceIsIgnored() { + var sourceText = SourceText.fromString("", " blah "); + + var tokens = Tokenizer.tokenize(sourceText); + + assertThat(tokens.tokens(), isSequence( + equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 2, 6))) + )); + } + + @Test + public void whitespaceSeparatesOtherTokens() { + var sourceText = SourceText.fromString("", "one two \t three"); + + var tokens = Tokenizer.tokenize(sourceText); + + assertThat(tokens.tokens(), isSequence( + equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 0, 3))), + equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 4, 7))), + equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 10, 15))) + )); + } + + private static SourceRange sourceRange(SourceText sourceText, int fromCharacterIndex, int toCharacterIndex) { + return sourceText.characterPosition(fromCharacterIndex).to(sourceText.characterPosition(toCharacterIndex)); + } +} diff --git a/src/test/java/org/zwobble/hobgoblin/parser/ParserNamespaceTests.java b/src/test/java/org/zwobble/hobgoblin/parser/ParserNamespaceTests.java deleted file mode 100644 index 89527e9..0000000 --- a/src/test/java/org/zwobble/hobgoblin/parser/ParserNamespaceTests.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.zwobble.hobgoblin.parser; - -import org.junit.jupiter.api.Test; -import org.zwobble.hobgoblin.ast.untyped.UntypedNamespaceNode; -import org.zwobble.hobgoblin.types.NamespaceName; - -import static org.zwobble.hobgoblin.parser.ParserTesting.parseString; -import static org.zwobble.precisely.AssertThat.assertThat; -import static org.zwobble.precisely.Matchers.*; - -public class ParserNamespaceTests { - @Test - public void emptyNamespaceHasEmptyBody() { - var source = ""; - - var node = parseString( - source, - tokens -> Parser.parseNamespace(tokens, NamespaceName.of("Example", "Project")) - ); - - assertThat(node, instanceOf( - UntypedNamespaceNode.class, - has("namespaceName", x -> x.namespaceName(), equalTo(NamespaceName.of("Example", "Project"))), - has("body", x -> x.body(), isSequence()) - )); - } -} diff --git a/src/test/java/org/zwobble/hobgoblin/parser/ParserTesting.java b/src/test/java/org/zwobble/hobgoblin/parser/ParserTesting.java deleted file mode 100644 index 14cfb62..0000000 --- a/src/test/java/org/zwobble/hobgoblin/parser/ParserTesting.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.zwobble.hobgoblin.parser; - -import org.zwobble.sourcetext.SourceText; - -import java.util.function.Function; - -import static org.zwobble.precisely.AssertThat.assertThat; -import static org.zwobble.precisely.Matchers.equalTo; - -public class ParserTesting { - public static T parseString( - String sourceContents, - Function parseTokens - ) { - var sourceText = SourceText.fromString("", sourceContents); - var tokens = Tokenizer.tokenize(sourceText); - var tokenIterator = tokens.iterator(); - var result = parseTokens.apply(tokenIterator); - assertThat(tokenIterator.peek().tokenType(), equalTo(TokenType.END)); - return result; - } -} diff --git a/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java b/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java deleted file mode 100644 index 6f77470..0000000 --- a/src/test/java/org/zwobble/hobgoblin/parser/TokenizerTests.java +++ /dev/null @@ -1,92 +0,0 @@ -package org.zwobble.hobgoblin.parser; - -import org.junit.jupiter.api.Test; -import org.zwobble.sourcetext.SourceRange; -import org.zwobble.sourcetext.SourceText; - -import static org.zwobble.precisely.AssertThat.assertThat; -import static org.zwobble.precisely.Matchers.equalTo; -import static org.zwobble.precisely.Matchers.isSequence; - -public class TokenizerTests { - @Test - public void emptySourceTextIsTokenizedToEmpty() { - var sourceText = SourceText.fromString("", ""); - - var tokens = Tokenizer.tokenize(sourceText); - - assertThat(tokens.tokens(), isSequence()); - } - - @Test - public void keywordStructIsTokenized() { - var sourceText = SourceText.fromString("", "struct"); - - var tokens = Tokenizer.tokenize(sourceText); - - assertThat(tokens.tokens(), isSequence( - equalTo(new Token(TokenType.KEYWORD_STRUCT, sourceRange(sourceText, 0, 6))) - )); - } - - @Test - public void identifierIsTokenized() { - var sourceText = SourceText.fromString("", "blah"); - - var tokens = Tokenizer.tokenize(sourceText); - - assertThat(tokens.tokens(), isSequence( - equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 0, 4))) - )); - } - - @Test - public void symbolBraceOpenIsTokenized() { - var sourceText = SourceText.fromString("", "{"); - - var tokens = Tokenizer.tokenize(sourceText); - - assertThat(tokens.tokens(), isSequence( - equalTo(new Token(TokenType.SYMBOL_BRACE_OPEN, sourceRange(sourceText, 0, 1))) - )); - } - - @Test - public void symbolBraceCloseIsTokenized() { - var sourceText = SourceText.fromString("", "}"); - - var tokens = Tokenizer.tokenize(sourceText); - - assertThat(tokens.tokens(), isSequence( - equalTo(new Token(TokenType.SYMBOL_BRACE_CLOSE, sourceRange(sourceText, 0, 1))) - )); - } - - @Test - public void whitespaceIsIgnored() { - var sourceText = SourceText.fromString("", " blah "); - - var tokens = Tokenizer.tokenize(sourceText); - - assertThat(tokens.tokens(), isSequence( - equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 2, 6))) - )); - } - - @Test - public void whitespaceSeparatesOtherTokens() { - var sourceText = SourceText.fromString("", "one two \t three"); - - var tokens = Tokenizer.tokenize(sourceText); - - assertThat(tokens.tokens(), isSequence( - equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 0, 3))), - equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 4, 7))), - equalTo(new Token(TokenType.IDENTIFIER, sourceRange(sourceText, 10, 15))) - )); - } - - private static SourceRange sourceRange(SourceText sourceText, int fromCharacterIndex, int toCharacterIndex) { - return sourceText.characterPosition(fromCharacterIndex).to(sourceText.characterPosition(toCharacterIndex)); - } -} -- cgit v1.2.3