package org.zwobble.hobgoblin.compiler.parser; import org.zwobble.sourcetext.SourceCharacterIterator; import org.zwobble.sourcetext.SourcePosition; import org.zwobble.sourcetext.SourceRange; import org.zwobble.sourcetext.SourceText; import java.util.ArrayList; import java.util.Optional; class Tokenizer { private Tokenizer() { } static Tokens tokenize(SourceText sourceText) { var iterator = new CharacterIterator(sourceText); var tokens = new ArrayList(); while (!iterator.isEnd()) { if (trySkipWhitespace(iterator)) { // Skip whitespace. } else if (trySkipComment(iterator)) { // Skip comments. } else { var token = tokenizeToken(iterator); if (token.isPresent()) { tokens.add(token.get()); } else { throw ParseError.unexpectedTextError( "token", describeCharacter(iterator.peek()), iterator.characterSourceRange() ); } } } var tokenEnd = new Token(TokenType.END, iterator.characterSourceRange()); return new Tokens(tokens, tokenEnd); } private static boolean trySkipComment(CharacterIterator iterator) { if (!"//".contentEquals(iterator.peekSequence(2)) || "///".contentEquals(iterator.peekSequence(3))) { return false; } skipToEndOfLine(iterator); return true; } private static Optional tokenizeToken(CharacterIterator iterator) { var identifierOrKeyword = tokenizeIdentifierOrKeyword(iterator); if (identifierOrKeyword.isPresent()) { return identifierOrKeyword; } var docComment = tokenizeDocComment(iterator); if (docComment.isPresent()) { return docComment; } var symbol = tokenizeSymbol(iterator); if (symbol.isPresent()) { return symbol; } throw ParseError.unexpectedTextError( "token", describeCharacter(iterator.peek()), iterator.characterSourceRange() ); } private static boolean trySkipWhitespace(CharacterIterator iterator) { var whitespace = false; while (isWhitespace(iterator.peek())) { iterator.skip(); whitespace = true; } return whitespace; } private static boolean isWhitespace(int character) { return Character.isWhitespace(character); } private static Optional tokenizeDocComment(CharacterIterator iterator) { if (!"///".contentEquals(iterator.peekSequence(3))) { return Optional.empty(); } var start = iterator.position(); skipToEndOfLine(iterator); var end = iterator.position(); var token = new Token(TokenType.DOC_COMMENT, start.to(end)); return Optional.of(token); } private static void skipToEndOfLine(CharacterIterator iterator) { while (!iterator.isEnd() && iterator.peek() != '\n') { iterator.skip(); } if (iterator.peek() == '\n') { iterator.skip(); } } private static Optional tokenizeIdentifierOrKeyword(CharacterIterator iterator) { if (!Character.isAlphabetic(iterator.peek())) { return Optional.empty(); } var start = iterator.position(); do { iterator.skip(); } while (Character.isAlphabetic(iterator.peek()) || Character.isDigit(iterator.peek())); var end = iterator.position(); var sourceRange = start.to(end); TokenType tokenType; if (CharSequence.compare(sourceRange.charSequence(), "enum") == 0) { tokenType = TokenType.KEYWORD_ENUM; } else if (CharSequence.compare(sourceRange.charSequence(), "field") == 0) { tokenType = TokenType.KEYWORD_FIELD; } else if (CharSequence.compare(sourceRange.charSequence(), "from") == 0) { tokenType = TokenType.KEYWORD_FROM; } else if (CharSequence.compare(sourceRange.charSequence(), "import") == 0) { tokenType = TokenType.KEYWORD_IMPORT; } else if (CharSequence.compare(sourceRange.charSequence(), "native") == 0) { tokenType = TokenType.KEYWORD_NATIVE; } else if (CharSequence.compare(sourceRange.charSequence(), "struct") == 0) { tokenType = TokenType.KEYWORD_STRUCT; } else if (CharSequence.compare(sourceRange.charSequence(), "sum") == 0) { tokenType = TokenType.KEYWORD_SUM; } else if (CharSequence.compare(sourceRange.charSequence(), "variant") == 0) { tokenType = TokenType.KEYWORD_VARIANT; } else { tokenType = TokenType.IDENTIFIER; } var token = new Token(tokenType, start.to(end)); return Optional.of(token); } private static Optional tokenizeSymbol(CharacterIterator iterator) { TokenType tokenType; var start = iterator.position(); if ("..".contentEquals(iterator.peekSequence(2))) { tokenType = TokenType.SYMBOL_DOT_DOT; iterator.skip(); iterator.skip(); } else { switch (iterator.peek()) { case '{': tokenType = TokenType.SYMBOL_BRACE_OPEN; iterator.skip(); break; case '}': tokenType = TokenType.SYMBOL_BRACE_CLOSE; iterator.skip(); break; case ':': tokenType = TokenType.SYMBOL_COLON; iterator.skip(); break; case ',': tokenType = TokenType.SYMBOL_COMMA; iterator.skip(); break; case '.': tokenType = TokenType.SYMBOL_DOT; iterator.skip(); break; case '/': tokenType = TokenType.SYMBOL_FORWARD_SLASH; iterator.skip(); break; case '(': tokenType = TokenType.SYMBOL_PAREN_OPEN; iterator.skip(); break; case ')': tokenType = TokenType.SYMBOL_PAREN_CLOSE; iterator.skip(); break; case ';': tokenType = TokenType.SYMBOL_SEMICOLON; iterator.skip(); break; case '[': tokenType = TokenType.SYMBOL_SQUARE_OPEN; iterator.skip(); break; case ']': tokenType = TokenType.SYMBOL_SQUARE_CLOSE; iterator.skip(); break; default: return Optional.empty(); } } var end = iterator.position(); var token = new Token(tokenType, start.to(end)); return Optional.of(token); } private static String describeCharacter(int character) { // TODO: handle characters that should be escaped if (character == -1) { return "end of document"; } else { return String.format("'%s'", new String(new int[]{character}, 0, 1)); } } private static class CharacterIterator { private final SourceCharacterIterator iterator; CharacterIterator(SourceText sourceText) { this.iterator = sourceText.characterIterator(); } boolean isEnd() { return this.iterator.isEnd(); } SourceRange characterSourceRange() { return this.iterator.characterSourceRange(); } int peek() { return this.iterator.peek(); } CharSequence peekSequence(int length) { if (length > this.iterator.remaining()) { length = this.iterator.remaining(); } return this.iterator.peekSequence(length); } void skip() { this.iterator.skip(); } public SourcePosition position() { return this.iterator.position(); } } }