package org.shedlang.compiler.tokeniser; import org.junit.jupiter.api.Test; import org.zwobble.sourcetext.SourceText; import java.util.List; import static org.junit.jupiter.api.Assertions.fail; import static org.zwobble.precisely.AssertThat.assertThat; import static org.zwobble.precisely.Matchers.equalTo; public class TokenIteratorTests { enum TokenType { IDENTIFIER, SYMBOL, END } @Test public void skipSucceedsWhenNextTokenHasMatchingTokenType() { var sourceText = SourceText.fromString("", "!x!"); var tokens = new TokenIterator<>( List.of( token(sourceText, 0, 1, TokenType.SYMBOL), token(sourceText, 1, 2, TokenType.IDENTIFIER), token(sourceText, 2, 3, TokenType.SYMBOL) ), token(sourceText, 3, 3, TokenType.END) ); tokens.skip(TokenType.SYMBOL); tokens.skip(TokenType.IDENTIFIER); tokens.skip(TokenType.SYMBOL); tokens.skip(TokenType.END); tokens.skip(TokenType.END); } @Test public void skipFailsWhenNextTokenDoesNotHaveMatchingTokenType() { var sourceText = SourceText.fromString("", "!x!"); var tokens = new TokenIterator<>( List.of( token(sourceText, 0, 1, TokenType.SYMBOL), token(sourceText, 1, 2, TokenType.IDENTIFIER), token(sourceText, 2, 3, TokenType.SYMBOL) ), token(sourceText, 3, 3, TokenType.END) ); tokens.skip(TokenType.SYMBOL); try { tokens.skip(TokenType.SYMBOL); fail("expected exception"); } catch (UnexpectedTokenException exception) { assertThat(exception.getMessage(), equalTo("Expected: SYMBOL\nBut got: IDENTIFIER \"x\"")); } } @Test public void trySkipAdvancesIteratorAndReturnsTrueWhenNextTokenHasMatchingTokenType() { var sourceText = SourceText.fromString("", "!x!"); var tokens = new TokenIterator<>( List.of( token(sourceText, 0, 1, TokenType.SYMBOL), token(sourceText, 1, 2, TokenType.IDENTIFIER), token(sourceText, 2, 3, TokenType.SYMBOL) ), token(sourceText, 3, 3, TokenType.END) ); assertThat(tokens.trySkip(TokenType.SYMBOL), equalTo(true)); assertThat(tokens.trySkip(TokenType.IDENTIFIER), equalTo(true)); assertThat(tokens.trySkip(TokenType.SYMBOL), equalTo(true)); assertThat(tokens.trySkip(TokenType.END), equalTo(true)); assertThat(tokens.trySkip(TokenType.END), equalTo(true)); } @Test public void trySkipDoesNotAdvanceIteratorAndReturnsFalseWhenNextTokenDoesNotHaveMatchingTokenType() { var sourceText = SourceText.fromString("", "!x!"); var tokens = new TokenIterator<>( List.of( token(sourceText, 0, 1, TokenType.SYMBOL), token(sourceText, 1, 2, TokenType.IDENTIFIER), token(sourceText, 2, 3, TokenType.SYMBOL) ), token(sourceText, 3, 3, TokenType.END) ); tokens.skip(TokenType.SYMBOL); assertThat(tokens.trySkip(TokenType.SYMBOL), equalTo(false)); assertThat(tokens.trySkip(TokenType.IDENTIFIER), equalTo(true)); } @Test public void nextValueReturnsValueWhenCurrentTokenHasMatchingTokenType() { var sourceText = SourceText.fromString("", "!x!"); var tokens = new TokenIterator<>( List.of( token(sourceText, 0, 1, TokenType.SYMBOL), token(sourceText, 1, 2, TokenType.IDENTIFIER), token(sourceText, 2, 3, TokenType.SYMBOL) ), token(sourceText, 3, 3, TokenType.END) ); assertThat(tokens.nextValue(TokenType.SYMBOL), equalTo("!")); assertThat(tokens.nextValue(TokenType.IDENTIFIER), equalTo("x")); } @Test public void nextValueFailsWhenCurrentTokenDoesNotHaveMatchingTokenType() { var sourceText = SourceText.fromString("", "!x!"); var tokens = new TokenIterator<>( List.of( token(sourceText, 0, 1, TokenType.SYMBOL), token(sourceText, 1, 2, TokenType.IDENTIFIER), token(sourceText, 2, 3, TokenType.SYMBOL) ), token(sourceText, 3, 3, TokenType.END) ); tokens.skip(TokenType.SYMBOL); try { tokens.nextValue(TokenType.SYMBOL); fail("expected exception"); } catch (UnexpectedTokenException exception) { assertThat(exception.getMessage(), equalTo("Expected: SYMBOL\nBut got: IDENTIFIER \"x\"")); } } @Test public void isNextWithOneArgReturnsTrueIfNextTokenIsOfMatchingType() { var sourceText = SourceText.fromString("", "!x!"); var tokens = new TokenIterator<>( List.of( token(sourceText, 0, 1, TokenType.SYMBOL), token(sourceText, 1, 2, TokenType.IDENTIFIER), token(sourceText, 2, 3, TokenType.SYMBOL) ), token(sourceText, 3, 3, TokenType.END) ); tokens.skip(TokenType.SYMBOL); assertThat(tokens.isNext(TokenType.IDENTIFIER), equalTo(true)); } @Test public void isNextWithOneArgReturnsFalseIfNextTokenIsNotOfMatchingType() { var sourceText = SourceText.fromString("", "!x!"); var tokens = new TokenIterator<>( List.of( token(sourceText, 0, 1, TokenType.SYMBOL), token(sourceText, 1, 2, TokenType.IDENTIFIER), token(sourceText, 2, 3, TokenType.SYMBOL) ), token(sourceText, 3, 3, TokenType.END) ); tokens.skip(TokenType.SYMBOL); assertThat(tokens.isNext(TokenType.SYMBOL), equalTo(false)); } @Test public void isNextWithTwoArgReturnsTrueIfBothNextTokensMatchRespectiveTypes() { var sourceText = SourceText.fromString("", "!x!"); var tokens = new TokenIterator<>( List.of( token(sourceText, 0, 1, TokenType.SYMBOL), token(sourceText, 1, 2, TokenType.IDENTIFIER), token(sourceText, 2, 3, TokenType.SYMBOL) ), token(sourceText, 3, 3, TokenType.END) ); tokens.skip(TokenType.SYMBOL); assertThat(tokens.isNext(TokenType.IDENTIFIER, TokenType.SYMBOL), equalTo(true)); } @Test public void isNextWithTwoArgReturnsFalseIfFirstTokenDoesNotMatchRespectiveType() { var sourceText = SourceText.fromString("", "!x!"); var tokens = new TokenIterator<>( List.of( token(sourceText, 0, 1, TokenType.SYMBOL), token(sourceText, 1, 2, TokenType.IDENTIFIER), token(sourceText, 2, 3, TokenType.SYMBOL) ), token(sourceText, 3, 3, TokenType.END) ); tokens.skip(TokenType.SYMBOL); assertThat(tokens.isNext(TokenType.SYMBOL, TokenType.SYMBOL), equalTo(false)); } @Test public void isNextWithTwoArgReturnsFalseIfSecondTokenDoesNotMatchRespectiveType() { var sourceText = SourceText.fromString("", "!x!"); var tokens = new TokenIterator<>( List.of( token(sourceText, 0, 1, TokenType.SYMBOL), token(sourceText, 1, 2, TokenType.IDENTIFIER), token(sourceText, 2, 3, TokenType.SYMBOL) ), token(sourceText, 3, 3, TokenType.END) ); tokens.skip(TokenType.SYMBOL); assertThat(tokens.isNext(TokenType.IDENTIFIER, TokenType.IDENTIFIER), equalTo(false)); } @Test public void peekReturnsNextTokenWithoutAdvancing() { var sourceText = SourceText.fromString("", "!x!"); var tokens = new TokenIterator<>( List.of( token(sourceText, 0, 1, TokenType.SYMBOL), token(sourceText, 1, 2, TokenType.IDENTIFIER), token(sourceText, 2, 3, TokenType.SYMBOL) ), token(sourceText, 3, 3, TokenType.END) ); assertThat(tokens.peek(), equalTo(token(sourceText, 0, 1, TokenType.SYMBOL))); assertThat(tokens.peek(), equalTo(token(sourceText, 0, 1, TokenType.SYMBOL))); tokens.skip(TokenType.SYMBOL); assertThat(tokens.peek(), equalTo(token(sourceText, 1, 2, TokenType.IDENTIFIER))); } @Test public void canPeekMultipleTokensAhead() { var sourceText = SourceText.fromString("", "!x!"); var tokens = new TokenIterator<>( List.of( token(sourceText, 0, 1, TokenType.SYMBOL), token(sourceText, 1, 2, TokenType.IDENTIFIER), token(sourceText, 2, 3, TokenType.SYMBOL) ), token(sourceText, 3, 3, TokenType.END) ); assertThat(tokens.peek(0), equalTo(token(sourceText, 0, 1, TokenType.SYMBOL))); assertThat(tokens.peek(1), equalTo(token(sourceText, 1, 2, TokenType.IDENTIFIER))); assertThat(tokens.peek(2), equalTo(token(sourceText, 2, 3, TokenType.SYMBOL))); tokens.skip(TokenType.SYMBOL); assertThat(tokens.peek(0), equalTo(token(sourceText, 1, 2, TokenType.IDENTIFIER))); assertThat(tokens.peek(1), equalTo(token(sourceText, 2, 3, TokenType.SYMBOL))); assertThat(tokens.peek(2), equalTo(token(sourceText, 3, 3, TokenType.END))); } private Token token( SourceText sourceText, int startCharacterIndex, int endCharacterIndex, TokenType tokenType ) { var start = sourceText.characterPosition(startCharacterIndex); var end = sourceText.characterPosition(endCharacterIndex); return new Token<>(start.to(end), tokenType); } }