package org.shedlang.compiler.parser; import org.junit.jupiter.api.Test; import org.shedlang.compiler.sources.SourceTexts; import static org.zwobble.precisely.AssertThat.assertThat; import static org.zwobble.precisely.Matchers.equalTo; public class TokeniserTests { @Test public void whitespaceTokensAreIgnored() { var sourceText = SourceTexts.string(""" one two three """); var result = Tokeniser.tokenise(sourceText); assertThat(result.nextValue(TokenType.IDENTIFIER), equalTo("one")); assertThat(result.nextValue(TokenType.IDENTIFIER), equalTo("two")); assertThat(result.nextValue(TokenType.IDENTIFIER), equalTo("three")); assertThat(result.trySkip(TokenType.END), equalTo(true)); } @Test public void singleLineCommentsAreIgnored() { var sourceText = SourceTexts.string(""" // a one // b two """); var result = Tokeniser.tokenise(sourceText); assertThat(result.nextValue(TokenType.IDENTIFIER), equalTo("one")); assertThat(result.nextValue(TokenType.IDENTIFIER), equalTo("two")); assertThat(result.trySkip(TokenType.END), equalTo(true)); } }