src/test/java/org/shedlang/compiler/parser/TokeniserTests.java


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

package org.shedlang.compiler.parser;

import org.junit.jupiter.api.Test;
import org.shedlang.compiler.sources.SourceTexts;

import static org.zwobble.precisely.AssertThat.assertThat;
import static org.zwobble.precisely.Matchers.equalTo;

public class TokeniserTests {
    @Test
    public void whitespaceTokensAreIgnored() {
        var sourceText = SourceTexts.string("""
            one two three
            """);

        var result = Tokeniser.tokenise(sourceText);

        assertThat(result.nextValue(TokenType.IDENTIFIER), equalTo("one"));
        assertThat(result.nextValue(TokenType.IDENTIFIER), equalTo("two"));
        assertThat(result.nextValue(TokenType.IDENTIFIER), equalTo("three"));
        assertThat(result.trySkip(TokenType.END), equalTo(true));
    }

    @Test
    public void singleLineCommentsAreIgnored() {
        var sourceText = SourceTexts.string("""
            // a
            one
            // b
            two
            """);

        var result = Tokeniser.tokenise(sourceText);

        assertThat(result.nextValue(TokenType.IDENTIFIER), equalTo("one"));
        assertThat(result.nextValue(TokenType.IDENTIFIER), equalTo("two"));
        assertThat(result.trySkip(TokenType.END), equalTo(true));
    }
}