src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

package org.zwobble.mammoth.internal.styles.parsing;

import java.util.List;

import static org.zwobble.mammoth.internal.util.Lists.list;

public class StyleMappingTokeniser {
    public static TokenIterator<TokenType> tokenise(String line) {
        return new TokenIterator<>(
            tokeniseToList(line),
            new Token<>(line.length(), TokenType.EOF, "")
        );
    }

    public static List<Token<TokenType>> tokeniseToList(String line) {
        String stringPrefix = "'(?:(?:\\\\(?:.|$)|[^'\\\\])*)";
        String identifierCharacter = "(?:[a-zA-Z\\-_]|\\\\.)";

        RegexTokeniser<TokenType> tokeniser = new RegexTokeniser<>(
            TokenType.UNKNOWN,
            list(
                RegexTokeniser.rule(TokenType.IDENTIFIER, identifierCharacter + "(?:" + identifierCharacter + "|[0-9])*"),
                RegexTokeniser.rule(TokenType.SYMBOL, ":|>|=>|\\^=|=|\\(|\\)|\\[|\\]|\\||!|\\."),
                RegexTokeniser.rule(TokenType.WHITESPACE, "\\s+"),
                RegexTokeniser.rule(TokenType.STRING, stringPrefix + "'"),
                RegexTokeniser.rule(TokenType.UNTERMINATED_STRING, stringPrefix),
                RegexTokeniser.rule(TokenType.INTEGER, "[0-9]+")
            )
        );
        return tokeniser.tokenise(line);
    }
}