diff options
Diffstat (limited to 'src/main/java')
| -rw-r--r-- | src/main/java/org/shedlang/compiler/parser/TokenType.java | 1 | ||||
| -rw-r--r-- | src/main/java/org/shedlang/compiler/parser/Tokeniser.java | 6 |
2 files changed, 6 insertions, 1 deletions
diff --git a/src/main/java/org/shedlang/compiler/parser/TokenType.java b/src/main/java/org/shedlang/compiler/parser/TokenType.java index 4ce657e7..d9cd95aa 100644 --- a/src/main/java/org/shedlang/compiler/parser/TokenType.java +++ b/src/main/java/org/shedlang/compiler/parser/TokenType.java @@ -6,6 +6,7 @@ public enum TokenType { INT, STRING, UNKNOWN, + COMMENT_SINGLE_LINE, WHITESPACE, KEYWORD_ASSIGN, diff --git a/src/main/java/org/shedlang/compiler/parser/Tokeniser.java b/src/main/java/org/shedlang/compiler/parser/Tokeniser.java index f983e0dc..3793417a 100644 --- a/src/main/java/org/shedlang/compiler/parser/Tokeniser.java +++ b/src/main/java/org/shedlang/compiler/parser/Tokeniser.java @@ -32,6 +32,7 @@ public class Tokeniser { RegexTokeniser.rule(TokenType.INT, "-?[0-9]+"), RegexTokeniser.rule(TokenType.STRING, "\"(?:[^\\\\\"\n\r]|\\\\.)*\""), RegexTokeniser.rule(TokenType.WHITESPACE, "[\r\n\t ]+"), + RegexTokeniser.rule(TokenType.COMMENT_SINGLE_LINE, "//[^\r\n]*[\r\n]"), RegexTokeniser.rule(TokenType.SYMBOL_BANG_EQUALS, "!="), RegexTokeniser.rule(TokenType.SYMBOL_COLON_EQUALS, ":="), @@ -56,7 +57,10 @@ public class Tokeniser { public static TokenIterator<TokenType> tokenise(Source source) { var tokens = tokeniser.tokenise(source).stream() - .filter(token -> token.tokenType() != TokenType.WHITESPACE) + .filter(token -> + token.tokenType() != TokenType.WHITESPACE && + token.tokenType() != TokenType.COMMENT_SINGLE_LINE + ) .map(token -> token.tokenType() == TokenType.IDENTIFIER ? new Token<>(token.source(), KEYWORDS.getOrDefault(token.value(), token.tokenType()), token.value()) |