diff options
| author | Michael Williamson <mike@zwobble.org> | 2023-09-23 16:55:24 +0100 |
|---|---|---|
| committer | Michael Williamson <mike@zwobble.org> | 2023-09-23 16:55:24 +0100 |
| commit | 07ff6a2866980660b5a68fe8d54088b015a1d6de (patch) | |
| tree | ca5679f8a437443dd681bd1f3f1f59e9bdc84655 /src/main/java | |
| parent | 39b0d3dda4dc5a8c9c27f73a3c7f0be1dfe5a34d (diff) | |
Support single line comments
Diffstat (limited to 'src/main/java')
| -rw-r--r-- | src/main/java/org/shedlang/compiler/parser/TokenType.java | 1 | ||||
| -rw-r--r-- | src/main/java/org/shedlang/compiler/parser/Tokeniser.java | 6 |
2 files changed, 6 insertions, 1 deletions
diff --git a/src/main/java/org/shedlang/compiler/parser/TokenType.java b/src/main/java/org/shedlang/compiler/parser/TokenType.java index 4ce657e7..d9cd95aa 100644 --- a/src/main/java/org/shedlang/compiler/parser/TokenType.java +++ b/src/main/java/org/shedlang/compiler/parser/TokenType.java @@ -6,6 +6,7 @@ public enum TokenType { INT, STRING, UNKNOWN, + COMMENT_SINGLE_LINE, WHITESPACE, KEYWORD_ASSIGN, diff --git a/src/main/java/org/shedlang/compiler/parser/Tokeniser.java b/src/main/java/org/shedlang/compiler/parser/Tokeniser.java index f983e0dc..3793417a 100644 --- a/src/main/java/org/shedlang/compiler/parser/Tokeniser.java +++ b/src/main/java/org/shedlang/compiler/parser/Tokeniser.java @@ -32,6 +32,7 @@ public class Tokeniser { RegexTokeniser.rule(TokenType.INT, "-?[0-9]+"), RegexTokeniser.rule(TokenType.STRING, "\"(?:[^\\\\\"\n\r]|\\\\.)*\""), RegexTokeniser.rule(TokenType.WHITESPACE, "[\r\n\t ]+"), + RegexTokeniser.rule(TokenType.COMMENT_SINGLE_LINE, "//[^\r\n]*[\r\n]"), RegexTokeniser.rule(TokenType.SYMBOL_BANG_EQUALS, "!="), RegexTokeniser.rule(TokenType.SYMBOL_COLON_EQUALS, ":="), @@ -56,7 +57,10 @@ public class Tokeniser { public static TokenIterator<TokenType> tokenise(Source source) { var tokens = tokeniser.tokenise(source).stream() - .filter(token -> token.tokenType() != TokenType.WHITESPACE) + .filter(token -> + token.tokenType() != TokenType.WHITESPACE && + token.tokenType() != TokenType.COMMENT_SINGLE_LINE + ) .map(token -> token.tokenType() == TokenType.IDENTIFIER ? new Token<>(token.source(), KEYWORDS.getOrDefault(token.value(), token.tokenType()), token.value()) |