From 07ff6a2866980660b5a68fe8d54088b015a1d6de Mon Sep 17 00:00:00 2001 From: Michael Williamson Date: Sat, 23 Sep 2023 16:55:24 +0100 Subject: Support single line comments --- src/main/java/org/shedlang/compiler/parser/TokenType.java | 1 + src/main/java/org/shedlang/compiler/parser/Tokeniser.java | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) (limited to 'src/main/java') diff --git a/src/main/java/org/shedlang/compiler/parser/TokenType.java b/src/main/java/org/shedlang/compiler/parser/TokenType.java index 4ce657e7..d9cd95aa 100644 --- a/src/main/java/org/shedlang/compiler/parser/TokenType.java +++ b/src/main/java/org/shedlang/compiler/parser/TokenType.java @@ -6,6 +6,7 @@ public enum TokenType { INT, STRING, UNKNOWN, + COMMENT_SINGLE_LINE, WHITESPACE, KEYWORD_ASSIGN, diff --git a/src/main/java/org/shedlang/compiler/parser/Tokeniser.java b/src/main/java/org/shedlang/compiler/parser/Tokeniser.java index f983e0dc..3793417a 100644 --- a/src/main/java/org/shedlang/compiler/parser/Tokeniser.java +++ b/src/main/java/org/shedlang/compiler/parser/Tokeniser.java @@ -32,6 +32,7 @@ public class Tokeniser { RegexTokeniser.rule(TokenType.INT, "-?[0-9]+"), RegexTokeniser.rule(TokenType.STRING, "\"(?:[^\\\\\"\n\r]|\\\\.)*\""), RegexTokeniser.rule(TokenType.WHITESPACE, "[\r\n\t ]+"), + RegexTokeniser.rule(TokenType.COMMENT_SINGLE_LINE, "//[^\r\n]*[\r\n]"), RegexTokeniser.rule(TokenType.SYMBOL_BANG_EQUALS, "!="), RegexTokeniser.rule(TokenType.SYMBOL_COLON_EQUALS, ":="), @@ -56,7 +57,10 @@ public class Tokeniser { public static TokenIterator tokenise(Source source) { var tokens = tokeniser.tokenise(source).stream() - .filter(token -> token.tokenType() != TokenType.WHITESPACE) + .filter(token -> + token.tokenType() != TokenType.WHITESPACE && + token.tokenType() != TokenType.COMMENT_SINGLE_LINE + ) .map(token -> token.tokenType() == TokenType.IDENTIFIER ? new Token<>(token.source(), KEYWORDS.getOrDefault(token.value(), token.tokenType()), token.value()) -- cgit v1.2.3