Return unknown tokens instead of throwing

3 files changed, 10 insertions, 2 deletions

diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/RegexTokeniser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/RegexTokeniser.java
index 06ecbd3..8ff4392 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/RegexTokeniser.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/RegexTokeniser.java
@@ -23,8 +23,9 @@ public class RegexTokeniser<T> {
 
     private final List<TokenRule<T>> rules;
 
-    public RegexTokeniser(List<TokenRule<T>> rules) {
-        this.rules = rules;
+    public RegexTokeniser(T unknown, List<TokenRule<T>> rules) {
+        this.rules = new ArrayList<>(rules);
+        this.rules.add(rule(unknown, "."));
     }
 
     public List<Token<T>> tokenise(String value) {
@@ -42,6 +43,7 @@ public class RegexTokeniser<T> {
                 }
             }
             if (!matched) {
+                // Should be impossible
                 throw new RuntimeException("Remaining: " + remaining);
             }
         }
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java
index 1b7c881..1eec870 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java
@@ -9,6 +9,7 @@ public class StyleMappingTokeniser {
         String stringPrefix = "'((?:\\.|[^'])*)";
 
         RegexTokeniser<TokenType> tokeniser = new RegexTokeniser<>(
+            TokenType.UNKNOWN,
             list(
                 RegexTokeniser.rule(TokenType.IDENTIFIER, "([a-zA-Z][a-zA-Z0-9\\-]*)"),
                 RegexTokeniser.rule(TokenType.CLASS_NAME, "(\\.(?:[a-zA-Z0-9\\-]|\\\\.)+)"),
diff --git a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java
index 1106003..a7edf9f 100644
--- a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java
+++ b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java
@@ -16,6 +16,11 @@ import static org.zwobble.mammoth.internal.util.Lists.list;
 
 public class StyleMappingTokeniserTests {
     @Test
+    public void unknownTokensAreTokenised() {
+        assertTokens("~", isToken(TokenType.UNKNOWN, "~"));
+    }
+
+    @Test
     public void emptyStringIsTokenisedToEndOfFileToken() {
         assertTokens("");
     }