From b4099505ca1ceab2f0633381265f27ddfd4fd1be Mon Sep 17 00:00:00 2001
From: Michael Williamson <mike@zwobble.org>
Date: Wed, 16 Nov 2016 19:21:49 +0000
Subject: Return unknown tokens instead of throwing

---
 .../org/zwobble/mammoth/internal/styles/parsing/RegexTokeniser.java | 6 ++++--
 .../mammoth/internal/styles/parsing/StyleMappingTokeniser.java      | 1 +
 .../mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java    | 5 +++++
 3 files changed, 10 insertions(+), 2 deletions(-)

(limited to 'src')

diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/RegexTokeniser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/RegexTokeniser.java
index 06ecbd3..8ff4392 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/RegexTokeniser.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/RegexTokeniser.java
@@ -23,8 +23,9 @@ public class RegexTokeniser<T> {
 
     private final List<TokenRule<T>> rules;
 
-    public RegexTokeniser(List<TokenRule<T>> rules) {
-        this.rules = rules;
+    public RegexTokeniser(T unknown, List<TokenRule<T>> rules) {
+        this.rules = new ArrayList<>(rules);
+        this.rules.add(rule(unknown, "."));
     }
 
     public List<Token<T>> tokenise(String value) {
@@ -42,6 +43,7 @@ public class RegexTokeniser<T> {
                 }
             }
             if (!matched) {
+                // Should be impossible
                 throw new RuntimeException("Remaining: " + remaining);
             }
         }
diff --git a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java
index 1b7c881..1eec870 100644
--- a/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java
+++ b/src/main/java/org/zwobble/mammoth/internal/styles/parsing/StyleMappingTokeniser.java
@@ -9,6 +9,7 @@ public class StyleMappingTokeniser {
         String stringPrefix = "'((?:\\.|[^'])*)";
 
         RegexTokeniser<TokenType> tokeniser = new RegexTokeniser<>(
+            TokenType.UNKNOWN,
             list(
                 RegexTokeniser.rule(TokenType.IDENTIFIER, "([a-zA-Z][a-zA-Z0-9\\-]*)"),
                 RegexTokeniser.rule(TokenType.CLASS_NAME, "(\\.(?:[a-zA-Z0-9\\-]|\\\\.)+)"),
diff --git a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java
index 1106003..a7edf9f 100644
--- a/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java
+++ b/src/test/java/org/zwobble/mammoth/tests/styles/parsing/StyleMappingTokeniserTests.java
@@ -15,6 +15,11 @@ import static org.zwobble.mammoth.internal.util.Lists.eagerConcat;
 import static org.zwobble.mammoth.internal.util.Lists.list;
 
 public class StyleMappingTokeniserTests {
+    @Test
+    public void unknownTokensAreTokenised() {
+        assertTokens("~", isToken(TokenType.UNKNOWN, "~"));
+    }
+
     @Test
     public void emptyStringIsTokenisedToEndOfFileToken() {
         assertTokens("");
-- 
cgit v1.2.3