Use recursive descent parser instead of parsimonious

7 files changed, 27 insertions, 228 deletions

diff --git a/mammoth/options.py b/mammoth/options.py
index d33b7ba..fcdba74 100644
--- a/mammoth/options.py
+++ b/mammoth/options.py
@@ -1,4 +1,5 @@
-from . import style_reader, lists, results
+from .styles.parser import read_style_mapping
+from . import lists, results
 
 
 def read_options(options):
@@ -24,7 +25,7 @@ def read_options(options):
 
 def _read_style_map(style_text):
     lines = filter(None, map(_get_line, style_text.split("\n")))
-    return results.combine(lists.map(style_reader.read_style, lines)) \
+    return results.combine(lists.map(read_style_mapping, lines)) \
         .map(lambda style_mappings: lists.filter(None, style_mappings))
     
 
diff --git a/mammoth/style_reader/__init__.py b/mammoth/style_reader/__init__.py
deleted file mode 100644
index aca07df..0000000
--- a/mammoth/style_reader/__init__.py
+++ /dev/null
@@ -1,37 +0,0 @@
-from parsimonious.grammar import Grammar
-from parsimonious.exceptions import ParseError
-
-from .html_path_reader import read_html_path, grammar_text as html_path_grammar, read_html_path_node
-from .document_matcher_reader import read_document_matcher, grammar_text as document_matcher_grammar, read_document_matcher_node
-from ..styles import Style
-from .. import results
-
-
-__all__ = ["read_style", "read_html_path", "read_document_matcher"]
-
-
-def read_style(string):
-    try:
-        style_node = _grammar.parse(string)
-    except ParseError:
-        warning = "Did not understand this style mapping, so ignored it: " + string
-        return results.Result(None, [results.warning(warning)])
-    
-    return results.success(_read_style_node(style_node))
-
-
-def _read_style_node(style_node):
-    document_matcher_node = style_node.children[0]
-    html_path_node = style_node.children[4]
-    
-    document_matcher = read_document_matcher_node(document_matcher_node)
-    html_path = read_html_path_node(html_path_node)
-    
-    return Style(document_matcher, html_path)
-    
-
-
-_grammar = Grammar("""
-style = document_matcher whitespace "=>" whitespace html_path
-""" + html_path_grammar + document_matcher_grammar
-)
diff --git a/mammoth/style_reader/document_matcher_reader.py b/mammoth/style_reader/document_matcher_reader.py
deleted file mode 100644
index b22b6a0..0000000
--- a/mammoth/style_reader/document_matcher_reader.py
+++ /dev/null
@@ -1,95 +0,0 @@
-from parsimonious.grammar import Grammar
-
-from .. import document_matchers, documents
-
-
-def read_document_matcher(string):
-    root_node = _grammar.parse(string)
-    return read_document_matcher_node(root_node)
-
-
-def read_document_matcher_node(root_node):
-    element_node = root_node.children[0]
-    if element_node.expr_name == "paragraph":
-        return _read_paragraph_node(element_node)
-    elif element_node.expr_name == "run":
-        return _read_run_node(element_node)
-    elif element_node.expr_name == "bold":
-        return document_matchers.bold
-    elif element_node.expr_name == "italic":
-        return document_matchers.italic
-    elif element_node.expr_name == "underline":
-        return document_matchers.underline
-    elif element_node.expr_name == "strikethrough":
-        return document_matchers.strikethrough
-    elif element_node.expr_name == "comment_reference":
-        return document_matchers.comment_reference
-    
-
-def _read_paragraph_node(paragraph_node):
-    style_id = _read_style_id_node(paragraph_node.children[1])
-    style_name = _read_style_name_node(paragraph_node.children[2])
-    numbering = _read_list_node(paragraph_node.children[3])
-    return document_matchers.paragraph(style_id=style_id, style_name=style_name, numbering=numbering)
-    
-
-def _read_run_node(run_node):
-    style_id = _read_style_id_node(run_node.children[1])
-    style_name = _read_style_name_node(run_node.children[2])
-    return document_matchers.run(style_id=style_id, style_name=style_name)
-
-
-def _read_style_id_node(style_id_node):
-    if style_id_node.children:
-        return style_id_node.children[0].children[1].text
-    else:
-        return None
-
-
-def _read_style_name_node(style_name_node):
-    if style_name_node.children:
-        return style_name_node.children[0].children[1].text
-    else:
-        return None
-
-
-def _read_list_node(list_node):
-    if list_node.children:
-        return documents.numbering_level(
-            int(list_node.children[0].children[3].text) - 1,
-            is_ordered=list_node.children[0].children[1].text == "ordered-list",
-        )
-    else:
-        return None
-
-grammar_text = r"""
-document_matcher = paragraph / run / underline / strikethrough / bold / italic / comment_reference
-
-underline = "u"
-
-strikethrough = "strike"
-
-bold = "b"
-
-italic = "i"
-
-comment_reference = "comment-reference"
-
-paragraph = "p" style_id? style_name_specifier? list?
-
-run = "r" style_id? style_name_specifier?
-
-style_id = "." style_identifier
-
-style_identifier = ~"[A-Z0-9]*"i
-
-style_name_specifier = "[style-name='" style_name "']"
-
-style_name = ~"[^']+"
-
-list = ":" list_type "(" ~"[0-9]+" ")"
-
-list_type = "ordered-list" / "unordered-list"
-"""
-_grammar = Grammar(grammar_text)
-
diff --git a/mammoth/style_reader/html_path_reader.py b/mammoth/style_reader/html_path_reader.py
deleted file mode 100644
index f70c8e3..0000000
--- a/mammoth/style_reader/html_path_reader.py
+++ /dev/null
@@ -1,93 +0,0 @@
-from parsimonious.grammar import Grammar
-
-from .. import html_paths
-
-
-def read_html_path(string):
-    path_node = _grammar.parse(string)
-    return read_html_path_node(path_node)
-
-
-def read_html_path_node(path_node):
-    if path_node.children[0].expr_name == "ignore":
-        return html_paths.ignore
-    elif path_node.children[0].children:
-        return _read_html_path_elements_node(path_node.children[0].children[0])
-    else:
-        return html_paths.empty
-
-
-def _read_html_path_elements_node(path_node):
-    elements = [
-        _read_element_node(child)
-        for child in _repeated_children_with_separator(path_node, has_whitespace=True)
-    ]
-    return html_paths.path(elements)
-
-
-def _read_element_node(node):
-    tag_names = _read_tag_names_node(node.children[0])
-    class_names = _read_class_names_node(node.children[1])
-    fresh = _read_fresh_node(node.children[2])
-    return html_paths.element(tag_names, class_names=class_names, fresh=fresh)
-
-
-def _read_tag_names_node(node):
-    return [
-        child.text
-        for child in _repeated_children_with_separator(node, has_whitespace=False)
-    ]
-
-
-def _read_class_names_node(class_names_node):
-    return [
-        _read_class_name_node(node)
-        for node in class_names_node.children
-    ]
-
-
-def _read_class_name_node(node):
-    return node.children[1].text
-
-
-def _read_fresh_node(node):
-    return len(node.children) > 0
-
-
-def _repeated_children_with_separator(node, has_whitespace):
-    yield node.children[0]
-    
-    if has_whitespace:
-        sequence_node_index = 3
-    else:
-        sequence_node_index = 1
-    
-    sequence_node = node.children[1]
-    for child in sequence_node.children:
-        yield child.children[sequence_node_index]
-
-
-grammar_text = r"""
-
-html_path = ignore / html_path_elements?
-
-ignore = "!"
-
-html_path_elements = element (whitespace* ">" whitespace* element)*
-
-element = tag_names class_name* fresh?
-
-tag_names = identifier ("|" identifier)*
-
-class_name = "." identifier
-
-fresh = ":fresh"
-
-identifier = ~"[A-Z0-9]+"i
-
-whitespace = ~"\s"*
-
-"""
-
-_grammar = Grammar(grammar_text)
-
diff --git a/mammoth/styles/parser/__init__.py b/mammoth/styles/parser/__init__.py
index e69de29..2266467 100644
--- a/mammoth/styles/parser/__init__.py
+++ b/mammoth/styles/parser/__init__.py
@@ -0,0 +1,14 @@
+from .errors import LineParseError
+from .style_mapping_parser import parse_style_mapping
+from .tokeniser import tokenise
+from .token_iterator import TokenIterator
+from ... import results
+
+
+def read_style_mapping(string):
+    try:
+        tokens = tokenise(string)
+        return results.success(parse_style_mapping(TokenIterator(tokens)))
+    except LineParseError:
+        warning = "Did not understand this style mapping, so ignored it: " + string
+        return results.Result(None, [results.warning(warning)])
diff --git a/mammoth/styles/parser/errors.py b/mammoth/styles/parser/errors.py
new file mode 100644
index 0000000..ba45e87
--- /dev/null
+++ b/mammoth/styles/parser/errors.py
@@ -0,0 +1,2 @@
+class LineParseError(Exception):
+    pass
diff --git a/mammoth/styles/parser/token_iterator.py b/mammoth/styles/parser/token_iterator.py
index 114ce28..027775b 100644
--- a/mammoth/styles/parser/token_iterator.py
+++ b/mammoth/styles/parser/token_iterator.py
@@ -1,4 +1,8 @@
 # TODO: check indices
+# TODO: proper tests for unexpected tokens
+
+from .errors import LineParseError
+
 
 class TokenIterator(object):
     def __init__(self, tokens):
@@ -46,4 +50,7 @@ class TokenIterator(object):
                 self._index += 1
         
         return True
-                
+    
+    def _unexpected_token_type(self, token_type, token):
+        raise LineParseError()
+