Use recursive descent parser instead of parsimonious

11 files changed, 35 insertions, 424 deletions

diff --git a/mammoth/options.py b/mammoth/options.py
index d33b7ba..fcdba74 100644
--- a/mammoth/options.py
+++ b/mammoth/options.py
@@ -1,4 +1,5 @@
-from . import style_reader, lists, results
+from .styles.parser import read_style_mapping
+from . import lists, results
 
 
 def read_options(options):
@@ -24,7 +25,7 @@ def read_options(options):
 
 def _read_style_map(style_text):
     lines = filter(None, map(_get_line, style_text.split("\n")))
-    return results.combine(lists.map(style_reader.read_style, lines)) \
+    return results.combine(lists.map(read_style_mapping, lines)) \
         .map(lambda style_mappings: lists.filter(None, style_mappings))
     
 
diff --git a/mammoth/style_reader/__init__.py b/mammoth/style_reader/__init__.py
deleted file mode 100644
index aca07df..0000000
--- a/mammoth/style_reader/__init__.py
+++ /dev/null
@@ -1,37 +0,0 @@
-from parsimonious.grammar import Grammar
-from parsimonious.exceptions import ParseError
-
-from .html_path_reader import read_html_path, grammar_text as html_path_grammar, read_html_path_node
-from .document_matcher_reader import read_document_matcher, grammar_text as document_matcher_grammar, read_document_matcher_node
-from ..styles import Style
-from .. import results
-
-
-__all__ = ["read_style", "read_html_path", "read_document_matcher"]
-
-
-def read_style(string):
-    try:
-        style_node = _grammar.parse(string)
-    except ParseError:
-        warning = "Did not understand this style mapping, so ignored it: " + string
-        return results.Result(None, [results.warning(warning)])
-    
-    return results.success(_read_style_node(style_node))
-
-
-def _read_style_node(style_node):
-    document_matcher_node = style_node.children[0]
-    html_path_node = style_node.children[4]
-    
-    document_matcher = read_document_matcher_node(document_matcher_node)
-    html_path = read_html_path_node(html_path_node)
-    
-    return Style(document_matcher, html_path)
-    
-
-
-_grammar = Grammar("""
-style = document_matcher whitespace "=>" whitespace html_path
-""" + html_path_grammar + document_matcher_grammar
-)
diff --git a/mammoth/style_reader/document_matcher_reader.py b/mammoth/style_reader/document_matcher_reader.py
deleted file mode 100644
index b22b6a0..0000000
--- a/mammoth/style_reader/document_matcher_reader.py
+++ /dev/null
@@ -1,95 +0,0 @@
-from parsimonious.grammar import Grammar
-
-from .. import document_matchers, documents
-
-
-def read_document_matcher(string):
-    root_node = _grammar.parse(string)
-    return read_document_matcher_node(root_node)
-
-
-def read_document_matcher_node(root_node):
-    element_node = root_node.children[0]
-    if element_node.expr_name == "paragraph":
-        return _read_paragraph_node(element_node)
-    elif element_node.expr_name == "run":
-        return _read_run_node(element_node)
-    elif element_node.expr_name == "bold":
-        return document_matchers.bold
-    elif element_node.expr_name == "italic":
-        return document_matchers.italic
-    elif element_node.expr_name == "underline":
-        return document_matchers.underline
-    elif element_node.expr_name == "strikethrough":
-        return document_matchers.strikethrough
-    elif element_node.expr_name == "comment_reference":
-        return document_matchers.comment_reference
-    
-
-def _read_paragraph_node(paragraph_node):
-    style_id = _read_style_id_node(paragraph_node.children[1])
-    style_name = _read_style_name_node(paragraph_node.children[2])
-    numbering = _read_list_node(paragraph_node.children[3])
-    return document_matchers.paragraph(style_id=style_id, style_name=style_name, numbering=numbering)
-    
-
-def _read_run_node(run_node):
-    style_id = _read_style_id_node(run_node.children[1])
-    style_name = _read_style_name_node(run_node.children[2])
-    return document_matchers.run(style_id=style_id, style_name=style_name)
-
-
-def _read_style_id_node(style_id_node):
-    if style_id_node.children:
-        return style_id_node.children[0].children[1].text
-    else:
-        return None
-
-
-def _read_style_name_node(style_name_node):
-    if style_name_node.children:
-        return style_name_node.children[0].children[1].text
-    else:
-        return None
-
-
-def _read_list_node(list_node):
-    if list_node.children:
-        return documents.numbering_level(
-            int(list_node.children[0].children[3].text) - 1,
-            is_ordered=list_node.children[0].children[1].text == "ordered-list",
-        )
-    else:
-        return None
-
-grammar_text = r"""
-document_matcher = paragraph / run / underline / strikethrough / bold / italic / comment_reference
-
-underline = "u"
-
-strikethrough = "strike"
-
-bold = "b"
-
-italic = "i"
-
-comment_reference = "comment-reference"
-
-paragraph = "p" style_id? style_name_specifier? list?
-
-run = "r" style_id? style_name_specifier?
-
-style_id = "." style_identifier
-
-style_identifier = ~"[A-Z0-9]*"i
-
-style_name_specifier = "[style-name='" style_name "']"
-
-style_name = ~"[^']+"
-
-list = ":" list_type "(" ~"[0-9]+" ")"
-
-list_type = "ordered-list" / "unordered-list"
-"""
-_grammar = Grammar(grammar_text)
-
diff --git a/mammoth/style_reader/html_path_reader.py b/mammoth/style_reader/html_path_reader.py
deleted file mode 100644
index f70c8e3..0000000
--- a/mammoth/style_reader/html_path_reader.py
+++ /dev/null
@@ -1,93 +0,0 @@
-from parsimonious.grammar import Grammar
-
-from .. import html_paths
-
-
-def read_html_path(string):
-    path_node = _grammar.parse(string)
-    return read_html_path_node(path_node)
-
-
-def read_html_path_node(path_node):
-    if path_node.children[0].expr_name == "ignore":
-        return html_paths.ignore
-    elif path_node.children[0].children:
-        return _read_html_path_elements_node(path_node.children[0].children[0])
-    else:
-        return html_paths.empty
-
-
-def _read_html_path_elements_node(path_node):
-    elements = [
-        _read_element_node(child)
-        for child in _repeated_children_with_separator(path_node, has_whitespace=True)
-    ]
-    return html_paths.path(elements)
-
-
-def _read_element_node(node):
-    tag_names = _read_tag_names_node(node.children[0])
-    class_names = _read_class_names_node(node.children[1])
-    fresh = _read_fresh_node(node.children[2])
-    return html_paths.element(tag_names, class_names=class_names, fresh=fresh)
-
-
-def _read_tag_names_node(node):
-    return [
-        child.text
-        for child in _repeated_children_with_separator(node, has_whitespace=False)
-    ]
-
-
-def _read_class_names_node(class_names_node):
-    return [
-        _read_class_name_node(node)
-        for node in class_names_node.children
-    ]
-
-
-def _read_class_name_node(node):
-    return node.children[1].text
-
-
-def _read_fresh_node(node):
-    return len(node.children) > 0
-
-
-def _repeated_children_with_separator(node, has_whitespace):
-    yield node.children[0]
-    
-    if has_whitespace:
-        sequence_node_index = 3
-    else:
-        sequence_node_index = 1
-    
-    sequence_node = node.children[1]
-    for child in sequence_node.children:
-        yield child.children[sequence_node_index]
-
-
-grammar_text = r"""
-
-html_path = ignore / html_path_elements?
-
-ignore = "!"
-
-html_path_elements = element (whitespace* ">" whitespace* element)*
-
-element = tag_names class_name* fresh?
-
-tag_names = identifier ("|" identifier)*
-
-class_name = "." identifier
-
-fresh = ":fresh"
-
-identifier = ~"[A-Z0-9]+"i
-
-whitespace = ~"\s"*
-
-"""
-
-_grammar = Grammar(grammar_text)
-
diff --git a/mammoth/styles/parser/__init__.py b/mammoth/styles/parser/__init__.py
index e69de29..2266467 100644
--- a/mammoth/styles/parser/__init__.py
+++ b/mammoth/styles/parser/__init__.py
@@ -0,0 +1,14 @@
+from .errors import LineParseError
+from .style_mapping_parser import parse_style_mapping
+from .tokeniser import tokenise
+from .token_iterator import TokenIterator
+from ... import results
+
+
+def read_style_mapping(string):
+    try:
+        tokens = tokenise(string)
+        return results.success(parse_style_mapping(TokenIterator(tokens)))
+    except LineParseError:
+        warning = "Did not understand this style mapping, so ignored it: " + string
+        return results.Result(None, [results.warning(warning)])
diff --git a/mammoth/styles/parser/errors.py b/mammoth/styles/parser/errors.py
new file mode 100644
index 0000000..ba45e87
--- /dev/null
+++ b/mammoth/styles/parser/errors.py
@@ -0,0 +1,2 @@
+class LineParseError(Exception):
+    pass
diff --git a/mammoth/styles/parser/token_iterator.py b/mammoth/styles/parser/token_iterator.py
index 114ce28..027775b 100644
--- a/mammoth/styles/parser/token_iterator.py
+++ b/mammoth/styles/parser/token_iterator.py
@@ -1,4 +1,8 @@
 # TODO: check indices
+# TODO: proper tests for unexpected tokens
+
+from .errors import LineParseError
+
 
 class TokenIterator(object):
     def __init__(self, tokens):
@@ -46,4 +50,7 @@ class TokenIterator(object):
                 self._index += 1
         
         return True
-                
+    
+    def _unexpected_token_type(self, token_type, token):
+        raise LineParseError()
+    
diff --git a/setup.py b/setup.py
index 46817da..7392025 100644
--- a/setup.py
+++ b/setup.py
@@ -9,7 +9,6 @@ def read(fname):
 
 
 _install_requires = [
-    "parsimonious>=0.5,<0.6",
     "cobble>=0.1.1,<0.2",
 ]
 
@@ -24,7 +23,7 @@ setup(
     author='Michael Williamson',
     author_email='mike@zwobble.org',
     url='http://github.com/mwilliamson/python-mammoth',
-    packages=['mammoth', 'mammoth.docx', 'mammoth.html', 'mammoth.style_reader', 'mammoth.styles', 'mammoth.styles.parser', 'mammoth.writers'],
+    packages=['mammoth', 'mammoth.docx', 'mammoth.html', 'mammoth.styles', 'mammoth.styles.parser', 'mammoth.writers'],
     entry_points={
         "console_scripts": [
             "mammoth=mammoth.cli:main"
diff --git a/tests/conversion_tests.py b/tests/conversion_tests.py
index e91d4ed..0d08b20 100644
--- a/tests/conversion_tests.py
+++ b/tests/conversion_tests.py
@@ -6,9 +6,10 @@ import io
 
 from nose.tools import istest, assert_equal
 
-from mammoth import documents, style_reader, results, html
+from mammoth import documents, results, html
 from mammoth.conversion import convert_document_element_to_html, _comment_author_label
 from mammoth.docx.xmlparser import parse_xml
+from mammoth.styles.parser import read_style_mapping
 
 
 @istest
@@ -502,6 +503,6 @@ def _run_with_text(text):
 
 
 def _style_mapping(text):
-    result = style_reader.read_style(text)
+    result = read_style_mapping(text)
     assert_equal([], result.messages)
     return result.value
diff --git a/tests/options_tests.py b/tests/options_tests.py
index f0b0beb..182678e 100644
--- a/tests/options_tests.py
+++ b/tests/options_tests.py
@@ -1,7 +1,7 @@
 from nose.tools import istest, assert_equal
 
-from mammoth import style_reader
 from mammoth.options import read_options, _default_style_map
+from mammoth.styles.parser import read_style_mapping
 
 
 @istest
@@ -14,7 +14,7 @@ def custom_style_mappings_are_prepended_to_default_style_mappings():
     style_map = read_options({
         "style_map": "p.SectionTitle => h2"
     }).value["style_map"]
-    assert_equal(style_reader.read_style("p.SectionTitle => h2").value, style_map[0])
+    assert_equal(read_style_mapping("p.SectionTitle => h2").value, style_map[0])
     assert_equal(_default_style_map, style_map[1:])
 
 
@@ -24,7 +24,7 @@ def default_style_mappings_are_ignored_if_include_default_style_map_is_false():
         "style_map": "p.SectionTitle => h2",
         "include_default_style_map": False
     }).value["style_map"]
-    assert_equal([style_reader.read_style("p.SectionTitle => h2").value], style_map)
+    assert_equal([read_style_mapping("p.SectionTitle => h2").value], style_map)
 
 
 @istest
@@ -33,4 +33,4 @@ def lines_starting_with_hash_in_custom_style_map_are_ignored():
         "style_map": "#p.SectionTitle => h3\np.SectionTitle => h2",
         "include_default_style_map": False
     }).value["style_map"]
-    assert_equal([style_reader.read_style("p.SectionTitle => h2").value], style_map)
+    assert_equal([read_style_mapping("p.SectionTitle => h2").value], style_map)
diff --git a/tests/style_reader_tests.py b/tests/style_reader_tests.py
deleted file mode 100644
index 61bc911..0000000
--- a/tests/style_reader_tests.py
+++ /dev/null
@@ -1,188 +0,0 @@
-from nose.tools import istest, assert_equal
-
-from mammoth import html_paths, document_matchers, documents, styles
-from mammoth.style_reader import read_html_path, read_document_matcher, read_style
-
-
-@istest
-class ReadHtmlPathTests(object):
-    @istest
-    def can_read_empty_path(self):
-        assert_equal(
-            html_paths.empty,
-            read_html_path("")
-        )
-
-    @istest
-    def can_read_single_element(self):
-        assert_equal(
-            html_paths.path([html_paths.element(["p"])]),
-            read_html_path("p")
-        )
-
-    
-    @istest
-    def can_read_choice_of_two_elements(self):
-        assert_equal(
-            html_paths.path([html_paths.element(["ul", "ol"])]),
-            read_html_path("ul|ol")
-        )
-
-    
-    @istest
-    def can_read_choice_of_three_elements(self):
-        assert_equal(
-            html_paths.path([html_paths.element(["ul", "ol", "p"])]),
-            read_html_path("ul|ol|p")
-        )
-
-    
-    @istest
-    def can_read_nested_elements(self):
-        assert_equal(
-            html_paths.path([html_paths.element(["ul"]), html_paths.element(["li"])]),
-            read_html_path("ul > li")
-        )
-
-    
-    @istest
-    def can_read_class_on_element(self):
-        assert_equal(
-            html_paths.path([html_paths.element(["p"], class_names=["tip"])]),
-            read_html_path("p.tip")
-        )
-
-    
-    @istest
-    def can_read_multiple_classes_on_element(self):
-        assert_equal(
-            html_paths.path([html_paths.element(["p"], class_names=["tip", "help"])]),
-            read_html_path("p.tip.help")
-        )
-
-    
-    @istest
-    def can_read_when_element_must_be_fresh(self):
-        assert_equal(
-            html_paths.path([html_paths.element(["p"], fresh=True)]),
-            read_html_path("p:fresh")
-        )
-
-    
-    @istest
-    def can_read_ignore_element(self):
-        assert_equal(
-            html_paths.ignore,
-            read_html_path("!")
-        )
-
-
-@istest
-class ReadDocumentMatcherTests(object):
-    @istest
-    def reads_plain_paragraph(self):
-        assert_equal(
-            document_matchers.paragraph(),
-            read_document_matcher("p")
-        )
-    
-    
-    @istest
-    def reads_paragraph_with_style_id(self):
-        assert_equal(
-            document_matchers.paragraph(style_id="Heading1"),
-            read_document_matcher("p.Heading1")
-        )
-    
-    
-    @istest
-    def reads_paragraph_with_style_name(self):
-        assert_equal(
-            document_matchers.paragraph(style_name="Heading 1"),
-            read_document_matcher("p[style-name='Heading 1']")
-        )
-    
-    
-    @istest
-    def reads_paragraph_ordered_list(self):
-        assert_equal(
-            document_matchers.paragraph(numbering=documents.numbering_level(1, is_ordered=True)),
-            read_document_matcher("p:ordered-list(2)")
-        )
-    
-    
-    @istest
-    def reads_paragraph_unordered_list(self):
-        assert_equal(
-            document_matchers.paragraph(numbering=documents.numbering_level(1, is_ordered=False)),
-            read_document_matcher("p:unordered-list(2)")
-        )
-    
-    
-    @istest
-    def reads_plain_run(self):
-        assert_equal(
-            document_matchers.run(),
-            read_document_matcher("r")
-        )
-    
-    
-    @istest
-    def reads_run_with_style_id(self):
-        assert_equal(
-            document_matchers.run(style_id="Emphasis"),
-            read_document_matcher("r.Emphasis")
-        )
-    
-    
-    @istest
-    def reads_run_with_style_name(self):
-        assert_equal(
-            document_matchers.run(style_name="Emphasis"),
-            read_document_matcher("r[style-name='Emphasis']")
-        )
-
-    @istest
-    def reads_bold(self):
-        assert_equal(
-            document_matchers.bold,
-            read_document_matcher("b")
-        )
-
-    @istest
-    def reads_italic(self):
-        assert_equal(
-            document_matchers.italic,
-            read_document_matcher("i")
-        )
-
-    @istest
-    def reads_underline(self):
-        assert_equal(
-            document_matchers.underline,
-            read_document_matcher("u")
-        )
-
-    @istest
-    def reads_strikethrough(self):
-        assert_equal(
-            document_matchers.strikethrough,
-            read_document_matcher("strike")
-        )
-
-    @istest
-    def reads_comment_reference(self):
-        assert_equal(
-            document_matchers.comment_reference,
-            read_document_matcher("comment-reference")
-        )
-
-
-@istest
-def document_matcher_is_mapped_to_html_path_using_fat_arrow():
-    style_result = read_style("p => h1")
-    assert_equal(
-        styles.style(document_matchers.paragraph(), html_paths.path([html_paths.element(["h1"])])),
-        style_result.value
-    )
-    assert_equal([], style_result.messages)