From 2c57c1014ae8e44e62021ea2cc36c54352389632 Mon Sep 17 00:00:00 2001 From: Michael Williamson Date: Mon, 14 Mar 2016 20:46:13 +0000 Subject: Implement collapsing using HTML AST --- mammoth/conversion.py | 174 +++++++++++++++++++++------------------------- mammoth/html/__init__.py | 42 ++++++++--- mammoth/html_paths.py | 33 ++++++++- mammoth/images.py | 7 +- mammoth/underline.py | 7 +- tests/conversion_tests.py | 6 +- tests/mammoth_tests.py | 6 +- 7 files changed, 159 insertions(+), 116 deletions(-) diff --git a/mammoth/conversion.py b/mammoth/conversion.py index 94ffeae..fb7a9c3 100644 --- a/mammoth/conversion.py +++ b/mammoth/conversion.py @@ -5,8 +5,7 @@ from __future__ import unicode_literals import base64 import random -from . import documents, results, html_paths, images, writers -from .html_generation import HtmlGenerator, satisfy_html_path, append_html_path +from . import documents, results, html_paths, images, writers, html, lists from .docx.files import InvalidFileReferenceError @@ -27,10 +26,6 @@ def convert_document_element_to_html(element, if convert_image is None: convert_image = images.inline(_generate_image_attributes) - def create_writer(): - return writers.writer(output_format) - - html_generator = HtmlGenerator(create_writer) messages = [] converter = _DocumentConverter( messages=messages, @@ -39,11 +34,12 @@ def convert_document_element_to_html(element, convert_underline=convert_underline, id_prefix=id_prefix, ignore_empty_paragraphs=ignore_empty_paragraphs, - html_generator=html_generator, note_references=[]) - converter.visit(element) - html_generator.end_all() - return results.Result(html_generator.as_string(), messages) + nodes = converter.visit(element) + + writer = writers.writer(output_format) + html.write(writer, html.collapse(html.strip_empty(nodes))) + return results.Result(writer.as_string(), messages) def _generate_image_attributes(image): @@ -56,7 +52,7 @@ def _generate_image_attributes(image): class _DocumentConverter(documents.ElementVisitor): - def __init__(self, messages, style_map, convert_image, convert_underline, id_prefix, ignore_empty_paragraphs, html_generator, note_references): + def __init__(self, messages, style_map, convert_image, convert_underline, id_prefix, ignore_empty_paragraphs, note_references): self._messages = messages self._style_map = style_map self._id_prefix = id_prefix @@ -64,84 +60,70 @@ class _DocumentConverter(documents.ElementVisitor): self._note_references = note_references self._convert_underline = convert_underline or self._default_convert_underline self._convert_image = convert_image - self._html_generator = html_generator - - def _with_html_generator(self, html_generator): - return _DocumentConverter( - messages=self._messages, - style_map=self._style_map, - convert_image=self._convert_image, - convert_underline=self._convert_underline, - id_prefix=self._id_prefix, - ignore_empty_paragraphs=self._ignore_empty_paragraphs, - html_generator=html_generator, - note_references=self._note_references - ) - + def visit_image(self, image): try: - self._convert_image(image, self._html_generator) + return self._convert_image(image) except InvalidFileReferenceError as error: self._messages.append(results.warning(str(error))) + return [] def visit_document(self, document): - self._visit_all(document.children) - self._html_generator.end_all() - self._html_generator.start("ol") + nodes = self._visit_all(document.children) notes = [ document.notes.resolve(reference) for reference in self._note_references ] - self._visit_all(notes) - self._html_generator.end() + notes_list = html.element("ol", {}, self._visit_all(notes)) + return nodes + [notes_list] def visit_paragraph(self, paragraph): + content = self._visit_all(paragraph.children) + if self._ignore_empty_paragraphs: + children = content + else: + children = [html.force_write] + content + html_path = self._find_html_path_for_paragraph(paragraph) - satisfy_html_path(self._html_generator, html_path) - if not self._ignore_empty_paragraphs: - self._html_generator.write_all() - self._visit_all(paragraph.children) + return html_path.wrap(children) def visit_run(self, run): - run_generator = self._html_generator.child() + nodes = self._visit_all(run.children) + if run.is_strikethrough: + nodes = self._find_style_for_run_property("strikethrough", default="s").wrap(nodes) + if run.is_underline: + nodes = self._convert_underline(nodes) + if run.vertical_alignment == documents.VerticalAlignment.subscript: + nodes = html_paths.element(["sub"], fresh=False).wrap(nodes) + if run.vertical_alignment == documents.VerticalAlignment.superscript: + nodes = html_paths.element(["sup"], fresh=False).wrap(nodes) + if run.is_italic: + nodes = self._find_style_for_run_property("italic", default="em").wrap(nodes) + if run.is_bold: + nodes = self._find_style_for_run_property("bold", default="strong").wrap(nodes) html_path = self._find_html_path_for_run(run) if html_path: - satisfy_html_path(run_generator, html_path) - if run.is_bold: - self._convert_run_property(run_generator, "bold", default="strong") - if run.is_italic: - self._convert_run_property(run_generator, "italic", default="em") - if run.vertical_alignment == documents.VerticalAlignment.superscript: - run_generator.start("sup") - if run.vertical_alignment == documents.VerticalAlignment.subscript: - run_generator.start("sub") - if run.is_underline: - self._convert_underline(run_generator) - if run.is_strikethrough: - self._convert_run_property(run_generator, "strikethrough", default="s") - self._with_html_generator(run_generator)._visit_all(run.children) - run_generator.end_all() - self._html_generator.append(run_generator) + nodes = html_path.wrap(nodes) + return nodes - def _default_convert_underline(self, run_generator): - style = self._find_style(None, "underline") - if style is not None: - append_html_path(run_generator, style.html_path) + def _default_convert_underline(self, nodes): + return self._find_style_for_run_property("underline").wrap(nodes) - def _convert_run_property(self, run_generator, element_type, default): + def _find_style_for_run_property(self, element_type, default=None): style = self._find_style(None, element_type) - if style is None: - run_generator.start(default) + if style is not None: + return style.html_path + elif default is not None: + return html_paths.element(default, fresh=False) else: - append_html_path(run_generator, style.html_path) - + return html_paths.empty def visit_text(self, text): - self._html_generator.text(text.value) + return [html.text(text.value)] def visit_hyperlink(self, hyperlink): @@ -149,60 +131,65 @@ class _DocumentConverter(documents.ElementVisitor): href = hyperlink.href else: href = "#{0}".format(self._html_id(hyperlink.anchor)) - self._html_generator.start("a", {"href": href}) - self._visit_all(hyperlink.children) - self._html_generator.end() + + nodes = self._visit_all(hyperlink.children) + return [html.collapsible_element("a", {"href": href}, nodes)] def visit_bookmark(self, bookmark): - self._html_generator.start("a", {"id": self._html_id(bookmark.name)}, always_write=True) - self._html_generator.end() + element = html.collapsible_element( + "a", + {"id": self._html_id(bookmark.name)}, + [html.force_write]) + return [element] def visit_tab(self, tab): - self._html_generator.text("\t") + return [html.text("\t")] def visit_table(self, table): - self._html_generator.end_all() - self._html_generator.start("table") - self._visit_all(table.children) - self._html_generator.end() + return [html.element("table", {}, self._visit_all(table.children))] def visit_table_row(self, table_row): - self._html_generator.start("tr") - self._visit_all(table_row.children) - self._html_generator.end() + return [html.element("tr", {}, self._visit_all(table_row.children))] def visit_table_cell(self, table_cell): - self._html_generator.start("td", always_write=True) - for child in table_cell.children: - child_generator = self._html_generator.child() - self._with_html_generator(child_generator).visit(child) - child_generator.end_all() - self._html_generator.append(child_generator) - - self._html_generator.end() + nodes = [html.force_write] + self._visit_all(table_cell.children) + return [ + html.element("td", {}, nodes) + ] def visit_line_break(self, line_break): - self._html_generator.self_closing("br") + return [html.self_closing_element("br")] def visit_note_reference(self, note_reference): - self._html_generator.start("sup") - self._html_generator.start("a", { - "href": "#" + self._note_html_id(note_reference), - "id": self._note_ref_html_id(note_reference), - }) self._note_references.append(note_reference); note_number = len(self._note_references) - self._html_generator.text("[{0}]".format(note_number)) - self._html_generator.end() - self._html_generator.end() + return [ + html.element("sup", {}, [ + html.element("a", { + "href": "#" + self._note_html_id(note_reference), + "id": self._note_ref_html_id(note_reference), + }, [html.text("[{0}]".format(note_number))]) + ]) + ] def visit_note(self, note): + note_body = self._visit_all(note.body) + [ + html.collapsible_element("p", {}, [ + html.text(" "), + html.element("a", {"href": "#" + self._note_ref_html_id(note)}, [ + html.text(_up_arrow) + ]), + ]) + ] + return [ + html.element("li", {"id": self._note_html_id(note)}, note_body) + ] self._html_generator.start("li", {"id": self._note_html_id(note)}) note_generator = self._html_generator.child() self._with_html_generator(note_generator)._visit_all(note.body) @@ -215,8 +202,7 @@ class _DocumentConverter(documents.ElementVisitor): def _visit_all(self, elements): - for element in elements: - self.visit(element) + return lists.flat_map(self.visit, elements) def _find_html_path_for_paragraph(self, paragraph): diff --git a/mammoth/html/__init__.py b/mammoth/html/__init__.py index 54bb87d..889ec68 100644 --- a/mammoth/html/__init__.py +++ b/mammoth/html/__init__.py @@ -6,15 +6,7 @@ def text(value): return TextNode(value) -def element(tag_name, attributes=None, children=None): - return _element(tag_name, attributes, children, collapsible=False) - - -def collapsible_element(tag_name, attributes=None, children=None): - return _element(tag_name, attributes, children, collapsible=True) - - -def _element(tag_names, attributes, children, collapsible): +def element(tag_names, attributes=None, children=None, collapsible=False): if not isinstance(tag_names, list): tag_names = [tag_names] if attributes is None: @@ -22,6 +14,10 @@ def _element(tag_names, attributes, children, collapsible): if children is None: children = [] return Element(tag_names, attributes, children, collapsible=collapsible) + + +def collapsible_element(tag_names, attributes=None, children=None): + return element(tag_names, attributes, children, collapsible=True) def self_closing_element(tag_name, attributes=None): @@ -120,3 +116,31 @@ def _try_collapse(collapsed, node): def _is_match(first, second): return first.tag_name in second.tag_names and first.attributes == second.attributes + + +def write(writer, nodes): + visitor = _NodeWriter(writer) + visitor.visit_all(nodes) + + +class _NodeWriter(NodeVisitor): + def __init__(self, writer): + self._writer = writer + + def visit_text_node(self, node): + self._writer.text(node.value) + + def visit_element(self, element): + self._writer.start(element.tag_name, element.attributes) + self.visit_all(element.children) + self._writer.end(element.tag_name) + + def visit_self_closing_element(self, element): + self._writer.self_closing(element.tag_name, element.attributes) + + def visit_force_write(self, element): + pass + + def visit_all(self, nodes): + for node in nodes: + self.visit(node) diff --git a/mammoth/html_paths.py b/mammoth/html_paths.py index 1428015..e043e23 100644 --- a/mammoth/html_paths.py +++ b/mammoth/html_paths.py @@ -1,4 +1,6 @@ -import collections +import cobble + +from . import html def path(elements): @@ -13,7 +15,32 @@ def element(names, class_names=None, fresh=None): return HtmlPathElement(names, class_names, fresh) -HtmlPath = collections.namedtuple("HtmlPath", ["elements"]) -HtmlPathElement = collections.namedtuple("HtmlPathElement", ["names", "class_names", "fresh"]) +@cobble.data +class HtmlPath(object): + elements = cobble.field() + + def wrap(self, nodes): + for element in reversed(self.elements): + nodes = element.wrap(nodes) + + return nodes + +@cobble.data +class HtmlPathElement(object): + names = cobble.field() + class_names = cobble.field() + fresh = cobble.field() + + def wrap(self, nodes): + if self.class_names: + attributes = {"class": " ".join(self.class_names)} + else: + attributes = {} + element = html.element( + self.names, + attributes, + nodes, + collapsible=not self.fresh) + return [element] empty = path([]) diff --git a/mammoth/images.py b/mammoth/images.py index 713f739..3def424 100644 --- a/mammoth/images.py +++ b/mammoth/images.py @@ -1,9 +1,12 @@ +from . import html + + def inline(func): - def convert_image(image, html_generator): + def convert_image(image): attributes = func(image).copy() if image.alt_text: attributes["alt"] = image.alt_text - html_generator.self_closing("img", attributes) + return [html.self_closing_element("img", attributes)] return convert_image diff --git a/mammoth/underline.py b/mammoth/underline.py index bb7d270..4a6b346 100644 --- a/mammoth/underline.py +++ b/mammoth/underline.py @@ -1,5 +1,8 @@ +from . import html + + def element(name): - def convert_underline(html_generator): - html_generator.start(name) + def convert_underline(nodes): + return [html.collapsible_element(name, {}, nodes)] return convert_underline diff --git a/tests/conversion_tests.py b/tests/conversion_tests.py index 98359a7..cb6f961 100644 --- a/tests/conversion_tests.py +++ b/tests/conversion_tests.py @@ -7,7 +7,7 @@ import io from nose.tools import istest, assert_equal import mammoth -from mammoth import documents, style_reader, results +from mammoth import documents, style_reader, results, html from mammoth.conversion import convert_document_element_to_html from mammoth.docx.xmlparser import parse_xml @@ -365,9 +365,9 @@ def images_have_alt_tags_if_available(): @istest def can_define_custom_conversion_for_images(): - def convert_image(image, html_generator): + def convert_image(image): with image.open() as image_file: - html_generator.self_closing("img", {"alt": image_file.read().decode("ascii")}) + return [html.self_closing_element("img", {"alt": image_file.read().decode("ascii")})] image = documents.image(alt_text=None, content_type="image/png", open=lambda: io.BytesIO(b"abc")) result = convert_document_element_to_html(image, convert_image=convert_image) diff --git a/tests/mammoth_tests.py b/tests/mammoth_tests.py index d749b4c..677a97b 100644 --- a/tests/mammoth_tests.py +++ b/tests/mammoth_tests.py @@ -195,21 +195,21 @@ def text_boxes_are_read(): def underline_is_ignored_by_default(): with open(test_path("underline.docx"), "rb") as fileobj: result = mammoth.convert_to_html(fileobj=fileobj) - assert_equal('

The Sunset Tree

', result.value) + assert_equal('

The Sunset Tree

', result.value) @istest def underline_can_be_configured_with_convert_underline_option(): with open(test_path("underline.docx"), "rb") as fileobj: result = mammoth.convert_to_html(fileobj=fileobj, convert_underline=mammoth.underline.element("em")) - assert_equal('

The Sunset Tree

', result.value) + assert_equal('

The Sunset Tree

', result.value) @istest def underline_can_be_configured_with_style_mapping(): with open(test_path("underline.docx"), "rb") as fileobj: result = mammoth.convert_to_html(fileobj=fileobj, style_map="u => em") - assert_equal('

The Sunset Tree

', result.value) + assert_equal('

The Sunset Tree

', result.value) @istest -- cgit v1.2.3