From f8d72dbba832db282a3ec04a5108fbd090f49a4f Mon Sep 17 00:00:00 2001 From: Michael Williamson Date: Sun, 1 Dec 2013 11:47:48 +0000 Subject: Move docx reading into separate package --- mammoth/docx.py | 88 -------------------------------------- mammoth/docx/__init__.py | 88 ++++++++++++++++++++++++++++++++++++++ mammoth/docx/xmlparser.py | 99 +++++++++++++++++++++++++++++++++++++++++++ mammoth/xmlparser.py | 99 ------------------------------------------- setup.py | 2 +- tests/docx/__init__.py | 0 tests/docx/docx_tests.py | 97 ++++++++++++++++++++++++++++++++++++++++++ tests/docx/xmlparser_tests.py | 69 ++++++++++++++++++++++++++++++ tests/docx_tests.py | 97 ------------------------------------------ tests/xmlparser_tests.py | 69 ------------------------------ 10 files changed, 354 insertions(+), 354 deletions(-) delete mode 100644 mammoth/docx.py create mode 100644 mammoth/docx/__init__.py create mode 100644 mammoth/docx/xmlparser.py delete mode 100644 mammoth/xmlparser.py create mode 100644 tests/docx/__init__.py create mode 100644 tests/docx/docx_tests.py create mode 100644 tests/docx/xmlparser_tests.py delete mode 100644 tests/docx_tests.py delete mode 100644 tests/xmlparser_tests.py diff --git a/mammoth/docx.py b/mammoth/docx.py deleted file mode 100644 index 01087ce..0000000 --- a/mammoth/docx.py +++ /dev/null @@ -1,88 +0,0 @@ -import zipfile - -from . import documents -from .results import Result -from .xmlparser import parse_xml, node_types - - -_namespaces = [ - ("w", "http://schemas.openxmlformats.org/wordprocessingml/2006/main"), -] - - -def read(fileobj): - zip_file = zipfile.ZipFile(fileobj) - document_fileobj = zip_file.open("word/document.xml") - document_xml = parse_xml(document_fileobj, _namespaces) - return Result(read_xml_element(document_xml), []) - - -_handlers = {} - - -def handler(name): - def add(func): - _handlers[name] = func - - return add - - -def read_xml_element(element): - handler = _handlers.get(element.name) - if handler is None: - return None - else: - return handler(element) - - -@handler("w:t") -def text(element): - return documents.Text(_inner_text(element)) - - -@handler("w:r") -def run(element): - return documents.Run(_read_xml_elements(element.children)) - - -@handler("w:p") -def paragraph(element): - properties = element.find_child_or_null("w:pPr") - style_name = properties \ - .find_child_or_null("w:pStyle") \ - .attributes.get("w:val") - - children = _read_xml_elements(element.children) - return documents.paragraph(children, style_name) - - -@handler("w:body") -def body(element): - return _read_xml_elements(element.children) - - -@handler("w:document") -def document(element): - body_element = _find_child(element, "w:body") - return documents.Document(_read_xml_elements(body_element.children)) - - -def _find(predicate, iterable): - for item in iterable: - if predicate(item): - return item - - -def _find_child(element, name): - return _find(lambda child: child.name == name, element.children) - - -def _read_xml_elements(elements): - return filter(None, map(read_xml_element, elements)) - - -def _inner_text(node): - if node.node_type == node_types.text: - return node.value - else: - return "".join(_inner_text(child) for child in node.children) diff --git a/mammoth/docx/__init__.py b/mammoth/docx/__init__.py new file mode 100644 index 0000000..de2b8a8 --- /dev/null +++ b/mammoth/docx/__init__.py @@ -0,0 +1,88 @@ +import zipfile + +from .. import documents +from ..results import Result +from .xmlparser import parse_xml, node_types + + +_namespaces = [ + ("w", "http://schemas.openxmlformats.org/wordprocessingml/2006/main"), +] + + +def read(fileobj): + zip_file = zipfile.ZipFile(fileobj) + document_fileobj = zip_file.open("word/document.xml") + document_xml = parse_xml(document_fileobj, _namespaces) + return Result(read_xml_element(document_xml), []) + + +_handlers = {} + + +def handler(name): + def add(func): + _handlers[name] = func + + return add + + +def read_xml_element(element): + handler = _handlers.get(element.name) + if handler is None: + return None + else: + return handler(element) + + +@handler("w:t") +def text(element): + return documents.Text(_inner_text(element)) + + +@handler("w:r") +def run(element): + return documents.Run(_read_xml_elements(element.children)) + + +@handler("w:p") +def paragraph(element): + properties = element.find_child_or_null("w:pPr") + style_name = properties \ + .find_child_or_null("w:pStyle") \ + .attributes.get("w:val") + + children = _read_xml_elements(element.children) + return documents.paragraph(children, style_name) + + +@handler("w:body") +def body(element): + return _read_xml_elements(element.children) + + +@handler("w:document") +def document(element): + body_element = _find_child(element, "w:body") + return documents.Document(_read_xml_elements(body_element.children)) + + +def _find(predicate, iterable): + for item in iterable: + if predicate(item): + return item + + +def _find_child(element, name): + return _find(lambda child: child.name == name, element.children) + + +def _read_xml_elements(elements): + return filter(None, map(read_xml_element, elements)) + + +def _inner_text(node): + if node.node_type == node_types.text: + return node.value + else: + return "".join(_inner_text(child) for child in node.children) diff --git a/mammoth/docx/xmlparser.py b/mammoth/docx/xmlparser.py new file mode 100644 index 0000000..d58ea75 --- /dev/null +++ b/mammoth/docx/xmlparser.py @@ -0,0 +1,99 @@ +import collections +import xml.sax + + +XmlElementBase = collections.namedtuple("XmlElement", ["name", "attributes", "children"]) +class XmlElement(XmlElementBase): + def find_child_or_null(self, name): + for child in self.children: + if child.name == name: + return child + return _null_xml_element + + +class NullXmlElement(object): + attributes = {} + + def find_child_or_null(self, name): + return self + + +_null_xml_element = NullXmlElement() + + +XmlText = collections.namedtuple("XmlText", ["value"]) + + +def element(name, attributes=None, children=None): + return XmlElement(name, attributes or {}, children or []) + +text = XmlText + + +class node_types(object): + element = 1 + text = 3 + + +XmlElement.node_type = node_types.element +XmlText.node_type = node_types.text + + + +def parse_xml(fileobj, namespace_mapping=None): + if namespace_mapping is None: + namespace_prefixes = {} + else: + namespace_prefixes = dict((uri, prefix) for prefix, uri in namespace_mapping) + + handler = Handler(namespace_prefixes) + parser = xml.sax.make_parser() + parser.setFeature(xml.sax.handler.feature_namespaces, True) + parser.setContentHandler(handler) + parser.parse(fileobj) + return handler.root() + + +class Handler(xml.sax.handler.ContentHandler): + def __init__(self, namespace_prefixes): + self._namespace_prefixes = namespace_prefixes + self._element_stack = [RootElement()] + self._character_buffer = [] + + def root(self): + return self._element_stack[0].children[0] + + def startElementNS(self, name, qname, attrs): + self._flush_character_buffer() + attributes = dict((self._read_name(key), value) for key, value in attrs.items()) + element = XmlElement(self._read_name(name), attributes, []) + self._element_stack[-1].children.append(element) + self._element_stack.append(element) + + def endElementNS(self, name, qname): + self._flush_character_buffer() + self._element_stack.pop() + + def characters(self, content): + self._character_buffer.append(content) + + def _flush_character_buffer(self): + if self._character_buffer: + text = "".join(self._character_buffer) + self._element_stack[-1].children.append(XmlText(text)) + self._character_buffer = [] + + def _read_name(self, name): + uri, local_name = name + if uri is None: + return local_name + else: + prefix = self._namespace_prefixes.get(uri) + if prefix is None: + return "{%s}%s" % (uri, local_name) + else: + return "%s:%s" % (prefix, local_name) + +class RootElement(object): + def __init__(self): + self.children = [] diff --git a/mammoth/xmlparser.py b/mammoth/xmlparser.py deleted file mode 100644 index d58ea75..0000000 --- a/mammoth/xmlparser.py +++ /dev/null @@ -1,99 +0,0 @@ -import collections -import xml.sax - - -XmlElementBase = collections.namedtuple("XmlElement", ["name", "attributes", "children"]) -class XmlElement(XmlElementBase): - def find_child_or_null(self, name): - for child in self.children: - if child.name == name: - return child - return _null_xml_element - - -class NullXmlElement(object): - attributes = {} - - def find_child_or_null(self, name): - return self - - -_null_xml_element = NullXmlElement() - - -XmlText = collections.namedtuple("XmlText", ["value"]) - - -def element(name, attributes=None, children=None): - return XmlElement(name, attributes or {}, children or []) - -text = XmlText - - -class node_types(object): - element = 1 - text = 3 - - -XmlElement.node_type = node_types.element -XmlText.node_type = node_types.text - - - -def parse_xml(fileobj, namespace_mapping=None): - if namespace_mapping is None: - namespace_prefixes = {} - else: - namespace_prefixes = dict((uri, prefix) for prefix, uri in namespace_mapping) - - handler = Handler(namespace_prefixes) - parser = xml.sax.make_parser() - parser.setFeature(xml.sax.handler.feature_namespaces, True) - parser.setContentHandler(handler) - parser.parse(fileobj) - return handler.root() - - -class Handler(xml.sax.handler.ContentHandler): - def __init__(self, namespace_prefixes): - self._namespace_prefixes = namespace_prefixes - self._element_stack = [RootElement()] - self._character_buffer = [] - - def root(self): - return self._element_stack[0].children[0] - - def startElementNS(self, name, qname, attrs): - self._flush_character_buffer() - attributes = dict((self._read_name(key), value) for key, value in attrs.items()) - element = XmlElement(self._read_name(name), attributes, []) - self._element_stack[-1].children.append(element) - self._element_stack.append(element) - - def endElementNS(self, name, qname): - self._flush_character_buffer() - self._element_stack.pop() - - def characters(self, content): - self._character_buffer.append(content) - - def _flush_character_buffer(self): - if self._character_buffer: - text = "".join(self._character_buffer) - self._element_stack[-1].children.append(XmlText(text)) - self._character_buffer = [] - - def _read_name(self, name): - uri, local_name = name - if uri is None: - return local_name - else: - prefix = self._namespace_prefixes.get(uri) - if prefix is None: - return "{%s}%s" % (uri, local_name) - else: - return "%s:%s" % (prefix, local_name) - -class RootElement(object): - def __init__(self): - self.children = [] diff --git a/setup.py b/setup.py index 5f529c6..50c3cf3 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ setup( author='Michael Williamson', author_email='mike@zwobble.org', url='http://github.com/mwilliamson/python-mammoth', - packages=['mammoth'], + packages=['mammoth', 'mammoth.docx'], keywords="docx word office clean html", ) diff --git a/tests/docx/__init__.py b/tests/docx/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/docx/docx_tests.py b/tests/docx/docx_tests.py new file mode 100644 index 0000000..9e9efe2 --- /dev/null +++ b/tests/docx/docx_tests.py @@ -0,0 +1,97 @@ +from nose.tools import istest, assert_equal + +from mammoth import docx, documents +from mammoth.docx.xmlparser import element as xml_element, text as xml_text +from ..testing import test_path + +@istest +class ReadTests(object): + @istest + def can_read_document_with_single_paragraph_with_single_run_of_text(self): + with open(test_path("single-paragraph.docx")) as fileobj: + result = docx.read(fileobj=fileobj) + expected_document = documents.Document([ + documents.paragraph([ + documents.Run([ + documents.Text("Walking on imported air") + ]) + ]) + ]) + assert_equal(expected_document, result.value) + + +@istest +class ReadXmlElementTests(object): + @istest + def text_from_text_element_is_read(self): + element = _text_element("Hello!") + assert_equal(documents.Text("Hello!"), docx.read_xml_element(element)) + + @istest + def can_read_text_within_run(self): + element = _run_element_with_text("Hello!") + assert_equal( + documents.Run([documents.Text("Hello!")]), + docx.read_xml_element(element) + ) + + @istest + def can_read_text_within_paragraph(self): + element = _paragraph_element_with_text("Hello!") + assert_equal( + documents.paragraph([documents.Run([documents.Text("Hello!")])]), + docx.read_xml_element(element) + ) + + @istest + def can_read_text_within_document(self): + element = _document_element_with_text("Hello!") + assert_equal( + documents.Document([documents.paragraph([documents.Run([documents.Text("Hello!")])])]), + docx.read_xml_element(element) + ) + + @istest + def paragraph_has_no_style_if_it_has_no_properties(self): + element = xml_element("w:p") + assert_equal(None, docx.read_xml_element(element).style_name) + + @istest + def paragraph_has_style_name_read_from_paragraph_properties_if_present(self): + style_xml = xml_element("w:pStyle", {"w:val": "Heading1"}) + properties_xml = xml_element("w:pPr", {}, [style_xml]) + paragraph_xml = xml_element("w:p", {}, [properties_xml]) + paragraph = docx.read_xml_element(paragraph_xml) + assert_equal("Heading1", paragraph.style_name) + + @istest + def unrecognised_elements_are_ignored(self): + element = xml_element("w:huh", {}, []) + assert_equal(None, docx.read_xml_element(element)) + + @istest + def unrecognised_children_are_ignored(self): + element = xml_element("w:r", {}, [_text_element("Hello!"), xml_element("w:huh", {}, [])]) + assert_equal( + documents.Run([documents.Text("Hello!")]), + docx.read_xml_element(element) + ) + + +def _document_element_with_text(text): + return xml_element("w:document", {}, [ + xml_element("w:body", {}, [_paragraph_element_with_text(text)]) + ]) + + +def _paragraph_element_with_text(text): + return xml_element("w:p", {}, [_run_element_with_text(text)]) + + +def _run_element_with_text(text): + return xml_element("w:r", {}, [_text_element(text)]) + + +def _text_element(value): + return xml_element("w:t", {}, [xml_text(value)]) + diff --git a/tests/docx/xmlparser_tests.py b/tests/docx/xmlparser_tests.py new file mode 100644 index 0000000..5e33778 --- /dev/null +++ b/tests/docx/xmlparser_tests.py @@ -0,0 +1,69 @@ +import io + +from nose.tools import istest, assert_equal + +from mammoth.docx.xmlparser import parse_xml, element as xml_element, text as xml_text + + +@istest +def can_parse_self_closing_element(): + xml = _parse_xml_string(b"") + assert_equal(xml_element("body", {}, []), xml) + + +@istest +def can_parse_empty_element_with_separate_closing_tag(): + xml = _parse_xml_string(b"") + assert_equal(xml_element("body", {}, []), xml) + + +@istest +def can_parse_attributes_of_tag(): + xml = _parse_xml_string(b"") + assert_equal(xml_element("body", {"name": "bob"}, []), xml) + + +@istest +def can_parse_text_element(): + xml = _parse_xml_string(b"Hello!") + assert_equal(xml_element("body", {}, [xml_text("Hello!")]), xml) + + +@istest +def can_parse_text_element_before_new_tag(): + xml = _parse_xml_string(b"Hello!
") + assert_equal(xml_element("body", {}, [xml_text("Hello!"), xml_element("br", {}, [])]), xml) + + +@istest +def can_parse_element_with_children(): + xml = _parse_xml_string(b"") + assert_equal([xml_element("a", {}, []), xml_element("b", {}, [])], xml.children) + + +@istest +def unmapped_namespaces_uris_are_included_in_braces_as_prefix(): + xml = _parse_xml_string(b'') + assert_equal("{word}body", xml.name) + + +@istest +def mapped_namespaces_uris_are_translated_using_namespace_map(): + xml = _parse_xml_string(b'', [("x", "word")]) + assert_equal("x:body", xml.name) + + +@istest +def namespace_of_attributes_is_mapped_to_prefix(): + xml = _parse_xml_string(b'', [("x", "word")]) + assert_equal("Hello!", xml.attributes["x:val"]) + + +@istest +def whitespace_between_xml_declaration_and_root_tag_is_ignored(): + xml = _parse_xml_string(b'\n') + assert_equal("body", xml.name) + + +def _parse_xml_string(string, namespace_mapping=None): + return parse_xml(io.BytesIO(string), namespace_mapping) diff --git a/tests/docx_tests.py b/tests/docx_tests.py deleted file mode 100644 index 792ff14..0000000 --- a/tests/docx_tests.py +++ /dev/null @@ -1,97 +0,0 @@ -from nose.tools import istest, assert_equal - -from mammoth import docx, documents -from mammoth.xmlparser import element as xml_element, text as xml_text -from .testing import test_path - -@istest -class ReadTests(object): - @istest - def can_read_document_with_single_paragraph_with_single_run_of_text(self): - with open(test_path("single-paragraph.docx")) as fileobj: - result = docx.read(fileobj=fileobj) - expected_document = documents.Document([ - documents.paragraph([ - documents.Run([ - documents.Text("Walking on imported air") - ]) - ]) - ]) - assert_equal(expected_document, result.value) - - -@istest -class ReadXmlElementTests(object): - @istest - def text_from_text_element_is_read(self): - element = _text_element("Hello!") - assert_equal(documents.Text("Hello!"), docx.read_xml_element(element)) - - @istest - def can_read_text_within_run(self): - element = _run_element_with_text("Hello!") - assert_equal( - documents.Run([documents.Text("Hello!")]), - docx.read_xml_element(element) - ) - - @istest - def can_read_text_within_paragraph(self): - element = _paragraph_element_with_text("Hello!") - assert_equal( - documents.paragraph([documents.Run([documents.Text("Hello!")])]), - docx.read_xml_element(element) - ) - - @istest - def can_read_text_within_document(self): - element = _document_element_with_text("Hello!") - assert_equal( - documents.Document([documents.paragraph([documents.Run([documents.Text("Hello!")])])]), - docx.read_xml_element(element) - ) - - @istest - def paragraph_has_no_style_if_it_has_no_properties(self): - element = xml_element("w:p") - assert_equal(None, docx.read_xml_element(element).style_name) - - @istest - def paragraph_has_style_name_read_from_paragraph_properties_if_present(self): - style_xml = xml_element("w:pStyle", {"w:val": "Heading1"}) - properties_xml = xml_element("w:pPr", {}, [style_xml]) - paragraph_xml = xml_element("w:p", {}, [properties_xml]) - paragraph = docx.read_xml_element(paragraph_xml) - assert_equal("Heading1", paragraph.style_name) - - @istest - def unrecognised_elements_are_ignored(self): - element = xml_element("w:huh", {}, []) - assert_equal(None, docx.read_xml_element(element)) - - @istest - def unrecognised_children_are_ignored(self): - element = xml_element("w:r", {}, [_text_element("Hello!"), xml_element("w:huh", {}, [])]) - assert_equal( - documents.Run([documents.Text("Hello!")]), - docx.read_xml_element(element) - ) - - -def _document_element_with_text(text): - return xml_element("w:document", {}, [ - xml_element("w:body", {}, [_paragraph_element_with_text(text)]) - ]) - - -def _paragraph_element_with_text(text): - return xml_element("w:p", {}, [_run_element_with_text(text)]) - - -def _run_element_with_text(text): - return xml_element("w:r", {}, [_text_element(text)]) - - -def _text_element(value): - return xml_element("w:t", {}, [xml_text(value)]) - diff --git a/tests/xmlparser_tests.py b/tests/xmlparser_tests.py deleted file mode 100644 index 33563dd..0000000 --- a/tests/xmlparser_tests.py +++ /dev/null @@ -1,69 +0,0 @@ -import io - -from nose.tools import istest, assert_equal - -from mammoth.xmlparser import parse_xml, element as xml_element, text as xml_text - - -@istest -def can_parse_self_closing_element(): - xml = _parse_xml_string(b"") - assert_equal(xml_element("body", {}, []), xml) - - -@istest -def can_parse_empty_element_with_separate_closing_tag(): - xml = _parse_xml_string(b"") - assert_equal(xml_element("body", {}, []), xml) - - -@istest -def can_parse_attributes_of_tag(): - xml = _parse_xml_string(b"") - assert_equal(xml_element("body", {"name": "bob"}, []), xml) - - -@istest -def can_parse_text_element(): - xml = _parse_xml_string(b"Hello!") - assert_equal(xml_element("body", {}, [xml_text("Hello!")]), xml) - - -@istest -def can_parse_text_element_before_new_tag(): - xml = _parse_xml_string(b"Hello!
") - assert_equal(xml_element("body", {}, [xml_text("Hello!"), xml_element("br", {}, [])]), xml) - - -@istest -def can_parse_element_with_children(): - xml = _parse_xml_string(b"") - assert_equal([xml_element("a", {}, []), xml_element("b", {}, [])], xml.children) - - -@istest -def unmapped_namespaces_uris_are_included_in_braces_as_prefix(): - xml = _parse_xml_string(b'') - assert_equal("{word}body", xml.name) - - -@istest -def mapped_namespaces_uris_are_translated_using_namespace_map(): - xml = _parse_xml_string(b'', [("x", "word")]) - assert_equal("x:body", xml.name) - - -@istest -def namespace_of_attributes_is_mapped_to_prefix(): - xml = _parse_xml_string(b'', [("x", "word")]) - assert_equal("Hello!", xml.attributes["x:val"]) - - -@istest -def whitespace_between_xml_declaration_and_root_tag_is_ignored(): - xml = _parse_xml_string(b'\n') - assert_equal("body", xml.name) - - -def _parse_xml_string(string, namespace_mapping=None): - return parse_xml(io.BytesIO(string), namespace_mapping) -- cgit v1.2.3