Merge branch 'master' into image-errors

Conflicts:
	test/mammoth.tests.js

10 files changed, 194 insertions, 83 deletions

diff --git a/lib/document-matchers.js b/lib/document-matchers.js
index 60bc65f..bdc213c 100644
--- a/lib/document-matchers.js
+++ b/lib/document-matchers.js
@@ -4,7 +4,7 @@ exports.bold = new Matcher("bold");
 exports.italic = new Matcher("italic");
 exports.underline = new Matcher("underline");
 exports.strikethrough = new Matcher("strikethrough");
-exports.commentReference = new Matcher("comment-reference");
+exports.commentReference = new Matcher("commentReference");
 
 
 function paragraph(options) {
diff --git a/lib/document-to-html.js b/lib/document-to-html.js
index 6aa7a9c..e022946 100644
--- a/lib/document-to-html.js
+++ b/lib/document-to-html.js
@@ -13,18 +13,24 @@ exports.DocumentConverter = DocumentConverter;
 
 function DocumentConverter(options) {
     return {
-        convertToHtml: function() {
-            var conversion = new DocumentConversion(options);
-            return conversion.convertToHtml.apply(this, arguments);
+        convertToHtml: function(element) {
+            var comments = _.indexBy(
+                element.type === documents.types.document ? element.comments : [],
+                "commentId"
+            );
+            var conversion = new DocumentConversion(options, comments);
+            return conversion.convertToHtml(element);
         }
     };
 }
 
-function DocumentConversion(options) {
+function DocumentConversion(options, comments) {
     var noteNumber = 1;
     
     var noteReferences = [];
     
+    var referencedComments = [];
+    
     options = _.extend({ignoreEmptyParagraphs: true}, options);
     var idPrefix = options.idPrefix === undefined ? "" : options.idPrefix;
     
@@ -89,16 +95,17 @@ function DocumentConversion(options) {
     }
     
     function convertParagraph(element, messages) {
-        var style = styleForParagraph(element, messages);
-        
-        var children = convertElements(element.children, messages);
-        if (!options.ignoreEmptyParagraphs) {
-            children = [Html.forceWrite].concat(children);
-        }
-        return Html.pathToNodes(style, children);
+        return htmlPathForParagraph(element, messages).wrap(function() {
+            var content = convertElements(element.children, messages);
+            if (options.ignoreEmptyParagraphs) {
+                return content;
+            } else {
+                return [Html.forceWrite].concat(content);
+            }
+        });
     }
     
-    function styleForParagraph(element, messages) {
+    function htmlPathForParagraph(element, messages) {
         var style = findStyle(element);
         
         if (style) {
@@ -112,58 +119,60 @@ function DocumentConversion(options) {
     }
     
     function convertRun(run, messages) {
-        var style;
-        if (run.styleId) {
-            style = findStyle(run);
-            if (!style) {
-                messages.push(unrecognisedStyleWarning("run", run));
-            }
-        }
-        var result = convertElements(run.children, messages);
+        var nodes = function() {
+            return convertElements(run.children, messages);
+        };
+        var paths = [];
         if (run.isStrikethrough) {
-            result = convertRunProperty(result, "strikethrough", "s");
+            paths.push(findHtmlPathForRunProperty("strikethrough", "s"));
         }
         if (run.isUnderline) {
-            result = convertUnderline(result);
+            paths.push(findHtmlPathForRunProperty("underline"));
         }
         if (run.verticalAlignment === documents.verticalAlignment.subscript) {
-            result = wrapInNonFreshElement(result, "sub");
+            paths.push(htmlPaths.element("sub", {}, {fresh: false}));
         }
         if (run.verticalAlignment === documents.verticalAlignment.superscript) {
-            result = wrapInNonFreshElement(result, "sup");
+            paths.push(htmlPaths.element("sup", {}, {fresh: false}));
         }
         if (run.isItalic) {
-            result = convertRunProperty(result, "italic", "em");
+            paths.push(findHtmlPathForRunProperty("italic", "em"));
         }
         if (run.isBold) {
-            result = convertRunProperty(result, "bold", "strong");
+            paths.push(findHtmlPathForRunProperty("bold", "strong"));
         }
-        if (style) {
-            result = Html.pathToNodes(style.to, result);
+        var stylePath = htmlPaths.empty;
+        if (run.styleId) {
+            var style = findStyle(run);
+            if (style) {
+                stylePath = style.to;
+            } else {
+                messages.push(unrecognisedStyleWarning("run", run));
+            }
         }
-        return result;
-    }
-    
-    function wrapInNonFreshElement(nodes, tagName) {
-        return [Html.nonFreshElement(tagName, {}, nodes)];
+        paths.push(stylePath);
+            
+        paths.forEach(function(path) {
+            nodes = path.wrap.bind(path, nodes);
+        });
+        
+        return nodes();
     }
     
-    function convertUnderline(runHtml) {
-        var style = findStyle({type: "underline"});
-        if (style) {
-            return Html.pathToNodes(style.to, runHtml);
+    function findHtmlPathForRunProperty(elementType, defaultTagName) {
+        var path = findHtmlPath({type: elementType});
+        if (path) {
+            return path;
+        } else if (defaultTagName) {
+            return htmlPaths.element(defaultTagName, {}, {fresh: false});
         } else {
-            return runHtml;
+            return htmlPaths.empty;
         }
     }
     
-    function convertRunProperty(result, elementType, defaultTagName) {
-        var style = findStyle({type: elementType});
-        if (style) {
-            return Html.pathToNodes(style.to, result);
-        } else {
-            return wrapInNonFreshElement(result, defaultTagName);
-        }
+    function findHtmlPath(element, defaultPath) {
+        var style = findStyle(element);
+        return style ? style.to : defaultPath;
     }
     
     function findStyle(element) {
@@ -192,11 +201,19 @@ function DocumentConversion(options) {
     }
 
     function noteHtmlId(note) {
-        return htmlId(note.noteType + "-" + note.noteId);
+        return referentHtmlId(note.noteType, note.noteId);
     }
     
     function noteRefHtmlId(note) {
-        return htmlId(note.noteType + "-ref-" + note.noteId);
+        return referenceHtmlId(note.noteType, note.noteId);
+    }
+    
+    function referentHtmlId(referenceType, referenceId) {
+        return htmlId(referenceType + "-" + referenceId);
+    }
+    
+    function referenceHtmlId(referenceType, referenceId) {
+        return htmlId(referenceType + "-ref-" + referenceId);
     }
     
     function htmlId(suffix) {
@@ -226,6 +243,46 @@ function DocumentConversion(options) {
         ];
     }
     
+    function convertCommentReference(reference, messages) {
+        return findHtmlPath(reference, htmlPaths.ignore).wrap(function() {
+            var comment = comments[reference.commentId];
+            var count = referencedComments.length + 1;
+            var label = "[" + commentAuthorLabel(comment) + count + "]";
+            referencedComments.push({label: label, comment: comment});
+            // TODO: remove duplication with note references
+            return [
+                Html.freshElement("a", {
+                    href: "#" + referentHtmlId("comment", reference.commentId),
+                    id: referenceHtmlId("comment", reference.commentId)
+                }, [Html.text(label)])
+            ];
+        });
+    }
+    
+    function convertComment(referencedComment, messages) {
+        // TODO: remove duplication with note references
+        
+        var label = referencedComment.label;
+        var comment = referencedComment.comment;
+        var body = convertElements(comment.body).concat([
+            Html.nonFreshElement("p", {}, [
+                Html.text(" "),
+                Html.freshElement("a", {"href": "#" + referenceHtmlId("comment", comment.commentId)}, [
+                    Html.text("↑")
+                ])
+            ])
+        ]);
+        
+        return [
+            Html.freshElement(
+                "dt",
+                {"id": referentHtmlId("comment", comment.commentId)},
+                [Html.text("Comment " + label)]
+            ),
+            Html.freshElement("dd", {}, body)
+        ];
+    }
+    
     function wrapChildrenInFreshElement(element, wrapElementName, messages) {
         var children = convertElements(element.children, messages);
         return [
@@ -241,7 +298,10 @@ function DocumentConversion(options) {
             });
             var notesNodes = convertElements(notes, messages);
             return children.concat([
-                Html.freshElement("ol", {}, notesNodes)
+                Html.freshElement("ol", {}, notesNodes),
+                Html.freshElement("dl", {}, flatMap(referencedComments, function(referencedComment) {
+                    return convertComment(referencedComment, messages);
+                }))
             ]);
         },
         "paragraph": convertParagraph,
@@ -283,6 +343,8 @@ function DocumentConversion(options) {
             
             return Html.freshElement("li", {id: noteHtmlId(element)}, body);
         },
+        "commentReference": convertCommentReference,
+        "comment": convertComment,
         "image": deferredConversion(recoveringConvertImage(options.convertImage || defaultConvertImage)),
         "table": convertTable,
         "tableRow": convertTableRow,
@@ -331,3 +393,7 @@ function walkHtml(nodes, callback) {
         }
     });
 }
+
+var commentAuthorLabel = exports.commentAuthorLabel = function commentAuthorLabel(comment) {
+    return comment.authorInitials || "";
+};
diff --git a/lib/documents.js b/lib/documents.js
index d4a28af..428dc09 100644
--- a/lib/documents.js
+++ b/lib/documents.js
@@ -24,7 +24,8 @@ function Document(children, options) {
     return {
         type: types.document,
         children: children,
-        notes: options.notes || new Notes({})
+        notes: options.notes || new Notes({}),
+        comments: options.comments || []
     };
 }
 
@@ -181,9 +182,9 @@ function BookmarkStart(options) {
     };
 }
 
-exports.Document = Document;
-exports.Paragraph = Paragraph;
-exports.Run = Run;
+exports.document = exports.Document = Document;
+exports.paragraph = exports.Paragraph = Paragraph;
+exports.run = exports.Run = Run;
 exports.Text = Text;
 exports.Tab = Tab;
 exports.Hyperlink = Hyperlink;
diff --git a/lib/docx/body-reader.js b/lib/docx/body-reader.js
index b514363..7b3120d 100644
--- a/lib/docx/body-reader.js
+++ b/lib/docx/body-reader.js
@@ -379,13 +379,13 @@ var ignoreElements = {
     "office-word:wrap": true,
     "v:shadow": true,
     "v:shapetype": true,
+    "w:annotationRef": true,
     "w:bookmarkEnd": true,
     "w:sectPr": true,
     "w:proofErr": true,
     "w:lastRenderedPageBreak": true,
     "w:commentRangeStart": true,
     "w:commentRangeEnd": true,
-    "w:commentReference": true,
     "w:del": true,
     "w:footnoteRef": true,
     "w:endnoteRef": true,
diff --git a/lib/docx/document-xml-reader.js b/lib/docx/document-xml-reader.js
index 12ca19e..bb912e4 100644
--- a/lib/docx/document-xml-reader.js
+++ b/lib/docx/document-xml-reader.js
@@ -12,7 +12,10 @@ function DocumentXmlReader(options) {
         
         var result = bodyReader.readXmlElements(body.children)
             .map(function(children) {
-                return new documents.Document(children, {notes: options.notes});
+                return new documents.Document(children, {
+                    notes: options.notes,
+                    comments: options.comments
+                });
             });
         return new Result(result.value, result.messages);
     }
diff --git a/lib/docx/docx-reader.js b/lib/docx/docx-reader.js
index 6ad63ab..0c7d57f 100644
--- a/lib/docx/docx-reader.js
+++ b/lib/docx/docx-reader.js
@@ -14,6 +14,7 @@ var contentTypesReader = require("./content-types-reader");
 var numberingXml = require("./numbering-xml");
 var stylesReader = require("./styles-reader");
 var notesReader = require("./notes-reader");
+var commentsReader = require("./comments-reader");
 var Files = require("./files").Files;
 
 
@@ -40,6 +41,13 @@ function read(docxFile, input) {
                 } else {
                     return new Result([]);
                 }
+            }),
+            comments: readXmlFileWithBody("comments", result, function(bodyReader, xml) {
+                if (xml) {
+                    return commentsReader.createCommentsReader(bodyReader)(xml);
+                } else {
+                    return new Result([]);
+                }
             })
         };
     }).also(function(result) {
@@ -52,13 +60,20 @@ function read(docxFile, input) {
         };
     }).then(function(result) {
         return readXmlFileWithBody("document", result, function(bodyReader, xml) {
-            return result.notes.flatMap(function(notes) {
-                var reader = new DocumentXmlReader({
-                    bodyReader: bodyReader,
-                    notes: notes
+            if (xml) {
+                return result.notes.flatMap(function(notes) {
+                    return result.comments.flatMap(function(comments) {
+                        var reader = new DocumentXmlReader({
+                            bodyReader: bodyReader,
+                            notes: notes,
+                            comments: comments
+                        });
+                        return reader.convertXmlToDocument(xml);
+                    });
                 });
-                return reader.convertXmlToDocument(xml);
-            });
+            } else {
+                throw new Error("Could not find word/document.xml in ZIP file. Are you sure this is a valid .docx file?");
+            }
         });
     });
 }
diff --git a/lib/html-paths.js b/lib/html-paths.js
index 04bc655..5a00cd8 100644
--- a/lib/html-paths.js
+++ b/lib/html-paths.js
@@ -1,5 +1,7 @@
 var _ = require("underscore");
 
+var html = require("./html");
+
 exports.topLevelElement = topLevelElement;
 exports.elements = elements;
 exports.element = element;
@@ -9,15 +11,27 @@ function topLevelElement(tagName, attributes) {
 }
 
 function elements(elementStyles) {
-    return elementStyles.map(function(elementStyle) {
+    return new HtmlPath(elementStyles.map(function(elementStyle) {
         if (_.isString(elementStyle)) {
             return element(elementStyle);
         } else {
             return elementStyle;
         }
-    });
+    }));
+}
+
+function HtmlPath(elements) {
+    this._elements = elements;
 }
 
+HtmlPath.prototype.wrap = function wrap(children) {
+    var result = children();
+    for (var index = this._elements.length - 1; index >= 0; index--) {
+        result = this._elements[index].wrapNodes(result);
+    }
+    return result;
+};
+
 function element(tagName, attributes, options) {
     options = options || {};
     return new Element(tagName, attributes, options);
@@ -44,4 +58,17 @@ Element.prototype.matchesElement = function(element) {
     return this.tagNames[element.tagName] && _.isEqual(this.attributes || {}, element.attributes || {});
 };
 
+Element.prototype.wrap = function wrap(generateNodes) {
+    return this.wrapNodes(generateNodes());
+};
+
+Element.prototype.wrapNodes = function wrapNodes(nodes) {
+    return [html.elementWithTag(this, nodes)];
+};
+
 exports.empty = elements([]);
+exports.ignore = {
+    wrap: function() {
+        return [];
+    }
+};
diff --git a/lib/html/index.js b/lib/html/index.js
index edc8256..600d783 100644
--- a/lib/html/index.js
+++ b/lib/html/index.js
@@ -7,15 +7,6 @@ exports.selfClosingElement = ast.selfClosingElement;
 exports.text = ast.text;
 exports.forceWrite = ast.forceWrite;
 
-function pathToNodes(path, children) {
-    var result = children;
-    for (var index = path.length - 1; index >= 0; index--) {
-        result = [ast.elementWithTag(path[index], result)];
-    }
-    return result;
-}
-
-exports.pathToNodes = pathToNodes;
 exports.simplify = require("./simplify");
 
 function write(writer, nodes) {
diff --git a/lib/options-reader.js b/lib/options-reader.js
index 19934d8..3765fd3 100644
--- a/lib/options-reader.js
+++ b/lib/options-reader.js
@@ -26,6 +26,8 @@ var standardOptions = exports._standardOptions = {
         "r[style-name='footnote reference'] =>",
         "p[style-name='endnote text'] => p",
         "r[style-name='endnote reference'] =>",
+        "p[style-name='annotation text'] => p",
+        "r[style-name='annotation reference'] =>",
         
         // LibreOffice
         "p[style-name='Footnote'] => p",
diff --git a/lib/style-reader.js b/lib/style-reader.js
index ab9a171..fe60930 100644
--- a/lib/style-reader.js
+++ b/lib/style-reader.js
@@ -177,16 +177,21 @@ function htmlPathRule() {
         return htmlPaths.element(tagName, attributes, options);
     });
     
-    return lop.rules.then(
-        lop.rules.zeroOrMoreWithSeparator(
-            styleElementRule,
-            lop.rules.sequence(
-                whitespaceRule,
-                lop.rules.tokenOfType("gt"),
-                whitespaceRule
-            )
-        ),
-        htmlPaths.elements
+    return lop.rules.firstOf("html path",
+        lop.rules.then(lop.rules.tokenOfType("bang"), function() {
+            return htmlPaths.ignore;
+        }),
+        lop.rules.then(
+            lop.rules.zeroOrMoreWithSeparator(
+                styleElementRule,
+                lop.rules.sequence(
+                    whitespaceRule,
+                    lop.rules.tokenOfType("gt"),
+                    whitespaceRule
+                )
+            ),
+            htmlPaths.elements
+        )
     );
 }
     
@@ -234,7 +239,8 @@ function tokenise(string) {
         {name: "close-square-bracket", regex: /\]/},
         {name: "string", regex: /'([^']*)'/},
         {name: "integer", regex: /([0-9]+)/},
-        {name: "choice", regex: /\|/}
+        {name: "choice", regex: /\|/},
+        {name: "bang", regex: /(!)/}
     ]);
     return tokeniser.tokenise(string);
 }