From 1de9198f0c69152fe6f474185ac351e9500a4ce8 Mon Sep 17 00:00:00 2001 From: Michael Williamson Date: Thu, 24 Nov 2016 21:38:10 +0000 Subject: Implement document matcher parser --- mammoth/styles/parser/document_matcher_parser.py | 71 ++++++++++++++ mammoth/styles/parser/token_iterator.py | 31 ++++++ mammoth/styles/parser/token_parser.py | 37 +++++++ .../styles/parser/document_matcher_parser_tests.py | 109 +++++++++++++++++++++ tests/styles/parser/token_parser_tests.py | 50 ++++++++++ 5 files changed, 298 insertions(+) create mode 100644 mammoth/styles/parser/document_matcher_parser.py create mode 100644 mammoth/styles/parser/token_iterator.py create mode 100644 mammoth/styles/parser/token_parser.py create mode 100644 tests/styles/parser/document_matcher_parser_tests.py create mode 100644 tests/styles/parser/token_parser_tests.py diff --git a/mammoth/styles/parser/document_matcher_parser.py b/mammoth/styles/parser/document_matcher_parser.py new file mode 100644 index 0000000..77eae3e --- /dev/null +++ b/mammoth/styles/parser/document_matcher_parser.py @@ -0,0 +1,71 @@ +from ... import documents, document_matchers +from .tokeniser import TokenType +from .token_parser import try_parse_class_name, parse_string + + +def parse_document_matcher(tokens): + if tokens.try_skip(TokenType.IDENTIFIER, "p"): + style_id = try_parse_class_name(tokens) + style_name = _parse_style_name(tokens) + numbering = _parse_numbering(tokens) + + return document_matchers.paragraph( + style_id=style_id, + style_name=style_name, + numbering=numbering, + ) + + elif tokens.try_skip(TokenType.IDENTIFIER, "r"): + style_id = try_parse_class_name(tokens) + style_name = _parse_style_name(tokens) + + return document_matchers.run( + style_id=style_id, + style_name=style_name, + ) + + elif tokens.try_skip(TokenType.IDENTIFIER, "b"): + return document_matchers.bold + + elif tokens.try_skip(TokenType.IDENTIFIER, "i"): + return document_matchers.italic + + elif tokens.try_skip(TokenType.IDENTIFIER, "u"): + return document_matchers.underline + + elif tokens.try_skip(TokenType.IDENTIFIER, "strike"): + return document_matchers.strikethrough + + elif tokens.try_skip(TokenType.IDENTIFIER, "comment-reference"): + return document_matchers.comment_reference + + +def _parse_style_name(tokens): + if tokens.try_skip(TokenType.SYMBOL, "["): + tokens.skip(TokenType.IDENTIFIER, "style-name") + tokens.skip(TokenType.SYMBOL, "=") + name = parse_string(tokens) + tokens.skip(TokenType.SYMBOL, "]") + return name + else: + return None + + +def _parse_numbering(tokens): + if tokens.try_skip(TokenType.SYMBOL, ":"): + is_ordered = _parse_list_type(tokens) + tokens.skip(TokenType.SYMBOL, "(") + level = int(tokens.next_value(TokenType.INTEGER)) - 1 + tokens.skip(TokenType.SYMBOL, ")") + return documents.numbering_level(level, is_ordered=is_ordered) + + +def _parse_list_type(tokens): + list_type = tokens.next_value(TokenType.IDENTIFIER) + if list_type == "ordered-list": + return True + elif list_type == "unordered-list": + return False + else: + pass + # TODO: raise diff --git a/mammoth/styles/parser/token_iterator.py b/mammoth/styles/parser/token_iterator.py new file mode 100644 index 0000000..810e50b --- /dev/null +++ b/mammoth/styles/parser/token_iterator.py @@ -0,0 +1,31 @@ +class TokenIterator(object): + def __init__(self, tokens): + self._tokens = tokens + self._index = 0 + + def next_value(self, token_type): + return self._next(token_type).value + + def _next(self, token_type): + token = self._tokens[self._index] + if token.type == token_type: + self._index += 1 + return token + else: + raise self._unexpected_token_type(token_type, token) + + def skip(self, token_type, token_value): + token = self._tokens[self._index] + if token.type == token_type and token.value == token_value: + self._index += 1 + return True + else: + raise self._unexpected_token_type(token_type, token) + + def try_skip(self, token_type, token_value): + token = self._tokens[self._index] + if token.type == token_type and token.value == token_value: + self._index += 1 + return True + else: + return False diff --git a/mammoth/styles/parser/token_parser.py b/mammoth/styles/parser/token_parser.py new file mode 100644 index 0000000..495d790 --- /dev/null +++ b/mammoth/styles/parser/token_parser.py @@ -0,0 +1,37 @@ +import re + +from .tokeniser import TokenType + + +def try_parse_class_name(tokens): + if tokens.try_skip(TokenType.SYMBOL, "."): + return parse_identifier(tokens) + else: + return None + + +def parse_identifier(tokens): + return decode_escape_sequences(tokens.next_value(TokenType.IDENTIFIER)) + + +def parse_string(tokens): + return decode_escape_sequences(tokens.next_value(TokenType.STRING)[1:-1]) + + +_ESCAPE_SEQUENCE_REGEX = re.compile(r"\\(.)") + + +def decode_escape_sequences(value): + return _ESCAPE_SEQUENCE_REGEX.sub(_decode_escape_sequence, value) + + +def _decode_escape_sequence(match): + code = match.group(1) + if code == "n": + return "\n" + elif code == "r": + return "\r" + elif code == "t": + return "\t" + else: + return code diff --git a/tests/styles/parser/document_matcher_parser_tests.py b/tests/styles/parser/document_matcher_parser_tests.py new file mode 100644 index 0000000..18109ca --- /dev/null +++ b/tests/styles/parser/document_matcher_parser_tests.py @@ -0,0 +1,109 @@ +from nose.tools import istest, assert_equal + +from mammoth import documents, document_matchers +from mammoth.styles.parser.document_matcher_parser import parse_document_matcher +from mammoth.styles.parser.tokeniser import tokenise +from mammoth.styles.parser.token_iterator import TokenIterator + + +@istest +def reads_plain_paragraph(): + assert_equal( + document_matchers.paragraph(), + read_document_matcher("p") + ) + + +@istest +def reads_paragraph_with_style_id(): + assert_equal( + document_matchers.paragraph(style_id="Heading1"), + read_document_matcher("p.Heading1") + ) + + +@istest +def reads_paragraph_with_exact_style_name(): + assert_equal( + document_matchers.paragraph(style_name="Heading 1"), + read_document_matcher("p[style-name='Heading 1']") + ) + + +@istest +def reads_paragraph_ordered_list(): + assert_equal( + document_matchers.paragraph(numbering=documents.numbering_level(1, is_ordered=True)), + read_document_matcher("p:ordered-list(2)") + ) + + +@istest +def reads_paragraph_unordered_list(): + assert_equal( + document_matchers.paragraph(numbering=documents.numbering_level(1, is_ordered=False)), + read_document_matcher("p:unordered-list(2)") + ) + + +@istest +def reads_plain_run(): + assert_equal( + document_matchers.run(), + read_document_matcher("r") + ) + + +@istest +def reads_run_with_style_id(): + assert_equal( + document_matchers.run(style_id="Emphasis"), + read_document_matcher("r.Emphasis") + ) + + +@istest +def reads_run_with_style_name(): + assert_equal( + document_matchers.run(style_name="Emphasis"), + read_document_matcher("r[style-name='Emphasis']") + ) + +@istest +def reads_bold(): + assert_equal( + document_matchers.bold, + read_document_matcher("b") + ) + +@istest +def reads_italic(): + assert_equal( + document_matchers.italic, + read_document_matcher("i") + ) + +@istest +def reads_underline(): + assert_equal( + document_matchers.underline, + read_document_matcher("u") + ) + +@istest +def reads_strikethrough(): + assert_equal( + document_matchers.strikethrough, + read_document_matcher("strike") + ) + +@istest +def reads_comment_reference(): + assert_equal( + document_matchers.comment_reference, + read_document_matcher("comment-reference") + ) + + +def read_document_matcher(string): + return parse_document_matcher(TokenIterator(tokenise(string))) diff --git a/tests/styles/parser/token_parser_tests.py b/tests/styles/parser/token_parser_tests.py new file mode 100644 index 0000000..0e278c7 --- /dev/null +++ b/tests/styles/parser/token_parser_tests.py @@ -0,0 +1,50 @@ +from nose.tools import istest, assert_equal + +from mammoth.styles.parser.tokeniser import Token, TokenType +from mammoth.styles.parser.token_parser import decode_escape_sequences, parse_identifier, parse_string +from mammoth.styles.parser.token_iterator import TokenIterator + + +@istest +def escape_sequences_in_identifiers_are_decoded(): + assert_equal( + ":", + parse_identifier(TokenIterator([ + Token(0, TokenType.IDENTIFIER, r"\:"), + ])), + ) + + +@istest +def escape_sequences_in_strings_are_decoded(): + assert_equal( + "\n", + parse_string(TokenIterator([ + Token(0, TokenType.STRING, r"'\n'"), + ])), + ) + + +@istest +def line_feeds_are_decoded(): + assert_equal("\n", decode_escape_sequences(r"\n")) + + +@istest +def carriage_returns_are_decoded(): + assert_equal("\r", decode_escape_sequences(r"\r")) + + +@istest +def tabs_are_decoded(): + assert_equal("\t", decode_escape_sequences(r"\t")) + + +@istest +def backslashes_are_decoded(): + assert_equal("\\", decode_escape_sequences(r"\\")) + + +@istest +def colons_are_decoded(): + assert_equal(":", decode_escape_sequences(r"\:")) -- cgit v1.2.3