From 52564dd2ba555b26048f54a1f3e9708fedefbb42 Mon Sep 17 00:00:00 2001 From: Nikolai Kondrashov Date: Wed, 9 Sep 2020 13:10:39 +0300 Subject: Add tests for data preservation Add tests verifying various value types and combinations are preserved after passing through the program. --- tests/jq_tests.py | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 1 deletion(-) diff --git a/tests/jq_tests.py b/tests/jq_tests.py index 27734e6..6b63556 100644 --- a/tests/jq_tests.py +++ b/tests/jq_tests.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals -from nose.tools import istest, assert_equal, assert_raises +from nose.tools import istest, assert_equal, assert_is, assert_raises import jq @@ -197,6 +197,107 @@ def program_string_can_be_retrieved_from_program(): assert_equal(".", program.program_string) +@istest +def program_preserves_null(): + program = jq.compile(".") + assert_is(None, program.input(text='null').first()) + + +@istest +def program_preserves_bools(): + program = jq.compile(".") + assert_is(False, program.input(text="false").first()) + assert_is(True, program.input(text="true").first()) + + +@istest +def program_preserves_ints(): + program = jq.compile(".") + assert_equal(int, type(program.input(text="0").first())) + assert_equal(0, program.input(text="0").first()) + assert_equal(1, program.input(text="1").first()) + assert_equal(-1, program.input(text="-1").first()) + assert_equal(12345, program.input(text="12345").first()) + assert_equal(-12345, program.input(text="-12345").first()) + + +@istest +def program_preserves_floats(): + program = jq.compile(".") + assert_equal(float, type(program.input(text="1.1").first())) + assert_equal(3.14159, program.input(text="3.14159").first()) + assert_equal(-3.14159, program.input(text="-3.14159").first()) + assert_equal(42E100, program.input(text="42E100").first()) + assert_equal(-42E100, program.input(text="-42E100").first()) + + +@istest +def program_preserves_strings(): + program = jq.compile(".") + assert_equal(type(""), type(program.input(text='""').first())) + assert_equal("", program.input(text='""').first()) + assert_equal("x", program.input(text='"x"').first()) + assert_equal("xyz", program.input(text='"xyz"').first()) + + +@istest +def program_preserves_arrays(): + program = jq.compile(".") + + assert_equal(list, type(program.input(text='[]').first())) + + assert_equal([], program.input(text='[]').first()) + + assert_equal([1], program.input(text='[1]').first()) + assert_equal([1, 2], program.input(text='[1, 2]').first()) + + assert_equal([3.14159], program.input(text='[3.14159]').first()) + assert_equal([3.14159, 95141.3], program.input(text='[3.14159, 95141.3]').first()) + + assert_equal([False], program.input(text='[false]').first()) + assert_equal([False, True], program.input(text='[false, true]').first()) + + assert_equal([[]], program.input(text='[[]]').first()) + assert_equal([[[]]], program.input(text='[[[]]]').first()) + assert_equal([[], []], program.input(text='[[], []]').first()) + assert_equal([[[], []], [[], []]], program.input(text='[[[], []], [[], []]]').first()) + + assert_equal([{}], program.input(text='[{}]').first()) + assert_equal([{"": []}], program.input(text='[{"": []}]').first()) + assert_equal([{"1": [], "2": []}], program.input(text='[{"1": [], "2": []}]').first()) + + +@istest +def program_preserves_objects(): + program = jq.compile(".") + + assert_equal(dict, type(program.input(text='{}').first())) + + assert_equal({}, program.input(text='{}').first()) + + assert_equal({"": True}, program.input(text='{"": true}').first()) + assert_equal({"": False}, program.input(text='{"": false}').first()) + + assert_equal({"": 0}, program.input(text='{"": 0}').first()) + assert_equal({"": 1}, program.input(text='{"": 1}').first()) + assert_equal({"a": 0, "b": 1}, program.input(text='{"a": 0, "b": 1}').first()) + assert_equal({"": 3.14159}, program.input(text='{"": 3.14159}').first()) + assert_equal({"a": 3.14159, "b": 95141.3}, + program.input(text='{"a": 3.14159, "b": 95141.3}').first()) + + assert_equal({"": ""}, program.input(text='{"": ""}').first()) + assert_equal({"": "x"}, program.input(text='{"": "x"}').first()) + + assert_equal({"": []}, program.input(text='{"": []}').first()) + assert_equal({"": [{}]}, program.input(text='{"": [{}]}').first()) + assert_equal({"a": [1], "b": [2]}, + program.input(text='{"a": [1], "b": [2]}').first()) + + assert_equal({"": {}}, program.input(text='{"": {}}').first()) + assert_equal({"a": {}, "b": {}}, program.input(text='{"a": {}, "b": {}}').first()) + assert_equal({"": {"": {}}}, program.input(text='{"": {"": {}}}').first()) + + @istest class ConvenienceFunctions(object): @istest -- cgit v1.2.3 From a24ce4ccc0ace9aa0397472f5f4889e5756d8381 Mon Sep 17 00:00:00 2001 From: Nikolai Kondrashov Date: Tue, 8 Sep 2020 15:57:14 +0300 Subject: Free invalid values Even though "message-less" invalid values don't technically need "freeing", doing so is supported by the library, and could be good for uniformity and future-proofing. --- jq.pyx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jq.pyx b/jq.pyx index bcfb457..083faa7 100644 --- a/jq.pyx +++ b/jq.pyx @@ -260,6 +260,7 @@ cdef class _ResultIterator(object): jv_free(error_message) raise ValueError(message) else: + jv_free(result) self._ready = False cdef bint _ready_next_input(self) except 1: @@ -274,6 +275,7 @@ cdef class _ResultIterator(object): jv_free(error_message) raise ValueError(u"parse error: " + message) else: + jv_free(value) raise StopIteration() -- cgit v1.2.3 From 7f6137b8602393ebbe6c23b00a482b2917495d3f Mon Sep 17 00:00:00 2001 From: Nikolai Kondrashov Date: Tue, 8 Sep 2020 16:08:06 +0300 Subject: Unpack jv values into Python values directly Instead of dumping and parsing JSON, convert JQ's "jv" structures into Python values directly by recursively walking them. The naive implementation is still twice as fast. --- jq.pyx | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 57 insertions(+), 14 deletions(-) diff --git a/jq.pyx b/jq.pyx index 083faa7..cbb7b5d 100644 --- a/jq.pyx +++ b/jq.pyx @@ -26,6 +26,15 @@ cdef extern from "jv.h": int jv_invalid_has_msg(jv) char* jv_string_value(jv) jv jv_dump_string(jv, int flags) + int jv_is_integer(jv) + double jv_number_value(jv) + int jv_array_length(jv) + jv jv_array_get(jv, int) + int jv_object_iter(jv) + int jv_object_iter_next(jv, int) + int jv_object_iter_valid(jv, int) + jv jv_object_iter_key(jv, int) + jv jv_object_iter_value(jv, int) cdef struct jv_parser: pass @@ -51,6 +60,52 @@ cdef extern from "jq.h": void jq_get_error_cb(jq_state *, jq_err_cb *, void **) +cdef object _jv_to_python(jv value): + """Unpack a jv value into a Python value""" + cdef jv_kind kind = jv_get_kind(value) + cdef int idx + cdef jv property_key + cdef jv property_value + cdef object python_value + + if kind == JV_KIND_INVALID: + raise ValueError("Invalid value") + elif kind == JV_KIND_NULL: + python_value = None + elif kind == JV_KIND_FALSE: + python_value = False + elif kind == JV_KIND_TRUE: + python_value = True + elif kind == JV_KIND_NUMBER: + if jv_is_integer(value): + python_value = int(jv_number_value(value)) + else: + python_value = float(jv_number_value(value)) + elif kind == JV_KIND_STRING: + python_value = jv_string_value(value).decode("utf-8") + elif kind == JV_KIND_ARRAY: + python_value = [] + for idx in range(0, jv_array_length(jv_copy(value))): + property_value = jv_array_get(jv_copy(value), idx) + python_value.append(_jv_to_python(property_value)) + elif kind == JV_KIND_OBJECT: + python_value = {} + idx = jv_object_iter(value) + while jv_object_iter_valid(value, idx): + property_key = jv_object_iter_key(value, idx) + property_value = jv_object_iter_value(value, idx) + try: + python_value[jv_string_value(property_key).decode("utf-8")] = \ + _jv_to_python(property_value) + finally: + jv_free(property_key) + idx = jv_object_iter_next(value, idx) + else: + raise ValueError("Invalid value kind: " + str(kind)) + jv_free(value) + return python_value + + def compile(object program): cdef object program_bytes = program.encode("utf8") return _Program(program_bytes) @@ -199,13 +254,7 @@ cdef class _ProgramWithInput(object): return _ResultIterator(self._jq_state_pool, self._bytes_input) def text(self): - iterator = self._make_iterator() - results = [] - while True: - try: - results.append(iterator._next_string()) - except StopIteration: - return "\n".join(results) + return "\n".join(json.dumps(v) for v in self) def all(self): return list(self) @@ -239,9 +288,6 @@ cdef class _ResultIterator(object): return self def __next__(self): - return json.loads(self._next_string()) - - cdef unicode _next_string(self): cdef int dumpopts = 0 while True: if not self._ready: @@ -250,10 +296,7 @@ cdef class _ResultIterator(object): result = jq_next(self._jq) if jv_is_valid(result): - dumped = jv_dump_string(result, dumpopts) - value = jv_string_value(dumped).decode("utf8") - jv_free(dumped) - return value + return _jv_to_python(result) elif jv_invalid_has_msg(jv_copy(result)): error_message = jv_invalid_get_msg(result) message = jv_string_value(error_message).decode("utf8") -- cgit v1.2.3