diff --git a/.repo-metadata.json b/.repo-metadata.json index a6e99c74..bb0b0f08 100644 --- a/.repo-metadata.json +++ b/.repo-metadata.json @@ -9,6 +9,6 @@ "repo": "googleapis/python-ndb", "distribution_name": "google-cloud-ndb", "default_version": "", - "codeowner_team": "@googleapis/firestore-dpe @googleapis/cloud-storage-dpe", + "codeowner_team": "@googleapis/firestore-dpe @googleapis/gcs-sdk-team", "api_shortname": "datastore" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 02865039..927bcf49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-cloud-ndb/#history +## [2.3.3](https://github.com/googleapis/python-ndb/compare/v2.3.2...v2.3.3) (2025-05-09) + + +### Bug Fixes + +* Support sub-meanings for datastore v2.20.3 ([#1014](https://github.com/googleapis/python-ndb/issues/1014)) ([88f14fa](https://github.com/googleapis/python-ndb/commit/88f14fa462b7f7caf72688374682bb1b7a2d933c)) + ## [2.3.2](https://github.com/googleapis/python-ndb/compare/v2.3.1...v2.3.2) (2024-07-15) diff --git a/google/cloud/ndb/key.py b/google/cloud/ndb/key.py index 3c3af888..b168e55a 100644 --- a/google/cloud/ndb/key.py +++ b/google/cloud/ndb/key.py @@ -716,13 +716,13 @@ def reference(self): >>> key = ndb.Key("Trampoline", 88, project="xy", database="wv", namespace="zt") >>> key.reference() app: "xy" - name_space: "zt" path { element { type: "Trampoline" id: 88 } } + name_space: "zt" database_id: "wv" """ diff --git a/google/cloud/ndb/model.py b/google/cloud/ndb/model.py index acfd10a8..c4d3cdb6 100644 --- a/google/cloud/ndb/model.py +++ b/google/cloud/ndb/model.py @@ -2698,14 +2698,26 @@ def _from_datastore(self, ds_entity, value): Need to check the ds_entity for a compressed meaning that would indicate we are getting a compressed value. """ - if self._name in ds_entity._meanings: - meaning = ds_entity._meanings[self._name][0] - if meaning == _MEANING_COMPRESSED and not self._compressed: - if self._repeated: - for sub_value in value: - sub_value.b_val = zlib.decompress(sub_value.b_val) - else: - value.b_val = zlib.decompress(value.b_val) + if self._name in ds_entity._meanings and not self._compressed: + root_meaning = ds_entity._meanings[self._name][0] + sub_meanings = None + # meaning may be a tuple. Attempt unwrap + if isinstance(root_meaning, tuple): + root_meaning, sub_meanings = root_meaning + # decompress values if needed + if root_meaning == _MEANING_COMPRESSED and not self._repeated: + value.b_val = zlib.decompress(value.b_val) + elif root_meaning == _MEANING_COMPRESSED and self._repeated: + for sub_value in value: + sub_value.b_val = zlib.decompress(sub_value.b_val) + elif isinstance(sub_meanings, list) and self._repeated: + for idx, sub_value in enumerate(value): + try: + if sub_meanings[idx] == _MEANING_COMPRESSED: + sub_value.b_val = zlib.decompress(sub_value.b_val) + except IndexError: + # value list size exceeds sub_meanings list + break return value def _db_set_compressed_meaning(self, p): diff --git a/google/cloud/ndb/version.py b/google/cloud/ndb/version.py index 871b248f..fcf3ff30 100644 --- a/google/cloud/ndb/version.py +++ b/google/cloud/ndb/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "2.3.2" +__version__ = "2.3.3" diff --git a/setup.py b/setup.py index 2bee63fb..fd6e94e2 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ def main(): readme = readme_file.read() dependencies = [ "google-api-core[grpc] >= 1.34.0, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*", - "google-cloud-datastore >= 2.16.0, < 3.0.0dev", + "google-cloud-datastore >= 2.16.0, != 2.20.2, < 3.0.0dev", "protobuf >= 3.20.2, <6.0.0dev,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5", "pymemcache >= 2.1.0, < 5.0.0dev", "pytz >= 2018.3", diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py index 14f03cef..b642aa3b 100644 --- a/tests/unit/test_model.py +++ b/tests/unit/test_model.py @@ -1929,9 +1929,8 @@ class ThisKind(model.Model): ds_entity = model._entity_to_ds_entity(entity) assert ds_entity["foo"] == compressed_value - @staticmethod @pytest.mark.usefixtures("in_context") - def test__from_datastore_compressed_repeated_to_compressed(): + def test__from_datastore_compressed_repeated_to_compressed(self): class ThisKind(model.Model): foo = model.BlobProperty(compressed=True, repeated=True) @@ -1955,9 +1954,48 @@ class ThisKind(model.Model): ds_entity = model._entity_to_ds_entity(entity) assert ds_entity["foo"] == [compressed_value_one, compressed_value_two] - @staticmethod + @pytest.mark.skipif( + [int(v) for v in datastore.__version__.split(".")] < [2, 20, 2], + reason="uses meanings semantics from datastore v2.20.2 and later", + ) + @pytest.mark.parametrize( + "meaning", + [ + (model._MEANING_COMPRESSED, None), # set root meaning + (model._MEANING_COMPRESSED, []), + (model._MEANING_COMPRESSED, [1, 1]), + (None, [model._MEANING_COMPRESSED] * 2), # set sub-meanings + ], + ) @pytest.mark.usefixtures("in_context") - def test__from_datastore_compressed_repeated_to_uncompressed(): + def test__from_datastore_compressed_repeated_to_compressed_tuple_meaning( + self, meaning + ): + class ThisKind(model.Model): + foo = model.BlobProperty(compressed=True, repeated=True) + + key = datastore.Key("ThisKind", 123, project="testing") + datastore_entity = datastore.Entity(key=key) + uncompressed_value_one = b"abc" * 1000 + compressed_value_one = zlib.compress(uncompressed_value_one) + uncompressed_value_two = b"xyz" * 1000 + compressed_value_two = zlib.compress(uncompressed_value_two) + compressed_value = [compressed_value_one, compressed_value_two] + datastore_entity.update({"foo": compressed_value}) + meanings = { + "foo": ( + meaning, + compressed_value, + ) + } + datastore_entity._meanings = meanings + protobuf = helpers.entity_to_protobuf(datastore_entity) + entity = model._entity_from_protobuf(protobuf) + ds_entity = model._entity_to_ds_entity(entity) + assert ds_entity["foo"] == [compressed_value_one, compressed_value_two] + + @pytest.mark.usefixtures("in_context") + def test__from_datastore_compressed_repeated_to_uncompressed(self): class ThisKind(model.Model): foo = model.BlobProperty(compressed=False, repeated=True) @@ -1981,6 +2019,170 @@ class ThisKind(model.Model): ds_entity = model._entity_to_ds_entity(entity) assert ds_entity["foo"] == [uncompressed_value_one, uncompressed_value_two] + @pytest.mark.skipif( + [int(v) for v in datastore.__version__.split(".")] < [2, 20, 2], + reason="uses meanings semantics from datastore v2.20.2 and later", + ) + @pytest.mark.parametrize( + "meaning", + [ + (model._MEANING_COMPRESSED, None), # set root meaning + (model._MEANING_COMPRESSED, []), + (model._MEANING_COMPRESSED, [1, 1]), + (None, [model._MEANING_COMPRESSED] * 2), # set sub-meanings + ], + ) + @pytest.mark.usefixtures("in_context") + def test__from_datastore_compressed_repeated_to_uncompressed_tuple_meaning( + self, meaning + ): + class ThisKind(model.Model): + foo = model.BlobProperty(compressed=False, repeated=True) + + key = datastore.Key("ThisKind", 123, project="testing") + datastore_entity = datastore.Entity(key=key) + uncompressed_value_one = b"abc" * 1000 + compressed_value_one = zlib.compress(uncompressed_value_one) + uncompressed_value_two = b"xyz" * 1000 + compressed_value_two = zlib.compress(uncompressed_value_two) + compressed_value = [compressed_value_one, compressed_value_two] + datastore_entity.update({"foo": compressed_value}) + meanings = { + "foo": ( + meaning, + compressed_value, + ) + } + datastore_entity._meanings = meanings + protobuf = helpers.entity_to_protobuf(datastore_entity) + entity = model._entity_from_protobuf(protobuf) + ds_entity = model._entity_to_ds_entity(entity) + assert ds_entity["foo"] == [uncompressed_value_one, uncompressed_value_two] + + @pytest.mark.skipif( + [int(v) for v in datastore.__version__.split(".")] < [2, 20, 2], + reason="uses meanings semantics from datastore v2.20.2 and later", + ) + @pytest.mark.parametrize( + "meaning", + [ + (None, [model._MEANING_COMPRESSED, None]), + (None, [model._MEANING_COMPRESSED, None, None]), + (1, [model._MEANING_COMPRESSED, 1]), + (None, [model._MEANING_COMPRESSED]), + ], + ) + @pytest.mark.usefixtures("in_context") + def test__from_datastore_compressed_repeated_to_uncompressed_mixed_meaning( + self, meaning + ): + """ + One item is compressed, one uncompressed + """ + + class ThisKind(model.Model): + foo = model.BlobProperty(compressed=False, repeated=True) + + key = datastore.Key("ThisKind", 123, project="testing") + datastore_entity = datastore.Entity(key=key) + uncompressed_value_one = b"abc" * 1000 + compressed_value_one = zlib.compress(uncompressed_value_one) + uncompressed_value_two = b"xyz" * 1000 + compressed_value_two = zlib.compress(uncompressed_value_two) + compressed_value = [compressed_value_one, compressed_value_two] + datastore_entity.update({"foo": compressed_value}) + meanings = { + "foo": ( + meaning, + compressed_value, + ) + } + datastore_entity._meanings = meanings + protobuf = helpers.entity_to_protobuf(datastore_entity) + entity = model._entity_from_protobuf(protobuf) + ds_entity = model._entity_to_ds_entity(entity) + assert ds_entity["foo"] == [uncompressed_value_one, compressed_value_two] + + @pytest.mark.skipif( + [int(v) for v in datastore.__version__.split(".")] < [2, 20, 2], + reason="uses meanings semantics from datastore v2.20.2 and later", + ) + @pytest.mark.parametrize( + "meaning", + [ + (None, None), + (None, []), + (None, [None]), + (None, [None, None]), + (1, []), + (1, [1]), + (1, [1, 1]), + ], + ) + @pytest.mark.usefixtures("in_context") + def test__from_datastore_compressed_repeated_no_meaning(self, meaning): + """ + could be uncompressed, but meaning not set + """ + + class ThisKind(model.Model): + foo = model.BlobProperty(compressed=False, repeated=True) + + key = datastore.Key("ThisKind", 123, project="testing") + datastore_entity = datastore.Entity(key=key) + uncompressed_value_one = b"abc" * 1000 + compressed_value_one = zlib.compress(uncompressed_value_one) + uncompressed_value_two = b"xyz" * 1000 + compressed_value_two = zlib.compress(uncompressed_value_two) + compressed_value = [compressed_value_one, compressed_value_two] + datastore_entity.update({"foo": compressed_value}) + meanings = { + "foo": ( + meaning, + compressed_value, + ) + } + datastore_entity._meanings = meanings + protobuf = helpers.entity_to_protobuf(datastore_entity) + entity = model._entity_from_protobuf(protobuf) + ds_entity = model._entity_to_ds_entity(entity) + assert ds_entity["foo"] == [compressed_value_one, compressed_value_two] + + @staticmethod + @pytest.mark.usefixtures("in_context") + def test__from_datastore_large_value_list(): + """ + try calling _from_datastore with a meaning list smaller than the value list + """ + + prop = model.BlobProperty(compressed=False, repeated=True, name="foo") + + key = datastore.Key("ThisKind", 123, project="testing") + datastore_entity = datastore.Entity(key=key) + uncompressed_value_one = b"abc" * 1000 + compressed_value_one = zlib.compress(uncompressed_value_one) + uncompressed_value_two = b"xyz" * 1000 + compressed_value_two = zlib.compress(uncompressed_value_two) + compressed_value = [ + model._BaseValue(compressed_value_one), + model._BaseValue(compressed_value_two), + ] + datastore_entity.update({"foo": compressed_value}) + meanings = { + "foo": ( + (None, [model._MEANING_COMPRESSED]), + compressed_value, + ) + } + + datastore_entity._meanings = meanings + + updated_value = prop._from_datastore(datastore_entity, compressed_value) + assert len(updated_value) == 2 + assert updated_value[0].b_val == uncompressed_value_one + # second value should remain compressed + assert updated_value[1].b_val == compressed_value_two + @staticmethod @pytest.mark.usefixtures("in_context") def test__from_datastore_uncompressed_to_uncompressed():