From 306e43dc49aadf0151428a13c2efcd97a9a024da Mon Sep 17 00:00:00 2001 From: furkanonder Date: Sun, 17 Aug 2025 17:48:55 +0300 Subject: [PATCH 1/8] Fix shelve tests for backend compatibility --- Lib/test/test_shelve.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 64609ab9dd9a62..35544a98795241 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -173,6 +173,8 @@ def test_custom_serializer_and_deserializer(self): def serializer(obj, protocol): if isinstance(obj, (bytes, bytearray, str)): if protocol == 5: + if isinstance(obj, bytearray): + return bytes(obj) return obj return type(obj).__name__ elif isinstance(obj, array.array): @@ -223,11 +225,10 @@ def deserializer(data): ) def test_custom_incomplete_serializer_and_deserializer(self): - dbm_sqlite3 = import_helper.import_module("dbm.sqlite3") os.mkdir(self.dirname) self.addCleanup(os_helper.rmtree, self.dirname) - with self.assertRaises(dbm_sqlite3.error): + with self.assertRaises((TypeError, dbm.error)): def serializer(obj, protocol=None): pass From 79e335ebe7ccd5dec0c1d0076d06e6f1d36ad973 Mon Sep 17 00:00:00 2001 From: furkanonder Date: Mon, 18 Aug 2025 23:10:47 +0300 Subject: [PATCH 2/8] split incomplete serializer/deserializer in test_shelve --- Lib/test/test_shelve.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 35544a98795241..bb589e544c269d 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -224,21 +224,25 @@ def deserializer(data): s["array_data"], array_data.tobytes().decode() ) - def test_custom_incomplete_serializer_and_deserializer(self): + def test_custom_incomplete_serializer(self): os.mkdir(self.dirname) self.addCleanup(os_helper.rmtree, self.dirname) - with self.assertRaises((TypeError, dbm.error)): - def serializer(obj, protocol=None): - pass + def serializer(obj, protocol=None): + pass - def deserializer(data): - return data.decode("utf-8") + def deserializer(data): + return data.decode("utf-8") + with self.assertRaises((TypeError, dbm.error)): with shelve.open(self.fn, serializer=serializer, deserializer=deserializer) as s: s["foo"] = "bar" + def test_custom_incomplete_deserializer(self): + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + def serializer(obj, protocol=None): return type(obj).__name__.encode("utf-8") @@ -353,7 +357,7 @@ def type_name_len(obj): self.assertEqual(s["bytearray_data"], "bytearray") self.assertEqual(s["array_data"], "array") - def test_custom_incomplete_serializer_and_deserializer_bsd_db_shelf(self): + def test_custom_incomplete_deserializer_bsd_db_shelf(self): berkeleydb = import_helper.import_module("berkeleydb") os.mkdir(self.dirname) self.addCleanup(os_helper.rmtree, self.dirname) @@ -371,6 +375,11 @@ def deserializer(data): self.assertIsNone(s["foo"]) self.assertNotEqual(s["foo"], "bar") + def test_custom_incomplete_serializer_bsd_db_shelf(self): + berkeleydb = import_helper.import_module("berkeleydb") + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + def serializer(obj, protocol=None): pass From 5b9f61b4ff3547e7c56b92233954de39ea76f4aa Mon Sep 17 00:00:00 2001 From: furkanonder Date: Tue, 19 Aug 2025 02:03:17 +0300 Subject: [PATCH 3/8] add a test case for custom serializer that returning wrong type --- Lib/test/test_shelve.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index bb589e544c269d..760bb5096bef2d 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -409,6 +409,28 @@ def deserializer(data): self.assertRaises(shelve.ShelveError, shelve.Shelf, {}, **kwargs) self.assertRaises(shelve.ShelveError, shelve.BsdDbShelf, {}, **kwargs) + def test_custom_serializer_returns_wrong_type_for_key(self): + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + + def serializer(obj, protocol): + # Return None instead of bytes, which is wrong for dbm keys + return None + + def deserializer(data): + return data.decode("utf-8") if data else "" + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto), shelve.open( + self.fn, + protocol=proto, + serializer=serializer, + deserializer=deserializer + ) as s: + # Serializer returns None for the value, but dbm expects bytes + with self.assertRaises((TypeError, dbm.error)): + s["foo"] = "bar" + class TestShelveBase: type2test = shelve.Shelf From 2e3ea9bf98f78c36461c573614b942d7afd0abbf Mon Sep 17 00:00:00 2001 From: furkanonder Date: Fri, 29 Aug 2025 15:50:19 +0300 Subject: [PATCH 4/8] Add comments clarifying exceptions when serializer returns None --- Lib/test/test_shelve.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 760bb5096bef2d..4ba96138ac4938 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -174,7 +174,7 @@ def serializer(obj, protocol): if isinstance(obj, (bytes, bytearray, str)): if protocol == 5: if isinstance(obj, bytearray): - return bytes(obj) + return bytes(obj) # DBM backends expect bytes return obj return type(obj).__name__ elif isinstance(obj, array.array): @@ -234,6 +234,8 @@ def serializer(obj, protocol=None): def deserializer(data): return data.decode("utf-8") + # Since the serializer returns None, dbm.error is raised + # by dbm.sqlite3 and TypeError is raised by other backends. with self.assertRaises((TypeError, dbm.error)): with shelve.open(self.fn, serializer=serializer, deserializer=deserializer) as s: @@ -427,7 +429,8 @@ def deserializer(data): serializer=serializer, deserializer=deserializer ) as s: - # Serializer returns None for the value, but dbm expects bytes + # Since the serializer returns None, dbm.error is raised + # by dbm.sqlite3 and TypeError is raised by other backends. with self.assertRaises((TypeError, dbm.error)): s["foo"] = "bar" From e1b0584acdd9f80d31693b3764d7c0f6e27cdbbf Mon Sep 17 00:00:00 2001 From: furkanonder Date: Sat, 30 Aug 2025 10:06:44 +0300 Subject: [PATCH 5/8] Remove test case --- Lib/test/test_shelve.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 4ba96138ac4938..585098054a7653 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -411,29 +411,6 @@ def deserializer(data): self.assertRaises(shelve.ShelveError, shelve.Shelf, {}, **kwargs) self.assertRaises(shelve.ShelveError, shelve.BsdDbShelf, {}, **kwargs) - def test_custom_serializer_returns_wrong_type_for_key(self): - os.mkdir(self.dirname) - self.addCleanup(os_helper.rmtree, self.dirname) - - def serializer(obj, protocol): - # Return None instead of bytes, which is wrong for dbm keys - return None - - def deserializer(data): - return data.decode("utf-8") if data else "" - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.subTest(proto=proto), shelve.open( - self.fn, - protocol=proto, - serializer=serializer, - deserializer=deserializer - ) as s: - # Since the serializer returns None, dbm.error is raised - # by dbm.sqlite3 and TypeError is raised by other backends. - with self.assertRaises((TypeError, dbm.error)): - s["foo"] = "bar" - class TestShelveBase: type2test = shelve.Shelf From f0af214772a906cded0ae32ab815ef5ed5ea104f Mon Sep 17 00:00:00 2001 From: furkanonder Date: Mon, 8 Sep 2025 23:55:52 +0300 Subject: [PATCH 6/8] Add test for invalid custom serializer --- Lib/test/test_shelve.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 585098054a7653..2f448ee46f2277 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -241,6 +241,23 @@ def deserializer(data): deserializer=deserializer) as s: s["foo"] = "bar" + def test_custom_invalid_serializer(self): + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + + def serializer(obj, protocol=None): + return ["value with invalid type"] + + def deserializer(data): + return data.decode("utf-8") + + # Since the serializer returns None, dbm.error is raised + # by dbm.sqlite3 and TypeError is raised by other backends. + with self.assertRaises((TypeError, dbm.error)): + with shelve.open(self.fn, serializer=serializer, + deserializer=deserializer) as s: + s["foo"] = "bar" + def test_custom_incomplete_deserializer(self): os.mkdir(self.dirname) self.addCleanup(os_helper.rmtree, self.dirname) From e64777ba435bf2c4871619d8a96db359a7867417 Mon Sep 17 00:00:00 2001 From: furkanonder Date: Fri, 19 Sep 2025 20:32:46 +0300 Subject: [PATCH 7/8] Add parameterized test for invalid custom serializers --- Lib/test/test_shelve.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 2f448ee46f2277..071195ee7891e2 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -5,7 +5,7 @@ import pickle import os -from test.support import import_helper, os_helper +from test.support import import_helper, os_helper, subTests from collections.abc import MutableMapping from test.test_dbm import dbm_iterator @@ -241,20 +241,24 @@ def deserializer(data): deserializer=deserializer) as s: s["foo"] = "bar" - def test_custom_invalid_serializer(self): - os.mkdir(self.dirname) - self.addCleanup(os_helper.rmtree, self.dirname) + @subTests("serialized", [None, ["invalid type"]]) + def test_custom_invalid_serializer(self, serialized): + test_dir = f"{self.dirname}_{id(serialized)}" + os.mkdir(test_dir) + self.addCleanup(os_helper.rmtree, test_dir) + test_fn = os.path.join(test_dir, "shelftemp.db") def serializer(obj, protocol=None): - return ["value with invalid type"] + return serialized def deserializer(data): return data.decode("utf-8") - # Since the serializer returns None, dbm.error is raised - # by dbm.sqlite3 and TypeError is raised by other backends. + # Since the serializer returns an invalid type or None, + # dbm.error is raised by dbm.sqlite3 and TypeError is raised + # by other backends. with self.assertRaises((TypeError, dbm.error)): - with shelve.open(self.fn, serializer=serializer, + with shelve.open(test_fn, serializer=serializer, deserializer=deserializer) as s: s["foo"] = "bar" From c683173d8e55eafde1532fa6474adc34b6c5288f Mon Sep 17 00:00:00 2001 From: furkanonder Date: Fri, 19 Sep 2025 20:36:47 +0300 Subject: [PATCH 8/8] Remove test_custom_incomplete_serializer --- Lib/test/test_shelve.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 071195ee7891e2..5f6a030e018f96 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -224,23 +224,6 @@ def deserializer(data): s["array_data"], array_data.tobytes().decode() ) - def test_custom_incomplete_serializer(self): - os.mkdir(self.dirname) - self.addCleanup(os_helper.rmtree, self.dirname) - - def serializer(obj, protocol=None): - pass - - def deserializer(data): - return data.decode("utf-8") - - # Since the serializer returns None, dbm.error is raised - # by dbm.sqlite3 and TypeError is raised by other backends. - with self.assertRaises((TypeError, dbm.error)): - with shelve.open(self.fn, serializer=serializer, - deserializer=deserializer) as s: - s["foo"] = "bar" - @subTests("serialized", [None, ["invalid type"]]) def test_custom_invalid_serializer(self, serialized): test_dir = f"{self.dirname}_{id(serialized)}"