diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index e9d367f9ea95aa..835b8eda1cabb3 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -5363,6 +5363,55 @@ def test_priming_unpickler_memo(self): self.assertEqual(unpickled_data2, data) self.assertTrue(unpickled_data2 is unpickled_data1) + def test_assign_unpickler_memo_dict(self): + value = [] + for index in (0, 1, 1000): + with self.subTest(index=index): + unpickler = self.unpickler_class( + io.BytesIO(f'g{index}\n.'.encode())) + memo = {index: value} + unpickler.memo = memo + self.assertIs(unpickler.load(), value) + + def test_assign_unpickler_memo_continuation(self): + for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): + for copy_memo in (False, True): + with self.subTest(proto=proto, copy_memo=copy_memo): + stream = io.BytesIO() + pickler = self.pickler_class(stream, protocol=proto) + pickler.dump(['first']) + value = ['second'] + pickler.dump([value, value]) + stream.seek(0) + unpickler = self.unpickler_class(stream) + unpickler.load() + primed = self.unpickler_class(stream) + memo = unpickler.memo + primed.memo = memo.copy() if copy_memo else memo + result = primed.load() + self.assertEqual(result, [value, value]) + self.assertIs(result[0], result[1]) + + def test_assign_unpickler_memo_sparse(self): + source = self.unpickler_class(io.BytesIO(b']p1000\n.')) + value = source.load() + unpickler = self.unpickler_class(io.BytesIO(b'g1000\n.')) + unpickler.memo = source.memo + self.assertIs(unpickler.load(), value) + + def test_assign_empty_unpickler_memo(self): + unpickler = self.unpickler_class(io.BytesIO(b']p1000\n.g1000\n.')) + unpickler.load() + unpickler.memo = {} + with self.assertRaises(pickle.UnpicklingError): + unpickler.load() + + def test_assign_unpickler_own_memo(self): + unpickler = self.unpickler_class(io.BytesIO(b']q\x00.h\x00.')) + value = unpickler.load() + unpickler.memo = unpickler.memo + self.assertIs(unpickler.load(), value) + def test_reusing_unpickler_objects(self): data1 = ["abcdefg", "abcdefg", 44] f = io.BytesIO() diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 315aac3bf7a921..b990a2002c9e23 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -470,6 +470,42 @@ def test_issue18339(self): unpickler.memo = {-1: None} unpickler.memo = {1: None} + def test_invalid_unpickler_memo_preserves_entries(self): + for key, error in (('invalid', TypeError), (-1, ValueError), + (sys.maxsize + 1, OverflowError)): + with self.subTest(key=key): + unpickler = self.unpickler_class(io.BytesIO(b']q\x00.h\x00.')) + value = unpickler.load() + with self.assertRaises(error): + unpickler.memo = {0: None, key: None} + self.assertIs(unpickler.load(), value) + + def test_assign_unpickler_memo_copies_dict(self): + for index in (0, 1, 1000): + with self.subTest(index=index): + unpickler = self.unpickler_class( + io.BytesIO(f'g{index}\n.'.encode())) + value = [] + memo = {index: value} + unpickler.memo = memo + memo.clear() + self.assertEqual(unpickler.memo.copy(), {index: value}) + self.assertIs(unpickler.load(), value) + + def test_assign_unpickler_memo_reentrant(self): + unpickler = self.unpickler_class(io.BytesIO(b'h\x00.')) + observed = [] + + class Observer: + def __del__(self): + observed.append(unpickler.memo.copy()) + + unpickler.memo = {0: Observer()} + value = [] + unpickler.memo = {0: value} + self.assertEqual(observed, [{0: value}]) + self.assertIs(unpickler.load(), value) + def test_concurrent_pickler_dump(self): f = io.BytesIO() pickler = self.pickler_class(f) diff --git a/Misc/NEWS.d/next/Library/2026-11-20-12-00-00.gh-issue-111066.memo.rst b/Misc/NEWS.d/next/Library/2026-11-20-12-00-00.gh-issue-111066.memo.rst new file mode 100644 index 00000000000000..ec3b7168feedcb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-20-12-00-00.gh-issue-111066.memo.rst @@ -0,0 +1,3 @@ +Fix assigning a dictionary or another unpickler's memo to +:class:`pickle.Unpickler`. Preserve the memo entries and the index used for +subsequent memoization. diff --git a/Modules/_pickle.c b/Modules/_pickle.c index b8f9b1c92f2072..82a7e2556a8c25 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -7940,7 +7940,8 @@ _pickle_UnpicklerMemoProxy_copy_impl(UnpicklerMemoProxyObject *self) /*[clinic end generated code: output=e12af7e9bc1e4c77 input=97769247ce032c1d]*/ { size_t i; - PyObject *new_memo = PyDict_New(); + PyObject *new_memo = self->unpickler->memo_dict == NULL + ? PyDict_New() : PyDict_Copy(self->unpickler->memo_dict); if (new_memo == NULL) return NULL; @@ -8074,6 +8075,8 @@ Unpickler_set_memo(PyObject *op, PyObject *obj, void *Py_UNUSED(closure)) PyObject **new_memo; UnpicklerObject *self = UnpicklerObject_CAST(op); size_t new_memo_size = 0; + size_t new_memo_len = 0; + PyObject *new_memo_dict = NULL; if (obj == NULL) { PyErr_SetString(PyExc_TypeError, @@ -8091,6 +8094,13 @@ Unpickler_set_memo(PyObject *op, PyObject *obj, void *Py_UNUSED(closure)) if (new_memo == NULL) return -1; + new_memo_len = unpickler->memo_len; + if (unpickler->memo_dict != NULL) { + new_memo_dict = PyDict_Copy(unpickler->memo_dict); + if (new_memo_dict == NULL) + goto error; + } + for (size_t i = 0; i < new_memo_size; i++) { new_memo[i] = Py_XNewRef(unpickler->memo[i]); } @@ -8100,6 +8110,7 @@ Unpickler_set_memo(PyObject *op, PyObject *obj, void *Py_UNUSED(closure)) PyObject *key, *value; new_memo_size = PyDict_GET_SIZE(obj); + new_memo_len = new_memo_size; new_memo = _Unpickler_NewMemo(new_memo_size); if (new_memo == NULL) return -1; @@ -8119,8 +8130,23 @@ Unpickler_set_memo(PyObject *op, PyObject *obj, void *Py_UNUSED(closure)) "memo key must be positive integers."); goto error; } - if (_Unpickler_MemoPut(self, idx, value) < 0) - goto error; + if ((size_t)idx < new_memo_size) { + Py_XSETREF(new_memo[idx], Py_NewRef(value)); + } + else { + if (new_memo_dict == NULL) { + new_memo_dict = PyDict_New(); + if (new_memo_dict == NULL) + goto error; + } + PyObject *index = PyLong_FromSsize_t(idx); + if (index == NULL) + goto error; + int status = PyDict_SetItem(new_memo_dict, index, value); + Py_DECREF(index); + if (status < 0) + goto error; + } } } else { @@ -8130,19 +8156,28 @@ Unpickler_set_memo(PyObject *op, PyObject *obj, void *Py_UNUSED(closure)) return -1; } - _Unpickler_MemoCleanup(self, NULL); - self->memo_size = new_memo_size; + /* Install all of the new state before releasing the old entries, which + may call back into the unpickler. */ + PyObject **old_memo = self->memo; + size_t old_memo_size = self->memo_size; + PyObject *old_memo_dict = self->memo_dict; self->memo = new_memo; - + self->memo_size = new_memo_size; + self->memo_len = new_memo_len; + self->memo_dict = new_memo_dict; + for (size_t i = 0; i < old_memo_size; i++) { + Py_XDECREF(old_memo[i]); + } + PyMem_Free(old_memo); + Py_XDECREF(old_memo_dict); return 0; error: - if (new_memo_size) { - for (size_t i = new_memo_size - 1; i != SIZE_MAX; i--) { - Py_XDECREF(new_memo[i]); - } - PyMem_Free(new_memo); + for (size_t i = 0; i < new_memo_size; i++) { + Py_XDECREF(new_memo[i]); } + PyMem_Free(new_memo); + Py_XDECREF(new_memo_dict); return -1; }