Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions Lib/test/pickletester.py
Original file line number Diff line number Diff line change
Expand Up @@ -5363,6 +5363,55 @@ def test_priming_unpickler_memo(self):
self.assertEqual(unpickled_data2, data)
self.assertTrue(unpickled_data2 is unpickled_data1)

def test_assign_unpickler_memo_dict(self):
value = []
for index in (0, 1, 1000):
with self.subTest(index=index):
unpickler = self.unpickler_class(
io.BytesIO(f'g{index}\n.'.encode()))
memo = {index: value}
unpickler.memo = memo
self.assertIs(unpickler.load(), value)

def test_assign_unpickler_memo_continuation(self):
for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
for copy_memo in (False, True):
with self.subTest(proto=proto, copy_memo=copy_memo):
stream = io.BytesIO()
pickler = self.pickler_class(stream, protocol=proto)
pickler.dump(['first'])
value = ['second']
pickler.dump([value, value])
stream.seek(0)
unpickler = self.unpickler_class(stream)
unpickler.load()
primed = self.unpickler_class(stream)
memo = unpickler.memo
primed.memo = memo.copy() if copy_memo else memo
result = primed.load()
self.assertEqual(result, [value, value])
self.assertIs(result[0], result[1])

def test_assign_unpickler_memo_sparse(self):
source = self.unpickler_class(io.BytesIO(b']p1000\n.'))
value = source.load()
unpickler = self.unpickler_class(io.BytesIO(b'g1000\n.'))
unpickler.memo = source.memo
self.assertIs(unpickler.load(), value)

def test_assign_empty_unpickler_memo(self):
unpickler = self.unpickler_class(io.BytesIO(b']p1000\n.g1000\n.'))
unpickler.load()
unpickler.memo = {}
with self.assertRaises(pickle.UnpicklingError):
unpickler.load()

def test_assign_unpickler_own_memo(self):
unpickler = self.unpickler_class(io.BytesIO(b']q\x00.h\x00.'))
value = unpickler.load()
unpickler.memo = unpickler.memo
self.assertIs(unpickler.load(), value)

def test_reusing_unpickler_objects(self):
data1 = ["abcdefg", "abcdefg", 44]
f = io.BytesIO()
Expand Down
36 changes: 36 additions & 0 deletions Lib/test/test_pickle.py
Original file line number Diff line number Diff line change
Expand Up @@ -470,6 +470,42 @@ def test_issue18339(self):
unpickler.memo = {-1: None}
unpickler.memo = {1: None}

def test_invalid_unpickler_memo_preserves_entries(self):
for key, error in (('invalid', TypeError), (-1, ValueError),
(sys.maxsize + 1, OverflowError)):
with self.subTest(key=key):
unpickler = self.unpickler_class(io.BytesIO(b']q\x00.h\x00.'))
value = unpickler.load()
with self.assertRaises(error):
unpickler.memo = {0: None, key: None}
self.assertIs(unpickler.load(), value)

def test_assign_unpickler_memo_copies_dict(self):
for index in (0, 1, 1000):
with self.subTest(index=index):
unpickler = self.unpickler_class(
io.BytesIO(f'g{index}\n.'.encode()))
value = []
memo = {index: value}
unpickler.memo = memo
memo.clear()
self.assertEqual(unpickler.memo.copy(), {index: value})
self.assertIs(unpickler.load(), value)

def test_assign_unpickler_memo_reentrant(self):
unpickler = self.unpickler_class(io.BytesIO(b'h\x00.'))
observed = []

class Observer:
def __del__(self):
observed.append(unpickler.memo.copy())

unpickler.memo = {0: Observer()}
value = []
unpickler.memo = {0: value}
self.assertEqual(observed, [{0: value}])
self.assertIs(unpickler.load(), value)

def test_concurrent_pickler_dump(self):
f = io.BytesIO()
pickler = self.pickler_class(f)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Fix assigning a dictionary or another unpickler's memo to
:class:`pickle.Unpickler`. Preserve the memo entries and the index used for
subsequent memoization.
57 changes: 46 additions & 11 deletions Modules/_pickle.c
Original file line number Diff line number Diff line change
Expand Up @@ -7940,7 +7940,8 @@ _pickle_UnpicklerMemoProxy_copy_impl(UnpicklerMemoProxyObject *self)
/*[clinic end generated code: output=e12af7e9bc1e4c77 input=97769247ce032c1d]*/
{
size_t i;
PyObject *new_memo = PyDict_New();
PyObject *new_memo = self->unpickler->memo_dict == NULL
? PyDict_New() : PyDict_Copy(self->unpickler->memo_dict);
if (new_memo == NULL)
return NULL;

Expand Down Expand Up @@ -8074,6 +8075,8 @@ Unpickler_set_memo(PyObject *op, PyObject *obj, void *Py_UNUSED(closure))
PyObject **new_memo;
UnpicklerObject *self = UnpicklerObject_CAST(op);
size_t new_memo_size = 0;
size_t new_memo_len = 0;
PyObject *new_memo_dict = NULL;

if (obj == NULL) {
PyErr_SetString(PyExc_TypeError,
Expand All @@ -8091,6 +8094,13 @@ Unpickler_set_memo(PyObject *op, PyObject *obj, void *Py_UNUSED(closure))
if (new_memo == NULL)
return -1;

new_memo_len = unpickler->memo_len;
if (unpickler->memo_dict != NULL) {
new_memo_dict = PyDict_Copy(unpickler->memo_dict);
if (new_memo_dict == NULL)
goto error;
}

for (size_t i = 0; i < new_memo_size; i++) {
new_memo[i] = Py_XNewRef(unpickler->memo[i]);
}
Expand All @@ -8100,6 +8110,7 @@ Unpickler_set_memo(PyObject *op, PyObject *obj, void *Py_UNUSED(closure))
PyObject *key, *value;

new_memo_size = PyDict_GET_SIZE(obj);
new_memo_len = new_memo_size;
new_memo = _Unpickler_NewMemo(new_memo_size);
if (new_memo == NULL)
return -1;
Expand All @@ -8119,8 +8130,23 @@ Unpickler_set_memo(PyObject *op, PyObject *obj, void *Py_UNUSED(closure))
"memo key must be positive integers.");
goto error;
}
if (_Unpickler_MemoPut(self, idx, value) < 0)
goto error;
if ((size_t)idx < new_memo_size) {
Py_XSETREF(new_memo[idx], Py_NewRef(value));
}
else {
if (new_memo_dict == NULL) {
new_memo_dict = PyDict_New();
if (new_memo_dict == NULL)
goto error;
}
PyObject *index = PyLong_FromSsize_t(idx);
if (index == NULL)
goto error;
int status = PyDict_SetItem(new_memo_dict, index, value);
Py_DECREF(index);
if (status < 0)
goto error;
}
}
}
else {
Expand All @@ -8130,19 +8156,28 @@ Unpickler_set_memo(PyObject *op, PyObject *obj, void *Py_UNUSED(closure))
return -1;
}

_Unpickler_MemoCleanup(self, NULL);
self->memo_size = new_memo_size;
/* Install all of the new state before releasing the old entries, which
may call back into the unpickler. */
PyObject **old_memo = self->memo;
size_t old_memo_size = self->memo_size;
PyObject *old_memo_dict = self->memo_dict;
self->memo = new_memo;

self->memo_size = new_memo_size;
self->memo_len = new_memo_len;
self->memo_dict = new_memo_dict;
for (size_t i = 0; i < old_memo_size; i++) {
Py_XDECREF(old_memo[i]);
}
PyMem_Free(old_memo);
Py_XDECREF(old_memo_dict);
return 0;

error:
if (new_memo_size) {
for (size_t i = new_memo_size - 1; i != SIZE_MAX; i--) {
Py_XDECREF(new_memo[i]);
}
PyMem_Free(new_memo);
for (size_t i = 0; i < new_memo_size; i++) {
Py_XDECREF(new_memo[i]);
}
PyMem_Free(new_memo);
Py_XDECREF(new_memo_dict);
return -1;
}

Expand Down
Loading