From 9d8d8acdf1279cf43b4e40e9fc3f7326e2552aca Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Oct 2017 18:34:46 +0200 Subject: [PATCH] bpo-31626: Rewrite _PyMem_DebugRawRealloc() Previously, _PyMem_DebugRawRealloc() expected that the old memory block remains readable and unchanged after realloc(). This assumption is wrong on OpenBSD. Rewrite _PyMem_DebugRawRealloc() as malloc()+free() rather than realloc(). Debug hooks are reused. The old memory block remains valid after the new memory block was allocated, so bytes can be safely copied in a portable way. The drawback is that _PyMem_DebugRawRealloc() now allocates a new memory block while the old memory block remains allocated, until the old memory block is freed, so the peak memory usage can be the double in the worst case. The second drawback is that the system realloc() is no more used in debug mode. --- .../2017-10-05-18-38-47.bpo-31626.tDXcnr.rst | 18 +++++++ Objects/obmalloc.c | 54 +++++-------------- 2 files changed, 31 insertions(+), 41 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-10-05-18-38-47.bpo-31626.tDXcnr.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-10-05-18-38-47.bpo-31626.tDXcnr.rst b/Misc/NEWS.d/next/Core and Builtins/2017-10-05-18-38-47.bpo-31626.tDXcnr.rst new file mode 100644 index 00000000000000..2d4882c3849d43 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-10-05-18-38-47.bpo-31626.tDXcnr.rst @@ -0,0 +1,18 @@ +Rewrite _PyMem_DebugRawRealloc(). + +Previously, _PyMem_DebugRawRealloc() expected that the old memory +block remains readable and unchanged after realloc(). This assumption +is wrong on OpenBSD. + +Rewrite _PyMem_DebugRawRealloc() as malloc()+free() rather than +realloc(). Debug hooks are reused. The old memory block remains valid +after the new memory block was allocated, so bytes can be safely +copied in a portable way. + +The drawback is that _PyMem_DebugRawRealloc() now allocates a new +memory block while the old memory block remains allocated, until the +old memory block is freed, so the peak memory usage can be the double +in the worst case. + +The second drawback is that the system realloc() is no more used in +debug mode. diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index f2651d7574b20f..7ef497a97de6bf 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1460,53 +1460,25 @@ static void * _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) { debug_alloc_api_t *api = (debug_alloc_api_t *)ctx; - uint8_t *q = (uint8_t *)p, *oldq; - uint8_t *tail; - size_t total; /* nbytes + 4*SST */ - size_t original_nbytes; - int i; + void *q; + size_t old_size; - if (p == NULL) - return _PyMem_DebugRawAlloc(0, ctx, nbytes); - - _PyMem_DebugCheckAddress(api->api_id, p); - bumpserialno(); - original_nbytes = read_size_t(q - 2*SST); - total = nbytes + 4*SST; - if (nbytes > PY_SSIZE_T_MAX - 4*SST) - /* overflow: can't represent total as a Py_ssize_t */ - return NULL; + if (p != NULL) { + uint8_t *p2 = (uint8_t *)p - 2*SST; /* address returned from malloc */ + _PyMem_DebugCheckAddress(api->api_id, p); + old_size = read_size_t(p2); + } - /* Resize and add decorations. We may get a new pointer here, in which - * case we didn't get the chance to mark the old memory with DEADBYTE, - * but we live with that. - */ - oldq = q; - q = (uint8_t *)api->alloc.realloc(api->alloc.ctx, q - 2*SST, total); - if (q == NULL) + q = _PyMem_DebugRawAlloc(0, ctx, nbytes); + if (q == NULL) { return NULL; - - if (q == oldq && nbytes < original_nbytes) { - /* shrinking: mark old extra memory dead */ - memset(q + nbytes, DEADBYTE, original_nbytes - nbytes); } - write_size_t(q, nbytes); - assert(q[SST] == (uint8_t)api->api_id); - for (i = 1; i < SST; ++i) - assert(q[SST + i] == FORBIDDENBYTE); - q += 2*SST; - - tail = q + nbytes; - memset(tail, FORBIDDENBYTE, SST); - write_size_t(tail + SST, _PyRuntime.mem.serialno); - - if (nbytes > original_nbytes) { - /* growing: mark new extra memory clean */ - memset(q + original_nbytes, CLEANBYTE, - nbytes - original_nbytes); + if (p != NULL) { + size_t copied_bytes = Py_MIN(nbytes, old_size); + memcpy(q, p, copied_bytes); + _PyMem_DebugRawFree(ctx, p); } - return q; }