Revert changeset 6661a8154eb3: Issue #3329: Add new APIs to customize memory allocators

The new API require more discussion.

Doc/c-api/memory.rst

@@ -84,46 +84,6 @@ the C library allocator as shown in the previous example, the allocated memory
 for the I/O buffer escapes completely the Python memory manager.
 
 
-Raw Memory Interface
-====================
-
-The following function are wrappers to system allocators: :c:func:`malloc`,
-:c:func:`realloc`, :c:func:`free`. These functions are thread-safe, the
-:term:`GIL <global interpreter lock>` does not need to be held to use these
-functions.
-
-The behaviour of requesting zero bytes is not defined: return *NULL* or a
-distinct non-*NULL* pointer depending on the platform. Use
-:c:func:`PyMem_Malloc` and :c:func:`PyMem_Realloc` to have a well defined
-behaviour.
-
-.. versionadded:: 3.4
-
-.. c:function:: void* PyMem_RawMalloc(size_t n)
-
-   Allocates *n* bytes and returns a pointer of type :c:type:`void\*` to the
-   allocated memory, or *NULL* if the request fails. The memory
-   will not have been initialized in any way.
-
-
-.. c:function:: void* PyMem_RawRealloc(void *p, size_t n)
-
-   Resizes the memory block pointed to by *p* to *n* bytes. The contents will
-   be unchanged to the minimum of the old and the new sizes. If *p* is *NULL*,
-   the call is equivalent to ``PyMem_RawMalloc(n)``. Unless *p* is *NULL*, it
-   must have been returned by a previous call to :c:func:`PyMem_RawMalloc` or
-   :c:func:`PyMem_RawRealloc`. If the request fails, :c:func:`PyMem_RawRealloc`
-   returns *NULL* and *p* remains a valid pointer to the previous memory area.
-
-
-.. c:function:: void PyMem_RawFree(void *p)
-
-   Frees the memory block pointed to by *p*, which must have been returned by a
-   previous call to :c:func:`PyMem_RawMalloc` or :c:func:`PyMem_RawRealloc`.
-   Otherwise, or if ``PyMem_Free(p)`` has been called before, undefined
-   behavior occurs. If *p* is *NULL*, no operation is performed.
-
-
 .. _memoryinterface:
 
 Memory Interface
@@ -131,12 +91,8 @@ Memory Interface
 
 The following function sets, modeled after the ANSI C standard, but specifying
 behavior when requesting zero bytes, are available for allocating and releasing
-memory from the Python heap.
+memory from the Python heap:
 
-.. warning::
-
-   The :term:`GIL <global interpreter lock>` must be held when using these
-   functions.
 
 .. c:function:: void* PyMem_Malloc(size_t n)
 
@@ -199,81 +155,6 @@ versions and is therefore deprecated in extension modules.
 :c:func:`PyMem_NEW`, :c:func:`PyMem_RESIZE`, :c:func:`PyMem_DEL`.
 
 
-Customize Memory Allocators
-===========================
-
-.. versionadded:: 3.4
-
-.. c:type:: PyMemAllocators
-
-   Structure used to describe memory allocator.  This structure has
-   four fields:
-
-   +----------------------------------------------------------+-----------------+
-   | Field                                                    | Meaning         |
-   +==========================================================+=================+
-   | ``void *ctx``                                            | user data       |
-   +----------------------------------------------------------+-----------------+
-   | ``void* malloc(void *ctx, size_t size)``                 | allocate memory |
-   +----------------------------------------------------------+-----------------+
-   | ``void* realloc(void *ctx, void *ptr, size_t new_size)`` | allocate memory |
-   |                                                          | or resize a     |
-   |                                                          | memory block    |
-   +----------------------------------------------------------+-----------------+
-   | ``void free(void *ctx, void *ptr)``                      | release memory  |
-   +----------------------------------------------------------+-----------------+
-
-.. c:function:: void PyMem_GetRawAllocators(PyMemAllocators *allocators)
-
-   Get internal functions of :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc`
-   and :c:func:`PyMem_RawFree`.
-
-.. c:function:: void PyMem_SetRawAllocators(PyMemAllocators *allocators)
-
-   Set internal functions of :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc`
-   and :c:func:`PyMem_RawFree`.
-
-   :c:func:`PyMem_SetupDebugHooks` should be called to reinstall debug hooks if
-   new functions do no call original functions anymore.
-
-.. c:function:: void PyMem_GetAllocators(PyMemAllocators *allocators)
-
-   Get internal functions of :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc`
-   and :c:func:`PyMem_Free`.
-
-.. c:function:: void PyMem_SetAllocators(PyMemAllocators *allocators)
-
-   Set internal functions of :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc`
-   and :c:func:`PyMem_Free`.
-
-   ``malloc(ctx, 0)`` and ``realloc(ctx, ptr, 0)`` must not return *NULL*: it
-   would be treated as an error.
-
-   :c:func:`PyMem_SetupDebugHooks` should be called to reinstall debug hooks if
-   new functions do no call original functions anymore.
-
-.. c:function:: void PyMem_SetupDebugHooks(void)
-
-   Setup hooks to detect bugs in the following Python memory allocator
-   functions:
-
-   - :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc`,
-     :c:func:`PyMem_RawFree`
-   - :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc`, :c:func:`PyMem_Free`
-   - :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc`,
-     :c:func:`PyObject_Free`
-
-   Newly allocated memory is filled with the byte ``0xCB``, freed memory is
-   filled with the byte ``0xDB``. Additionnal checks:
-
-   - detect API violations, ex: :c:func:`PyObject_Free` called on a buffer
-     allocated by :c:func:`PyMem_Malloc`
-   - detect write before the start of the buffer (buffer underflow)
-   - detect write after the end of the buffer (buffer overflow)
-
-   The function does nothing if Python is not compiled is debug mode.
-
-
 .. _memoryexamples:
 
 Examples

Include/objimpl.h

@@ -94,9 +94,9 @@ PyObject_{New, NewVar, Del}.
    the object gets initialized via PyObject_{Init, InitVar} after obtaining
    the raw memory.
 */
-PyAPI_FUNC(void *) PyObject_Malloc(size_t size);
-PyAPI_FUNC(void *) PyObject_Realloc(void *ptr, size_t new_size);
-PyAPI_FUNC(void) PyObject_Free(void *ptr);
+PyAPI_FUNC(void *) PyObject_Malloc(size_t);
+PyAPI_FUNC(void *) PyObject_Realloc(void *, size_t);
+PyAPI_FUNC(void) PyObject_Free(void *);
 
 /* This function returns the number of allocated memory blocks, regardless of size */
 PyAPI_FUNC(Py_ssize_t) _Py_GetAllocatedBlocks(void);
@@ -106,46 +106,41 @@ PyAPI_FUNC(Py_ssize_t) _Py_GetAllocatedBlocks(void);
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(void) _PyObject_DebugMallocStats(FILE *out);
 #endif /* #ifndef Py_LIMITED_API */
-#endif
-
-/* Macros */
+#ifdef PYMALLOC_DEBUG   /* WITH_PYMALLOC && PYMALLOC_DEBUG */
+PyAPI_FUNC(void *) _PyObject_DebugMalloc(size_t nbytes);
+PyAPI_FUNC(void *) _PyObject_DebugRealloc(void *p, size_t nbytes);
+PyAPI_FUNC(void) _PyObject_DebugFree(void *p);
+PyAPI_FUNC(void) _PyObject_DebugDumpAddress(const void *p);
+PyAPI_FUNC(void) _PyObject_DebugCheckAddress(const void *p);
+PyAPI_FUNC(void *) _PyObject_DebugMallocApi(char api, size_t nbytes);
+PyAPI_FUNC(void *) _PyObject_DebugReallocApi(char api, void *p, size_t nbytes);
+PyAPI_FUNC(void) _PyObject_DebugFreeApi(char api, void *p);
+PyAPI_FUNC(void) _PyObject_DebugCheckAddressApi(char api, const void *p);
+PyAPI_FUNC(void *) _PyMem_DebugMalloc(size_t nbytes);
+PyAPI_FUNC(void *) _PyMem_DebugRealloc(void *p, size_t nbytes);
+PyAPI_FUNC(void) _PyMem_DebugFree(void *p);
+#define PyObject_MALLOC         _PyObject_DebugMalloc
+#define PyObject_Malloc         _PyObject_DebugMalloc
+#define PyObject_REALLOC        _PyObject_DebugRealloc
+#define PyObject_Realloc        _PyObject_DebugRealloc
+#define PyObject_FREE           _PyObject_DebugFree
+#define PyObject_Free           _PyObject_DebugFree
+
+#else   /* WITH_PYMALLOC && ! PYMALLOC_DEBUG */
 #define PyObject_MALLOC         PyObject_Malloc
 #define PyObject_REALLOC        PyObject_Realloc
 #define PyObject_FREE           PyObject_Free
+#endif
+
+#else   /* ! WITH_PYMALLOC */
+#define PyObject_MALLOC         PyMem_MALLOC
+#define PyObject_REALLOC        PyMem_REALLOC
+#define PyObject_FREE           PyMem_FREE
+
+#endif  /* WITH_PYMALLOC */
+
 #define PyObject_Del            PyObject_Free
-#define PyObject_DEL            PyObject_Free
-
-/* Get internal functions of PyObject_Malloc(), PyObject_Realloc() and
-   PyObject_Free(). *ctx_p is an arbitrary user value. */
-PyAPI_FUNC(void) PyObject_GetAllocators(PyMemAllocators *allocators);
-
-/* Set internal functions of PyObject_Malloc(), PyObject_Realloc() and PyObject_Free().
-   ctx is an arbitrary user value.
-
-   malloc(ctx, 0) and realloc(ctx, ptr, 0) must not return NULL: it would be
-   treated as an error.
-
-   PyMem_SetupDebugHooks() should be called to reinstall debug hooks if new
-   functions do no call original functions anymore. */
-PyAPI_FUNC(void) PyObject_SetAllocators(PyMemAllocators *allocators);
-
-/* Get internal functions allocating and deallocating arenas for
-   PyObject_Malloc(), PyObject_Realloc() and PyObject_Free().
-   *ctx_p is an arbitrary user value. */
-PyAPI_FUNC(void) _PyObject_GetArenaAllocators(
-    void **ctx_p,
-    void* (**malloc_p) (void *ctx, size_t size),
-    void (**free_p) (void *ctx, void *ptr, size_t size)
-    );
-
-/* Get internal functions allocating and deallocating arenas for
-   PyObject_Malloc(), PyObject_Realloc() and PyObject_Free().
-   ctx is an arbitrary user value. */
-PyAPI_FUNC(void) _PyObject_SetArenaAllocators(
-    void *ctx,
-    void* (*malloc) (void *ctx, size_t size),
-    void (*free) (void *ctx, void *ptr, size_t size)
-    );
+#define PyObject_DEL            PyObject_FREE
 
 /*
  * Generic object allocator interface

Include/pymem.h

@@ -11,40 +11,6 @@
 extern "C" {
 #endif
 
-typedef struct {
-    /* user context passed as the first argument to the 3 functions */
-    void *ctx;
-
-    /* allocate memory */
-    void* (*malloc) (void *ctx, size_t size);
-
-    /* allocate memory or resize a memory buffer */
-    void* (*realloc) (void *ctx, void *ptr, size_t new_size);
-
-    /* release memory */
-    void (*free) (void *ctx, void *ptr);
-} PyMemAllocators;
-
-/* Raw memory allocators, system functions: malloc(), realloc(), free().
-
-   These functions are thread-safe, the GIL does not need to be held. */
-
-/* Get internal functions of PyMem_RawMalloc(), PyMem_RawRealloc() and
-   PyMem_RawFree(). *ctx_p is an arbitrary user value. */
-PyAPI_FUNC(void) PyMem_GetRawAllocators(PyMemAllocators *allocators);
-
-/* Set internal functions of PyMem_RawMalloc(), PyMem_RawRealloc() and
-   PyMem_RawFree(). ctx is an arbitrary user value.
-
-   PyMem_SetupDebugHooks() should be called to reinstall debug hooks if new
-   functions do no call original functions anymore. */
-PyAPI_FUNC(void) PyMem_SetRawAllocators(PyMemAllocators *allocators);
-
-PyAPI_FUNC(void *) PyMem_RawMalloc(size_t size);
-PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size);
-PyAPI_FUNC(void) PyMem_RawFree(void *ptr);
-
-
 /* BEWARE:
 
    Each interface exports both functions and macros.  Extension modules should
@@ -83,11 +49,21 @@ PyAPI_FUNC(void) PyMem_RawFree(void *ptr);
    performed on failure (no exception is set, no warning is printed, etc).
 */
 
-PyAPI_FUNC(void *) PyMem_Malloc(size_t size);
-PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size);
-PyAPI_FUNC(void) PyMem_Free(void *ptr);
+PyAPI_FUNC(void *) PyMem_Malloc(size_t);
+PyAPI_FUNC(void *) PyMem_Realloc(void *, size_t);
+PyAPI_FUNC(void) PyMem_Free(void *);
+
+/* Starting from Python 1.6, the wrappers Py_{Malloc,Realloc,Free} are
+   no longer supported. They used to call PyErr_NoMemory() on failure. */
 
 /* Macros. */
+#ifdef PYMALLOC_DEBUG
+/* Redirect all memory operations to Python's debugging allocator. */
+#define PyMem_MALLOC		_PyMem_DebugMalloc
+#define PyMem_REALLOC		_PyMem_DebugRealloc
+#define PyMem_FREE		_PyMem_DebugFree
+
+#else	/* ! PYMALLOC_DEBUG */
 
 /* PyMem_MALLOC(0) means malloc(1). Some systems would return NULL
    for malloc(0), which would be treated as an error. Some platforms
@@ -95,9 +71,13 @@ PyAPI_FUNC(void) PyMem_Free(void *ptr);
    pymalloc. To solve these problems, allocate an extra byte. */
 /* Returns NULL to indicate error if a negative size or size larger than
    Py_ssize_t can represent is supplied.  Helps prevents security holes. */
-#define PyMem_MALLOC(n)         PyMem_Malloc(n)
-#define PyMem_REALLOC(p, n)     PyMem_Realloc(p, n)
-#define PyMem_FREE(p)           PyMem_Free(p)
+#define PyMem_MALLOC(n)		((size_t)(n) > (size_t)PY_SSIZE_T_MAX ? NULL \
+				: malloc((n) ? (n) : 1))
+#define PyMem_REALLOC(p, n)	((size_t)(n) > (size_t)PY_SSIZE_T_MAX  ? NULL \
+				: realloc((p), (n) ? (n) : 1))
+#define PyMem_FREE		free
+
+#endif	/* PYMALLOC_DEBUG */
 
 /*
  * Type-oriented memory interface
@@ -135,37 +115,6 @@ PyAPI_FUNC(void) PyMem_Free(void *ptr);
 #define PyMem_Del		PyMem_Free
 #define PyMem_DEL		PyMem_FREE
 
-/* Get internal functions of PyMem_Malloc(), PyMem_Realloc()
-   and PyMem_Free() */
-PyAPI_FUNC(void) PyMem_GetAllocators(PyMemAllocators *allocators);
-
-/* Set internal functions of PyMem_Malloc(), PyMem_Realloc() and PyMem_Free().
-
-   malloc(ctx, 0) and realloc(ctx, ptr, 0) must not return NULL: it would be
-   treated as an error.
-
-   PyMem_SetupDebugHooks() should be called to reinstall debug hooks if new
-   functions do no call original functions anymore. */
-PyAPI_FUNC(void) PyMem_SetAllocators(PyMemAllocators *allocators);
-
-/* Setup hooks to detect bugs in the following Python memory allocator
-   functions:
-
-   - PyMem_RawMalloc(), PyMem_RawRealloc(), PyMem_RawFree()
-   - PyMem_Malloc(), PyMem_Realloc(), PyMem_Free()
-   - PyObject_Malloc(), PyObject_Realloc() and PyObject_Free()
-
-   Newly allocated memory is filled with the byte 0xCB, freed memory is filled
-   with the byte 0xDB. Additionnal checks:
-
-   - detect API violations, ex: PyObject_Free() called on a buffer allocated
-     by PyMem_Malloc()
-   - detect write before the start of the buffer (buffer underflow)
-   - detect write after the end of the buffer (buffer overflow)
-
-   The function does nothing if Python is not compiled is debug mode. */
-PyAPI_FUNC(void) PyMem_SetupDebugHooks(void);
-
 #ifdef __cplusplus
 }
 #endif

Modules/_testcapimodule.c

@@ -2511,176 +2511,6 @@ test_decref_doesnt_leak(PyObject *ob)
     Py_RETURN_NONE;
 }
 
-static PyObject *
-test_pymem_alloc0(PyObject *self)
-{
-    void *ptr;
-
-    ptr = PyMem_Malloc(0);
-    if (ptr == NULL) {
-        PyErr_SetString(PyExc_RuntimeError, "PyMem_Malloc(0) returns NULL");
-        return NULL;
-    }
-    PyMem_Free(ptr);
-
-    ptr = PyObject_Malloc(0);
-    if (ptr == NULL) {
-        PyErr_SetString(PyExc_RuntimeError, "PyObject_Malloc(0) returns NULL");
-        return NULL;
-    }
-    PyObject_Free(ptr);
-
-    Py_RETURN_NONE;
-}
-
-typedef struct {
-    PyMemAllocators alloc;
-
-    size_t malloc_size;
-    void *realloc_ptr;
-    size_t realloc_new_size;
-    void *free_ptr;
-} alloc_hook_t;
-
-static void* hook_malloc (void* ctx, size_t size)
-{
-    alloc_hook_t *hook = (alloc_hook_t *)ctx;
-    hook->malloc_size = size;
-    return hook->alloc.malloc(hook->alloc.ctx, size);
-}
-
-static void* hook_realloc (void* ctx, void* ptr, size_t new_size)
-{
-    alloc_hook_t *hook = (alloc_hook_t *)ctx;
-    hook->realloc_ptr = ptr;
-    hook->realloc_new_size = new_size;
-    return hook->alloc.realloc(hook->alloc.ctx, ptr, new_size);
-}
-
-static void hook_free (void *ctx, void *ptr)
-{
-    alloc_hook_t *hook = (alloc_hook_t *)ctx;
-    printf("HOOK\n");
-    hook->free_ptr = ptr;
-    hook->alloc.free(hook->alloc.ctx, ptr);
-}
-
-static PyObject *
-test_setallocators(char api)
-{
-    PyObject *res = NULL;
-    const char *error_msg;
-    alloc_hook_t hook;
-    PyMemAllocators alloc;
-    size_t size, size2;
-    void *ptr, *ptr2;
-
-    hook.malloc_size = 0;
-    hook.realloc_ptr = NULL;
-    hook.realloc_new_size = 0;
-    hook.free_ptr = NULL;
-
-    alloc.ctx = &hook;
-    alloc.malloc = &hook_malloc;
-    alloc.realloc = &hook_realloc;
-    alloc.free = &hook_free;
-    if (api == 'o') {
-        PyObject_GetAllocators(&hook.alloc);
-        PyObject_SetAllocators(&alloc);
-    }
-    else if (api == 'r') {
-        PyMem_GetRawAllocators(&hook.alloc);
-        PyMem_SetRawAllocators(&alloc);
-    }
-    else {
-        PyMem_GetAllocators(&hook.alloc);
-        PyMem_SetAllocators(&alloc);
-    }
-
-    size = 42;
-    if (api == 'o')
-        ptr = PyObject_Malloc(size);
-    else if (api == 'r')
-        ptr = PyMem_RawMalloc(size);
-    else
-        ptr = PyMem_Malloc(size);
-    if (ptr == NULL) {
-        error_msg = "malloc failed";
-        goto fail;
-    }
-
-    if (hook.malloc_size != size) {
-        error_msg = "malloc invalid size";
-        goto fail;
-    }
-
-    size2 = 200;
-    if (api == 'o')
-        ptr2 = PyObject_Realloc(ptr, size2);
-    else if (api == 'r')
-        ptr2 = PyMem_RawRealloc(ptr, size2);
-    else
-        ptr2 = PyMem_Realloc(ptr, size2);
-    if (ptr2 == NULL) {
-        error_msg = "realloc failed";
-        goto fail;
-    }
-
-    if (hook.realloc_ptr != ptr
-        || hook.realloc_new_size != size2) {
-        error_msg = "realloc invalid parameters";
-        goto fail;
-    }
-
-    if (api == 'o')
-        PyObject_Free(ptr2);
-    else if (api == 'r')
-        PyMem_RawFree(ptr2);
-    else {
-        printf("PyMem_Free\n");
-        PyMem_Free(ptr2);
-    }
-
-    if (hook.free_ptr != ptr2) {
-        error_msg = "free invalid pointer";
-        goto fail;
-    }
-
-    Py_INCREF(Py_None);
-    res = Py_None;
-    goto finally;
-
-fail:
-    PyErr_SetString(PyExc_RuntimeError, error_msg);
-
-finally:
-    if (api == 'o')
-        PyObject_SetAllocators(&hook.alloc);
-    else if (api == 'r')
-        PyMem_SetRawAllocators(&hook.alloc);
-    else
-        PyMem_SetAllocators(&hook.alloc);
-    return res;
-}
-
-static PyObject *
-test_pymem_setrawallocators(PyObject *self)
-{
-    return test_setallocators('r');
-}
-
-static PyObject *
-test_pymem_setallocators(PyObject *self)
-{
-    return test_setallocators('m');
-}
-
-static PyObject *
-test_pyobject_setallocators(PyObject *self)
-{
-    return test_setallocators('o');
-}
-
 static PyMethodDef TestMethods[] = {
     {"raise_exception",         raise_exception,                 METH_VARARGS},
     {"raise_memoryerror",   (PyCFunction)raise_memoryerror,  METH_NOARGS},
@@ -2781,14 +2611,6 @@ static PyMethodDef TestMethods[] = {
     {"pytime_object_to_time_t", test_pytime_object_to_time_t,  METH_VARARGS},
     {"pytime_object_to_timeval", test_pytime_object_to_timeval,  METH_VARARGS},
     {"pytime_object_to_timespec", test_pytime_object_to_timespec,  METH_VARARGS},
-    {"test_pymem",
-     (PyCFunction)test_pymem_alloc0, METH_NOARGS},
-    {"test_pymem_alloc0",
-     (PyCFunction)test_pymem_setrawallocators, METH_NOARGS},
-    {"test_pymem_setallocators",
-     (PyCFunction)test_pymem_setallocators, METH_NOARGS},
-    {"test_pyobject_setallocators",
-     (PyCFunction)test_pyobject_setallocators, METH_NOARGS},
     {NULL, NULL} /* sentinel */
 };
 

Objects/object.c

@@ -1859,6 +1859,26 @@ PyTypeObject *_PyCapsule_hack = &PyCapsule_Type;
 Py_ssize_t (*_Py_abstract_hack)(PyObject *) = PyObject_Size;
 
 
+/* Python's malloc wrappers (see pymem.h) */
+
+void *
+PyMem_Malloc(size_t nbytes)
+{
+    return PyMem_MALLOC(nbytes);
+}
+
+void *
+PyMem_Realloc(void *p, size_t nbytes)
+{
+    return PyMem_REALLOC(p, nbytes);
+}
+
+void
+PyMem_Free(void *p)
+{
+    PyMem_FREE(p);
+}
+
 void
 _PyObject_DebugTypeStats(FILE *out)
 {