Issue #18874: Implement the PEP 454 (tracemalloc)

Doc/library/debug.rst

@@ -15,3 +15,4 @@ allowing you to identify bottlenecks in your programs.
    profile.rst
    timeit.rst
    trace.rst
+   tracemalloc.rst

Doc/license.rst

@@ -893,3 +893,44 @@ used for the build::
   Jean-loup Gailly        Mark Adler
   jloup@gzip.org          madler@alumni.caltech.edu
 
+
+cfuhash
+-------
+
+The implementtation of the hash table used by the :mod:`tracemalloc` is based
+on the cfuhash project::
+
+   Copyright (c) 2005 Don Owens
+   All rights reserved.
+
+   This code is released under the BSD license:
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+     * Redistributions of source code must retain the above copyright
+       notice, this list of conditions and the following disclaimer.
+
+     * Redistributions in binary form must reproduce the above
+       copyright notice, this list of conditions and the following
+       disclaimer in the documentation and/or other materials provided
+       with the distribution.
+
+     * Neither the name of the author nor the names of its
+       contributors may be used to endorse or promote products derived
+       from this software without specific prior written permission.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+   FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+   COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+   INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+   (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+   SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+   HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+   STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+   OF THE POSSIBILITY OF SUCH DAMAGE.
+

Doc/using/cmdline.rst

@@ -376,11 +376,15 @@ Miscellaneous options
 .. cmdoption:: -X
 
    Reserved for various implementation-specific options.  CPython currently
-   defines two possible values:
+   defines the following possible values:
 
    * ``-X faulthandler`` to enable :mod:`faulthandler`;
    * ``-X showrefcount`` to enable the output of the total reference count
      and memory blocks (only works on debug builds);
+   * ``-X tracemalloc`` to enable :mod:`tracemalloc`.
+   * ``-X tracemalloc=NFRAME`` to enable :mod:`tracemalloc`, *NFRAME* is the
+     maximum number of frames stored in a trace: see the
+     :func:`tracemalloc.set_traceback_limit` function.
 
    It also allows to pass arbitrary values and retrieve them through the
    :data:`sys._xoptions` dictionary.
@@ -392,7 +396,7 @@ Miscellaneous options
       The ``-X faulthandler`` option.
 
    .. versionadded:: 3.4
-      The ``-X showrefcount`` option.
+      The ``-X showrefcount`` and ``-X tracemalloc`` options.
 
 
 Options you shouldn't use
@@ -594,6 +598,16 @@ conflict.
    .. versionadded:: 3.3
 
 
+.. envvar:: PYTHONTRACEMALLOC
+
+   If this environment variable is set to a non-empty string, all memory
+   allocations made by Python are traced by the :mod:`tracemalloc` module.
+   The value of the variable is the maximum number of frames stored in a trace:
+   see the :func:`tracemalloc.set_traceback_limit` function.
+
+   .. versionadded:: 3.4
+
+
 Debug-mode variables
 ~~~~~~~~~~~~~~~~~~~~
 

Lib/test/support/__init__.py

@@ -2154,3 +2154,22 @@ def patch(test_instance, object_to_patch, attr_name, new_value):
 
     # actually override the attribute
     setattr(object_to_patch, attr_name, new_value)
+
+
+def run_in_subinterp(code):
+    """
+    Run code in a subinterpreter. Raise unittest.SkipTest if the tracemalloc
+    module is enabled.
+    """
+    # Issue #10915, #15751: PyGILState_*() functions don't work with
+    # sub-interpreters, the tracemalloc module uses these functions internally
+    try:
+        import tracemalloc
+    except ImportError:
+        pass
+    else:
+        if tracemalloc.is_tracing():
+            raise unittest.SkipTest("run_in_subinterp() cannot be used "
+                                     "if tracemalloc module is tracing "
+                                     "memory allocations")
+    return _testcapi.run_in_subinterp(code)

Lib/test/test_atexit.py

@@ -158,7 +158,7 @@ class SubinterpreterTest(unittest.TestCase):
             atexit.register(f)
             del atexit
             """
-        ret = _testcapi.run_in_subinterp(code)
+        ret = support.run_in_subinterp(code)
         self.assertEqual(ret, 0)
         self.assertEqual(atexit._ncallbacks(), n)
 
@@ -173,7 +173,7 @@ class SubinterpreterTest(unittest.TestCase):
             atexit.register(f)
             atexit.__atexit = atexit
             """
-        ret = _testcapi.run_in_subinterp(code)
+        ret = support.run_in_subinterp(code)
         self.assertEqual(ret, 0)
         self.assertEqual(atexit._ncallbacks(), n)
 

Lib/test/test_capi.py

@@ -205,7 +205,7 @@ class SubinterpreterTest(unittest.TestCase):
                 pickle.dump(id(builtins), f)
             """.format(w)
         with open(r, "rb") as f:
-            ret = _testcapi.run_in_subinterp(code)
+            ret = support.run_in_subinterp(code)
             self.assertEqual(ret, 0)
             self.assertNotEqual(pickle.load(f), id(sys.modules))
             self.assertNotEqual(pickle.load(f), id(builtins))

Lib/test/test_threading.py

@@ -853,7 +853,7 @@ class SubinterpThreadingTests(BaseTestCase):
                 os.write(%d, b"x")
             threading.Thread(target=f).start()
             """ % (w,)
-        ret = _testcapi.run_in_subinterp(code)
+        ret = test.support.run_in_subinterp(code)
         self.assertEqual(ret, 0)
         # The thread was joined properly.
         self.assertEqual(os.read(r, 1), b"x")
@@ -885,7 +885,7 @@ class SubinterpThreadingTests(BaseTestCase):
                 os.write(%d, b"x")
             threading.Thread(target=f).start()
             """ % (w,)
-        ret = _testcapi.run_in_subinterp(code)
+        ret = test.support.run_in_subinterp(code)
         self.assertEqual(ret, 0)
         # The thread was joined properly.
         self.assertEqual(os.read(r, 1), b"x")

Modules/Setup.dist

@@ -102,7 +102,7 @@ PYTHONPATH=$(COREPYTHONPATH)
 # various reasons; therefore they are listed here instead of in the
 # normal order.
 
-# This only contains the minimal set of modules required to run the 
+# This only contains the minimal set of modules required to run the
 # setup.py script in the root of the Python source tree.
 
 posix posixmodule.c		# posix (UNIX) system calls
@@ -115,7 +115,7 @@ _weakref _weakref.c		# weak references
 _functools _functoolsmodule.c   # Tools for working with functions and callable objects
 _operator _operator.c	        # operator.add() and similar goodies
 _collections _collectionsmodule.c # Container types
-itertools itertoolsmodule.c    # Functions creating iterators for efficient looping 
+itertools itertoolsmodule.c    # Functions creating iterators for efficient looping
 atexit atexitmodule.c      # Register functions to be run at interpreter-shutdown
 _stat _stat.c			# stat.h interface
 
@@ -132,12 +132,15 @@ zipimport zipimport.c
 # faulthandler module
 faulthandler faulthandler.c
 
+# debug tool to trace memory blocks allocated by Python
+_tracemalloc _tracemalloc.c hashtable.c
+
 # The rest of the modules listed in this file are all commented out by
 # default.  Usually they can be detected and built as dynamically
 # loaded modules by the new setup.py script added in Python 2.1.  If
-# you're on a platform that doesn't support dynamic loading, want to 
-# compile modules statically into the Python binary, or need to 
-# specify some odd set of compiler switches, you can uncomment the 
+# you're on a platform that doesn't support dynamic loading, want to
+# compile modules statically into the Python binary, or need to
+# specify some odd set of compiler switches, you can uncomment the
 # appropriate lines below.
 
 # ======================================================================
@@ -186,7 +189,7 @@ _symtable symtablemodule.c
 # supported...)
 
 #fcntl fcntlmodule.c	# fcntl(2) and ioctl(2)
-#spwd spwdmodule.c		# spwd(3) 
+#spwd spwdmodule.c		# spwd(3)
 #grp grpmodule.c		# grp(3)
 #select selectmodule.c	# select(2); not on ancient System V
 
@@ -302,7 +305,7 @@ _symtable symtablemodule.c
 
 #_curses _cursesmodule.c -lcurses -ltermcap
 # Wrapper for the panel library that's part of ncurses and SYSV curses.
-#_curses_panel _curses_panel.c -lpanel -lncurses 
+#_curses_panel _curses_panel.c -lpanel -lncurses
 
 
 # Modules that provide persistent dictionary-like semantics.  You will

Modules/hashtable.h

+#ifndef Py_HASHTABLE_H
+#define Py_HASHTABLE_H
+
+/* The whole API is private */
+#ifndef Py_LIMITED_API
+
+typedef struct _Py_slist_item_s {
+    struct _Py_slist_item_s *next;
+} _Py_slist_item_t;
+
+typedef struct {
+    _Py_slist_item_t *head;
+} _Py_slist_t;
+
+#define _Py_SLIST_ITEM_NEXT(ITEM) (((_Py_slist_item_t *)ITEM)->next)
+
+#define _Py_SLIST_HEAD(SLIST) (((_Py_slist_t *)SLIST)->head)
+
+typedef struct {
+    /* used by _Py_hashtable_t.buckets to link entries */
+    _Py_slist_item_t _Py_slist_item;
+
+    const void *key;
+    Py_uhash_t key_hash;
+
+    /* data follows */
+} _Py_hashtable_entry_t;
+
+#define _PY_HASHTABLE_ENTRY_DATA(ENTRY) \
+        ((char *)(ENTRY) + sizeof(_Py_hashtable_entry_t))
+
+#define _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ENTRY) \
+        (*(void **)_PY_HASHTABLE_ENTRY_DATA(ENTRY))
+
+#define _Py_HASHTABLE_ENTRY_READ_DATA(TABLE, DATA, DATA_SIZE, ENTRY) \
+    do { \
+        assert((DATA_SIZE) == (TABLE)->data_size); \
+        memcpy(DATA, _PY_HASHTABLE_ENTRY_DATA(ENTRY), DATA_SIZE); \
+    } while (0)
+
+typedef Py_uhash_t (*_Py_hashtable_hash_func) (const void *key);
+typedef int (*_Py_hashtable_compare_func) (const void *key, const _Py_hashtable_entry_t *he);
+typedef void* (*_Py_hashtable_copy_data_func)(void *data);
+typedef void (*_Py_hashtable_free_data_func)(void *data);
+typedef size_t (*_Py_hashtable_get_data_size_func)(void *data);
+
+typedef struct {
+    /* allocate a memory block */
+    void* (*malloc) (size_t size);
+
+    /* release a memory block */
+    void (*free) (void *ptr);
+} _Py_hashtable_allocator_t;
+
+typedef struct {
+    size_t num_buckets;
+    size_t entries; /* Total number of entries in the table. */
+    _Py_slist_t *buckets;
+    size_t data_size;
+
+    _Py_hashtable_hash_func hash_func;
+    _Py_hashtable_compare_func compare_func;
+    _Py_hashtable_copy_data_func copy_data_func;
+    _Py_hashtable_free_data_func free_data_func;
+    _Py_hashtable_get_data_size_func get_data_size_func;
+    _Py_hashtable_allocator_t alloc;
+} _Py_hashtable_t;
+
+/* hash and compare functions for integers and pointers */
+PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_ptr(const void *key);
+PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_int(const void *key);
+PyAPI_FUNC(int) _Py_hashtable_compare_direct(const void *key, const _Py_hashtable_entry_t *entry);
+
+PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new(
+    size_t data_size,
+    _Py_hashtable_hash_func hash_func,
+    _Py_hashtable_compare_func compare_func);
+PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new_full(
+    size_t data_size,
+    size_t init_size,
+    _Py_hashtable_hash_func hash_func,
+    _Py_hashtable_compare_func compare_func,
+    _Py_hashtable_copy_data_func copy_data_func,
+    _Py_hashtable_free_data_func free_data_func,
+    _Py_hashtable_get_data_size_func get_data_size_func,
+    _Py_hashtable_allocator_t *allocator);
+PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_copy(_Py_hashtable_t *src);
+PyAPI_FUNC(void) _Py_hashtable_clear(_Py_hashtable_t *ht);
+PyAPI_FUNC(void) _Py_hashtable_destroy(_Py_hashtable_t *ht);
+
+typedef int (*_Py_hashtable_foreach_func) (_Py_hashtable_entry_t *entry, void *arg);
+
+PyAPI_FUNC(int) _Py_hashtable_foreach(
+    _Py_hashtable_t *ht,
+    _Py_hashtable_foreach_func func, void *arg);
+PyAPI_FUNC(size_t) _Py_hashtable_size(_Py_hashtable_t *ht);
+
+PyAPI_FUNC(_Py_hashtable_entry_t*) _Py_hashtable_get_entry(
+    _Py_hashtable_t *ht,
+    const void *key);
+PyAPI_FUNC(int) _Py_hashtable_set(
+    _Py_hashtable_t *ht,
+    const void *key,
+    void *data,
+    size_t data_size);
+PyAPI_FUNC(int) _Py_hashtable_get(
+    _Py_hashtable_t *ht,
+    const void *key,
+    void *data,
+    size_t data_size);
+PyAPI_FUNC(int) _Py_hashtable_pop(
+    _Py_hashtable_t *ht,
+    const void *key,
+    void *data,
+    size_t data_size);
+PyAPI_FUNC(void) _Py_hashtable_delete(
+    _Py_hashtable_t *ht,
+    const void *key);
+
+#define _Py_HASHTABLE_SET(TABLE, KEY, DATA) \
+    _Py_hashtable_set(TABLE, KEY, &(DATA), sizeof(DATA))
+
+#define _Py_HASHTABLE_GET(TABLE, KEY, DATA) \
+    _Py_hashtable_get(TABLE, KEY, &(DATA), sizeof(DATA))
+
+#endif   /* Py_LIMITED_API */
+
+#endif

PC/config.c

@@ -13,6 +13,7 @@ extern PyObject* PyInit_binascii(void);
 extern PyObject* PyInit_cmath(void);
 extern PyObject* PyInit_errno(void);
 extern PyObject* PyInit_faulthandler(void);
+extern PyObject* PyInit__tracemalloc(void);
 extern PyObject* PyInit_gc(void);
 extern PyObject* PyInit_math(void);
 extern PyObject* PyInit__md5(void);
@@ -102,6 +103,7 @@ struct _inittab _PyImport_Inittab[] = {
     {"msvcrt", PyInit_msvcrt},
     {"_locale", PyInit__locale},
 #endif
+    {"_tracemalloc", PyInit__tracemalloc},
     /* XXX Should _winapi go in a WIN32 block?  not WIN64? */
     {"_winapi", PyInit__winapi},
 

PCbuild/pythoncore.vcxproj

@@ -449,6 +449,7 @@
     <ClInclude Include="..\Include\unicodeobject.h" />
     <ClInclude Include="..\Include\weakrefobject.h" />
     <ClInclude Include="..\Modules\_math.h" />
+    <ClInclude Include="..\Modules\hashtable.h" />
     <ClInclude Include="..\Modules\rotatingtree.h" />
     <ClInclude Include="..\Modules\sre.h" />
     <ClInclude Include="..\Modules\sre_constants.h" />
@@ -517,6 +518,7 @@
     <ClCompile Include="..\Modules\errnomodule.c" />
     <ClCompile Include="..\Modules\faulthandler.c" />
     <ClCompile Include="..\Modules\gcmodule.c" />
+    <ClCompile Include="..\Modules\hashtable.c" />
     <ClCompile Include="..\Modules\itertoolsmodule.c" />
     <ClCompile Include="..\Modules\main.c" />
     <ClCompile Include="..\Modules\mathmodule.c" />
@@ -532,6 +534,7 @@
     <ClCompile Include="..\Modules\signalmodule.c" />
     <ClCompile Include="..\Modules\symtablemodule.c" />
     <ClCompile Include="..\Modules\_threadmodule.c" />
+    <ClCompile Include="..\Modules\_tracemalloc.c" />
     <ClCompile Include="..\Modules\timemodule.c" />
     <ClCompile Include="..\Modules\xxsubtype.c" />
     <ClCompile Include="..\Modules\zipimport.c" />
@@ -684,4 +687,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>
\ No newline at end of file

Python/pythonrun.c

@@ -105,6 +105,7 @@ extern void PyLong_Fini(void);
 extern int _PyFaulthandler_Init(void);
 extern void _PyFaulthandler_Fini(void);
 extern void _PyHash_Fini(void);
+extern int _PyTraceMalloc_Init(void);
 
 #ifdef WITH_THREAD
 extern void _PyGILState_Init(PyInterpreterState *, PyThreadState *);
@@ -454,6 +455,9 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
     if (install_sigs)
         initsigs(); /* Signal handling stuff, including initintr() */
 
+    if (_PyTraceMalloc_Init() < 0)
+        Py_FatalError("Py_Initialize: can't initialize tracemalloc");
+
     initmain(interp); /* Module __main__ */
     if (initstdio() < 0)
         Py_FatalError(