bpo-34170: Add _Py_InitializeFromConfig() (GH-8454)

* If _Py_InitializeCore() is called twice, the second call now copies
  and apply (partially) the new configuration.
* Rename _Py_CommandLineDetails to _PyCmdline
* Move more code into pymain_init(). The core configuration created
  by Py_Main() is new destroyed before running Python to reduce the
  memory footprint.
* _Py_InitializeCore() now returns the created interpreter.
  _Py_InitializeMainInterpreter() now expects an interpreter.
* Remove _Py_InitializeEx_Private(): _freeze_importlib now uses
  _Py_InitializeFromConfig()
* _PyCoreConfig_InitPathConfig() now only computes the path
  configuration if needed.

Include/pylifecycle.h

@@ -51,7 +51,9 @@ PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding,
                                              const char *errors);
 
 /* PEP 432 Multi-phase initialization API (Private while provisional!) */
-PyAPI_FUNC(_PyInitError) _Py_InitializeCore(const _PyCoreConfig *);
+PyAPI_FUNC(_PyInitError) _Py_InitializeCore(
+    PyInterpreterState **interp,
+    const _PyCoreConfig *);
 PyAPI_FUNC(int) _Py_IsCoreInitialized(void);
 
 PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *config);
@@ -73,14 +75,17 @@ PyAPI_FUNC(int) _PyMainInterpreterConfig_Copy(
     _PyMainInterpreterConfig *config,
     const _PyMainInterpreterConfig *config2);
 
-PyAPI_FUNC(_PyInitError) _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *);
+PyAPI_FUNC(_PyInitError) _Py_InitializeMainInterpreter(
+    PyInterpreterState *interp,
+    const _PyMainInterpreterConfig *);
 #endif
 
 /* Initialization and finalization */
 PyAPI_FUNC(void) Py_Initialize(void);
 PyAPI_FUNC(void) Py_InitializeEx(int);
 #ifndef Py_LIMITED_API
-PyAPI_FUNC(_PyInitError) _Py_InitializeEx_Private(int, int);
+PyAPI_FUNC(_PyInitError) _Py_InitializeFromConfig(
+    const _PyCoreConfig *config);
 PyAPI_FUNC(void) _Py_FatalInitError(_PyInitError err) _Py_NO_RETURN;
 #endif
 PyAPI_FUNC(void) Py_Finalize(void);

Include/pystate.h

@@ -200,8 +200,7 @@ typedef struct {
     /* --- Private fields -------- */
 
     /* Install importlib? If set to 0, importlib is not initialized at all.
-       Needed by freeze_importlib: see install_importlib argument of
-       _Py_InitializeEx_Private(). */
+       Needed by freeze_importlib. */
     int _install_importlib;
 
     /* Value of the --check-hash-based-pycs configure option. Valid values:

Programs/_freeze_importlib.c

@@ -74,14 +74,20 @@ main(int argc, char *argv[])
     }
     text[text_size] = '\0';
 
-    Py_NoUserSiteDirectory++;
-    Py_NoSiteFlag++;
-    Py_IgnoreEnvironmentFlag++;
+    _PyCoreConfig config = _PyCoreConfig_INIT;
+    config.user_site_directory = 0;
+    config.site_import = 0;
+    config.ignore_environment = 1;
+    config.program_name = L"./_freeze_importlib";
+    /* Don't install importlib, since it could execute outdated bytecode. */
+    config._install_importlib = 0;
+    config.install_signal_handlers = 1;
+
     Py_FrozenFlag++;
 
-    Py_SetProgramName(L"./_freeze_importlib");
-    /* Don't install importlib, since it could execute outdated bytecode. */
-    _PyInitError err = _Py_InitializeEx_Private(1, 0);
+    _PyInitError err = _Py_InitializeFromConfig(&config);
+    /* No need to call _PyCoreConfig_Clear() since we didn't allocate any
+       memory: program_name is a constant string. */
     if (_Py_INIT_FAILED(err)) {
         _Py_FatalInitError(err);
     }

Python/pathconfig.c

@@ -282,8 +282,8 @@ core_config_init_module_search_paths(_PyCoreConfig *config,
 }
 
 
-_PyInitError
-_PyCoreConfig_InitPathConfig(_PyCoreConfig *config)
+static _PyInitError
+_PyCoreConfig_CalculatePathConfig(_PyCoreConfig *config)
 {
     _PyPathConfig path_config = _PyPathConfig_INIT;
     _PyInitError err;
@@ -332,18 +332,6 @@ _PyCoreConfig_InitPathConfig(_PyCoreConfig *config)
     }
 #endif
 
-    if (config->base_prefix == NULL) {
-        if (copy_wstr(&config->base_prefix, config->prefix) < 0) {
-            goto no_memory;
-        }
-    }
-
-    if (config->base_exec_prefix == NULL) {
-        if (copy_wstr(&config->base_exec_prefix, config->exec_prefix) < 0) {
-            goto no_memory;
-        }
-    }
-
     if (path_config.isolated != -1) {
         config->isolated = path_config.isolated;
     }
@@ -363,6 +351,39 @@ error:
 }
 
 
+_PyInitError
+_PyCoreConfig_InitPathConfig(_PyCoreConfig *config)
+{
+    /* Do we need to calculate the path? */
+    if ((config->nmodule_search_path < 0)
+        || (config->executable == NULL)
+        || (config->prefix == NULL)
+#ifdef MS_WINDOWS
+        || (config->dll_path == NULL)
+#endif
+        || (config->exec_prefix == NULL))
+    {
+        _PyInitError err = _PyCoreConfig_CalculatePathConfig(config);
+        if (_Py_INIT_FAILED(err)) {
+            return err;
+        }
+    }
+
+    if (config->base_prefix == NULL) {
+        if (copy_wstr(&config->base_prefix, config->prefix) < 0) {
+            return _Py_INIT_NO_MEMORY();
+        }
+    }
+
+    if (config->base_exec_prefix == NULL) {
+        if (copy_wstr(&config->base_exec_prefix, config->exec_prefix) < 0) {
+            return _Py_INIT_NO_MEMORY();
+        }
+    }
+    return _Py_INIT_OK();
+}
+
+
 static void
 pathconfig_global_init(void)
 {

Python/pylifecycle.c

@@ -567,7 +567,7 @@ _Py_SetLocaleFromEnv(int category)
 /* Global initializations.  Can be undone by Py_Finalize().  Don't
    call this twice without an intervening Py_Finalize() call.
 
-   Every call to Py_InitializeCore, Py_Initialize or Py_InitializeEx
+   Every call to _Py_InitializeCore, Py_Initialize or Py_InitializeEx
    must have a corresponding call to Py_Finalize.
 
    Locking: you must hold the interpreter lock while calling these APIs.
@@ -576,6 +576,35 @@ _Py_SetLocaleFromEnv(int category)
 
 */
 
+static _PyInitError
+_Py_Initialize_ReconfigureCore(PyInterpreterState *interp,
+                               const _PyCoreConfig *core_config)
+{
+    if (core_config->allocator != NULL) {
+        const char *allocator = _PyMem_GetAllocatorsName();
+        if (allocator == NULL || strcmp(core_config->allocator, allocator) != 0) {
+            return _Py_INIT_USER_ERR("cannot modify memory allocator "
+                                     "after first Py_Initialize()");
+        }
+    }
+
+    _PyCoreConfig_SetGlobalConfig(core_config);
+
+    if (_PyCoreConfig_Copy(&interp->core_config, core_config) < 0) {
+        return _Py_INIT_ERR("failed to copy core config");
+    }
+    core_config = &interp->core_config;
+
+    if (core_config->_install_importlib) {
+        _PyInitError err = _PyCoreConfig_SetPathConfig(core_config);
+        if (_Py_INIT_FAILED(err)) {
+            return err;
+        }
+    }
+    return _Py_INIT_OK();
+}
+
+
 /* Begin interpreter initialization
  *
  * On return, the first thread and interpreter state have been created,
@@ -592,16 +621,41 @@ _Py_SetLocaleFromEnv(int category)
  * Any code invoked from this function should *not* assume it has access
  * to the Python C API (unless the API is explicitly listed as being
  * safe to call without calling Py_Initialize first)
+ *
+ * The caller is responsible to call _PyCoreConfig_Read().
  */
 
-_PyInitError
-_Py_InitializeCore(const _PyCoreConfig *core_config)
+static _PyInitError
+_Py_InitializeCore_impl(PyInterpreterState **interp_p,
+                        const _PyCoreConfig *core_config)
 {
-    assert(core_config != NULL);
+    PyInterpreterState *interp;
+    _PyInitError err;
+
+    /* bpo-34008: For backward compatibility reasons, calling Py_Main() after
+       Py_Initialize() ignores the new configuration. */
+    if (_PyRuntime.core_initialized) {
+        PyThreadState *tstate = PyThreadState_GET();
+        if (!tstate) {
+            return _Py_INIT_ERR("failed to read thread state");
+        }
+
+        interp = tstate->interp;
+        if (interp == NULL) {
+            return _Py_INIT_ERR("can't make main interpreter");
+        }
+        *interp_p = interp;
+
+        return _Py_Initialize_ReconfigureCore(interp, core_config);
+    }
+
+    if (_PyRuntime.initialized) {
+        return _Py_INIT_ERR("main interpreter already initialized");
+    }
 
     _PyCoreConfig_SetGlobalConfig(core_config);
 
-    _PyInitError err = _PyRuntime_Initialize();
+    err = _PyRuntime_Initialize();
     if (_Py_INIT_FAILED(err)) {
         return err;
     }
@@ -612,13 +666,6 @@ _Py_InitializeCore(const _PyCoreConfig *core_config)
         }
     }
 
-    if (_PyRuntime.initialized) {
-        return _Py_INIT_ERR("main interpreter already initialized");
-    }
-    if (_PyRuntime.core_initialized) {
-        return _Py_INIT_ERR("runtime core already initialized");
-    }
-
     /* Py_Finalize leaves _Py_Finalizing set in order to help daemon
      * threads behave a little more gracefully at interpreter shutdown.
      * We clobber it here so the new interpreter can start with a clean
@@ -648,10 +695,11 @@ _Py_InitializeCore(const _PyCoreConfig *core_config)
         return err;
     }
 
-    PyInterpreterState *interp = PyInterpreterState_New();
+    interp = PyInterpreterState_New();
     if (interp == NULL) {
         return _Py_INIT_ERR("can't make main interpreter");
     }
+    *interp_p = interp;
 
     if (_PyCoreConfig_Copy(&interp->core_config, core_config) < 0) {
         return _Py_INIT_ERR("failed to copy core config");
@@ -773,6 +821,43 @@ _Py_InitializeCore(const _PyCoreConfig *core_config)
     return _Py_INIT_OK();
 }
 
+_PyInitError
+_Py_InitializeCore(PyInterpreterState **interp_p,
+                   const _PyCoreConfig *src_config)
+{
+    assert(src_config != NULL);
+
+
+    PyMemAllocatorEx old_alloc;
+    _PyInitError err;
+
+    /* Copy the configuration, since _PyCoreConfig_Read() modifies it
+       (and the input configuration is read only). */
+    _PyCoreConfig config = _PyCoreConfig_INIT;
+
+    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+    if (_PyCoreConfig_Copy(&config, src_config) >= 0) {
+        err = _PyCoreConfig_Read(&config);
+    }
+    else {
+        err = _Py_INIT_ERR("failed to copy core config");
+    }
+    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+
+    if (_Py_INIT_FAILED(err)) {
+        goto done;
+    }
+
+    err = _Py_InitializeCore_impl(interp_p, &config);
+
+done:
+    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+    _PyCoreConfig_Clear(&config);
+    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+
+    return err;
+}
+
 /* Py_Initialize() has already been called: update the main interpreter
    configuration. Example of bpo-34008: Py_Main() called after
    Py_Initialize(). */
@@ -801,27 +886,19 @@ _Py_ReconfigureMainInterpreter(PyInterpreterState *interp,
  * non-zero return code.
  */
 _PyInitError
-_Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config)
+_Py_InitializeMainInterpreter(PyInterpreterState *interp,
+                              const _PyMainInterpreterConfig *config)
 {
     if (!_PyRuntime.core_initialized) {
         return _Py_INIT_ERR("runtime core not initialized");
     }
 
-    /* Get current thread state and interpreter pointer */
-    PyThreadState *tstate = PyThreadState_GET();
-    if (!tstate) {
-        return _Py_INIT_ERR("failed to read thread state");
-    }
-    PyInterpreterState *interp = tstate->interp;
-    if (!interp) {
-        return _Py_INIT_ERR("failed to get interpreter");
-    }
-    _PyCoreConfig *core_config = &interp->core_config;
-
-    /* Now finish configuring the main interpreter */
+    /* Configure the main interpreter */
     if (_PyMainInterpreterConfig_Copy(&interp->config, config) < 0) {
         return _Py_INIT_ERR("failed to copy main interpreter config");
     }
+    config = &interp->config;
+    _PyCoreConfig *core_config = &interp->core_config;
 
     if (_PyRuntime.initialized) {
         return _Py_ReconfigureMainInterpreter(interp, config);
@@ -908,51 +985,44 @@ _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config)
 #undef _INIT_DEBUG_PRINT
 
 _PyInitError
-_Py_InitializeEx_Private(int install_sigs, int install_importlib)
+_Py_InitializeFromConfig(const _PyCoreConfig *config)
 {
-    if (_PyRuntime.initialized) {
-        /* bpo-33932: Calling Py_Initialize() twice does nothing. */
-        return _Py_INIT_OK();
-    }
-
-    _PyCoreConfig config = _PyCoreConfig_INIT;
+    PyInterpreterState *interp;
     _PyInitError err;
-
-    config._install_importlib = install_importlib;
-    config.install_signal_handlers = install_sigs;
-
-    err = _PyCoreConfig_Read(&config);
-    if (_Py_INIT_FAILED(err)) {
-        goto done;
-    }
-
-    err = _Py_InitializeCore(&config);
+    err = _Py_InitializeCore(&interp, config);
     if (_Py_INIT_FAILED(err)) {
-        goto done;
+        return err;
     }
+    config = &interp->core_config;
 
     _PyMainInterpreterConfig main_config = _PyMainInterpreterConfig_INIT;
-    err = _PyMainInterpreterConfig_Read(&main_config, &config);
+    err = _PyMainInterpreterConfig_Read(&main_config, config);
     if (!_Py_INIT_FAILED(err)) {
-        err = _Py_InitializeMainInterpreter(&main_config);
+        err = _Py_InitializeMainInterpreter(interp, &main_config);
     }
     _PyMainInterpreterConfig_Clear(&main_config);
     if (_Py_INIT_FAILED(err)) {
-        goto done;
+        return err;
     }
-
-    err = _Py_INIT_OK();
-
-done:
-    _PyCoreConfig_Clear(&config);
-    return err;
+    return _Py_INIT_OK();
 }
 
 
 void
 Py_InitializeEx(int install_sigs)
 {
-    _PyInitError err = _Py_InitializeEx_Private(install_sigs, 1);
+    if (_PyRuntime.initialized) {
+        /* bpo-33932: Calling Py_Initialize() twice does nothing. */
+        return;
+    }
+
+    _PyInitError err;
+    _PyCoreConfig config = _PyCoreConfig_INIT;
+    config.install_signal_handlers = install_sigs;
+
+    err = _Py_InitializeFromConfig(&config);
+    _PyCoreConfig_Clear(&config);
+
     if (_Py_INIT_FAILED(err)) {
         _Py_FatalInitError(err);
     }