Revert "bpo-32197: Try to fix a compiler error on OS X introduced in bpo-32030. (#4681)" (#4694)

* Revert "bpo-32197: Try to fix a compiler error on OS X introduced in bpo-32030. (#4681)"

This reverts commit 13badcbc.

Re-apply commits:

* "bpo-32030: _PyPathConfig_Init() sets home and program_name (#4673)"
  commit af5a8950.
* "bpo-32030: Fix config_get_program_name() on macOS (#4669)"
  commit e23c06e2.
* "bpo-32030: Add Python/pathconfig.c (#4668)"
  commit 0ea395ae.
* "bpo-32030: Don't call _PyPathConfig_Fini() in Py_FinalizeEx() (#4667)"
  commit ebac19da.
* "bpo-32030: Fix Py_GetPath(): init program_name (#4665)"
  commit 9ac3d888.

* Fix compilation error on macOS

Doc/c-api/init.rst

@@ -40,7 +40,6 @@ The following functions can be safely called before Python is initialized:
   * :c:func:`Py_GetCompiler`
   * :c:func:`Py_GetCopyright`
   * :c:func:`Py_GetPlatform`
-  * :c:func:`Py_GetProgramName`
   * :c:func:`Py_GetVersion`
 
 * Utilities:
@@ -59,8 +58,8 @@ The following functions can be safely called before Python is initialized:
    The following functions **should not be called** before
    :c:func:`Py_Initialize`: :c:func:`Py_EncodeLocale`, :c:func:`Py_GetPath`,
    :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`,
-   :c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome` and
-   :c:func:`PyEval_InitThreads`.
+   :c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome`,
+   :c:func:`Py_GetProgramName` and :c:func:`PyEval_InitThreads`.
 
 
 .. _global-conf-vars:

Include/internal/pystate.h

@@ -48,9 +48,36 @@ typedef struct {
 #endif
     /* Set by Py_SetPath(), or computed by _PyPathConfig_Init() */
     wchar_t *module_search_path;
+    /* Python program name */
+    wchar_t *program_name;
+    /* Set by Py_SetPythonHome() or PYTHONHOME environment variable */
+    wchar_t *home;
 } _PyPathConfig;
 
-#define _PyPathConfig_INIT {.module_search_path = NULL}
+#ifdef MS_WINDOWS
+#define _PyPathConfig_INIT \
+    {.program_full_path = NULL, \
+     .prefix = NULL, \
+     .dll_path = NULL, \
+     .module_search_path = NULL, \
+     .program_name = NULL, \
+     .home = NULL}
+#else
+#define _PyPathConfig_INIT \
+    {.program_full_path = NULL, \
+     .prefix = NULL, \
+     .exec_prefix = NULL, \
+     .module_search_path = NULL, \
+     .program_name = NULL, \
+     .home = NULL}
+#endif
+
+PyAPI_DATA(_PyPathConfig) _Py_path_config;
+
+PyAPI_FUNC(_PyInitError) _PyPathConfig_Calculate(
+    _PyPathConfig *config,
+    const _PyMainInterpreterConfig *main_config);
+PyAPI_FUNC(void) _PyPathConfig_Clear(_PyPathConfig *config);
 
 
 /* Full Python runtime state */

Include/pylifecycle.h

@@ -105,11 +105,10 @@ PyAPI_FUNC(wchar_t *) Py_GetPath(void);
 #ifdef Py_BUILD_CORE
 PyAPI_FUNC(_PyInitError) _PyPathConfig_Init(
     const _PyMainInterpreterConfig *main_config);
-PyAPI_FUNC(void) _PyPathConfig_Fini(void);
 #endif
 PyAPI_FUNC(void)      Py_SetPath(const wchar_t *);
 #ifdef MS_WINDOWS
-int _Py_CheckPython3();
+int _Py_CheckPython3(void);
 #endif
 
 /* In their own files */

Include/pystate.h

@@ -72,7 +72,8 @@ typedef struct {
     (_PyMainInterpreterConfig){\
      .install_signal_handlers = -1, \
      .module_search_path_env = NULL, \
-     .home = NULL}
+     .home = NULL, \
+     .program_name = NULL}
 
 typedef struct _is {
 

Makefile.pre.in

@@ -337,8 +337,9 @@ PYTHON_OBJS=	\
 		Python/importdl.o \
 		Python/marshal.o \
 		Python/modsupport.o \
-		Python/mystrtoul.o \
 		Python/mysnprintf.o \
+		Python/mystrtoul.o \
+		Python/pathconfig.o \
 		Python/peephole.o \
 		Python/pyarena.o \
 		Python/pyctype.o \

Modules/main.c

@@ -412,7 +412,6 @@ typedef struct {
     /* non-zero if filename, command (-c) or module (-m) is set
        on the command line */
     int run_code;
-    wchar_t *program_name;
     /* Error message if a function failed */
     _PyInitError err;
     /* PYTHONWARNINGS env var */
@@ -429,7 +428,6 @@ typedef struct {
      .config = _PyMainInterpreterConfig_INIT, \
      .main_importer_path = NULL, \
      .run_code = -1, \
-     .program_name = NULL, \
      .err = _Py_INIT_OK(), \
      .env_warning_options = {0, NULL}}
 
@@ -455,7 +453,6 @@ pymain_free_impl(_PyMain *pymain)
 
     pymain_optlist_clear(&pymain->env_warning_options);
     Py_CLEAR(pymain->main_importer_path);
-    PyMem_RawFree(pymain->program_name);
 
     _PyMainInterpreterConfig_Clear(&pymain->config);
 
@@ -874,14 +871,21 @@ pymain_init_stdio(_PyMain *pymain)
 
 
 /* Get the program name: use PYTHONEXECUTABLE and __PYVENV_LAUNCHER__
-   environment variables on macOS if available, use argv[0] by default.
-
-   Return 0 on success.
-   Set pymain->err and return -1 on error. */
-static int
-pymain_get_program_name(_PyMain *pymain)
+   environment variables on macOS if available. */
+static _PyInitError
+config_get_program_name(_PyMainInterpreterConfig *config)
 {
-    assert(pymain->program_name == NULL);
+    assert(config->program_name == NULL);
+
+    /* If Py_SetProgramName() was called, use its value */
+    wchar_t *program_name = _Py_path_config.program_name;
+    if (program_name != NULL) {
+        config->program_name = _PyMem_RawWcsdup(program_name);
+        if (config->program_name == NULL) {
+            return _Py_INIT_NO_MEMORY();
+        }
+    }
+
 #ifdef __APPLE__
     char *p;
     /* On MacOS X, when the Python interpreter is embedded in an
@@ -894,17 +898,13 @@ pymain_get_program_name(_PyMain *pymain)
        See Lib/plat-mac/bundlebuiler.py for details about the bootstrap
        script. */
     if ((p = Py_GETENV("PYTHONEXECUTABLE")) && *p != '\0') {
-        wchar_t* buffer;
-        size_t len = strlen(p) + 1;
-
-        buffer = PyMem_RawMalloc(len * sizeof(wchar_t));
-        if (buffer == NULL) {
-            pymain->err = _Py_INIT_NO_MEMORY();
-            return -1;
+        size_t len;
+        wchar_t* program_name = Py_DecodeLocale(p, &len);
+        if (program_name == NULL) {
+            return DECODE_LOCALE_ERR("PYTHONEXECUTABLE environment "
+                                     "variable", (Py_ssize_t)len);
         }
-
-        mbstowcs(buffer, p, len);
-        pymain->program_name = buffer;
+        config->program_name = program_name;
     }
 #ifdef WITH_NEXT_FRAMEWORK
     else {
@@ -914,21 +914,30 @@ pymain_get_program_name(_PyMain *pymain)
              * the argv0 of the stub executable
              */
             size_t len;
-            wchar_t* wbuf = Py_DecodeLocale(pyvenv_launcher, &len);
-            if (wbuf == NULL) {
-                SET_DECODE_ERROR("__PYVENV_LAUNCHER__", len);
-                return -1;
+            wchar_t* program_name = Py_DecodeLocale(pyvenv_launcher, &len);
+            if (program_name == NULL) {
+                return DECODE_LOCALE_ERR("__PYVENV_LAUNCHER__ environment "
+                                         "variable", (Py_ssize_t)len);
             }
-            pymain->program_name = wbuf;
+            config->program_name = program_name;
         }
     }
 #endif   /* WITH_NEXT_FRAMEWORK */
 #endif   /* __APPLE__ */
 
-    if (pymain->program_name == NULL) {
+    return _Py_INIT_OK();
+}
+
+
+/* If config_get_program_name() found no program name: use argv[0] by default.
+   Return 0 on success. Set pymain->err and return -1 on error. */
+static int
+pymain_get_program_name(_PyMain *pymain)
+{
+    if (pymain->config.program_name == NULL) {
         /* Use argv[0] by default */
-        pymain->program_name = pymain_wstrdup(pymain, pymain->argv[0]);
-        if (pymain->program_name == NULL) {
+        pymain->config.program_name = pymain_wstrdup(pymain, pymain->argv[0]);
+        if (pymain->config.program_name == NULL) {
             return -1;
         }
     }
@@ -950,13 +959,6 @@ pymain_init_main_interpreter(_PyMain *pymain)
 {
     _PyInitError err;
 
-    /* TODO: Print any exceptions raised by these operations */
-    err = _PyMainInterpreterConfig_Read(&pymain->config);
-    if (_Py_INIT_FAILED(err)) {
-        pymain->err = err;
-        return -1;
-    }
-
     err = _Py_InitializeMainInterpreter(&pymain->config);
     if (_Py_INIT_FAILED(err)) {
         pymain->err = err;
@@ -1414,14 +1416,13 @@ config_init_pythonpath(_PyMainInterpreterConfig *config)
 
 
 static _PyInitError
-config_init_pythonhome(_PyMainInterpreterConfig *config)
+config_init_home(_PyMainInterpreterConfig *config)
 {
     wchar_t *home;
 
-    home = Py_GetPythonHome();
+    /* If Py_SetPythonHome() was called, use its value */
+    home = _Py_path_config.home;
     if (home) {
-        /* Py_SetPythonHome() has been called before Py_Main(),
-           use its value */
         config->home = _PyMem_RawWcsdup(home);
         if (config->home == NULL) {
             return _Py_INIT_NO_MEMORY();
@@ -1441,7 +1442,7 @@ config_init_pythonhome(_PyMainInterpreterConfig *config)
 _PyInitError
 _PyMainInterpreterConfig_ReadEnv(_PyMainInterpreterConfig *config)
 {
-    _PyInitError err = config_init_pythonhome(config);
+    _PyInitError err = config_init_home(config);
     if (_Py_INIT_FAILED(err)) {
         return err;
     }
@@ -1451,11 +1452,9 @@ _PyMainInterpreterConfig_ReadEnv(_PyMainInterpreterConfig *config)
         return err;
     }
 
-    /* FIXME: _PyMainInterpreterConfig_Read() has the same code. Remove it
-       here? See also pymain_get_program_name() and pymain_parse_envvars(). */
-    config->program_name = _PyMem_RawWcsdup(Py_GetProgramName());
-    if (config->program_name == NULL) {
-        return _Py_INIT_NO_MEMORY();
+    err = config_get_program_name(config);
+    if (_Py_INIT_FAILED(err)) {
+        return err;
     }
 
     return _Py_INIT_OK();
@@ -1481,25 +1480,17 @@ pymain_parse_envvars(_PyMain *pymain)
     if (pymain_warnings_envvar(pymain) < 0) {
         return -1;
     }
-    if (pymain_get_program_name(pymain) < 0) {
-        return -1;
-    }
-    core_config->allocator = Py_GETENV("PYTHONMALLOC");
-
-    /* FIXME: move pymain_get_program_name() code into
-       _PyMainInterpreterConfig_ReadEnv().
-       Problem: _PyMainInterpreterConfig_ReadEnv() doesn't have access
-       to argv[0]. */
-    Py_SetProgramName(pymain->program_name);
-    /* Don't free program_name here: the argument to Py_SetProgramName
-       must remain valid until Py_FinalizeEx is called. The string is freed
-       by pymain_free(). */
 
     _PyInitError err = _PyMainInterpreterConfig_ReadEnv(&pymain->config);
     if (_Py_INIT_FAILED(pymain->err)) {
         pymain->err = err;
         return -1;
     }
+    if (pymain_get_program_name(pymain) < 0) {
+        return -1;
+    }
+
+    core_config->allocator = Py_GETENV("PYTHONMALLOC");
 
     /* -X options */
     if (pymain_get_xoption(pymain, L"showrefcount")) {
@@ -1555,6 +1546,12 @@ pymain_parse_cmdline_envvars_impl(_PyMain *pymain)
         return -1;
     }
 
+    _PyInitError err = _PyMainInterpreterConfig_Read(&pymain->config);
+    if (_Py_INIT_FAILED(err)) {
+        pymain->err = err;
+        return -1;
+    }
+
     return 0;
 }
 
@@ -1671,6 +1668,14 @@ pymain_impl(_PyMain *pymain)
            other special meaning */
         pymain->status = 120;
     }
+
+    /* _PyPathConfig_Clear() cannot be called in Py_FinalizeEx().
+       Py_Initialize() and Py_Finalize() can be called multiple times, but it
+       must not "forget" parameters set by Py_SetProgramName(), Py_SetPath() or
+       Py_SetPythonHome(), whereas _PyPathConfig_Clear() clear all these
+       parameters. */
+    _PyPathConfig_Clear(&_Py_path_config);
+
     return 0;
 }
 

PCbuild/pythoncore.vcxproj

@@ -381,6 +381,7 @@
     <ClCompile Include="..\Python\modsupport.c" />
     <ClCompile Include="..\Python\mysnprintf.c" />
     <ClCompile Include="..\Python\mystrtoul.c" />
+    <ClCompile Include="..\Python\pathconfig.c" />
     <ClCompile Include="..\Python\peephole.c" />
     <ClCompile Include="..\Python\pyarena.c" />
     <ClCompile Include="..\Python\pyctype.c" />

PCbuild/pythoncore.vcxproj.filters

@@ -896,6 +896,9 @@
     <ClCompile Include="..\Python\mystrtoul.c">
       <Filter>Python</Filter>
     </ClCompile>
+    <ClCompile Include="..\Python\pathconfig.c">
+      <Filter>Python</Filter>
+    </ClCompile>
     <ClCompile Include="..\Python\peephole.c">
       <Filter>Python</Filter>
     </ClCompile>

Python/pathconfig.c

+/* Path configuration like module_search_path (sys.path) */
+
+#include "Python.h"
+#include "osdefs.h"
+#include "internal/pystate.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+_PyPathConfig _Py_path_config = _PyPathConfig_INIT;
+
+
+void
+_PyPathConfig_Clear(_PyPathConfig *config)
+{
+    /* _PyMem_SetDefaultAllocator() is needed to get a known memory allocator,
+       since Py_SetPath(), Py_SetPythonHome() and Py_SetProgramName() can be
+       called before Py_Initialize() which can changes the memory allocator. */
+    PyMemAllocatorEx old_alloc;
+    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+
+#define CLEAR(ATTR) \
+    do { \
+        PyMem_RawFree(ATTR); \
+        ATTR = NULL; \
+    } while (0)
+
+    CLEAR(config->prefix);
+    CLEAR(config->program_full_path);
+#ifdef MS_WINDOWS
+    CLEAR(config->dll_path);
+#else
+    CLEAR(config->exec_prefix);
+#endif
+    CLEAR(config->module_search_path);
+    CLEAR(config->home);
+    CLEAR(config->program_name);
+#undef CLEAR
+
+    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+}
+
+
+/* Initialize paths for Py_GetPath(), Py_GetPrefix(), Py_GetExecPrefix()
+   and Py_GetProgramFullPath() */
+_PyInitError
+_PyPathConfig_Init(const _PyMainInterpreterConfig *main_config)
+{
+    if (_Py_path_config.module_search_path) {
+        /* Already initialized */
+        return _Py_INIT_OK();
+    }
+
+    _PyInitError err;
+    _PyPathConfig new_config = _PyPathConfig_INIT;
+
+    PyMemAllocatorEx old_alloc;
+    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+
+    /* Calculate program_full_path, prefix, exec_prefix (Unix)
+       or dll_path (Windows), and module_search_path */
+    err = _PyPathConfig_Calculate(&new_config, main_config);
+    if (_Py_INIT_FAILED(err)) {
+        _PyPathConfig_Clear(&new_config);
+        goto done;
+    }
+
+    /* Copy home and program_name from main_config */
+    if (main_config->home != NULL) {
+        new_config.home = _PyMem_RawWcsdup(main_config->home);
+        if (new_config.home == NULL) {
+            err = _Py_INIT_NO_MEMORY();
+            goto done;
+        }
+    }
+    else {
+        new_config.home = NULL;
+    }
+
+    new_config.program_name = _PyMem_RawWcsdup(main_config->program_name);
+    if (new_config.program_name == NULL) {
+        err = _Py_INIT_NO_MEMORY();
+        goto done;
+    }
+
+    _PyPathConfig_Clear(&_Py_path_config);
+    _Py_path_config = new_config;
+
+    err = _Py_INIT_OK();
+
+done:
+    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+    return err;
+}
+
+
+static void
+pathconfig_global_init(void)
+{
+    if (_Py_path_config.module_search_path) {
+        /* Already initialized */
+        return;
+    }
+
+    _PyInitError err;
+    _PyMainInterpreterConfig config = _PyMainInterpreterConfig_INIT;
+
+    err = _PyMainInterpreterConfig_ReadEnv(&config);
+    if (_Py_INIT_FAILED(err)) {
+        goto error;
+    }
+
+    err = _PyMainInterpreterConfig_Read(&config);
+    if (_Py_INIT_FAILED(err)) {
+        goto error;
+    }
+
+    err = _PyPathConfig_Init(&config);
+    if (_Py_INIT_FAILED(err)) {
+        goto error;
+    }
+
+    _PyMainInterpreterConfig_Clear(&config);
+    return;
+
+error:
+    _PyMainInterpreterConfig_Clear(&config);
+    _Py_FatalInitError(err);
+}
+
+
+/* External interface */
+
+void
+Py_SetPath(const wchar_t *path)
+{
+    if (path == NULL) {
+        _PyPathConfig_Clear(&_Py_path_config);
+        return;
+    }
+
+    PyMemAllocatorEx old_alloc;
+    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+
+    _PyPathConfig new_config;
+    new_config.program_full_path = _PyMem_RawWcsdup(Py_GetProgramName());
+    new_config.prefix = _PyMem_RawWcsdup(L"");
+#ifdef MS_WINDOWS
+    new_config.dll_path = _PyMem_RawWcsdup(L"");
+#else
+    new_config.exec_prefix = _PyMem_RawWcsdup(L"");
+#endif
+    new_config.module_search_path = _PyMem_RawWcsdup(path);
+
+    /* steal the home and program_name values (to leave them unchanged) */
+    new_config.home = _Py_path_config.home;
+    _Py_path_config.home = NULL;
+    new_config.program_name = _Py_path_config.program_name;
+    _Py_path_config.program_name = NULL;
+
+    _PyPathConfig_Clear(&_Py_path_config);
+    _Py_path_config = new_config;
+
+    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+}
+
+
+void
+Py_SetPythonHome(wchar_t *home)
+{
+    if (home == NULL) {
+        return;
+    }
+
+    PyMemAllocatorEx old_alloc;
+    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+
+    PyMem_RawFree(_Py_path_config.home);
+    _Py_path_config.home = _PyMem_RawWcsdup(home);
+
+    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+
+    if (_Py_path_config.home == NULL) {
+        Py_FatalError("Py_SetPythonHome() failed: out of memory");
+    }
+}
+
+
+void
+Py_SetProgramName(wchar_t *program_name)
+{
+    if (program_name == NULL || program_name[0] == L'\0') {
+        return;
+    }
+
+    PyMemAllocatorEx old_alloc;
+    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+
+    PyMem_RawFree(_Py_path_config.program_name);
+    _Py_path_config.program_name = _PyMem_RawWcsdup(program_name);
+
+    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+
+    if (_Py_path_config.program_name == NULL) {
+        Py_FatalError("Py_SetProgramName() failed: out of memory");
+    }
+}
+
+
+wchar_t *
+Py_GetPath(void)
+{
+    pathconfig_global_init();
+    return _Py_path_config.module_search_path;
+}
+
+
+wchar_t *
+Py_GetPrefix(void)
+{
+    pathconfig_global_init();
+    return _Py_path_config.prefix;
+}
+
+
+wchar_t *
+Py_GetExecPrefix(void)
+{
+#ifdef MS_WINDOWS
+    return Py_GetPrefix();
+#else
+    pathconfig_global_init();
+    return _Py_path_config.exec_prefix;
+#endif
+}
+
+
+wchar_t *
+Py_GetProgramFullPath(void)
+{
+    pathconfig_global_init();
+    return _Py_path_config.program_full_path;
+}
+
+
+wchar_t*
+Py_GetPythonHome(void)
+{
+    pathconfig_global_init();
+    return _Py_path_config.home;
+}
+
+
+wchar_t *
+Py_GetProgramName(void)
+{
+    pathconfig_global_init();
+    return _Py_path_config.program_name;
+}
+
+
+#ifdef __cplusplus
+}
+#endif

Python/pylifecycle.c

@@ -804,7 +804,12 @@ _PyMainInterpreterConfig_Read(_PyMainInterpreterConfig *config)
     }
 
     if (config->program_name == NULL) {
-        config->program_name = _PyMem_RawWcsdup(Py_GetProgramName());
+#ifdef MS_WINDOWS
+        const wchar_t *program_name = L"python";
+#else
+        const wchar_t *program_name = L"python3";
+#endif
+        config->program_name = _PyMem_RawWcsdup(program_name);
         if (config->program_name == NULL) {
             return _Py_INIT_NO_MEMORY();
         }
@@ -1273,8 +1278,6 @@ Py_FinalizeEx(void)
 
     call_ll_exitfuncs();
 
-    _PyPathConfig_Fini();
-
     _PyRuntime_Finalize();
     return status;
 }
@@ -1491,61 +1494,6 @@ Py_EndInterpreter(PyThreadState *tstate)
     PyInterpreterState_Delete(interp);
 }
 
-#ifdef MS_WINDOWS
-static wchar_t *progname = L"python";
-#else
-static wchar_t *progname = L"python3";
-#endif
-
-void
-Py_SetProgramName(wchar_t *pn)
-{
-    if (pn && *pn)
-        progname = pn;
-}
-
-wchar_t *
-Py_GetProgramName(void)
-{
-    return progname;
-}
-
-static wchar_t *default_home = NULL;
-
-void
-Py_SetPythonHome(wchar_t *home)
-{
-    default_home = home;
-}
-
-
-wchar_t*
-Py_GetPythonHome(void)
-{
-    /* Use a static buffer to avoid heap memory allocation failure.
-       Py_GetPythonHome() doesn't allow to report error, and the caller
-       doesn't release memory. */
-    static wchar_t buffer[MAXPATHLEN+1];
-
-    if (default_home) {
-        return default_home;
-    }
-
-    char *home = Py_GETENV("PYTHONHOME");
-    if (!home) {
-        return NULL;
-    }
-
-    size_t size = Py_ARRAY_LENGTH(buffer);
-    size_t r = mbstowcs(buffer, home, size);
-    if (r == (size_t)-1 || r >= size) {
-        /* conversion failed or the static buffer is too small */
-        return NULL;
-    }
-
-    return buffer;
-}
-
 /* Add the __main__ module */
 
 static _PyInitError