bpo-34170, test_embed: write Py_Initialize() tests (GH-8484)

Include/pystate.h

@@ -55,7 +55,12 @@ typedef struct {
     int malloc_stats;       /* PYTHONMALLOCSTATS */
     int coerce_c_locale;    /* PYTHONCOERCECLOCALE, -1 means unknown */
     int coerce_c_locale_warn; /* PYTHONCOERCECLOCALE=warn */
-    int utf8_mode;          /* PYTHONUTF8, -X utf8; -1 means unknown */
+
+    /* Enable UTF-8 mode?
+       Set by -X utf8 command line option and PYTHONUTF8 environment variable.
+       If set to -1 (default), inherit Py_UTF8Mode value. */
+    int utf8_mode;
+
     wchar_t *pycache_prefix; /* PYTHONPYCACHEPREFIX, -X pycache_prefix=PATH */
 
     wchar_t *program_name;  /* Program name, see also Py_GetProgramName() */

Lib/test/test_embed.py

@@ -9,7 +9,7 @@ import subprocess
 import sys
 
 
-class EmbeddingTests(unittest.TestCase):
+class EmbeddingTestsMixin:
     def setUp(self):
         here = os.path.abspath(__file__)
         basepath = os.path.dirname(os.path.dirname(os.path.dirname(here)))
@@ -110,6 +110,8 @@ class EmbeddingTests(unittest.TestCase):
                 yield current_run
                 current_run = []
 
+
+class EmbeddingTests(EmbeddingTestsMixin, unittest.TestCase):
     def test_subinterps_main(self):
         for run in self.run_repeated_init_and_subinterpreters():
             main = run[0]
@@ -247,5 +249,160 @@ class EmbeddingTests(unittest.TestCase):
         self.assertEqual(err, '')
 
 
+class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
+    maxDiff = 4096
+    DEFAULT_CONFIG = {
+        'install_signal_handlers': 1,
+        'use_environment': 1,
+        'use_hash_seed': 0,
+        'hash_seed': 0,
+        'allocator': '(null)',
+        'dev_mode': 0,
+        'faulthandler': 0,
+        'tracemalloc': 0,
+        'import_time': 0,
+        'show_ref_count': 0,
+        'show_alloc_count': 0,
+        'dump_refs': 0,
+        'malloc_stats': 0,
+        'utf8_mode': 0,
+
+        'coerce_c_locale': 0,
+        'coerce_c_locale_warn': 0,
+
+        'pycache_prefix': '(null)',
+        'program_name': './_testembed',
+        'program': '(null)',
+
+        'isolated': 0,
+        'site_import': 1,
+        'bytes_warning': 0,
+        'inspect': 0,
+        'interactive': 0,
+        'optimization_level': 0,
+        'debug': 0,
+        'write_bytecode': 1,
+        'verbose': 0,
+        'quiet': 0,
+        'user_site_directory': 1,
+        'unbuffered_stdio': 0,
+
+        '_install_importlib': 1,
+        '_check_hash_pycs_mode': 'default',
+    }
+
+    def check_config(self, testname, expected):
+        env = dict(os.environ)
+        for key in list(env):
+            if key.startswith('PYTHON'):
+                del env[key]
+        # Disable C locale coercion and UTF-8 mode to not depend
+        # on the current locale
+        env['PYTHONCOERCECLOCALE'] = '0'
+        env['PYTHONUTF8'] = '0'
+        out, err = self.run_embedded_interpreter(testname, env=env)
+        # Ignore err
+
+        expected = dict(self.DEFAULT_CONFIG, **expected)
+        for key, value in expected.items():
+            expected[key] = str(value)
+
+        config = {}
+        for line in out.splitlines():
+            key, value = line.split(' = ', 1)
+            config[key] = value
+        self.assertEqual(config, expected)
+
+    def test_init_default_config(self):
+        self.check_config("init_default_config", {})
+
+    def test_init_global_config(self):
+        config = {
+            'program_name': './globalvar',
+            'site_import': 0,
+            'bytes_warning': 1,
+            'inspect': 1,
+            'interactive': 1,
+            'optimization_level': 2,
+            'write_bytecode': 0,
+            'verbose': 1,
+            'quiet': 1,
+            'unbuffered_stdio': 1,
+            'utf8_mode': 1,
+            'user_site_directory': 0,
+        }
+        self.check_config("init_global_config", config)
+
+    def test_init_from_config(self):
+        config = {
+            'install_signal_handlers': 0,
+            'use_hash_seed': 1,
+            'hash_seed': 123,
+            'allocator': 'malloc_debug',
+            'tracemalloc': 2,
+            'import_time': 1,
+            'show_ref_count': 1,
+            'show_alloc_count': 1,
+            'malloc_stats': 1,
+
+            'utf8_mode': 1,
+
+            'pycache_prefix': 'conf_pycache_prefix',
+            'program_name': './conf_program_name',
+            'program': 'conf_program',
+
+            'site_import': 0,
+            'bytes_warning': 1,
+            'inspect': 1,
+            'interactive': 1,
+            'optimization_level': 2,
+            'write_bytecode': 0,
+            'verbose': 1,
+            'quiet': 1,
+            'unbuffered_stdio': 1,
+            'user_site_directory': 0,
+            'faulthandler': 1,
+            '_check_hash_pycs_mode': 'always',
+        }
+        self.check_config("init_from_config", config)
+
+    def test_init_env(self):
+        config = {
+            'use_hash_seed': 1,
+            'hash_seed': 42,
+            'allocator': 'malloc_debug',
+            'tracemalloc': 2,
+            'import_time': 1,
+            'malloc_stats': 1,
+            'utf8_mode': 1,
+            'inspect': 1,
+            'optimization_level': 2,
+            'pycache_prefix': 'env_pycache_prefix',
+            'write_bytecode': 0,
+            'verbose': 1,
+            'unbuffered_stdio': 1,
+            'user_site_directory': 0,
+            'faulthandler': 1,
+            'dev_mode': 1,
+        }
+        self.check_config("init_env", config)
+
+    def test_init_dev_mode(self):
+        config = {
+            'dev_mode': 1,
+            'faulthandler': 1,
+            'allocator': 'debug',
+        }
+        self.check_config("init_dev_mode", config)
+
+    def test_init_isolated(self):
+        config = {
+            'isolated': 1,
+            'use_environment': 0,
+            'user_site_directory': 0,
+        }
+        self.check_config("init_isolated", config)
+
+
 if __name__ == "__main__":
     unittest.main()

Programs/_testembed.c

@@ -292,6 +292,331 @@ static int test_initialize_pymain(void)
 }
 
 
+static void
+dump_config(void)
+{
+#define ASSERT_EQUAL(a, b) \
+    if ((a) != (b)) { \
+        printf("ERROR: %s != %s (%i != %i)\n", #a, #b, (a), (b)); \
+        exit(1); \
+    }
+#define ASSERT_STR_EQUAL(a, b) \
+    if ((a) == NULL || (b == NULL) || wcscmp((a), (b)) != 0) { \
+        printf("ERROR: %s != %s ('%ls' != '%ls')\n", #a, #b, (a), (b)); \
+        exit(1); \
+    }
+
+    PyInterpreterState *interp = PyThreadState_Get()->interp;
+    _PyCoreConfig *config = &interp->core_config;
+
+    printf("install_signal_handlers = %i\n", config->install_signal_handlers);
+
+    printf("use_environment = %i\n", config->use_environment);
+    ASSERT_EQUAL(config->use_environment, !Py_IgnoreEnvironmentFlag);
+
+    printf("use_hash_seed = %i\n", config->use_hash_seed);
+    printf("hash_seed = %lu\n", config->hash_seed);
+
+    printf("allocator = %s\n", config->allocator);
+
+    printf("dev_mode = %i\n", config->dev_mode);
+    printf("faulthandler = %i\n", config->faulthandler);
+    printf("tracemalloc = %i\n", config->tracemalloc);
+    printf("import_time = %i\n", config->import_time);
+    printf("show_ref_count = %i\n", config->show_ref_count);
+    printf("show_alloc_count = %i\n", config->show_alloc_count);
+    printf("dump_refs = %i\n", config->dump_refs);
+    printf("malloc_stats = %i\n", config->malloc_stats);
+
+    printf("coerce_c_locale = %i\n", config->coerce_c_locale);
+    printf("coerce_c_locale_warn = %i\n", config->coerce_c_locale_warn);
+    printf("utf8_mode = %i\n", config->utf8_mode);
+
+    printf("pycache_prefix = %ls\n", config->pycache_prefix);
+    printf("program_name = %ls\n", config->program_name);
+    ASSERT_STR_EQUAL(config->program_name, Py_GetProgramName());
+    /* FIXME: test argc/argv */
+    printf("program = %ls\n", config->program);
+    /* FIXME: test xoptions */
+    /* FIXME: test warnoptions */
+    /* FIXME: test module_search_path_env */
+    /* FIXME: test home */
+    /* FIXME: test module_search_paths */
+    /* FIXME: test executable */
+    /* FIXME: test prefix */
+    /* FIXME: test base_prefix */
+    /* FIXME: test exec_prefix */
+    /* FIXME: test base_exec_prefix */
+    /* FIXME: test dll_path */
+
+    printf("isolated = %i\n", config->isolated);
+    ASSERT_EQUAL(config->isolated, Py_IsolatedFlag);
+    printf("site_import = %i\n", config->site_import);
+    printf("bytes_warning = %i\n", config->bytes_warning);
+    printf("inspect = %i\n", config->inspect);
+    printf("interactive = %i\n", config->interactive);
+    printf("optimization_level = %i\n", config->optimization_level);
+    printf("debug = %i\n", config->debug);
+    printf("write_bytecode = %i\n", config->write_bytecode);
+    printf("verbose = %i\n", config->verbose);
+    ASSERT_EQUAL(config->verbose, Py_VerboseFlag);
+    printf("quiet = %i\n", config->quiet);
+    printf("user_site_directory = %i\n", config->user_site_directory);
+    printf("unbuffered_stdio = %i\n", config->unbuffered_stdio);
+    /* FIXME: test legacy_windows_fs_encoding */
+    /* FIXME: test legacy_windows_stdio */
+
+    printf("_install_importlib = %i\n", config->_install_importlib);
+    printf("_check_hash_pycs_mode = %s\n", config->_check_hash_pycs_mode);
+
+#undef ASSERT_EQUAL
+#undef ASSERT_STR_EQUAL
+}
+
+
+static int test_init_default_config(void)
+{
+    _testembed_Py_Initialize();
+    dump_config();
+    Py_Finalize();
+    return 0;
+}
+
+
+static int test_init_global_config(void)
+{
+    /* FIXME: test Py_IgnoreEnvironmentFlag */
+
+    putenv("PYTHONUTF8=0");
+    Py_UTF8Mode = 1;
+
+    /* Test initialization from global configuration variables (Py_xxx) */
+    Py_SetProgramName(L"./globalvar");
+
+    /* Py_IsolatedFlag is not tested */
+    Py_NoSiteFlag = 1;
+    Py_BytesWarningFlag = 1;
+
+    putenv("PYTHONINSPECT=");
+    Py_InspectFlag = 1;
+
+    putenv("PYTHONOPTIMIZE=0");
+    Py_InteractiveFlag = 1;
+
+    putenv("PYTHONDEBUG=0");
+    Py_OptimizeFlag = 2;
+
+    /* Py_DebugFlag is not tested */
+
+    putenv("PYTHONDONTWRITEBYTECODE=");
+    Py_DontWriteBytecodeFlag = 1;
+
+    putenv("PYTHONVERBOSE=0");
+    Py_VerboseFlag = 1;
+
+    Py_QuietFlag = 1;
+    Py_NoUserSiteDirectory = 1;
+
+    putenv("PYTHONUNBUFFERED=");
+    Py_UnbufferedStdioFlag = 1;
+
+    /* FIXME: test Py_LegacyWindowsFSEncodingFlag */
+    /* FIXME: test Py_LegacyWindowsStdioFlag */
+
+    /* _Py_CheckHashBasedPycsMode is not public, and so not tested */
+
+    Py_Initialize();
+    dump_config();
+    Py_Finalize();
+    return 0;
+}
+
+
+static int test_init_from_config(void)
+{
+    /* Test _Py_InitializeFromConfig() */
+    _PyCoreConfig config = _PyCoreConfig_INIT;
+    config.install_signal_handlers = 0;
+
+    /* FIXME: test use_environment */
+
+    putenv("PYTHONHASHSEED=42");
+    config.use_hash_seed = 1;
+    config.hash_seed = 123;
+
+    putenv("PYTHONMALLOC=malloc");
+    config.allocator = "malloc_debug";
+
+    /* dev_mode=1 is tested in test_init_dev_mode() */
+
+    putenv("PYTHONFAULTHANDLER=");
+    config.faulthandler = 1;
+
+    putenv("PYTHONTRACEMALLOC=0");
+    config.tracemalloc = 2;
+
+    putenv("PYTHONPROFILEIMPORTTIME=0");
+    config.import_time = 1;
+
+    config.show_ref_count = 1;
+    config.show_alloc_count = 1;
+    /* FIXME: test dump_refs: bpo-34223 */
+
+    putenv("PYTHONMALLOCSTATS=0");
+    config.malloc_stats = 1;
+
+    /* FIXME: test coerce_c_locale and coerce_c_locale_warn */
+
+    putenv("PYTHONUTF8=0");
+    Py_UTF8Mode = 0;
+    config.utf8_mode = 1;
+
+    putenv("PYTHONPYCACHEPREFIX=env_pycache_prefix");
+    config.pycache_prefix = L"conf_pycache_prefix";
+
+    Py_SetProgramName(L"./globalvar");
+    config.program_name = L"./conf_program_name";
+
+    /* FIXME: test argc/argv */
+    config.program = L"conf_program";
+    /* FIXME: test xoptions */
+    /* FIXME: test warnoptions */
+    /* FIXME: test module_search_path_env */
+    /* FIXME: test home */
+    /* FIXME: test path config: module_search_path .. dll_path */
+
+    putenv("PYTHONVERBOSE=0");
+    Py_VerboseFlag = 0;
+    config.verbose = 1;
+
+    Py_NoSiteFlag = 0;
+    config.site_import = 0;
+
+    Py_BytesWarningFlag = 0;
+    config.bytes_warning = 1;
+
+    putenv("PYTHONINSPECT=");
+    Py_InspectFlag = 0;
+    config.inspect = 1;
+
+    Py_InteractiveFlag = 0;
+    config.interactive = 1;
+
+    putenv("PYTHONOPTIMIZE=0");
+    Py_OptimizeFlag = 1;
+    config.optimization_level = 2;
+
+    /* FIXME: test debug */
+
+    putenv("PYTHONDONTWRITEBYTECODE=");
+    Py_DontWriteBytecodeFlag = 0;
+    config.write_bytecode = 0;
+
+    Py_QuietFlag = 0;
+    config.quiet = 1;
+
+    putenv("PYTHONUNBUFFERED=");
+    Py_UnbufferedStdioFlag = 0;
+    config.unbuffered_stdio = 1;
+
+    putenv("PYTHONNOUSERSITE=");
+    Py_NoUserSiteDirectory = 0;
+    config.user_site_directory = 0;
+
+    config._check_hash_pycs_mode = "always";
+
+    _PyInitError err = _Py_InitializeFromConfig(&config);
+    /* Don't call _PyCoreConfig_Clear() since all strings are static */
+    if (_Py_INIT_FAILED(err)) {
+        _Py_FatalInitError(err);
+    }
+    dump_config();
+    Py_Finalize();
+    return 0;
+}
+
+
+static void test_init_env_putenvs(void)
+{
+    putenv("PYTHONHASHSEED=42");
+    putenv("PYTHONMALLOC=malloc_debug");
+    putenv("PYTHONTRACEMALLOC=2");
+    putenv("PYTHONPROFILEIMPORTTIME=1");
+    putenv("PYTHONMALLOCSTATS=1");
+    putenv("PYTHONUTF8=1");
+    putenv("PYTHONVERBOSE=1");
+    putenv("PYTHONINSPECT=1");
+    putenv("PYTHONOPTIMIZE=2");
+    putenv("PYTHONDONTWRITEBYTECODE=1");
+    putenv("PYTHONUNBUFFERED=1");
+    putenv("PYTHONPYCACHEPREFIX=env_pycache_prefix");
+    putenv("PYTHONNOUSERSITE=1");
+    putenv("PYTHONFAULTHANDLER=1");
+    putenv("PYTHONDEVMODE=1");
+    /* FIXME: test PYTHONWARNINGS */
+    /* FIXME: test PYTHONEXECUTABLE */
+    /* FIXME: test PYTHONHOME */
+    /* FIXME: test PYTHONDEBUG */
+    /* FIXME: test PYTHONDUMPREFS */
+    /* FIXME: test PYTHONCOERCECLOCALE */
+    /* FIXME: test PYTHONPATH */
+}
+
+
+static int test_init_env(void)
+{
+    /* Test initialization from environment variables */
+    Py_IgnoreEnvironmentFlag = 0;
+    test_init_env_putenvs();
+    _testembed_Py_Initialize();
+    dump_config();
+    Py_Finalize();
+    return 0;
+}
+
+
+static int test_init_isolated(void)
+{
+    /* Test _PyCoreConfig.isolated=1 */
+    _PyCoreConfig config = _PyCoreConfig_INIT;
+
+    /* Set coerce_c_locale and utf8_mode to not depend on the locale */
+    config.coerce_c_locale = 0;
+    config.utf8_mode = 0;
+    /* Use path starting with "./" avoids a search along the PATH */
+    config.program_name = L"./_testembed";
+
+    Py_IsolatedFlag = 0;
+    config.isolated = 1;
+
+    test_init_env_putenvs();
+    _PyInitError err = _Py_InitializeFromConfig(&config);
+    if (_Py_INIT_FAILED(err)) {
+        _Py_FatalInitError(err);
+    }
+    dump_config();
+    Py_Finalize();
+    return 0;
+}
+
+
+static int test_init_dev_mode(void)
+{
+    _PyCoreConfig config = _PyCoreConfig_INIT;
+    putenv("PYTHONFAULTHANDLER=");
+    putenv("PYTHONMALLOC=");
+    config.dev_mode = 1;
+    config.program_name = L"./_testembed";
+    _PyInitError err = _Py_InitializeFromConfig(&config);
+    if (_Py_INIT_FAILED(err)) {
+        _Py_FatalInitError(err);
+    }
+    dump_config();
+    Py_Finalize();
+    return 0;
+}
+
+
 /* *********************************************************
  * List of test cases and the function that implements it.
  *
@@ -318,6 +643,12 @@ static struct TestCase TestCases[] = {
     { "bpo20891", test_bpo20891 },
     { "initialize_twice", test_initialize_twice },
     { "initialize_pymain", test_initialize_pymain },
+    { "init_default_config", test_init_default_config },
+    { "init_global_config", test_init_global_config },
+    { "init_from_config", test_init_from_config },
+    { "init_env", test_init_env },
+    { "init_dev_mode", test_init_dev_mode },
+    { "init_isolated", test_init_isolated },
     { NULL, NULL }
 };