Issue #14928: Fix importlib bootstrap issues by using a custom executable…

Issue #14928: Fix importlib bootstrap issues by using a custom executable (Modules/_freeze_importlib) to build Python/importlib.h.

.hgignore

@@ -79,6 +79,7 @@ PCbuild/x64-temp-*
 PCbuild/amd64
 BuildLog.htm
 __pycache__
+Modules/_freeze_importlib
 Modules/_testembed
 .coverage
 coverage/

Include/pythonrun.h

@@ -30,6 +30,9 @@ PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void);
 
 PyAPI_FUNC(void) Py_Initialize(void);
 PyAPI_FUNC(void) Py_InitializeEx(int);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(void) _Py_InitializeEx_Private(int, int);
+#endif
 PyAPI_FUNC(void) Py_Finalize(void);
 PyAPI_FUNC(int) Py_IsInitialized(void);
 PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void);

Makefile.pre.in

@@ -324,7 +324,6 @@ PYTHON_OBJS=	\
 		Python/codecs.o \
 		Python/dynamic_annotations.o \
 		Python/errors.o \
-		Python/frozen.o \
 		Python/frozenmain.o \
 		Python/future.o \
 		Python/getargs.o \
@@ -410,7 +409,7 @@ SYSCONFIGDATA=$(srcdir)/Lib/_sysconfigdata.py
 
 ##########################################################################
 # objects that get linked into the Python library
-LIBRARY_OBJS=	\
+LIBRARY_OBJS_OMIT_FROZEN=	\
 		Modules/getbuildinfo.o \
 		$(PARSER_OBJS) \
 		$(OBJECT_OBJS) \
@@ -419,6 +418,10 @@ LIBRARY_OBJS=	\
 		$(SIGNAL_OBJS) \
 		$(MODOBJS)
 
+LIBRARY_OBJS=	\
+		$(LIBRARY_OBJS_OMIT_FROZEN) \
+		Python/frozen.o
+
 #########################################################################
 # Rules
 
@@ -478,7 +481,7 @@ $(LIBRARY): $(LIBRARY_OBJS)
 	$(AR) $(ARFLAGS) $@ Modules/getbuildinfo.o
 	$(AR) $(ARFLAGS) $@ $(PARSER_OBJS)
 	$(AR) $(ARFLAGS) $@ $(OBJECT_OBJS)
-	$(AR) $(ARFLAGS) $@ $(PYTHON_OBJS)
+	$(AR) $(ARFLAGS) $@ $(PYTHON_OBJS) Python/frozen.o
 	$(AR) $(ARFLAGS) $@ $(MODULE_OBJS) $(SIGNAL_OBJS)
 	$(AR) $(ARFLAGS) $@ $(MODOBJS)
 	$(RANLIB) $@
@@ -578,18 +581,14 @@ Modules/_testembed: Modules/_testembed.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY)
 ############################################################################
 # Importlib
 
-Python/importlib.h: $(srcdir)/Lib/importlib/_bootstrap.py $(srcdir)/Python/freeze_importlib.py
-	@if test -f ./$(BUILDPYTHON); then \
-	    $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Python/freeze_importlib.py \
-		$(srcdir)/Lib/importlib/_bootstrap.py Python/importlib.h; \
-	else \
-	    echo "----------------------------------------------------------"; \
-	    echo "Python/importlib.h needs to be rebuilt, but no interpreter"; \
-	    echo "is available to do so. Leaving the previous version in"; \
-	    echo "place. You may want to run ''make'' a second time after"; \
-	    echo "this build is complete."; \
-	    echo "----------------------------------------------------------"; \
-	fi
+Modules/_freeze_importlib: Modules/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN)
+	$(LINKCC) $(PY_LDFLAGS) -o $@ Modules/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST)
+
+Python/importlib.h: $(srcdir)/Lib/importlib/_bootstrap.py Modules/_freeze_importlib.c
+	$(MAKE) Modules/_freeze_importlib
+	./Modules/_freeze_importlib \
+		$(srcdir)/Lib/importlib/_bootstrap.py Python/importlib.h
+
 
 ############################################################################
 # Special rules for object files
@@ -1389,7 +1388,7 @@ clean: pycremoval
 	find build -name 'fficonfig.py' -exec rm -f {} ';' || true
 	-rm -f Lib/lib2to3/*Grammar*.pickle
 	-rm -f $(SYSCONFIGDATA)
-	-rm -f Modules/_testembed
+	-rm -f Modules/_testembed Modules/_freeze_importlib
 
 profile-removal:
 	find . -name '*.gc??' -exec rm -f {} ';'

Misc/NEWS

@@ -154,6 +154,12 @@ Tests
 - Issue #14963: Add test cases for exception handling behaviour
   in contextlib.ExitStack (Initial patch by Alon Horev)
 
+Build
+-----
+
+- Issue #14928: Fix importlib bootstrap issues by using a custom executable
+  (Modules/_freeze_importlib) to build Python/importlib.h.
+
 
 What's New in Python 3.3.0 Alpha 4?
 ===================================

Modules/_freeze_importlib.c

+/* This is built as a stand-alone executable by the Makefile, and helps turn
+   Lib/importlib/_bootstrap.py into a frozen module in Python/importlib.h
+*/
+
+#include <Python.h>
+#include <marshal.h>
+
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+
+/* To avoid a circular dependency on frozen.o, we create our own structure
+   of frozen modules instead, left deliberately blank so as to avoid
+   unintentional import of a stale version of _frozen_importlib. */
+
+static struct _frozen _PyImport_FrozenModules[] = {
+    {0, 0, 0} /* sentinel */
+};
+
+struct _frozen *PyImport_FrozenModules = _PyImport_FrozenModules;
+
+
+const char header[] = "/* Auto-generated by Modules/_freeze_importlib.c */";
+
+int
+main(int argc, char *argv[])
+{
+    char *inpath, *outpath;
+    FILE *infile, *outfile = NULL;
+    struct stat st;
+    size_t text_size, data_size, n;
+    char *text, *data;
+    PyObject *code, *marshalled;
+
+    if (argc != 3) {
+        fprintf(stderr, "need to specify input and output paths\n");
+        return 2;
+    }
+    inpath = argv[1];
+    outpath = argv[2];
+    infile = fopen(inpath, "rb");
+    if (infile == NULL) {
+        fprintf(stderr, "cannot open '%s' for reading\n", inpath);
+        return 1;
+    }
+    if (fstat(fileno(infile), &st)) {
+        fclose(infile);
+        fprintf(stderr, "cannot fstat '%s'\n", inpath);
+        return 1;
+    }
+    text_size = st.st_size;
+    text = (char *) malloc(text_size + 1);
+    if (text == NULL) {
+        fclose(infile);
+        fprintf(stderr, "could not allocate %ld bytes\n", (long) text_size);
+        return 1;
+    }
+    n = fread(text, 1, text_size, infile);
+    fclose(infile);
+    infile = NULL;
+    if (n < text_size) {
+        fprintf(stderr, "read too short: got %ld instead of %ld bytes\n",
+                (long) n, (long) text_size);
+        return 1;
+    }
+    text[text_size] = '\0';
+
+    Py_NoUserSiteDirectory++;
+    Py_NoSiteFlag++;
+    Py_IgnoreEnvironmentFlag++;
+
+    Py_SetProgramName(L"./_freeze_importlib");
+    /* Don't install importlib, since it could execute outdated bytecode. */
+    _Py_InitializeEx_Private(1, 0);
+
+    code = Py_CompileStringExFlags(text, "<frozen importlib._bootstrap>",
+                                   Py_file_input, NULL, 0);
+    if (code == NULL)
+        goto error;
+    marshalled = PyMarshal_WriteObjectToString(code, Py_MARSHAL_VERSION);
+    Py_DECREF(code);
+    if (marshalled == NULL)
+        goto error;
+
+    assert(PyBytes_CheckExact(marshalled));
+    data = PyBytes_AS_STRING(marshalled);
+    data_size = PyBytes_GET_SIZE(marshalled);
+
+    outfile = fopen(outpath, "wb");
+    if (outfile == NULL) {
+        fprintf(stderr, "cannot open '%s' for writing\n", outpath);
+        return 1;
+    }
+    fprintf(outfile, "%s\n", header);
+    fprintf(outfile, "unsigned char _Py_M__importlib[] = {\n");
+    for (n = 0; n < data_size; n += 16) {
+        size_t i, end = Py_MIN(n + 16, data_size);
+        fprintf(outfile, "    ");
+        for (i = n; i < end; i++) {
+            fprintf(outfile, "%d,", (int) data[i]);
+        }
+        fprintf(outfile, "\n");
+    }
+    fprintf(outfile, "};\n");
+
+    Py_DECREF(marshalled);
+
+    Py_Finalize();
+    if (infile)
+        fclose(infile);
+    if (outfile) {
+        if (ferror(outfile)) {
+            fprintf(stderr, "error when writing to '%s'\n", outpath);
+            fclose(outfile);
+            return 1;
+        }
+        fclose(outfile);
+    }
+    return 0;
+
+error:
+    PyErr_Print();
+    Py_Finalize();
+    if (infile)
+        fclose(infile);
+    if (outfile)
+        fclose(outfile);
+    return 1;
+}

Python/freeze_importlib.py

-#! /usr/bin/env python
-"""Freeze importlib for use as the implementation of import."""
-import marshal
-
-
-header = """/* Auto-generated by Python/freeze_importlib.py */"""
-
-
-def main(input_path, output_path):
-    with open(input_path, 'r', encoding='utf-8') as input_file:
-        source = input_file.read()
-
-    code = compile(source, '<frozen importlib._bootstrap>', 'exec')
-
-    lines = [header]
-    lines.append('unsigned char _Py_M__importlib[] = {')
-    data = marshal.dumps(code)
-    # Code from Tools/freeze/makefreeze.py:writecode()
-    for i in range(0, len(data), 16):
-        line = ['    ']
-        for c in data[i:i+16]:
-            line.append('%d,' % c)
-        lines.append(''.join(line))
-    lines.append('};\n')
-    with open(output_path, 'w', encoding='utf-8') as output_file:
-        output_file.write('\n'.join(lines))
-        # Avoid a compiler warning for lack of EOL
-        output_file.write('\n')
-
-
-if __name__ == '__main__':
-    import sys
-
-    args = sys.argv[1:]
-    if len(args) != 2:
-        print('Need to specify input and output file paths', file=sys.stderr)
-        sys.exit(1)
-
-    main(*args)

Python/pythonrun.c

@@ -242,7 +242,7 @@ import_init(PyInterpreterState *interp, PyObject *sysmod)
 
 
 void
-Py_InitializeEx(int install_sigs)
+_Py_InitializeEx_Private(int install_sigs, int install_importlib)
 {
     PyInterpreterState *interp;
     PyThreadState *tstate;
@@ -363,6 +363,9 @@ Py_InitializeEx(int install_sigs)
     /* Initialize _warnings. */
     _PyWarnings_Init();
 
+    if (!install_importlib)
+        return;
+
     import_init(interp, sysmod);
 
     _PyTime_Init();
@@ -392,6 +395,12 @@ Py_InitializeEx(int install_sigs)
         initsite(); /* Module site */
 }
 
+void
+Py_InitializeEx(int install_sigs)
+{
+    _Py_InitializeEx_Private(install_sigs, 1);
+}
+
 void
 Py_Initialize(void)
 {