Reimplement addbuilddir() in C inside getpath.c, so as to execute it

at interpreter startup before importing any non-builtin modules.
Should fix #9589.

.bzrignore

@@ -11,6 +11,7 @@ python
 build
 Makefile.pre
 platform
+pybuilddir.txt
 pyconfig.h
 libpython*.a
 python.exe

.hgignore

@@ -32,6 +32,7 @@ Modules/config.c
 Parser/pgen$
 ^core
 ^python-gdb.py
+^pybuilddir.txt
 
 syntax: glob
 libpython*.a

Include/Python.h

@@ -129,8 +129,9 @@ extern "C" {
 /* _Py_Mangle is defined in compile.c */
 PyAPI_FUNC(PyObject*) _Py_Mangle(PyObject *p, PyObject *name);
 
-/* _Py_char2wchar lives in main.c */
+/* These functions live in main.c */
 PyAPI_FUNC(wchar_t *) _Py_char2wchar(char *);
+PyAPI_FUNC(FILE *) _Py_wfopen(const wchar_t *path, const wchar_t *mode);
 #ifdef __cplusplus
 }
 #endif

Lib/site.py

@@ -107,18 +107,6 @@ def removeduppaths():
     sys.path[:] = L
     return known_paths
 
-# XXX This should not be part of site.py, since it is needed even when
-# using the -S option for Python.  See http://www.python.org/sf/586680
-def addbuilddir():
-    """Append ./build/lib.<platform> in case we're running in the build dir
-    (especially for Guido :-)"""
-    from sysconfig import get_platform
-    s = "build/lib.%s-%.3s" % (get_platform(), sys.version)
-    if hasattr(sys, 'gettotalrefcount'):
-        s += '-pydebug'
-    s = os.path.join(os.path.dirname(sys.path.pop()), s)
-    sys.path.append(s)
-
 
 def _init_pathinfo():
     """Return a set containing all existing directory entries from sys.path"""
@@ -529,9 +517,6 @@ def main():
 
     abs_paths()
     known_paths = removeduppaths()
-    if (os.name == "posix" and sys.path and
-        os.path.basename(sys.path[-1]) == "Modules"):
-        addbuilddir()
     if ENABLE_USER_SITE is None:
         ENABLE_USER_SITE = check_enableusersite()
     known_paths = addusersitepackages(known_paths)

Makefile.pre.in

@@ -1213,6 +1213,7 @@ distclean: clobber
 		Modules/Setup Modules/Setup.local Modules/Setup.config \
 		Misc/python.pc
 	-rm -f python*-gdb.py
+	-rm -f pybuilddir.txt
 	find $(srcdir) '(' -name '*.fdc' -o -name '*~' \
 			   -o -name '[@,#]*' -o -name '*.old' \
 			   -o -name '*.orig' -o -name '*.rej' \

Modules/getpath.c

@@ -394,12 +394,35 @@ search_for_exec_prefix(wchar_t *argv0_path, wchar_t *home)
         return 1;
     }
 
-    /* Check to see if argv[0] is in the build directory */
+    /* Check to see if argv[0] is in the build directory. "pybuilddir.txt"
+       is written by setup.py and contains the relative path to the location
+       of shared library modules. */
     wcscpy(exec_prefix, argv0_path);
-    joinpath(exec_prefix, L"Modules/Setup");
+    joinpath(exec_prefix, L"pybuilddir.txt");
     if (isfile(exec_prefix)) {
-        reduce(exec_prefix);
-        return -1;
+        FILE *f = _Py_wfopen(exec_prefix, L"rb");
+        if (f == NULL)
+            errno = 0;
+        else {
+            char buf[MAXPATHLEN+1];
+            PyObject *decoded;
+            wchar_t rel_builddir_path[MAXPATHLEN+1];
+            size_t n;
+            n = fread(buf, 1, MAXPATHLEN, f);
+            buf[n] = '\0';
+            fclose(f);
+            decoded = PyUnicode_DecodeUTF8(buf, n, "surrogateescape");
+            if (decoded != NULL) {
+                n = PyUnicode_AsWideChar(decoded, rel_builddir_path, MAXPATHLEN);
+                Py_DECREF(decoded);
+                if (n >= 0) {
+                    rel_builddir_path[n] = L'\0';
+                    wcscpy(exec_prefix, argv0_path);
+                    joinpath(exec_prefix, rel_builddir_path);
+                    return -1;
+                }
+            }
+        }
     }
 
     /* Search from argv0_path, until root is found */

Modules/main.c

@@ -101,10 +101,10 @@ PYTHONCASEOK : ignore case in 'import' statements (Windows).\n\
 PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.\n\
 ";
 
-#ifndef MS_WINDOWS
-static FILE*
-_wfopen(const wchar_t *path, const wchar_t *mode)
+FILE *
+_Py_wfopen(const wchar_t *path, const wchar_t *mode)
 {
+#ifndef MS_WINDOWS
     char cpath[PATH_MAX];
     char cmode[10];
     size_t r;
@@ -119,8 +119,10 @@ _wfopen(const wchar_t *path, const wchar_t *mode)
         return NULL;
     }
     return fopen(cpath, cmode);
-}
+#else
+    return _wfopen(path, mode);
 #endif
+}
 
 
 static int
@@ -640,7 +642,7 @@ Py_Main(int argc, wchar_t **argv)
         }
 
         if (sts==-1 && filename!=NULL) {
-            if ((fp = _wfopen(filename, L"r")) == NULL) {
+            if ((fp = _Py_wfopen(filename, L"r")) == NULL) {
                 char cfilename[PATH_MAX];
                 size_t r = wcstombs(cfilename, filename, PATH_MAX);
                 if (r == PATH_MAX)

setup.py

@@ -22,6 +22,10 @@ COMPILED_WITH_PYDEBUG = hasattr(sys, 'gettotalrefcount')
 # This global variable is used to hold the list of modules to be disabled.
 disabled_module_list = []
 
+# File which contains the directory for shared mods (for sys.path fixup
+# when running from the build dir, see Modules/getpath.c)
+_BUILDDIR_COOKIE = "pybuilddir.txt"
+
 def add_dir_to_list(dirlist, dir):
     """Add the directory 'dir' to the list 'dirlist' (at the front) if
     1) 'dir' is not already in 'dirlist'
@@ -224,6 +228,16 @@ class PyBuildExt(build_ext):
             args['compiler_so'] = compiler + ' ' + ccshared + ' ' + cflags
         self.compiler.set_executables(**args)
 
+        # Not only do we write the builddir cookie, but we manually install
+        # the shared modules directory if it isn't already in sys.path.
+        # Otherwise trying to import the extensions after building them
+        # will fail.
+        with open(_BUILDDIR_COOKIE, "wb") as f:
+            f.write(self.build_lib.encode('utf-8', 'surrogateescape'))
+        abs_build_lib = os.path.join(os.path.dirname(__file__), self.build_lib)
+        if abs_build_lib not in sys.path:
+            sys.path.append(abs_build_lib)
+
         build_ext.build_extensions(self)
 
         longest = max([len(e.name) for e in self.extensions])