[3.6] bpo-30389 Adds detection of VS 2017 to distutils._msvccompiler GH-1632 (#3425)

Lib/distutils/_msvccompiler.py

@@ -17,6 +17,7 @@ import os
 import shutil
 import stat
 import subprocess
+import winreg
 
 from distutils.errors import DistutilsExecError, DistutilsPlatformError, \
                              CompileError, LibError, LinkError
@@ -24,10 +25,9 @@ from distutils.ccompiler import CCompiler, gen_lib_options
 from distutils import log
 from distutils.util import get_platform
 
-import winreg
 from itertools import count
 
-def _find_vcvarsall(plat_spec):
+def _find_vc2015():
     try:
         key = winreg.OpenKeyEx(
             winreg.HKEY_LOCAL_MACHINE,
@@ -38,9 +38,9 @@ def _find_vcvarsall(plat_spec):
         log.debug("Visual C++ is not registered")
         return None, None
 
+    best_version = 0
+    best_dir = None
     with key:
-        best_version = 0
-        best_dir = None
         for i in count():
             try:
                 v, vc_dir, vt = winreg.EnumValue(key, i)
@@ -53,25 +53,74 @@ def _find_vcvarsall(plat_spec):
                     continue
                 if version >= 14 and version > best_version:
                     best_version, best_dir = version, vc_dir
-        if not best_version:
-            log.debug("No suitable Visual C++ version found")
-            return None, None
+    return best_version, best_dir
+
+def _find_vc2017():
+    import _findvs
+    import threading
+
+    best_version = 0,   # tuple for full version comparisons
+    best_dir = None
+
+    # We need to call findall() on its own thread because it will
+    # initialize COM.
+    all_packages = []
+    def _getall():
+        all_packages.extend(_findvs.findall())
+    t = threading.Thread(target=_getall)
+    t.start()
+    t.join()
+
+    for name, version_str, path, packages in all_packages:
+        if 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64' in packages:
+            vc_dir = os.path.join(path, 'VC', 'Auxiliary', 'Build')
+            if not os.path.isdir(vc_dir):
+                continue
+            try:
+                version = tuple(int(i) for i in version_str.split('.'))
+            except (ValueError, TypeError):
+                continue
+            if version > best_version:
+                best_version, best_dir = version, vc_dir
+    try:
+        best_version = best_version[0]
+    except IndexError:
+        best_version = None
+    return best_version, best_dir
 
-        vcvarsall = os.path.join(best_dir, "vcvarsall.bat")
-        if not os.path.isfile(vcvarsall):
-            log.debug("%s cannot be found", vcvarsall)
-            return None, None
+def _find_vcvarsall(plat_spec):
+    best_version, best_dir = _find_vc2017()
+    vcruntime = None
+    vcruntime_plat = 'x64' if 'amd64' in plat_spec else 'x86'
+    if best_version:
+        vcredist = os.path.join(best_dir, "..", "..", "redist", "MSVC", "**",
+            "Microsoft.VC141.CRT", "vcruntime140.dll")
+        try:
+            import glob
+            vcruntime = glob.glob(vcredist, recursive=True)[-1]
+        except (ImportError, OSError, LookupError):
+            vcruntime = None
+
+    if not best_version:
+        best_version, best_dir = _find_vc2015()
+        if best_version:
+            vcruntime = os.path.join(best_dir, 'redist', vcruntime_plat,
+                "Microsoft.VC140.CRT", "vcruntime140.dll")
+
+    if not best_version:
+        log.debug("No suitable Visual C++ version found")
+        return None, None
 
+    vcvarsall = os.path.join(best_dir, "vcvarsall.bat")
+    if not os.path.isfile(vcvarsall):
+        log.debug("%s cannot be found", vcvarsall)
+        return None, None
+
+    if not vcruntime or not os.path.isfile(vcruntime):
+        log.debug("%s cannot be found", vcruntime)
         vcruntime = None
-        vcruntime_spec = _VCVARS_PLAT_TO_VCRUNTIME_REDIST.get(plat_spec)
-        if vcruntime_spec:
-            vcruntime = os.path.join(best_dir,
-                vcruntime_spec.format(best_version))
-            if not os.path.isfile(vcruntime):
-                log.debug("%s cannot be found", vcruntime)
-                vcruntime = None
 
-        return vcvarsall, vcruntime
+    return vcvarsall, vcruntime
 
 def _get_vc_env(plat_spec):
     if os.getenv("DISTUTILS_USE_SDK"):
@@ -130,14 +179,6 @@ PLAT_TO_VCVARS = {
     'win-amd64' : 'x86_amd64',
 }
 
-# A map keyed by get_platform() return values to the file under
-# the VC install directory containing the vcruntime redistributable.
-_VCVARS_PLAT_TO_VCRUNTIME_REDIST = {
-    'x86' : 'redist\\x86\\Microsoft.VC{0}0.CRT\\vcruntime{0}0.dll',
-    'amd64' : 'redist\\x64\\Microsoft.VC{0}0.CRT\\vcruntime{0}0.dll',
-    'x86_amd64' : 'redist\\x64\\Microsoft.VC{0}0.CRT\\vcruntime{0}0.dll',
-}
-
 # A set containing the DLLs that are guaranteed to be available for
 # all micro versions of this Python version. Known extension
 # dependencies that are not in this set will be copied to the output

Lib/distutils/tests/test_msvccompiler.py

@@ -76,12 +76,12 @@ class msvccompilerTestCase(support.TempdirManager,
         compiler = _msvccompiler.MSVCCompiler()
         compiler.initialize()
         dll = compiler._vcruntime_redist
-        self.assertTrue(os.path.isfile(dll))
+        self.assertTrue(os.path.isfile(dll), dll or "<None>")
 
         compiler._copy_vcruntime(tempdir)
 
         self.assertFalse(os.path.isfile(os.path.join(
-            tempdir, os.path.basename(dll))))
+            tempdir, os.path.basename(dll))), dll or "<None>")
 
     def test_get_vc_env_unicode(self):
         import distutils._msvccompiler as _msvccompiler
@@ -101,6 +101,30 @@ class msvccompilerTestCase(support.TempdirManager,
             if old_distutils_use_sdk:
                 os.environ['DISTUTILS_USE_SDK'] = old_distutils_use_sdk
 
+    def test_get_vc2017(self):
+        import distutils._msvccompiler as _msvccompiler
+
+        # This function cannot be mocked, so pass it if we find VS 2017
+        # and mark it skipped if we do not.
+        version, path = _msvccompiler._find_vc2017()
+        if version:
+            self.assertGreaterEqual(version, 15)
+            self.assertTrue(os.path.isdir(path))
+        else:
+            raise unittest.SkipTest("VS 2017 is not installed")
+
+    def test_get_vc2015(self):
+        import distutils._msvccompiler as _msvccompiler
+
+        # This function cannot be mocked, so pass it if we find VS 2015
+        # and mark it skipped if we do not.
+        version, path = _msvccompiler._find_vc2015()
+        if version:
+            self.assertGreaterEqual(version, 14)
+            self.assertTrue(os.path.isdir(path))
+        else:
+            raise unittest.SkipTest("VS 2015 is not installed")
+
 def test_suite():
     return unittest.makeSuite(msvccompilerTestCase)
 

Misc/NEWS.d/next/Windows/2017-09-06-17-14-54.bpo-30389.9Dizrx.rst

+Adds detection of Visual Studio 2017 to distutils on Windows.

PC/_findvs.cpp

+//
+// Helper library for location Visual Studio installations
+// using the COM-based query API.
+//
+// Copyright (c) Microsoft Corporation
+// Licensed to PSF under a contributor agreement
+//
+
+// Version history
+//  2017-05: Initial contribution (Steve Dower)
+
+#include <Windows.h>
+#include <Strsafe.h>
+#include "external\include\Setup.Configuration.h"
+#pragma comment(lib, "ole32.lib")
+#pragma comment(lib, "oleaut32.lib")
+#pragma comment(lib, "version.lib")
+#pragma comment(lib, "Microsoft.VisualStudio.Setup.Configuration.Native.lib")
+
+#include <Python.h>
+
+static PyObject *error_from_hr(HRESULT hr)
+{
+    if (FAILED(hr))
+        PyErr_Format(PyExc_OSError, "Error %08x", hr);
+    assert(PyErr_Occurred());
+    return nullptr;
+}
+
+static PyObject *get_install_name(ISetupInstance2 *inst)
+{
+    HRESULT hr;
+    BSTR name;
+    PyObject *str = nullptr;
+    if (FAILED(hr = inst->GetDisplayName(LOCALE_USER_DEFAULT, &name)))
+        goto error;
+    str = PyUnicode_FromWideChar(name, SysStringLen(name));
+    SysFreeString(name);
+    return str;
+error:
+
+    return error_from_hr(hr);
+}
+
+static PyObject *get_install_version(ISetupInstance *inst)
+{
+    HRESULT hr;
+    BSTR ver;
+    PyObject *str = nullptr;
+    if (FAILED(hr = inst->GetInstallationVersion(&ver)))
+        goto error;
+    str = PyUnicode_FromWideChar(ver, SysStringLen(ver));
+    SysFreeString(ver);
+    return str;
+error:
+
+    return error_from_hr(hr);
+}
+
+static PyObject *get_install_path(ISetupInstance *inst)
+{
+    HRESULT hr;
+    BSTR path;
+    PyObject *str = nullptr;
+    if (FAILED(hr = inst->GetInstallationPath(&path)))
+        goto error;
+    str = PyUnicode_FromWideChar(path, SysStringLen(path));
+    SysFreeString(path);
+    return str;
+error:
+
+    return error_from_hr(hr);
+}
+
+static PyObject *get_installed_packages(ISetupInstance2 *inst)
+{
+    HRESULT hr;
+    PyObject *res = nullptr;
+    LPSAFEARRAY sa_packages = nullptr;
+    LONG ub = 0;
+    IUnknown **packages = nullptr;
+    PyObject *str = nullptr;
+
+    if (FAILED(hr = inst->GetPackages(&sa_packages)) ||
+        FAILED(hr = SafeArrayAccessData(sa_packages, (void**)&packages)) ||
+        FAILED(SafeArrayGetUBound(sa_packages, 1, &ub)) ||
+        !(res = PyList_New(0)))
+        goto error;
+
+    for (LONG i = 0; i < ub; ++i) {
+        ISetupPackageReference *package = nullptr;
+        BSTR id = nullptr;
+        PyObject *str = nullptr;
+
+        if (FAILED(hr = packages[i]->QueryInterface(&package)) ||
+            FAILED(hr = package->GetId(&id)))
+            goto iter_error;
+
+        str = PyUnicode_FromWideChar(id, SysStringLen(id));
+        SysFreeString(id);
+
+        if (!str || PyList_Append(res, str) < 0)
+            goto iter_error;
+
+        Py_CLEAR(str);
+        package->Release();
+        continue;
+
+    iter_error:
+        if (package) package->Release();
+        Py_XDECREF(str);
+
+        goto error;
+    }
+
+    SafeArrayUnaccessData(sa_packages);
+    SafeArrayDestroy(sa_packages);
+
+    return res;
+error:
+    if (sa_packages && packages) SafeArrayUnaccessData(sa_packages);
+    if (sa_packages) SafeArrayDestroy(sa_packages);
+    Py_XDECREF(res);
+
+    return error_from_hr(hr);
+}
+
+static PyObject *find_all_instances()
+{
+    ISetupConfiguration *sc = nullptr;
+    ISetupConfiguration2 *sc2 = nullptr;
+    IEnumSetupInstances *enm = nullptr;
+    ISetupInstance *inst = nullptr;
+    ISetupInstance2 *inst2 = nullptr;
+    PyObject *res = nullptr;
+    ULONG fetched;
+    HRESULT hr;
+
+    if (!(res = PyList_New(0)))
+        goto error;
+
+    if (FAILED(hr = CoCreateInstance(
+        __uuidof(SetupConfiguration),
+        NULL,
+        CLSCTX_INPROC_SERVER,
+        __uuidof(ISetupConfiguration),
+        (LPVOID*)&sc
+    )) && hr != REGDB_E_CLASSNOTREG)
+        goto error;
+
+    // If the class is not registered, there are no VS instances installed
+    if (hr == REGDB_E_CLASSNOTREG)
+        return res;
+
+    if (FAILED(hr = sc->QueryInterface(&sc2)) ||
+        FAILED(hr = sc2->EnumAllInstances(&enm)))
+        goto error;
+
+    while (SUCCEEDED(enm->Next(1, &inst, &fetched)) && fetched) {
+        PyObject *name = nullptr;
+        PyObject *version = nullptr;
+        PyObject *path = nullptr;
+        PyObject *packages = nullptr;
+
+        if (FAILED(hr = inst->QueryInterface(&inst2)) ||
+            !(name = get_install_name(inst2)) ||
+            !(version = get_install_version(inst)) ||
+            !(path = get_install_path(inst)) ||
+            !(packages = get_installed_packages(inst2)) ||
+            PyList_Append(res, PyTuple_Pack(4, name, version, path, packages)) < 0)
+            goto iter_error;
+
+        continue;
+    iter_error:
+        if (inst2) inst2->Release();
+        Py_XDECREF(packages);
+        Py_XDECREF(path);
+        Py_XDECREF(version);
+        Py_XDECREF(name);
+        goto error;
+    }
+
+    enm->Release();
+    sc2->Release();
+    sc->Release();
+    return res;
+
+error:
+    if (enm) enm->Release();
+    if (sc2) sc2->Release();
+    if (sc) sc->Release();
+    Py_XDECREF(res);
+
+    return error_from_hr(hr);
+}
+
+PyDoc_STRVAR(findvs_findall_doc, "findall()\
+\
+Finds all installed versions of Visual Studio.\
+\
+This function will initialize COM temporarily. To avoid impact on other parts\
+of your application, use a new thread to make this call.");
+
+static PyObject *findvs_findall(PyObject *self, PyObject *args, PyObject *kwargs)
+{
+    HRESULT hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED);
+    if (hr == RPC_E_CHANGED_MODE)
+        hr = CoInitializeEx(nullptr, COINIT_APARTMENTTHREADED);
+    if (FAILED(hr))
+        return error_from_hr(hr);
+    PyObject *res = find_all_instances();
+    CoUninitialize();
+    return res;
+}
+
+// List of functions to add to findvs in exec_findvs().
+static PyMethodDef findvs_functions[] = {
+    { "findall", (PyCFunction)findvs_findall, METH_VARARGS | METH_KEYWORDS, findvs_findall_doc },
+    { NULL, NULL, 0, NULL }
+};
+
+// Initialize findvs. May be called multiple times, so avoid
+// using static state.
+static int exec_findvs(PyObject *module)
+{
+    PyModule_AddFunctions(module, findvs_functions);
+
+    return 0; // success
+}
+
+PyDoc_STRVAR(findvs_doc, "The _findvs helper module");
+
+static PyModuleDef_Slot findvs_slots[] = {
+    { Py_mod_exec, exec_findvs },
+    { 0, NULL }
+};
+
+static PyModuleDef findvs_def = {
+    PyModuleDef_HEAD_INIT,
+    "_findvs",
+    findvs_doc,
+    0,              // m_size
+    NULL,           // m_methods
+    findvs_slots,
+    NULL,           // m_traverse
+    NULL,           // m_clear
+    NULL,           // m_free
+};
+
+extern "C" {
+    PyMODINIT_FUNC PyInit__findvs(void)
+    {
+        return PyModuleDef_Init(&findvs_def);
+    }
+}
\ No newline at end of file

PC/config.c

 /* Module configuration */
 
 /* This file contains the table of built-in modules.
-   See create_builtin() in import.c. */
+    See create_builtin() in import.c. */
 
 #include "Python.h"
 
@@ -69,6 +69,7 @@ extern PyObject* _PyWarnings_Init(void);
 extern PyObject* PyInit__string(void);
 extern PyObject* PyInit__stat(void);
 extern PyObject* PyInit__opcode(void);
+extern PyObject* PyInit__findvs(void);
 
 /* tools/freeze/makeconfig.py marker for additional "extern" */
 /* -- ADDMODULE MARKER 1 -- */
@@ -165,6 +166,8 @@ struct _inittab _PyImport_Inittab[] = {
     {"_stat", PyInit__stat},
     {"_opcode", PyInit__opcode},
 
+    {"_findvs", PyInit__findvs},
+
     /* Sentinel */
     {0, 0}
 };

PC/external/Externals.txt

+The files in this folder are from the Microsoft.VisualStudio.Setup.Configuration.Native package on Nuget.
+
+They are licensed under the MIT license.

PCbuild/_lzma.vcxproj

@@ -65,7 +65,7 @@
       <PreprocessorDefinitions>WIN32;_FILE_OFFSET_BITS=64;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_DEPRECATE;LZMA_API_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>
     </ClCompile>
     <Link>
-      <AdditionalDependencies>$(OutDir)/liblzma$(PyDebugExt).lib</AdditionalDependencies>
+      <AdditionalDependencies>$(OutDir)liblzma$(PyDebugExt).lib;%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>

PCbuild/build.bat

@@ -77,7 +77,7 @@ if "%~1"=="-e" (set IncludeExternals=true) & shift & goto CheckOpts
 if "%~1"=="--no-ssl" (set IncludeSSL=false) & shift & goto CheckOpts
 if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts
 
-if "%IncludeExternals%"=="" set IncludeExternals=false
+if "%IncludeExternals%"=="" set IncludeExternals=true
 if "%IncludeSSL%"=="" set IncludeSSL=true
 if "%IncludeTkinter%"=="" set IncludeTkinter=true
 

PCbuild/pythoncore.vcxproj

@@ -71,6 +71,7 @@
     </ClCompile>
     <Link>
       <AdditionalDependencies>version.lib;shlwapi.lib;ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>%(AdditionalLibraryDirectories);$(PySourcePath)PC\external\$(PlatformToolset)\$(ArchName)</AdditionalLibraryDirectories>
       <BaseAddress>0x1e000000</BaseAddress>
     </Link>
   </ItemDefinitionGroup>
@@ -214,6 +215,7 @@
     <ClInclude Include="..\Python\wordcode_helpers.h" />
   </ItemGroup>
   <ItemGroup>
+    <ClCompile Include="..\Modules\_asynciomodule.c" />
     <ClCompile Include="..\Modules\_bisectmodule.c" />
     <ClCompile Include="..\Modules\_blake2\blake2module.c" />
     <ClCompile Include="..\Modules\_blake2\blake2b_impl.c" />
@@ -348,6 +350,7 @@
     <ClCompile Include="..\PC\config.c" />
     <ClCompile Include="..\PC\getpathp.c" />
     <ClCompile Include="..\PC\msvcrtmodule.c" />
+    <ClCompile Include="..\PC\_findvs.cpp" />
     <ClCompile Include="..\Python\pyhash.c" />
     <ClCompile Include="..\Python\random.c" />
     <ClCompile Include="..\Python\_warnings.c" />