bpo-30687: Fixes build scripts to find msbuild.exe and stop relying on…

bpo-30687: Fixes build scripts to find msbuild.exe and stop relying on vcvarsall.bat (#2252) (#2280)

* Fixes build scripts to find msbuild.exe and stop relying on vcvarsall.bat
Also fixes bdist_wininst.vcxproj to use correct version in generated name.

Misc/NEWS

@@ -339,6 +339,9 @@ Tests
 Windows
 -------
 
+- bpo-30687: Locate msbuild.exe on Windows when building rather than
+  vcvarsall.bat
+
 - bpo-30450: The build process on Windows no longer depends on Subversion,
   instead pulling external code from GitHub via a Python script.  If Python 3.6
   is not found on the system (via ``py -3.6``), NuGet is used to download a

PC/bdist_wininst/bdist_wininst.vcxproj

@@ -58,6 +58,10 @@
     <OutDir>$(PySourcePath)lib\distutils\command\</OutDir>
     <LinkIncremental>false</LinkIncremental>
     <TargetName>wininst-$(VisualStudioVersion)</TargetName>
+    <TargetName Condition="$(PlatformToolset) == 'v140'">wininst-14.0</TargetName>
+    <TargetName Condition="$(PlatformToolset) == 'v120'">wininst-12.0</TargetName>
+    <TargetName Condition="$(PlatformToolset) == 'v110'">wininst-11.0</TargetName>
+    <TargetName Condition="$(PlatformToolset) == 'v100'">wininst-10.0</TargetName>
     <TargetName Condition="$(Platform) == 'x64'">$(TargetName)-amd64</TargetName>
     <TargetExt>.exe</TargetExt>
   </PropertyGroup>

PC/bdist_wininst/build.bat

@@ -7,16 +7,13 @@ set PCBUILD=%~dp0..\..\PCBuild\
 
 echo Building Lib\distutils\command\wininst-xx.0.exe
 
-call "%PCBUILD%env.bat" x86
-if errorlevel 1 goto :eof
+call "%PCBUILD%find_msbuild.bat" %MSBUILD%
+if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2)
 
-msbuild "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=Win32
+%MSBUILD% "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=Win32
 if errorlevel 1 goto :eof
 
 
 echo Building Lib\distutils\command\wininst-xx.0-amd64.exe
 
-call "%PCBUILD%env.bat" x86_amd64
-if errorlevel 1 goto :eof
-
-msbuild "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=x64
+%MSBUILD% "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=x64

PCbuild/build.bat

@@ -48,7 +48,6 @@ exit /b 127
 :Run
 setlocal
 set platf=Win32
-set vs_platf=x86
 set conf=Release
 set target=Build
 set dir=%~dp0
@@ -57,10 +56,6 @@ set verbose=/nologo /v:m
 set kill=
 set do_pgo=
 set pgo_job=-m test --pgo
-set on_64_bit=true
-
-rem This may not be 100% accurate, but close enough.
-if "%ProgramFiles(x86)%"=="" (set on_64_bit=false)
 
 :CheckOpts
 if "%~1"=="-h" goto Usage
@@ -90,18 +85,12 @@ if "%IncludeTkinter%"=="" set IncludeTkinter=true
 
 if "%IncludeExternals%"=="true" call "%dir%get_externals.bat"
 
-if "%platf%"=="x64" (
-    if "%on_64_bit%"=="true" (
-        rem This ought to always be correct these days...
-        set vs_platf=amd64
-    ) else (
-        if "%do_pgo%"=="true" (
-            echo.ERROR: Cannot cross-compile with PGO
-            echo.    32bit operating system detected, if this is incorrect,
-            echo.    make sure the ProgramFiles(x86^) environment variable is set
-            exit /b 1
-        )
-        set vs_platf=x86_amd64
+if "%do_pgo%" EQU "true" if "%platf%" EQU "x64" (
+    if "%PROCESSOR_ARCHITEW6432%" NEQ "AMD64" if "%PROCESSOR_ARCHITECTURE%" NEQ "AMD64" (
+        echo.ERROR: Cannot cross-compile with PGO 
+        echo.       32bit operating system detected. Ensure your PROCESSOR_ARCHITECTURE
+        echo.       and PROCESSOR_ARCHITEW6432 environment variables are correct.
+        exit /b 1 
     )
 )
 
@@ -110,7 +99,8 @@ if exist "%GIT%" set GITProperty=/p:GIT="%GIT%"
 if not exist "%GIT%" echo Cannot find Git on PATH & set GITProperty=
 
 rem Setup the environment
-call "%dir%env.bat" %vs_platf% >nul
+call "%dir%find_msbuild.bat" %MSBUILD%
+if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2)
 
 if "%kill%"=="true" call :Kill
 
@@ -129,7 +119,7 @@ if "%do_pgo%"=="true" (
 goto Build
 :Kill
 echo on
-msbuild "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^
+%MSBUILD% "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^
  /p:Configuration=%conf% /p:Platform=%platf%^
  /p:KillPython=true
 
@@ -141,7 +131,7 @@ rem Call on MSBuild to do the work, echo the command.
 rem Passing %1-9 is not the preferred option, but argument parsing in
 rem batch is, shall we say, "lackluster"
 echo on
-msbuild "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^
+%MSBUILD% "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^
  /p:Configuration=%conf% /p:Platform=%platf%^
  /p:IncludeExternals=%IncludeExternals%^
  /p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^
@@ -153,4 +143,4 @@ goto :eof
 
 :Version
 rem Display the current build version information
-msbuild "%dir%python.props" /t:ShowVersionInfo /v:m /nologo %1 %2 %3 %4 %5 %6 %7 %8 %9
+%MSBUILD% "%dir%python.props" /t:ShowVersionInfo /v:m /nologo %1 %2 %3 %4 %5 %6 %7 %8 %9

PCbuild/find_msbuild.bat

+@rem
+@rem Searches for MSBuild.exe. This is the only tool we need to initiate
+@rem a build, so we no longer search for the full VC toolset.
+@rem
+@rem This file is supposed to modify the state of the caller (specifically
+@rem the MSBUILD variable), so we do not use setlocal or echo, and avoid
+@rem changing any other persistent state.
+@rem
+
+@rem No arguments provided means do full search
+@if '%1' EQU '' goto :begin_search
+
+@rem One argument may be the full path. Use a goto so we don't try to
+@rem parse the next if statement - incorrect quoting in the multi-arg
+@rem case can cause us to break immediately.
+@if '%2' EQU '' goto :one_arg
+
+@rem Entire command line may represent the full path if quoting failed.
+@if exist "%*" (set MSBUILD="%*") & (set _Py_MSBuild_Source=environment) & goto :found
+@goto :begin_search
+
+:one_arg
+@if exist "%~1" (set MSBUILD="%~1") & (set _Py_MSBuild_Source=environment) & goto :found
+
+:begin_search
+@set MSBUILD=
+
+@rem If msbuild.exe is on the PATH, assume that the user wants that one.
+@where msbuild > "%TEMP%\msbuild.loc" 2> nul && set /P MSBUILD= < "%TEMP%\msbuild.loc" & del "%TEMP%\msbuild.loc"
+@if exist "%MSBUILD%" set MSBUILD="%MSBUILD%" & (set _Py_MSBuild_Source=PATH) & goto :found
+
+@rem VS 2017 sets exactly one install as the "main" install, so we may find MSBuild in there.
+@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32 >nul 2>nul
+@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32') DO @(
+    @if "%%i"=="15.0" @if exist "%%k\MSBuild\15.0\Bin\msbuild.exe" @(set MSBUILD="%%k\MSBuild\15.0\Bin\msbuild.exe")
+)
+@if exist %MSBUILD% (set _Py_MSBuild_Source=Visual Studio 2017 registry) & goto :found
+
+@rem VS 2015 and earlier register MSBuild separately, so we can find it.
+@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32 >nul 2>nul
+@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32') DO @(
+    @if "%%i"=="MSBuildToolsPath" @if exist "%%k\msbuild.exe" @(set MSBUILD="%%k\msbuild.exe")
+)
+@if exist %MSBUILD% (set _Py_MSBuild_Source=registry) & goto :found
+
+
+@exit /b 1
+
+:found
+@echo Using %MSBUILD% (found in the %_Py_MSBuild_Source%)
+@set _Py_MSBuild_Source=

PCbuild/python.vcxproj

@@ -83,7 +83,7 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-  <Target Name="ValidateUcrtbase" AfterTargets="AfterBuild">
+  <Target Name="ValidateUcrtbase" AfterTargets="AfterBuild" Condition="$(Configuration) != 'PGInstrument'">
     <PropertyGroup>
       <UcrtName>ucrtbase</UcrtName>
       <UcrtName Condition="'$(Configuration)' == 'Debug'">ucrtbased</UcrtName>
@@ -94,6 +94,8 @@ set PYTHONPATH=$(PySourcePath)Lib
   </Target>
   <Target Name="GeneratePythonBat" AfterTargets="AfterBuild">
     <PropertyGroup>
+      <_PGOPath Condition="$(Configuration) == 'PGInstrument' and $(Platform) == 'Win32'">@set PATH=%PATH%%3B$(VCInstallDir)bin</_PGOPath>
+      <_PGOPath Condition="$(Configuration) == 'PGInstrument' and $(Platform) == 'x64'">@set PATH=%PATH%%3B$(VCInstallDir)bin\amd64</_PGOPath>
       <_Content>@rem This script invokes the most recently built Python with all arguments
 @rem passed through to the interpreter.  This file is generated by the
 @rem build process and any changes *will* be thrown away by the next
@@ -103,6 +105,7 @@ set PYTHONPATH=$(PySourcePath)Lib
 @echo Running $(Configuration)^|$(Platform) interpreter...
 @setlocal
 @set PYTHONHOME=$(PySourcePath)
+$(_PGOPath)
 @"$(OutDir)python$(PyDebugExt).exe" %*
 </_Content>
       <_ExistingContent Condition="Exists('$(PySourcePath)python.bat')">$([System.IO.File]::ReadAllText('$(PySourcePath)python.bat'))</_ExistingContent>

Tools/msi/build.bat

@@ -22,8 +22,8 @@ if "%~1" EQU "-r" (set REBUILD=-r) && shift && goto CheckOpts
 if not defined BUILDX86 if not defined BUILDX64 (set BUILDX86=1) && (set BUILDX64=1)
 
 call "%D%get_externals.bat"
-
-call "%PCBUILD%env.bat" x86
+call "%PCBUILD%find_msbuild.bat" %MSBUILD%
+if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2)
 
 if defined BUILDX86 (
     call "%PCBUILD%build.bat" -d -e %REBUILD% %BUILDTEST%
@@ -44,7 +44,7 @@ if defined BUILDDOC (
 )
 
 rem Build the launcher MSI separately
-msbuild "%D%launcher\launcher.wixproj" /p:Platform=x86
+%MSBUILD% "%D%launcher\launcher.wixproj" /p:Platform=x86
 
 set BUILD_CMD="%D%bundle\snapshot.wixproj"
 if defined BUILDTEST (
@@ -58,11 +58,11 @@ if defined REBUILD (
 )
 
 if defined BUILDX86 (
-    msbuild %BUILD_CMD%
+    %MSBUILD% %BUILD_CMD%
     if errorlevel 1 goto :eof
 )
 if defined BUILDX64 (
-    msbuild /p:Platform=x64 %BUILD_CMD%
+    %MSBUILD% /p:Platform=x64 %BUILD_CMD%
     if errorlevel 1 goto :eof
 )
 

Tools/msi/buildrelease.bat

@@ -69,6 +69,8 @@ if not exist "%GIT%" where git > "%TEMP%\git.loc" 2> nul && set /P GIT= < "%TEMP
 if not exist "%GIT%" echo Cannot find Git on PATH && exit /B 1
 
 call "%D%get_externals.bat"
+call "%PCBUILD%find_msbuild.bat" %MSBUILD%
+if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2)
 
 :builddoc
 if "%SKIPBUILD%" EQU "1" goto skipdoc
@@ -165,28 +167,27 @@ if not "%SKIPBUILD%" EQU "1" (
     @echo off
 )
 
-call "%PCBUILD%env.bat"
 if "%OUTDIR_PLAT%" EQU "win32" (
-    msbuild "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI%
+    %MSBUILD% "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI%
     if errorlevel 1 exit /B
 ) else if not exist "%PCBUILD%win32\en-us\launcher.msi" (
-    msbuild "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI%
+    %MSBUILD% "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI%
     if errorlevel 1 exit /B
 )
 
 set BUILDOPTS=/p:Platform=%1 /p:BuildForRelease=true /p:DownloadUrl=%DOWNLOAD_URL% /p:DownloadUrlBase=%DOWNLOAD_URL_BASE% /p:ReleaseUri=%RELEASE_URI%
-msbuild "%D%bundle\releaselocal.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=true
+%MSBUILD% "%D%bundle\releaselocal.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=true
 if errorlevel 1 exit /B
-msbuild "%D%bundle\releaseweb.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=false
+%MSBUILD% "%D%bundle\releaseweb.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=false
 if errorlevel 1 exit /B
 
 if defined BUILDZIP (
-    msbuild "%D%make_zip.proj" /t:Build %BUILDOPTS% %CERTOPTS% /p:OutputPath="%BUILD%en-us"
+    %MSBUILD% "%D%make_zip.proj" /t:Build %BUILDOPTS% %CERTOPTS% /p:OutputPath="%BUILD%en-us"
     if errorlevel 1 exit /B
 )
 
 if defined BUILDNUGET (
-    msbuild "%D%..\nuget\make_pkg.proj" /t:Build /p:Configuration=Release /p:Platform=%1 /p:OutputPath="%BUILD%en-us"
+    %MSBUILD% "%D%..\nuget\make_pkg.proj" /t:Build /p:Configuration=Release /p:Platform=%1 /p:OutputPath="%BUILD%en-us"
     if errorlevel 1 exit /B
 )
 

Tools/msi/uploadrelease.bat

@@ -49,16 +49,17 @@ if defined NOGPG (
     echo Found gpg2.exe at %GPG%
 )
 
-call "%PCBUILD%env.bat" > nul 2> nul
+call "%PCBUILD%find_msbuild.bat" %MSBUILD%
+if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2)
 pushd "%D%"
-msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 %PURGE_OPTION%
-msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false %PURGE_OPTION%
+%MSBUILD% /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 %PURGE_OPTION%
+%MSBUILD% /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false %PURGE_OPTION%
 if not defined NOTEST (
-    msbuild /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x86
-    msbuild /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x64
+    %MSBUILD% /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x86
+    %MSBUILD% /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x64
 )
-msbuild /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x86
-msbuild /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x64 /p:IncludeDoc=false
+%MSBUILD% /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x86
+%MSBUILD% /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x64 /p:IncludeDoc=false
 popd
 exit /B 0
 

Tools/nuget/build.bat

@@ -21,7 +21,8 @@ if "%~1" EQU "-p" (set PACKAGES=%PACKAGES% %~2) && shift && shift && goto CheckO
 if not defined BUILDX86 if not defined BUILDX64 (set BUILDX86=1) && (set BUILDX64=1)
 
 call "%D%..\msi\get_externals.bat"
-call "%PCBUILD%env.bat" x86
+call "%PCBUILD%find_msbuild.bat" %MSBUILD%
+if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2)
 
 if defined PACKAGES set PACKAGES="/p:Packages=%PACKAGES%"
 
@@ -30,7 +31,7 @@ if defined BUILDX86 (
     ) else if not exist "%PCBUILD%win32\python.exe" call "%PCBUILD%build.bat" -e
     if errorlevel 1 goto :eof
 
-    msbuild "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x86 %OUTPUT% %PACKAGES%
+    %MSBUILD% "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x86 %OUTPUT% %PACKAGES%
     if errorlevel 1 goto :eof
 )
 
@@ -39,7 +40,7 @@ if defined BUILDX64 (
     ) else if not exist "%PCBUILD%amd64\python.exe" call "%PCBUILD%build.bat" -p x64 -e
     if errorlevel 1 goto :eof
 
-    msbuild "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x64 %OUTPUT% %PACKAGES%
+    %MSBUILD% "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x64 %OUTPUT% %PACKAGES%
     if errorlevel 1 goto :eof
 )