[3.6] bpo-30450: Improved logic for obtaining dependencies (#2751)

Adds alternate download approach for nuget.exe
Fall back to git.exe if no Python is found. (#2739)
Also check whether git.exe is on PATH if it will be used.
Add support for HOST_PYTHON variable.
Clear internal environment variables used in find_python.bat
Use HOST_PYTHON as the actual Python if it is recent enough.
Adds HOST_PYTHON variable to AppVeyor configuration

.github/appveyor.yml

@@ -9,6 +9,8 @@ build_script:
 - cmd: PCbuild\build.bat -e
 test_script:
 - cmd: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
+environment:
+  HOST_PYTHON: C:\Python36\python.exe
 
 # Only trigger AppVeyor if actual code or its configuration changes
 only_commits:

Doc/make.bat

@@ -5,8 +5,19 @@ pushd %~dp0
 
 set this=%~n0
 
-if "%SPHINXBUILD%" EQU "" set SPHINXBUILD=sphinx-build
+call ..\PCBuild\find_python.bat %PYTHON%
+if "%SPHINXBUILD%" EQU "" if "%PYTHON%" NEQ "" (
+    set SPHINXBUILD=%PYTHON%\..\Scripts\sphinx-build.exe
+    rem Cannot use %SPHINXBUILD% in the same block where we set it
+    if not exist "%PYTHON%\..\Scripts\sphinx-build.exe" (
+        echo Installing sphinx with %PYTHON%
+        "%PYTHON%" -m pip install sphinx
+        if errorlevel 1 exit /B
+    )
+)
+
 if "%PYTHON%" EQU "" set PYTHON=py
+if "%SPHINXBUILD%" EQU "" set SPHINXBUILD=sphinx-build
 
 if "%1" NEQ "htmlhelp" goto :skiphhcsearch
 if exist "%HTMLHELP%" goto :skiphhcsearch

PCbuild/build.bat

@@ -5,8 +5,6 @@ echo.%~nx0 [flags and arguments] [quoted MSBuild options]
 echo.
 echo.Build CPython from the command line.  Requires the appropriate
 echo.version(s) of Microsoft Visual Studio to be installed (see readme.txt).
-echo.Also requires Subversion (svn.exe) to be on PATH if the '-e' flag is
-echo.given.
 echo.
 echo.After the flags recognized by this script, up to 9 arguments to be passed
 echo.directly to MSBuild may be passed.  If the argument contains an '=', the

PCbuild/find_python.bat

+@rem
+@rem Searches for python.exe and may download a private copy from nuget.
+@rem
+@rem This file is supposed to modify the state of the caller (specifically
+@rem the MSBUILD variable), so we do not use setlocal or echo, and avoid
+@rem changing any other persistent state.
+@rem
+
+@rem No arguments provided means do full search
+@if '%1' EQU '' goto :begin_search
+
+@rem One argument may be the full path. Use a goto so we don't try to
+@rem parse the next if statement - incorrect quoting in the multi-arg
+@rem case can cause us to break immediately.
+@if '%2' EQU '' goto :one_arg
+
+@rem Entire command line may represent the full path if quoting failed.
+@if exist "%*" (set PYTHON="%*") & (set _Py_Python_Source=from environment) & goto :found
+@goto :begin_search
+
+:one_arg
+@if exist "%~1" (set PYTHON="%~1") & (set _Py_Python_Source=from environment) & goto :found
+
+:begin_search
+@set PYTHON=
+
+@set _Py_EXTERNALS_DIR=%EXTERNAL_DIR%
+@if "%_Py_EXTERNALS_DIR%"=="" (set _Py_EXTERNALS_DIR=%~dp0\..\externals)
+
+@rem If we have Python in externals, use that one
+@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found in externals directory) & goto :found
+
+@rem If HOST_PYTHON is recent enough, use that
+@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -c "import sys; assert sys.version_info[:2] >= (3, 6)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found
+
+@rem If py.exe finds a recent enough version, use that one
+@py -3.6 -V >nul 2>&1 && (set PYTHON=py -3.6) && (set _Py_Python_Source=found with py.exe) && goto :found
+
+@if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%"
+@set _Py_NUGET=%NUGET%
+@set _Py_NUGET_URL=%NUGET_URL%
+@set _Py_HOST_PYTHON=%HOST_PYTHON%
+@if "%_Py_HOST_PYTHON%"=="" set _Py_HOST_PYTHON=py
+@if "%_Py_NUGET%"=="" (set _Py_NUGET=%_Py_EXTERNALS_DIR%\nuget.exe)
+@if "%_Py_NUGET_URL%"=="" (set _Py_NUGET_URL=https://aka.ms/nugetclidl)
+@if NOT exist "%_Py_NUGET%" (
+    @echo Downloading nuget...
+    @rem NB: Must use single quotes around NUGET here, NOT double!
+    @rem Otherwise, a space in the path would break things
+    @rem If it fails, retry with any available copy of Python
+    @powershell.exe -Command Invoke-WebRequest %_Py_NUGET_URL% -OutFile '%_Py_NUGET%'
+    @if errorlevel 1 (
+        @%_Py_HOST_PYTHON% "%~dp0\urlretrieve.py" "%_Py_NUGET_URL%" "%_Py_NUGET%"
+    )
+)
+@echo Installing Python via nuget...
+@"%_Py_NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%"
+@rem Quote it here; it's not quoted later because "py -3.6" wouldn't work
+@if not errorlevel 1 (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found on nuget.org) & goto :found
+
+
+@set _Py_Python_Source=
+@set _Py_EXTERNALS_DIR=
+@set _Py_NUGET=
+@set _Py_NUGET_URL=
+@set _Py_HOST_PYTHON=
+@exit /b 1
+
+:found
+@echo Using %PYTHON% (%_Py_Python_Source%)
+@set _Py_Python_Source=
+@set _Py_EXTERNALS_DIR=
+@set _Py_NUGET=
+@set _Py_NUGET_URL=
+@set _Py_HOST_PYTHON=

PCbuild/get_externals.bat

@@ -4,8 +4,6 @@ rem Simple script to fetch source for external libraries
 
 if "%PCBUILD%"=="" (set PCBUILD=%~dp0)
 if "%EXTERNALS_DIR%"=="" (set EXTERNALS_DIR=%PCBUILD%\..\externals)
-if "%NUGET%"=="" (set NUGET=%EXTERNALS_DIR%\nuget.exe)
-if "%NUGET_URL%"=="" (set NUGET_URL=https://aka.ms/nugetclidl)
 
 set DO_FETCH=true
 set DO_CLEAN=false
@@ -13,7 +11,7 @@ set DO_CLEAN=false
 :CheckOpts
 if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts
 if "%~1"=="--no-openssl" (set IncludeSSL=false) & shift & goto CheckOpts
-if "%~1"=="--python" (set PYTHON_FOR_BUILD=%2) & shift & shift & goto CheckOpts
+if "%~1"=="--python" (set PYTHON=%2) & shift & shift & goto CheckOpts
 if "%~1"=="--organization" (set ORG=%2) & shift & shift & goto CheckOpts
 if "%~1"=="-c" (set DO_CLEAN=true) & shift & goto CheckOpts
 if "%~1"=="--clean" (set DO_CLEAN=true) & shift & goto CheckOpts
@@ -33,22 +31,10 @@ if "%DO_FETCH%"=="false" goto end
 
 if "%ORG%"=="" (set ORG=python)
 
-if "%PYTHON_FOR_BUILD%"=="" (
-    echo Checking for installed python...
-    py -3.6 -V >nul 2>&1 && (set PYTHON_FOR_BUILD=py -3.6)
-)
-if "%PYTHON_FOR_BUILD%"=="" (
-    if NOT exist "%EXTERNALS_DIR%" mkdir "%EXTERNALS_DIR%"
-    if NOT exist "%NUGET%" (
-        echo Downloading nuget...
-        rem NB: Must use single quotes around NUGET here, NOT double!
-        rem Otherwise, a space in the path would break things
-        powershell.exe -Command Invoke-WebRequest %NUGET_URL% -OutFile '%NUGET%'
-    )
-    echo Installing Python via nuget...
-    "%NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%EXTERNALS_DIR%"
-    rem Quote it here; it's not quoted later because "py -3.6" wouldn't work
-    set PYTHON_FOR_BUILD="%EXTERNALS_DIR%\pythonx86\tools\python.exe"
+call "%PCBUILD%find_python.bat" "%PYTHON%"
+
+if "%PYTHON%"=="" (
+    where /Q git || echo Python 3.6 could not be found or installed, and git.exe is not on your PATH && exit /B 1
 )
 
 echo.Fetching external libraries...
@@ -65,9 +51,12 @@ set libraries=%libraries%                                    xz-5.2.2
 for %%e in (%libraries%) do (
     if exist "%EXTERNALS_DIR%\%%e" (
         echo.%%e already exists, skipping.
+    ) else if "%PYTHON%"=="" (
+        echo.Fetching %%e with git...
+        git clone --depth 1 https://github.com/%ORG%/cpython-source-deps --branch %%e "%EXTERNALS_DIR%\%%e"
     ) else (
         echo.Fetching %%e...
-        %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -O %ORG% %%e
+        %PYTHON% "%PCBUILD%get_external.py" -O %ORG% %%e
     )
 )
 
@@ -80,9 +69,12 @@ if NOT "%IncludeSSL%"=="false" set binaries=%binaries%     nasm-2.11.06
 for %%b in (%binaries%) do (
     if exist "%EXTERNALS_DIR%\%%b" (
         echo.%%b already exists, skipping.
+    ) else if "%PYTHON%"=="" (
+        echo.Fetching %%b with git...
+        git clone --depth 1 https://github.com/%ORG%/cpython-bin-deps --branch %%b "%EXTERNALS_DIR%\%%b"
     ) else (
         echo.Fetching %%b...
-        %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -b -O %ORG% %%b
+        %PYTHON% "%PCBUILD%get_external.py" -b -O %ORG% %%b
     )
 )
 

PCbuild/urlretrieve.py

+# Simple Python script to download a file. Used as a fallback
+# when other more reliable methods fail.
+#
+from __future__ import print_function
+import sys
+
+try:
+    from requests import get
+except ImportError:
+    try:
+        from urllib.request import urlretrieve
+        USING = "urllib.request.urlretrieve"
+    except ImportError:
+        try:
+            from urllib import urlretrieve
+            USING = "urllib.retrieve"
+        except ImportError:
+            print("Python at", sys.executable, "is not suitable",
+                  "for downloading files.", file=sys.stderr)
+            sys.exit(2)
+else:
+    USING = "requests.get"
+
+    def urlretrieve(url, filename):
+        r = get(url, stream=True)
+        r.raise_for_status()
+        with open(filename, 'wb') as f:
+            for chunk in r.iter_content(chunk_size=1024):
+                f.write(chunk)
+        return filename
+
+if __name__ == '__main__':
+    if len(sys.argv) != 3:
+        print("Usage: urlretrieve.py [url] [filename]", file=sys.stderr)
+        sys.exit(1)
+    URL = sys.argv[1]
+    FILENAME = sys.argv[2]
+    print("Downloading from", URL, "to", FILENAME, "using", USING)
+    urlretrieve(URL, FILENAME)

Tools/msi/buildrelease.bat

@@ -76,9 +76,6 @@ if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable &
 if "%SKIPBUILD%" EQU "1" goto skipdoc
 if "%SKIPDOC%" EQU "1" goto skipdoc
 
-if not defined PYTHON where py -q || echo Cannot find py on path and PYTHON is not set. && exit /B 1
-if not defined SPHINXBUILD where sphinx-build -q || echo Cannot find sphinx-build on path and SPHINXBUILD is not set. && exit /B 1
-
 call "%D%..\..\doc\make.bat" htmlhelp
 if errorlevel 1 goto :eof
 :skipdoc

Tools/msi/get_externals.bat

@@ -12,7 +12,7 @@ set DO_FETCH=true
 set DO_CLEAN=false
 
 :CheckOpts
-if "%~1"=="--python" (set PYTHON_FOR_BUILD=%2) & shift & shift & goto CheckOpts
+if "%~1"=="--python" (set PYTHON=%2) & shift & shift & goto CheckOpts
 if "%~1"=="--organization" (set ORG=%2) & shift & shift & goto CheckOpts
 if "%~1"=="-c" (set DO_CLEAN=true) & shift & goto CheckOpts
 if "%~1"=="--clean" (set DO_CLEAN=true) & shift & goto CheckOpts
@@ -32,23 +32,7 @@ if "%DO_FETCH%"=="false" goto end
 
 if "%ORG%"=="" (set ORG=python)
 
-if "%PYTHON_FOR_BUILD%"=="" (
-    echo Checking for installed python...
-    py -3.6 -V >nul 2>&1 && (set PYTHON_FOR_BUILD=py -3.6)
-)
-if "%PYTHON_FOR_BUILD%"=="" (
-    if NOT exist "%EXTERNALS_DIR%" mkdir "%EXTERNALS_DIR%"
-    if NOT exist "%NUGET%" (
-        echo Downloading nuget...
-        rem NB: Must use single quotes around NUGET here, NOT double!
-        rem Otherwise, a space in the path would break things
-        powershell.exe -Command Invoke-WebRequest %NUGET_URL% -OutFile '%NUGET%'
-    )
-    echo Installing Python via nuget...
-    "%NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%EXTERNALS_DIR%"
-    rem Quote it here; it's not quoted later because "py -3.6" wouldn't work
-    set PYTHON_FOR_BUILD="%EXTERNALS_DIR%\pythonx86\tools\python.exe"
-)
+call "%PCBUILD%\find_python.bat" "%PYTHON%"
 
 echo.Fetching external libraries...
 
@@ -59,7 +43,7 @@ for %%e in (%libraries%) do (
         echo.%%e already exists, skipping.
     ) else (
         echo.Fetching %%e...
-        %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -O %ORG% %%e
+        %PYTHON% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -O %ORG% %%e
     )
 )
 
@@ -79,7 +63,7 @@ for %%b in (%binaries%) do (
         echo.%%b already exists, skipping.
     ) else (
         echo.Fetching %%b...
-        %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -b -O %ORG% %%b
+        %PYTHON% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -b -O %ORG% %%b
     )
 )