Issue #26268: Merge with 3.5

Misc/NEWS

@@ -751,11 +751,17 @@ Build
   Previously it could fail to compile properly if the math and cmath builds
   were concurrent.
 
-- Issue #26268: Update OS X 10.5 installer to use OpenSSL 1.0.2f.
+- Issue #26268: Update OS X 10.5 installer and Windows builds to use
+  OpenSSL 1.0.2f.
+
 
 Windows
 -------
 
+- Issue #26268: Update the prepare_ssl.py script to handle OpenSSL releases
+  that don't include the contents of the include directory (that is, 1.0.2e
+  and later).
+
 - Issue #26071: bdist_wininst created binaries fail to start and find
   32bit Python
 

PCbuild/_hashlib.vcxproj

@@ -61,7 +61,7 @@
   </PropertyGroup>
   <ItemDefinitionGroup>
     <ClCompile>
-      <AdditionalIncludeDirectories>$(opensslDir)include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(opensslIncludeDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
       <AdditionalDependencies>ws2_32.lib;$(OutDir)libeay$(PyDebugExt).lib;$(OutDir)ssleay$(PyDebugExt).lib;%(AdditionalDependencies)</AdditionalDependencies>
@@ -87,4 +87,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
\ No newline at end of file
+</Project>

PCbuild/_ssl.vcxproj

@@ -61,7 +61,7 @@
   </PropertyGroup>
   <ItemDefinitionGroup>
     <ClCompile>
-      <AdditionalIncludeDirectories>$(opensslDir)include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(opensslIncludeDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
       <AdditionalDependencies>ws2_32.lib;crypt32.lib;$(OutDir)libeay$(PyDebugExt).lib;$(OutDir)ssleay$(PyDebugExt).lib;%(AdditionalDependencies)</AdditionalDependencies>
@@ -91,4 +91,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
\ No newline at end of file
+</Project>

PCbuild/get_externals.bat

@@ -54,7 +54,7 @@ echo.Fetching external libraries...
 set libraries=
 set libraries=%libraries%                                    bzip2-1.0.6
 if NOT "%IncludeSSL%"=="false" set libraries=%libraries%     nasm-2.11.06
-if NOT "%IncludeSSL%"=="false" set libraries=%libraries%     openssl-1.0.2d
+if NOT "%IncludeSSL%"=="false" set libraries=%libraries%     openssl-1.0.2f
 set libraries=%libraries%                                    sqlite-3.8.11.0
 if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tcl-core-8.6.4.2
 if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tk-8.6.4.2

PCbuild/openssl.props

@@ -17,9 +17,8 @@
     <PreprocessorDefinitions Include="_CRT_SECURE_NO_WARNINGS" />
     <PreprocessorDefinitions Include="_CRT_SECURE_NO_DEPRECATE" />
     <PreprocessorDefinitions Include="OPENSSL_THREADS" />
-    <PreprocessorDefinitions Include="OPENSSL_SYSNAME_WIN32" />
+    <!-- <PreprocessorDefinitions Include="OPENSSL_SYSNAME_WIN32" /> -->
     <PreprocessorDefinitions Include="OPENSSL_IA32_SSE2" />
-    <PreprocessorDefinitions Include="OPENSSL_CPUID_OBJ" />
     <PreprocessorDefinitions Include="SHA1_ASM" />
     <PreprocessorDefinitions Include="SHA256_ASM" />
     <PreprocessorDefinitions Include="SHA512_ASM" />
@@ -53,7 +52,7 @@
     <ClCompile>
       <!-- Suppress 64-bit truncation warnings - they aren't ours to worry about -->
       <DisableSpecificWarnings>4244;4267</DisableSpecificWarnings>
-      <AdditionalIncludeDirectories>$(opensslDir);$(opensslDir)include;$(opensslDir)crypto;$(opensslDir)crypto\asn1;$(opensslDir)crypto\evp;$(opensslDir)crypto\modes</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(opensslDir);$(opensslIncludeDir);$(opensslDir)crypto;$(opensslDir)crypto\asn1;$(opensslDir)crypto\evp;$(opensslDir)crypto\modes</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>$(_PreprocessorDefinitionList);%(PreprocessorDefinitions)</PreprocessorDefinitions>
     </ClCompile>
   </ItemDefinitionGroup>
@@ -74,4 +73,4 @@ $(nasm) -o "$(IntDir)%(NasmCompile.Filename).obj" "%(NasmCompile.FullPath)"' />
       <Lib Include="$(IntDir)%(NasmCompile.Filename).obj" />
     </ItemGroup>
   </Target>
-</Project>
\ No newline at end of file
+</Project>

PCbuild/prepare_ssl.py

@@ -21,11 +21,11 @@
 import os
 import re
 import sys
-import shutil
 import subprocess
+from shutil import copy
 
 # Find all "foo.exe" files on the PATH.
-def find_all_on_path(filename, extras = None):
+def find_all_on_path(filename, extras=None):
     entries = os.environ["PATH"].split(os.pathsep)
     ret = []
     for p in entries:
@@ -39,6 +39,7 @@ def find_all_on_path(filename, extras = None):
                 ret.append(fname)
     return ret
 
+
 # Find a suitable Perl installation for OpenSSL.
 # cygwin perl does *not* work.  ActivePerl does.
 # Being a Perl dummy, the simplest way I can check is if the "Win32" package
@@ -61,82 +62,47 @@ def find_working_perl(perls):
         print("NO perl interpreters were found on this machine at all!")
     print(" Please install ActivePerl and ensure it appears on your path")
 
-def create_makefile64(makefile, m32):
-    """Create and fix makefile for 64bit
-
-    Replace 32 with 64bit directories
-    """
-    if not os.path.isfile(m32):
-        return
-    with open(m32) as fin:
-        with open(makefile, 'w') as fout:
-            for line in fin:
-                line = line.replace("=tmp32", "=tmp64")
-                line = line.replace("=out32", "=out64")
-                line = line.replace("=inc32", "=inc64")
-                # force 64 bit machine
-                line = line.replace("MKLIB=lib", "MKLIB=lib /MACHINE:X64")
-                line = line.replace("LFLAGS=", "LFLAGS=/MACHINE:X64 ")
-                # don't link against the lib on 64bit systems
-                line = line.replace("bufferoverflowu.lib", "")
-                fout.write(line)
-    os.unlink(m32)
-
-def create_asms(makefile):
+
+def create_asms(makefile, tmp_d):
     #create a custom makefile out of the provided one
     asm_makefile = os.path.splitext(makefile)[0] + '.asm.mak'
-    with open(makefile) as fin:
-        with open(asm_makefile, 'w') as fout:
-            for line in fin:
-                # Keep everything up to the install target (it's convenient)
-                if line.startswith('install: all'):
-                    break
-                else:
+    with open(makefile) as fin, open(asm_makefile, 'w') as fout:
+        for line in fin:
+            # Keep everything up to the install target (it's convenient)
+            if line.startswith('install: all'):
+                break
+            fout.write(line)
+        asms = []
+        for line in fin:
+            if '.asm' in line and line.strip().endswith('.pl'):
+                asms.append(line.split(':')[0])
+                while line.strip():
                     fout.write(line)
-            asms = []
-            for line in fin:
-                if '.asm' in line and line.strip().endswith('.pl'):
-                    asms.append(line.split(':')[0])
-                    while line.strip():
-                        fout.write(line)
-                        line = next(fin)
-                    fout.write('\n')
+                    line = next(fin)
+                fout.write('\n')
 
-            fout.write('asms: $(TMP_D) ')
-            fout.write(' '.join(asms))
-            fout.write('\n')
+        fout.write('asms: $(TMP_D) ')
+        fout.write(' '.join(asms))
+        fout.write('\n')
+    os.system('nmake /f {} PERL=perl TMP_D={} asms'.format(asm_makefile, tmp_d))
 
-    os.system('nmake /f {} PERL=perl asms'.format(asm_makefile))
-    os.unlink(asm_makefile)
 
-
-
-def fix_makefile(makefile):
-    """Fix some stuff in all makefiles
-    """
-    if not os.path.isfile(makefile):
-        return
+def copy_includes(makefile, suffix):
+    dir = 'include'+suffix+'\\openssl'
+    os.makedirs(dir, exist_ok=True)
     copy_if_different = r'$(PERL) $(SRC_D)\util\copy-if-different.pl'
     with open(makefile) as fin:
-        lines = fin.readlines()
-    with open(makefile, 'w') as fout:
-        for line in lines:
-            if line.startswith("PERL="):
-                continue
-            if line.startswith("CP="):
-                line = "CP=copy\n"
-            if line.startswith("MKDIR="):
-                line = "MKDIR=mkdir\n"
-            if line.startswith("CFLAG="):
-                line = line.strip()
-                for algo in ("RC5", "MDC2", "IDEA"):
-                    noalgo = " -DOPENSSL_NO_%s" % algo
-                    if noalgo not in line:
-                        line = line + noalgo
-                line = line + '\n'
+        for line in fin:
             if copy_if_different in line:
-                line = line.replace(copy_if_different, 'copy /Y')
-            fout.write(line)
+                perl, script, src, dest = line.split()
+                if not '$(INCO_D)' in dest:
+                    continue
+                # We're in the root of the source tree
+                src = src.replace('$(SRC_D)', '.').strip('"')
+                dest = dest.strip('"').replace('$(INCO_D)', dir)
+                print('copying', src, 'to', dest)
+                copy(src, dest)
+
 
 def run_configure(configure, do_script):
     print("perl Configure "+configure+" no-idea no-mdc2")
@@ -144,61 +110,37 @@ def run_configure(configure, do_script):
     print(do_script)
     os.system(do_script)
 
-def cmp(f1, f2):
-    bufsize = 1024 * 8
-    with open(f1, 'rb') as fp1, open(f2, 'rb') as fp2:
-        while True:
-            b1 = fp1.read(bufsize)
-            b2 = fp2.read(bufsize)
-            if b1 != b2:
-                return False
-            if not b1:
-                return True
-
-def copy(src, dst):
-    if os.path.isfile(dst) and cmp(src, dst):
-        return
-    shutil.copy(src, dst)
 
 def prep(arch):
+    makefile_template = "ms\\nt{}.mak"
+    generated_makefile = makefile_template.format('')
     if arch == "x86":
         configure = "VC-WIN32"
         do_script = "ms\\do_nasm"
-        makefile="ms\\nt.mak"
-        m32 = makefile
-        dirsuffix = "32"
+        suffix = "32"
     elif arch == "amd64":
         configure = "VC-WIN64A"
         do_script = "ms\\do_win64a"
-        makefile = "ms\\nt64.mak"
-        m32 = makefile.replace('64', '')
-        dirsuffix = "64"
+        suffix = "64"
         #os.environ["VSEXTCOMP_USECL"] = "MS_OPTERON"
     else:
         raise ValueError('Unrecognized platform: %s' % arch)
 
-    # rebuild makefile when we do the role over from 32 to 64 build
-    if arch == "amd64" and os.path.isfile(m32) and not os.path.isfile(makefile):
-        os.unlink(m32)
-
-    # If the ssl makefiles do not exist, we invoke Perl to generate them.
-    # Due to a bug in this script, the makefile sometimes ended up empty
-    # Force a regeneration if it is.
-    if not os.path.isfile(makefile) or os.path.getsize(makefile)==0:
-        print("Creating the makefiles...")
-        sys.stdout.flush()
-        run_configure(configure, do_script)
-
-        if arch == "amd64":
-            create_makefile64(makefile, m32)
-        fix_makefile(makefile)
-        copy(r"crypto\buildinf.h", r"crypto\buildinf_%s.h" % arch)
-        copy(r"crypto\opensslconf.h", r"crypto\opensslconf_%s.h" % arch)
-    else:
-        print(makefile, 'already exists!')
+    print("Creating the makefiles...")
+    sys.stdout.flush()
+    # run configure, copy includes, create asms
+    run_configure(configure, do_script)
+    makefile = makefile_template.format(suffix)
+    try:
+        os.unlink(makefile)
+    except FileNotFoundError:
+        pass
+    os.rename(generated_makefile, makefile)
+    copy_includes(makefile, suffix)
 
     print('creating asms...')
-    create_asms(makefile)
+    create_asms(makefile, 'tmp'+suffix)
+
 
 def main():
     if len(sys.argv) == 1:
@@ -229,6 +171,9 @@ def main():
         print("Found a working perl at '%s'" % (perl,))
     else:
         sys.exit(1)
+    if not find_all_on_path('nmake.exe'):
+        print('Could not find nmake.exe, try running env.bat')
+        sys.exit(1)
     sys.stdout.flush()
 
     # Put our working Perl at the front of our path

PCbuild/python.props

@@ -41,7 +41,9 @@
     <sqlite3Dir>$(ExternalsDir)sqlite-3.8.11.0\</sqlite3Dir>
     <bz2Dir>$(ExternalsDir)bzip2-1.0.6\</bz2Dir>
     <lzmaDir>$(ExternalsDir)xz-5.0.5\</lzmaDir>
-    <opensslDir>$(ExternalsDir)openssl-1.0.2d\</opensslDir>
+    <opensslDir>$(ExternalsDir)openssl-1.0.2f\</opensslDir>
+    <opensslIncludeDir>$(opensslDir)include32</opensslIncludeDir>
+    <opensslIncludeDir Condition="'$(ArchName)' == 'amd64'">$(opensslDir)include64</opensslIncludeDir>
     <nasmDir>$(ExternalsDir)\nasm-2.11.06\</nasmDir>
     
     <!-- Suffix for all binaries when building for debug -->

PCbuild/readme.txt

@@ -169,7 +169,7 @@ _lzma
     Homepage:
         http://tukaani.org/xz/
 _ssl
-    Python wrapper for version 1.0.2d of the OpenSSL secure sockets
+    Python wrapper for version 1.0.2f of the OpenSSL secure sockets
     library, which is built by ssl.vcxproj
     Homepage:
         http://www.openssl.org/