Issue #26268: Update the prepare_ssl.py script

It can now handle OpenSSL versions 1.0.2e and greater, which don't
include include files in include/.

Note that sources prepared by this script no longer support the old
project files for 2.7; you now have to have Perl available to use
the old build_ssl.py script with sources from svn.python.org.

Misc/NEWS

@@ -263,6 +263,10 @@ Build
 Windows
 -------
 
+- Issue #26268: Update the prepare_ssl.py script to handle OpenSSL releases
+  that don't include the contents of the include directory (that is, 1.0.2e
+  and later).
+
 - Issue #26071: bdist_wininst created binaries fail to start and find
   32bit Python
 

PCbuild/prepare_ssl.py

@@ -21,11 +21,11 @@
 import os
 import re
 import sys
-import shutil
 import subprocess
+from shutil import copy
 
 # Find all "foo.exe" files on the PATH.
-def find_all_on_path(filename, extras = None):
+def find_all_on_path(filename, extras=None):
     entries = os.environ["PATH"].split(os.pathsep)
     ret = []
     for p in entries:
@@ -39,6 +39,7 @@ def find_all_on_path(filename, extras = None):
                 ret.append(fname)
     return ret
 
+
 # Find a suitable Perl installation for OpenSSL.
 # cygwin perl does *not* work.  ActivePerl does.
 # Being a Perl dummy, the simplest way I can check is if the "Win32" package
@@ -61,82 +62,47 @@ def find_working_perl(perls):
         print("NO perl interpreters were found on this machine at all!")
     print(" Please install ActivePerl and ensure it appears on your path")
 
-def create_makefile64(makefile, m32):
-    """Create and fix makefile for 64bit
-
-    Replace 32 with 64bit directories
-    """
-    if not os.path.isfile(m32):
-        return
-    with open(m32) as fin:
-        with open(makefile, 'w') as fout:
-            for line in fin:
-                line = line.replace("=tmp32", "=tmp64")
-                line = line.replace("=out32", "=out64")
-                line = line.replace("=inc32", "=inc64")
-                # force 64 bit machine
-                line = line.replace("MKLIB=lib", "MKLIB=lib /MACHINE:X64")
-                line = line.replace("LFLAGS=", "LFLAGS=/MACHINE:X64 ")
-                # don't link against the lib on 64bit systems
-                line = line.replace("bufferoverflowu.lib", "")
-                fout.write(line)
-    os.unlink(m32)
-
-def create_asms(makefile):
+
+def create_asms(makefile, tmp_d):
     #create a custom makefile out of the provided one
     asm_makefile = os.path.splitext(makefile)[0] + '.asm.mak'
-    with open(makefile) as fin:
-        with open(asm_makefile, 'w') as fout:
-            for line in fin:
-                # Keep everything up to the install target (it's convenient)
-                if line.startswith('install: all'):
-                    break
-                else:
+    with open(makefile) as fin, open(asm_makefile, 'w') as fout:
+        for line in fin:
+            # Keep everything up to the install target (it's convenient)
+            if line.startswith('install: all'):
+                break
+            fout.write(line)
+        asms = []
+        for line in fin:
+            if '.asm' in line and line.strip().endswith('.pl'):
+                asms.append(line.split(':')[0])
+                while line.strip():
                     fout.write(line)
-            asms = []
-            for line in fin:
-                if '.asm' in line and line.strip().endswith('.pl'):
-                    asms.append(line.split(':')[0])
-                    while line.strip():
-                        fout.write(line)
-                        line = next(fin)
-                    fout.write('\n')
+                    line = next(fin)
+                fout.write('\n')
 
-            fout.write('asms: $(TMP_D) ')
-            fout.write(' '.join(asms))
-            fout.write('\n')
+        fout.write('asms: $(TMP_D) ')
+        fout.write(' '.join(asms))
+        fout.write('\n')
+    os.system('nmake /f {} PERL=perl TMP_D={} asms'.format(asm_makefile, tmp_d))
 
-    os.system('nmake /f {} PERL=perl asms'.format(asm_makefile))
-    os.unlink(asm_makefile)
 
-
-
-def fix_makefile(makefile):
-    """Fix some stuff in all makefiles
-    """
-    if not os.path.isfile(makefile):
-        return
+def copy_includes(makefile, suffix):
+    dir = 'include'+suffix+'\\openssl'
+    os.makedirs(dir, exist_ok=True)
     copy_if_different = r'$(PERL) $(SRC_D)\util\copy-if-different.pl'
     with open(makefile) as fin:
-        lines = fin.readlines()
-    with open(makefile, 'w') as fout:
-        for line in lines:
-            if line.startswith("PERL="):
-                continue
-            if line.startswith("CP="):
-                line = "CP=copy\n"
-            if line.startswith("MKDIR="):
-                line = "MKDIR=mkdir\n"
-            if line.startswith("CFLAG="):
-                line = line.strip()
-                for algo in ("RC5", "MDC2", "IDEA"):
-                    noalgo = " -DOPENSSL_NO_%s" % algo
-                    if noalgo not in line:
-                        line = line + noalgo
-                line = line + '\n'
+        for line in fin:
             if copy_if_different in line:
-                line = line.replace(copy_if_different, 'copy /Y')
-            fout.write(line)
+                perl, script, src, dest = line.split()
+                if not '$(INCO_D)' in dest:
+                    continue
+                # We're in the root of the source tree
+                src = src.replace('$(SRC_D)', '.').strip('"')
+                dest = dest.strip('"').replace('$(INCO_D)', dir)
+                print('copying', src, 'to', dest)
+                copy(src, dest)
+
 
 def run_configure(configure, do_script):
     print("perl Configure "+configure+" no-idea no-mdc2")
@@ -144,61 +110,37 @@ def run_configure(configure, do_script):
     print(do_script)
     os.system(do_script)
 
-def cmp(f1, f2):
-    bufsize = 1024 * 8
-    with open(f1, 'rb') as fp1, open(f2, 'rb') as fp2:
-        while True:
-            b1 = fp1.read(bufsize)
-            b2 = fp2.read(bufsize)
-            if b1 != b2:
-                return False
-            if not b1:
-                return True
-
-def copy(src, dst):
-    if os.path.isfile(dst) and cmp(src, dst):
-        return
-    shutil.copy(src, dst)
 
 def prep(arch):
+    makefile_template = "ms\\nt{}.mak"
+    generated_makefile = makefile_template.format('')
     if arch == "x86":
         configure = "VC-WIN32"
         do_script = "ms\\do_nasm"
-        makefile="ms\\nt.mak"
-        m32 = makefile
-        dirsuffix = "32"
+        suffix = "32"
     elif arch == "amd64":
         configure = "VC-WIN64A"
         do_script = "ms\\do_win64a"
-        makefile = "ms\\nt64.mak"
-        m32 = makefile.replace('64', '')
-        dirsuffix = "64"
+        suffix = "64"
         #os.environ["VSEXTCOMP_USECL"] = "MS_OPTERON"
     else:
         raise ValueError('Unrecognized platform: %s' % arch)
 
-    # rebuild makefile when we do the role over from 32 to 64 build
-    if arch == "amd64" and os.path.isfile(m32) and not os.path.isfile(makefile):
-        os.unlink(m32)
-
-    # If the ssl makefiles do not exist, we invoke Perl to generate them.
-    # Due to a bug in this script, the makefile sometimes ended up empty
-    # Force a regeneration if it is.
-    if not os.path.isfile(makefile) or os.path.getsize(makefile)==0:
-        print("Creating the makefiles...")
-        sys.stdout.flush()
-        run_configure(configure, do_script)
-
-        if arch == "amd64":
-            create_makefile64(makefile, m32)
-        fix_makefile(makefile)
-        copy(r"crypto\buildinf.h", r"crypto\buildinf_%s.h" % arch)
-        copy(r"crypto\opensslconf.h", r"crypto\opensslconf_%s.h" % arch)
-    else:
-        print(makefile, 'already exists!')
+    print("Creating the makefiles...")
+    sys.stdout.flush()
+    # run configure, copy includes, create asms
+    run_configure(configure, do_script)
+    makefile = makefile_template.format(suffix)
+    try:
+        os.unlink(makefile)
+    except FileNotFoundError:
+        pass
+    os.rename(generated_makefile, makefile)
+    copy_includes(makefile, suffix)
 
     print('creating asms...')
-    create_asms(makefile)
+    create_asms(makefile, 'tmp'+suffix)
+
 
 def main():
     if len(sys.argv) == 1:
@@ -229,6 +171,9 @@ def main():
         print("Found a working perl at '%s'" % (perl,))
     else:
         sys.exit(1)
+    if not find_all_on_path('nmake.exe'):
+        print('Could not find nmake.exe, try running env.bat')
+        sys.exit(1)
     sys.stdout.flush()
 
     # Put our working Perl at the front of our path