make_zip.py 7.58 KB
Newer Older
1
import argparse
2
import py_compile
3 4 5
import re
import sys
import shutil
6
import stat
7 8 9
import os
import tempfile

10
from itertools import chain
11 12
from pathlib import Path
from zipfile import ZipFile, ZIP_DEFLATED
13

14 15

TKTCL_RE = re.compile(r'^(_?tk|tcl).+\.(pyd|dll)', re.IGNORECASE)
16
DEBUG_RE = re.compile(r'_d\.(pyd|dll|exe|pdb|lib)$', re.IGNORECASE)
17 18
PYTHON_DLL_RE = re.compile(r'python\d\d?\.dll$', re.IGNORECASE)

19 20 21 22
DEBUG_FILES = {
    '_ctypes_test',
    '_testbuffer',
    '_testcapi',
23
    '_testconsole',
24 25 26 27 28 29
    '_testimportmultiple',
    '_testmultiphase',
    'xxlimited',
    'python3_dstub',
}

30 31 32 33 34 35 36
EXCLUDE_FROM_LIBRARY = {
    '__pycache__',
    'idlelib',
    'pydoc_data',
    'site-packages',
    'tkinter',
    'turtledemo',
37 38 39 40
}

EXCLUDE_FROM_EMBEDDABLE_LIBRARY = {
    'ensurepip',
41
    'venv',
42 43 44 45 46 47
}

EXCLUDE_FILE_FROM_LIBRARY = {
    'bdist_wininst.py',
}

48
EXCLUDE_FILE_FROM_LIBS = {
49
    'liblzma',
50 51 52 53 54
    'ssleay',
    'libeay',
    'python3stub',
}

55 56 57 58
EXCLUDED_FILES = {
    'pyshellext',
}

59
def is_not_debug(p):
60 61 62 63 64 65
    if DEBUG_RE.search(p.name):
        return False

    if TKTCL_RE.search(p.name):
        return False

66
    return p.stem.lower() not in DEBUG_FILES and p.stem.lower() not in EXCLUDED_FILES
67 68 69 70 71 72 73

def is_not_debug_or_python(p):
    return is_not_debug(p) and not PYTHON_DLL_RE.search(p.name)

def include_in_lib(p):
    name = p.name.lower()
    if p.is_dir():
74
        if name in EXCLUDE_FROM_LIBRARY:
75 76 77
            return False
        if name == 'test' and p.parts[-2].lower() == 'lib':
            return False
78 79
        if name in {'test', 'tests'} and p.parts[-3].lower() == 'lib':
            return False
80 81
        return True

82 83 84
    if name in EXCLUDE_FILE_FROM_LIBRARY:
        return False

85
    suffix = p.suffix.lower()
86
    return suffix not in {'.pyc', '.pyo', '.exe'}
87

88 89 90 91 92 93
def include_in_embeddable_lib(p):
    if p.is_dir() and p.name.lower() in EXCLUDE_FROM_EMBEDDABLE_LIBRARY:
        return False

    return include_in_lib(p)

94 95 96 97 98 99
def include_in_libs(p):
    if not is_not_debug(p):
        return False

    return p.stem.lower() not in EXCLUDE_FILE_FROM_LIBS

100 101 102 103 104 105
def include_in_tools(p):
    if p.is_dir() and p.name.lower() in {'scripts', 'i18n', 'pynche', 'demo', 'parser'}:
        return True

    return p.suffix.lower() in {'.py', '.pyw', '.txt'}

106 107
BASE_NAME = 'python{0.major}{0.minor}'.format(sys.version_info)

108
FULL_LAYOUT = [
109 110
    ('/', 'PCBuild/$arch', 'python.exe', is_not_debug),
    ('/', 'PCBuild/$arch', 'pythonw.exe', is_not_debug),
111 112
    ('/', 'PCBuild/$arch', 'python{}.dll'.format(sys.version_info.major), is_not_debug),
    ('/', 'PCBuild/$arch', '{}.dll'.format(BASE_NAME), is_not_debug),
113 114
    ('DLLs/', 'PCBuild/$arch', '*.pyd', is_not_debug),
    ('DLLs/', 'PCBuild/$arch', '*.dll', is_not_debug_or_python),
115 116 117
    ('include/', 'include', '*.h', None),
    ('include/', 'PC', 'pyconfig.h', None),
    ('Lib/', 'Lib', '**/*', include_in_lib),
118
    ('libs/', 'PCBuild/$arch', '*.lib', include_in_libs),
119 120 121 122
    ('Tools/', 'Tools', '**/*', include_in_tools),
]

EMBED_LAYOUT = [
123 124 125
    ('/', 'PCBuild/$arch', 'python*.exe', is_not_debug),
    ('/', 'PCBuild/$arch', '*.pyd', is_not_debug),
    ('/', 'PCBuild/$arch', '*.dll', is_not_debug),
126
    ('{}.zip'.format(BASE_NAME), 'Lib', '**/*', include_in_embeddable_lib),
127 128
]

129 130 131 132 133 134
if os.getenv('DOC_FILENAME'):
    FULL_LAYOUT.append(('Doc/', 'Doc/build/htmlhelp', os.getenv('DOC_FILENAME'), None))
if os.getenv('VCREDIST_PATH'):
    FULL_LAYOUT.append(('/', os.getenv('VCREDIST_PATH'), 'vcruntime*.dll', None))
    EMBED_LAYOUT.append(('/', os.getenv('VCREDIST_PATH'), 'vcruntime*.dll', None))

135
def copy_to_layout(target, rel_sources):
136 137 138 139 140 141 142
    count = 0

    if target.suffix.lower() == '.zip':
        if target.exists():
            target.unlink()

        with ZipFile(str(target), 'w', ZIP_DEFLATED) as f:
143 144 145 146 147 148 149 150 151 152
            with tempfile.TemporaryDirectory() as tmpdir:
                for s, rel in rel_sources:
                    if rel.suffix.lower() == '.py':
                        pyc = Path(tmpdir) / rel.with_suffix('.pyc').name
                        try:
                            py_compile.compile(str(s), str(pyc), str(rel), doraise=True, optimize=2)
                        except py_compile.PyCompileError:
                            f.write(str(s), str(rel))
                        else:
                            f.write(str(pyc), str(rel.with_suffix('.pyc')))
153
                    else:
154 155
                        f.write(str(s), str(rel))
                    count += 1
156 157 158

    else:
        for s, rel in rel_sources:
159
            dest = target / rel
160
            try:
161
                dest.parent.mkdir(parents=True)
162 163
            except FileExistsError:
                pass
164 165 166 167 168
            if dest.is_file():
                dest.chmod(stat.S_IWRITE)
            shutil.copy(str(s), str(dest))
            if dest.is_file():
                dest.chmod(stat.S_IWRITE)
169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186
            count += 1

    return count

def rglob(root, pattern, condition):
    dirs = [root]
    recurse = pattern[:3] in {'**/', '**\\'}
    while dirs:
        d = dirs.pop(0)
        for f in d.glob(pattern[3:] if recurse else pattern):
            if recurse and f.is_dir() and (not condition or condition(f)):
                dirs.append(f)
            elif f.is_file() and (not condition or condition(f)):
                yield f, f.relative_to(root)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-s', '--source', metavar='dir', help='The directory containing the repository root', type=Path)
187
    parser.add_argument('-o', '--out', metavar='file', help='The name of the output archive', type=Path, default=None)
188 189
    parser.add_argument('-t', '--temp', metavar='dir', help='A directory to temporarily extract files into', type=Path, default=None)
    parser.add_argument('-e', '--embed', help='Create an embedding layout', action='store_true', default=False)
190
    parser.add_argument('-a', '--arch', help='Specify the architecture to use (win32/amd64)', type=str, default="win32")
191 192
    ns = parser.parse_args()

193
    source = ns.source or (Path(__file__).resolve().parent.parent.parent)
194
    out = ns.out
195
    arch = ns.arch
196
    assert isinstance(source, Path)
197
    assert not out or isinstance(out, Path)
198
    assert isinstance(arch, str)
199 200 201 202 203 204 205 206

    if ns.temp:
        temp = ns.temp
        delete_temp = False
    else:
        temp = Path(tempfile.mkdtemp())
        delete_temp = True

207 208 209 210 211
    if out:
        try:
            out.parent.mkdir(parents=True)
        except FileExistsError:
            pass
212 213 214 215 216 217 218 219 220
    try:
        temp.mkdir(parents=True)
    except FileExistsError:
        pass

    layout = EMBED_LAYOUT if ns.embed else FULL_LAYOUT

    try:
        for t, s, p, c in layout:
221
            fs = source / s.replace("$arch", arch)
222 223 224 225
            files = rglob(fs, p, c)
            extra_files = []
            if s == 'Lib' and p == '**/*':
                extra_files.append((
226 227
                    source / 'tools' / 'msi' / 'distutils.command.bdist_wininst.py',
                    Path('distutils') / 'command' / 'bdist_wininst.py'
228 229
                ))
            copied = copy_to_layout(temp / t.rstrip('/'), chain(files, extra_files))
230 231
            print('Copied {} files'.format(copied))

232
        if ns.embed:
233 234
            with open(str(temp / (BASE_NAME + '._pth')), 'w') as f:
                print(BASE_NAME + '.zip', file=f)
235
                print('.', file=f)
236 237 238
                print('', file=f)
                print('# Uncomment to run site.main() automatically', file=f)
                print('#import site', file=f)
239

240 241 242
        if out:
            total = copy_to_layout(out, rglob(temp, '**/*', None))
            print('Wrote {} files to {}'.format(total, out))
243 244 245 246 247 248 249
    finally:
        if delete_temp:
            shutil.rmtree(temp, True)


if __name__ == "__main__":
    sys.exit(int(main() or 0))