test_compileall.py 14.7 KB
Newer Older
1
import sys
2 3 4 5 6 7
import compileall
import imp
import os
import py_compile
import shutil
import struct
Barry Warsaw's avatar
Barry Warsaw committed
8
import subprocess
9
import tempfile
10
import time
11
import unittest
12
import io
13

14
from test import support, script_helper
15 16 17 18 19 20

class CompileallTests(unittest.TestCase):

    def setUp(self):
        self.directory = tempfile.mkdtemp()
        self.source_path = os.path.join(self.directory, '_test.py')
Barry Warsaw's avatar
Barry Warsaw committed
21
        self.bc_path = imp.cache_from_source(self.source_path)
22 23
        with open(self.source_path, 'w') as file:
            file.write('x = 123\n')
24
        self.source_path2 = os.path.join(self.directory, '_test2.py')
Barry Warsaw's avatar
Barry Warsaw committed
25
        self.bc_path2 = imp.cache_from_source(self.source_path2)
26
        shutil.copyfile(self.source_path, self.source_path2)
27 28 29 30
        self.subdirectory = os.path.join(self.directory, '_subdir')
        os.mkdir(self.subdirectory)
        self.source_path3 = os.path.join(self.subdirectory, '_test3.py')
        shutil.copyfile(self.source_path, self.source_path3)
31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65

    def tearDown(self):
        shutil.rmtree(self.directory)

    def data(self):
        with open(self.bc_path, 'rb') as file:
            data = file.read(8)
        mtime = int(os.stat(self.source_path).st_mtime)
        compare = struct.pack('<4sl', imp.get_magic(), mtime)
        return data, compare

    def recreation_check(self, metadata):
        """Check that compileall recreates bytecode when the new metadata is
        used."""
        if not hasattr(os, 'stat'):
            return
        py_compile.compile(self.source_path)
        self.assertEqual(*self.data())
        with open(self.bc_path, 'rb') as file:
            bc = file.read()[len(metadata):]
        with open(self.bc_path, 'wb') as file:
            file.write(metadata)
            file.write(bc)
        self.assertNotEqual(*self.data())
        compileall.compile_dir(self.directory, force=False, quiet=True)
        self.assertTrue(*self.data())

    def test_mtime(self):
        # Test a change in mtime leads to a new .pyc.
        self.recreation_check(struct.pack('<4sl', imp.get_magic(), 1))

    def test_magic_number(self):
        # Test a change in mtime leads to a new .pyc.
        self.recreation_check(b'\0\0\0\0')

66 67 68 69 70 71 72 73
    def test_compile_files(self):
        # Test compiling a single file, and complete directory
        for fn in (self.bc_path, self.bc_path2):
            try:
                os.unlink(fn)
            except:
                pass
        compileall.compile_file(self.source_path, force=False, quiet=True)
Barry Warsaw's avatar
Barry Warsaw committed
74 75
        self.assertTrue(os.path.isfile(self.bc_path) and
                        not os.path.isfile(self.bc_path2))
76 77
        os.unlink(self.bc_path)
        compileall.compile_dir(self.directory, force=False, quiet=True)
Barry Warsaw's avatar
Barry Warsaw committed
78 79
        self.assertTrue(os.path.isfile(self.bc_path) and
                        os.path.isfile(self.bc_path2))
80 81
        os.unlink(self.bc_path)
        os.unlink(self.bc_path2)
82

83 84 85 86 87 88 89 90 91 92 93 94
    def test_no_pycache_in_non_package(self):
        # Bug 8563 reported that __pycache__ directories got created by
        # compile_file() for non-.py files.
        data_dir = os.path.join(self.directory, 'data')
        data_file = os.path.join(data_dir, 'file')
        os.mkdir(data_dir)
        # touch data/file
        with open(data_file, 'w'):
            pass
        compileall.compile_file(data_file)
        self.assertFalse(os.path.exists(os.path.join(data_dir, '__pycache__')))

95 96 97 98 99 100 101 102
    def test_optimize(self):
        # make sure compiling with different optimization settings than the
        # interpreter's creates the correct file names
        optimize = 1 if __debug__ else 0
        compileall.compile_dir(self.directory, quiet=True, optimize=optimize)
        cached = imp.cache_from_source(self.source_path,
                                       debug_override=not optimize)
        self.assertTrue(os.path.isfile(cached))
103 104 105 106 107 108
        cached2 = imp.cache_from_source(self.source_path2,
                                       debug_override=not optimize)
        self.assertTrue(os.path.isfile(cached2))
        cached3 = imp.cache_from_source(self.source_path3,
                                       debug_override=not optimize)
        self.assertTrue(os.path.isfile(cached3))
109

Barry Warsaw's avatar
Barry Warsaw committed
110

111
class EncodingTest(unittest.TestCase):
Barry Warsaw's avatar
Barry Warsaw committed
112 113
    """Issue 6716: compileall should escape source code when printing errors
    to stdout."""
114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132

    def setUp(self):
        self.directory = tempfile.mkdtemp()
        self.source_path = os.path.join(self.directory, '_test.py')
        with open(self.source_path, 'w', encoding='utf-8') as file:
            file.write('# -*- coding: utf-8 -*-\n')
            file.write('print u"\u20ac"\n')

    def tearDown(self):
        shutil.rmtree(self.directory)

    def test_error(self):
        try:
            orig_stdout = sys.stdout
            sys.stdout = io.TextIOWrapper(io.BytesIO(),encoding='ascii')
            compileall.compile_dir(self.directory)
        finally:
            sys.stdout = orig_stdout

133

Barry Warsaw's avatar
Barry Warsaw committed
134
class CommandLineTests(unittest.TestCase):
135
    """Test compileall's CLI."""
Barry Warsaw's avatar
Barry Warsaw committed
136

137 138 139 140 141 142
    def _get_run_args(self, args):
        interp_args = ['-S']
        if sys.flags.optimize:
            interp_args.append({1 : '-O', 2 : '-OO'}[sys.flags.optimize])
        return interp_args + ['-m', 'compileall'] + list(args)

143 144
    def assertRunOK(self, *args, **env_vars):
        rc, out, err = script_helper.assert_python_ok(
145
                         *self._get_run_args(args), **env_vars)
146 147 148
        self.assertEqual(b'', err)
        return out

149
    def assertRunNotOK(self, *args, **env_vars):
150
        rc, out, err = script_helper.assert_python_failure(
151
                        *self._get_run_args(args), **env_vars)
152 153 154 155 156 157 158 159
        return rc, out, err

    def assertCompiled(self, fn):
        self.assertTrue(os.path.exists(imp.cache_from_source(fn)))

    def assertNotCompiled(self, fn):
        self.assertFalse(os.path.exists(imp.cache_from_source(fn)))

Barry Warsaw's avatar
Barry Warsaw committed
160 161 162 163 164
    def setUp(self):
        self.addCleanup(self._cleanup)
        self.directory = tempfile.mkdtemp()
        self.pkgdir = os.path.join(self.directory, 'foo')
        os.mkdir(self.pkgdir)
165 166 167 168
        self.pkgdir_cachedir = os.path.join(self.pkgdir, '__pycache__')
        # Create the __init__.py and a package module.
        self.initfn = script_helper.make_script(self.pkgdir, '__init__', '')
        self.barfn = script_helper.make_script(self.pkgdir, 'bar', '')
Barry Warsaw's avatar
Barry Warsaw committed
169 170 171

    def _cleanup(self):
        support.rmtree(self.directory)
172 173 174 175 176 177 178 179

    def test_no_args_compiles_path(self):
        # Note that -l is implied for the no args case.
        bazfn = script_helper.make_script(self.directory, 'baz', '')
        self.assertRunOK(PYTHONPATH=self.directory)
        self.assertCompiled(bazfn)
        self.assertNotCompiled(self.initfn)
        self.assertNotCompiled(self.barfn)
Barry Warsaw's avatar
Barry Warsaw committed
180

181 182 183 184 185
    # Ensure that the default behavior of compileall's CLI is to create
    # PEP 3147 pyc/pyo files.
    for name, ext, switch in [
        ('normal', 'pyc', []),
        ('optimize', 'pyo', ['-O']),
186
        ('doubleoptimize', 'pyo', ['-OO']),
187 188
    ]:
        def f(self, ext=ext, switch=switch):
189 190
            script_helper.assert_python_ok(*(switch +
                ['-m', 'compileall', '-q', self.pkgdir]))
191
            # Verify the __pycache__ directory contents.
192
            self.assertTrue(os.path.exists(self.pkgdir_cachedir))
193 194
            expected = sorted(base.format(imp.get_tag(), ext) for base in
                              ('__init__.{}.{}', 'bar.{}.{}'))
195
            self.assertEqual(sorted(os.listdir(self.pkgdir_cachedir)), expected)
196
            # Make sure there are no .pyc files in the source directory.
197 198
            self.assertFalse([fn for fn in os.listdir(self.pkgdir)
                              if fn.endswith(ext)])
199
        locals()['test_pep3147_paths_' + name] = f
Barry Warsaw's avatar
Barry Warsaw committed
200 201 202 203

    def test_legacy_paths(self):
        # Ensure that with the proper switch, compileall leaves legacy
        # pyc/pyo files, and no __pycache__ directory.
204
        self.assertRunOK('-b', '-q', self.pkgdir)
Barry Warsaw's avatar
Barry Warsaw committed
205
        # Verify the __pycache__ directory contents.
206
        self.assertFalse(os.path.exists(self.pkgdir_cachedir))
207 208 209
        opt = 'c' if __debug__ else 'o'
        expected = sorted(['__init__.py', '__init__.py' + opt, 'bar.py',
                           'bar.py' + opt])
Barry Warsaw's avatar
Barry Warsaw committed
210 211
        self.assertEqual(sorted(os.listdir(self.pkgdir)), expected)

212 213 214
    def test_multiple_runs(self):
        # Bug 8527 reported that multiple calls produced empty
        # __pycache__/__pycache__ directories.
215
        self.assertRunOK('-q', self.pkgdir)
216
        # Verify the __pycache__ directory contents.
217 218
        self.assertTrue(os.path.exists(self.pkgdir_cachedir))
        cachecachedir = os.path.join(self.pkgdir_cachedir, '__pycache__')
219 220
        self.assertFalse(os.path.exists(cachecachedir))
        # Call compileall again.
221 222
        self.assertRunOK('-q', self.pkgdir)
        self.assertTrue(os.path.exists(self.pkgdir_cachedir))
223 224
        self.assertFalse(os.path.exists(cachecachedir))

225
    def test_force(self):
226 227
        self.assertRunOK('-q', self.pkgdir)
        pycpath = imp.cache_from_source(self.barfn)
228 229
        # set atime/mtime backward to avoid file timestamp resolution issues
        os.utime(pycpath, (time.time()-60,)*2)
230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250
        mtime = os.stat(pycpath).st_mtime
        # without force, no recompilation
        self.assertRunOK('-q', self.pkgdir)
        mtime2 = os.stat(pycpath).st_mtime
        self.assertEqual(mtime, mtime2)
        # now force it.
        self.assertRunOK('-q', '-f', self.pkgdir)
        mtime2 = os.stat(pycpath).st_mtime
        self.assertNotEqual(mtime, mtime2)

    def test_recursion_control(self):
        subpackage = os.path.join(self.pkgdir, 'spam')
        os.mkdir(subpackage)
        subinitfn = script_helper.make_script(subpackage, '__init__', '')
        hamfn = script_helper.make_script(subpackage, 'ham', '')
        self.assertRunOK('-q', '-l', self.pkgdir)
        self.assertNotCompiled(subinitfn)
        self.assertFalse(os.path.exists(os.path.join(subpackage, '__pycache__')))
        self.assertRunOK('-q', self.pkgdir)
        self.assertCompiled(subinitfn)
        self.assertCompiled(hamfn)
251 252

    def test_quiet(self):
253 254 255 256
        noisy = self.assertRunOK(self.pkgdir)
        quiet = self.assertRunOK('-q', self.pkgdir)
        self.assertNotEqual(b'', noisy)
        self.assertEqual(b'', quiet)
257 258

    def test_regexp(self):
259
        self.assertRunOK('-q', '-x', r'ba[^\\/]*$', self.pkgdir)
260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336
        self.assertNotCompiled(self.barfn)
        self.assertCompiled(self.initfn)

    def test_multiple_dirs(self):
        pkgdir2 = os.path.join(self.directory, 'foo2')
        os.mkdir(pkgdir2)
        init2fn = script_helper.make_script(pkgdir2, '__init__', '')
        bar2fn = script_helper.make_script(pkgdir2, 'bar2', '')
        self.assertRunOK('-q', self.pkgdir, pkgdir2)
        self.assertCompiled(self.initfn)
        self.assertCompiled(self.barfn)
        self.assertCompiled(init2fn)
        self.assertCompiled(bar2fn)

    def test_d_takes_exactly_one_dir(self):
        rc, out, err = self.assertRunNotOK('-d', 'foo')
        self.assertEqual(out, b'')
        self.assertRegex(err, b'-d')
        rc, out, err = self.assertRunNotOK('-d', 'foo', 'bar')
        self.assertEqual(out, b'')
        self.assertRegex(err, b'-d')

    def test_d_compile_error(self):
        script_helper.make_script(self.pkgdir, 'crunchyfrog', 'bad(syntax')
        rc, out, err = self.assertRunNotOK('-q', '-d', 'dinsdale', self.pkgdir)
        self.assertRegex(out, b'File "dinsdale')

    def test_d_runtime_error(self):
        bazfn = script_helper.make_script(self.pkgdir, 'baz', 'raise Exception')
        self.assertRunOK('-q', '-d', 'dinsdale', self.pkgdir)
        fn = script_helper.make_script(self.pkgdir, 'bing', 'import baz')
        pyc = imp.cache_from_source(bazfn)
        os.rename(pyc, os.path.join(self.pkgdir, 'baz.pyc'))
        os.remove(bazfn)
        rc, out, err = script_helper.assert_python_failure(fn)
        self.assertRegex(err, b'File "dinsdale')

    def test_include_bad_file(self):
        rc, out, err = self.assertRunNotOK(
            '-i', os.path.join(self.directory, 'nosuchfile'), self.pkgdir)
        self.assertRegex(out, b'rror.*nosuchfile')
        self.assertNotRegex(err, b'Traceback')
        self.assertFalse(os.path.exists(imp.cache_from_source(
                                            self.pkgdir_cachedir)))

    def test_include_file_with_arg(self):
        f1 = script_helper.make_script(self.pkgdir, 'f1', '')
        f2 = script_helper.make_script(self.pkgdir, 'f2', '')
        f3 = script_helper.make_script(self.pkgdir, 'f3', '')
        f4 = script_helper.make_script(self.pkgdir, 'f4', '')
        with open(os.path.join(self.directory, 'l1'), 'w') as l1:
            l1.write(os.path.join(self.pkgdir, 'f1.py')+os.linesep)
            l1.write(os.path.join(self.pkgdir, 'f2.py')+os.linesep)
        self.assertRunOK('-i', os.path.join(self.directory, 'l1'), f4)
        self.assertCompiled(f1)
        self.assertCompiled(f2)
        self.assertNotCompiled(f3)
        self.assertCompiled(f4)

    def test_include_file_no_arg(self):
        f1 = script_helper.make_script(self.pkgdir, 'f1', '')
        f2 = script_helper.make_script(self.pkgdir, 'f2', '')
        f3 = script_helper.make_script(self.pkgdir, 'f3', '')
        f4 = script_helper.make_script(self.pkgdir, 'f4', '')
        with open(os.path.join(self.directory, 'l1'), 'w') as l1:
            l1.write(os.path.join(self.pkgdir, 'f2.py')+os.linesep)
        self.assertRunOK('-i', os.path.join(self.directory, 'l1'))
        self.assertNotCompiled(f1)
        self.assertCompiled(f2)
        self.assertNotCompiled(f3)
        self.assertNotCompiled(f4)

    def test_include_on_stdin(self):
        f1 = script_helper.make_script(self.pkgdir, 'f1', '')
        f2 = script_helper.make_script(self.pkgdir, 'f2', '')
        f3 = script_helper.make_script(self.pkgdir, 'f3', '')
        f4 = script_helper.make_script(self.pkgdir, 'f4', '')
337
        p = script_helper.spawn_python(*(self._get_run_args(()) + ['-i', '-']))
338 339 340 341 342 343 344 345 346 347 348
        p.stdin.write((f3+os.linesep).encode('ascii'))
        script_helper.kill_python(p)
        self.assertNotCompiled(f1)
        self.assertNotCompiled(f2)
        self.assertCompiled(f3)
        self.assertNotCompiled(f4)

    def test_compiles_as_much_as_possible(self):
        bingfn = script_helper.make_script(self.pkgdir, 'bing', 'syntax(error')
        rc, out, err = self.assertRunNotOK('nosuchfile', self.initfn,
                                           bingfn, self.barfn)
349
        self.assertRegex(out, b'rror')
350 351 352 353
        self.assertNotCompiled(bingfn)
        self.assertCompiled(self.initfn)
        self.assertCompiled(self.barfn)

354 355
    def test_invalid_arg_produces_message(self):
        out = self.assertRunOK('badfilename')
356
        self.assertRegex(out, b"Can't list 'badfilename'")
357

Barry Warsaw's avatar
Barry Warsaw committed
358

359
def test_main():
Barry Warsaw's avatar
Barry Warsaw committed
360 361 362 363 364
    support.run_unittest(
        CommandLineTests,
        CompileallTests,
        EncodingTest,
        )
365 366 367 368


if __name__ == "__main__":
    test_main()