Clean up byte-compilation code in packaging (#11254 followup).

- Don't use keyword arguments for debug_override; I find it more
  readable to have a comment explaining that True makes pyc and False
  pyo than to write out the non-obvious (when you haven’t read the doc)
  argument name

- Move duplicate code from build_py and install_lib into cmd

- Remove obsolete verbose argument of util.byte_compile

- Remove obsolete passing of -O/-OO to the Python process spawned by
  util.byte_compile (I’ll remove the whole spawning later, after I write
  more tests to check the contents of pyc and pyo files; now that
  byte_compile does not depend on the value of __debug__ in the calling
  Python, we can call py_compile or compileall directly)

Doc/library/packaging.util.rst

@@ -90,7 +90,7 @@ This module contains various helpers for the other modules.
    Search the path for a given executable name.
 
 
-.. function:: execute(func, args[, msg=None, verbose=0, dry_run=0])
+.. function:: execute(func, args, msg=None, verbose=0, dry_run=0)
 
    Perform some action that affects the outside world (for instance, writing to
    the filesystem).  Such actions are special because they are disabled by the
@@ -117,7 +117,8 @@ This module contains various helpers for the other modules.
    :exc:`ValueError` if *val* is anything else.
 
 
-.. function:: byte_compile(py_files[, optimize=0, force=0, prefix=None, base_dir=None, verbose=1, dry_run=0, direct=None])
+.. function:: byte_compile(py_files, optimize=0, force=0, prefix=None, \
+                           base_dir=None, dry_run=0, direct=None)
 
    Byte-compile a collection of Python source files to either :file:`.pyc` or
    :file:`.pyo` files in a :file:`__pycache__` subdirectory (see :pep:`3147`),
@@ -131,6 +132,9 @@ This module contains various helpers for the other modules.
    * ``1`` - normal optimization (like ``python -O``)
    * ``2`` - extra optimization (like ``python -OO``)
 
+   This function is independent from the running Python's :option:`-O` or
+   :option:`-B` options; it is fully controlled by the parameters passed in.
+
    If *force* is true, all files are recompiled regardless of timestamps.
 
    The source filename encoded in each :term:`bytecode` file defaults to the filenames
@@ -149,6 +153,3 @@ This module contains various helpers for the other modules.
    figure out to use direct compilation or not (see the source for details).
    The *direct* flag is used by the script generated in indirect mode; unless
    you know what you're doing, leave it set to ``None``.
-
-   This function is independent from the running Python's :option:`-O` or
-   :option:`-B` options; it is fully controlled by the parameters passed in.

Lib/packaging/command/build_py.py

@@ -18,7 +18,7 @@ class build_py(Command, Mixin2to3):
 
     description = "build pure Python modules (copy to build directory)"
 
-    # The options for controlling byte compilations are two independent sets;
+    # The options for controlling byte compilation are two independent sets;
     # more info in install_lib or the reST docs
 
     user_options = [
@@ -113,7 +113,8 @@ class build_py(Command, Mixin2to3):
             self.run_2to3(self._updated_files, self._doctests_2to3,
                                             self.use_2to3_fixers)
 
-        self.byte_compile(self.get_outputs(include_bytecode=False))
+        self.byte_compile(self.get_outputs(include_bytecode=False),
+                          prefix=self.build_lib)
 
     # -- Top-level worker functions ------------------------------------
 
@@ -335,11 +336,9 @@ class build_py(Command, Mixin2to3):
             outputs.append(filename)
             if include_bytecode:
                 if self.compile:
-                    outputs.append(imp.cache_from_source(filename,
-                                                         debug_override=True))
-                if self.optimize > 0:
-                    outputs.append(imp.cache_from_source(filename,
-                                                         debug_override=False))
+                    outputs.append(imp.cache_from_source(filename, True))
+                if self.optimize:
+                    outputs.append(imp.cache_from_source(filename, False))
 
         outputs += [
             os.path.join(build_dir, filename)
@@ -391,19 +390,3 @@ class build_py(Command, Mixin2to3):
             for package_, module, module_file in modules:
                 assert package == package_
                 self.build_module(module, module_file, package)
-
-    def byte_compile(self, files):
-        from packaging.util import byte_compile  # FIXME use compileall
-        prefix = self.build_lib
-        if prefix[-1] != os.sep:
-            prefix = prefix + os.sep
-
-        # XXX this code is essentially the same as the 'byte_compile()
-        # method of the "install_lib" command, except for the determination
-        # of the 'prefix' string.  Hmmm.
-        if self.compile:
-            byte_compile(files, optimize=0,
-                         force=self.force, prefix=prefix, dry_run=self.dry_run)
-        if self.optimize > 0:
-            byte_compile(files, optimize=self.optimize,
-                         force=self.force, prefix=prefix, dry_run=self.dry_run)

Lib/packaging/command/cmd.py

@@ -10,7 +10,7 @@ from packaging.errors import PackagingOptionError
 
 class Command:
     """Abstract base class for defining command classes, the "worker bees"
-    of the Packaging.  A useful analogy for command classes is to think of
+    of Packaging.  A useful analogy for command classes is to think of
     them as subroutines with local variables called "options".  The options
     are "declared" in 'initialize_options()' and "defined" (given their
     final values, aka "finalized") in 'finalize_options()', both of which
@@ -386,7 +386,6 @@ class Command:
         if self.dry_run:
             return  # see if we want to display something
 
-
         return util.copy_tree(infile, outfile, preserve_mode, preserve_times,
             preserve_symlinks, not self.force, dry_run=self.dry_run)
 
@@ -439,3 +438,20 @@ class Command:
         # Otherwise, print the "skip" message
         else:
             logger.debug(skip_msg)
+
+    def byte_compile(self, files, prefix=None):
+        """Byte-compile files to pyc and/or pyo files.
+
+        This method requires that the calling class define compile and
+        optimize options, like build_py and install_lib.  It also
+        automatically respects the force and dry-run options.
+
+        prefix, if given, is a string that will be stripped off the
+        filenames encoded in bytecode files.
+        """
+        if self.compile:
+            util.byte_compile(files, optimize=False, prefix=prefix,
+                              force=self.force, dry_run=self.dry_run)
+        if self.optimize:
+            util.byte_compile(files, optimize=self.optimize, prefix=prefix,
+                              force=self.force, dry_run=self.dry_run)

Lib/packaging/command/install_lib.py

@@ -2,7 +2,6 @@
 
 import os
 import imp
-import logging
 
 from packaging import logger
 from packaging.command.cmd import Command
@@ -21,7 +20,7 @@ class install_lib(Command):
 
     description = "install all modules (extensions and pure Python)"
 
-    # The options for controlling byte compilations are two independent sets:
+    # The options for controlling byte compilation are two independent sets:
     # 'compile' is strictly boolean, and only decides whether to
     # generate .pyc files.  'optimize' is three-way (0, 1, or 2), and
     # decides both whether to generate .pyo files and what level of
@@ -84,9 +83,14 @@ class install_lib(Command):
         # having a build directory!)
         outfiles = self.install()
 
-        # (Optionally) compile .py to .pyc
+        # (Optionally) compile .py to .pyc and/or .pyo
         if outfiles is not None and self.distribution.has_pure_modules():
-            self.byte_compile(outfiles)
+            # XXX comment from distutils: "This [prefix stripping] is far from
+            # complete, but it should at least generate usable bytecode in RPM
+            # distributions." -> need to find exact requirements for
+            # byte-compiled files and fix it
+            install_root = self.get_finalized_command('install_dist').root
+            self.byte_compile(outfiles, prefix=install_root)
 
     # -- Top-level worker functions ------------------------------------
     # (called from 'run()')
@@ -108,28 +112,6 @@ class install_lib(Command):
             return
         return outfiles
 
-    def byte_compile(self, files):
-        from packaging.util import byte_compile  # FIXME use compileall
-
-        # Get the "--root" directory supplied to the "install_dist" command,
-        # and use it as a prefix to strip off the purported filename
-        # encoded in bytecode files.  This is far from complete, but it
-        # should at least generate usable bytecode in RPM distributions.
-        install_root = self.get_finalized_command('install_dist').root
-
-        # Temporary kludge until we remove the verbose arguments and use
-        # logging everywhere
-        verbose = logger.getEffectiveLevel() >= logging.DEBUG
-
-        if self.compile:
-            byte_compile(files, optimize=0,
-                         force=self.force, prefix=install_root,
-                         verbose=verbose, dry_run=self.dry_run)
-        if self.optimize > 0:
-            byte_compile(files, optimize=self.optimize,
-                         force=self.force, prefix=install_root,
-                         verbose=verbose, dry_run=self.dry_run)
-
     # -- Utility methods -----------------------------------------------
 
     def _mutate_outputs(self, has_any, build_cmd, cmd_option, output_dir):
@@ -157,11 +139,9 @@ class install_lib(Command):
             if ext != PYTHON_SOURCE_EXTENSION:
                 continue
             if self.compile:
-                bytecode_files.append(imp.cache_from_source(
-                    py_file, debug_override=True))
-            if self.optimize > 0:
-                bytecode_files.append(imp.cache_from_source(
-                    py_file, debug_override=False))
+                bytecode_files.append(imp.cache_from_source(py_file, True))
+            if self.optimize:
+                bytecode_files.append(imp.cache_from_source(py_file, False))
 
         return bytecode_files
 

Lib/packaging/tests/test_command_install_lib.py

@@ -44,8 +44,8 @@ class InstallLibTestCase(support.TempdirManager,
         f = os.path.join(project_dir, 'foo.py')
         self.write_file(f, '# python file')
         cmd.byte_compile([f])
-        pyc_file = imp.cache_from_source('foo.py', debug_override=True)
-        pyo_file = imp.cache_from_source('foo.py', debug_override=False)
+        pyc_file = imp.cache_from_source('foo.py', True)
+        pyo_file = imp.cache_from_source('foo.py', False)
         self.assertTrue(os.path.exists(pyc_file))
         self.assertTrue(os.path.exists(pyo_file))
 

Lib/packaging/util.py

@@ -296,7 +296,7 @@ def strtobool(val):
 
 
 def byte_compile(py_files, optimize=0, force=False, prefix=None,
-                 base_dir=None, verbose=0, dry_run=False, direct=None):
+                 base_dir=None, dry_run=False, direct=None):
     """Byte-compile a collection of Python source files to either .pyc
     or .pyo files in a __pycache__ subdirectory.
 
@@ -305,6 +305,9 @@ def byte_compile(py_files, optimize=0, force=False, prefix=None,
       0 - don't optimize (generate .pyc)
       1 - normal optimization (like "python -O")
       2 - extra optimization (like "python -OO")
+    This function is independent from the running Python's -O or -B options;
+    it is fully controlled by the parameters passed in.
+
     If 'force' is true, all files are recompiled regardless of
     timestamps.
 
@@ -325,10 +328,9 @@ def byte_compile(py_files, optimize=0, force=False, prefix=None,
     the source for details).  The 'direct' flag is used by the script
     generated in indirect mode; unless you know what you're doing, leave
     it set to None.
-
-    This function is independent from the running Python's -O or -B options;
-    it is fully controlled by the parameters passed in.
     """
+    # FIXME use compileall + remove direct/indirect shenanigans
+
     # First, if the caller didn't force us into direct or indirect mode,
     # figure out which mode we should be in.  We take a conservative
     # approach: choose direct mode *only* if the current interpreter is
@@ -381,15 +383,11 @@ files = [
                 script.write("""
 byte_compile(files, optimize=%r, force=%r,
              prefix=%r, base_dir=%r,
-             verbose=%r, dry_run=False,
+             dry_run=False,
              direct=True)
-""" % (optimize, force, prefix, base_dir, verbose))
+""" % (optimize, force, prefix, base_dir))
 
         cmd = [sys.executable, script_name]
-        if optimize == 1:
-            cmd.insert(1, "-O")
-        elif optimize == 2:
-            cmd.insert(1, "-OO")
 
         env = os.environ.copy()
         env['PYTHONPATH'] = os.path.pathsep.join(sys.path)
@@ -415,8 +413,10 @@ byte_compile(files, optimize=%r, force=%r,
             # Terminology from the py_compile module:
             #   cfile - byte-compiled file
             #   dfile - purported source filename (same as 'file' by default)
-            debug_override = not optimize
-            cfile = imp.cache_from_source(file, debug_override)
+            # The second argument to cache_from_source forces the extension to
+            # be .pyc (if true) or .pyo (if false); without it, the extension
+            # would depend on the calling Python's -O option
+            cfile = imp.cache_from_source(file, not optimize)
             dfile = file
 
             if prefix:
@@ -1334,7 +1334,7 @@ def copy_tree(src, dst, preserve_mode=True, preserve_times=True,
               preserve_symlinks=False, update=False, verbose=True,
               dry_run=False):
     # FIXME use of this function is why we get spurious logging message on
-    # stdout when tests run; kill and replace by shuil!
+    # stdout when tests run; kill and replace by shutil!
     from distutils.file_util import copy_file
 
     if not dry_run and not os.path.isdir(src):
@@ -1448,8 +1448,7 @@ def encode_multipart(fields, files, boundary=None):
 
     Returns (content_type: bytes, body: bytes) ready for http.client.HTTP.
     """
-    # Taken from
-    # http://code.activestate.com/recipes/146306-http-client-to-post-using-multipartform-data/
+    # Taken from http://code.activestate.com/recipes/146306
 
     if boundary is None:
         boundary = b'--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'