Issue #19700: set __spec__ appropriately in runpy

Note that __spec__.name is not currently guaranteed to be in
sys.modules when the code is running, only __name__ is.

The "running module is in sys.modules" invariant will be
expanded to also cover __spec__.name in a subsequent patch.

Doc/library/runpy.rst

@@ -44,28 +44,22 @@ The :mod:`runpy` module provides two functions:
    below are defined in the supplied dictionary, those definitions are
    overridden by :func:`run_module`.
 
-   The special global variables ``__name__``, ``__file__``, ``__cached__``,
-   ``__loader__``
-   and ``__package__`` are set in the globals dictionary before the module
-   code is executed (Note that this is a minimal set of variables - other
-   variables may be set implicitly as an interpreter implementation detail).
+   The special global variables ``__name__``, ``__spec__``, ``__file__``,
+   ``__cached__``, ``__loader__`` and ``__package__`` are set in the globals
+   dictionary before the module code is executed (Note that this is a
+   minimal set of variables - other variables may be set implicitly as an
+   interpreter implementation detail).
 
    ``__name__`` is set to *run_name* if this optional argument is not
    :const:`None`, to ``mod_name + '.__main__'`` if the named module is a
    package and to the *mod_name* argument otherwise.
 
-   ``__file__`` is set to the name provided by the module loader. If the
-   loader does not make filename information available, this variable is set
-   to :const:`None`.
+   ``__spec__`` will be set appropriately for the *actually* imported
+   module (that is, ``__spec__.name`` will always be *mod_name* or
+   ``mod_name + '.__main__``, never *run_name*).
 
-   ``__cached__`` will be set to ``None``.
-
-   ``__loader__`` is set to the :pep:`302` module loader used to retrieve the
-   code for the module (This loader may be a wrapper around the standard
-   import mechanism).
-
-   ``__package__`` is set to *mod_name* if the named module is a package and
-   to ``mod_name.rpartition('.')[0]`` otherwise.
+   ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` are
+   :ref:`set as normal <import-mod-attrs>` based on the module spec.
 
    If the argument *alter_sys* is supplied and evaluates to :const:`True`,
    then ``sys.argv[0]`` is updated with the value of ``__file__`` and
@@ -83,8 +77,13 @@ The :mod:`runpy` module provides two functions:
       Added ability to execute packages by looking for a ``__main__`` submodule.
 
    .. versionchanged:: 3.2
-      Added ``__cached__`` global variable (see :PEP:`3147`).
+      Added ``__cached__`` global variable (see :pep:`3147`).
 
+   .. versionchanged:: 3.4
+      Updated to take advantage of the module spec feature added by
+      :pep:`451`. This allows ``__cached__`` to be set correctly for modules
+      run this way, as well as ensuring the real module name is always
+      accessible as ``__spec__.name``.
 
 .. function:: run_path(file_path, init_globals=None, run_name=None)
 
@@ -108,23 +107,28 @@ The :mod:`runpy` module provides two functions:
    below are defined in the supplied dictionary, those definitions are
    overridden by :func:`run_path`.
 
-   The special global variables ``__name__``, ``__file__``, ``__loader__``
-   and ``__package__`` are set in the globals dictionary before the module
-   code is executed (Note that this is a minimal set of variables - other
-   variables may be set implicitly as an interpreter implementation detail).
+   The special global variables ``__name__``, ``__spec__``, ``__file__``,
+   ``__cached__``, ``__loader__`` and ``__package__`` are set in the globals
+   dictionary before the module code is executed (Note that this is a
+   minimal set of variables - other variables may be set implicitly as an
+   interpreter implementation detail).
 
    ``__name__`` is set to *run_name* if this optional argument is not
    :const:`None` and to ``'<run_path>'`` otherwise.
 
-   ``__file__`` is set to the name provided by the module loader. If the
-   loader does not make filename information available, this variable is set
-   to :const:`None`. For a simple script, this will be set to ``file_path``.
+   If the supplied path directly references a script file (whether as source
+   or as precompiled byte code), then ``__file__`` will be set to the
+   supplied path, and ``__spec__``, ``__cached__``, ``__loader__`` and
+   ``__package__`` will all be set to :const:`None`.
 
-   ``__loader__`` is set to the :pep:`302` module loader used to retrieve the
-   code for the module (This loader may be a wrapper around the standard
-   import mechanism). For a simple script, this will be set to :const:`None`.
+   ``__spec__`` will be set to :const:`None` if the supplied path is a
+   direct path to a script (as source or as precompiled bytecode).
 
-   ``__package__`` is set to ``__name__.rpartition('.')[0]``.
+   If the supplied path is a reference to a valid sys.path entry, then
+   ``__spec__`` will be set appropriately for the imported ``__main__``
+   module (that is, ``__spec__.name`` will always be ``__main__``).
+   ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` will be
+   :ref:`set as normal <import-mod-attrs>` based on the module spec.
 
    A number of alterations are also made to the :mod:`sys` module. Firstly,
    ``sys.path`` may be altered as described above. ``sys.argv[0]`` is updated
@@ -141,6 +145,12 @@ The :mod:`runpy` module provides two functions:
 
    .. versionadded:: 3.2
 
+   .. versionchanged:: 3.4
+      Updated to take advantage of the module spec feature added by
+      :pep:`451`. This allows ``__cached__`` to be set correctly in the
+      case where ``__main__`` is imported from a valid sys.path entry rather
+      than being executed directly.
+
 .. seealso::
 
    :pep:`338` - Executing modules as scripts
@@ -149,6 +159,9 @@ The :mod:`runpy` module provides two functions:
    :pep:`366` - Main module explicit relative imports
       PEP written and implemented by Nick Coghlan.
 
+   :pep:`451` - A ModuleSpec Type for the Import System
+      PEP written and implemented by Eric Snow
+
    :ref:`using-on-general` - CPython command line details
 
    The :func:`importlib.import_module` function

Lib/runpy.py

@@ -14,7 +14,9 @@ import os
 import sys
 import importlib.machinery # importlib first so we can test #15386 via -m
 import types
-from pkgutil import read_code, get_loader, get_importer
+from importlib import find_spec
+from importlib.util import spec_from_loader
+from pkgutil import read_code, get_importer
 
 __all__ = [
     "run_module", "run_path",
@@ -58,51 +60,76 @@ class _ModifiedArgv0(object):
         self.value = self._sentinel
         sys.argv[0] = self._saved_value
 
+# TODO: Replace these helpers with importlib._bootstrap._SpecMethods
 def _run_code(code, run_globals, init_globals=None,
-              mod_name=None, mod_fname=None,
-              mod_loader=None, pkg_name=None):
+              mod_name=None, mod_spec=None,
+              pkg_name=None, script_name=None):
     """Helper to run code in nominated namespace"""
     if init_globals is not None:
         run_globals.update(init_globals)
+    if mod_spec is None:
+        loader = None
+        fname = script_name
+        cached = None
+    else:
+        loader = mod_spec.loader
+        fname = mod_spec.origin
+        cached = mod_spec.cached
+        if pkg_name is None:
+            pkg_name = mod_spec.parent
     run_globals.update(__name__ = mod_name,
-                       __file__ = mod_fname,
-                       __cached__ = None,
+                       __file__ = fname,
+                       __cached__ = cached,
                        __doc__ = None,
-                       __loader__ = mod_loader,
-                       __package__ = pkg_name)
+                       __loader__ = loader,
+                       __package__ = pkg_name,
+                       __spec__ = mod_spec)
     exec(code, run_globals)
     return run_globals
 
 def _run_module_code(code, init_globals=None,
-                    mod_name=None, mod_fname=None,
-                    mod_loader=None, pkg_name=None):
+                    mod_name=None, mod_spec=None,
+                    pkg_name=None, script_name=None):
     """Helper to run code in new namespace with sys modified"""
-    with _TempModule(mod_name) as temp_module, _ModifiedArgv0(mod_fname):
+    fname = script_name if mod_spec is None else mod_spec.origin
+    with _TempModule(mod_name) as temp_module, _ModifiedArgv0(fname):
         mod_globals = temp_module.module.__dict__
         _run_code(code, mod_globals, init_globals,
-                  mod_name, mod_fname, mod_loader, pkg_name)
+                  mod_name, mod_spec, pkg_name, script_name)
     # Copy the globals of the temporary module, as they
     # may be cleared when the temporary module goes away
     return mod_globals.copy()
 
 
-# This helper is needed due to a missing component in the PEP 302
-# loader protocol (specifically, "get_filename" is non-standard)
-# Since we can't introduce new features in maintenance releases,
-# support was added to zipimporter under the name '_get_filename'
-def _get_filename(loader, mod_name):
-    for attr in ("get_filename", "_get_filename"):
-        meth = getattr(loader, attr, None)
-        if meth is not None:
-            return os.path.abspath(meth(mod_name))
-    return None
+def _fixed_find_spec(mod_name):
+    # find_spec has the same annoying behaviour as find_loader did (it
+    # fails to work properly for dotted names), so this is a fixed version
+    # ala pkgutil.get_loader
+    if mod_name.startswith('.'):
+        msg = "Relative module name {!r} not supported".format(mod_name)
+        raise ImportError(msg)
+    path = None
+    pkg_name = mod_name.rpartition(".")[0]
+    if pkg_name:
+        pkg = importlib.import_module(pkg_name)
+        path = getattr(pkg, "__path__", None)
+        if path is None:
+            return None
+    try:
+        return importlib.find_spec(mod_name, path)
+    except (ImportError, AttributeError, TypeError, ValueError) as ex:
+        # This hack fixes an impedance mismatch between pkgutil and
+        # importlib, where the latter raises other errors for cases where
+        # pkgutil previously raised ImportError
+        msg = "Error while finding spec for {!r} ({}: {})"
+        raise ImportError(msg.format(mod_name, type(ex), ex)) from ex
 
 # Helper to get the loader, code and filename for a module
 def _get_module_details(mod_name):
-    loader = get_loader(mod_name)
-    if loader is None:
+    spec = _fixed_find_spec(mod_name)
+    if spec is None:
         raise ImportError("No module named %s" % mod_name)
-    if loader.is_package(mod_name):
+    if spec.submodule_search_locations is not None:
         if mod_name == "__main__" or mod_name.endswith(".__main__"):
             raise ImportError("Cannot use package as __main__ module")
         try:
@@ -111,11 +138,14 @@ def _get_module_details(mod_name):
         except ImportError as e:
             raise ImportError(("%s; %r is a package and cannot " +
                                "be directly executed") %(e, mod_name))
+    loader = spec.loader
+    if loader is None:
+        raise ImportError("%r is a namespace package and cannot be executed"
+                                                                 % mod_name)
     code = loader.get_code(mod_name)
     if code is None:
         raise ImportError("No code object available for %s" % mod_name)
-    filename = _get_filename(loader, mod_name)
-    return mod_name, loader, code, filename
+    return mod_name, spec, code
 
 # XXX ncoghlan: Should this be documented and made public?
 # (Current thoughts: don't repeat the mistake that lead to its
@@ -137,9 +167,9 @@ def _run_module_as_main(mod_name, alter_argv=True):
     """
     try:
         if alter_argv or mod_name != "__main__": # i.e. -m switch
-            mod_name, loader, code, fname = _get_module_details(mod_name)
+            mod_name, mod_spec, code = _get_module_details(mod_name)
         else:          # i.e. directory or zipfile execution
-            mod_name, loader, code, fname = _get_main_module_details()
+            mod_name, mod_spec, code = _get_main_module_details()
     except ImportError as exc:
         # Try to provide a good error message
         # for directories, zip files and the -m switch
@@ -152,12 +182,11 @@ def _run_module_as_main(mod_name, alter_argv=True):
             info = "can't find '__main__' module in %r" % sys.argv[0]
         msg = "%s: %s" % (sys.executable, info)
         sys.exit(msg)
-    pkg_name = mod_name.rpartition('.')[0]
     main_globals = sys.modules["__main__"].__dict__
     if alter_argv:
-        sys.argv[0] = fname
+        sys.argv[0] = mod_spec.origin
     return _run_code(code, main_globals, None,
-                     "__main__", fname, loader, pkg_name)
+                     "__main__", mod_spec)
 
 def run_module(mod_name, init_globals=None,
                run_name=None, alter_sys=False):
@@ -165,17 +194,14 @@ def run_module(mod_name, init_globals=None,
 
        Returns the resulting top level namespace dictionary
     """
-    mod_name, loader, code, fname = _get_module_details(mod_name)
+    mod_name, mod_spec, code = _get_module_details(mod_name)
     if run_name is None:
         run_name = mod_name
-    pkg_name = mod_name.rpartition('.')[0]
     if alter_sys:
-        return _run_module_code(code, init_globals, run_name,
-                                fname, loader, pkg_name)
+        return _run_module_code(code, init_globals, run_name, mod_spec)
     else:
         # Leave the sys module alone
-        return _run_code(code, {}, init_globals, run_name,
-                         fname, loader, pkg_name)
+        return _run_code(code, {}, init_globals, run_name, mod_spec)
 
 def _get_main_module_details():
     # Helper that gives a nicer error message when attempting to
@@ -204,10 +230,7 @@ def _get_code_from_file(run_name, fname):
         # That didn't work, so try it as normal source code
         with open(fname, "rb") as f:
             code = compile(f.read(), fname, 'exec')
-            loader = importlib.machinery.SourceFileLoader(run_name, fname)
-    else:
-        loader = importlib.machinery.SourcelessFileLoader(run_name, fname)
-    return code, loader
+    return code, fname
 
 def run_path(path_name, init_globals=None, run_name=None):
     """Execute code located at the specified filesystem location
@@ -231,9 +254,9 @@ def run_path(path_name, init_globals=None, run_name=None):
     if isinstance(importer, type(None)) or is_NullImporter:
         # Not a valid sys.path entry, so run the code directly
         # execfile() doesn't help as we want to allow compiled files
-        code, mod_loader = _get_code_from_file(run_name, path_name)
-        return _run_module_code(code, init_globals, run_name, path_name,
-                                mod_loader, pkg_name)
+        code, fname = _get_code_from_file(run_name, path_name)
+        return _run_module_code(code, init_globals, run_name,
+                                pkg_name=pkg_name, script_name=fname)
     else:
         # Importer is defined for path, so add it to
         # the start of sys.path
@@ -245,12 +268,12 @@ def run_path(path_name, init_globals=None, run_name=None):
             # have no choice and we have to remove it even while we read the
             # code. If we don't do this, a __loader__ attribute in the
             # existing __main__ module may prevent location of the new module.
-            mod_name, loader, code, fname = _get_main_module_details()
+            mod_name, mod_spec, code = _get_main_module_details()
             with _TempModule(run_name) as temp_module, \
                  _ModifiedArgv0(path_name):
                 mod_globals = temp_module.module.__dict__
                 return _run_code(code, mod_globals, init_globals,
-                                    run_name, fname, loader, pkg_name).copy()
+                                    run_name, mod_spec, pkg_name).copy()
         finally:
             try:
                 sys.path.remove(path_name)

Lib/test/script_helper.py

@@ -101,8 +101,10 @@ def kill_python(p):
     subprocess._cleanup()
     return data
 
-def make_script(script_dir, script_basename, source):
-    script_filename = script_basename+os.extsep+'py'
+def make_script(script_dir, script_basename, source, omit_suffix=False):
+    script_filename = script_basename
+    if not omit_suffix:
+        script_filename += os.extsep + 'py'
     script_name = os.path.join(script_dir, script_filename)
     # The script should be encoded to UTF-8, the default string encoding
     script_file = open(script_name, 'w', encoding='utf-8')

Lib/test/test_cmd_line_script.py

@@ -41,11 +41,28 @@ from importlib.machinery import BuiltinImporter
 _loader = __loader__ if __loader__ is BuiltinImporter else type(__loader__)
 print('__loader__==%a' % _loader)
 print('__file__==%a' % __file__)
-assertEqual(__cached__, None)
+print('__cached__==%a' % __cached__)
 print('__package__==%r' % __package__)
+# Check PEP 451 details
+import os.path
+if __package__ is not None:
+    print('__main__ was located through the import system')
+    assertIdentical(__spec__.loader, __loader__)
+    expected_spec_name = os.path.splitext(os.path.basename(__file__))[0]
+    if __package__:
+        expected_spec_name = __package__ + "." + expected_spec_name
+    assertEqual(__spec__.name, expected_spec_name)
+    assertEqual(__spec__.parent, __package__)
+    assertIdentical(__spec__.submodule_search_locations, None)
+    assertEqual(__spec__.origin, __file__)
+    if __spec__.cached is not None:
+        assertEqual(__spec__.cached, __cached__)
 # Check the sys module
 import sys
 assertIdentical(globals(), sys.modules[__name__].__dict__)
+if __spec__ is not None:
+    # XXX: We're not currently making __main__ available under its real name
+    pass # assertIdentical(globals(), sys.modules[__spec__.name].__dict__)
 from test import test_cmd_line_script
 example_args_list = test_cmd_line_script.example_args
 assertEqual(sys.argv[1:], example_args_list)