Issue #19700: set __spec__ appropriately in runpy

Note that __spec__.name is not currently guaranteed to be in
sys.modules when the code is running, only __name__ is.

The "running module is in sys.modules" invariant will be
expanded to also cover __spec__.name in a subsequent patch.

Doc/library/runpy.rst

@@ -44,28 +44,22 @@ The :mod:`runpy` module provides two functions:
    below are defined in the supplied dictionary, those definitions are
    overridden by :func:`run_module`.
 
-   The special global variables ``__name__``, ``__file__``, ``__cached__``,
-   ``__loader__``
-   and ``__package__`` are set in the globals dictionary before the module
-   code is executed (Note that this is a minimal set of variables - other
-   variables may be set implicitly as an interpreter implementation detail).
+   The special global variables ``__name__``, ``__spec__``, ``__file__``,
+   ``__cached__``, ``__loader__`` and ``__package__`` are set in the globals
+   dictionary before the module code is executed (Note that this is a
+   minimal set of variables - other variables may be set implicitly as an
+   interpreter implementation detail).
 
    ``__name__`` is set to *run_name* if this optional argument is not
    :const:`None`, to ``mod_name + '.__main__'`` if the named module is a
    package and to the *mod_name* argument otherwise.
 
-   ``__file__`` is set to the name provided by the module loader. If the
-   loader does not make filename information available, this variable is set
-   to :const:`None`.
+   ``__spec__`` will be set appropriately for the *actually* imported
+   module (that is, ``__spec__.name`` will always be *mod_name* or
+   ``mod_name + '.__main__``, never *run_name*).
 
-   ``__cached__`` will be set to ``None``.
-
-   ``__loader__`` is set to the :pep:`302` module loader used to retrieve the
-   code for the module (This loader may be a wrapper around the standard
-   import mechanism).
-
-   ``__package__`` is set to *mod_name* if the named module is a package and
-   to ``mod_name.rpartition('.')[0]`` otherwise.
+   ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` are
+   :ref:`set as normal <import-mod-attrs>` based on the module spec.
 
    If the argument *alter_sys* is supplied and evaluates to :const:`True`,
    then ``sys.argv[0]`` is updated with the value of ``__file__`` and
@@ -83,8 +77,13 @@ The :mod:`runpy` module provides two functions:
       Added ability to execute packages by looking for a ``__main__`` submodule.
 
    .. versionchanged:: 3.2
-      Added ``__cached__`` global variable (see :PEP:`3147`).
+      Added ``__cached__`` global variable (see :pep:`3147`).
 
+   .. versionchanged:: 3.4
+      Updated to take advantage of the module spec feature added by
+      :pep:`451`. This allows ``__cached__`` to be set correctly for modules
+      run this way, as well as ensuring the real module name is always
+      accessible as ``__spec__.name``.
 
 .. function:: run_path(file_path, init_globals=None, run_name=None)
 
@@ -108,23 +107,28 @@ The :mod:`runpy` module provides two functions:
    below are defined in the supplied dictionary, those definitions are
    overridden by :func:`run_path`.
 
-   The special global variables ``__name__``, ``__file__``, ``__loader__``
-   and ``__package__`` are set in the globals dictionary before the module
-   code is executed (Note that this is a minimal set of variables - other
-   variables may be set implicitly as an interpreter implementation detail).
+   The special global variables ``__name__``, ``__spec__``, ``__file__``,
+   ``__cached__``, ``__loader__`` and ``__package__`` are set in the globals
+   dictionary before the module code is executed (Note that this is a
+   minimal set of variables - other variables may be set implicitly as an
+   interpreter implementation detail).
 
    ``__name__`` is set to *run_name* if this optional argument is not
    :const:`None` and to ``'<run_path>'`` otherwise.
 
-   ``__file__`` is set to the name provided by the module loader. If the
-   loader does not make filename information available, this variable is set
-   to :const:`None`. For a simple script, this will be set to ``file_path``.
+   If the supplied path directly references a script file (whether as source
+   or as precompiled byte code), then ``__file__`` will be set to the
+   supplied path, and ``__spec__``, ``__cached__``, ``__loader__`` and
+   ``__package__`` will all be set to :const:`None`.
 
-   ``__loader__`` is set to the :pep:`302` module loader used to retrieve the
-   code for the module (This loader may be a wrapper around the standard
-   import mechanism). For a simple script, this will be set to :const:`None`.
+   ``__spec__`` will be set to :const:`None` if the supplied path is a
+   direct path to a script (as source or as precompiled bytecode).
 
-   ``__package__`` is set to ``__name__.rpartition('.')[0]``.
+   If the supplied path is a reference to a valid sys.path entry, then
+   ``__spec__`` will be set appropriately for the imported ``__main__``
+   module (that is, ``__spec__.name`` will always be ``__main__``).
+   ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` will be
+   :ref:`set as normal <import-mod-attrs>` based on the module spec.
 
    A number of alterations are also made to the :mod:`sys` module. Firstly,
    ``sys.path`` may be altered as described above. ``sys.argv[0]`` is updated
@@ -141,6 +145,12 @@ The :mod:`runpy` module provides two functions:
 
    .. versionadded:: 3.2
 
+   .. versionchanged:: 3.4
+      Updated to take advantage of the module spec feature added by
+      :pep:`451`. This allows ``__cached__`` to be set correctly in the
+      case where ``__main__`` is imported from a valid sys.path entry rather
+      than being executed directly.
+
 .. seealso::
 
    :pep:`338` - Executing modules as scripts
@@ -149,6 +159,9 @@ The :mod:`runpy` module provides two functions:
    :pep:`366` - Main module explicit relative imports
       PEP written and implemented by Nick Coghlan.
 
+   :pep:`451` - A ModuleSpec Type for the Import System
+      PEP written and implemented by Eric Snow
+
    :ref:`using-on-general` - CPython command line details
 
    The :func:`importlib.import_module` function

Lib/runpy.py

@@ -14,7 +14,9 @@ import os
 import sys
 import importlib.machinery # importlib first so we can test #15386 via -m
 import types
-from pkgutil import read_code, get_loader, get_importer
+from importlib import find_spec
+from importlib.util import spec_from_loader
+from pkgutil import read_code, get_importer
 
 __all__ = [
     "run_module", "run_path",
@@ -58,51 +60,76 @@ class _ModifiedArgv0(object):
         self.value = self._sentinel
         sys.argv[0] = self._saved_value
 
+# TODO: Replace these helpers with importlib._bootstrap._SpecMethods
 def _run_code(code, run_globals, init_globals=None,
-              mod_name=None, mod_fname=None,
-              mod_loader=None, pkg_name=None):
+              mod_name=None, mod_spec=None,
+              pkg_name=None, script_name=None):
     """Helper to run code in nominated namespace"""
     if init_globals is not None:
         run_globals.update(init_globals)
+    if mod_spec is None:
+        loader = None
+        fname = script_name
+        cached = None
+    else:
+        loader = mod_spec.loader
+        fname = mod_spec.origin
+        cached = mod_spec.cached
+        if pkg_name is None:
+            pkg_name = mod_spec.parent
     run_globals.update(__name__ = mod_name,
-                       __file__ = mod_fname,
-                       __cached__ = None,
+                       __file__ = fname,
+                       __cached__ = cached,
                        __doc__ = None,
-                       __loader__ = mod_loader,
-                       __package__ = pkg_name)
+                       __loader__ = loader,
+                       __package__ = pkg_name,
+                       __spec__ = mod_spec)
     exec(code, run_globals)
     return run_globals
 
 def _run_module_code(code, init_globals=None,
-                    mod_name=None, mod_fname=None,
-                    mod_loader=None, pkg_name=None):
+                    mod_name=None, mod_spec=None,
+                    pkg_name=None, script_name=None):
     """Helper to run code in new namespace with sys modified"""
-    with _TempModule(mod_name) as temp_module, _ModifiedArgv0(mod_fname):
+    fname = script_name if mod_spec is None else mod_spec.origin
+    with _TempModule(mod_name) as temp_module, _ModifiedArgv0(fname):
         mod_globals = temp_module.module.__dict__
         _run_code(code, mod_globals, init_globals,
-                  mod_name, mod_fname, mod_loader, pkg_name)
+                  mod_name, mod_spec, pkg_name, script_name)
     # Copy the globals of the temporary module, as they
     # may be cleared when the temporary module goes away
     return mod_globals.copy()
 
 
-# This helper is needed due to a missing component in the PEP 302
-# loader protocol (specifically, "get_filename" is non-standard)
-# Since we can't introduce new features in maintenance releases,
-# support was added to zipimporter under the name '_get_filename'
-def _get_filename(loader, mod_name):
-    for attr in ("get_filename", "_get_filename"):
-        meth = getattr(loader, attr, None)
-        if meth is not None:
-            return os.path.abspath(meth(mod_name))
-    return None
+def _fixed_find_spec(mod_name):
+    # find_spec has the same annoying behaviour as find_loader did (it
+    # fails to work properly for dotted names), so this is a fixed version
+    # ala pkgutil.get_loader
+    if mod_name.startswith('.'):
+        msg = "Relative module name {!r} not supported".format(mod_name)
+        raise ImportError(msg)
+    path = None
+    pkg_name = mod_name.rpartition(".")[0]
+    if pkg_name:
+        pkg = importlib.import_module(pkg_name)
+        path = getattr(pkg, "__path__", None)
+        if path is None:
+            return None
+    try:
+        return importlib.find_spec(mod_name, path)
+    except (ImportError, AttributeError, TypeError, ValueError) as ex:
+        # This hack fixes an impedance mismatch between pkgutil and
+        # importlib, where the latter raises other errors for cases where
+        # pkgutil previously raised ImportError
+        msg = "Error while finding spec for {!r} ({}: {})"
+        raise ImportError(msg.format(mod_name, type(ex), ex)) from ex
 
 # Helper to get the loader, code and filename for a module
 def _get_module_details(mod_name):
-    loader = get_loader(mod_name)
-    if loader is None:
+    spec = _fixed_find_spec(mod_name)
+    if spec is None:
         raise ImportError("No module named %s" % mod_name)
-    if loader.is_package(mod_name):
+    if spec.submodule_search_locations is not None:
         if mod_name == "__main__" or mod_name.endswith(".__main__"):
             raise ImportError("Cannot use package as __main__ module")
         try:
@@ -111,11 +138,14 @@ def _get_module_details(mod_name):
         except ImportError as e:
             raise ImportError(("%s; %r is a package and cannot " +
                                "be directly executed") %(e, mod_name))
+    loader = spec.loader
+    if loader is None:
+        raise ImportError("%r is a namespace package and cannot be executed"
+                                                                 % mod_name)
     code = loader.get_code(mod_name)
     if code is None:
         raise ImportError("No code object available for %s" % mod_name)
-    filename = _get_filename(loader, mod_name)
-    return mod_name, loader, code, filename
+    return mod_name, spec, code
 
 # XXX ncoghlan: Should this be documented and made public?
 # (Current thoughts: don't repeat the mistake that lead to its
@@ -137,9 +167,9 @@ def _run_module_as_main(mod_name, alter_argv=True):
     """
     try:
         if alter_argv or mod_name != "__main__": # i.e. -m switch
-            mod_name, loader, code, fname = _get_module_details(mod_name)
+            mod_name, mod_spec, code = _get_module_details(mod_name)
         else:          # i.e. directory or zipfile execution
-            mod_name, loader, code, fname = _get_main_module_details()
+            mod_name, mod_spec, code = _get_main_module_details()
     except ImportError as exc:
         # Try to provide a good error message
         # for directories, zip files and the -m switch
@@ -152,12 +182,11 @@ def _run_module_as_main(mod_name, alter_argv=True):
             info = "can't find '__main__' module in %r" % sys.argv[0]
         msg = "%s: %s" % (sys.executable, info)
         sys.exit(msg)
-    pkg_name = mod_name.rpartition('.')[0]
     main_globals = sys.modules["__main__"].__dict__
     if alter_argv:
-        sys.argv[0] = fname
+        sys.argv[0] = mod_spec.origin
     return _run_code(code, main_globals, None,
-                     "__main__", fname, loader, pkg_name)
+                     "__main__", mod_spec)
 
 def run_module(mod_name, init_globals=None,
                run_name=None, alter_sys=False):
@@ -165,17 +194,14 @@ def run_module(mod_name, init_globals=None,
 
        Returns the resulting top level namespace dictionary
     """
-    mod_name, loader, code, fname = _get_module_details(mod_name)
+    mod_name, mod_spec, code = _get_module_details(mod_name)
     if run_name is None:
         run_name = mod_name
-    pkg_name = mod_name.rpartition('.')[0]
     if alter_sys:
-        return _run_module_code(code, init_globals, run_name,
-                                fname, loader, pkg_name)
+        return _run_module_code(code, init_globals, run_name, mod_spec)
     else:
         # Leave the sys module alone
-        return _run_code(code, {}, init_globals, run_name,
-                         fname, loader, pkg_name)
+        return _run_code(code, {}, init_globals, run_name, mod_spec)
 
 def _get_main_module_details():
     # Helper that gives a nicer error message when attempting to
@@ -204,10 +230,7 @@ def _get_code_from_file(run_name, fname):
         # That didn't work, so try it as normal source code
         with open(fname, "rb") as f:
             code = compile(f.read(), fname, 'exec')
-            loader = importlib.machinery.SourceFileLoader(run_name, fname)
-    else:
-        loader = importlib.machinery.SourcelessFileLoader(run_name, fname)
-    return code, loader
+    return code, fname
 
 def run_path(path_name, init_globals=None, run_name=None):
     """Execute code located at the specified filesystem location
@@ -231,9 +254,9 @@ def run_path(path_name, init_globals=None, run_name=None):
     if isinstance(importer, type(None)) or is_NullImporter:
         # Not a valid sys.path entry, so run the code directly
         # execfile() doesn't help as we want to allow compiled files
-        code, mod_loader = _get_code_from_file(run_name, path_name)
-        return _run_module_code(code, init_globals, run_name, path_name,
-                                mod_loader, pkg_name)
+        code, fname = _get_code_from_file(run_name, path_name)
+        return _run_module_code(code, init_globals, run_name,
+                                pkg_name=pkg_name, script_name=fname)
     else:
         # Importer is defined for path, so add it to
         # the start of sys.path
@@ -245,12 +268,12 @@ def run_path(path_name, init_globals=None, run_name=None):
             # have no choice and we have to remove it even while we read the
             # code. If we don't do this, a __loader__ attribute in the
             # existing __main__ module may prevent location of the new module.
-            mod_name, loader, code, fname = _get_main_module_details()
+            mod_name, mod_spec, code = _get_main_module_details()
             with _TempModule(run_name) as temp_module, \
                  _ModifiedArgv0(path_name):
                 mod_globals = temp_module.module.__dict__
                 return _run_code(code, mod_globals, init_globals,
-                                    run_name, fname, loader, pkg_name).copy()
+                                    run_name, mod_spec, pkg_name).copy()
         finally:
             try:
                 sys.path.remove(path_name)

Lib/test/script_helper.py

@@ -101,8 +101,10 @@ def kill_python(p):
     subprocess._cleanup()
     return data
 
-def make_script(script_dir, script_basename, source):
-    script_filename = script_basename+os.extsep+'py'
+def make_script(script_dir, script_basename, source, omit_suffix=False):
+    script_filename = script_basename
+    if not omit_suffix:
+        script_filename += os.extsep + 'py'
     script_name = os.path.join(script_dir, script_filename)
     # The script should be encoded to UTF-8, the default string encoding
     script_file = open(script_name, 'w', encoding='utf-8')

Lib/test/test_cmd_line_script.py

@@ -41,11 +41,28 @@ from importlib.machinery import BuiltinImporter
 _loader = __loader__ if __loader__ is BuiltinImporter else type(__loader__)
 print('__loader__==%a' % _loader)
 print('__file__==%a' % __file__)
-assertEqual(__cached__, None)
+print('__cached__==%a' % __cached__)
 print('__package__==%r' % __package__)
+# Check PEP 451 details
+import os.path
+if __package__ is not None:
+    print('__main__ was located through the import system')
+    assertIdentical(__spec__.loader, __loader__)
+    expected_spec_name = os.path.splitext(os.path.basename(__file__))[0]
+    if __package__:
+        expected_spec_name = __package__ + "." + expected_spec_name
+    assertEqual(__spec__.name, expected_spec_name)
+    assertEqual(__spec__.parent, __package__)
+    assertIdentical(__spec__.submodule_search_locations, None)
+    assertEqual(__spec__.origin, __file__)
+    if __spec__.cached is not None:
+        assertEqual(__spec__.cached, __cached__)
 # Check the sys module
 import sys
 assertIdentical(globals(), sys.modules[__name__].__dict__)
+if __spec__ is not None:
+    # XXX: We're not currently making __main__ available under its real name
+    pass # assertIdentical(globals(), sys.modules[__spec__.name].__dict__)
 from test import test_cmd_line_script
 example_args_list = test_cmd_line_script.example_args
 assertEqual(sys.argv[1:], example_args_list)

Lib/test/test_runpy.py

@@ -5,7 +5,7 @@ import os.path
 import sys
 import re
 import tempfile
-import importlib
+import importlib, importlib.machinery, importlib.util
 import py_compile
 from test.support import (
     forget, make_legacy_pyc, run_unittest, unload, verbose, no_tracing,
@@ -47,7 +47,7 @@ implicit_namespace = {
     "__cached__": None,
     "__package__": None,
     "__doc__": None,
-#    "__spec__": None,  # XXX Uncomment.
+    "__spec__": None
 }
 example_namespace =  {
     "sys": sys,
@@ -57,7 +57,7 @@ example_namespace =  {
     "run_name_in_sys_modules": False,
     "module_in_sys_modules": False,
     "nested": dict(implicit_namespace,
-                   x=1, __name__="<run>", __loader__=None, __spec__=None),
+                   x=1, __name__="<run>", __loader__=None),
 }
 example_namespace.update(implicit_namespace)
 
@@ -68,11 +68,19 @@ class CodeExecutionMixin:
     # testing occurs at those upper layers as well, not just at the utility
     # layer
 
+    # Figuring out the loader details in advance is hard to do, so we skip
+    # checking the full details of loader and loader_state
+    CHECKED_SPEC_ATTRIBUTES = ["name", "parent", "origin", "cached",
+                               "has_location", "submodule_search_locations"]
+
     def assertNamespaceMatches(self, result_ns, expected_ns):
         """Check two namespaces match.
 
            Ignores any unspecified interpreter created names
         """
+        # Avoid side effects
+        result_ns = result_ns.copy()
+        expected_ns = expected_ns.copy()
         # Impls are permitted to add extra names, so filter them out
         for k in list(result_ns):
             if k.startswith("__") and k.endswith("__"):
@@ -80,7 +88,25 @@ class CodeExecutionMixin:
                     result_ns.pop(k)
                 if k not in expected_ns["nested"]:
                     result_ns["nested"].pop(k)
-        # Don't use direct dict comparison - the diffs are too hard to debug
+        # Spec equality includes the loader, so we take the spec out of the
+        # result namespace and check that separately
+        result_spec = result_ns.pop("__spec__")
+        expected_spec = expected_ns.pop("__spec__")
+        if expected_spec is None:
+            self.assertIsNone(result_spec)
+        else:
+            # If an expected loader is set, we just check we got the right
+            # type, rather than checking for full equality
+            if expected_spec.loader is not None:
+                self.assertEqual(type(result_spec.loader),
+                                 type(expected_spec.loader))
+            for attr in self.CHECKED_SPEC_ATTRIBUTES:
+                k = "__spec__." + attr
+                actual = (k, getattr(result_spec, attr))
+                expected = (k, getattr(expected_spec, attr))
+                self.assertEqual(actual, expected)
+        # For the rest, we still don't use direct dict comparison on the
+        # namespace, as the diffs are too hard to debug if anything breaks
         self.assertEqual(set(result_ns), set(expected_ns))
         for k in result_ns:
             actual = (k, result_ns[k])
@@ -131,12 +157,16 @@ class ExecutionLayerTestCase(unittest.TestCase, CodeExecutionMixin):
         mod_fname = "Some other nonsense"
         mod_loader = "Now you're just being silly"
         mod_package = '' # Treat as a top level module
+        mod_spec = importlib.machinery.ModuleSpec(mod_name,
+                                                  origin=mod_fname,
+                                                  loader=mod_loader)
         expected_ns = example_namespace.copy()
         expected_ns.update({
             "__name__": mod_name,
             "__file__": mod_fname,
             "__loader__": mod_loader,
             "__package__": mod_package,
+            "__spec__": mod_spec,
             "run_argv0": mod_fname,
             "run_name_in_sys_modules": True,
             "module_in_sys_modules": True,
@@ -145,9 +175,7 @@ class ExecutionLayerTestCase(unittest.TestCase, CodeExecutionMixin):
             return _run_module_code(example_source,
                                     init_globals,
                                     mod_name,
-                                    mod_fname,
-                                    mod_loader,
-                                    mod_package)
+                                    mod_spec)
         self.check_code_execution(create_ns, expected_ns)
 
 # TODO: Use self.addCleanup to get rid of a lot of try-finally blocks
@@ -177,31 +205,43 @@ class RunModuleTestCase(unittest.TestCase, CodeExecutionMixin):
     def test_library_module(self):
         self.assertEqual(run_module("runpy")["__name__"], "runpy")
 
-    def _add_pkg_dir(self, pkg_dir):
+    def _add_pkg_dir(self, pkg_dir, namespace=False):
         os.mkdir(pkg_dir)
+        if namespace:
+            return None
         pkg_fname = os.path.join(pkg_dir, "__init__.py")
         create_empty_file(pkg_fname)
         return pkg_fname
 
-    def _make_pkg(self, source, depth, mod_base="runpy_test"):
+    def _make_pkg(self, source, depth, mod_base="runpy_test",
+                     *, namespace=False, parent_namespaces=False):
+        # Enforce a couple of internal sanity checks on test cases
+        if (namespace or parent_namespaces) and not depth:
+            raise RuntimeError("Can't mark top level module as a "
+                               "namespace package")
         pkg_name = "__runpy_pkg__"
         test_fname = mod_base+os.extsep+"py"
         pkg_dir = sub_dir = os.path.realpath(tempfile.mkdtemp())
         if verbose > 1: print("  Package tree in:", sub_dir)
         sys.path.insert(0, pkg_dir)
         if verbose > 1: print("  Updated sys.path:", sys.path[0])
-        for i in range(depth):
-            sub_dir = os.path.join(sub_dir, pkg_name)
-            pkg_fname = self._add_pkg_dir(sub_dir)
-            if verbose > 1: print("  Next level in:", sub_dir)
-            if verbose > 1: print("  Created:", pkg_fname)
+        if depth:
+            namespace_flags = [parent_namespaces] * depth
+            namespace_flags[-1] = namespace
+            for namespace_flag in namespace_flags:
+                sub_dir = os.path.join(sub_dir, pkg_name)
+                pkg_fname = self._add_pkg_dir(sub_dir, namespace_flag)
+                if verbose > 1: print("  Next level in:", sub_dir)
+                if verbose > 1: print("  Created:", pkg_fname)
         mod_fname = os.path.join(sub_dir, test_fname)
         mod_file = open(mod_fname, "w")
         mod_file.write(source)
         mod_file.close()
         if verbose > 1: print("  Created:", mod_fname)
         mod_name = (pkg_name+".")*depth + mod_base
-        return pkg_dir, mod_fname, mod_name
+        mod_spec = importlib.util.spec_from_file_location(mod_name,
+                                                          mod_fname)
+        return pkg_dir, mod_fname, mod_name, mod_spec
 
     def _del_pkg(self, top, depth, mod_name):
         for entry in list(sys.modules):
@@ -231,20 +271,29 @@ class RunModuleTestCase(unittest.TestCase, CodeExecutionMixin):
     def _fix_ns_for_legacy_pyc(self, ns, alter_sys):
         char_to_add = "c" if __debug__ else "o"
         ns["__file__"] += char_to_add
+        ns["__cached__"] = ns["__file__"]
+        spec = ns["__spec__"]
+        new_spec = importlib.util.spec_from_file_location(spec.name,
+                                                          ns["__file__"])
+        ns["__spec__"] = new_spec
         if alter_sys:
             ns["run_argv0"] += char_to_add
 
 
-    def _check_module(self, depth, alter_sys=False):
-        pkg_dir, mod_fname, mod_name = (
-               self._make_pkg(example_source, depth))
+    def _check_module(self, depth, alter_sys=False,
+                         *, namespace=False, parent_namespaces=False):
+        pkg_dir, mod_fname, mod_name, mod_spec = (
+               self._make_pkg(example_source, depth,
+                              namespace=namespace,
+                              parent_namespaces=parent_namespaces))
         forget(mod_name)
         expected_ns = example_namespace.copy()
         expected_ns.update({
             "__name__": mod_name,
             "__file__": mod_fname,
+            "__cached__": mod_spec.cached,
             "__package__": mod_name.rpartition(".")[0],
-#            "__spec__": None,  # XXX Needs to be set.
+            "__spec__": mod_spec,
         })
         if alter_sys:
             expected_ns.update({
@@ -271,17 +320,21 @@ class RunModuleTestCase(unittest.TestCase, CodeExecutionMixin):
             self._del_pkg(pkg_dir, depth, mod_name)
         if verbose > 1: print("Module executed successfully")
 
-    def _check_package(self, depth, alter_sys=False):
-        pkg_dir, mod_fname, mod_name = (
-               self._make_pkg(example_source, depth, "__main__"))
+    def _check_package(self, depth, alter_sys=False,
+                          *, namespace=False, parent_namespaces=False):
+        pkg_dir, mod_fname, mod_name, mod_spec = (
+               self._make_pkg(example_source, depth, "__main__",
+                              namespace=namespace,
+                              parent_namespaces=parent_namespaces))
         pkg_name = mod_name.rpartition(".")[0]
         forget(mod_name)
         expected_ns = example_namespace.copy()
         expected_ns.update({
             "__name__": mod_name,
             "__file__": mod_fname,
+            "__cached__": importlib.util.cache_from_source(mod_fname),
             "__package__": pkg_name,
-#            "__spec__": None,  # XXX Needs to be set.
+            "__spec__": mod_spec,
         })
         if alter_sys:
             expected_ns.update({
@@ -337,7 +390,7 @@ from __future__ import absolute_import
 from . import sibling
 from ..uncle.cousin import nephew
 """
-        pkg_dir, mod_fname, mod_name = (
+        pkg_dir, mod_fname, mod_name, mod_spec = (
                self._make_pkg(contents, depth))
         if run_name is None:
             expected_name = mod_name
@@ -376,11 +429,31 @@ from ..uncle.cousin import nephew
             if verbose > 1: print("Testing package depth:", depth)
             self._check_module(depth)
 
+    def test_run_module_in_namespace_package(self):
+        for depth in range(1, 4):
+            if verbose > 1: print("Testing package depth:", depth)
+            self._check_module(depth, namespace=True, parent_namespaces=True)
+
     def test_run_package(self):
         for depth in range(1, 4):
             if verbose > 1: print("Testing package depth:", depth)
             self._check_package(depth)
 
+    def test_run_package_in_namespace_package(self):
+        for depth in range(1, 4):
+            if verbose > 1: print("Testing package depth:", depth)
+            self._check_package(depth, parent_namespaces=True)
+
+    def test_run_namespace_package(self):
+        for depth in range(1, 4):
+            if verbose > 1: print("Testing package depth:", depth)
+            self._check_package(depth, namespace=True)
+
+    def test_run_namespace_package_in_namespace_package(self):
+        for depth in range(1, 4):
+            if verbose > 1: print("Testing package depth:", depth)
+            self._check_package(depth, namespace=True, parent_namespaces=True)
+
     def test_run_module_alter_sys(self):
         for depth in range(4):
             if verbose > 1: print("Testing package depth:", depth)
@@ -404,14 +477,16 @@ from ..uncle.cousin import nephew
     def test_run_name(self):
         depth = 1
         run_name = "And now for something completely different"
-        pkg_dir, mod_fname, mod_name = (
+        pkg_dir, mod_fname, mod_name, mod_spec = (
                self._make_pkg(example_source, depth))
         forget(mod_name)
         expected_ns = example_namespace.copy()
         expected_ns.update({
             "__name__": run_name,
             "__file__": mod_fname,
+            "__cached__": importlib.util.cache_from_source(mod_fname),
             "__package__": mod_name.rpartition(".")[0],
+            "__spec__": mod_spec,
         })
         def create_ns(init_globals):
             return run_module(mod_name, init_globals, run_name)
@@ -440,7 +515,7 @@ from ..uncle.cousin import nephew
         pkg_name = ".".join([base_name] * max_depth)
         expected_packages.add(pkg_name)
         expected_modules.add(pkg_name + ".runpy_test")
-        pkg_dir, mod_fname, mod_name = (
+        pkg_dir, mod_fname, mod_name, mod_spec = (
                self._make_pkg("", max_depth))
         self.addCleanup(self._del_pkg, pkg_dir, max_depth, mod_name)
         for depth in range(2, max_depth+1):
@@ -458,21 +533,39 @@ from ..uncle.cousin import nephew
 class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
     """Unit tests for runpy.run_path"""
 
-    def _make_test_script(self, script_dir, script_basename, source=None):
+    def _make_test_script(self, script_dir, script_basename,
+                          source=None, omit_suffix=False):
         if source is None:
             source = example_source
-        return make_script(script_dir, script_basename, source)
+        return make_script(script_dir, script_basename,
+                           source, omit_suffix)
 
     def _check_script(self, script_name, expected_name, expected_file,
-                            expected_argv0):
+                            expected_argv0, mod_name=None,
+                            expect_spec=True, check_loader=True):
         # First check is without run_name
         def create_ns(init_globals):
             return run_path(script_name, init_globals)
         expected_ns = example_namespace.copy()
+        if mod_name is None:
+            spec_name = expected_name
+        else:
+            spec_name = mod_name
+        if expect_spec:
+            mod_spec = importlib.util.spec_from_file_location(spec_name,
+                                                              expected_file)
+            mod_cached = mod_spec.cached
+            if not check_loader:
+                mod_spec.loader = None
+        else:
+            mod_spec = mod_cached = None
+
         expected_ns.update({
             "__name__": expected_name,
             "__file__": expected_file,
+            "__cached__": mod_cached,
             "__package__": "",
+            "__spec__": mod_spec,
             "run_argv0": expected_argv0,
             "run_name_in_sys_modules": True,
             "module_in_sys_modules": True,
@@ -482,6 +575,12 @@ class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
         run_name = "prove.issue15230.is.fixed"
         def create_ns(init_globals):
             return run_path(script_name, init_globals, run_name)
+        if expect_spec and mod_name is None:
+            mod_spec = importlib.util.spec_from_file_location(run_name,
+                                                              expected_file)
+            if not check_loader:
+                mod_spec.loader = None
+            expected_ns["__spec__"] = mod_spec
         expected_ns["__name__"] = run_name
         expected_ns["__package__"] = run_name.rpartition(".")[0]
         self.check_code_execution(create_ns, expected_ns)
@@ -495,7 +594,15 @@ class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
             mod_name = 'script'
             script_name = self._make_test_script(script_dir, mod_name)
             self._check_script(script_name, "<run_path>", script_name,
-                               script_name)
+                               script_name, expect_spec=False)
+
+    def test_basic_script_no_suffix(self):
+        with temp_dir() as script_dir:
+            mod_name = 'script'
+            script_name = self._make_test_script(script_dir, mod_name,
+                                                 omit_suffix=True)
+            self._check_script(script_name, "<run_path>", script_name,
+                               script_name, expect_spec=False)
 
     def test_script_compiled(self):
         with temp_dir() as script_dir:
@@ -504,14 +611,14 @@ class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
             compiled_name = py_compile.compile(script_name, doraise=True)
             os.remove(script_name)
             self._check_script(compiled_name, "<run_path>", compiled_name,
-                               compiled_name)
+                               compiled_name, expect_spec=False)
 
     def test_directory(self):
         with temp_dir() as script_dir:
             mod_name = '__main__'
             script_name = self._make_test_script(script_dir, mod_name)
             self._check_script(script_dir, "<run_path>", script_name,
-                               script_dir)
+                               script_dir, mod_name=mod_name)
 
     def test_directory_compiled(self):
         with temp_dir() as script_dir:
@@ -522,7 +629,7 @@ class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
             if not sys.dont_write_bytecode:
                 legacy_pyc = make_legacy_pyc(script_name)
                 self._check_script(script_dir, "<run_path>", legacy_pyc,
-                                   script_dir)
+                                   script_dir, mod_name=mod_name)
 
     def test_directory_error(self):
         with temp_dir() as script_dir:
@@ -536,7 +643,8 @@ class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
             mod_name = '__main__'
             script_name = self._make_test_script(script_dir, mod_name)
             zip_name, fname = make_zip_script(script_dir, 'test_zip', script_name)
-            self._check_script(zip_name, "<run_path>", fname, zip_name)
+            self._check_script(zip_name, "<run_path>", fname, zip_name,
+                               mod_name=mod_name, check_loader=False)
 
     def test_zipfile_compiled(self):
         with temp_dir() as script_dir:
@@ -545,7 +653,8 @@ class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
             compiled_name = py_compile.compile(script_name, doraise=True)
             zip_name, fname = make_zip_script(script_dir, 'test_zip',
                                               compiled_name)
-            self._check_script(zip_name, "<run_path>", fname, zip_name)
+            self._check_script(zip_name, "<run_path>", fname, zip_name,
+                               mod_name=mod_name, check_loader=False)
 
     def test_zipfile_error(self):
         with temp_dir() as script_dir: