Issue #13959: Re-implement imp.get_suffixes() in Lib/imp.py.

This introduces a new function, imp.extension_suffixes(), which is
currently undocumented. That is forthcoming once issue #14657 is
resolved and how to expose file suffixes is decided.

Lib/imp.py

@@ -9,11 +9,9 @@ functionality over this module.
 from _imp import (lock_held, acquire_lock, release_lock,
                   load_dynamic, get_frozen_object, is_frozen_package,
                   init_builtin, init_frozen, is_builtin, is_frozen,
-                  _fix_co_filename)
+                  _fix_co_filename, extension_suffixes)
 # Could move out of _imp, but not worth the code
 from _imp import get_magic, get_tag
-# Can (probably) move to importlib
-from _imp import get_suffixes
 
 from importlib._bootstrap import new_module
 from importlib._bootstrap import cache_from_source
@@ -38,6 +36,14 @@ PY_CODERESOURCE = 8
 IMP_HOOK = 9
 
 
+def get_suffixes():
+    extensions = [(s, 'rb', C_EXTENSION) for s in extension_suffixes()]
+    source = [(s, 'U', PY_SOURCE) for s in _bootstrap._SOURCE_SUFFIXES]
+    bytecode = [(_bootstrap._BYTECODE_SUFFIX, 'rb', PY_COMPILED)]
+
+    return extensions + source + bytecode
+
+
 def source_from_cache(path):
     """Given the path to a .pyc./.pyo file, return the path to its .py file.
 
@@ -120,8 +126,8 @@ def load_compiled(name, pathname, file=None):
 # XXX deprecate
 def load_package(name, path):
     if os.path.isdir(path):
-        extensions = _bootstrap._suffix_list(PY_SOURCE)
-        extensions += _bootstrap._suffix_list(PY_COMPILED)
+        extensions = _bootstrap._SOURCE_SUFFIXES
+        extensions += [_bootstrap._BYTECODE_SUFFIX]
         for extension in extensions:
             path = os.path.join(path, '__init__'+extension)
             if os.path.exists(path):

Lib/importlib/_bootstrap.py

@@ -95,16 +95,6 @@ def _path_split(path):
     return front, tail
 
 
-def _path_exists(path):
-    """Replacement for os.path.exists."""
-    try:
-        _os.stat(path)
-    except OSError:
-        return False
-    else:
-        return True
-
-
 def _path_is_mode_type(path, mode):
     """Test whether the path is the specified mode type."""
     try:
@@ -128,28 +118,6 @@ def _path_isdir(path):
     return _path_is_mode_type(path, 0o040000)
 
 
-def _path_without_ext(path, ext_type):
-    """Replacement for os.path.splitext()[0]."""
-    for suffix in _suffix_list(ext_type):
-        if path.endswith(suffix):
-            return path[:-len(suffix)]
-    else:
-        raise ValueError("path is not of the specified type")
-
-
-def _path_absolute(path):
-    """Replacement for os.path.abspath."""
-    if not path:
-        path = _os.getcwd()
-    try:
-        return _os._getfullpathname(path)
-    except AttributeError:
-        if path.startswith('/'):
-            return path
-        else:
-            return _path_join(_os.getcwd(), path)
-
-
 def _write_atomic(path, data):
     """Best-effort function to write data to a path atomically.
     Be prepared to handle a FileExistsError if concurrent writing of the
@@ -338,12 +306,6 @@ def _requires_frozen(fxn):
     return _requires_frozen_wrapper
 
 
-def _suffix_list(suffix_type):
-    """Return a list of file suffixes based on the imp file type."""
-    return [suffix[0] for suffix in _imp.get_suffixes()
-            if suffix[2] == suffix_type]
-
-
 # Loaders #####################################################################
 
 class BuiltinImporter:
@@ -1196,8 +1158,9 @@ def _install(sys_module, _imp_module):
 
     """
     _setup(sys_module, _imp_module)
-    supported_loaders = [(ExtensionFileLoader, _suffix_list(3), False),
-                         (SourceFileLoader, _suffix_list(1), True),
-                         (SourcelessFileLoader, _suffix_list(2), True)]
+    extensions = ExtensionFileLoader, _imp_module.extension_suffixes(), False
+    source = SourceFileLoader, _SOURCE_SUFFIXES, True
+    bytecode = SourcelessFileLoader, [_BYTECODE_SUFFIX], True
+    supported_loaders = [extensions, source, bytecode]
     sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)])
     sys.meta_path.extend([BuiltinImporter, FrozenImporter, PathFinder])

Lib/importlib/test/extension/test_case_sensitivity.py

@@ -16,7 +16,7 @@ class ExtensionModuleCaseSensitivityTest(unittest.TestCase):
         assert good_name != bad_name
         finder = _bootstrap.FileFinder(ext_util.PATH,
                                         (_bootstrap.ExtensionFileLoader,
-                                            _bootstrap._suffix_list(imp.C_EXTENSION),
+                                            imp.extension_suffixes(),
                                             False))
         return finder.find_module(bad_name)
 

Lib/importlib/test/extension/test_finder.py

@@ -12,7 +12,7 @@ class FinderTests(abc.FinderTests):
     def find_module(self, fullname):
         importer = _bootstrap.FileFinder(util.PATH,
                                           (_bootstrap.ExtensionFileLoader,
-                                              _bootstrap._suffix_list(imp.C_EXTENSION),
+                                              imp.extension_suffixes(),
                                               False))
         return importer.find_module(fullname)
 

Lib/importlib/test/extension/test_path_hook.py

@@ -15,7 +15,7 @@ class PathHookTests(unittest.TestCase):
 
     def hook(self, entry):
         return _bootstrap.FileFinder.path_hook((_bootstrap.ExtensionFileLoader,
-            _bootstrap._suffix_list(imp.C_EXTENSION), False))(entry)
+            imp.extension_suffixes(), False))(entry)
 
     def test_success(self):
         # Path hook should handle a directory where a known extension module

Lib/importlib/test/source/test_case_sensitivity.py

@@ -22,10 +22,10 @@ class CaseSensitivityTest(unittest.TestCase):
     def find(self, path):
         finder = _bootstrap.FileFinder(path,
                                         (_bootstrap.SourceFileLoader,
-                                            _bootstrap._suffix_list(imp.PY_SOURCE),
+                                            _bootstrap._SOURCE_SUFFIXES,
                                             True),
                                         (_bootstrap.SourcelessFileLoader,
-                                            _bootstrap._suffix_list(imp.PY_COMPILED),
+                                            [_bootstrap._BYTECODE_SUFFIX],
                                             True))
         return finder.find_module(self.name)
 

Lib/importlib/test/source/test_finder.py

@@ -37,9 +37,9 @@ class FinderTests(abc.FinderTests):
 
     def import_(self, root, module):
         loader_details = [(_bootstrap.SourceFileLoader,
-                            _bootstrap._suffix_list(imp.PY_SOURCE), True),
+                            _bootstrap._SOURCE_SUFFIXES, True),
                           (_bootstrap.SourcelessFileLoader,
-                            _bootstrap._suffix_list(imp.PY_COMPILED), True)]
+                            [_bootstrap._BYTECODE_SUFFIX], True)]
         finder = _bootstrap.FileFinder(root, *loader_details)
         return finder.find_module(module)
 
@@ -139,7 +139,7 @@ class FinderTests(abc.FinderTests):
     def test_empty_string_for_dir(self):
         # The empty string from sys.path means to search in the cwd.
         finder = _bootstrap.FileFinder('', (_bootstrap.SourceFileLoader,
-            _bootstrap._suffix_list(imp.PY_SOURCE), True))
+            _bootstrap._SOURCE_SUFFIXES, True))
         with open('mod.py', 'w') as file:
             file.write("# test file for importlib")
         try:
@@ -151,7 +151,7 @@ class FinderTests(abc.FinderTests):
     def test_invalidate_caches(self):
         # invalidate_caches() should reset the mtime.
         finder = _bootstrap.FileFinder('', (_bootstrap.SourceFileLoader,
-            _bootstrap._suffix_list(imp.PY_SOURCE), True))
+            _bootstrap._SOURCE_SUFFIXES, True))
         finder._path_mtime = 42
         finder.invalidate_caches()
         self.assertEqual(finder._path_mtime, -1)

Lib/importlib/test/source/test_path_hook.py

@@ -11,7 +11,7 @@ class PathHookTest(unittest.TestCase):
 
     def path_hook(self):
         return _bootstrap.FileFinder.path_hook((_bootstrap.SourceFileLoader,
-            _bootstrap._suffix_list(imp.PY_SOURCE), True))
+            _bootstrap._SOURCE_SUFFIXES, True))
 
     def test_success(self):
         with source_util.create_modules('dummy') as mapping:

Python/dynload_aix.c

@@ -26,10 +26,7 @@ typedef struct Module {
     void          *entry;
 } Module, *ModulePtr;
 
-const struct filedescr _PyImport_DynLoadFiletab[] = {
-    {".so", "rb", C_EXTENSION},
-    {0, 0}
-};
+const char *_PyImport_DynLoadFiletab[] = {".so", NULL};
 
 static int
 aix_getoldmodules(void **modlistptr)

Python/dynload_dl.c

@@ -9,10 +9,7 @@
 
 extern char *Py_GetProgramName(void);
 
-const struct filedescr _PyImport_DynLoadFiletab[] = {
-    {".o", "rb", C_EXTENSION},
-    {0, 0}
-};
+const char *_PyImport_DynLoadFiletab[] = {".o", NULL};
 
 
 dl_funcptr _PyImport_GetDynLoadFunc(const char *shortname,

Python/dynload_hpux.c

@@ -13,10 +13,7 @@
 #define FUNCNAME_PATTERN "PyInit_%.200s"
 #endif
 
-const struct filedescr _PyImport_DynLoadFiletab[] = {
-    {SHLIB_EXT, "rb", C_EXTENSION},
-    {0, 0}
-};
+const char *_PyImport_DynLoadFiletab[] = {SHLIB_EXT, NULL};
 
 dl_funcptr _PyImport_GetDynLoadFunc(const char *shortname,
                                     const char *pathname, FILE *fp)

Python/dynload_next.c

@@ -8,10 +8,7 @@
 
 #include <mach-o/dyld.h>
 
-const struct filedescr _PyImport_DynLoadFiletab[] = {
-    {".so", "rb", C_EXTENSION},
-    {0, 0}
-};
+const char *_PyImport_DynLoadFiletab[] = {".so", NULL};
 
 /*
 ** Python modules are Mach-O MH_BUNDLE files. The best way to load these

Python/dynload_os2.c

@@ -9,11 +9,7 @@
 #include "importdl.h"
 
 
-const struct filedescr _PyImport_DynLoadFiletab[] = {
-    {".pyd", "rb", C_EXTENSION},
-    {".dll", "rb", C_EXTENSION},
-    {0, 0}
-};
+const char *_PyImport_DynLoadFiletab[] = {".pyd", ".dll", NULL};
 
 dl_funcptr _PyImport_GetDynLoadFunc(const char *shortname,
                                     const char *pathname, FILE *fp)

Python/dynload_shlib.c

@@ -36,25 +36,25 @@
    live in the same directory.  E.g. foomodule.cpython-32.so
 */
 
-const struct filedescr _PyImport_DynLoadFiletab[] = {
+const char *_PyImport_DynLoadFiletab[] = {
 #ifdef __CYGWIN__
-    {".dll", "rb", C_EXTENSION},
+    ".dll",
 #else  /* !__CYGWIN__ */
 #if defined(PYOS_OS2) && defined(PYCC_GCC)
-    {".pyd", "rb", C_EXTENSION},
-    {".dll", "rb", C_EXTENSION},
+    ".pyd",
+    ".dll",
 #else  /* !(defined(PYOS_OS2) && defined(PYCC_GCC)) */
 #ifdef __VMS
-    {".exe", "rb", C_EXTENSION},
-    {".EXE", "rb", C_EXTENSION},
+    ".exe",
+    ".EXE",
 #else  /* !__VMS */
-    {"." SOABI ".so", "rb", C_EXTENSION},
-    {".abi" PYTHON_ABI_STRING ".so", "rb", C_EXTENSION},
-    {".so", "rb", C_EXTENSION},
+    "." SOABI ".so",
+    ".abi" PYTHON_ABI_STRING ".so",
+    ".so",
 #endif  /* __VMS */
 #endif  /* defined(PYOS_OS2) && defined(PYCC_GCC) */
 #endif  /* __CYGWIN__ */
-    {0, 0}
+    NULL,
 };
 
 static struct {

Python/dynload_stub.c

@@ -6,6 +6,4 @@
 #include "importdl.h"
 
 
-const struct filedescr _PyImport_DynLoadFiletab[] = {
-	{0, 0}
-};
+const char *_PyImport_DynLoadFiletab[] = {NULL};

Python/dynload_win.c

@@ -15,13 +15,13 @@
 extern ULONG_PTR _Py_ActivateActCtx();
 void _Py_DeactivateActCtx(ULONG_PTR cookie);
 
-const struct filedescr _PyImport_DynLoadFiletab[] = {
+const char *_PyImport_DynLoadFiletab[] = {
 #ifdef _DEBUG
-    {"_d.pyd", "rb", C_EXTENSION},
+    "_d.pyd",
 #else
-    {".pyd", "rb", C_EXTENSION},
+    ".pyd",
 #endif
-    {0, 0}
+    NULL
 };
 
 

Python/import.c

@@ -140,18 +140,6 @@ extern struct _inittab _PyImport_Inittab[];
 
 struct _inittab *PyImport_Inittab = _PyImport_Inittab;
 
-/* these tables define the module suffixes that Python recognizes */
-struct filedescr * _PyImport_Filetab = NULL;
-
-static const struct filedescr _PyImport_StandardFiletab[] = {
-    {".py", "U", PY_SOURCE},
-#ifdef MS_WINDOWS
-    {".pyw", "U", PY_SOURCE},
-#endif
-    {".pyc", "rb", PY_COMPILED},
-    {0, 0}
-};
-
 static PyObject *initstr = NULL;
 
 /* Initialize things */
@@ -159,44 +147,9 @@ static PyObject *initstr = NULL;
 void
 _PyImport_Init(void)
 {
-    const struct filedescr *scan;
-    struct filedescr *filetab;
-    int countD = 0;
-    int countS = 0;
-
     initstr = PyUnicode_InternFromString("__init__");
     if (initstr == NULL)
         Py_FatalError("Can't initialize import variables");
-
-    /* prepare _PyImport_Filetab: copy entries from
-       _PyImport_DynLoadFiletab and _PyImport_StandardFiletab.
-     */
-#ifdef HAVE_DYNAMIC_LOADING
-    for (scan = _PyImport_DynLoadFiletab; scan->suffix != NULL; ++scan)
-        ++countD;
-#endif
-    for (scan = _PyImport_StandardFiletab; scan->suffix != NULL; ++scan)
-        ++countS;
-    filetab = PyMem_NEW(struct filedescr, countD + countS + 1);
-    if (filetab == NULL)
-        Py_FatalError("Can't initialize import file table.");
-#ifdef HAVE_DYNAMIC_LOADING
-    memcpy(filetab, _PyImport_DynLoadFiletab,
-           countD * sizeof(struct filedescr));
-#endif
-    memcpy(filetab + countD, _PyImport_StandardFiletab,
-           countS * sizeof(struct filedescr));
-    filetab[countD + countS].suffix = NULL;
-
-    _PyImport_Filetab = filetab;
-
-    if (Py_OptimizeFlag) {
-        /* Replace ".pyc" with ".pyo" in _PyImport_Filetab */
-        for (; filetab->suffix != NULL; filetab++) {
-            if (strcmp(filetab->suffix, ".pyc") == 0)
-                filetab->suffix = ".pyo";
-        }
-    }
 }
 
 void
@@ -400,8 +353,6 @@ _PyImport_Fini(void)
 {
     Py_XDECREF(extensions);
     extensions = NULL;
-    PyMem_DEL(_PyImport_Filetab);
-    _PyImport_Filetab = NULL;
 #ifdef WITH_THREAD
     if (import_lock != NULL) {
         PyThread_free_lock(import_lock);
@@ -1911,17 +1862,18 @@ imp_get_tag(PyObject *self, PyObject *noargs)
 }
 
 static PyObject *
-imp_get_suffixes(PyObject *self, PyObject *noargs)
+imp_extension_suffixes(PyObject *self, PyObject *noargs)
 {
     PyObject *list;
-    struct filedescr *fdp;
+    const char *suffix;
+    unsigned int index = 0;
 
     list = PyList_New(0);
     if (list == NULL)
         return NULL;
-    for (fdp = _PyImport_Filetab; fdp->suffix != NULL; fdp++) {
-        PyObject *item = Py_BuildValue("ssi",
-                               fdp->suffix, fdp->mode, fdp->type);
+#ifdef HAVE_DYNAMIC_LOADING
+    while ((suffix = _PyImport_DynLoadFiletab[index])) {
+        PyObject *item = PyUnicode_FromString(suffix);
         if (item == NULL) {
             Py_DECREF(list);
             return NULL;
@@ -1932,7 +1884,9 @@ imp_get_suffixes(PyObject *self, PyObject *noargs)
             return NULL;
         }
         Py_DECREF(item);
+        index += 1;
     }
+#endif
     return list;
 }
 
@@ -2101,10 +2055,9 @@ PyDoc_STRVAR(doc_get_tag,
 "get_tag() -> string\n\
 Return the magic tag for .pyc or .pyo files.");
 
-PyDoc_STRVAR(doc_get_suffixes,
-"get_suffixes() -> [(suffix, mode, type), ...]\n\
-Return a list of (suffix, mode, type) tuples describing the files\n\
-that find_module() looks for.");
+PyDoc_STRVAR(doc_extension_suffixes,
+"extension_suffixes() -> list of strings\n\
+Returns the list of file suffixes used to identify extension modules.");
 
 PyDoc_STRVAR(doc_lock_held,
 "lock_held() -> boolean\n\
@@ -2126,7 +2079,8 @@ On platforms without threads, this function does nothing.");
 static PyMethodDef imp_methods[] = {
     {"get_magic",        imp_get_magic,    METH_NOARGS,  doc_get_magic},
     {"get_tag",          imp_get_tag,      METH_NOARGS,  doc_get_tag},
-    {"get_suffixes", imp_get_suffixes, METH_NOARGS,  doc_get_suffixes},
+    {"extension_suffixes", imp_extension_suffixes, METH_NOARGS,
+        doc_extension_suffixes},
     {"lock_held",        imp_lock_held,    METH_NOARGS,  doc_lock_held},
     {"acquire_lock", imp_acquire_lock, METH_NOARGS,  doc_acquire_lock},
     {"release_lock", imp_release_lock, METH_NOARGS,  doc_release_lock},

Python/importdl.h

@@ -20,13 +20,8 @@ enum filetype {
     IMP_HOOK
 };
 
-struct filedescr {
-    char *suffix;
-    char *mode;
-    enum filetype type;
-};
-extern struct filedescr * _PyImport_Filetab;
-extern const struct filedescr _PyImport_DynLoadFiletab[];
+
+extern const char *_PyImport_DynLoadFiletab[];
 
 extern PyObject *_PyImport_LoadDynamicModule(PyObject *name, PyObject *pathname,
                                              FILE *);