pkgutil.py 20.3 KB
Newer Older
1 2
"""Utilities to support packages."""

3 4
from functools import singledispatch as simplegeneric
import importlib
5
import importlib.util
6
import importlib.machinery
7
import os
8
import os.path
9
import sys
10
from types import ModuleType
11
import warnings
12 13 14

__all__ = [
    'get_importer', 'iter_importers', 'get_loader', 'find_loader',
15
    'walk_packages', 'iter_modules', 'get_data',
16 17 18 19 20 21 22 23 24
    'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',
]

def read_code(stream):
    # This helper is needed in order for the PEP 302 emulation to
    # correctly handle compiled files
    import marshal

    magic = stream.read(4)
25
    if magic != importlib.util.MAGIC_NUMBER:
26 27
        return None

28
    stream.read(8) # Skip timestamp and size
29 30 31 32
    return marshal.load(stream)


def walk_packages(path=None, prefix='', onerror=None):
33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59
    """Yields (module_loader, name, ispkg) for all modules recursively
    on path, or, if path is None, all accessible modules.

    'path' should be either None or a list of paths to look for
    modules in.

    'prefix' is a string to output on the front of every module name
    on output.

    Note that this function must import all *packages* (NOT all
    modules!) on the given path, in order to access the __path__
    attribute to find submodules.

    'onerror' is a function which gets called with one argument (the
    name of the package which was being imported) if any exception
    occurs while trying to import a package.  If no onerror function is
    supplied, ImportErrors are caught and ignored, while all other
    exceptions are propagated, terminating the search.

    Examples:

    # list all modules python can access
    walk_packages()

    # list all submodules of ctypes
    walk_packages(ctypes.__path__, ctypes.__name__+'.')
    """
60

61 62 63
    def seen(p, m={}):
        if p in m:
            return True
64 65 66 67 68 69 70 71 72 73
        m[p] = True

    for importer, name, ispkg in iter_modules(path, prefix):
        yield importer, name, ispkg

        if ispkg:
            try:
                __import__(name)
            except ImportError:
                if onerror is not None:
74 75 76 77 78 79
                    onerror(name)
            except Exception:
                if onerror is not None:
                    onerror(name)
                else:
                    raise
80 81 82 83 84 85
            else:
                path = getattr(sys.modules[name], '__path__', None) or []

                # don't traverse path items we've seen before
                path = [p for p in path if not seen(p)]

Philip Jenvey's avatar
Philip Jenvey committed
86
                yield from walk_packages(path, name+'.', onerror)
87 88 89


def iter_modules(path=None, prefix=''):
90 91 92 93 94 95 96 97 98 99
    """Yields (module_loader, name, ispkg) for all submodules on path,
    or, if path is None, all top-level modules on sys.path.

    'path' should be either None or a list of paths to look for
    modules in.

    'prefix' is a string to output on the front of every module name
    on output.
    """

100 101 102 103 104 105 106 107 108 109 110 111 112
    if path is None:
        importers = iter_importers()
    else:
        importers = map(get_importer, path)

    yielded = {}
    for i in importers:
        for name, ispkg in iter_importer_modules(i, prefix):
            if name not in yielded:
                yielded[name] = 1
                yield i, name, ispkg


113
@simplegeneric
114
def iter_importer_modules(importer, prefix=''):
115
    if not hasattr(importer, 'iter_modules'):
116 117 118 119
        return []
    return importer.iter_modules(prefix)


120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162
# Implement a file walker for the normal importlib path hook
def _iter_file_finder_modules(importer, prefix=''):
    if importer.path is None or not os.path.isdir(importer.path):
        return

    yielded = {}
    import inspect
    try:
        filenames = os.listdir(importer.path)
    except OSError:
        # ignore unreadable directories like import does
        filenames = []
    filenames.sort()  # handle packages before same-named modules

    for fn in filenames:
        modname = inspect.getmodulename(fn)
        if modname=='__init__' or modname in yielded:
            continue

        path = os.path.join(importer.path, fn)
        ispkg = False

        if not modname and os.path.isdir(path) and '.' not in fn:
            modname = fn
            try:
                dircontents = os.listdir(path)
            except OSError:
                # ignore unreadable directories like import does
                dircontents = []
            for fn in dircontents:
                subname = inspect.getmodulename(fn)
                if subname=='__init__':
                    ispkg = True
                    break
            else:
                continue    # not a package

        if modname and '.' not in modname:
            yielded[modname] = 1
            yield prefix + modname, ispkg

iter_importer_modules.register(
    importlib.machinery.FileFinder, _iter_file_finder_modules)
163

164 165 166 167 168 169 170

def _import_imp():
    global imp
    with warnings.catch_warnings():
        warnings.simplefilter('ignore', PendingDeprecationWarning)
        imp = importlib.import_module('imp')

171 172 173 174 175 176 177 178 179 180 181 182
class ImpImporter:
    """PEP 302 Importer that wraps Python's "classic" import algorithm

    ImpImporter(dirname) produces a PEP 302 importer that searches that
    directory.  ImpImporter(None) produces a PEP 302 importer that searches
    the current sys.path, plus any modules that are frozen or built-in.

    Note that ImpImporter does not currently support being used by placement
    on sys.meta_path.
    """

    def __init__(self, path=None):
183 184
        global imp
        warnings.warn("This emulation is deprecated, use 'importlib' instead",
185
             DeprecationWarning)
186
        _import_imp()
187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209
        self.path = path

    def find_module(self, fullname, path=None):
        # Note: we ignore 'path' argument since it is only used via meta_path
        subname = fullname.split(".")[-1]
        if subname != fullname and self.path is None:
            return None
        if self.path is None:
            path = None
        else:
            path = [os.path.realpath(self.path)]
        try:
            file, filename, etc = imp.find_module(subname, path)
        except ImportError:
            return None
        return ImpLoader(fullname, file, filename, etc)

    def iter_modules(self, prefix=''):
        if self.path is None or not os.path.isdir(self.path):
            return

        yielded = {}
        import inspect
210 211 212 213 214
        try:
            filenames = os.listdir(self.path)
        except OSError:
            # ignore unreadable directories like import does
            filenames = []
215 216 217 218 219 220 221 222 223 224 225 226
        filenames.sort()  # handle packages before same-named modules

        for fn in filenames:
            modname = inspect.getmodulename(fn)
            if modname=='__init__' or modname in yielded:
                continue

            path = os.path.join(self.path, fn)
            ispkg = False

            if not modname and os.path.isdir(path) and '.' not in fn:
                modname = fn
227 228 229 230 231 232
                try:
                    dircontents = os.listdir(path)
                except OSError:
                    # ignore unreadable directories like import does
                    dircontents = []
                for fn in dircontents:
233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250
                    subname = inspect.getmodulename(fn)
                    if subname=='__init__':
                        ispkg = True
                        break
                else:
                    continue    # not a package

            if modname and '.' not in modname:
                yielded[modname] = 1
                yield prefix + modname, ispkg


class ImpLoader:
    """PEP 302 Loader that wraps Python's "classic" import algorithm
    """
    code = source = None

    def __init__(self, fullname, file, filename, etc):
251 252 253
        warnings.warn("This emulation is deprecated, use 'importlib' instead",
                      DeprecationWarning)
        _import_imp()
254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270
        self.file = file
        self.filename = filename
        self.fullname = fullname
        self.etc = etc

    def load_module(self, fullname):
        self._reopen()
        try:
            mod = imp.load_module(fullname, self.file, self.filename, self.etc)
        finally:
            if self.file:
                self.file.close()
        # Note: we don't set __loader__ because we want the module to look
        # normal; i.e. this is just a wrapper for standard import machinery
        return mod

    def get_data(self, pathname):
271 272
        with open(pathname, "rb") as file:
            return file.read()
273 274 275

    def _reopen(self):
        if self.file and self.file.closed:
276
            mod_type = self.etc[2]
277
            if mod_type==imp.PY_SOURCE:
278
                self.file = open(self.filename, 'r')
279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322
            elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION):
                self.file = open(self.filename, 'rb')

    def _fix_name(self, fullname):
        if fullname is None:
            fullname = self.fullname
        elif fullname != self.fullname:
            raise ImportError("Loader for module %s cannot handle "
                              "module %s" % (self.fullname, fullname))
        return fullname

    def is_package(self, fullname):
        fullname = self._fix_name(fullname)
        return self.etc[2]==imp.PKG_DIRECTORY

    def get_code(self, fullname=None):
        fullname = self._fix_name(fullname)
        if self.code is None:
            mod_type = self.etc[2]
            if mod_type==imp.PY_SOURCE:
                source = self.get_source(fullname)
                self.code = compile(source, self.filename, 'exec')
            elif mod_type==imp.PY_COMPILED:
                self._reopen()
                try:
                    self.code = read_code(self.file)
                finally:
                    self.file.close()
            elif mod_type==imp.PKG_DIRECTORY:
                self.code = self._get_delegate().get_code()
        return self.code

    def get_source(self, fullname=None):
        fullname = self._fix_name(fullname)
        if self.source is None:
            mod_type = self.etc[2]
            if mod_type==imp.PY_SOURCE:
                self._reopen()
                try:
                    self.source = self.file.read()
                finally:
                    self.file.close()
            elif mod_type==imp.PY_COMPILED:
                if os.path.exists(self.filename[:-1]):
323 324
                    with open(self.filename[:-1], 'r') as f:
                        self.source = f.read()
325 326 327 328 329 330 331 332 333 334 335
            elif mod_type==imp.PKG_DIRECTORY:
                self.source = self._get_delegate().get_source()
        return self.source


    def _get_delegate(self):
        return ImpImporter(self.filename).find_module('__init__')

    def get_filename(self, fullname=None):
        fullname = self._fix_name(fullname)
        mod_type = self.etc[2]
336
        if mod_type==imp.PKG_DIRECTORY:
337
            return self._get_delegate().get_filename()
338
        elif mod_type in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION):
339 340 341 342 343 344 345 346 347
            return self.filename
        return None


try:
    import zipimport
    from zipimport import zipimporter

    def iter_zipimport_modules(importer, prefix=''):
348
        dirlist = sorted(zipimport._zip_directory_cache[importer.archive])
349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395
        _prefix = importer.prefix
        plen = len(_prefix)
        yielded = {}
        import inspect
        for fn in dirlist:
            if not fn.startswith(_prefix):
                continue

            fn = fn[plen:].split(os.sep)

            if len(fn)==2 and fn[1].startswith('__init__.py'):
                if fn[0] not in yielded:
                    yielded[fn[0]] = 1
                    yield fn[0], True

            if len(fn)!=1:
                continue

            modname = inspect.getmodulename(fn[0])
            if modname=='__init__':
                continue

            if modname and '.' not in modname and modname not in yielded:
                yielded[modname] = 1
                yield prefix + modname, False

    iter_importer_modules.register(zipimporter, iter_zipimport_modules)

except ImportError:
    pass


def get_importer(path_item):
    """Retrieve a PEP 302 importer for the given path item

    The returned importer is cached in sys.path_importer_cache
    if it was newly created by a path hook.

    The cache (or part of it) can be cleared manually if a
    rescan of sys.path_hooks is necessary.
    """
    try:
        importer = sys.path_importer_cache[path_item]
    except KeyError:
        for path_hook in sys.path_hooks:
            try:
                importer = path_hook(path_item)
396
                sys.path_importer_cache.setdefault(path_item, importer)
397 398 399 400
                break
            except ImportError:
                pass
        else:
401
            importer = None
402 403 404 405 406 407 408
    return importer


def iter_importers(fullname=""):
    """Yield PEP 302 importers for the given module name

    If fullname contains a '.', the importers will be for the package
409 410
    containing fullname, otherwise they will be all registered top level
    importers (i.e. those on both sys.meta_path and sys.path_hooks).
411

412 413
    If the named module is in a package, that package is imported as a side
    effect of invoking this function.
414

415
    If no module name is specified, all top level importers are produced.
416 417
    """
    if fullname.startswith('.'):
418 419
        msg = "Relative module name {!r} not supported".format(fullname)
        raise ImportError(msg)
420 421
    if '.' in fullname:
        # Get the containing package's __path__
422
        pkg_name = fullname.rpartition(".")[0]
423 424
        pkg = importlib.import_module(pkg_name)
        path = getattr(pkg, '__path__', None)
425 426
        if path is None:
            return
427
    else:
428
        yield from sys.meta_path
429 430 431 432
        path = sys.path
    for item in path:
        yield get_importer(item)

433

434 435 436
def get_loader(module_or_name):
    """Get a PEP 302 "loader" object for module_or_name

437
    Returns None if the module cannot be found or imported.
438 439 440 441 442 443 444
    If the named module is not already imported, its containing package
    (if any) is imported, in order to establish the package __path__.
    """
    if module_or_name in sys.modules:
        module_or_name = sys.modules[module_or_name]
    if isinstance(module_or_name, ModuleType):
        module = module_or_name
445
        loader = getattr(module, '__loader__', None)
446 447 448 449 450 451 452
        if loader is not None:
            return loader
        fullname = module.__name__
    else:
        fullname = module_or_name
    return find_loader(fullname)

453

454 455 456
def find_loader(fullname):
    """Find a PEP 302 "loader" object for fullname

457 458 459 460
    This is s convenience wrapper around :func:`importlib.find_loader` that
    sets the *path* argument correctly when searching for submodules, and
    also ensures parent packages (if any) are imported before searching for
    submodules.
461
    """
462 463 464 465 466 467 468 469 470 471 472 473 474 475
    if fullname.startswith('.'):
        msg = "Relative module name {!r} not supported".format(fullname)
        raise ImportError(msg)
    path = None
    pkg_name = fullname.rpartition(".")[0]
    if pkg_name:
        pkg = importlib.import_module(pkg_name)
        path = getattr(pkg, "__path__", None)
        if path is None:
            return None
    try:
        return importlib.find_loader(fullname, path)
    except (ImportError, AttributeError, TypeError, ValueError) as ex:
        # This hack fixes an impedance mismatch between pkgutil and
476
        # importlib, where the latter raises other errors for cases where
477
        # pkgutil previously raised ImportError
478 479
        msg = "Error while finding loader for {!r} ({}: {})"
        raise ImportError(msg.format(fullname, type(ex), ex)) from ex
480

481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519

def extend_path(path, name):
    """Extend a package's path.

    Intended use is to place the following code in a package's __init__.py:

        from pkgutil import extend_path
        __path__ = extend_path(__path__, __name__)

    This will add to the package's __path__ all subdirectories of
    directories on sys.path named after the package.  This is useful
    if one wants to distribute different parts of a single logical
    package as multiple directories.

    It also looks for *.pkg files beginning where * matches the name
    argument.  This feature is similar to *.pth files (see site.py),
    except that it doesn't special-case lines starting with 'import'.
    A *.pkg file is trusted at face value: apart from checking for
    duplicates, all entries found in a *.pkg file are added to the
    path, regardless of whether they are exist the filesystem.  (This
    is a feature.)

    If the input path is not a list (as is the case for frozen
    packages) it is returned unchanged.  The input path is not
    modified; an extended copy is returned.  Items are only appended
    to the copy at the end.

    It is assumed that sys.path is a sequence.  Items of sys.path that
    are not (unicode or 8-bit) strings referring to existing
    directories are ignored.  Unicode items of sys.path that cause
    errors when used as filenames may cause this function to raise an
    exception (in line with os.path.isdir() behavior).
    """

    if not isinstance(path, list):
        # This could happen e.g. when this is called from inside a
        # frozen package.  Return the path unchanged in that case.
        return path

Skip Montanaro's avatar
Skip Montanaro committed
520
    sname_pkg = name + ".pkg"
521 522 523

    path = path[:] # Start with a copy of the existing path

524 525 526 527 528 529 530 531 532 533 534 535
    parent_package, _, final_name = name.rpartition('.')
    if parent_package:
        try:
            search_path = sys.modules[parent_package].__path__
        except (KeyError, AttributeError):
            # We can't do anything: find_loader() returns None when
            # passed a dotted name.
            return path
    else:
        search_path = sys.path

    for dir in search_path:
536
        if not isinstance(dir, str):
537
            continue
538 539 540 541 542

        finder = get_importer(dir)
        if finder is not None:
            # Is this finder PEP 420 compliant?
            if hasattr(finder, 'find_loader'):
543
                loader, portions = finder.find_loader(final_name)
544 545 546 547 548 549 550 551 552 553 554
            else:
                # No, no need to call it
                loader = None
                portions = []

            for portion in portions:
                # XXX This may still add duplicate entries to path on
                # case-insensitive filesystems
                if portion not in path:
                    path.append(portion)

555 556 557 558 559 560
        # XXX Is this the right thing for subpackages like zope.app?
        # It looks for a file named "zope.app.pkg"
        pkgfile = os.path.join(dir, sname_pkg)
        if os.path.isfile(pkgfile):
            try:
                f = open(pkgfile)
561
            except OSError as msg:
562 563 564
                sys.stderr.write("Can't open %s: %s\n" %
                                 (pkgfile, msg))
            else:
565 566 567 568 569 570
                with f:
                    for line in f:
                        line = line.rstrip('\n')
                        if not line or line.startswith('#'):
                            continue
                        path.append(line) # Don't check for existence!
571 572

    return path
Christian Heimes's avatar
Christian Heimes committed
573

574

Christian Heimes's avatar
Christian Heimes committed
575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596
def get_data(package, resource):
    """Get a resource from a package.

    This is a wrapper round the PEP 302 loader get_data API. The package
    argument should be the name of a package, in standard module format
    (foo.bar). The resource argument should be in the form of a relative
    filename, using '/' as the path separator. The parent directory name '..'
    is not allowed, and nor is a rooted name (starting with a '/').

    The function returns a binary string, which is the contents of the
    specified resource.

    For packages located in the filesystem, which have already been imported,
    this is the rough equivalent of

        d = os.path.dirname(sys.modules[package].__file__)
        data = open(os.path.join(d, resource), 'rb').read()

    If the package cannot be located or loaded, or it uses a PEP 302 loader
    which does not support get_data(), then None is returned.
    """

597 598 599 600
    spec = importlib.find_spec(package)
    if spec is None:
        return None
    loader = spec.loader
Christian Heimes's avatar
Christian Heimes committed
601 602
    if loader is None or not hasattr(loader, 'get_data'):
        return None
603 604 605
    # XXX needs test
    mod = (sys.modules.get(package) or
           importlib._bootstrap._SpecMethods(spec).load())
Christian Heimes's avatar
Christian Heimes committed
606 607 608 609 610 611 612 613 614 615
    if mod is None or not hasattr(mod, '__file__'):
        return None

    # Modify the resource name to be compatible with the loader.get_data
    # signature - an os.path format "filename" starting with the dirname of
    # the package's __file__
    parts = resource.split('/')
    parts.insert(0, os.path.dirname(mod.__file__))
    resource_name = os.path.join(*parts)
    return loader.get_data(resource_name)