install.py 17.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
"""Building blocks for installers.

When used as a script, this module installs a release thanks to info
obtained from an index (e.g. PyPI), with dependencies.

This is a higher-level module built on packaging.database and
packaging.pypi.
"""
import os
import sys
import stat
import errno
import shutil
import logging
import tempfile
16
from sysconfig import get_config_var, get_path, is_python_build
17 18 19 20 21 22 23 24 25 26 27 28 29

from packaging import logger
from packaging.dist import Distribution
from packaging.util import (_is_archive_file, ask, get_install_method,
                            egginfo_to_distinfo)
from packaging.pypi import wrapper
from packaging.version import get_version_predicate
from packaging.database import get_distributions, get_distribution
from packaging.depgraph import generate_graph

from packaging.errors import (PackagingError, InstallationException,
                              InstallationConflict, CCompilerError)
from packaging.pypi.errors import ProjectNotFound, ReleaseNotFound
30 31
from packaging import database

32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47

__all__ = ['install_dists', 'install_from_infos', 'get_infos', 'remove',
           'install', 'install_local_project']


def _move_files(files, destination):
    """Move the list of files in the destination folder, keeping the same
    structure.

    Return a list of tuple (old, new) emplacement of files

    :param files: a list of files to move.
    :param destination: the destination directory to put on the files.
    """

    for old in files:
48 49
        filename = os.path.split(old)[-1]
        new = os.path.join(destination, filename)
50 51 52 53
        # try to make the paths.
        try:
            os.makedirs(os.path.dirname(new))
        except OSError as e:
54 55
            if e.errno != errno.EEXIST:
                raise
56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73
        os.rename(old, new)
        yield old, new


def _run_distutils_install(path):
    # backward compat: using setuptools or plain-distutils
    cmd = '%s setup.py install --record=%s'
    record_file = os.path.join(path, 'RECORD')
    os.system(cmd % (sys.executable, record_file))
    if not os.path.exists(record_file):
        raise ValueError('failed to install')
    else:
        egginfo_to_distinfo(record_file, remove_egginfo=True)


def _run_setuptools_install(path):
    cmd = '%s setup.py install --record=%s --single-version-externally-managed'
    record_file = os.path.join(path, 'RECORD')
74

75 76 77 78 79 80 81 82 83 84 85 86 87
    os.system(cmd % (sys.executable, record_file))
    if not os.path.exists(record_file):
        raise ValueError('failed to install')
    else:
        egginfo_to_distinfo(record_file, remove_egginfo=True)


def _run_packaging_install(path):
    # XXX check for a valid setup.cfg?
    dist = Distribution()
    dist.parse_config_files()
    try:
        dist.run_command('install_dist')
Éric Araujo's avatar
Éric Araujo committed
88
        name = dist.metadata['Name']
89
        return database.get_distribution(name) is not None
90
    except (IOError, os.error, PackagingError, CCompilerError) as msg:
91
        raise ValueError("Failed to install, " + str(msg))
92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116


def _install_dist(dist, path):
    """Install a distribution into a path.

    This:

    * unpack the distribution
    * copy the files in "path"
    * determine if the distribution is packaging or distutils1.
    """
    where = dist.unpack()

    if where is None:
        raise ValueError('Cannot locate the unpacked archive')

    return _run_install_from_archive(where)


def install_local_project(path):
    """Install a distribution from a source directory.

    If the source directory contains a setup.py install using distutils1.
    If a setup.cfg is found, install using the install_dist command.

117
    Returns True on success, False on Failure.
118 119 120
    """
    path = os.path.abspath(path)
    if os.path.isdir(path):
Éric Araujo's avatar
Éric Araujo committed
121
        logger.info('Installing from source directory: %r', path)
122
        return _run_install_from_dir(path)
123
    elif _is_archive_file(path):
Éric Araujo's avatar
Éric Araujo committed
124
        logger.info('Installing from archive: %r', path)
125
        _unpacked_dir = tempfile.mkdtemp()
126 127 128 129 130
        try:
            shutil.unpack_archive(path, _unpacked_dir)
            return _run_install_from_archive(_unpacked_dir)
        finally:
            shutil.rmtree(_unpacked_dir)
131
    else:
Éric Araujo's avatar
Éric Araujo committed
132
        logger.warning('No project to install.')
133
        return False
134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158


def _run_install_from_archive(source_dir):
    # XXX need a better way
    for item in os.listdir(source_dir):
        fullpath = os.path.join(source_dir, item)
        if os.path.isdir(fullpath):
            source_dir = fullpath
            break
    return _run_install_from_dir(source_dir)


install_methods = {
    'packaging': _run_packaging_install,
    'setuptools': _run_setuptools_install,
    'distutils': _run_distutils_install}


def _run_install_from_dir(source_dir):
    old_dir = os.getcwd()
    os.chdir(source_dir)
    install_method = get_install_method(source_dir)
    func = install_methods[install_method]
    try:
        func = install_methods[install_method]
159 160 161 162 163 164 165
        try:
            func(source_dir)
            return True
        except ValueError as err:
            # failed to install
            logger.info(str(err))
            return False
166 167 168 169
    finally:
        os.chdir(old_dir)


170
def install_dists(dists, path, paths=None):
171 172 173 174 175 176 177 178 179 180 181 182 183 184
    """Install all distributions provided in dists, with the given prefix.

    If an error occurs while installing one of the distributions, uninstall all
    the installed distribution (in the context if this function).

    Return a list of installed dists.

    :param dists: distributions to install
    :param path: base path to install distribution in
    :param paths: list of paths (defaults to sys.path) to look for info
    """

    installed_dists = []
    for dist in dists:
185
        logger.info('Installing %r %s...', dist.name, dist.version)
186 187 188 189
        try:
            _install_dist(dist, path)
            installed_dists.append(dist)
        except Exception as e:
190
            logger.info('Failed: %s', e)
191 192 193

            # reverting
            for installed_dist in installed_dists:
Éric Araujo's avatar
Éric Araujo committed
194
                logger.info('Reverting %r', installed_dist)
195
                remove(installed_dist.name, paths)
196 197 198 199 200
            raise e
    return installed_dists


def install_from_infos(install_path=None, install=[], remove=[], conflicts=[],
201
                       paths=None):
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302
    """Install and remove the given distributions.

    The function signature is made to be compatible with the one of get_infos.
    The aim of this script is to povide a way to install/remove what's asked,
    and to rollback if needed.

    So, it's not possible to be in an inconsistant state, it could be either
    installed, either uninstalled, not half-installed.

    The process follow those steps:

        1. Move all distributions that will be removed in a temporary location
        2. Install all the distributions that will be installed in a temp. loc.
        3. If the installation fails, rollback (eg. move back) those
           distributions, or remove what have been installed.
        4. Else, move the distributions to the right locations, and remove for
           real the distributions thats need to be removed.

    :param install_path: the installation path where we want to install the
                         distributions.
    :param install: list of distributions that will be installed; install_path
                    must be provided if this list is not empty.
    :param remove: list of distributions that will be removed.
    :param conflicts: list of conflicting distributions, eg. that will be in
                      conflict once the install and remove distribution will be
                      processed.
    :param paths: list of paths (defaults to sys.path) to look for info
    """
    # first of all, if we have conflicts, stop here.
    if conflicts:
        raise InstallationConflict(conflicts)

    if install and not install_path:
        raise ValueError("Distributions are to be installed but `install_path`"
                         " is not provided.")

    # before removing the files, we will start by moving them away
    # then, if any error occurs, we could replace them in the good place.
    temp_files = {}  # contains lists of {dist: (old, new)} paths
    temp_dir = None
    if remove:
        temp_dir = tempfile.mkdtemp()
        for dist in remove:
            files = dist.list_installed_files()
            temp_files[dist] = _move_files(files, temp_dir)
    try:
        if install:
            install_dists(install, install_path, paths)
    except:
        # if an error occurs, put back the files in the right place.
        for files in temp_files.values():
            for old, new in files:
                shutil.move(new, old)
        if temp_dir:
            shutil.rmtree(temp_dir)
        # now re-raising
        raise

    # we can remove them for good
    for files in temp_files.values():
        for old, new in files:
            os.remove(new)
    if temp_dir:
        shutil.rmtree(temp_dir)


def _get_setuptools_deps(release):
    # NotImplementedError
    pass


def get_infos(requirements, index=None, installed=None, prefer_final=True):
    """Return the informations on what's going to be installed and upgraded.

    :param requirements: is a *string* containing the requirements for this
                         project (for instance "FooBar 1.1" or "BarBaz (<1.2)")
    :param index: If an index is specified, use this one, otherwise, use
                  :class index.ClientWrapper: to get project metadatas.
    :param installed: a list of already installed distributions.
    :param prefer_final: when picking up the releases, prefer a "final" one
                         over a beta/alpha/etc one.

    The results are returned in a dict, containing all the operations
    needed to install the given requirements::

        >>> get_install_info("FooBar (<=1.2)")
        {'install': [<FooBar 1.1>], 'remove': [], 'conflict': []}

    Conflict contains all the conflicting distributions, if there is a
    conflict.
    """
    # this function does several things:
    # 1. get a release specified by the requirements
    # 2. gather its metadata, using setuptools compatibility if needed
    # 3. compare this tree with what is currently installed on the system,
    #    return the requirements of what is missing
    # 4. do that recursively and merge back the results
    # 5. return a dict containing information about what is needed to install
    #    or remove

    if not installed:
303
        logger.debug('Reading installed distributions')
304 305 306 307 308 309 310 311 312 313 314 315 316
        installed = list(get_distributions(use_egg_info=True))

    infos = {'install': [], 'remove': [], 'conflict': []}
    # Is a compatible version of the project already installed ?
    predicate = get_version_predicate(requirements)
    found = False

    # check that the project isn't already installed
    for installed_project in installed:
        # is it a compatible project ?
        if predicate.name.lower() != installed_project.name.lower():
            continue
        found = True
Éric Araujo's avatar
Éric Araujo committed
317 318
        logger.info('Found %r %s', installed_project.name,
                    installed_project.version)
319 320 321

        # if we already have something installed, check it matches the
        # requirements
Éric Araujo's avatar
Éric Araujo committed
322
        if predicate.match(installed_project.version):
323 324 325 326
            return infos
        break

    if not found:
327
        logger.debug('Project not installed')
328 329 330 331 332 333 334 335 336 337 338

    if not index:
        index = wrapper.ClientWrapper()

    if not installed:
        installed = get_distributions(use_egg_info=True)

    # Get all the releases that match the requirements
    try:
        release = index.get_release(requirements)
    except (ReleaseNotFound, ProjectNotFound):
Éric Araujo's avatar
Éric Araujo committed
339
        raise InstallationException('Release not found: %r' % requirements)
340 341

    if release is None:
342
        logger.info('Could not find a matching project')
343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358
        return infos

    metadata = release.fetch_metadata()

    # we need to build setuptools deps if any
    if 'requires_dist' not in metadata:
        metadata['requires_dist'] = _get_setuptools_deps(release)

    # build the dependency graph with local and required dependencies
    dists = list(installed)
    dists.append(release)
    depgraph = generate_graph(dists)

    # Get what the missing deps are
    dists = depgraph.missing[release]
    if dists:
359
        logger.info("Missing dependencies found, retrieving metadata")
360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381
        # we have missing deps
        for dist in dists:
            _update_infos(infos, get_infos(dist, index, installed))

    # Fill in the infos
    existing = [d for d in installed if d.name == release.name]
    if existing:
        infos['remove'].append(existing[0])
        infos['conflict'].extend(depgraph.reverse_list[existing[0]])
    infos['install'].append(release)
    return infos


def _update_infos(infos, new_infos):
    """extends the lists contained in the `info` dict with those contained
    in the `new_info` one
    """
    for key, value in infos.items():
        if key in new_infos:
            infos[key].extend(new_infos[key])


382
def remove(project_name, paths=None, auto_confirm=True):
383 384 385 386
    """Removes a single project from the installation.

    Returns True on success
    """
387 388
    dist = get_distribution(project_name, use_egg_info=True, paths=paths)
    if dist is None:
Éric Araujo's avatar
Éric Araujo committed
389
        raise PackagingError('Distribution %r not found' % project_name)
390 391 392 393
    files = dist.list_installed_files(local=True)
    rmdirs = []
    rmfiles = []
    tmp = tempfile.mkdtemp(prefix=project_name + '-uninstall')
394 395 396 397 398 399 400 401 402 403

    def _move_file(source, target):
        try:
            os.rename(source, target)
        except OSError as err:
            return err
        return None

    success = True
    error = None
404 405 406 407 408 409
    try:
        for file_, md5, size in files:
            if os.path.isfile(file_):
                dirname, filename = os.path.split(file_)
                tmpfile = os.path.join(tmp, filename)
                try:
410 411 412 413
                    error = _move_file(file_, tmpfile)
                    if error is not None:
                        success = False
                        break
414 415 416 417 418 419 420 421 422 423
                finally:
                    if not os.path.isfile(file_):
                        os.rename(tmpfile, file_)
                if file_ not in rmfiles:
                    rmfiles.append(file_)
                if dirname not in rmdirs:
                    rmdirs.append(dirname)
    finally:
        shutil.rmtree(tmp)

424 425
    if not success:
        logger.info('%r cannot be removed.', project_name)
Éric Araujo's avatar
Éric Araujo committed
426
        logger.info('Error: %s', error)
427 428
        return False

429
    logger.info('Removing %r: ', project_name)
430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471

    for file_ in rmfiles:
        logger.info('  %s', file_)

    # Taken from the pip project
    if auto_confirm:
        response = 'y'
    else:
        response = ask('Proceed (y/n)? ', ('y', 'n'))

    if response == 'y':
        file_count = 0
        for file_ in rmfiles:
            os.remove(file_)
            file_count += 1

        dir_count = 0
        for dirname in rmdirs:
            if not os.path.exists(dirname):
                # could
                continue

            files_count = 0
            for root, dir, files in os.walk(dirname):
                files_count += len(files)

            if files_count > 0:
                # XXX Warning
                continue

            # empty dirs with only empty dirs
            if os.stat(dirname).st_mode & stat.S_IWUSR:
                # XXX Add a callable in shutil.rmtree to count
                # the number of deleted elements
                shutil.rmtree(dirname)
                dir_count += 1

        # removing the top path
        # XXX count it ?
        if os.path.exists(dist.path):
            shutil.rmtree(dist.path)

472
        logger.info('Success: removed %d files and %d dirs',
473 474
                    file_count, dir_count)

475 476
    return True

477 478

def install(project):
479 480 481 482
    """Installs a project.

    Returns True on success, False on failure
    """
483 484 485 486 487 488 489 490 491
    if is_python_build():
        # Python would try to install into the site-packages directory under
        # $PREFIX, but when running from an uninstalled code checkout we don't
        # want to create directories under the installation root
        message = ('installing third-party projects from an uninstalled '
                   'Python is not supported')
        logger.error(message)
        return False

492 493
    logger.info('Checking the installation location...')
    purelib_path = get_path('purelib')
494

495 496 497 498 499 500
    # trying to write a file there
    try:
        with tempfile.NamedTemporaryFile(suffix=project,
                                         dir=purelib_path) as testfile:
            testfile.write(b'test')
    except OSError:
501 502 503
        # FIXME this should check the errno, or be removed altogether (race
        # condition: the directory permissions could be changed between here
        # and the actual install)
504 505 506 507
        logger.info('Unable to write in "%s". Do you have the permissions ?'
                    % purelib_path)
        return False

508
    logger.info('Getting information about %r...', project)
509 510 511
    try:
        info = get_infos(project)
    except InstallationException:
512
        logger.info('Cound not find %r', project)
513
        return False
514 515

    if info['install'] == []:
516
        logger.info('Nothing to install')
517
        return False
518 519 520 521 522 523 524 525

    install_path = get_config_var('base')
    try:
        install_from_infos(install_path,
                           info['install'], info['remove'], info['conflict'])

    except InstallationConflict as e:
        if logger.isEnabledFor(logging.INFO):
Éric Araujo's avatar
Éric Araujo committed
526
            projects = ('%r %s' % (p.name, p.version) for p in e.args[0])
527 528
            logger.info('%r conflicts with %s', project, ','.join(projects))

529
    return True