util.py 17 KB
Newer Older
1 2
"""distutils.util

3
Miscellaneous utility functions -- anything that doesn't fit into
Greg Ward's avatar
Greg Ward committed
4 5
one of the other *util.py modules.
"""
6 7 8

# created 1999/03/08, Greg Ward

9
__revision__ = "$Id$"
10

11 12 13
import sys, os, string, re
from distutils.errors import DistutilsPlatformError
from distutils.dep_util import newer
14
from distutils.spawn import spawn
15

16

17
def get_platform ():
18 19 20 21 22 23 24 25 26 27 28 29 30 31 32
    """Return a string that identifies the current platform.  This is used
    mainly to distinguish platform-specific build directories and
    platform-specific built distributions.  Typically includes the OS name
    and version and the architecture (as supplied by 'os.uname()'),
    although the exact information included depends on the OS; eg. for IRIX
    the architecture isn't particularly important (IRIX only runs on SGI
    hardware), but for Linux the kernel version isn't particularly
    important.

    Examples of returned values:
       linux-i586
       linux-alpha (?)
       solaris-2.6-sun4u
       irix-5.3
       irix64-6.2
Fred Drake's avatar
Fred Drake committed
33

34
    For non-POSIX platforms, currently just returns 'sys.platform'.
35
    """
36
    if os.name != "posix" or not hasattr(os, 'uname'):
37 38 39 40 41 42 43
        # XXX what about the architecture? NT is Intel or Alpha,
        # Mac OS is M68k or PPC, etc.
        return sys.platform

    # Try to distinguish various flavours of Unix

    (osname, host, release, version, machine) = os.uname()
44 45 46

    # Convert the OS name to lowercase and remove '/' characters
    # (to accommodate BSD/OS)
Fred Drake's avatar
Fred Drake committed
47
    osname = string.lower(osname)
48
    osname = string.replace(osname, '/', '')
Fred Drake's avatar
Fred Drake committed
49

50 51 52 53 54 55 56 57 58 59 60 61
    if osname[:5] == "linux":
        # At least on Linux/Intel, 'machine' is the processor --
        # i386, etc.
        # XXX what about Alpha, SPARC, etc?
        return  "%s-%s" % (osname, machine)
    elif osname[:5] == "sunos":
        if release[0] >= "5":           # SunOS 5 == Solaris 2
            osname = "solaris"
            release = "%d.%s" % (int(release[0]) - 3, release[2:])
        # fall through to standard osname-release-machine representation
    elif osname[:4] == "irix":              # could be "irix64"!
        return "%s-%s" % (osname, release)
Fred Drake's avatar
Fred Drake committed
62
    elif osname[:3] == "aix":
63
        return "%s-%s.%s" % (osname, version, release)
64
    elif osname[:6] == "cygwin":
65
        osname = "cygwin"
66 67 68 69
        rel_re = re.compile (r'[\d.]+')
        m = rel_re.match(release)
        if m:
            release = m.group()
Fred Drake's avatar
Fred Drake committed
70

71 72 73
    return "%s-%s-%s" % (osname, release, machine)

# get_platform ()
74 75


76
def convert_path (pathname):
Greg Ward's avatar
Greg Ward committed
77 78 79 80 81
    """Return 'pathname' as a name that will work on the native filesystem,
    i.e. split it on '/' and put it back together again using the current
    directory separator.  Needed because filenames in the setup script are
    always supplied in Unix style, and have to be converted to the local
    convention before we can actually use them in the filesystem.  Raises
Greg Ward's avatar
Greg Ward committed
82 83
    ValueError on non-Unix-ish systems if 'pathname' either starts or
    ends with a slash.
Greg Ward's avatar
Greg Ward committed
84
    """
85 86
    if os.sep == '/':
        return pathname
87
    if pathname and pathname[0] == '/':
88
        raise ValueError, "path '%s' cannot be absolute" % pathname
89
    if pathname and pathname[-1] == '/':
90
        raise ValueError, "path '%s' cannot end with '/'" % pathname
91 92

    paths = string.split(pathname, '/')
93 94 95 96
    while '.' in paths:
        paths.remove('.')
    if not paths:
        return os.curdir
97
    return apply(os.path.join, paths)
98

99
# convert_path ()
100 101


102 103 104 105
def change_root (new_root, pathname):
    """Return 'pathname' with 'new_root' prepended.  If 'pathname' is
    relative, this is equivalent to "os.path.join(new_root,pathname)".
    Otherwise, it requires making 'pathname' relative and then joining the
106 107 108
    two, which is tricky on DOS/Windows and Mac OS.
    """
    if os.name == 'posix':
109 110
        if not os.path.isabs(pathname):
            return os.path.join(new_root, pathname)
111
        else:
112
            return os.path.join(new_root, pathname[1:])
113 114

    elif os.name == 'nt':
115
        (drive, path) = os.path.splitdrive(pathname)
116 117
        if path[0] == '\\':
            path = path[1:]
118
        return os.path.join(new_root, path)
119

120 121 122 123 124 125
    elif os.name == 'os2':
        (drive, path) = os.path.splitdrive(pathname)
        if path[0] == os.sep:
            path = path[1:]
        return os.path.join(new_root, path)

126
    elif os.name == 'mac':
127 128 129 130 131 132 133
        if not os.path.isabs(pathname):
            return os.path.join(new_root, pathname)
        else:
            # Chop off volume name from start of path
            elements = string.split(pathname, ":", 1)
            pathname = ":" + elements[1]
            return os.path.join(new_root, pathname)
134 135 136 137 138 139

    else:
        raise DistutilsPlatformError, \
              "nothing known about platform '%s'" % os.name


140 141
_environ_checked = 0
def check_environ ():
142
    """Ensure that 'os.environ' has all the environment variables we
Greg Ward's avatar
Greg Ward committed
143 144 145 146 147
    guarantee that users can use in config files, command-line options,
    etc.  Currently this includes:
      HOME - user's home directory (Unix only)
      PLAT - description of the current platform, including hardware
             and OS (see 'get_platform()')
148
    """
149 150 151 152
    global _environ_checked
    if _environ_checked:
        return

153 154
    if os.name == 'posix' and not os.environ.has_key('HOME'):
        import pwd
155
        os.environ['HOME'] = pwd.getpwuid(os.getuid())[5]
156 157

    if not os.environ.has_key('PLAT'):
158
        os.environ['PLAT'] = get_platform()
159

160 161
    _environ_checked = 1

162

163
def subst_vars (s, local_vars):
Greg Ward's avatar
Greg Ward committed
164
    """Perform shell/Perl-style variable substitution on 'string'.  Every
Greg Ward's avatar
Greg Ward committed
165 166 167 168 169 170
    occurrence of '$' followed by a name is considered a variable, and
    variable is substituted by the value found in the 'local_vars'
    dictionary, or in 'os.environ' if it's not in 'local_vars'.
    'os.environ' is first checked/augmented to guarantee that it contains
    certain values: see 'check_environ()'.  Raise ValueError for any
    variables not found in either 'local_vars' or 'os.environ'.
Greg Ward's avatar
Greg Ward committed
171
    """
172
    check_environ()
173 174
    def _subst (match, local_vars=local_vars):
        var_name = match.group(1)
175 176
        if local_vars.has_key(var_name):
            return str(local_vars[var_name])
177 178 179
        else:
            return os.environ[var_name]

Greg Ward's avatar
Greg Ward committed
180
    try:
181
        return re.sub(r'\$([a-zA-Z_][a-zA-Z_0-9]*)', _subst, s)
Greg Ward's avatar
Greg Ward committed
182 183
    except KeyError, var:
        raise ValueError, "invalid variable '$%s'" % var
184 185

# subst_vars ()
186 187


188 189 190 191 192 193 194 195 196
def grok_environment_error (exc, prefix="error: "):
    """Generate a useful error message from an EnvironmentError (IOError or
    OSError) exception object.  Handles Python 1.5.1 and 1.5.2 styles, and
    does what it can to deal with exception objects that don't have a
    filename (which happens when the error is due to a two-file operation,
    such as 'rename()' or 'link()'.  Returns the error message as a string
    prefixed with 'prefix'.
    """
    # check for Python 1.5.2-style {IO,OS}Error exception objects
197
    if hasattr(exc, 'filename') and hasattr(exc, 'strerror'):
198 199 200 201 202 203 204 205 206 207
        if exc.filename:
            error = prefix + "%s: %s" % (exc.filename, exc.strerror)
        else:
            # two-argument functions in posix module don't
            # include the filename in the exception object!
            error = prefix + "%s" % exc.strerror
    else:
        error = prefix + str(exc[-1])

    return error
208 209 210


# Needed by 'split_quoted()'
211
_wordchars_re = re.compile(r'[^\\\'\"%s ]*' % string.whitespace)
212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240
_squote_re = re.compile(r"'(?:[^'\\]|\\.)*'")
_dquote_re = re.compile(r'"(?:[^"\\]|\\.)*"')

def split_quoted (s):
    """Split a string up according to Unix shell-like rules for quotes and
    backslashes.  In short: words are delimited by spaces, as long as those
    spaces are not escaped by a backslash, or inside a quoted string.
    Single and double quotes are equivalent, and the quote characters can
    be backslash-escaped.  The backslash is stripped from any two-character
    escape sequence, leaving only the escaped character.  The quote
    characters are stripped from any quoted string.  Returns a list of
    words.
    """

    # This is a nice algorithm for splitting up a single string, since it
    # doesn't require character-by-character examination.  It was a little
    # bit of a brain-bender to get it working right, though...

    s = string.strip(s)
    words = []
    pos = 0

    while s:
        m = _wordchars_re.match(s, pos)
        end = m.end()
        if end == len(s):
            words.append(s[:end])
            break

241
        if s[end] in string.whitespace: # unescaped, unquoted whitespace: now
242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274
            words.append(s[:end])       # we definitely have a word delimiter
            s = string.lstrip(s[end:])
            pos = 0

        elif s[end] == '\\':            # preserve whatever is being escaped;
                                        # will become part of the current word
            s = s[:end] + s[end+1:]
            pos = end+1

        else:
            if s[end] == "'":           # slurp singly-quoted string
                m = _squote_re.match(s, end)
            elif s[end] == '"':         # slurp doubly-quoted string
                m = _dquote_re.match(s, end)
            else:
                raise RuntimeError, \
                      "this can't happen (bad char '%c')" % s[end]

            if m is None:
                raise ValueError, \
                      "bad string (mismatched %s quotes?)" % s[end]

            (beg, end) = m.span()
            s = s[:beg] + s[beg+1:end-1] + s[end:]
            pos = m.end() - 2

        if pos >= len(s):
            words.append(s)
            break

    return words

# split_quoted ()
275 276 277 278 279 280 281 282 283 284 285 286 287 288


def execute (func, args, msg=None, verbose=0, dry_run=0):
    """Perform some action that affects the outside world (eg.  by writing
    to the filesystem).  Such actions are special because they are disabled
    by the 'dry_run' flag, and announce themselves if 'verbose' is true.
    This method takes care of all that bureaucracy for you; all you have to
    do is supply the function to call and an argument tuple for it (to
    embody the "external action" being performed), and an optional message
    to print.
    """
    # Generate a message if we weren't passed one
    if msg is None:
        msg = "%s%s" % (func.__name__, `args`)
Fred Drake's avatar
Fred Drake committed
289
        if msg[-2:] == ',)':        # correct for singleton tuple
290 291 292 293 294 295 296 297 298 299 300
            msg = msg[0:-2] + ')'

    # Print it if verbosity level is high enough
    if verbose:
        print msg

    # And do it, as long as we're not in dry-run mode
    if not dry_run:
        apply(func, args)

# execute()
301 302 303 304 305 306 307 308 309 310 311 312 313 314 315


def strtobool (val):
    """Convert a string representation of truth to true (1) or false (0).
    True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values
    are 'n', 'no', 'f', 'false', 'off', and '0'.  Raises ValueError if
    'val' is anything else.
    """
    val = string.lower(val)
    if val in ('y', 'yes', 't', 'true', 'on', '1'):
        return 1
    elif val in ('n', 'no', 'f', 'false', 'off', '0'):
        return 0
    else:
        raise ValueError, "invalid truth value %s" % `val`
316 317 318 319 320 321 322


def byte_compile (py_files,
                  optimize=0, force=0,
                  prefix=None, base_dir=None,
                  verbose=1, dry_run=0,
                  direct=None):
323 324 325 326
    """Byte-compile a collection of Python source files to either .pyc
    or .pyo files in the same directory.  'py_files' is a list of files
    to compile; any files that don't end in ".py" are silently skipped.
    'optimize' must be one of the following:
327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378
      0 - don't optimize (generate .pyc)
      1 - normal optimization (like "python -O")
      2 - extra optimization (like "python -OO")
    If 'force' is true, all files are recompiled regardless of
    timestamps.

    The source filename encoded in each bytecode file defaults to the
    filenames listed in 'py_files'; you can modify these with 'prefix' and
    'basedir'.  'prefix' is a string that will be stripped off of each
    source filename, and 'base_dir' is a directory name that will be
    prepended (after 'prefix' is stripped).  You can supply either or both
    (or neither) of 'prefix' and 'base_dir', as you wish.

    If 'verbose' is true, prints out a report of each file.  If 'dry_run'
    is true, doesn't actually do anything that would affect the filesystem.

    Byte-compilation is either done directly in this interpreter process
    with the standard py_compile module, or indirectly by writing a
    temporary script and executing it.  Normally, you should let
    'byte_compile()' figure out to use direct compilation or not (see
    the source for details).  The 'direct' flag is used by the script
    generated in indirect mode; unless you know what you're doing, leave
    it set to None.
    """

    # First, if the caller didn't force us into direct or indirect mode,
    # figure out which mode we should be in.  We take a conservative
    # approach: choose direct mode *only* if the current interpreter is
    # in debug mode and optimize is 0.  If we're not in debug mode (-O
    # or -OO), we don't know which level of optimization this
    # interpreter is running with, so we can't do direct
    # byte-compilation and be certain that it's the right thing.  Thus,
    # always compile indirectly if the current interpreter is in either
    # optimize mode, or if either optimization level was requested by
    # the caller.
    if direct is None:
        direct = (__debug__ and optimize == 0)

    # "Indirect" byte-compilation: write a temporary script and then
    # run it with the appropriate flags.
    if not direct:
        from tempfile import mktemp
        script_name = mktemp(".py")
        if verbose:
            print "writing byte-compilation script '%s'" % script_name
        if not dry_run:
            script = open(script_name, "w")

            script.write("""\
from distutils.util import byte_compile
files = [
""")
379 380 381 382 383 384 385 386 387 388 389 390 391 392 393

            # XXX would be nice to write absolute filenames, just for
            # safety's sake (script should be more robust in the face of
            # chdir'ing before running it).  But this requires abspath'ing
            # 'prefix' as well, and that breaks the hack in build_lib's
            # 'byte_compile()' method that carefully tacks on a trailing
            # slash (os.sep really) to make sure the prefix here is "just
            # right".  This whole prefix business is rather delicate -- the
            # problem is that it's really a directory, but I'm treating it
            # as a dumb string, so trailing slashes and so forth matter.

            #py_files = map(os.path.abspath, py_files)
            #if prefix:
            #    prefix = os.path.abspath(prefix)

394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409
            script.write(string.join(map(repr, py_files), ",\n") + "]\n")
            script.write("""
byte_compile(files, optimize=%s, force=%s,
             prefix=%s, base_dir=%s,
             verbose=%s, dry_run=0,
             direct=1)
""" % (`optimize`, `force`, `prefix`, `base_dir`, `verbose`))

            script.close()

        cmd = [sys.executable, script_name]
        if optimize == 1:
            cmd.insert(1, "-O")
        elif optimize == 2:
            cmd.insert(1, "-OO")
        spawn(cmd, verbose=verbose, dry_run=dry_run)
410 411
        execute(os.remove, (script_name,), "removing %s" % script_name,
                verbose=verbose, dry_run=dry_run)
Fred Drake's avatar
Fred Drake committed
412

413 414 415 416 417 418 419 420 421
    # "Direct" byte-compilation: use the py_compile module to compile
    # right here, right now.  Note that the script generated in indirect
    # mode simply calls 'byte_compile()' in direct mode, a weird sort of
    # cross-process recursion.  Hey, it works!
    else:
        from py_compile import compile

        for file in py_files:
            if file[-3:] != ".py":
422 423 424
                # This lets us be lazy and not filter filenames in
                # the "install_lib" command.
                continue
425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452

            # Terminology from the py_compile module:
            #   cfile - byte-compiled file
            #   dfile - purported source filename (same as 'file' by default)
            cfile = file + (__debug__ and "c" or "o")
            dfile = file
            if prefix:
                if file[:len(prefix)] != prefix:
                    raise ValueError, \
                          ("invalid prefix: filename %s doesn't start with %s"
                           % (`file`, `prefix`))
                dfile = dfile[len(prefix):]
            if base_dir:
                dfile = os.path.join(base_dir, dfile)

            cfile_base = os.path.basename(cfile)
            if direct:
                if force or newer(file, cfile):
                    if verbose:
                        print "byte-compiling %s to %s" % (file, cfile_base)
                    if not dry_run:
                        compile(file, cfile, dfile)
                else:
                    if verbose:
                        print "skipping byte-compilation of %s to %s" % \
                              (file, cfile_base)

# byte_compile ()
453 454 455

def rfc822_escape (header):
    """Return a version of the string escaped for inclusion in an
456
    RFC-822 header, by ensuring there are 8 spaces space after each newline.
457
    """
458 459 460
    lines = string.split(header, '\n')
    lines = map(string.strip, lines)
    header = string.join(lines, '\n' + 8*' ')
461
    return header