filelist.py 12.4 KB
Newer Older
1 2 3 4 5 6
"""distutils.filelist

Provides the FileList class, used for poking about the filesystem
and building lists of files.
"""

7
import os, re
8 9
import fnmatch
from distutils.util import convert_path
10
from distutils.errors import DistutilsTemplateError, DistutilsInternalError
11
from distutils import log
12 13

class FileList:
14 15 16 17 18 19 20 21 22 23 24 25 26
    """A list of files built by on exploring the filesystem and filtered by
    applying various patterns to what we find there.

    Instance attributes:
      dir
        directory from which files will be taken -- only used if
        'allfiles' not supplied to constructor
      files
        list of filenames currently being built/filtered/manipulated
      allfiles
        complete list of files under consideration (ie. without any
        filtering applied)
    """
27

28
    def __init__(self, warn=None, debug_print=None):
29 30
        # ignore argument to FileList, but keep them for backwards
        # compatibility
31 32
        self.allfiles = None
        self.files = []
33

34
    def set_allfiles(self, allfiles):
35 36
        self.allfiles = allfiles

37
    def findall(self, dir=os.curdir):
38 39
        self.allfiles = findall(dir)

40
    def debug_print(self, msg):
41 42 43
        """Print 'msg' to stdout if the global DEBUG (taken from the
        DISTUTILS_DEBUG environment variable) flag is true.
        """
44
        from distutils.debug import DEBUG
45
        if DEBUG:
46
            print(msg)
47

48 49
    # -- List-like methods ---------------------------------------------

50
    def append(self, item):
51 52
        self.files.append(item)

53
    def extend(self, items):
54 55
        self.files.extend(items)

56
    def sort(self):
57
        # Not a strict lexical sort!
58
        sortable_files = sorted(map(os.path.split, self.files))
59 60
        self.files = []
        for sort_tuple in sortable_files:
Neal Norwitz's avatar
Neal Norwitz committed
61
            self.files.append(os.path.join(*sort_tuple))
62 63 64 65


    # -- Other miscellaneous utility methods ---------------------------

66
    def remove_duplicates(self):
67
        # Assumes list has been sorted!
68 69
        for i in range(len(self.files) - 1, 0, -1):
            if self.files[i] == self.files[i - 1]:
70 71 72 73
                del self.files[i]


    # -- "File template" methods ---------------------------------------
Fred Drake's avatar
Fred Drake committed
74

75
    def _parse_template_line(self, line):
76
        words = line.split()
77 78
        action = words[0]

79 80 81 82
        patterns = dir = dir_pattern = None

        if action in ('include', 'exclude',
                      'global-include', 'global-exclude'):
83
            if len(words) < 2:
84 85
                raise DistutilsTemplateError(
                      "'%s' expects <pattern1> <pattern2> ..." % action)
86
            patterns = [convert_path(w) for w in words[1:]]
87
        elif action in ('recursive-include', 'recursive-exclude'):
88
            if len(words) < 3:
89 90
                raise DistutilsTemplateError(
                      "'%s' expects <dir> <pattern1> <pattern2> ..." % action)
91
            dir = convert_path(words[1])
92
            patterns = [convert_path(w) for w in words[2:]]
93
        elif action in ('graft', 'prune'):
94
            if len(words) != 2:
95 96
                raise DistutilsTemplateError(
                      "'%s' expects a single <dir_pattern>" % action)
97
            dir_pattern = convert_path(words[1])
98
        else:
99
            raise DistutilsTemplateError("unknown action '%s'" % action)
100

Greg Ward's avatar
Greg Ward committed
101
        return (action, patterns, dir, dir_pattern)
102

103
    def process_template_line(self, line):
104
        # Parse the line: split it up, make sure the right number of words
105
        # is there, and return the relevant words.  'action' is always
106 107 108
        # defined: it's the first word of the line.  Which of the other
        # three are defined depends on the action; it'll be either
        # patterns, (dir and patterns), or (dir_pattern).
109
        (action, patterns, dir, dir_pattern) = self._parse_template_line(line)
110 111 112

        # OK, now we know that the action is valid and we have the
        # right number of words on the line for that action -- so we
113
        # can proceed with minimal error-checking.
114
        if action == 'include':
115
            self.debug_print("include " + ' '.join(patterns))
116
            for pattern in patterns:
117
                if not self.include_pattern(pattern, anchor=1):
118 119
                    log.warn("warning: no files found matching '%s'",
                             pattern)
120 121

        elif action == 'exclude':
122
            self.debug_print("exclude " + ' '.join(patterns))
123
            for pattern in patterns:
124
                if not self.exclude_pattern(pattern, anchor=1):
125 126
                    log.warn(("warning: no previously-included files "
                              "found matching '%s'"), pattern)
127 128

        elif action == 'global-include':
129
            self.debug_print("global-include " + ' '.join(patterns))
130
            for pattern in patterns:
131
                if not self.include_pattern(pattern, anchor=0):
132
                    log.warn(("warning: no files found matching '%s' "
133
                              "anywhere in distribution"), pattern)
134 135

        elif action == 'global-exclude':
136
            self.debug_print("global-exclude " + ' '.join(patterns))
137
            for pattern in patterns:
138
                if not self.exclude_pattern(pattern, anchor=0):
139 140 141
                    log.warn(("warning: no previously-included files matching "
                              "'%s' found anywhere in distribution"),
                             pattern)
142 143 144

        elif action == 'recursive-include':
            self.debug_print("recursive-include %s %s" %
145
                             (dir, ' '.join(patterns)))
146
            for pattern in patterns:
147
                if not self.include_pattern(pattern, prefix=dir):
148
                    log.warn(("warning: no files found matching '%s' "
149
                                "under directory '%s'"),
150
                             pattern, dir)
151 152 153

        elif action == 'recursive-exclude':
            self.debug_print("recursive-exclude %s %s" %
154
                             (dir, ' '.join(patterns)))
155
            for pattern in patterns:
156
                if not self.exclude_pattern(pattern, prefix=dir):
157 158 159
                    log.warn(("warning: no previously-included files matching "
                              "'%s' found under directory '%s'"),
                             pattern, dir)
Fred Drake's avatar
Fred Drake committed
160

161 162
        elif action == 'graft':
            self.debug_print("graft " + dir_pattern)
163
            if not self.include_pattern(None, prefix=dir_pattern):
164 165
                log.warn("warning: no directories found matching '%s'",
                         dir_pattern)
166 167 168 169

        elif action == 'prune':
            self.debug_print("prune " + dir_pattern)
            if not self.exclude_pattern(None, prefix=dir_pattern):
170
                log.warn(("no previously-included directories found "
171
                          "matching '%s'"), dir_pattern)
172
        else:
173 174
            raise DistutilsInternalError(
                  "this cannot happen: invalid action '%s'" % action)
175

176

177 178
    # -- Filtering/selection methods -----------------------------------

179
    def include_pattern(self, pattern, anchor=1, prefix=None, is_regex=0):
180
        """Select strings (presumably filenames) from 'self.files' that
181 182 183 184
        match 'pattern', a Unix-style wildcard (glob) pattern.  Patterns
        are not quite the same as implemented by the 'fnmatch' module: '*'
        and '?'  match non-special characters, where "special" is platform-
        dependent: slash on Unix; colon, slash, and backslash on
185
        DOS/Windows; and colon on Mac OS.
186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201

        If 'anchor' is true (the default), then the pattern match is more
        stringent: "*.py" will match "foo.py" but not "foo/bar.py".  If
        'anchor' is false, both of these will match.

        If 'prefix' is supplied, then only filenames starting with 'prefix'
        (itself a pattern) and ending with 'pattern', with anything in between
        them, will match.  'anchor' is ignored in this case.

        If 'is_regex' is true, 'anchor' and 'prefix' are ignored, and
        'pattern' is assumed to be either a string containing a regex or a
        regex object -- no translation is done, the regex is just compiled
        and used as-is.

        Selected strings will be added to self.files.

202
        Return True if files are found, False otherwise.
203
        """
204
        # XXX docstring lying about what the special chars are?
205
        files_found = False
206
        pattern_re = translate_pattern(pattern, anchor, prefix, is_regex)
207
        self.debug_print("include_pattern: applying regex r'%s'" %
208 209 210
                         pattern_re.pattern)

        # delayed loading of allfiles list
211 212
        if self.allfiles is None:
            self.findall()
213 214

        for name in self.allfiles:
215
            if pattern_re.search(name):
216
                self.debug_print(" adding " + name)
217
                self.files.append(name)
218
                files_found = True
219 220 221
        return files_found


222 223
    def exclude_pattern (self, pattern,
                         anchor=1, prefix=None, is_regex=0):
224
        """Remove strings (presumably filenames) from 'files' that match
225 226 227 228
        'pattern'.  Other parameters are the same as for
        'include_pattern()', above.
        The list 'self.files' is modified in place.
        Return True if files are found, False otherwise.
229
        """
230
        files_found = False
231
        pattern_re = translate_pattern(pattern, anchor, prefix, is_regex)
232 233
        self.debug_print("exclude_pattern: applying regex r'%s'" %
                         pattern_re.pattern)
234 235
        for i in range(len(self.files)-1, -1, -1):
            if pattern_re.search(self.files[i]):
236 237
                self.debug_print(" removing " + self.files[i])
                del self.files[i]
238
                files_found = True
239 240 241 242 243 244
        return files_found


# ----------------------------------------------------------------------
# Utility functions

245
def findall(dir=os.curdir):
246 247 248 249 250 251 252 253 254 255 256 257
    """Find all files under 'dir' and return the list of full filenames
    (relative to 'dir').
    """
    from stat import ST_MODE, S_ISREG, S_ISDIR, S_ISLNK

    list = []
    stack = [dir]
    pop = stack.pop
    push = stack.append

    while stack:
        dir = pop()
258
        names = os.listdir(dir)
259 260 261

        for name in names:
            if dir != os.curdir:        # avoid the dreaded "./" syndrome
262
                fullname = os.path.join(dir, name)
263 264 265 266 267 268 269
            else:
                fullname = name

            # Avoid excess stat calls -- just one will do, thank you!
            stat = os.stat(fullname)
            mode = stat[ST_MODE]
            if S_ISREG(mode):
270
                list.append(fullname)
271
            elif S_ISDIR(mode) and not S_ISLNK(mode):
272
                push(fullname)
273 274 275
    return list


276
def glob_to_re(pattern):
277 278 279 280
    """Translate a shell-like glob pattern to a regular expression; return
    a string containing the regex.  Differs from 'fnmatch.translate()' in
    that '*' does not match "special characters" (which are
    platform-specific).
281
    """
282
    pattern_re = fnmatch.translate(pattern)
283 284 285 286 287

    # '?' and '*' in the glob pattern become '.' and '.*' in the RE, which
    # IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix,
    # and by extension they shouldn't match such "special characters" under
    # any OS.  So change all non-escaped dots in the RE to match any
288 289 290 291 292 293 294 295
    # character except the special characters (currently: just os.sep).
    sep = os.sep
    if os.sep == '\\':
        # we're using a regex to manipulate a regex, so we need
        # to escape the backslash twice
        sep = r'\\\\'
    escaped = r'\1[^%s]' % sep
    pattern_re = re.sub(r'((?<!\\)(\\\\)*)\.', escaped, pattern_re)
296 297 298
    return pattern_re


299
def translate_pattern(pattern, anchor=1, prefix=None, is_regex=0):
300
    """Translate a shell-like wildcard pattern to a compiled regular
301
    expression.  Return the compiled regex.  If 'is_regex' true,
302 303 304 305
    then 'pattern' is directly compiled to a regex (if it's a string)
    or just returned as-is (assumes it's a regex object).
    """
    if is_regex:
306
        if isinstance(pattern, str):
307 308 309 310 311
            return re.compile(pattern)
        else:
            return pattern

    if pattern:
312
        pattern_re = glob_to_re(pattern)
313 314
    else:
        pattern_re = ''
Fred Drake's avatar
Fred Drake committed
315

316
    if prefix is not None:
317 318
        # ditch end of pattern character
        empty_pattern = glob_to_re('')
319 320 321 322 323
        prefix_re = glob_to_re(prefix)[:-len(empty_pattern)]
        sep = os.sep
        if os.sep == '\\':
            sep = r'\\'
        pattern_re = "^" + sep.join((prefix_re, ".*" + pattern_re))
324 325 326
    else:                               # no prefix -- respect anchor flag
        if anchor:
            pattern_re = "^" + pattern_re
Fred Drake's avatar
Fred Drake committed
327

328
    return re.compile(pattern_re)