textwrap.py 16.1 KB
Newer Older
1
"""Text wrapping and filling.
2 3
"""

4
# Copyright (C) 1999-2001 Gregory P. Ward.
Greg Ward's avatar
Greg Ward committed
5
# Copyright (C) 2002, 2003 Python Software Foundation.
Greg Ward's avatar
Greg Ward committed
6 7
# Written by Greg Ward <gward@python.net>

8
import re
9

10
__all__ = ['TextWrapper', 'wrap', 'fill', 'dedent', 'indent']
11

12 13 14 15 16 17 18 19
# Hardcode the recognized whitespace characters to the US-ASCII
# whitespace characters.  The main reason for doing this is that in
# ISO-8859-1, 0xa0 is non-breaking whitespace, so in certain locales
# that character winds up in string.whitespace.  Respecting
# string.whitespace in those cases would 1) make textwrap treat 0xa0 the
# same as any other whitespace char, which is clearly wrong (it's a
# *non-breaking* space), 2) possibly cause problems with Unicode,
# since 0xa0 is not in range(128).
20
_whitespace = '\t\n\x0b\x0c\r '
21

22 23 24 25 26 27 28 29
class TextWrapper:
    """
    Object for wrapping/filling text.  The public interface consists of
    the wrap() and fill() methods; the other methods are just there for
    subclasses to override in order to tweak the default behaviour.
    If you want to completely replace the main wrapping algorithm,
    you'll probably have to override _wrap_chunks().

30 31 32 33
    Several instance attributes control various aspects of wrapping:
      width (default: 70)
        the maximum width of wrapped lines (unless break_long_words
        is false)
34 35 36 37 38 39
      initial_indent (default: "")
        string that will be prepended to the first line of wrapped
        output.  Counts towards the line's width.
      subsequent_indent (default: "")
        string that will be prepended to all lines save the first
        of wrapped output; also counts towards each line's width.
40 41
      expand_tabs (default: true)
        Expand tabs in input text to spaces before further processing.
42 43 44 45 46
        Each tab will become 0 .. 'tabsize' spaces, depending on its position
        in its line.  If false, each tab is treated as a single character.
      tabsize (default: 8)
        Expand tabs in input text to 0 .. 'tabsize' spaces, unless
        'expand_tabs' is false.
47 48 49 50 51 52 53
      replace_whitespace (default: true)
        Replace all whitespace characters in the input text by spaces
        after tab expansion.  Note that if expand_tabs is false and
        replace_whitespace is true, every tab will be converted to a
        single space!
      fix_sentence_endings (default: false)
        Ensure that sentence-ending punctuation is always followed
Andrew M. Kuchling's avatar
Andrew M. Kuchling committed
54
        by two spaces.  Off by default because the algorithm is
55 56
        (unavoidably) imperfect.
      break_long_words (default: true)
57 58
        Break words longer than 'width'.  If false, those words will not
        be broken, and some lines might be longer than 'width'.
59 60 61 62
      break_on_hyphens (default: true)
        Allow breaking hyphenated words. If true, wrapping will occur
        preferably on whitespaces and right after hyphens part of
        compound words.
63 64
      drop_whitespace (default: true)
        Drop leading and trailing whitespace from lines.
65 66
    """

67
    unicode_whitespace_trans = {}
68
    uspace = ord(' ')
69 70
    for x in _whitespace:
        unicode_whitespace_trans[ord(x)] = uspace
71

Tim Peters's avatar
Tim Peters committed
72
    # This funky little regex is just the trick for splitting
73 74 75 76 77
    # text up into word-wrappable chunks.  E.g.
    #   "Hello there -- you goof-ball, use the -b option!"
    # splits into
    #   Hello/ /there/ /--/ /you/ /goof-/ball,/ /use/ /the/ /-b/ /option!
    # (after stripping out empty strings).
78 79
    wordsep_re = re.compile(
        r'(\s+|'                                  # any whitespace
80
        r'[^\s\w]*\w+[^0-9\W]-(?=\w+[^0-9\W])|'   # hyphenated words
81
        r'(?<=[\w\!\"\'\&\.\,\?])-{2,}(?=\w))')   # em-dash
82

83 84 85 86 87 88 89 90
    # This less funky little regex just split on recognized spaces. E.g.
    #   "Hello there -- you goof-ball, use the -b option!"
    # splits into
    #   Hello/ /there/ /--/ /you/ /goof-ball,/ /use/ /the/ /-b/ /option!/
    wordsep_simple_re = re.compile(r'(\s+)')

    # XXX this is not locale- or charset-aware -- string.lowercase
    # is US-ASCII only (and therefore English-only)
91
    sentence_end_re = re.compile(r'[a-z]'             # lowercase letter
92 93
                                 r'[\.\!\?]'          # sentence-ending punct.
                                 r'[\"\']?'           # optional end-of-quote
94
                                 r'\Z')               # end of chunk
95

96

97 98 99 100 101 102 103
    def __init__(self,
                 width=70,
                 initial_indent="",
                 subsequent_indent="",
                 expand_tabs=True,
                 replace_whitespace=True,
                 fix_sentence_endings=False,
104
                 break_long_words=True,
105
                 drop_whitespace=True,
106 107
                 break_on_hyphens=True,
                 tabsize=8):
108
        self.width = width
109 110
        self.initial_indent = initial_indent
        self.subsequent_indent = subsequent_indent
111 112 113 114
        self.expand_tabs = expand_tabs
        self.replace_whitespace = replace_whitespace
        self.fix_sentence_endings = fix_sentence_endings
        self.break_long_words = break_long_words
115
        self.drop_whitespace = drop_whitespace
116
        self.break_on_hyphens = break_on_hyphens
117
        self.tabsize = tabsize
Tim Peters's avatar
Tim Peters committed
118

119 120 121 122

    # -- Private methods -----------------------------------------------
    # (possibly useful for subclasses to override)

123
    def _munge_whitespace(self, text):
124 125 126 127 128 129 130
        """_munge_whitespace(text : string) -> string

        Munge whitespace in text: expand tabs and convert all other
        whitespace characters to spaces.  Eg. " foo\tbar\n\nbaz"
        becomes " foo    bar  baz".
        """
        if self.expand_tabs:
131
            text = text.expandtabs(self.tabsize)
132
        if self.replace_whitespace:
133
            text = text.translate(self.unicode_whitespace_trans)
134 135 136
        return text


137
    def _split(self, text):
138 139 140
        """_split(text : string) -> [string]

        Split the text to wrap into indivisible chunks.  Chunks are
Benjamin Peterson's avatar
Benjamin Peterson committed
141
        not quite the same as words; see _wrap_chunks() for full
142 143 144 145 146
        details.  As an example, the text
          Look, goof-ball -- use the -b option!
        breaks into the following chunks:
          'Look,', ' ', 'goof-', 'ball', ' ', '--', ' ',
          'use', ' ', 'the', ' ', '-b', ' ', 'option!'
147 148 149 150
        if break_on_hyphens is True, or in:
          'Look,', ' ', 'goof-ball', ' ', '--', ' ',
          'use', ' ', 'the', ' ', '-b', ' ', option!'
        otherwise.
151
        """
152 153 154 155
        if self.break_on_hyphens is True:
            chunks = self.wordsep_re.split(text)
        else:
            chunks = self.wordsep_simple_re.split(text)
156
        chunks = [c for c in chunks if c]
157 158
        return chunks

159
    def _fix_sentence_endings(self, chunks):
160 161 162 163 164 165 166 167 168
        """_fix_sentence_endings(chunks : [string])

        Correct for sentence endings buried in 'chunks'.  Eg. when the
        original text contains "... foo.\nBar ...", munge_whitespace()
        and split() will convert that to [..., "foo.", " ", "Bar", ...]
        which has one too few spaces; this method simply changes the one
        space to two.
        """
        i = 0
Benjamin Peterson's avatar
Benjamin Peterson committed
169
        patsearch = self.sentence_end_re.search
170
        while i < len(chunks)-1:
Benjamin Peterson's avatar
Benjamin Peterson committed
171
            if chunks[i+1] == " " and patsearch(chunks[i]):
172 173 174 175 176
                chunks[i+1] = "  "
                i += 2
            else:
                i += 1

177
    def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width):
178 179
        """_handle_long_word(chunks : [string],
                             cur_line : [string],
180
                             cur_len : int, width : int)
181 182 183 184

        Handle a chunk of text (most likely a word, not whitespace) that
        is too long to fit in any line.
        """
185 186 187 188 189 190
        # Figure out when indent is larger than the specified width, and make
        # sure at least one character is stripped off on every pass
        if width < 1:
            space_left = 1
        else:
            space_left = width - cur_len
191 192 193 194

        # If we're allowed to break long words, then do so: put as much
        # of the next chunk onto the current line as will fit.
        if self.break_long_words:
195 196
            cur_line.append(reversed_chunks[-1][:space_left])
            reversed_chunks[-1] = reversed_chunks[-1][space_left:]
197 198 199 200 201

        # Otherwise, we have to preserve the long word intact.  Only add
        # it to the current line if there's nothing already there --
        # that minimizes how much we violate the width constraint.
        elif not cur_line:
202
            cur_line.append(reversed_chunks.pop())
203 204 205 206 207 208 209

        # If we're not allowed to break long words, and there's already
        # text on the current line, do nothing.  Next time through the
        # main loop of _wrap_chunks(), we'll wind up here again, but
        # cur_len will be zero, so the next line will be entirely
        # devoted to the long word that we can't handle right now.

210 211
    def _wrap_chunks(self, chunks):
        """_wrap_chunks(chunks : [string]) -> [string]
212 213

        Wrap a sequence of text chunks and return a list of lines of
214 215 216 217 218 219 220 221
        length 'self.width' or less.  (If 'break_long_words' is false,
        some lines may be longer than this.)  Chunks correspond roughly
        to words and the whitespace between them: each chunk is
        indivisible (modulo 'break_long_words'), but a line break can
        come between any two chunks.  Chunks should not have internal
        whitespace; ie. a chunk is either all whitespace or a "word".
        Whitespace chunks will be removed from the beginning and end of
        lines, but apart from that whitespace is preserved.
222 223
        """
        lines = []
224 225
        if self.width <= 0:
            raise ValueError("invalid width %r (must be > 0)" % self.width)
226

227 228 229 230
        # Arrange in reverse order so items can be efficiently popped
        # from a stack of chucks.
        chunks.reverse()

231 232
        while chunks:

233 234 235 236 237 238 239 240 241 242 243 244 245
            # Start the list of chunks that will make up the current line.
            # cur_len is just the length of all the chunks in cur_line.
            cur_line = []
            cur_len = 0

            # Figure out which static string will prefix this line.
            if lines:
                indent = self.subsequent_indent
            else:
                indent = self.initial_indent

            # Maximum width for this line.
            width = self.width - len(indent)
246

247 248
            # First chunk on line is whitespace -- drop it, unless this
            # is the very beginning of the text (ie. no lines started yet).
249
            if self.drop_whitespace and chunks[-1].strip() == '' and lines:
250
                del chunks[-1]
251 252

            while chunks:
253
                l = len(chunks[-1])
254 255 256

                # Can at least squeeze this chunk onto the current line.
                if cur_len + l <= width:
257
                    cur_line.append(chunks.pop())
258 259 260 261 262 263 264
                    cur_len += l

                # Nope, this line is full.
                else:
                    break

            # The current line is full, and the next chunk is too big to
Tim Peters's avatar
Tim Peters committed
265
            # fit on *any* line (not just this one).
266
            if chunks and len(chunks[-1]) > width:
267
                self._handle_long_word(chunks, cur_line, cur_len, width)
268 269

            # If the last chunk on this line is all whitespace, drop it.
270
            if self.drop_whitespace and cur_line and cur_line[-1].strip() == '':
271 272 273 274 275
                del cur_line[-1]

            # Convert current line back to a string and store it in list
            # of all lines (return value).
            if cur_line:
276
                lines.append(indent + ''.join(cur_line))
277 278 279 280 281 282

        return lines


    # -- Public interface ----------------------------------------------

283 284
    def wrap(self, text):
        """wrap(text : string) -> [string]
285

286 287 288 289 290
        Reformat the single paragraph in 'text' so it fits in lines of
        no more than 'self.width' columns, and return a list of wrapped
        lines.  Tabs in 'text' are expanded with string.expandtabs(),
        and all other whitespace characters (including newline) are
        converted to space.
291 292 293
        """
        text = self._munge_whitespace(text)
        chunks = self._split(text)
294 295
        if self.fix_sentence_endings:
            self._fix_sentence_endings(chunks)
296
        return self._wrap_chunks(chunks)
297

298 299
    def fill(self, text):
        """fill(text : string) -> string
300

301 302 303
        Reformat the single paragraph in 'text' to fit in lines of no
        more than 'self.width' columns, and return a new string
        containing the entire wrapped paragraph.
304
        """
305
        return "\n".join(self.wrap(text))
306 307


308
# -- Convenience interface ---------------------------------------------
309

310
def wrap(text, width=70, **kwargs):
311 312 313 314 315 316 317 318 319
    """Wrap a single paragraph of text, returning a list of wrapped lines.

    Reformat the single paragraph in 'text' so it fits in lines of no
    more than 'width' columns, and return a list of wrapped lines.  By
    default, tabs in 'text' are expanded with string.expandtabs(), and
    all other whitespace characters (including newline) are converted to
    space.  See TextWrapper class for available keyword args to customize
    wrapping behaviour.
    """
320 321
    w = TextWrapper(width=width, **kwargs)
    return w.wrap(text)
322

323
def fill(text, width=70, **kwargs):
324 325 326 327 328 329 330 331
    """Fill a single paragraph of text, returning a new string.

    Reformat the single paragraph in 'text' to fit in lines of no more
    than 'width' columns, and return a new string containing the entire
    wrapped paragraph.  As with wrap(), tabs are expanded and other
    whitespace characters converted to space.  See TextWrapper class for
    available keyword args to customize wrapping behaviour.
    """
332
    w = TextWrapper(width=width, **kwargs)
333
    return w.fill(text)
334 335 336 337


# -- Loosely related functionality -------------------------------------

338 339
_whitespace_only_re = re.compile('^[ \t]+$', re.MULTILINE)
_leading_whitespace_re = re.compile('(^[ \t]*)(?:[^ \t\n])', re.MULTILINE)
340

341 342
def dedent(text):
    """Remove any common leading whitespace from every line in `text`.
343

344 345 346
    This can be used to make triple-quoted strings line up with the left
    edge of the display, while still presenting them in the source code
    in indented form.
347

348 349 350 351 352
    Note that tabs and spaces are both treated as whitespace, but they
    are not equal: the lines "  hello" and "\thello" are
    considered to have no common leading whitespace.  (This behaviour is
    new in Python 2.5; older versions of this module incorrectly
    expanded tabs before searching for common leading whitespace.)
353
    """
354 355
    # Look for the longest leading string of spaces and tabs common to
    # all lines.
356
    margin = None
357 358 359
    text = _whitespace_only_re.sub('', text)
    indents = _leading_whitespace_re.findall(text)
    for indent in indents:
360 361 362
        if margin is None:
            margin = indent

363 364 365 366
        # Current line more deeply indented than previous winner:
        # no change (previous winner is still on top).
        elif indent.startswith(margin):
            pass
367

368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388
        # Current line consistent with and no deeper than previous winner:
        # it's the new winner.
        elif margin.startswith(indent):
            margin = indent

        # Current line and previous winner have no common whitespace:
        # there is no margin.
        else:
            margin = ""
            break

    # sanity check (testing/debugging only)
    if 0 and margin:
        for line in text.split("\n"):
            assert not line or line.startswith(margin), \
                   "line = %r, margin = %r" % (line, margin)

    if margin:
        text = re.sub(r'(?m)^' + margin, '', text)
    return text

389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407

def indent(text, prefix, predicate=None):
    """Adds 'prefix' to the beginning of selected lines in 'text'.

    If 'predicate' is provided, 'prefix' will only be added to the lines
    where 'predicate(line)' is True. If 'predicate' is not provided,
    it will default to adding 'prefix' to all non-empty lines that do not
    consist solely of whitespace characters.
    """
    if predicate is None:
        def predicate(line):
            return line.strip()

    def prefixed_lines():
        for line in text.splitlines(True):
            yield (prefix + line if predicate(line) else line)
    return ''.join(prefixed_lines())


408 409 410
if __name__ == "__main__":
    #print dedent("\tfoo\n\tbar")
    #print dedent("  \thello there\n  \t  how are you?")
411
    print(dedent("Hello there.\n  This is indented."))