doctest.py 95.1 KB
Newer Older
1
# Module doctest.
2
# Released to the public domain 16-Jan-2001, by Tim Peters (tim@python.org).
3
# Major enhancements and refactoring by:
4 5
#     Jim Fulton
#     Edward Loper
6 7 8

# Provided as-is; use at your own risk; no warranty; no promises; enjoy!

9
r"""Module doctest -- a framework for running examples in docstrings.
10

11
In simplest use, end each module M to be tested with:
12 13

def _test():
14
    import doctest
15
    doctest.testmod()
16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36

if __name__ == "__main__":
    _test()

Then running the module as a script will cause the examples in the
docstrings to get executed and verified:

python M.py

This won't display anything unless an example fails, in which case the
failing example(s) and the cause(s) of the failure(s) are printed to stdout
(why not stderr? because stderr is a lame hack <0.2 wink>), and the final
line of output is "Test failed.".

Run it with the -v switch instead:

python M.py -v

and a detailed report of all examples tried is printed to stdout, along
with assorted summaries at the end.

37 38
You can force verbose mode by passing "verbose=True" to testmod, or prohibit
it by passing "verbose=False".  In either of those cases, sys.argv is not
39 40
examined by testmod.

41 42 43 44 45
There are a variety of other ways to run doctests, including integration
with the unittest framework, and support for running non-Python text
files containing doctests.  There are also many ways to override parts
of doctest's default behaviors.  See the Library Reference Manual for
details.
46
"""
47

Edward Loper's avatar
Edward Loper committed
48
__docformat__ = 'reStructuredText en'
49

50
__all__ = [
51 52 53 54 55 56
    # 0, Option Flags
    'register_optionflag',
    'DONT_ACCEPT_TRUE_FOR_1',
    'DONT_ACCEPT_BLANKLINE',
    'NORMALIZE_WHITESPACE',
    'ELLIPSIS',
57
    'SKIP',
58
    'IGNORE_EXCEPTION_DETAIL',
59
    'COMPARISON_FLAGS',
60 61 62
    'REPORT_UDIFF',
    'REPORT_CDIFF',
    'REPORT_NDIFF',
63
    'REPORT_ONLY_FIRST_FAILURE',
64
    'REPORTING_FLAGS',
65 66
    # 1. Utility Functions
    # 2. Example & DocTest
67 68
    'Example',
    'DocTest',
69 70 71
    # 3. Doctest Parser
    'DocTestParser',
    # 4. Doctest Finder
72
    'DocTestFinder',
73
    # 5. Doctest Runner
74
    'DocTestRunner',
75 76 77 78 79
    'OutputChecker',
    'DocTestFailure',
    'UnexpectedException',
    'DebugRunner',
    # 6. Test Functions
80
    'testmod',
81
    'testfile',
82
    'run_docstring_examples',
Georg Brandl's avatar
Georg Brandl committed
83
    # 7. Unittest Support
84
    'DocTestSuite',
85
    'DocFileSuite',
86
    'set_unittest_reportflags',
Georg Brandl's avatar
Georg Brandl committed
87
    # 8. Debugging Support
88
    'script_from_examples',
89
    'testsource',
90
    'debug_src',
91
    'debug',
92 93 94
]

import __future__
95

96
import sys, traceback, inspect, linecache, os, re
97
import unittest, difflib, pdb, tempfile
98
import warnings
99
from io import StringIO
100 101 102
from collections import namedtuple

TestResults = namedtuple('TestResults', 'failed attempted')
103

104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122
# There are 4 basic classes:
#  - Example: a <source, want> pair, plus an intra-docstring line number.
#  - DocTest: a collection of examples, parsed from a docstring, plus
#    info about where the docstring came from (name, filename, lineno).
#  - DocTestFinder: extracts DocTests from a given object's docstring and
#    its contained objects' docstrings.
#  - DocTestRunner: runs DocTest cases, and accumulates statistics.
#
# So the basic picture is:
#
#                             list of:
# +------+                   +---------+                   +-------+
# |object| --DocTestFinder-> | DocTest | --DocTestRunner-> |results|
# +------+                   +---------+                   +-------+
#                            | Example |
#                            |   ...   |
#                            | Example |
#                            +---------+

123
# Option constants.
124

125 126
OPTIONFLAGS_BY_NAME = {}
def register_optionflag(name):
127 128
    # Create a new flag unless `name` is already known.
    return OPTIONFLAGS_BY_NAME.setdefault(name, 1 << len(OPTIONFLAGS_BY_NAME))
129 130 131 132 133

DONT_ACCEPT_TRUE_FOR_1 = register_optionflag('DONT_ACCEPT_TRUE_FOR_1')
DONT_ACCEPT_BLANKLINE = register_optionflag('DONT_ACCEPT_BLANKLINE')
NORMALIZE_WHITESPACE = register_optionflag('NORMALIZE_WHITESPACE')
ELLIPSIS = register_optionflag('ELLIPSIS')
134
SKIP = register_optionflag('SKIP')
135
IGNORE_EXCEPTION_DETAIL = register_optionflag('IGNORE_EXCEPTION_DETAIL')
136 137 138 139

COMPARISON_FLAGS = (DONT_ACCEPT_TRUE_FOR_1 |
                    DONT_ACCEPT_BLANKLINE |
                    NORMALIZE_WHITESPACE |
140
                    ELLIPSIS |
141
                    SKIP |
142
                    IGNORE_EXCEPTION_DETAIL)
143

144 145 146
REPORT_UDIFF = register_optionflag('REPORT_UDIFF')
REPORT_CDIFF = register_optionflag('REPORT_CDIFF')
REPORT_NDIFF = register_optionflag('REPORT_NDIFF')
147
REPORT_ONLY_FIRST_FAILURE = register_optionflag('REPORT_ONLY_FIRST_FAILURE')
148

149 150 151 152 153
REPORTING_FLAGS = (REPORT_UDIFF |
                   REPORT_CDIFF |
                   REPORT_NDIFF |
                   REPORT_ONLY_FIRST_FAILURE)

154 155 156 157
# Special string markers for use in `want` strings:
BLANKLINE_MARKER = '<BLANKLINE>'
ELLIPSIS_MARKER = '...'

158 159 160
######################################################################
## Table of Contents
######################################################################
161 162 163 164 165 166
#  1. Utility Functions
#  2. Example & DocTest -- store test cases
#  3. DocTest Parser -- extracts examples from strings
#  4. DocTest Finder -- extracts test cases from objects
#  5. DocTest Runner -- runs test cases
#  6. Test Functions -- convenient wrappers for testing
167 168 169
#  7. Unittest Support
#  8. Debugging Support
#  9. Example Usage
170 171 172 173

######################################################################
## 1. Utility Functions
######################################################################
174

175 176 177 178 179 180 181 182 183 184 185
def _extract_future_flags(globs):
    """
    Return the compiler-flags associated with the future features that
    have been imported into the given namespace (globs).
    """
    flags = 0
    for fname in __future__.all_feature_names:
        feature = globs.get(fname, None)
        if feature is getattr(__future__, fname):
            flags |= feature.compiler_flag
    return flags
186

187 188 189 190 191 192 193 194 195 196 197 198
def _normalize_module(module, depth=2):
    """
    Return the module specified by `module`.  In particular:
      - If `module` is a module, then return module.
      - If `module` is a string, then import and return the
        module with that name.
      - If `module` is None, then return the calling module.
        The calling module is assumed to be the module of
        the stack frame at the given depth in the call stack.
    """
    if inspect.ismodule(module):
        return module
199
    elif isinstance(module, str):
200 201 202 203 204
        return __import__(module, globals(), locals(), ["*"])
    elif module is None:
        return sys.modules[sys._getframe(depth).f_globals['__name__']]
    else:
        raise TypeError("Expected a module, string, or None")
205

206
def _load_testfile(filename, package, module_relative, encoding):
207 208 209 210 211
    if module_relative:
        package = _normalize_module(package, 3)
        filename = _module_relative_path(package, filename)
        if hasattr(package, '__loader__'):
            if hasattr(package.__loader__, 'get_data'):
212 213 214 215 216
                file_contents = package.__loader__.get_data(filename)
                file_contents = file_contents.decode(encoding)
                # get_data() opens files as 'rb', so one must do the equivalent
                # conversion as universal newlines would do.
                return file_contents.replace(os.linesep, '\n'), filename
217
    return open(filename, encoding=encoding).read(), filename
218

219
def _indent(s, indent=4):
220
    """
221 222
    Add the given number of space characters to the beginning every
    non-blank line in `s`, and return the result.
223
    """
224 225
    # This regexp matches the start of non-blank lines:
    return re.sub('(?m)^(?!$)', indent*' ', s)
226

Edward Loper's avatar
Edward Loper committed
227 228 229 230 231 232 233 234 235 236 237
def _exception_traceback(exc_info):
    """
    Return a string containing a traceback message for the given
    exc_info tuple (as returned by sys.exc_info()).
    """
    # Get a traceback message.
    excout = StringIO()
    exc_type, exc_val, exc_tb = exc_info
    traceback.print_exception(exc_type, exc_val, exc_tb, file=excout)
    return excout.getvalue()

238 239 240 241 242 243 244 245 246 247
# Override some StringIO methods.
class _SpoofOut(StringIO):
    def getvalue(self):
        result = StringIO.getvalue(self)
        # If anything at all was written, make sure there's a trailing
        # newline.  There's no way for the expected output to indicate
        # that a trailing newline is missing.
        if result and not result.endswith("\n"):
            result += "\n"
        return result
248

249
    def truncate(self, size=None):
250
        StringIO.truncate(self, size)
251

252
# Worst-case linear-time ellipsis matching.
253
def _ellipsis_match(want, got):
254 255
    """
    Essentially the only subtle case:
256
    >>> _ellipsis_match('aa...aa', 'aaa')
257 258
    False
    """
259 260
    if ELLIPSIS_MARKER not in want:
        return want == got
261

262 263 264
    # Find "the real" strings.
    ws = want.split(ELLIPSIS_MARKER)
    assert len(ws) >= 2
265 266 267

    # Deal with exact matches possibly needed at one or both ends.
    startpos, endpos = 0, len(got)
268
    w = ws[0]
269 270 271 272 273 274 275 276 277 278 279 280
    if w:   # starts with exact match
        if got.startswith(w):
            startpos = len(w)
            del ws[0]
        else:
            return False
    w = ws[-1]
    if w:   # ends with exact match
        if got.endswith(w):
            endpos -= len(w)
            del ws[-1]
        else:
281 282
            return False

283 284
    if startpos > endpos:
        # Exact end matches required more characters than we have, as in
285
        # _ellipsis_match('aa...aa', 'aaa')
286 287 288 289 290
        return False

    # For the rest, we only need to find the leftmost non-overlapping
    # match for each piece.  If there's no overall match that way alone,
    # there's no overall match period.
291 292 293
    for w in ws:
        # w may be '' at times, if there are consecutive ellipses, or
        # due to an ellipsis at the start or end of `want`.  That's OK.
294 295 296
        # Search for an empty string succeeds, and doesn't change startpos.
        startpos = got.find(w, startpos, endpos)
        if startpos < 0:
297
            return False
298 299 300
        startpos += len(w)

    return True
301

302 303 304 305 306 307 308 309
def _comment_line(line):
    "Return a commented form of the given line"
    line = line.rstrip()
    if line:
        return '# '+line
    else:
        return '#'

310 311 312 313 314 315 316 317
class _OutputRedirectingPdb(pdb.Pdb):
    """
    A specialized version of the python debugger that redirects stdout
    to a given stream when interacting with the user.  Stdout is *not*
    redirected when traced code is executed.
    """
    def __init__(self, out):
        self.__out = out
318
        self.__debugger_used = False
319
        pdb.Pdb.__init__(self, stdout=out)
320

321 322 323 324 325 326 327 328 329 330 331 332
    def set_trace(self, frame=None):
        self.__debugger_used = True
        if frame is None:
            frame = sys._getframe().f_back
        pdb.Pdb.set_trace(self, frame)

    def set_continue(self):
        # Calling set_continue unconditionally would break unit test
        # coverage reporting, as Bdb.set_continue calls sys.settrace(None).
        if self.__debugger_used:
            pdb.Pdb.set_continue(self)

333 334 335 336 337
    def trace_dispatch(self, *args):
        # Redirect stdout to the given stream.
        save_stdout = sys.stdout
        sys.stdout = self.__out
        # Call Pdb's trace dispatch method.
338 339 340 341
        try:
            return pdb.Pdb.trace_dispatch(self, *args)
        finally:
            sys.stdout = save_stdout
342

Edward Loper's avatar
Edward Loper committed
343
# [XX] Normalize with respect to os.path.pardir?
344 345
def _module_relative_path(module, path):
    if not inspect.ismodule(module):
346
        raise TypeError('Expected a module: %r' % module)
347
    if path.startswith('/'):
348
        raise ValueError('Module-relative files may not have absolute paths')
349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367

    # Find the base directory for the path.
    if hasattr(module, '__file__'):
        # A normal module/package
        basedir = os.path.split(module.__file__)[0]
    elif module.__name__ == '__main__':
        # An interactive session.
        if len(sys.argv)>0 and sys.argv[0] != '':
            basedir = os.path.split(sys.argv[0])[0]
        else:
            basedir = os.curdir
    else:
        # A module w/o __file__ (this includes builtins)
        raise ValueError("Can't resolve paths relative to the module " +
                         module + " (it has no __file__)")

    # Combine the base directory and the path.
    return os.path.join(basedir, *(path.split('/')))

368 369 370 371 372 373 374 375
######################################################################
## 2. Example & DocTest
######################################################################
## - An "example" is a <source, want> pair, where "source" is a
##   fragment of source code, and "want" is the expected output for
##   "source."  The Example class also includes information about
##   where the example was extracted from.
##
376 377 378
## - A "doctest" is a collection of examples, typically extracted from
##   a string (such as an object's docstring).  The DocTest class also
##   includes information about where the string was extracted from.
379 380 381 382

class Example:
    """
    A single doctest example, consisting of source code and expected
383
    output.  `Example` defines the following attributes:
384

385
      - source: A single Python statement, always ending with a newline.
386
        The constructor adds a newline if needed.
387

388
      - want: The expected output from running the source code (either
389 390 391
        from stdout, or a traceback in case of exception).  `want` ends
        with a newline unless it's empty, in which case it's an empty
        string.  The constructor adds a newline if needed.
392

393 394 395 396 397 398 399 400
      - exc_msg: The exception message generated by the example, if
        the example is expected to generate an exception; or `None` if
        it is not expected to generate an exception.  This exception
        message is compared against the return value of
        `traceback.format_exception_only()`.  `exc_msg` ends with a
        newline unless it's `None`.  The constructor adds a newline
        if needed.

401
      - lineno: The line number within the DocTest string containing
402 403
        this Example where the Example begins.  This line number is
        zero-based, with respect to the beginning of the DocTest.
404 405 406 407 408 409 410 411 412 413

      - indent: The example's indentation in the DocTest string.
        I.e., the number of space characters that preceed the
        example's first prompt.

      - options: A dictionary mapping from option flags to True or
        False, which is used to override default options for this
        example.  Any option flags not contained in this dictionary
        are left at their default value (as specified by the
        DocTestRunner's optionflags).  By default, no options are set.
414
    """
415 416
    def __init__(self, source, want, exc_msg=None, lineno=0, indent=0,
                 options=None):
417 418 419 420 421
        # Normalize inputs.
        if not source.endswith('\n'):
            source += '\n'
        if want and not want.endswith('\n'):
            want += '\n'
422 423
        if exc_msg is not None and not exc_msg.endswith('\n'):
            exc_msg += '\n'
424 425 426 427
        # Store properties.
        self.source = source
        self.want = want
        self.lineno = lineno
428 429 430
        self.indent = indent
        if options is None: options = {}
        self.options = options
431
        self.exc_msg = exc_msg
432 433 434 435

class DocTest:
    """
    A collection of doctest examples that should be run in a single
436
    namespace.  Each `DocTest` defines the following attributes:
437

438
      - examples: the list of examples.
439

440 441
      - globs: The namespace (aka globals) that the examples should
        be run in.
442

443 444
      - name: A name identifying the DocTest (typically, the name of
        the object whose docstring this DocTest was extracted from).
445

446
      - filename: The name of the file that this DocTest was extracted
447
        from, or `None` if the filename is unknown.
448

449
      - lineno: The line number within filename where this DocTest
450 451 452 453 454 455
        begins, or `None` if the line number is unavailable.  This
        line number is zero-based, with respect to the beginning of
        the file.

      - docstring: The string that the examples were extracted from,
        or `None` if the string is unavailable.
456
    """
457
    def __init__(self, examples, globs, name, filename, lineno, docstring):
458
        """
459 460
        Create a new DocTest containing the given examples.  The
        DocTest's globals are initialized with a copy of `globs`.
461
        """
462
        assert not isinstance(examples, str), \
463 464 465
               "DocTest no longer accepts str; use DocTestParser instead"
        self.examples = examples
        self.docstring = docstring
466 467 468 469
        self.globs = globs.copy()
        self.name = name
        self.filename = filename
        self.lineno = lineno
470

471 472 473 474 475 476 477 478 479
    def __repr__(self):
        if len(self.examples) == 0:
            examples = 'no examples'
        elif len(self.examples) == 1:
            examples = '1 example'
        else:
            examples = '%d examples' % len(self.examples)
        return ('<DocTest %s from %s:%s (%s)>' %
                (self.name, self.filename, self.lineno, examples))
480 481


482
    # This lets us sort tests by name:
483
    def __lt__(self, other):
484
        if not isinstance(other, DocTest):
485 486 487 488
            return NotImplemented
        return ((self.name, self.filename, self.lineno, id(self))
                <
                (other.name, other.filename, other.lineno, id(other)))
489

490
######################################################################
491
## 3. DocTestParser
492 493
######################################################################

494
class DocTestParser:
495
    """
496
    A class used to parse strings containing doctest examples.
497
    """
Edward Loper's avatar
Edward Loper committed
498 499 500 501 502
    # This regular expression is used to find doctest examples in a
    # string.  It defines three groups: `source` is the source code
    # (including leading indentation and prompts); `indent` is the
    # indentation of the first (PS1) line of the source code; and
    # `want` is the expected output (including leading indentation).
503
    _EXAMPLE_RE = re.compile(r'''
504 505 506 507 508 509 510 511 512 513 514
        # Source consists of a PS1 line followed by zero or more PS2 lines.
        (?P<source>
            (?:^(?P<indent> [ ]*) >>>    .*)    # PS1 line
            (?:\n           [ ]*  \.\.\. .*)*)  # PS2 lines
        \n?
        # Want consists of any non-blank lines that do not start with PS1.
        (?P<want> (?:(?![ ]*$)    # Not a blank line
                     (?![ ]*>>>)  # Not a line starting with PS1
                     .*$\n?       # But any other line
                  )*)
        ''', re.MULTILINE | re.VERBOSE)
Edward Loper's avatar
Edward Loper committed
515

516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537
    # A regular expression for handling `want` strings that contain
    # expected exceptions.  It divides `want` into three pieces:
    #    - the traceback header line (`hdr`)
    #    - the traceback stack (`stack`)
    #    - the exception message (`msg`), as generated by
    #      traceback.format_exception_only()
    # `msg` may have multiple lines.  We assume/require that the
    # exception message is the first non-indented line starting with a word
    # character following the traceback header line.
    _EXCEPTION_RE = re.compile(r"""
        # Grab the traceback header.  Different versions of Python have
        # said different things on the first traceback line.
        ^(?P<hdr> Traceback\ \(
            (?: most\ recent\ call\ last
            |   innermost\ last
            ) \) :
        )
        \s* $                # toss trailing whitespace on the header.
        (?P<stack> .*?)      # don't blink: absorb stuff until...
        ^ (?P<msg> \w+ .*)   #     a line *starts* with alphanum.
        """, re.VERBOSE | re.MULTILINE | re.DOTALL)

538 539
    # A callable returning a true value iff its argument is a blank line
    # or contains a single comment.
Edward Loper's avatar
Edward Loper committed
540
    _IS_BLANK_OR_COMMENT = re.compile(r'^[ ]*(#.*)?$').match
541

542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558
    def parse(self, string, name='<string>'):
        """
        Divide the given string into examples and intervening text,
        and return them as a list of alternating Examples and strings.
        Line numbers for the Examples are 0-based.  The optional
        argument `name` is a name identifying this string, and is only
        used for error messages.
        """
        string = string.expandtabs()
        # If all lines begin with the same indentation, then strip it.
        min_indent = self._min_indent(string)
        if min_indent > 0:
            string = '\n'.join([l[min_indent:] for l in string.split('\n')])

        output = []
        charno, lineno = 0, 0
        # Find all doctest examples in the string:
559
        for m in self._EXAMPLE_RE.finditer(string):
560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580
            # Add the pre-example text to `output`.
            output.append(string[charno:m.start()])
            # Update lineno (lines before this example)
            lineno += string.count('\n', charno, m.start())
            # Extract info from the regexp match.
            (source, options, want, exc_msg) = \
                     self._parse_example(m, name, lineno)
            # Create an Example, and add it to the list.
            if not self._IS_BLANK_OR_COMMENT(source):
                output.append( Example(source, want, exc_msg,
                                    lineno=lineno,
                                    indent=min_indent+len(m.group('indent')),
                                    options=options) )
            # Update lineno (lines inside this example)
            lineno += string.count('\n', m.start(), m.end())
            # Update charno.
            charno = m.end()
        # Add any remaining post-example text to `output`.
        output.append(string[charno:])
        return output

581 582 583 584 585 586 587 588
    def get_doctest(self, string, globs, name, filename, lineno):
        """
        Extract all doctest examples from the given string, and
        collect them into a `DocTest` object.

        `globs`, `name`, `filename`, and `lineno` are attributes for
        the new `DocTest` object.  See the documentation for `DocTest`
        for more information.
589
        """
590 591 592 593 594 595 596 597 598 599 600 601 602
        return DocTest(self.get_examples(string, name), globs,
                       name, filename, lineno, string)

    def get_examples(self, string, name='<string>'):
        """
        Extract all doctest examples from the given string, and return
        them as a list of `Example` objects.  Line numbers are
        0-based, because it's most common in doctests that nothing
        interesting appears on the same line as opening triple-quote,
        and so the first interesting line is called \"line 1\" then.

        The optional argument `name` is a name identifying this
        string, and is only used for error messages.
603
        """
604 605
        return [x for x in self.parse(string, name)
                if isinstance(x, Example)]
606

607 608 609 610 611 612 613 614 615 616 617
    def _parse_example(self, m, name, lineno):
        """
        Given a regular expression match from `_EXAMPLE_RE` (`m`),
        return a pair `(source, want)`, where `source` is the matched
        example's source code (with prompts and indentation stripped);
        and `want` is the example's expected output (with indentation
        stripped).

        `name` is the string's name, and `lineno` is the line number
        where the example starts; both are used for error messages.
        """
618 619 620 621 622 623
        # Get the example's indentation level.
        indent = len(m.group('indent'))

        # Divide source into lines; check that they're properly
        # indented; and then strip their indentation & prompts.
        source_lines = m.group('source').split('\n')
624
        self._check_prompt_blank(source_lines, indent, name, lineno)
625
        self._check_prefix(source_lines[1:], ' '*indent + '.', name, lineno)
626 627
        source = '\n'.join([sl[indent+4:] for sl in source_lines])

628 629 630
        # Divide want into lines; check that it's properly indented; and
        # then strip the indentation.  Spaces before the last newline should
        # be preserved, so plain rstrip() isn't good enough.
Jim Fulton's avatar
Jim Fulton committed
631 632
        want = m.group('want')
        want_lines = want.split('\n')
633 634
        if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
            del want_lines[-1]  # forget final newline & spaces after it
635
        self._check_prefix(want_lines, ' '*indent, name,
636
                           lineno + len(source_lines))
637 638
        want = '\n'.join([wl[indent:] for wl in want_lines])

639 640 641 642 643 644 645
        # If `want` contains a traceback message, then extract it.
        m = self._EXCEPTION_RE.match(want)
        if m:
            exc_msg = m.group('msg')
        else:
            exc_msg = None

646 647 648 649
        # Extract options from the source.
        options = self._find_options(source, name, lineno)

        return source, options, want, exc_msg
650

651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686
    # This regular expression looks for option directives in the
    # source code of an example.  Option directives are comments
    # starting with "doctest:".  Warning: this may give false
    # positives for string-literals that contain the string
    # "#doctest:".  Eliminating these false positives would require
    # actually parsing the string; but we limit them by ignoring any
    # line containing "#doctest:" that is *followed* by a quote mark.
    _OPTION_DIRECTIVE_RE = re.compile(r'#\s*doctest:\s*([^\n\'"]*)$',
                                      re.MULTILINE)

    def _find_options(self, source, name, lineno):
        """
        Return a dictionary containing option overrides extracted from
        option directives in the given source string.

        `name` is the string's name, and `lineno` is the line number
        where the example starts; both are used for error messages.
        """
        options = {}
        # (note: with the current regexp, this will match at most once:)
        for m in self._OPTION_DIRECTIVE_RE.finditer(source):
            option_strings = m.group(1).replace(',', ' ').split()
            for option in option_strings:
                if (option[0] not in '+-' or
                    option[1:] not in OPTIONFLAGS_BY_NAME):
                    raise ValueError('line %r of the doctest for %s '
                                     'has an invalid option: %r' %
                                     (lineno+1, name, option))
                flag = OPTIONFLAGS_BY_NAME[option[1:]]
                options[flag] = (option[0] == '+')
        if options and self._IS_BLANK_OR_COMMENT(source):
            raise ValueError('line %r of the doctest for %s has an option '
                             'directive on a line with no example: %r' %
                             (lineno, name, source))
        return options

687 688
    # This regular expression finds the indentation of every non-blank
    # line in a string.
689
    _INDENT_RE = re.compile('^([ ]*)(?=\S)', re.MULTILINE)
690 691 692

    def _min_indent(self, s):
        "Return the minimum indentation of any non-blank line in `s`"
693 694 695
        indents = [len(indent) for indent in self._INDENT_RE.findall(s)]
        if len(indents) > 0:
            return min(indents)
696
        else:
697
            return 0
698

699
    def _check_prompt_blank(self, lines, indent, name, lineno):
700 701 702 703 704 705
        """
        Given the lines of a source string (including prompts and
        leading indentation), check to make sure that every prompt is
        followed by a space character.  If any line is not followed by
        a space character, then raise ValueError.
        """
706 707 708 709
        for i, line in enumerate(lines):
            if len(line) >= indent+4 and line[indent+3] != ' ':
                raise ValueError('line %r of the docstring for %s '
                                 'lacks blank after %s: %r' %
710
                                 (lineno+i+1, name,
711 712
                                  line[indent:indent+3], line))

713
    def _check_prefix(self, lines, prefix, name, lineno):
714 715 716 717
        """
        Check that every line in the given list starts with the given
        prefix; if any line does not, then raise a ValueError.
        """
718 719 720 721
        for i, line in enumerate(lines):
            if line and not line.startswith(prefix):
                raise ValueError('line %r of the docstring for %s has '
                                 'inconsistent leading whitespace: %r' %
722
                                 (lineno+i+1, name, line))
723 724 725 726


######################################################################
## 4. DocTest Finder
727
######################################################################
728

729 730 731 732 733 734 735 736
class DocTestFinder:
    """
    A class used to extract the DocTests that are relevant to a given
    object, from its docstring and the docstrings of its contained
    objects.  Doctests can currently be extracted from the following
    object types: modules, functions, classes, methods, staticmethods,
    classmethods, and properties.
    """
737

738
    def __init__(self, verbose=False, parser=DocTestParser(),
739
                 recurse=True, exclude_empty=True):
740
        """
741
        Create a new doctest finder.
742

743
        The optional argument `parser` specifies a class or
744
        function that should be used to create new DocTest objects (or
Tim Peters's avatar
Tim Peters committed
745
        objects that implement the same interface as DocTest).  The
746 747 748
        signature for this factory function should match the signature
        of the DocTest constructor.

749 750
        If the optional argument `recurse` is false, then `find` will
        only examine the given object, and not any contained objects.
751

752 753
        If the optional argument `exclude_empty` is false, then `find`
        will include tests for objects with empty docstrings.
754
        """
755
        self._parser = parser
756 757
        self._verbose = verbose
        self._recurse = recurse
758
        self._exclude_empty = exclude_empty
759

760
    def find(self, obj, name=None, module=None, globs=None, extraglobs=None):
761
        """
762 763 764 765 766
        Return a list of the DocTests that are defined by the given
        object's docstring, or by any of its contained objects'
        docstrings.

        The optional parameter `module` is the module that contains
767 768
        the given object.  If the module is not specified or is None, then
        the test finder will attempt to automatically determine the
769 770 771 772
        correct module.  The object's module is used:

            - As a default namespace, if `globs` is not specified.
            - To prevent the DocTestFinder from extracting DocTests
773
              from objects that are imported from other modules.
774 775 776 777
            - To find the name of the file containing the object.
            - To help find the line number of the object within its
              file.

778 779 780 781 782 783 784 785
        Contained objects whose module does not match `module` are ignored.

        If `module` is False, no attempt to find the module will be made.
        This is obscure, of use mostly in tests:  if `module` is False, or
        is None but cannot be found automatically, then all objects are
        considered to belong to the (non-existent) module, so all contained
        objects will (recursively) be searched for doctests.

786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805
        The globals for each DocTest is formed by combining `globs`
        and `extraglobs` (bindings in `extraglobs` override bindings
        in `globs`).  A new copy of the globals dictionary is created
        for each DocTest.  If `globs` is not specified, then it
        defaults to the module's `__dict__`, if specified, or {}
        otherwise.  If `extraglobs` is not specified, then it defaults
        to {}.

        """
        # If name was not specified, then extract it from the object.
        if name is None:
            name = getattr(obj, '__name__', None)
            if name is None:
                raise ValueError("DocTestFinder.find: name must be given "
                        "when obj.__name__ doesn't exist: %r" %
                                 (type(obj),))

        # Find the module that contains the given object (if obj is
        # a module, then module=obj.).  Note: this may fail, in which
        # case module will be None.
806 807 808
        if module is False:
            module = None
        elif module is None:
809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831
            module = inspect.getmodule(obj)

        # Read the module's source code.  This is used by
        # DocTestFinder._find_lineno to find the line number for a
        # given object's docstring.
        try:
            file = inspect.getsourcefile(obj) or inspect.getfile(obj)
            source_lines = linecache.getlines(file)
            if not source_lines:
                source_lines = None
        except TypeError:
            source_lines = None

        # Initialize globals, and merge in extraglobs.
        if globs is None:
            if module is None:
                globs = {}
            else:
                globs = module.__dict__.copy()
        else:
            globs = globs.copy()
        if extraglobs is not None:
            globs.update(extraglobs)
832

833 834
        # Recursively expore `obj`, extracting DocTests.
        tests = []
835
        self._find(tests, obj, name, module, source_lines, globs, {})
836 837 838 839 840
        # Sort the tests by alpha order of names, for consistency in
        # verbose-mode output.  This was a feature of doctest in Pythons
        # <= 2.3 that got lost by accident in 2.4.  It was repaired in
        # 2.4.4 and 2.5.
        tests.sort()
841
        return tests
842

843 844 845 846
    def _from_module(self, module, object):
        """
        Return true if the given object is defined in the given
        module.
847
        """
848 849 850
        if module is None:
            return True
        elif inspect.isfunction(object):
851
            return module.__dict__ is object.__globals__
852 853 854 855 856 857 858 859 860 861
        elif inspect.isclass(object):
            return module.__name__ == object.__module__
        elif inspect.getmodule(object) is not None:
            return module is inspect.getmodule(object)
        elif hasattr(object, '__module__'):
            return module.__name__ == object.__module__
        elif isinstance(object, property):
            return True # [XX] no way not be sure.
        else:
            raise ValueError("object must be a class or function")
862

863
    def _find(self, tests, obj, name, module, source_lines, globs, seen):
864 865 866 867 868
        """
        Find tests for the given object and any contained objects, and
        add them to `tests`.
        """
        if self._verbose:
869
            print('Finding tests in %s' % name)
870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886

        # If we've already processed this object, then ignore it.
        if id(obj) in seen:
            return
        seen[id(obj)] = 1

        # Find a test for this object, and add it to the list of tests.
        test = self._get_test(obj, name, module, globs, source_lines)
        if test is not None:
            tests.append(test)

        # Look for tests in a module's contained objects.
        if inspect.ismodule(obj) and self._recurse:
            for valname, val in obj.__dict__.items():
                valname = '%s.%s' % (name, valname)
                # Recurse to functions & classes.
                if ((inspect.isfunction(val) or inspect.isclass(val)) and
887
                    self._from_module(module, val)):
888
                    self._find(tests, val, valname, module, source_lines,
889
                               globs, seen)
890 891 892 893

        # Look for tests in a module's __test__ dictionary.
        if inspect.ismodule(obj) and self._recurse:
            for valname, val in getattr(obj, '__test__', {}).items():
894
                if not isinstance(valname, str):
895 896 897 898 899
                    raise ValueError("DocTestFinder.find: __test__ keys "
                                     "must be strings: %r" %
                                     (type(valname),))
                if not (inspect.isfunction(val) or inspect.isclass(val) or
                        inspect.ismethod(val) or inspect.ismodule(val) or
900
                        isinstance(val, str)):
901 902 903 904
                    raise ValueError("DocTestFinder.find: __test__ values "
                                     "must be strings, functions, methods, "
                                     "classes, or modules: %r" %
                                     (type(val),))
905
                valname = '%s.__test__.%s' % (name, valname)
906
                self._find(tests, val, valname, module, source_lines,
907
                           globs, seen)
908 909 910 911 912 913 914 915

        # Look for tests in a class's contained objects.
        if inspect.isclass(obj) and self._recurse:
            for valname, val in obj.__dict__.items():
                # Special handling for staticmethod/classmethod.
                if isinstance(val, staticmethod):
                    val = getattr(obj, valname)
                if isinstance(val, classmethod):
916
                    val = getattr(obj, valname).__func__
917 918 919

                # Recurse to methods, properties, and nested classes.
                if ((inspect.isfunction(val) or inspect.isclass(val) or
920 921
                      isinstance(val, property)) and
                      self._from_module(module, val)):
922 923
                    valname = '%s.%s' % (name, valname)
                    self._find(tests, val, valname, module, source_lines,
924
                               globs, seen)
925 926 927 928 929 930 931 932

    def _get_test(self, obj, name, module, globs, source_lines):
        """
        Return a DocTest for the given object, if it defines a docstring;
        otherwise, return None.
        """
        # Extract the object's docstring.  If it doesn't have one,
        # then return None (no test for this object).
933
        if isinstance(obj, str):
934 935 936 937
            docstring = obj
        else:
            try:
                if obj.__doc__ is None:
938 939
                    docstring = ''
                else:
940
                    docstring = obj.__doc__
941
                    if not isinstance(docstring, str):
942
                        docstring = str(docstring)
943
            except (TypeError, AttributeError):
944
                docstring = ''
945 946 947 948

        # Find the docstring's location in the file.
        lineno = self._find_lineno(obj, source_lines)

949 950 951 952
        # Don't bother if the docstring is empty.
        if self._exclude_empty and not docstring:
            return None

953 954 955 956 957
        # Return a DocTest for this object.
        if module is None:
            filename = None
        else:
            filename = getattr(module, '__file__', module.__name__)
Jim Fulton's avatar
Jim Fulton committed
958 959
            if filename[-4:] in (".pyc", ".pyo"):
                filename = filename[:-1]
960 961
        return self._parser.get_doctest(docstring, globs, name,
                                        filename, lineno)
962

963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985
    def _find_lineno(self, obj, source_lines):
        """
        Return a line number of the given object's docstring.  Note:
        this method assumes that the object has a docstring.
        """
        lineno = None

        # Find the line number for modules.
        if inspect.ismodule(obj):
            lineno = 0

        # Find the line number for classes.
        # Note: this could be fooled if a class is defined multiple
        # times in a single file.
        if inspect.isclass(obj):
            if source_lines is None:
                return None
            pat = re.compile(r'^\s*class\s*%s\b' %
                             getattr(obj, '__name__', '-'))
            for i, line in enumerate(source_lines):
                if pat.match(line):
                    lineno = i
                    break
986

987
        # Find the line number for functions & methods.
988
        if inspect.ismethod(obj): obj = obj.__func__
989
        if inspect.isfunction(obj): obj = obj.__code__
990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011
        if inspect.istraceback(obj): obj = obj.tb_frame
        if inspect.isframe(obj): obj = obj.f_code
        if inspect.iscode(obj):
            lineno = getattr(obj, 'co_firstlineno', None)-1

        # Find the line number where the docstring starts.  Assume
        # that it's the first line that begins with a quote mark.
        # Note: this could be fooled by a multiline function
        # signature, where a continuation line begins with a quote
        # mark.
        if lineno is not None:
            if source_lines is None:
                return lineno+1
            pat = re.compile('(^|.*:)\s*\w*("|\')')
            for lineno in range(lineno, len(source_lines)):
                if pat.match(source_lines[lineno]):
                    return lineno

        # We couldn't find the line number.
        return None

######################################################################
1012
## 5. DocTest Runner
1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023
######################################################################

class DocTestRunner:
    """
    A class used to run DocTest test cases, and accumulate statistics.
    The `run` method is used to process a single DocTest case.  It
    returns a tuple `(f, t)`, where `t` is the number of test cases
    tried, and `f` is the number of test cases that failed.

        >>> tests = DocTestFinder().find(_TestClass)
        >>> runner = DocTestRunner(verbose=False)
1024
        >>> tests.sort(key = lambda test: test.name)
1025
        >>> for test in tests:
1026
        ...     print(test.name, '->', runner.run(test))
1027 1028 1029 1030
        _TestClass -> TestResults(failed=0, attempted=2)
        _TestClass.__init__ -> TestResults(failed=0, attempted=2)
        _TestClass.get -> TestResults(failed=0, attempted=2)
        _TestClass.square -> TestResults(failed=0, attempted=1)
1031

1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044
    The `summarize` method prints a summary of all the test cases that
    have been run by the runner, and returns an aggregated `(f, t)`
    tuple:

        >>> runner.summarize(verbose=1)
        4 items passed all tests:
           2 tests in _TestClass
           2 tests in _TestClass.__init__
           2 tests in _TestClass.get
           1 tests in _TestClass.square
        7 tests in 4 items.
        7 passed and 0 failed.
        Test passed.
1045
        TestResults(failed=0, attempted=7)
1046 1047 1048 1049 1050 1051 1052 1053 1054 1055

    The aggregated number of tried examples and failed examples is
    also available via the `tries` and `failures` attributes:

        >>> runner.tries
        7
        >>> runner.failures
        0

    The comparison between expected outputs and actual outputs is done
1056 1057 1058 1059 1060
    by an `OutputChecker`.  This comparison may be customized with a
    number of option flags; see the documentation for `testmod` for
    more information.  If the option flags are insufficient, then the
    comparison may also be customized by passing a subclass of
    `OutputChecker` to the constructor.
1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073

    The test runner's display output can be controlled in two ways.
    First, an output function (`out) can be passed to
    `TestRunner.run`; this function will be called with strings that
    should be displayed.  It defaults to `sys.stdout.write`.  If
    capturing the output is not sufficient, then the display output
    can be also customized by subclassing DocTestRunner, and
    overriding the methods `report_start`, `report_success`,
    `report_unexpected_exception`, and `report_failure`.
    """
    # This divider string is used to separate failure messages, and to
    # separate sections of the summary.
    DIVIDER = "*" * 70
1074

1075
    def __init__(self, checker=None, verbose=None, optionflags=0):
1076 1077
        """
        Create a new test runner.
1078

1079 1080 1081 1082
        Optional keyword arg `checker` is the `OutputChecker` that
        should be used to compare the expected outputs and actual
        outputs of doctest examples.

1083 1084 1085
        Optional keyword arg 'verbose' prints lots of stuff if true,
        only failures if false; by default, it's true iff '-v' is in
        sys.argv.
1086

1087 1088 1089 1090 1091
        Optional argument `optionflags` can be used to control how the
        test runner compares expected output to actual output, and how
        it displays failures.  See the documentation for `testmod` for
        more information.
        """
1092
        self._checker = checker or OutputChecker()
1093 1094 1095 1096
        if verbose is None:
            verbose = '-v' in sys.argv
        self._verbose = verbose
        self.optionflags = optionflags
Jim Fulton's avatar
Jim Fulton committed
1097
        self.original_optionflags = optionflags
1098

1099 1100 1101 1102
        # Keep track of the examples we've run.
        self.tries = 0
        self.failures = 0
        self._name2ft = {}
1103

1104 1105
        # Create a fake output target for capturing doctest output.
        self._fakeout = _SpoofOut()
1106

1107 1108 1109 1110 1111 1112 1113 1114
    #/////////////////////////////////////////////////////////////////
    # Reporting methods
    #/////////////////////////////////////////////////////////////////

    def report_start(self, out, test, example):
        """
        Report that the test runner is about to process the given
        example.  (Only displays a message if verbose=True)
1115
        """
1116
        if self._verbose:
1117 1118 1119 1120 1121 1122
            if example.want:
                out('Trying:\n' + _indent(example.source) +
                    'Expecting:\n' + _indent(example.want))
            else:
                out('Trying:\n' + _indent(example.source) +
                    'Expecting nothing\n')
1123

1124 1125 1126 1127 1128 1129 1130
    def report_success(self, out, test, example, got):
        """
        Report that the given example ran successfully.  (Only
        displays a message if verbose=True)
        """
        if self._verbose:
            out("ok\n")
1131

1132 1133 1134 1135
    def report_failure(self, out, test, example, got):
        """
        Report that the given example failed.
        """
Edward Loper's avatar
Edward Loper committed
1136
        out(self._failure_header(test, example) +
1137
            self._checker.output_difference(example, got, self.optionflags))
1138

1139 1140 1141 1142
    def report_unexpected_exception(self, out, test, example, exc_info):
        """
        Report that the given example raised an unexpected exception.
        """
Edward Loper's avatar
Edward Loper committed
1143
        out(self._failure_header(test, example) +
1144
            'Exception raised:\n' + _indent(_exception_traceback(exc_info)))
1145

Edward Loper's avatar
Edward Loper committed
1146
    def _failure_header(self, test, example):
Jim Fulton's avatar
Jim Fulton committed
1147 1148 1149 1150 1151 1152 1153 1154
        out = [self.DIVIDER]
        if test.filename:
            if test.lineno is not None and example.lineno is not None:
                lineno = test.lineno + example.lineno + 1
            else:
                lineno = '?'
            out.append('File "%s", line %s, in %s' %
                       (test.filename, lineno, test.name))
1155
        else:
Jim Fulton's avatar
Jim Fulton committed
1156 1157 1158
            out.append('Line %s, in %s' % (example.lineno+1, test.name))
        out.append('Failed example:')
        source = example.source
1159 1160
        out.append(_indent(source))
        return '\n'.join(out)
1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177

    #/////////////////////////////////////////////////////////////////
    # DocTest Running
    #/////////////////////////////////////////////////////////////////

    def __run(self, test, compileflags, out):
        """
        Run the examples in `test`.  Write the outcome of each example
        with one of the `DocTestRunner.report_*` methods, using the
        writer function `out`.  `compileflags` is the set of compiler
        flags that should be used to execute examples.  Return a tuple
        `(f, t)`, where `t` is the number of examples tried, and `f`
        is the number of examples that failed.  The examples are run
        in the namespace `test.globs`.
        """
        # Keep track of the number of failures and tries.
        failures = tries = 0
1178

1179 1180 1181
        # Save the option flags (since option directives can be used
        # to modify them).
        original_optionflags = self.optionflags
1182

1183 1184 1185 1186
        SUCCESS, FAILURE, BOOM = range(3) # `outcome` state

        check = self._checker.check_output

1187
        # Process each example.
1188 1189
        for examplenum, example in enumerate(test.examples):

1190 1191 1192 1193 1194
            # If REPORT_ONLY_FIRST_FAILURE is set, then supress
            # reporting after the first failure.
            quiet = (self.optionflags & REPORT_ONLY_FIRST_FAILURE and
                     failures > 0)

1195 1196 1197 1198 1199 1200 1201 1202
            # Merge in the example's options.
            self.optionflags = original_optionflags
            if example.options:
                for (optionflag, val) in example.options.items():
                    if val:
                        self.optionflags |= optionflag
                    else:
                        self.optionflags &= ~optionflag
1203

1204 1205 1206 1207
            # If 'SKIP' is set, then skip this example.
            if self.optionflags & SKIP:
                continue

1208 1209
            # Record that we started this example.
            tries += 1
1210 1211
            if not quiet:
                self.report_start(out, test, example)
1212

1213 1214 1215 1216 1217
            # Use a special filename for compile(), so we can retrieve
            # the source code during interactive debugging (see
            # __patched_linecache_getlines).
            filename = '<doctest %s[%d]>' % (test.name, examplenum)

1218 1219 1220 1221
            # Run the example in the given context (globs), and record
            # any exception that gets raised.  (But don't intercept
            # keyboard interrupts.)
            try:
1222
                # Don't blink!  This is where the user's code gets run.
1223 1224
                exec(compile(example.source, filename, "single",
                             compileflags, 1), test.globs)
1225
                self.debugger.set_continue() # ==== Example Finished ====
1226 1227 1228 1229 1230
                exception = None
            except KeyboardInterrupt:
                raise
            except:
                exception = sys.exc_info()
1231
                self.debugger.set_continue() # ==== Example Finished ====
1232

1233
            got = self._fakeout.getvalue()  # the actual output
1234
            self._fakeout.truncate(0)
1235
            outcome = FAILURE   # guilty until proved innocent or insane
1236 1237

            # If the example executed without raising any exceptions,
1238
            # verify its output.
1239
            if exception is None:
1240 1241 1242 1243
                if check(example.want, got, self.optionflags):
                    outcome = SUCCESS

            # The example raised an exception:  check if it was expected.
1244 1245 1246
            else:
                exc_info = sys.exc_info()
                exc_msg = traceback.format_exception_only(*exc_info[:2])[-1]
1247 1248
                if not quiet:
                    got += _exception_traceback(exc_info)
1249

1250 1251
                # If `example.exc_msg` is None, then we weren't expecting
                # an exception.
1252
                if example.exc_msg is None:
1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281
                    outcome = BOOM

                # We expected an exception:  see whether it matches.
                elif check(example.exc_msg, exc_msg, self.optionflags):
                    outcome = SUCCESS

                # Another chance if they didn't care about the detail.
                elif self.optionflags & IGNORE_EXCEPTION_DETAIL:
                    m1 = re.match(r'[^:]*:', example.exc_msg)
                    m2 = re.match(r'[^:]*:', exc_msg)
                    if m1 and m2 and check(m1.group(0), m2.group(0),
                                           self.optionflags):
                        outcome = SUCCESS

            # Report the outcome.
            if outcome is SUCCESS:
                if not quiet:
                    self.report_success(out, test, example, got)
            elif outcome is FAILURE:
                if not quiet:
                    self.report_failure(out, test, example, got)
                failures += 1
            elif outcome is BOOM:
                if not quiet:
                    self.report_unexpected_exception(out, test, example,
                                                     exc_info)
                failures += 1
            else:
                assert False, ("unknown outcome", outcome)
1282 1283 1284 1285 1286 1287

        # Restore the option flags (in case they were modified)
        self.optionflags = original_optionflags

        # Record and return the number of failures and tries.
        self.__record_outcome(test, failures, tries)
1288
        return TestResults(failures, tries)
1289

1290
    def __record_outcome(self, test, f, t):
1291
        """
1292 1293 1294 1295 1296 1297 1298
        Record the fact that the given DocTest (`test`) generated `f`
        failures out of `t` tried examples.
        """
        f2, t2 = self._name2ft.get(test.name, (0,0))
        self._name2ft[test.name] = (f+f2, t+t2)
        self.failures += f
        self.tries += t
1299

1300 1301 1302
    __LINECACHE_FILENAME_RE = re.compile(r'<doctest '
                                         r'(?P<name>[\w\.]+)'
                                         r'\[(?P<examplenum>\d+)\]>$')
1303
    def __patched_linecache_getlines(self, filename, module_globals=None):
1304 1305 1306 1307 1308
        m = self.__LINECACHE_FILENAME_RE.match(filename)
        if m and m.group('name') == self.test.name:
            example = self.test.examples[int(m.group('examplenum'))]
            return example.source.splitlines(True)
        else:
1309
            return self.save_linecache_getlines(filename, module_globals)
1310

1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329
    def run(self, test, compileflags=None, out=None, clear_globs=True):
        """
        Run the examples in `test`, and display the results using the
        writer function `out`.

        The examples are run in the namespace `test.globs`.  If
        `clear_globs` is true (the default), then this namespace will
        be cleared after the test runs, to help with garbage
        collection.  If you would like to examine the namespace after
        the test completes, then use `clear_globs=False`.

        `compileflags` gives the set of flags that should be used by
        the Python compiler when running the examples.  If not
        specified, then it will default to the set of future-import
        flags that apply to `globs`.

        The output of each example is checked using
        `DocTestRunner.check_output`, and the results are formatted by
        the `DocTestRunner.report_*` methods.
1330
        """
1331 1332
        self.test = test

1333 1334
        if compileflags is None:
            compileflags = _extract_future_flags(test.globs)
1335

1336
        save_stdout = sys.stdout
1337
        if out is None:
1338 1339 1340
            out = save_stdout.write
        sys.stdout = self._fakeout

1341 1342 1343 1344 1345
        # Patch pdb.set_trace to restore sys.stdout during interactive
        # debugging (so it's not still redirected to self._fakeout).
        # Note that the interactive output will go to *our*
        # save_stdout, even if that's not the real sys.stdout; this
        # allows us to write test cases for the set_trace behavior.
1346
        save_set_trace = pdb.set_trace
1347 1348 1349 1350 1351 1352 1353 1354 1355
        self.debugger = _OutputRedirectingPdb(save_stdout)
        self.debugger.reset()
        pdb.set_trace = self.debugger.set_trace

        # Patch linecache.getlines, so we can see the example's source
        # when we're inside the debugger.
        self.save_linecache_getlines = linecache.getlines
        linecache.getlines = self.__patched_linecache_getlines

1356
        try:
1357
            return self.__run(test, compileflags, out)
1358
        finally:
1359 1360
            sys.stdout = save_stdout
            pdb.set_trace = save_set_trace
1361
            linecache.getlines = self.save_linecache_getlines
1362 1363 1364 1365 1366 1367
            if clear_globs:
                test.globs.clear()

    #/////////////////////////////////////////////////////////////////
    # Summarization
    #/////////////////////////////////////////////////////////////////
1368 1369
    def summarize(self, verbose=None):
        """
1370 1371 1372 1373 1374 1375 1376 1377
        Print a summary of all the test cases that have been run by
        this DocTestRunner, and return a tuple `(f, t)`, where `f` is
        the total number of failed examples, and `t` is the total
        number of tried examples.

        The optional `verbose` argument controls how detailed the
        summary is.  If the verbosity is not specified, then the
        DocTestRunner's verbosity is used.
1378 1379
        """
        if verbose is None:
1380
            verbose = self._verbose
1381 1382 1383 1384
        notests = []
        passed = []
        failed = []
        totalt = totalf = 0
1385
        for x in self._name2ft.items():
1386 1387
            name, (f, t) = x
            assert f <= t
1388 1389
            totalt += t
            totalf += f
1390 1391 1392 1393 1394 1395 1396 1397
            if t == 0:
                notests.append(name)
            elif f == 0:
                passed.append( (name, t) )
            else:
                failed.append(x)
        if verbose:
            if notests:
1398
                print(len(notests), "items had no tests:")
1399 1400
                notests.sort()
                for thing in notests:
1401
                    print("   ", thing)
1402
            if passed:
1403
                print(len(passed), "items passed all tests:")
1404 1405
                passed.sort()
                for thing, count in passed:
1406
                    print(" %3d tests in %s" % (count, thing))
1407
        if failed:
1408 1409
            print(self.DIVIDER)
            print(len(failed), "items had failures:")
1410 1411
            failed.sort()
            for thing, (f, t) in failed:
1412
                print(" %3d of %3d in %s" % (f, t, thing))
1413
        if verbose:
1414 1415
            print(totalt, "tests in", len(self._name2ft), "items.")
            print(totalt - totalf, "passed and", totalf, "failed.")
1416
        if totalf:
1417
            print("***Test Failed***", totalf, "failures.")
1418
        elif verbose:
1419
            print("Test passed.")
1420
        return TestResults(totalf, totalt)
1421

1422 1423 1424 1425 1426 1427 1428
    #/////////////////////////////////////////////////////////////////
    # Backward compatibility cruft to maintain doctest.master.
    #/////////////////////////////////////////////////////////////////
    def merge(self, other):
        d = self._name2ft
        for name, (f, t) in other._name2ft.items():
            if name in d:
1429 1430
                print("*** DocTestRunner.merge: '" + name + "' in both" \
                    " testers; summing outcomes.")
1431 1432 1433 1434 1435
                f2, t2 = d[name]
                f = f + f2
                t = t + t2
            d[name] = f, t

1436 1437 1438 1439 1440 1441 1442 1443 1444 1445
class OutputChecker:
    """
    A class used to check the whether the actual output from a doctest
    example matches the expected output.  `OutputChecker` defines two
    methods: `check_output`, which compares a given pair of outputs,
    and returns true if they match; and `output_difference`, which
    returns a string describing the differences between two outputs.
    """
    def check_output(self, want, got, optionflags):
        """
1446 1447 1448 1449 1450 1451 1452
        Return True iff the actual output from an example (`got`)
        matches the expected output (`want`).  These strings are
        always considered to match if they are identical; but
        depending on what option flags the test runner is using,
        several non-exact match types are also possible.  See the
        documentation for `TestRunner` for more information about
        option flags.
1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480
        """
        # Handle the common case first, for efficiency:
        # if they're string-identical, always return true.
        if got == want:
            return True

        # The values True and False replaced 1 and 0 as the return
        # value for boolean comparisons in Python 2.3.
        if not (optionflags & DONT_ACCEPT_TRUE_FOR_1):
            if (got,want) == ("True\n", "1\n"):
                return True
            if (got,want) == ("False\n", "0\n"):
                return True

        # <BLANKLINE> can be used as a special sequence to signify a
        # blank line, unless the DONT_ACCEPT_BLANKLINE flag is used.
        if not (optionflags & DONT_ACCEPT_BLANKLINE):
            # Replace <BLANKLINE> in want with a blank line.
            want = re.sub('(?m)^%s\s*?$' % re.escape(BLANKLINE_MARKER),
                          '', want)
            # If a line in got contains only spaces, then remove the
            # spaces.
            got = re.sub('(?m)^\s*?$', '', got)
            if got == want:
                return True

        # This flag causes doctest to ignore any differences in the
        # contents of whitespace strings.  Note that this can be used
Tim Peters's avatar
Tim Peters committed
1481
        # in conjunction with the ELLIPSIS flag.
1482
        if optionflags & NORMALIZE_WHITESPACE:
1483 1484 1485 1486 1487 1488
            got = ' '.join(got.split())
            want = ' '.join(want.split())
            if got == want:
                return True

        # The ELLIPSIS flag says to let the sequence "..." in `want`
1489
        # match any substring in `got`.
1490
        if optionflags & ELLIPSIS:
1491
            if _ellipsis_match(want, got):
1492 1493 1494 1495 1496
                return True

        # We didn't find any match; return false.
        return False

Tim Peters's avatar
Tim Peters committed
1497 1498 1499
    # Should we do a fancy diff?
    def _do_a_fancy_diff(self, want, got, optionflags):
        # Not unless they asked for a fancy diff.
1500 1501 1502
        if not optionflags & (REPORT_UDIFF |
                              REPORT_CDIFF |
                              REPORT_NDIFF):
Tim Peters's avatar
Tim Peters committed
1503
            return False
1504

Tim Peters's avatar
Tim Peters committed
1505
        # If expected output uses ellipsis, a meaningful fancy diff is
1506 1507 1508 1509 1510 1511 1512
        # too hard ... or maybe not.  In two real-life failures Tim saw,
        # a diff was a major help anyway, so this is commented out.
        # [todo] _ellipsis_match() knows which pieces do and don't match,
        # and could be the basis for a kick-ass diff in this case.
        ##if optionflags & ELLIPSIS and ELLIPSIS_MARKER in want:
        ##    return False

Tim Peters's avatar
Tim Peters committed
1513
        # ndiff does intraline difference marking, so can be useful even
1514
        # for 1-line differences.
1515
        if optionflags & REPORT_NDIFF:
Tim Peters's avatar
Tim Peters committed
1516
            return True
1517

Tim Peters's avatar
Tim Peters committed
1518 1519 1520
        # The other diff types need at least a few lines to be helpful.
        return want.count('\n') > 2 and got.count('\n') > 2

1521
    def output_difference(self, example, got, optionflags):
1522 1523
        """
        Return a string describing the differences between the
1524 1525 1526
        expected output for a given example (`example`) and the actual
        output (`got`).  `optionflags` is the set of option flags used
        to compare `want` and `got`.
1527
        """
1528
        want = example.want
1529 1530
        # If <BLANKLINE>s are being used, then replace blank lines
        # with <BLANKLINE> in the actual output string.
1531
        if not (optionflags & DONT_ACCEPT_BLANKLINE):
1532
            got = re.sub('(?m)^[ ]*(?=\n)', BLANKLINE_MARKER, got)
1533

1534
        # Check if we should use diff.
Tim Peters's avatar
Tim Peters committed
1535
        if self._do_a_fancy_diff(want, got, optionflags):
1536
            # Split want & got into lines.
1537 1538
            want_lines = want.splitlines(True)  # True == keep line ends
            got_lines = got.splitlines(True)
1539
            # Use difflib to find their differences.
1540
            if optionflags & REPORT_UDIFF:
1541 1542 1543
                diff = difflib.unified_diff(want_lines, got_lines, n=2)
                diff = list(diff)[2:] # strip the diff header
                kind = 'unified diff with -expected +actual'
1544
            elif optionflags & REPORT_CDIFF:
1545 1546 1547
                diff = difflib.context_diff(want_lines, got_lines, n=2)
                diff = list(diff)[2:] # strip the diff header
                kind = 'context diff with expected followed by actual'
1548
            elif optionflags & REPORT_NDIFF:
Tim Peters's avatar
Tim Peters committed
1549 1550 1551
                engine = difflib.Differ(charjunk=difflib.IS_CHARACTER_JUNK)
                diff = list(engine.compare(want_lines, got_lines))
                kind = 'ndiff with -expected +actual'
1552 1553 1554 1555
            else:
                assert 0, 'Bad diff option'
            # Remove trailing whitespace on diff output.
            diff = [line.rstrip() + '\n' for line in diff]
1556
            return 'Differences (%s):\n' % kind + _indent(''.join(diff))
1557 1558 1559

        # If we're not using diff, then simply list the expected
        # output followed by the actual output.
1560 1561 1562 1563 1564 1565 1566 1567
        if want and got:
            return 'Expected:\n%sGot:\n%s' % (_indent(want), _indent(got))
        elif want:
            return 'Expected:\n%sGot nothing\n' % _indent(want)
        elif got:
            return 'Expected nothing\nGot:\n%s' % _indent(got)
        else:
            return 'Expected nothing\nGot nothing\n'
1568

1569 1570 1571 1572 1573 1574 1575
class DocTestFailure(Exception):
    """A DocTest example has failed in debugging mode.

    The exception instance has variables:

    - test: the DocTest object being run

Neal Norwitz's avatar
Neal Norwitz committed
1576
    - example: the Example object that failed
1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594

    - got: the actual output
    """
    def __init__(self, test, example, got):
        self.test = test
        self.example = example
        self.got = got

    def __str__(self):
        return str(self.test)

class UnexpectedException(Exception):
    """A DocTest example has encountered an unexpected exception

    The exception instance has variables:

    - test: the DocTest object being run

1595
    - example: the Example object that failed
1596 1597 1598 1599 1600 1601 1602 1603 1604 1605

    - exc_info: the exception info
    """
    def __init__(self, test, example, exc_info):
        self.test = test
        self.example = example
        self.exc_info = exc_info

    def __str__(self):
        return str(self.test)
Tim Peters's avatar
Tim Peters committed
1606

1607 1608 1609 1610 1611 1612 1613
class DebugRunner(DocTestRunner):
    r"""Run doc tests but raise an exception as soon as there is a failure.

       If an unexpected exception occurs, an UnexpectedException is raised.
       It contains the test, the example, and the original exception:

         >>> runner = DebugRunner(verbose=False)
1614 1615
         >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42',
         ...                                    {}, 'foo', 'foo.py', 0)
1616 1617
         >>> try:
         ...     runner.run(test)
1618 1619
         ... except UnexpectedException as f:
         ...     failure = f
1620 1621 1622 1623 1624 1625 1626 1627

         >>> failure.test is test
         True

         >>> failure.example.want
         '42\n'

         >>> exc_info = failure.exc_info
1628
         >>> raise exc_info[1] # Already has the traceback
1629 1630 1631 1632 1633 1634 1635 1636 1637
         Traceback (most recent call last):
         ...
         KeyError

       We wrap the original exception to give the calling application
       access to the test and example information.

       If the output doesn't match, then a DocTestFailure is raised:

1638
         >>> test = DocTestParser().get_doctest('''
1639 1640 1641 1642 1643 1644 1645
         ...      >>> x = 1
         ...      >>> x
         ...      2
         ...      ''', {}, 'foo', 'foo.py', 0)

         >>> try:
         ...    runner.run(test)
1646 1647
         ... except DocTestFailure as f:
         ...    failure = f
1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669

       DocTestFailure objects provide access to the test:

         >>> failure.test is test
         True

       As well as to the example:

         >>> failure.example.want
         '2\n'

       and the actual output:

         >>> failure.got
         '1\n'

       If a failure or error occurs, the globals are left intact:

         >>> del test.globs['__builtins__']
         >>> test.globs
         {'x': 1}

1670
         >>> test = DocTestParser().get_doctest('''
1671 1672 1673 1674 1675 1676 1677
         ...      >>> x = 2
         ...      >>> raise KeyError
         ...      ''', {}, 'foo', 'foo.py', 0)

         >>> runner.run(test)
         Traceback (most recent call last):
         ...
1678
         doctest.UnexpectedException: <DocTest foo from foo.py:0 (2 examples)>
Tim Peters's avatar
Tim Peters committed
1679

1680 1681 1682 1683 1684 1685
         >>> del test.globs['__builtins__']
         >>> test.globs
         {'x': 2}

       But the globals are cleared if there is no error:

1686
         >>> test = DocTestParser().get_doctest('''
1687 1688 1689 1690
         ...      >>> x = 2
         ...      ''', {}, 'foo', 'foo.py', 0)

         >>> runner.run(test)
1691
         TestResults(failed=0, attempted=1)
1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709

         >>> test.globs
         {}

       """

    def run(self, test, compileflags=None, out=None, clear_globs=True):
        r = DocTestRunner.run(self, test, compileflags, out, False)
        if clear_globs:
            test.globs.clear()
        return r

    def report_unexpected_exception(self, out, test, example, exc_info):
        raise UnexpectedException(test, example, exc_info)

    def report_failure(self, out, test, example, got):
        raise DocTestFailure(test, example, got)

1710
######################################################################
1711
## 6. Test Functions
1712 1713
######################################################################
# These should be backwards compatible.
1714

1715 1716 1717 1718
# For backward compatibility, a global instance of a DocTestRunner
# class, updated by testmod.
master = None

1719
def testmod(m=None, name=None, globs=None, verbose=None,
1720
            report=True, optionflags=0, extraglobs=None,
1721
            raise_on_error=False, exclude_empty=False):
1722 1723
    """m=None, name=None, globs=None, verbose=None, report=True,
       optionflags=0, extraglobs=None, raise_on_error=False,
1724
       exclude_empty=False
1725

1726 1727
    Test examples in docstrings in functions and classes reachable
    from module m (or the current module if m is not supplied), starting
1728
    with m.__doc__.
1729 1730

    Also test examples reachable from dict m.__test__ if it exists and is
1731
    not None.  m.__test__ maps names to functions, classes and strings;
1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746
    function and class docstrings are tested even if the name is private;
    strings are tested directly, as if they were docstrings.

    Return (#failures, #tests).

    See doctest.__doc__ for an overview.

    Optional keyword arg "name" gives the name of the module; by default
    use m.__name__.

    Optional keyword arg "globs" gives a dict to be used as the globals
    when executing examples; by default, use m.__dict__.  A copy of this
    dict is actually used for each docstring, so that each docstring's
    examples start with a clean slate.

1747 1748 1749 1750
    Optional keyword arg "extraglobs" gives a dictionary that should be
    merged into the globals that are used to execute examples.  By
    default, no extra globals are used.  This is new in 2.4.

1751 1752 1753 1754 1755 1756 1757
    Optional keyword arg "verbose" prints lots of stuff if true, prints
    only failures if false; by default, it's true iff "-v" is in sys.argv.

    Optional keyword arg "report" prints a summary at the end when true,
    else prints nothing at the end.  In verbose mode, the summary is
    detailed, else very brief (in fact, empty if all tests passed).

1758
    Optional keyword arg "optionflags" or's together module constants,
1759 1760
    and defaults to 0.  This is new in 2.3.  Possible values (see the
    docs for details):
1761 1762

        DONT_ACCEPT_TRUE_FOR_1
1763 1764 1765
        DONT_ACCEPT_BLANKLINE
        NORMALIZE_WHITESPACE
        ELLIPSIS
1766
        SKIP
1767
        IGNORE_EXCEPTION_DETAIL
1768 1769 1770
        REPORT_UDIFF
        REPORT_CDIFF
        REPORT_NDIFF
1771
        REPORT_ONLY_FIRST_FAILURE
1772 1773 1774 1775 1776

    Optional keyword arg "raise_on_error" raises an exception on the
    first unexpected exception or failure. This allows failures to be
    post-mortem debugged.

1777 1778 1779 1780 1781 1782 1783 1784
    Advanced tomfoolery:  testmod runs methods of a local instance of
    class doctest.Tester, then merges the results into (or creates)
    global Tester instance doctest.master.  Methods of doctest.master
    can be called directly too, if you want to do something unusual.
    Passing report=0 to testmod is especially useful then, to delay
    displaying a summary.  Invoke doctest.master.summarize(verbose)
    when you're done fiddling.
    """
1785 1786
    global master

1787
    # If no module was given, then use __main__.
1788 1789 1790 1791 1792 1793
    if m is None:
        # DWA - m will still be None if this wasn't invoked from the command
        # line, in which case the following TypeError is about as good an error
        # as we should expect
        m = sys.modules.get('__main__')

1794 1795
    # Check that we were actually given a module.
    if not inspect.ismodule(m):
1796
        raise TypeError("testmod: module required; %r" % (m,))
1797 1798

    # If no name was given, then use the module's name.
1799 1800
    if name is None:
        name = m.__name__
1801 1802

    # Find, parse, and run all tests in the given module.
1803
    finder = DocTestFinder(exclude_empty=exclude_empty)
1804 1805 1806 1807 1808 1809

    if raise_on_error:
        runner = DebugRunner(verbose=verbose, optionflags=optionflags)
    else:
        runner = DocTestRunner(verbose=verbose, optionflags=optionflags)

1810 1811 1812
    for test in finder.find(m, name, globs=globs, extraglobs=extraglobs):
        runner.run(test)

1813
    if report:
1814
        runner.summarize()
1815

1816 1817 1818 1819 1820
    if master is None:
        master = runner
    else:
        master.merge(runner)

1821
    return TestResults(runner.failures, runner.tries)
1822

1823 1824
def testfile(filename, module_relative=True, name=None, package=None,
             globs=None, verbose=None, report=True, optionflags=0,
1825 1826
             extraglobs=None, raise_on_error=False, parser=DocTestParser(),
             encoding=None):
1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844
    """
    Test examples in the given file.  Return (#failures, #tests).

    Optional keyword arg "module_relative" specifies how filenames
    should be interpreted:

      - If "module_relative" is True (the default), then "filename"
         specifies a module-relative path.  By default, this path is
         relative to the calling module's directory; but if the
         "package" argument is specified, then it is relative to that
         package.  To ensure os-independence, "filename" should use
         "/" characters to separate path segments, and should not
         be an absolute path (i.e., it may not begin with "/").

      - If "module_relative" is False, then "filename" specifies an
        os-specific path.  The path may be absolute or relative (to
        the current working directory).

Edward Loper's avatar
Edward Loper committed
1845 1846
    Optional keyword arg "name" gives the name of the test; by default
    use the file's basename.
1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877

    Optional keyword argument "package" is a Python package or the
    name of a Python package whose directory should be used as the
    base directory for a module relative filename.  If no package is
    specified, then the calling module's directory is used as the base
    directory for module relative filenames.  It is an error to
    specify "package" if "module_relative" is False.

    Optional keyword arg "globs" gives a dict to be used as the globals
    when executing examples; by default, use {}.  A copy of this dict
    is actually used for each docstring, so that each docstring's
    examples start with a clean slate.

    Optional keyword arg "extraglobs" gives a dictionary that should be
    merged into the globals that are used to execute examples.  By
    default, no extra globals are used.

    Optional keyword arg "verbose" prints lots of stuff if true, prints
    only failures if false; by default, it's true iff "-v" is in sys.argv.

    Optional keyword arg "report" prints a summary at the end when true,
    else prints nothing at the end.  In verbose mode, the summary is
    detailed, else very brief (in fact, empty if all tests passed).

    Optional keyword arg "optionflags" or's together module constants,
    and defaults to 0.  Possible values (see the docs for details):

        DONT_ACCEPT_TRUE_FOR_1
        DONT_ACCEPT_BLANKLINE
        NORMALIZE_WHITESPACE
        ELLIPSIS
1878
        SKIP
1879 1880 1881 1882 1883 1884 1885 1886 1887 1888
        IGNORE_EXCEPTION_DETAIL
        REPORT_UDIFF
        REPORT_CDIFF
        REPORT_NDIFF
        REPORT_ONLY_FIRST_FAILURE

    Optional keyword arg "raise_on_error" raises an exception on the
    first unexpected exception or failure. This allows failures to be
    post-mortem debugged.

1889 1890 1891
    Optional keyword arg "parser" specifies a DocTestParser (or
    subclass) that should be used to extract tests from the files.

1892 1893 1894
    Optional keyword arg "encoding" specifies an encoding that should
    be used to convert the file to unicode.

1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907
    Advanced tomfoolery:  testmod runs methods of a local instance of
    class doctest.Tester, then merges the results into (or creates)
    global Tester instance doctest.master.  Methods of doctest.master
    can be called directly too, if you want to do something unusual.
    Passing report=0 to testmod is especially useful then, to delay
    displaying a summary.  Invoke doctest.master.summarize(verbose)
    when you're done fiddling.
    """
    global master

    if package and not module_relative:
        raise ValueError("Package may only be specified for module-"
                         "relative paths.")
Tim Peters's avatar
Tim Peters committed
1908

1909
    # Relativize the path
1910 1911
    text, filename = _load_testfile(filename, package, module_relative,
                                    encoding or "utf-8")
1912 1913 1914

    # If no name was given, then use the file's name.
    if name is None:
Edward Loper's avatar
Edward Loper committed
1915
        name = os.path.basename(filename)
1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930

    # Assemble the globals.
    if globs is None:
        globs = {}
    else:
        globs = globs.copy()
    if extraglobs is not None:
        globs.update(extraglobs)

    if raise_on_error:
        runner = DebugRunner(verbose=verbose, optionflags=optionflags)
    else:
        runner = DocTestRunner(verbose=verbose, optionflags=optionflags)

    # Read the file, convert it to a test, and run it.
1931
    test = parser.get_doctest(text, globs, name, filename, 0)
1932 1933 1934 1935 1936 1937 1938 1939 1940 1941
    runner.run(test)

    if report:
        runner.summarize()

    if master is None:
        master = runner
    else:
        master.merge(runner)

1942
    return TestResults(runner.failures, runner.tries)
1943

1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967
def run_docstring_examples(f, globs, verbose=False, name="NoName",
                           compileflags=None, optionflags=0):
    """
    Test examples in the given object's docstring (`f`), using `globs`
    as globals.  Optional argument `name` is used in failure messages.
    If the optional argument `verbose` is true, then generate output
    even if there are no failures.

    `compileflags` gives the set of flags that should be used by the
    Python compiler when running the examples.  If not specified, then
    it will default to the set of future-import flags that apply to
    `globs`.

    Optional keyword arg `optionflags` specifies options for the
    testing and output.  See the documentation for `testmod` for more
    information.
    """
    # Find, parse, and run all tests in the given module.
    finder = DocTestFinder(verbose=verbose, recurse=False)
    runner = DocTestRunner(verbose=verbose, optionflags=optionflags)
    for test in finder.find(f, name, globs=globs):
        runner.run(test, compileflags=compileflags)

######################################################################
1968
## 7. Unittest Support
1969
######################################################################
1970

1971
_unittest_reportflags = 0
1972

1973
def set_unittest_reportflags(flags):
1974
    """Sets the unittest option flags.
1975 1976 1977 1978

    The old flag is returned so that a runner could restore the old
    value if it wished to:

1979 1980 1981
      >>> import doctest
      >>> old = doctest._unittest_reportflags
      >>> doctest.set_unittest_reportflags(REPORT_NDIFF |
1982 1983 1984 1985 1986 1987
      ...                          REPORT_ONLY_FIRST_FAILURE) == old
      True

      >>> doctest._unittest_reportflags == (REPORT_NDIFF |
      ...                                   REPORT_ONLY_FIRST_FAILURE)
      True
Tim Peters's avatar
Tim Peters committed
1988

1989 1990
    Only reporting flags can be set:

1991
      >>> doctest.set_unittest_reportflags(ELLIPSIS)
1992 1993
      Traceback (most recent call last):
      ...
1994
      ValueError: ('Only reporting flags allowed', 8)
1995

1996
      >>> doctest.set_unittest_reportflags(old) == (REPORT_NDIFF |
1997 1998 1999 2000
      ...                                   REPORT_ONLY_FIRST_FAILURE)
      True
    """
    global _unittest_reportflags
2001 2002 2003

    if (flags & REPORTING_FLAGS) != flags:
        raise ValueError("Only reporting flags allowed", flags)
2004 2005 2006
    old = _unittest_reportflags
    _unittest_reportflags = flags
    return old
Tim Peters's avatar
Tim Peters committed
2007

2008

2009
class DocTestCase(unittest.TestCase):
2010

2011 2012
    def __init__(self, test, optionflags=0, setUp=None, tearDown=None,
                 checker=None):
Jim Fulton's avatar
Jim Fulton committed
2013

2014
        unittest.TestCase.__init__(self)
2015
        self._dt_optionflags = optionflags
2016
        self._dt_checker = checker
2017 2018 2019
        self._dt_test = test
        self._dt_setUp = setUp
        self._dt_tearDown = tearDown
2020 2021

    def setUp(self):
2022
        test = self._dt_test
Tim Peters's avatar
Tim Peters committed
2023

2024
        if self._dt_setUp is not None:
2025
            self._dt_setUp(test)
2026 2027

    def tearDown(self):
2028 2029
        test = self._dt_test

2030
        if self._dt_tearDown is not None:
2031 2032 2033
            self._dt_tearDown(test)

        test.globs.clear()
2034 2035

    def runTest(self):
2036
        test = self._dt_test
2037 2038
        old = sys.stdout
        new = StringIO()
2039
        optionflags = self._dt_optionflags
Tim Peters's avatar
Tim Peters committed
2040

2041
        if not (optionflags & REPORTING_FLAGS):
2042 2043 2044
            # The option flags don't include any reporting flags,
            # so add the default reporting flags
            optionflags |= _unittest_reportflags
Tim Peters's avatar
Tim Peters committed
2045

2046
        runner = DocTestRunner(optionflags=optionflags,
2047
                               checker=self._dt_checker, verbose=False)
2048

2049
        try:
2050
            runner.DIVIDER = "-"*70
2051 2052
            failures, tries = runner.run(
                test, out=new.write, clear_globs=False)
2053 2054 2055 2056
        finally:
            sys.stdout = old

        if failures:
2057 2058 2059 2060 2061 2062 2063
            raise self.failureException(self.format_failure(new.getvalue()))

    def format_failure(self, err):
        test = self._dt_test
        if test.lineno is None:
            lineno = 'unknown line number'
        else:
Jim Fulton's avatar
Jim Fulton committed
2064
            lineno = '%s' % test.lineno
2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082
        lname = '.'.join(test.name.split('.')[-1:])
        return ('Failed doctest test for %s\n'
                '  File "%s", line %s, in %s\n\n%s'
                % (test.name, test.filename, lineno, lname, err)
                )

    def debug(self):
        r"""Run the test case without results and without catching exceptions

           The unit test framework includes a debug method on test cases
           and test suites to support post-mortem debugging.  The test code
           is run in such a way that errors are not caught.  This way a
           caller can catch the errors and initiate post-mortem debugging.

           The DocTestCase provides a debug method that raises
           UnexpectedException errors if there is an unexepcted
           exception:

2083
             >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42',
2084 2085 2086 2087
             ...                {}, 'foo', 'foo.py', 0)
             >>> case = DocTestCase(test)
             >>> try:
             ...     case.debug()
2088 2089
             ... except UnexpectedException as f:
             ...     failure = f
2090 2091 2092

           The UnexpectedException contains the test, the example, and
           the original exception:
2093

2094 2095 2096 2097 2098 2099 2100
             >>> failure.test is test
             True

             >>> failure.example.want
             '42\n'

             >>> exc_info = failure.exc_info
2101
             >>> raise exc_info[1] # Already has the traceback
2102 2103 2104 2105 2106 2107
             Traceback (most recent call last):
             ...
             KeyError

           If the output doesn't match, then a DocTestFailure is raised:

2108
             >>> test = DocTestParser().get_doctest('''
2109 2110 2111 2112 2113 2114 2115 2116
             ...      >>> x = 1
             ...      >>> x
             ...      2
             ...      ''', {}, 'foo', 'foo.py', 0)
             >>> case = DocTestCase(test)

             >>> try:
             ...    case.debug()
2117 2118
             ... except DocTestFailure as f:
             ...    failure = f
2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136

           DocTestFailure objects provide access to the test:

             >>> failure.test is test
             True

           As well as to the example:

             >>> failure.example.want
             '2\n'

           and the actual output:

             >>> failure.got
             '1\n'

           """

2137
        self.setUp()
2138 2139
        runner = DebugRunner(optionflags=self._dt_optionflags,
                             checker=self._dt_checker, verbose=False)
2140
        runner.run(self._dt_test, clear_globs=False)
2141
        self.tearDown()
2142 2143

    def id(self):
2144
        return self._dt_test.name
2145 2146

    def __repr__(self):
2147
        name = self._dt_test.name.split('.')
2148 2149 2150 2151 2152
        return "%s (%s)" % (name[-1], '.'.join(name[:-1]))

    __str__ = __repr__

    def shortDescription(self):
2153
        return "Doctest: " + self._dt_test.name
2154

2155 2156
def DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None,
                 **options):
2157
    """
Tim Peters's avatar
Tim Peters committed
2158
    Convert doctest tests for a module to a unittest test suite.
2159

2160 2161 2162 2163
    This converts each documentation string in a module that
    contains doctest tests to a unittest test case.  If any of the
    tests in a doc string fail, then the test case fails.  An exception
    is raised showing the name of the file containing the test and a
2164 2165
    (sometimes approximate) line number.

2166
    The `module` argument provides the module to be tested.  The argument
2167 2168 2169
    can be either a module or a module name.

    If no argument is given, the calling module is used.
2170 2171 2172 2173

    A number of options may be provided as keyword arguments:

    setUp
Edward Loper's avatar
Edward Loper committed
2174
      A set-up function.  This is called before running the
2175 2176 2177 2178 2179
      tests in each file. The setUp function will be passed a DocTest
      object.  The setUp function can access the test globals as the
      globs attribute of the test passed.

    tearDown
Edward Loper's avatar
Edward Loper committed
2180
      A tear-down function.  This is called after running the
2181 2182 2183 2184 2185 2186 2187 2188 2189
      tests in each file.  The tearDown function will be passed a DocTest
      object.  The tearDown function can access the test globals as the
      globs attribute of the test passed.

    globs
      A dictionary containing initial global variables for the tests.

    optionflags
       A set of doctest option flags expressed as an integer.
2190
    """
2191

2192 2193 2194
    if test_finder is None:
        test_finder = DocTestFinder()

2195 2196
    module = _normalize_module(module)
    tests = test_finder.find(module, globs=globs, extraglobs=extraglobs)
2197 2198 2199
    if not tests:
        # Why do we want to do this? Because it reveals a bug that might
        # otherwise be hidden.
2200
        raise ValueError(module, "has no tests")
2201 2202 2203

    tests.sort()
    suite = unittest.TestSuite()
2204
    for test in tests:
2205 2206
        if len(test.examples) == 0:
            continue
2207
        if not test.filename:
2208
            filename = module.__file__
Jim Fulton's avatar
Jim Fulton committed
2209
            if filename[-4:] in (".pyc", ".pyo"):
2210
                filename = filename[:-1]
2211
            test.filename = filename
2212
        suite.addTest(DocTestCase(test, **options))
2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229

    return suite

class DocFileCase(DocTestCase):

    def id(self):
        return '_'.join(self._dt_test.name.split('.'))

    def __repr__(self):
        return self._dt_test.filename
    __str__ = __repr__

    def format_failure(self, err):
        return ('Failed doctest test for %s\n  File "%s", line 0\n\n%s'
                % (self._dt_test.name, self._dt_test.filename, err)
                )

2230
def DocFileTest(path, module_relative=True, package=None,
2231 2232
                globs=None, parser=DocTestParser(),
                encoding=None, **options):
2233 2234
    if globs is None:
        globs = {}
2235 2236
    else:
        globs = globs.copy()
Edward Loper's avatar
Edward Loper committed
2237

2238 2239 2240
    if package and not module_relative:
        raise ValueError("Package may only be specified for module-"
                         "relative paths.")
Tim Peters's avatar
Tim Peters committed
2241

2242
    # Relativize the path.
2243 2244
    doc, path = _load_testfile(path, package, module_relative,
                               encoding or "utf-8")
2245

2246 2247
    if "__file__" not in globs:
        globs["__file__"] = path
Edward Loper's avatar
Edward Loper committed
2248

2249
    # Find the file and read it.
Edward Loper's avatar
Edward Loper committed
2250
    name = os.path.basename(path)
2251

2252
    # Convert it to a test, and wrap it in a DocFileCase.
2253
    test = parser.get_doctest(doc, globs, name, path, 0)
2254
    return DocFileCase(test, **options)
2255 2256

def DocFileSuite(*paths, **kw):
2257
    """A unittest suite for one or more doctest files.
Tim Peters's avatar
Tim Peters committed
2258

2259 2260 2261
    The path to each doctest file is given as a string; the
    interpretation of that string depends on the keyword argument
    "module_relative".
2262 2263 2264

    A number of options may be provided as keyword arguments:

2265 2266 2267 2268 2269 2270 2271 2272 2273
    module_relative
      If "module_relative" is True, then the given file paths are
      interpreted as os-independent module-relative paths.  By
      default, these paths are relative to the calling module's
      directory; but if the "package" argument is specified, then
      they are relative to that package.  To ensure os-independence,
      "filename" should use "/" characters to separate path
      segments, and may not be an absolute path (i.e., it may not
      begin with "/").
Tim Peters's avatar
Tim Peters committed
2274

2275 2276 2277 2278
      If "module_relative" is False, then the given file paths are
      interpreted as os-specific paths.  These paths may be absolute
      or relative (to the current working directory).

2279
    package
2280 2281 2282 2283 2284 2285
      A Python package or the name of a Python package whose directory
      should be used as the base directory for module relative paths.
      If "package" is not specified, then the calling module's
      directory is used as the base directory for module relative
      filenames.  It is an error to specify "package" if
      "module_relative" is False.
2286 2287

    setUp
Edward Loper's avatar
Edward Loper committed
2288
      A set-up function.  This is called before running the
2289 2290 2291
      tests in each file. The setUp function will be passed a DocTest
      object.  The setUp function can access the test globals as the
      globs attribute of the test passed.
2292 2293

    tearDown
Edward Loper's avatar
Edward Loper committed
2294
      A tear-down function.  This is called after running the
2295 2296 2297
      tests in each file.  The tearDown function will be passed a DocTest
      object.  The tearDown function can access the test globals as the
      globs attribute of the test passed.
2298 2299 2300

    globs
      A dictionary containing initial global variables for the tests.
2301 2302

    optionflags
2303 2304 2305 2306 2307
      A set of doctest option flags expressed as an integer.

    parser
      A DocTestParser (or subclass) that should be used to extract
      tests from the files.
2308 2309 2310

    encoding
      An encoding that will be used to convert the files to unicode.
2311 2312 2313 2314 2315 2316
    """
    suite = unittest.TestSuite()

    # We do this here so that _normalize_module is called at the right
    # level.  If it were called in DocFileTest, then this function
    # would be the caller and we might guess the package incorrectly.
2317 2318
    if kw.get('module_relative', True):
        kw['package'] = _normalize_module(kw.get('package'))
2319 2320 2321

    for path in paths:
        suite.addTest(DocFileTest(path, **kw))
2322

2323 2324
    return suite

2325
######################################################################
2326
## 8. Debugging Support
2327
######################################################################
2328

2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360
def script_from_examples(s):
    r"""Extract script from text with examples.

       Converts text with examples to a Python script.  Example input is
       converted to regular code.  Example output and all other words
       are converted to comments:

       >>> text = '''
       ...       Here are examples of simple math.
       ...
       ...           Python has super accurate integer addition
       ...
       ...           >>> 2 + 2
       ...           5
       ...
       ...           And very friendly error messages:
       ...
       ...           >>> 1/0
       ...           To Infinity
       ...           And
       ...           Beyond
       ...
       ...           You can use logic if you want:
       ...
       ...           >>> if 0:
       ...           ...    blah
       ...           ...    blah
       ...           ...
       ...
       ...           Ho hum
       ...           '''

2361
       >>> print(script_from_examples(text))
2362
       # Here are examples of simple math.
2363
       #
2364
       #     Python has super accurate integer addition
2365 2366 2367
       #
       2 + 2
       # Expected:
2368
       ## 5
2369
       #
2370
       #     And very friendly error messages:
2371 2372 2373
       #
       1/0
       # Expected:
2374 2375 2376
       ## To Infinity
       ## And
       ## Beyond
2377
       #
2378
       #     You can use logic if you want:
2379 2380 2381 2382 2383
       #
       if 0:
          blah
          blah
       #
2384
       #     Ho hum
2385
       <BLANKLINE>
2386
       """
2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407
    output = []
    for piece in DocTestParser().parse(s):
        if isinstance(piece, Example):
            # Add the example's source code (strip trailing NL)
            output.append(piece.source[:-1])
            # Add the expected output:
            want = piece.want
            if want:
                output.append('# Expected:')
                output += ['## '+l for l in want.split('\n')[:-1]]
        else:
            # Add non-example text.
            output += [_comment_line(l)
                       for l in piece.split('\n')[:-1]]

    # Trim junk on both ends.
    while output and output[-1] == '#':
        output.pop()
    while output and output[0] == '#':
        output.pop(0)
    # Combine the output, and return it.
2408 2409
    # Add a courtesy newline to prevent exec from choking (see bug #1172785)
    return '\n'.join(output) + '\n'
2410 2411

def testsource(module, name):
2412
    """Extract the test sources from a doctest docstring as a script.
2413 2414

    Provide the module (or dotted name of the module) containing the
2415 2416
    test to be debugged and the name (within the module) of the object
    with the doc string with tests to be debugged.
2417
    """
2418 2419 2420
    module = _normalize_module(module)
    tests = DocTestFinder().find(module)
    test = [t for t in tests if t.name == name]
2421 2422 2423
    if not test:
        raise ValueError(name, "not found in tests")
    test = test[0]
2424
    testsrc = script_from_examples(test.docstring)
2425 2426 2427
    return testsrc

def debug_src(src, pm=False, globs=None):
2428 2429
    """Debug a single doctest docstring, in argument `src`'"""
    testsrc = script_from_examples(src)
2430
    debug_script(testsrc, pm, globs)
2431 2432

def debug_script(src, pm=False, globs=None):
2433
    "Debug a test script.  `src` is the script, as a string."
2434 2435
    import pdb

2436 2437
    # Note that tempfile.NameTemporaryFile() cannot be used.  As the
    # docs say, a file so created cannot be opened by name a second time
2438
    # on modern Windows boxes, and exec() needs to open and read it.
2439
    srcfilename = tempfile.mktemp(".py", "doctestdebug")
2440 2441 2442 2443
    f = open(srcfilename, 'w')
    f.write(src)
    f.close()

2444 2445 2446 2447 2448
    try:
        if globs:
            globs = globs.copy()
        else:
            globs = {}
2449

2450 2451
        if pm:
            try:
2452
                exec(open(srcfilename).read(), globs, globs)
2453
            except:
2454
                print(sys.exc_info()[1])
2455 2456
                pdb.post_mortem(sys.exc_info()[2])
        else:
2457 2458 2459 2460 2461 2462
            fp = open(srcfilename)
            try:
                script = fp.read()
            finally:
                fp.close()
            pdb.run("exec(%r)" % script, globs, globs)
2463 2464 2465

    finally:
        os.remove(srcfilename)
2466

2467
def debug(module, name, pm=False):
2468
    """Debug a single doctest docstring.
2469 2470 2471

    Provide the module (or dotted name of the module) containing the
    test to be debugged and the name (within the module) of the object
2472
    with the docstring with tests to be debugged.
2473
    """
2474
    module = _normalize_module(module)
2475 2476 2477
    testsrc = testsource(module, name)
    debug_script(testsrc, pm, module.__dict__)

2478
######################################################################
2479
## 9. Example Usage
2480
######################################################################
2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498
class _TestClass:
    """
    A pointless class, for sanity-checking of docstring testing.

    Methods:
        square()
        get()

    >>> _TestClass(13).get() + _TestClass(-12).get()
    1
    >>> hex(_TestClass(13).square().get())
    '0xa9'
    """

    def __init__(self, val):
        """val -> _TestClass object with associated value val.

        >>> t = _TestClass(123)
2499
        >>> print(t.get())
2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518
        123
        """

        self.val = val

    def square(self):
        """square() -> square TestClass's associated value

        >>> _TestClass(13).square().get()
        169
        """

        self.val = self.val ** 2
        return self

    def get(self):
        """get() -> return TestClass's associated value.

        >>> x = _TestClass(-42)
2519
        >>> print(x.get())
2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530
        -42
        """

        return self.val

__test__ = {"_TestClass": _TestClass,
            "string": r"""
                      Example of a string object, searched as-is.
                      >>> x = 1; y = 2
                      >>> x + y, x * y
                      (3, 2)
2531
                      """,
Tim Peters's avatar
Tim Peters committed
2532

2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547
            "bool-int equivalence": r"""
                                    In 2.2, boolean expressions displayed
                                    0 or 1.  By default, we still accept
                                    them.  This can be disabled by passing
                                    DONT_ACCEPT_TRUE_FOR_1 to the new
                                    optionflags argument.
                                    >>> 4 == 4
                                    1
                                    >>> 4 == 4
                                    True
                                    >>> 4 > 4
                                    0
                                    >>> 4 > 4
                                    False
                                    """,
Tim Peters's avatar
Tim Peters committed
2548

2549
            "blank lines": r"""
Tim Peters's avatar
Tim Peters committed
2550
                Blank lines can be marked with <BLANKLINE>:
2551
                    >>> print('foo\n\nbar\n')
Tim Peters's avatar
Tim Peters committed
2552 2553 2554 2555 2556 2557 2558 2559 2560
                    foo
                    <BLANKLINE>
                    bar
                    <BLANKLINE>
            """,

            "ellipsis": r"""
                If the ellipsis flag is used, then '...' can be used to
                elide substrings in the desired output:
2561
                    >>> print(list(range(1000))) #doctest: +ELLIPSIS
Tim Peters's avatar
Tim Peters committed
2562 2563 2564 2565 2566 2567
                    [0, 1, 2, ..., 999]
            """,

            "whitespace normalization": r"""
                If the whitespace normalization flag is used, then
                differences in whitespace are ignored.
2568
                    >>> print(list(range(30))) #doctest: +NORMALIZE_WHITESPACE
Tim Peters's avatar
Tim Peters committed
2569 2570 2571
                    [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
                     15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
                     27, 28, 29]
2572
            """,
Tim Peters's avatar
Tim Peters committed
2573
           }
2574

2575
def _test():
2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586
    testfiles = [arg for arg in sys.argv[1:] if arg and arg[0] != '-']
    if testfiles:
        for filename in testfiles:
            if filename.endswith(".py"):
                # It is a module -- insert its dir into sys.path and try to
                # import it. If it is part of a package, that possibly won't work
                # because of package imports.
                dirname, filename = os.path.split(filename)
                sys.path.insert(0, dirname)
                m = __import__(filename[:-3])
                del sys.path[0]
2587
                failures, _ = testmod(m)
2588
            else:
2589 2590 2591
                failures, _ = testfile(filename, module_relative=False)
            if failures:
                return 1
2592 2593 2594
    else:
        r = unittest.TextTestRunner()
        r.run(DocTestSuite())
2595
    return 0
2596 2597

if __name__ == "__main__":
2598
    sys.exit(_test())