New regression test harness. See usage message / doc string.

Lib/test/regrtest.py

+#! /usr/bin/env python
+
+"""Regression test.
+
+This will find all modules whose name is "test_*" in the test
+directory, and run them.  Various command line options provide
+additional facilities.
+
+Command line options:
+
+-v: verbose -- print the name name of each test as it is being run
+-q: quiet -- don't print anything except if a test fails
+-g: generate -- write the output file for a test instead of comparing it
+-x: exclude -- arguments are tests to *exclude*
+
+If non-option arguments are present, they are names for tests to run,
+unless -x is given, in which case they are names for tests not to run.
+If no test names are given, all tests are run.
+"""
+
+import sys
+import string
+import os
+import getopt
+
+import test_support
+
+def main():
+    try:
+	opts, args = getopt.getopt(sys.argv[1:], 'vgqx')
+    except getopt.error, msg:
+	print msg
+	print __doc__
+	sys.exit(2)
+    verbose = 0
+    quiet = 0
+    generate = 0
+    exclude = 0
+    for o, a in opts:
+	if o == '-v': verbose = 1
+	if o == '-q': quiet = 1
+	if o == '-g': generate = 1
+	if o == '-x': exclude = 1
+    good = []
+    bad = []
+    skipped = []
+    if exclude:
+	nottests[:0] = args
+	args = []
+    tests = args or findtests()
+    test_support.verbose = 0		# Tell tests to be moderately quiet
+    for test in tests:
+	if verbose:
+	    print test
+	ok = runtest(test, generate)
+	if ok > 0:
+	    good.append(test)
+	elif ok == 0:
+	    bad.append(test)
+	else:
+	    if not quiet:
+		print "test", test,
+		print "skipped -- an optional feature could not be imported"
+	    skipped.append(test)
+    if good and not quiet:
+	if not bad and not skipped and len(good) > 1:
+	    print "All",
+	print count(len(good), "test"), "OK."
+    if bad:
+	print count(len(bad), "test"), "failed:",
+	print string.join(bad)
+    if skipped and not quiet:
+	print count(len(skipped), "test"), "skipped:",
+	print string.join(skipped)
+    sys.exit(len(bad) > 0)
+
+stdtests = [
+    'test_grammar',
+    'test_opcodes',
+    'test_operations',
+    'test_builtin',
+    'test_exceptions',
+    'test_types',
+   ]
+
+nottests = [
+    'test_support',
+    'test_b1',
+    'test_b2',
+    ]
+
+def findtests():
+    """Return a list of all applicable test modules."""
+    testdir = findtestdir()
+    names = os.listdir(testdir)
+    tests = []
+    for name in names:
+	if name[:5] == "test_" and name[-3:] == ".py":
+	    modname = name[:-3]
+	    if modname not in stdtests and modname not in nottests:
+		tests.append(modname)
+    tests.sort()
+    return stdtests + tests
+
+def runtest(test, generate):
+    test_support.unload(test)
+    testdir = findtestdir()
+    outputdir = os.path.join(testdir, "output")
+    outputfile = os.path.join(outputdir, test)
+    try:
+	if generate:
+	    cfp = open(outputfile, "w")
+	else:
+	    cfp = Compare(outputfile)
+    except IOError:
+	cfp = None
+	print "Warning: can't open", outputfile
+    try:
+	save_stdout = sys.stdout
+	try:
+	    if cfp:
+		sys.stdout = cfp
+		print test		# Output file starts with test name
+	    __import__(test)
+	finally:
+	    sys.stdout = save_stdout
+    except ImportError, msg:
+	return -1
+    except test_support.TestFailed, msg:
+	print "test", test, "failed --", msg
+	return 0
+    else:
+	return 1
+
+def findtestdir():
+    if __name__ == '__main__':
+	file = sys.argv[0]
+    else:
+	file = __file__
+    testdir = os.path.dirname(file) or os.curdir
+    return testdir
+
+def count(n, word):
+    if n == 1:
+	return "%d %s" % (n, word)
+    else:
+	return "%d %ss" % (n, word)
+
+class Compare:
+
+    def __init__(self, filename):
+	self.fp = open(filename, 'r')
+
+    def write(self, data):
+	expected = self.fp.read(len(data))
+	if data <> expected:
+	    raise test_support.TestFailed, \
+		    'Writing: '+`data`+', expected: '+`expected`
+
+    def close(self):
+	leftover = self.fp.read()
+	if leftover:
+	    raise test_support.TestFailed, 'Unread: '+`leftover`
+	self.fp.close()
+
+    def isatty(self):
+	return 0
+
+if __name__ == '__main__':
+    main()