Fixed #22102 -- Made SimpleTestCase tests run before unittest.TestCase ones

Thanks aptiko for the reporti and Tim Graham for the review.

django/test/runner.py

@@ -6,7 +6,7 @@ from unittest import TestSuite, defaultTestLoader
 
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
-from django.test import TestCase
+from django.test import SimpleTestCase, TestCase
 from django.test.utils import setup_test_environment, teardown_test_environment
 
 
@@ -18,7 +18,7 @@ class DiscoverRunner(object):
     test_suite = TestSuite
     test_runner = unittest.TextTestRunner
     test_loader = defaultTestLoader
-    reorder_by = (TestCase, )
+    reorder_by = (TestCase, SimpleTestCase)
     option_list = (
         make_option('-t', '--top-level-directory',
             action='store', dest='top_level', default=None,

docs/topics/testing/overview.txt

@@ -206,13 +206,13 @@ the Django test runner reorders tests in the following way:
 
 * All :class:`~django.test.TestCase` subclasses are run first.
 
-* Then, all other unittests (including :class:`unittest.TestCase`,
-  :class:`~django.test.SimpleTestCase` and
+* Then, all other Django-based tests (test cases based on
+  :class:`~django.test.SimpleTestCase`, including
   :class:`~django.test.TransactionTestCase`) are run with no particular
   ordering guaranteed nor enforced among them.
 
-* Then any other tests (e.g. doctests) that may alter the database without
-  restoring it to its original state are run.
+* Then any other :class:`unittest.TestCase` tests (including doctests) that may
+  alter the database without restoring it to its original state are run.
 
 .. note::
 

tests/test_discovery_sample/doctests.py

+"""
+Doctest example from the official Python documentation.
+https://docs.python.org/3/library/doctest.html
+"""
+
+def factorial(n):
+    """Return the factorial of n, an exact integer >= 0.
+
+    >>> [factorial(n) for n in range(6)]
+    [1, 1, 2, 6, 24, 120]
+    >>> factorial(30)
+    265252859812191058636308480000000
+    >>> factorial(-1)
+    Traceback (most recent call last):
+        ...
+    ValueError: n must be >= 0
+
+    Factorials of floats are OK, but the float must be an exact integer:
+    >>> factorial(30.1)
+    Traceback (most recent call last):
+        ...
+    ValueError: n must be exact integer
+    >>> factorial(30.0)
+    265252859812191058636308480000000
+
+    It must also not be ridiculously large:
+    >>> factorial(1e100)
+    Traceback (most recent call last):
+        ...
+    OverflowError: n too large
+    """
+
+    import math
+    if not n >= 0:
+        raise ValueError("n must be >= 0")
+    if math.floor(n) != n:
+        raise ValueError("n must be exact integer")
+    if n+1 == n:  # catch a value like 1e300
+        raise OverflowError("n too large")
+    result = 1
+    factor = 2
+    while factor <= n:
+        result *= factor
+        factor += 1
+    return result

tests/test_discovery_sample/tests_sample.py

+import doctest
 from unittest import TestCase
 
-from django.test import TestCase as DjangoTestCase
+from django.test import SimpleTestCase, TestCase as DjangoTestCase
+
+from . import doctests
 
 
 class TestVanillaUnittest(TestCase):
@@ -15,5 +18,17 @@ class TestDjangoTestCase(DjangoTestCase):
         self.assertEqual(1, 1)
 
 
+class TestZimpleTestCase(SimpleTestCase):
+    # Z is used to trick this test case to appear after Vanilla in default suite
+
+    def test_sample(self):
+        self.assertEqual(1, 1)
+
+
 class EmptyTestCase(TestCase):
     pass
+
+
+def load_tests(loader, tests, ignore):
+    tests.addTests(doctest.DocTestSuite(doctests))
+    return tests

tests/test_runner/test_discover_runner.py

@@ -25,7 +25,7 @@ class DiscoverRunnerTest(TestCase):
             ["test_discovery_sample.tests_sample"],
         ).countTestCases()
 
-        self.assertEqual(count, 2)
+        self.assertEqual(count, 4)
 
     def test_dotted_test_class_vanilla_unittest(self):
         count = DiscoverRunner().build_suite(
@@ -61,7 +61,7 @@ class DiscoverRunnerTest(TestCase):
                 ["test_discovery_sample/"],
             ).countTestCases()
 
-        self.assertEqual(count, 3)
+        self.assertEqual(count, 5)
 
     def test_empty_label(self):
         """
@@ -103,6 +103,20 @@ class DiscoverRunnerTest(TestCase):
 
         self.assertEqual(count, 0)
 
+    def test_testcase_ordering(self):
+        suite = DiscoverRunner().build_suite(["test_discovery_sample/"])
+        tc_names = [case.__class__.__name__ for case in suite._tests]
+        self.assertEqual(
+            suite._tests[0].__class__.__name__,
+            'TestDjangoTestCase',
+            msg="TestDjangoTestCase should be the first test case")
+        self.assertEqual(
+            suite._tests[1].__class__.__name__,
+            'TestZimpleTestCase',
+            msg="TestZimpleTestCase should be the second test case")
+        # All others can follow in unspecified order, including doctests
+        self.assertIn('DocTestCase', [t.__class__.__name__ for t in suite._tests[2:]])
+
     def test_overrideable_test_suite(self):
         self.assertEqual(DiscoverRunner().test_suite, TestSuite)