Closes issue 16709. unittest test discovery sorts test files for consistent test ordering

Doc/library/unittest.rst

@@ -1518,6 +1518,11 @@ Loading and running tests
          Modules that raise ``SkipTest`` on import are recorded as skips, not
          errors.
 
+      .. versionchanged:: 3.4
+         Paths are sorted before being imported to ensure execution order for a
+         given test suite is the same even if the underlying file system's ordering
+         is not dependent on file name like in ext3/4.
+
 
    The following attributes of a :class:`TestLoader` can be configured either by
    subclassing or assignment on an instance:

Lib/unittest/loader.py

@@ -177,6 +177,9 @@ class TestLoader(object):
         The pattern is deliberately not stored as a loader attribute so that
         packages can continue discovery themselves. top_level_dir is stored so
         load_tests does not need to pass this argument in to loader.discover().
+
+        Paths are sorted before being imported to ensure reproducible execution
+        order even on filesystems with non-alphabetical ordering like ext3/4.
         """
         set_implicit_top = False
         if top_level_dir is None and self._top_level_dir is not None:
@@ -253,7 +256,7 @@ class TestLoader(object):
 
     def _find_tests(self, start_dir, pattern):
         """Used by discovery. Yields test suites it loads."""
-        paths = os.listdir(start_dir)
+        paths = sorted(os.listdir(start_dir))
 
         for path in paths:
             full_path = os.path.join(start_dir, path)

Lib/unittest/test/test_discovery.py

@@ -46,9 +46,9 @@ class TestDiscovery(unittest.TestCase):
         def restore_isdir():
             os.path.isdir = original_isdir
 
-        path_lists = [['test1.py', 'test2.py', 'not_a_test.py', 'test_dir',
+        path_lists = [['test2.py', 'test1.py', 'not_a_test.py', 'test_dir',
                        'test.foo', 'test-not-a-module.py', 'another_dir'],
-                      ['test3.py', 'test4.py', ]]
+                      ['test4.py', 'test3.py', ]]
         os.listdir = lambda path: path_lists.pop(0)
         self.addCleanup(restore_listdir)
 
@@ -70,6 +70,8 @@ class TestDiscovery(unittest.TestCase):
         loader._top_level_dir = top_level
         suite = list(loader._find_tests(top_level, 'test*.py'))
 
+        # The test suites found should be sorted alphabetically for reliable
+        # execution order.
         expected = [name + ' module tests' for name in
                     ('test1', 'test2')]
         expected.extend([('test_dir.%s' % name) + ' module tests' for name in
@@ -132,6 +134,7 @@ class TestDiscovery(unittest.TestCase):
         # and directly from the test_directory2 package
         self.assertEqual(suite,
                          ['load_tests', 'test_directory2' + ' module tests'])
+        # The test module paths should be sorted for reliable execution order
         self.assertEqual(Module.paths, ['test_directory', 'test_directory2'])
 
         # load_tests should have been called once with loader, tests and pattern

Misc/NEWS

@@ -289,6 +289,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #16709: unittest discover order is no-longer filesystem specific. Patch
+  by Jeff Ramnani.
+
 - Issue #5024: sndhdr.whichhdr now returns the frame count for WAV files
   rather than -1.