bpo-26732: fix too many fds in processes started with the "forkserver" method (#2813)

* bpo-26732: fix too many fds in processes started with the "forkserver" method

A child process would inherit as many fds as the number of still-running children.

* Add blurb and test comment

Lib/multiprocessing/forkserver.py

@@ -236,8 +236,11 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None):
                             code = 1
                             try:
                                 listener.close()
+                                selector.close()
+                                unused_fds = [alive_r, child_w, sig_r, sig_w]
+                                unused_fds.extend(pid_to_fd.values())
                                 code = _serve_one(child_r, fds,
-                                                  (alive_r, child_w, sig_r, sig_w),
+                                                  unused_fds,
                                                   old_handlers)
                             except Exception:
                                 sys.excepthook(*sys.exc_info())

Lib/test/_test_multiprocessing.py

@@ -494,6 +494,40 @@ class _TestProcess(BaseTestCase):
         self.assertIs(wr(), None)
         self.assertEqual(q.get(), 5)
 
+    @classmethod
+    def _test_child_fd_inflation(self, evt, q):
+        q.put(test.support.fd_count())
+        evt.wait()
+
+    def test_child_fd_inflation(self):
+        # Number of fds in child processes should not grow with the
+        # number of running children.
+        if self.TYPE == 'threads':
+            self.skipTest('test not appropriate for {}'.format(self.TYPE))
+
+        sm = multiprocessing.get_start_method()
+        if sm == 'fork':
+            # The fork method by design inherits all fds from the parent,
+            # trying to go against it is a lost battle
+            self.skipTest('test not appropriate for {}'.format(sm))
+
+        N = 5
+        evt = self.Event()
+        q = self.Queue()
+
+        procs = [self.Process(target=self._test_child_fd_inflation, args=(evt, q))
+                 for i in range(N)]
+        for p in procs:
+            p.start()
+
+        try:
+            fd_counts = [q.get() for i in range(N)]
+            self.assertEqual(len(set(fd_counts)), 1, fd_counts)
+
+        finally:
+            evt.set()
+            for p in procs:
+                p.join()
 
 #
 #

Lib/test/libregrtest/refleak.py

@@ -7,36 +7,6 @@ from inspect import isabstract
 from test import support
 
 
-try:
-    MAXFD = os.sysconf("SC_OPEN_MAX")
-except Exception:
-    MAXFD = 256
-
-
-def fd_count():
-    """Count the number of open file descriptors"""
-    if sys.platform.startswith(('linux', 'freebsd')):
-        try:
-            names = os.listdir("/proc/self/fd")
-            return len(names)
-        except FileNotFoundError:
-            pass
-
-    count = 0
-    for fd in range(MAXFD):
-        try:
-            # Prefer dup() over fstat(). fstat() can require input/output
-            # whereas dup() doesn't.
-            fd2 = os.dup(fd)
-        except OSError as e:
-            if e.errno != errno.EBADF:
-                raise
-        else:
-            os.close(fd2)
-            count += 1
-    return count
-
-
 def dash_R(the_module, test, indirect_test, huntrleaks):
     """Run a test multiple times, looking for reference leaks.
 
@@ -174,7 +144,7 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs):
     func1 = sys.getallocatedblocks
     func2 = sys.gettotalrefcount
     gc.collect()
-    return func1(), func2(), fd_count()
+    return func1(), func2(), support.fd_count()
 
 
 def clear_caches():

Lib/test/support/__init__.py

@@ -107,7 +107,7 @@ __all__ = [
     "check_warnings", "check_no_resource_warning", "EnvironmentVarGuard",
     "run_with_locale", "swap_item",
     "swap_attr", "Matcher", "set_memlimit", "SuppressCrashReport", "sortdict",
-    "run_with_tz", "PGO", "missing_compiler_executable",
+    "run_with_tz", "PGO", "missing_compiler_executable", "fd_count",
     ]
 
 class Error(Exception):
@@ -2647,3 +2647,34 @@ def disable_faulthandler():
     finally:
         if is_enabled:
             faulthandler.enable(file=fd, all_threads=True)
+
+
+try:
+    MAXFD = os.sysconf("SC_OPEN_MAX")
+except Exception:
+    MAXFD = 256
+
+
+def fd_count():
+    """Count the number of open file descriptors.
+    """
+    if sys.platform.startswith(('linux', 'freebsd')):
+        try:
+            names = os.listdir("/proc/self/fd")
+            return len(names)
+        except FileNotFoundError:
+            pass
+
+    count = 0
+    for fd in range(MAXFD):
+        try:
+            # Prefer dup() over fstat(). fstat() can require input/output
+            # whereas dup() doesn't.
+            fd2 = os.dup(fd)
+        except OSError as e:
+            if e.errno != errno.EBADF:
+                raise
+        else:
+            os.close(fd2)
+            count += 1
+    return count

Misc/NEWS.d/next/Library/2017-07-22-12-12-42.bpo-26732.lYLWBH.rst

+Fix too many fds in processes started with the "forkserver" method.
+
+A child process would inherit as many fds as the number of still-running
+children.