bpo-29942: Fix the use of recursion in itertools.chain.from_iterable. (#913)

Fix the use of recursion in itertools.chain.from_iterable. Using recursion
is unnecessary, and can easily cause stack overflows, especially when
building in low optimization modes or with Py_DEBUG enabled.
(cherry picked from commit 5466d4af)

Lib/test/test_itertools.py

@@ -1465,6 +1465,14 @@ class RegressionTests(unittest.TestCase):
         self.assertRaises(AssertionError, list, cycle(gen1()))
         self.assertEqual(hist, [0,1])
 
+    def test_long_chain_of_empty_iterables(self):
+        # Make sure itertools.chain doesn't run into recursion limits when
+        # dealing with long chains of empty iterables. Even with a high
+        # number this would probably only fail in Py_DEBUG mode.
+        it = chain.from_iterable(() for unused in xrange(10000000))
+        with self.assertRaises(StopIteration):
+            next(it)
+
 class SubclassWithKwargsTest(unittest.TestCase):
     def test_keywords_in_subclass(self):
         # count is not subclassable...

Misc/NEWS

@@ -42,6 +42,9 @@ Extension Modules
 Library
 -------
 
+- bpo-29942: Fix a crash in itertools.chain.from_iterable when encountering
+  long runs of empty iterables.
+
 - bpo-29861: Release references to tasks, their arguments and their results
   as soon as they are finished in multiprocessing.Pool.
 

Modules/itertoolsmodule.c

@@ -1708,9 +1708,10 @@ chain_next(chainobject *lz)
 {
     PyObject *item;
 
-    if (lz->source == NULL)
-        return NULL;                                    /* already stopped */
-
+    /* lz->source is the iterator of iterables. If it's NULL, we've already
+     * consumed them all. lz->active is the current iterator. If it's NULL,
+     * we should grab a new one from lz->source. */
+    while (lz->source != NULL) {
         if (lz->active == NULL) {
             PyObject *iterable = PyIter_Next(lz->source);
             if (iterable == NULL) {
@@ -1733,8 +1734,11 @@ chain_next(chainobject *lz)
             else
                 return NULL;                  /* input raised an exception */
         }
+        /* lz->active is consumed, try with the next iterable. */
         Py_CLEAR(lz->active);
-    return chain_next(lz);                      /* recurse and use next active */
+    }
+    /* Everything had been consumed already. */
+    return NULL;
 }
 
 PyDoc_STRVAR(chain_doc,