bpo-30696: Fix the REPL looping endlessly when no memory (GH-4160)

Doc/c-api/veryhigh.rst

@@ -141,7 +141,8 @@ the same library that the Python runtime is using.
    Read and execute statements from a file associated with an interactive device
    until EOF is reached.  The user will be prompted using ``sys.ps1`` and
    ``sys.ps2``.  *filename* is decoded from the filesystem encoding
-   (:func:`sys.getfilesystemencoding`).  Returns ``0`` at EOF.
+   (:func:`sys.getfilesystemencoding`).  Returns ``0`` at EOF or a negative
+   number upon failure.
 
 
 .. c:var:: int (*PyOS_InputHook)(void)

Lib/test/test_repl.py

+"""Test the interactive interpreter."""
+
+import sys
+import os
+import unittest
+import subprocess
+from textwrap import dedent
+from test.support import cpython_only, SuppressCrashReport
+from test.support.script_helper import kill_python
+
+def spawn_repl(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw):
+    """Run the Python REPL with the given arguments.
+
+    kw is extra keyword args to pass to subprocess.Popen. Returns a Popen
+    object.
+    """
+
+    # To run the REPL without using a terminal, spawn python with the command
+    # line option '-i' and the process name set to '<stdin>'.
+    # The directory of argv[0] must match the directory of the Python
+    # executable for the Popen() call to python to succeed as the directory
+    # path may be used by Py_GetPath() to build the default module search
+    # path.
+    stdin_fname = os.path.join(os.path.dirname(sys.executable), "<stdin>")
+    cmd_line = [stdin_fname, '-E', '-i']
+    cmd_line.extend(args)
+
+    # Set TERM=vt100, for the rationale see the comments in spawn_python() of
+    # test.support.script_helper.
+    env = kw.setdefault('env', dict(os.environ))
+    env['TERM'] = 'vt100'
+    return subprocess.Popen(cmd_line, executable=sys.executable,
+                            stdin=subprocess.PIPE,
+                            stdout=stdout, stderr=stderr,
+                            **kw)
+
+class TestInteractiveInterpreter(unittest.TestCase):
+
+    @cpython_only
+    def test_no_memory(self):
+        # Issue #30696: Fix the interactive interpreter looping endlessly when
+        # no memory. Check also that the fix does not break the interactive
+        # loop when an exception is raised.
+        user_input = """
+            import sys, _testcapi
+            1/0
+            print('After the exception.')
+            _testcapi.set_nomemory(0)
+            sys.exit(0)
+        """
+        user_input = dedent(user_input)
+        user_input = user_input.encode()
+        p = spawn_repl()
+        with SuppressCrashReport():
+            p.stdin.write(user_input)
+        output = kill_python(p)
+        self.assertIn(b'After the exception.', output)
+        # Exit code 120: Py_FinalizeEx() failed to flush stdout and stderr.
+        self.assertIn(p.returncode, (1, 120))
+
+if __name__ == "__main__":
+    unittest.main()

Misc/NEWS.d/next/Core and Builtins/2017-10-28-22-06-03.bpo-30696.lhC3HE.rst

+Fix the interactive interpreter looping endlessly when no memory.

Python/pythonrun.c

@@ -65,6 +65,7 @@ static PyObject *run_pyc_file(FILE *, const char *, PyObject *, PyObject *,
                               PyCompilerFlags *);
 static void err_input(perrdetail *);
 static void err_free(perrdetail *);
+static int PyRun_InteractiveOneObjectEx(FILE *, PyObject *, PyCompilerFlags *);
 
 /* Parse input from a file and execute it */
 int
@@ -89,6 +90,7 @@ PyRun_InteractiveLoopFlags(FILE *fp, const char *filename_str, PyCompilerFlags *
     PyObject *filename, *v;
     int ret, err;
     PyCompilerFlags local_flags;
+    int nomem_count = 0;
 
     filename = PyUnicode_DecodeFSDefault(filename_str);
     if (filename == NULL) {
@@ -110,22 +112,32 @@ PyRun_InteractiveLoopFlags(FILE *fp, const char *filename_str, PyCompilerFlags *
         _PySys_SetObjectId(&PyId_ps2, v = PyUnicode_FromString("... "));
         Py_XDECREF(v);
     }
-    err = -1;
-    for (;;) {
-        ret = PyRun_InteractiveOneObject(fp, filename, flags);
+    err = 0;
+    do {
+        ret = PyRun_InteractiveOneObjectEx(fp, filename, flags);
+        if (ret == -1 && PyErr_Occurred()) {
+            /* Prevent an endless loop after multiple consecutive MemoryErrors
+             * while still allowing an interactive command to fail with a
+             * MemoryError. */
+            if (PyErr_ExceptionMatches(PyExc_MemoryError)) {
+                if (++nomem_count > 16) {
+                    PyErr_Clear();
+                    err = -1;
+                    break;
+                }
+            } else {
+                nomem_count = 0;
+            }
+            PyErr_Print();
+            flush_io();
+        } else {
+            nomem_count = 0;
+        }
 #ifdef Py_REF_DEBUG
         if (_PyDebug_XOptionShowRefCount() == Py_True)
             _PyDebug_PrintTotalRefs();
 #endif
-        if (ret == E_EOF) {
-            err = 0;
-            break;
-        }
-        /*
-        if (ret == E_NOMEM)
-            break;
-        */
-    }
+    } while (ret != E_EOF);
     Py_DECREF(filename);
     return err;
 }
@@ -154,8 +166,11 @@ static int PARSER_FLAGS(PyCompilerFlags *flags)
                    PyPARSE_WITH_IS_KEYWORD : 0)) : 0)
 #endif
 
-int
-PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags)
+/* A PyRun_InteractiveOneObject() auxiliary function that does not print the
+ * error on failure. */
+static int
+PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename,
+                             PyCompilerFlags *flags)
 {
     PyObject *m, *d, *v, *w, *oenc = NULL, *mod_name;
     mod_ty mod;
@@ -167,7 +182,6 @@ PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags)
 
     mod_name = _PyUnicode_FromId(&PyId___main__); /* borrowed */
     if (mod_name == NULL) {
-        PyErr_Print();
         return -1;
     }
 
@@ -227,7 +241,6 @@ PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags)
             PyErr_Clear();
             return E_EOF;
         }
-        PyErr_Print();
         return -1;
     }
     m = PyImport_AddModuleObject(mod_name);
@@ -239,8 +252,6 @@ PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags)
     v = run_mod(mod, filename, d, d, flags, arena);
     PyArena_Free(arena);
     if (v == NULL) {
-        PyErr_Print();
-        flush_io();
         return -1;
     }
     Py_DECREF(v);
@@ -248,6 +259,19 @@ PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags)
     return 0;
 }
 
+int
+PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags)
+{
+    int res;
+
+    res = PyRun_InteractiveOneObjectEx(fp, filename, flags);
+    if (res == -1) {
+        PyErr_Print();
+        flush_io();
+    }
+    return res;
+}
+
 int
 PyRun_InteractiveOneFlags(FILE *fp, const char *filename_str, PyCompilerFlags *flags)
 {