Issue #8844: Regular and recursive lock acquisitions can now be interrupted

by signals on platforms using pthreads.  Patch by Reid Kleckner.

Doc/library/_thread.rst

@@ -137,6 +137,10 @@ Lock objects have the following methods:
    .. versionchanged:: 3.2
       The *timeout* parameter is new.
 
+   .. versionchanged:: 3.2
+      Lock acquires can now be interrupted by signals on POSIX.
+
+
 .. method:: lock.release()
 
    Releases the lock.  The lock must have been acquired earlier, but not

Doc/library/threading.rst

@@ -408,6 +408,9 @@ All methods are executed atomically.
    .. versionchanged:: 3.2
       The *timeout* parameter is new.
 
+   .. versionchanged:: 3.2
+      Lock acquires can now be interrupted by signals on POSIX.
+
 
 .. method:: Lock.release()
 

Doc/whatsnew/3.2.rst

@@ -1212,6 +1212,12 @@ Multi-threading
 * Similarly, :meth:`threading.Semaphore.acquire` also gained a *timeout*
   argument.  (Contributed by Torsten Landschoff; :issue:`850728`.)
 
+* Regular and recursive lock acquisitions can now be interrupted by signals on
+  platforms using pthreads.  This means that Python programs that deadlock while
+  acquiring locks can be successfully killed by repeatedly sending SIGINT to the
+  process (ie, by pressing Ctl+C in most shells).
+  (Contributed by Reid Kleckner; :issue:`8844`.)
+
 
 Optimizations
 =============

Include/pythread.h

@@ -9,6 +9,14 @@ typedef void *PyThread_type_sema;
 extern "C" {
 #endif
 
+/* Return status codes for Python lock acquisition.  Chosen for maximum
+ * backwards compatibility, ie failure -> 0, success -> 1.  */
+typedef enum PyLockStatus {
+    PY_LOCK_FAILURE = 0,
+    PY_LOCK_ACQUIRED = 1,
+    PY_LOCK_INTR
+} PyLockStatus;
+
 PyAPI_FUNC(void) PyThread_init_thread(void);
 PyAPI_FUNC(long) PyThread_start_new_thread(void (*)(void *), void *);
 PyAPI_FUNC(void) PyThread_exit_thread(void);
@@ -49,11 +57,18 @@ PyAPI_FUNC(int) PyThread_acquire_lock(PyThread_type_lock, int);
    even when the lock can't be acquired.
    If microseconds > 0, the call waits up to the specified duration.
    If microseconds < 0, the call waits until success (or abnormal failure)
-   
+
    microseconds must be less than PY_TIMEOUT_MAX. Behaviour otherwise is
-   undefined. */
-PyAPI_FUNC(int) PyThread_acquire_lock_timed(PyThread_type_lock,
-					    PY_TIMEOUT_T microseconds);
+   undefined.
+
+   If intr_flag is true and the acquire is interrupted by a signal, then the
+   call will return PY_LOCK_INTR.  The caller may reattempt to acquire the
+   lock.
+*/
+PyAPI_FUNC(PyLockStatus) PyThread_acquire_lock_timed(PyThread_type_lock,
+                                                     PY_TIMEOUT_T microseconds,
+                                                     int intr_flag);
+
 PyAPI_FUNC(void) PyThread_release_lock(PyThread_type_lock);
 
 PyAPI_FUNC(size_t) PyThread_get_stacksize(void);

Lib/test/test_threadsignals.py

@@ -6,6 +6,7 @@ import os
 import sys
 from test.support import run_unittest, import_module
 thread = import_module('_thread')
+import time
 
 if sys.platform[:3] in ('win', 'os2') or sys.platform=='riscos':
     raise unittest.SkipTest("Can't test signal on %s" % sys.platform)
@@ -34,12 +35,12 @@ def send_signals():
     signalled_all.release()
 
 class ThreadSignals(unittest.TestCase):
-    """Test signal handling semantics of threads.
-       We spawn a thread, have the thread send two signals, and
-       wait for it to finish. Check that we got both signals
-       and that they were run by the main thread.
-    """
+
     def test_signals(self):
+        # Test signal handling semantics of threads.
+        # We spawn a thread, have the thread send two signals, and
+        # wait for it to finish. Check that we got both signals
+        # and that they were run by the main thread.
         signalled_all.acquire()
         self.spawnSignallingThread()
         signalled_all.acquire()
@@ -66,6 +67,115 @@ class ThreadSignals(unittest.TestCase):
     def spawnSignallingThread(self):
         thread.start_new_thread(send_signals, ())
 
+    def alarm_interrupt(self, sig, frame):
+        raise KeyboardInterrupt
+
+    def test_lock_acquire_interruption(self):
+        # Mimic receiving a SIGINT (KeyboardInterrupt) with SIGALRM while stuck
+        # in a deadlock.
+        oldalrm = signal.signal(signal.SIGALRM, self.alarm_interrupt)
+        try:
+            lock = thread.allocate_lock()
+            lock.acquire()
+            signal.alarm(1)
+            self.assertRaises(KeyboardInterrupt, lock.acquire)
+        finally:
+            signal.signal(signal.SIGALRM, oldalrm)
+
+    def test_rlock_acquire_interruption(self):
+        # Mimic receiving a SIGINT (KeyboardInterrupt) with SIGALRM while stuck
+        # in a deadlock.
+        oldalrm = signal.signal(signal.SIGALRM, self.alarm_interrupt)
+        try:
+            rlock = thread.RLock()
+            # For reentrant locks, the initial acquisition must be in another
+            # thread.
+            def other_thread():
+                rlock.acquire()
+            thread.start_new_thread(other_thread, ())
+            # Wait until we can't acquire it without blocking...
+            while rlock.acquire(blocking=False):
+                rlock.release()
+                time.sleep(0.01)
+            signal.alarm(1)
+            self.assertRaises(KeyboardInterrupt, rlock.acquire)
+        finally:
+            signal.signal(signal.SIGALRM, oldalrm)
+
+    def acquire_retries_on_intr(self, lock):
+        self.sig_recvd = False
+        def my_handler(signal, frame):
+            self.sig_recvd = True
+        old_handler = signal.signal(signal.SIGUSR1, my_handler)
+        try:
+            def other_thread():
+                # Acquire the lock in a non-main thread, so this test works for
+                # RLocks.
+                lock.acquire()
+                # Wait until the main thread is blocked in the lock acquire, and
+                # then wake it up with this.
+                time.sleep(0.5)
+                os.kill(process_pid, signal.SIGUSR1)
+                # Let the main thread take the interrupt, handle it, and retry
+                # the lock acquisition.  Then we'll let it run.
+                time.sleep(0.5)
+                lock.release()
+            thread.start_new_thread(other_thread, ())
+            # Wait until we can't acquire it without blocking...
+            while lock.acquire(blocking=False):
+                lock.release()
+                time.sleep(0.01)
+            result = lock.acquire()  # Block while we receive a signal.
+            self.assertTrue(self.sig_recvd)
+            self.assertTrue(result)
+        finally:
+            signal.signal(signal.SIGUSR1, old_handler)
+
+    def test_lock_acquire_retries_on_intr(self):
+        self.acquire_retries_on_intr(thread.allocate_lock())
+
+    def test_rlock_acquire_retries_on_intr(self):
+        self.acquire_retries_on_intr(thread.RLock())
+
+    def test_interrupted_timed_acquire(self):
+        # Test to make sure we recompute lock acquisition timeouts when we
+        # receive a signal.  Check this by repeatedly interrupting a lock
+        # acquire in the main thread, and make sure that the lock acquire times
+        # out after the right amount of time.
+        self.start = None
+        self.end = None
+        self.sigs_recvd = 0
+        done = thread.allocate_lock()
+        done.acquire()
+        lock = thread.allocate_lock()
+        lock.acquire()
+        def my_handler(signum, frame):
+            self.sigs_recvd += 1
+        old_handler = signal.signal(signal.SIGUSR1, my_handler)
+        try:
+            def timed_acquire():
+                self.start = time.time()
+                lock.acquire(timeout=0.5)
+                self.end = time.time()
+            def send_signals():
+                for _ in range(40):
+                    time.sleep(0.05)
+                    os.kill(process_pid, signal.SIGUSR1)
+                done.release()
+
+            # Send the signals from the non-main thread, since the main thread
+            # is the only one that can process signals.
+            thread.start_new_thread(send_signals, ())
+            timed_acquire()
+            # Wait for thread to finish
+            done.acquire()
+            # This allows for some timing and scheduling imprecision
+            self.assertLess(self.end - self.start, 2.0)
+            self.assertGreater(self.end - self.start, 0.3)
+            self.assertEqual(40, self.sigs_recvd)
+        finally:
+            signal.signal(signal.SIGUSR1, old_handler)
+
 
 def test_main():
     global signal_blackboard

Misc/ACKS

@@ -454,6 +454,7 @@ Paul Kippes
 Steve Kirsch
 Sebastian Kirsche
 Ron Klatchko
+Reid Kleckner
 Bastian Kleineidam
 Bob Kline
 Matthias Klose

Misc/NEWS

@@ -10,6 +10,9 @@ What's New in Python 3.2 Beta 2?
 Core and Builtins
 -----------------
 
+- Issue #8844: Regular and recursive lock acquisitions can now be interrupted
+  by signals on platforms using pthreads.  Patch by Reid Kleckner.
+
 - Issue #4236: PyModule_Create2 now checks the import machinery directly
   rather than the Py_IsInitialized flag, avoiding a Fatal Python
   error in certain circumstances when an import is done in __del__.

Modules/_threadmodule.c

@@ -40,6 +40,58 @@ lock_dealloc(lockobject *self)
     PyObject_Del(self);
 }
 
+/* Helper to acquire an interruptible lock with a timeout.  If the lock acquire
+ * is interrupted, signal handlers are run, and if they raise an exception,
+ * PY_LOCK_INTR is returned.  Otherwise, PY_LOCK_ACQUIRED or PY_LOCK_FAILURE
+ * are returned, depending on whether the lock can be acquired withing the
+ * timeout.
+ */
+static PyLockStatus
+acquire_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds)
+{
+    PyLockStatus r;
+    _PyTime_timeval curtime;
+    _PyTime_timeval endtime;
+
+    if (microseconds > 0) {
+        _PyTime_gettimeofday(&endtime);
+        endtime.tv_sec += microseconds / (1000 * 1000);
+        endtime.tv_usec += microseconds % (1000 * 1000);
+    }
+
+
+    do {
+        Py_BEGIN_ALLOW_THREADS
+        r = PyThread_acquire_lock_timed(lock, microseconds, 1);
+        Py_END_ALLOW_THREADS
+
+        if (r == PY_LOCK_INTR) {
+            /* Run signal handlers if we were interrupted.  Propagate
+             * exceptions from signal handlers, such as KeyboardInterrupt, by
+             * passing up PY_LOCK_INTR.  */
+            if (Py_MakePendingCalls() < 0) {
+                return PY_LOCK_INTR;
+            }
+
+            /* If we're using a timeout, recompute the timeout after processing
+             * signals, since those can take time.  */
+            if (microseconds >= 0) {
+                _PyTime_gettimeofday(&curtime);
+                microseconds = ((endtime.tv_sec - curtime.tv_sec) * 1000000 +
+                                (endtime.tv_usec - curtime.tv_usec));
+
+                /* Check for negative values, since those mean block forever.
+                 */
+                if (microseconds <= 0) {
+                    r = PY_LOCK_FAILURE;
+                }
+            }
+        }
+    } while (r == PY_LOCK_INTR);  /* Retry if we were interrupted. */
+
+    return r;
+}
+
 static PyObject *
 lock_PyThread_acquire_lock(lockobject *self, PyObject *args, PyObject *kwds)
 {
@@ -47,7 +99,7 @@ lock_PyThread_acquire_lock(lockobject *self, PyObject *args, PyObject *kwds)
     int blocking = 1;
     double timeout = -1;
     PY_TIMEOUT_T microseconds;
-    int r;
+    PyLockStatus r;
 
     if (!PyArg_ParseTupleAndKeywords(args, kwds, "|id:acquire", kwlist,
                                      &blocking, &timeout))
@@ -77,11 +129,12 @@ lock_PyThread_acquire_lock(lockobject *self, PyObject *args, PyObject *kwds)
         microseconds = (PY_TIMEOUT_T) timeout;
     }
 
-    Py_BEGIN_ALLOW_THREADS
-    r = PyThread_acquire_lock_timed(self->lock_lock, microseconds);
-    Py_END_ALLOW_THREADS
+    r = acquire_timed(self->lock_lock, microseconds);
+    if (r == PY_LOCK_INTR) {
+        return NULL;
+    }
 
-    return PyBool_FromLong(r);
+    return PyBool_FromLong(r == PY_LOCK_ACQUIRED);
 }
 
 PyDoc_STRVAR(acquire_doc,
@@ -93,7 +146,7 @@ locked (even by the same thread), waiting for another thread to release\n\
 the lock, and return None once the lock is acquired.\n\
 With an argument, this will only block if the argument is true,\n\
 and the return value reflects whether the lock is acquired.\n\
-The blocking operation is not interruptible.");
+The blocking operation is interruptible.");
 
 static PyObject *
 lock_PyThread_release_lock(lockobject *self)
@@ -218,7 +271,7 @@ rlock_acquire(rlockobject *self, PyObject *args, PyObject *kwds)
     double timeout = -1;
     PY_TIMEOUT_T microseconds;
     long tid;
-    int r = 1;
+    PyLockStatus r = PY_LOCK_ACQUIRED;
 
     if (!PyArg_ParseTupleAndKeywords(args, kwds, "|id:acquire", kwlist,
                                      &blocking, &timeout))
@@ -265,17 +318,18 @@ rlock_acquire(rlockobject *self, PyObject *args, PyObject *kwds)
         if (microseconds == 0) {
             Py_RETURN_FALSE;
         }
-        Py_BEGIN_ALLOW_THREADS
-        r = PyThread_acquire_lock_timed(self->rlock_lock, microseconds);
-        Py_END_ALLOW_THREADS
+        r = acquire_timed(self->rlock_lock, microseconds);
     }
-    if (r) {
+    if (r == PY_LOCK_ACQUIRED) {
         assert(self->rlock_count == 0);
         self->rlock_owner = tid;
         self->rlock_count = 1;
     }
+    else if (r == PY_LOCK_INTR) {
+        return NULL;
+    }
 
-    return PyBool_FromLong(r);
+    return PyBool_FromLong(r == PY_LOCK_ACQUIRED);
 }
 
 PyDoc_STRVAR(rlock_acquire_doc,
@@ -287,7 +341,7 @@ and another thread holds the lock, the method will return False\n\
 immediately.  If `blocking` is True and another thread holds\n\
 the lock, the method will wait for the lock to be released,\n\
 take it and then return True.\n\
-(note: the blocking operation is not interruptible.)\n\
+(note: the blocking operation is interruptible.)\n\
 \n\
 In all other cases, the method will return True immediately.\n\
 Precisely, if the current thread already holds the lock, its\n\

Python/thread_nt.h

@@ -238,10 +238,13 @@ PyThread_free_lock(PyThread_type_lock aLock)
  * and 0 if the lock was not acquired. This means a 0 is returned
  * if the lock has already been acquired by this thread!
  */
-int
-PyThread_acquire_lock_timed(PyThread_type_lock aLock, PY_TIMEOUT_T microseconds)
+PyLockStatus
+PyThread_acquire_lock_timed(PyThread_type_lock aLock,
+                            PY_TIMEOUT_T microseconds, int intr_flag)
 {
-    int success ;
+    /* Fow now, intr_flag does nothing on Windows, and lock acquires are
+     * uninterruptible.  */
+    PyLockStatus success;
     PY_TIMEOUT_T milliseconds;
 
     if (microseconds >= 0) {
@@ -258,7 +261,13 @@ PyThread_acquire_lock_timed(PyThread_type_lock aLock, PY_TIMEOUT_T microseconds)
     dprintf(("%ld: PyThread_acquire_lock_timed(%p, %lld) called\n",
              PyThread_get_thread_ident(), aLock, microseconds));
 
-    success = aLock && EnterNonRecursiveMutex((PNRMUTEX) aLock, (DWORD) milliseconds) == WAIT_OBJECT_0 ;
+    if (aLock && EnterNonRecursiveMutex((PNRMUTEX)aLock,
+                                        (DWORD)milliseconds) == WAIT_OBJECT_0) {
+        success = PY_LOCK_ACQUIRED;
+    }
+    else {
+        success = PY_LOCK_FAILURE;
+    }
 
     dprintf(("%ld: PyThread_acquire_lock(%p, %lld) -> %d\n",
              PyThread_get_thread_ident(), aLock, microseconds, success));
@@ -268,7 +277,7 @@ PyThread_acquire_lock_timed(PyThread_type_lock aLock, PY_TIMEOUT_T microseconds)
 int
 PyThread_acquire_lock(PyThread_type_lock aLock, int waitflag)
 {
-    return PyThread_acquire_lock_timed(aLock, waitflag ? -1 : 0);
+    return PyThread_acquire_lock_timed(aLock, waitflag ? -1 : 0, 0);
 }
 
 void

Python/thread_pthread.h

@@ -316,16 +316,17 @@ fix_status(int status)
     return (status == -1) ? errno : status;
 }
 
-int
-PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds)
+PyLockStatus
+PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds,
+                            int intr_flag)
 {
-    int success;
+    PyLockStatus success;
     sem_t *thelock = (sem_t *)lock;
     int status, error = 0;
     struct timespec ts;
 
-    dprintf(("PyThread_acquire_lock_timed(%p, %lld) called\n",
-             lock, microseconds));
+    dprintf(("PyThread_acquire_lock_timed(%p, %lld, %d) called\n",
+             lock, microseconds, intr_flag));
 
     if (microseconds > 0)
         MICROSECONDS_TO_TIMESPEC(microseconds, ts);
@@ -336,33 +337,38 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds)
             status = fix_status(sem_trywait(thelock));
         else
             status = fix_status(sem_wait(thelock));
-    } while (status == EINTR); /* Retry if interrupted by a signal */
-
-    if (microseconds > 0) {
-        if (status != ETIMEDOUT)
-            CHECK_STATUS("sem_timedwait");
-    }
-    else if (microseconds == 0) {
-        if (status != EAGAIN)
-            CHECK_STATUS("sem_trywait");
-    }
-    else {
-        CHECK_STATUS("sem_wait");
+        /* Retry if interrupted by a signal, unless the caller wants to be
+           notified.  */
+    } while (!intr_flag && status == EINTR);
+
+    /* Don't check the status if we're stopping because of an interrupt.  */
+    if (!(intr_flag && status == EINTR)) {
+        if (microseconds > 0) {
+            if (status != ETIMEDOUT)
+                CHECK_STATUS("sem_timedwait");
+        }
+        else if (microseconds == 0) {
+            if (status != EAGAIN)
+                CHECK_STATUS("sem_trywait");
+        }
+        else {
+            CHECK_STATUS("sem_wait");
+        }
     }
 
-    success = (status == 0) ? 1 : 0;
+    if (status == 0) {
+        success = PY_LOCK_ACQUIRED;
+    } else if (intr_flag && status == EINTR) {
+        success = PY_LOCK_INTR;
+    } else {
+        success = PY_LOCK_FAILURE;
+    }
 
-    dprintf(("PyThread_acquire_lock_timed(%p, %lld) -> %d\n",
-             lock, microseconds, success));
+    dprintf(("PyThread_acquire_lock_timed(%p, %lld, %d) -> %d\n",
+             lock, microseconds, intr_flag, success));
     return success;
 }
 
-int
-PyThread_acquire_lock(PyThread_type_lock lock, int waitflag)
-{
-    return PyThread_acquire_lock_timed(lock, waitflag ? -1 : 0);
-}
-
 void
 PyThread_release_lock(PyThread_type_lock lock)
 {
@@ -436,21 +442,25 @@ PyThread_free_lock(PyThread_type_lock lock)
     free((void *)thelock);
 }
 
-int
-PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds)
+PyLockStatus
+PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds,
+                            int intr_flag)
 {
-    int success;
+    PyLockStatus success;
     pthread_lock *thelock = (pthread_lock *)lock;
     int status, error = 0;
 
-    dprintf(("PyThread_acquire_lock_timed(%p, %lld) called\n",
-             lock, microseconds));
+    dprintf(("PyThread_acquire_lock_timed(%p, %lld, %d) called\n",
+             lock, microseconds, intr_flag));
 
     status = pthread_mutex_lock( &thelock->mut );
     CHECK_STATUS("pthread_mutex_lock[1]");
-    success = thelock->locked == 0;
 
-    if (!success && microseconds != 0) {
+    if (thelock->locked == 0) {
+        success = PY_LOCK_ACQUIRED;
+    } else if (microseconds == 0) {
+        success = PY_LOCK_FAILURE;
+    } else {
         struct timespec ts;
         if (microseconds > 0)
             MICROSECONDS_TO_TIMESPEC(microseconds, ts);
@@ -458,7 +468,8 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds)
 
         /* mut must be locked by me -- part of the condition
          * protocol */
-        while (thelock->locked) {
+        success = PY_LOCK_FAILURE;
+        while (success == PY_LOCK_FAILURE) {
             if (microseconds > 0) {
                 status = pthread_cond_timedwait(
                     &thelock->lock_released,
@@ -473,25 +484,30 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds)
                     &thelock->mut);
                 CHECK_STATUS("pthread_cond_wait");
             }
+
+            if (intr_flag && status == 0 && thelock->locked) {
+                /* We were woken up, but didn't get the lock.  We probably received
+                 * a signal.  Return PY_LOCK_INTR to allow the caller to handle
+                 * it and retry.  */
+                success = PY_LOCK_INTR;
+                break;
+            } else if (status == 0 && !thelock->locked) {
+                success = PY_LOCK_ACQUIRED;
+            } else {
+                success = PY_LOCK_FAILURE;
+            }
         }
-        success = (status == 0);
     }
-    if (success) thelock->locked = 1;
+    if (success == PY_LOCK_ACQUIRED) thelock->locked = 1;
     status = pthread_mutex_unlock( &thelock->mut );
     CHECK_STATUS("pthread_mutex_unlock[1]");
 
-    if (error) success = 0;
-    dprintf(("PyThread_acquire_lock_timed(%p, %lld) -> %d\n",
-             lock, microseconds, success));
+    if (error) success = PY_LOCK_FAILURE;
+    dprintf(("PyThread_acquire_lock_timed(%p, %lld, %d) -> %d\n",
+             lock, microseconds, intr_flag, success));
     return success;
 }
 
-int
-PyThread_acquire_lock(PyThread_type_lock lock, int waitflag)
-{
-    return PyThread_acquire_lock_timed(lock, waitflag ? -1 : 0);
-}
-
 void
 PyThread_release_lock(PyThread_type_lock lock)
 {
@@ -515,6 +531,12 @@ PyThread_release_lock(PyThread_type_lock lock)
 
 #endif /* USE_SEMAPHORES */
 
+int
+PyThread_acquire_lock(PyThread_type_lock lock, int waitflag)
+{
+    return PyThread_acquire_lock_timed(lock, waitflag ? -1 : 0, /*intr_flag=*/0);
+}
+
 /* set the thread stack size.
  * Return 0 if size is valid, -1 if size is invalid,
  * -2 if setting stack size is not supported.