Backported thread fixes from 2.4 (by mostly copying pystate.c over from 2.4):

* using malloc() and free() directly, as explained in the new comment
* coding style in the PyGILState_*() functions
* the recent destroy-tstate-without-holding-the-GIL bug
* lock fixes and many more comments in thread.c

Python/pystate.c

@@ -3,6 +3,16 @@
 
 #include "Python.h"
 
+/* --------------------------------------------------------------------------
+CAUTION
+
+Always use malloc() and free() directly in this file.  A number of these
+functions are advertised as safe to call when the GIL isn't held, and in
+a debug build Python redirects (e.g.) PyMem_NEW (etc) to Python's debugging
+obmalloc functions.  Those aren't thread-safe (they rely on the GIL to avoid
+the expense of doing their own locking).
+-------------------------------------------------------------------------- */
+
 #ifdef HAVE_DLOPEN
 #ifdef HAVE_DLFCN_H
 #include <dlfcn.h>
@@ -41,7 +51,8 @@ PyThreadFrameGetter _PyThreadState_GetFrame = NULL;
 PyInterpreterState *
 PyInterpreterState_New(void)
 {
-	PyInterpreterState *interp = PyMem_NEW(PyInterpreterState, 1);
+	PyInterpreterState *interp = (PyInterpreterState *)
+				     malloc(sizeof(PyInterpreterState));
 
 	if (interp != NULL) {
 		HEAD_INIT();
@@ -116,7 +127,7 @@ PyInterpreterState_Delete(PyInterpreterState *interp)
 		Py_FatalError("PyInterpreterState_Delete: remaining threads");
 	*p = interp->next;
 	HEAD_UNLOCK();
-	PyMem_DEL(interp);
+	free(interp);
 }
 
 
@@ -130,7 +141,8 @@ threadstate_getframe(PyThreadState *self)
 PyThreadState *
 PyThreadState_New(PyInterpreterState *interp)
 {
-	PyThreadState *tstate = PyMem_NEW(PyThreadState, 1);
+	PyThreadState *tstate = (PyThreadState *)malloc(sizeof(PyThreadState));
+
 	if (_PyThreadState_GetFrame == NULL)
 		_PyThreadState_GetFrame = threadstate_getframe;
 
@@ -223,7 +235,7 @@ tstate_delete_common(PyThreadState *tstate)
 	}
 	*p = tstate->next;
 	HEAD_UNLOCK();
-	PyMem_DEL(tstate);
+	free(tstate);
 }
 
 
@@ -268,7 +280,7 @@ PyThreadState_Swap(PyThreadState *new)
 
 	_PyThreadState_Current = new;
 	/* It should not be possible for more than one thread state
-	   to be used for a thread.  Check this the best we can in debug 
+	   to be used for a thread.  Check this the best we can in debug
 	   builds.
 	*/
 #if defined(Py_DEBUG) && defined(WITH_THREAD)
@@ -313,7 +325,7 @@ PyThreadState_GetDict(void)
 
 int
 PyThreadState_SetAsyncExc(long id, PyObject *exc) {
-	PyThreadState *tstate = PyThreadState_Get();
+	PyThreadState *tstate = PyThreadState_GET();
 	PyInterpreterState *interp = tstate->interp;
 	PyThreadState *p;
 	int count = 0;
@@ -382,21 +394,25 @@ PyThreadState_IsCurrent(PyThreadState *tstate)
 static PyInterpreterState *autoInterpreterState = NULL;
 static int autoTLSkey = 0;
 
-/* Internal initialization/finalization functions called by 
-   Py_Initialize/Py_Finalize 
+/* Internal initialization/finalization functions called by
+   Py_Initialize/Py_Finalize
 */
-void _PyGILState_Init(PyInterpreterState *i, PyThreadState *t)
+void
+_PyGILState_Init(PyInterpreterState *i, PyThreadState *t)
 {
-	assert(i && t); /* must init with a valid states */
+	assert(i && t); /* must init with valid states */
 	autoTLSkey = PyThread_create_key();
 	autoInterpreterState = i;
 	/* Now stash the thread state for this thread in TLS */
-	PyThread_set_key_value(autoTLSkey, (void *)t);
-	assert(t->gilstate_counter==0); /* must be a new thread state */
+	assert(PyThread_get_key_value(autoTLSkey) == NULL);
+	if (PyThread_set_key_value(autoTLSkey, (void *)t) < 0)
+		Py_FatalError("Couldn't create autoTLSkey mapping");
+	assert(t->gilstate_counter == 0); /* must be a new thread state */
 	t->gilstate_counter = 1;
 }
 
-void _PyGILState_Fini(void)
+void
+_PyGILState_Fini(void)
 {
 	PyThread_delete_key(autoTLSkey);
 	autoTLSkey = 0;
@@ -404,80 +420,84 @@ void _PyGILState_Fini(void)
 }
 
 /* The public functions */
-PyThreadState *PyGILState_GetThisThreadState(void)
+PyThreadState *
+PyGILState_GetThisThreadState(void)
 {
-	if (autoInterpreterState==NULL || autoTLSkey==0)
+	if (autoInterpreterState == NULL || autoTLSkey == 0)
 		return NULL;
-	return (PyThreadState *) PyThread_get_key_value(autoTLSkey);
+	return (PyThreadState *)PyThread_get_key_value(autoTLSkey);
 }
 
-PyGILState_STATE PyGILState_Ensure(void)
+PyGILState_STATE
+PyGILState_Ensure(void)
 {
 	int current;
 	PyThreadState *tcur;
-	/* Note that we do not auto-init Python here - apart from 
-	   potential races with 2 threads auto-initializing, pep-311 
+	/* Note that we do not auto-init Python here - apart from
+	   potential races with 2 threads auto-initializing, pep-311
 	   spells out other issues.  Embedders are expected to have
 	   called Py_Initialize() and usually PyEval_InitThreads().
 	*/
 	assert(autoInterpreterState); /* Py_Initialize() hasn't been called! */
 	tcur = PyThread_get_key_value(autoTLSkey);
-	if (tcur==NULL) {
+	if (tcur == NULL) {
 		/* Create a new thread state for this thread */
 		tcur = PyThreadState_New(autoInterpreterState);
-		if (tcur==NULL)
+		if (tcur == NULL)
 			Py_FatalError("Couldn't create thread-state for new thread");
-		PyThread_set_key_value(autoTLSkey, (void *)tcur);
+		if (PyThread_set_key_value(autoTLSkey, (void *)tcur) < 0)
+			Py_FatalError("Couldn't create autoTLSkey mapping");
 		current = 0; /* new thread state is never current */
-	} else
+	}
+	else
 		current = PyThreadState_IsCurrent(tcur);
-	if (!current)
+	if (current == 0)
 		PyEval_RestoreThread(tcur);
 	/* Update our counter in the thread-state - no need for locks:
 	   - tcur will remain valid as we hold the GIL.
-	   - the counter is safe as we are the only thread "allowed" 
+	   - the counter is safe as we are the only thread "allowed"
 	     to modify this value
 	*/
-	tcur->gilstate_counter++;
+	++tcur->gilstate_counter;
 	return current ? PyGILState_LOCKED : PyGILState_UNLOCKED;
 }
 
-void PyGILState_Release(PyGILState_STATE oldstate)
+void
+PyGILState_Release(PyGILState_STATE oldstate)
 {
 	PyThreadState *tcur = PyThread_get_key_value(autoTLSkey);
-	if (tcur==NULL)
+	if (tcur == NULL)
 		Py_FatalError("auto-releasing thread-state, "
 		              "but no thread-state for this thread");
 	/* We must hold the GIL and have our thread state current */
 	/* XXX - remove the check - the assert should be fine,
-	   but while this is very new (April 2003), the extra check 
+	   but while this is very new (April 2003), the extra check
 	   by release-only users can't hurt.
 	*/
-	if (!PyThreadState_IsCurrent(tcur))
+	if (! PyThreadState_IsCurrent(tcur))
 		Py_FatalError("This thread state must be current when releasing");
-	assert (PyThreadState_IsCurrent(tcur));
-	tcur->gilstate_counter -= 1;
-	assert (tcur->gilstate_counter >= 0); /* illegal counter value */
-
-	/* If we are about to destroy this thread-state, we must 
-	   clear it while the lock is held, as destructors may run
-	*/
-	if (tcur->gilstate_counter==0) {
+	assert(PyThreadState_IsCurrent(tcur));
+	--tcur->gilstate_counter;
+	assert(tcur->gilstate_counter >= 0); /* illegal counter value */
+
+	/* If we're going to destroy this thread-state, we must
+	 * clear it while the GIL is held, as destructors may run.
+	 */
+	if (tcur->gilstate_counter == 0) {
 		/* can't have been locked when we created it */
-		assert(oldstate==PyGILState_UNLOCKED);
+		assert(oldstate == PyGILState_UNLOCKED);
 		PyThreadState_Clear(tcur);
+		/* Delete the thread-state.  Note this releases the GIL too!
+		 * It's vital that the GIL be held here, to avoid shutdown
+		 * races; see bugs 225673 and 1061968 (that nasty bug has a
+		 * habit of coming back).
+		 */
+		PyThreadState_DeleteCurrent();
+		/* Delete this thread from our TLS. */
+		PyThread_delete_key_value(autoTLSkey);
 	}
-
 	/* Release the lock if necessary */
-	if (oldstate==PyGILState_UNLOCKED)
+	else if (oldstate == PyGILState_UNLOCKED)
 		PyEval_ReleaseThread(tcur);
-
-	/* Now complete destruction of the thread if necessary */
-	if (tcur->gilstate_counter==0) {
-		/* Delete this thread from our TLS */
-		PyThread_delete_key_value(autoTLSkey);
-		/* Delete the thread-state */
-		PyThreadState_Delete(tcur);
-	}
 }
 #endif /* WITH_THREAD */

Python/thread.c

@@ -146,54 +146,132 @@ void PyThread_init_thread(void)
    This code stolen from "thread_sgi.h", where it was the only
    implementation of an existing Python TLS API.
 */
-/*
- * Per-thread data ("key") support.
- */
+/* ------------------------------------------------------------------------
+Per-thread data ("key") support.
+
+Use PyThread_create_key() to create a new key.  This is typically shared
+across threads.
+
+Use PyThread_set_key_value(thekey, value) to associate void* value with
+thekey in the current thread.  Each thread has a distinct mapping of thekey
+to a void* value.  Caution:  if the current thread already has a mapping
+for thekey, value is ignored.
+
+Use PyThread_get_key_value(thekey) to retrieve the void* value associated
+with thekey in the current thread.  This returns NULL if no value is
+associated with thekey in the current thread.
+
+Use PyThread_delete_key_value(thekey) to forget the current thread's associated
+value for thekey.  PyThread_delete_key(thekey) forgets the values associated
+with thekey across *all* threads.
+
+While some of these functions have error-return values, none set any
+Python exception.
 
+None of the functions does memory management on behalf of the void* values.
+You need to allocate and deallocate them yourself.  If the void* values
+happen to be PyObject*, these functions don't do refcount operations on
+them either.
+
+The GIL does not need to be held when calling these functions; they supply
+their own locking.  This isn't true of PyThread_create_key(), though (see
+next paragraph).
+
+There's a hidden assumption that PyThread_create_key() will be called before
+any of the other functions are called.  There's also a hidden assumption
+that calls to PyThread_create_key() are serialized externally.
+------------------------------------------------------------------------ */
+
+/* A singly-linked list of struct key objects remembers all the key->value
+ * associations.  File static keyhead heads the list.  keymutex is used
+ * to enforce exclusion internally.
+ */
 struct key {
+	/* Next record in the list, or NULL if this is the last record. */
 	struct key *next;
+
+	/* The thread id, according to PyThread_get_thread_ident(). */
 	long id;
+
+	/* The key and its associated value. */
 	int key;
 	void *value;
 };
 
 static struct key *keyhead = NULL;
-static int nkeys = 0;
 static PyThread_type_lock keymutex = NULL;
-
-static struct key *find_key(int key, void *value)
+static int nkeys = 0;  /* PyThread_create_key() hands out nkeys+1 next */
+
+/* Internal helper.
+ * If the current thread has a mapping for key, the appropriate struct key*
+ * is returned.  NB:  value is ignored in this case!
+ * If there is no mapping for key in the current thread, then:
+ *     If value is NULL, NULL is returned.
+ *     Else a mapping of key to value is created for the current thread,
+ *     and a pointer to a new struct key* is returned; except that if
+ *     malloc() can't find room for a new struct key*, NULL is returned.
+ * So when value==NULL, this acts like a pure lookup routine, and when
+ * value!=NULL, this acts like dict.setdefault(), returning an existing
+ * mapping if one exists, else creating a new mapping.
+ *
+ * Caution:  this used to be too clever, trying to hold keymutex only
+ * around the "p->next = keyhead; keyhead = p" pair.  That allowed
+ * another thread to mutate the list, via key deletion, concurrent with
+ * find_key() crawling over the list.  Hilarity ensued.  For example, when
+ * the for-loop here does "p = p->next", p could end up pointing at a
+ * record that PyThread_delete_key_value() was concurrently free()'ing.
+ * That could lead to anything, from failing to find a key that exists, to
+ * segfaults.  Now we lock the whole routine.
+ */
+static struct key *
+find_key(int key, void *value)
 {
 	struct key *p;
 	long id = PyThread_get_thread_ident();
+
+	PyThread_acquire_lock(keymutex, 1);
 	for (p = keyhead; p != NULL; p = p->next) {
 		if (p->id == id && p->key == key)
-			return p;
+			goto Done;
+	}
+	if (value == NULL) {
+		assert(p == NULL);
+		goto Done;
 	}
-	if (value == NULL)
-		return NULL;
 	p = (struct key *)malloc(sizeof(struct key));
 	if (p != NULL) {
 		p->id = id;
 		p->key = key;
 		p->value = value;
-		PyThread_acquire_lock(keymutex, 1);
 		p->next = keyhead;
 		keyhead = p;
-		PyThread_release_lock(keymutex);
 	}
+ Done:
+	PyThread_release_lock(keymutex);
 	return p;
 }
 
-int PyThread_create_key(void)
+/* Return a new key.  This must be called before any other functions in
+ * this family, and callers must arrange to serialize calls to this
+ * function.  No violations are detected.
+ */
+int
+PyThread_create_key(void)
 {
+	/* All parts of this function are wrong if it's called by multiple
+	 * threads simultaneously.
+	 */
 	if (keymutex == NULL)
 		keymutex = PyThread_allocate_lock();
 	return ++nkeys;
 }
 
-void PyThread_delete_key(int key)
+/* Forget the associations for key across *all* threads. */
+void
+PyThread_delete_key(int key)
 {
 	struct key *p, **q;
+
 	PyThread_acquire_lock(keymutex, 1);
 	q = &keyhead;
 	while ((p = *q) != NULL) {
@@ -208,28 +286,46 @@ void PyThread_delete_key(int key)
 	PyThread_release_lock(keymutex);
 }
 
-int PyThread_set_key_value(int key, void *value)
+/* Confusing:  If the current thread has an association for key,
+ * value is ignored, and 0 is returned.  Else an attempt is made to create
+ * an association of key to value for the current thread.  0 is returned
+ * if that succeeds, but -1 is returned if there's not enough memory
+ * to create the association.  value must not be NULL.
+ */
+int
+PyThread_set_key_value(int key, void *value)
 {
-	struct key *p = find_key(key, value);
+	struct key *p;
+
+	assert(value != NULL);
+	p = find_key(key, value);
 	if (p == NULL)
 		return -1;
 	else
 		return 0;
 }
 
-void *PyThread_get_key_value(int key)
+/* Retrieve the value associated with key in the current thread, or NULL
+ * if the current thread doesn't have an association for key.
+ */
+void *
+PyThread_get_key_value(int key)
 {
 	struct key *p = find_key(key, NULL);
+
 	if (p == NULL)
 		return NULL;
 	else
 		return p->value;
 }
 
-void PyThread_delete_key_value(int key)
+/* Forget the current thread's association for key, if any. */
+void
+PyThread_delete_key_value(int key)
 {
 	long id = PyThread_get_thread_ident();
 	struct key *p, **q;
+
 	PyThread_acquire_lock(keymutex, 1);
 	q = &keyhead;
 	while ((p = *q) != NULL) {