Fixed SF bug #663074. The codec system was using global static

variables to store internal data. As a result, any atempts to use the
unicode system with multiple active interpreters, or successive
interpreter executions, would fail.

Now that information is stored into members of the PyInterpreterState
structure.

Include/pystate.h

@@ -22,6 +22,10 @@ typedef struct _is {
     PyObject *sysdict;
     PyObject *builtins;
 
+    PyObject *codec_search_path;
+    PyObject *codec_search_cache;
+    PyObject *codec_error_registry;
+
 #ifdef HAVE_DLOPEN
     int dlopenflags;
 #endif

Misc/NEWS

@@ -29,6 +29,11 @@ Core and builtins
 - On 64-bit systems, a dictionary could contain duplicate long/int keys
   if the key value was larger than 2**32.  See SF bug #689659.
 
+- Fixed SF bug #663074. The codec system was using global static
+  variables to store internal data. As a result, any atempts to use the
+  unicode system with multiple active interpreters, or successive
+  interpreter executions, would fail.
+
 Extension modules
 -----------------
 

Python/codecs.c

@@ -11,14 +11,6 @@ Copyright (c) Corporation for National Research Initiatives.
 #include "Python.h"
 #include <ctype.h>
 
-/* --- Globals ------------------------------------------------------------ */
-
-static PyObject *_PyCodec_SearchPath;
-static PyObject *_PyCodec_SearchCache;
-
-/* Flag used for lazy import of the standard encodings package */
-static int import_encodings_called = 0;
-
 /* --- Codec Registry ----------------------------------------------------- */
 
 /* Import the standard encodings package which will register the first
@@ -32,35 +24,13 @@ static int import_encodings_called = 0;
 
 */
 
-static
-int import_encodings(void)
-{
-    PyObject *mod;
-    
-    import_encodings_called = 1;
-    mod = PyImport_ImportModuleEx("encodings", NULL, NULL, NULL);
-    if (mod == NULL) {
-	if (PyErr_ExceptionMatches(PyExc_ImportError)) {
-	    /* Ignore ImportErrors... this is done so that
-	       distributions can disable the encodings package. Note
-	       that other errors are not masked, e.g. SystemErrors
-	       raised to inform the user of an error in the Python
-	       configuration are still reported back to the user. */
-	    PyErr_Clear();
-	    return 0;
-	}
-	return -1;
-    }
-    Py_DECREF(mod);
-    return 0;
-}
+static int _PyCodecRegistry_Init(void); /* Forward */
 
 int PyCodec_Register(PyObject *search_function)
 {
-    if (!import_encodings_called) {
-	if (import_encodings())
-	    goto onError;
-    }
+    PyInterpreterState *interp = PyThreadState_Get()->interp;
+    if (interp->codec_search_path == NULL && _PyCodecRegistry_Init())
+	goto onError;
     if (search_function == NULL) {
 	PyErr_BadArgument();
 	goto onError;
@@ -70,7 +40,7 @@ int PyCodec_Register(PyObject *search_function)
 			"argument must be callable");
 	goto onError;
     }
-    return PyList_Append(_PyCodec_SearchPath, search_function);
+    return PyList_Append(interp->codec_search_path, search_function);
 
  onError:
     return -1;
@@ -124,6 +94,7 @@ PyObject *normalizestring(const char *string)
 
 PyObject *_PyCodec_Lookup(const char *encoding)
 {
+    PyInterpreterState *interp;
     PyObject *result, *args = NULL, *v;
     int i, len;
 
@@ -131,16 +102,10 @@ PyObject *_PyCodec_Lookup(const char *encoding)
 	PyErr_BadArgument();
 	goto onError;
     }
-    if (_PyCodec_SearchCache == NULL || 
-	_PyCodec_SearchPath == NULL) {
-	PyErr_SetString(PyExc_SystemError,
-			"codec module not properly initialized");
+
+    interp = PyThreadState_Get()->interp;
+    if (interp->codec_search_path == NULL && _PyCodecRegistry_Init())
 	goto onError;
-    }
-    if (!import_encodings_called) {
-	if (import_encodings())
-	    goto onError;
-    }
 
     /* Convert the encoding to a normalized Python string: all
        characters are converted to lower case, spaces and hyphens are
@@ -151,7 +116,7 @@ PyObject *_PyCodec_Lookup(const char *encoding)
     PyString_InternInPlace(&v);
 
     /* First, try to lookup the name in the registry dictionary */
-    result = PyDict_GetItem(_PyCodec_SearchCache, v);
+    result = PyDict_GetItem(interp->codec_search_cache, v);
     if (result != NULL) {
 	Py_INCREF(result);
 	Py_DECREF(v);
@@ -164,7 +129,7 @@ PyObject *_PyCodec_Lookup(const char *encoding)
 	goto onError;
     PyTuple_SET_ITEM(args,0,v);
 
-    len = PyList_Size(_PyCodec_SearchPath);
+    len = PyList_Size(interp->codec_search_path);
     if (len < 0)
 	goto onError;
     if (len == 0) {
@@ -177,7 +142,7 @@ PyObject *_PyCodec_Lookup(const char *encoding)
     for (i = 0; i < len; i++) {
 	PyObject *func;
 	
-	func = PyList_GetItem(_PyCodec_SearchPath, i);
+	func = PyList_GetItem(interp->codec_search_path, i);
 	if (func == NULL)
 	    goto onError;
 	result = PyEval_CallObject(func, args);
@@ -203,7 +168,7 @@ PyObject *_PyCodec_Lookup(const char *encoding)
     }
 
     /* Cache and return the result */
-    PyDict_SetItem(_PyCodec_SearchCache, v, result);
+    PyDict_SetItem(interp->codec_search_cache, v, result);
     Py_DECREF(args);
     return result;
 
@@ -422,8 +387,6 @@ PyObject *PyCodec_Decode(PyObject *object,
     return NULL;
 }
 
-static PyObject *_PyCodec_ErrorRegistry;
-
 /* Register the error handling callback function error under the name
    name. This function will be called by the codec when it encounters
    an unencodable characters/undecodable bytes and doesn't know the
@@ -432,11 +395,15 @@ static PyObject *_PyCodec_ErrorRegistry;
    Return 0 on success, -1 on error */
 int PyCodec_RegisterError(const char *name, PyObject *error)
 {
+    PyInterpreterState *interp = PyThreadState_Get()->interp;
+    if (interp->codec_search_path == NULL && _PyCodecRegistry_Init())
+	return -1;
     if (!PyCallable_Check(error)) {
 	PyErr_SetString(PyExc_TypeError, "handler must be callable");
 	return -1;
     }
-    return PyDict_SetItemString( _PyCodec_ErrorRegistry, (char *)name, error);
+    return PyDict_SetItemString(interp->codec_error_registry,
+	    			(char *)name, error);
 }
 
 /* Lookup the error handling callback function registered under the
@@ -446,9 +413,13 @@ PyObject *PyCodec_LookupError(const char *name)
 {
     PyObject *handler = NULL;
 
+    PyInterpreterState *interp = PyThreadState_Get()->interp;
+    if (interp->codec_search_path == NULL && _PyCodecRegistry_Init())
+	return NULL;
+
     if (name==NULL)
 	name = "strict";
-    handler = PyDict_GetItemString(_PyCodec_ErrorRegistry, (char *)name);
+    handler = PyDict_GetItemString(interp->codec_error_registry, (char *)name);
     if (!handler)
 	PyErr_Format(PyExc_LookupError, "unknown error handler name '%.400s'", name);
     else
@@ -762,8 +733,7 @@ static PyObject *backslashreplace_errors(PyObject *self, PyObject *exc)
 }
 #endif
 
-
-void _PyCodecRegistry_Init(void)
+static int _PyCodecRegistry_Init(void)
 {
     static struct {
 	char *name;
@@ -813,38 +783,49 @@ void _PyCodecRegistry_Init(void)
 	}
 #endif
     };
-    if (_PyCodec_SearchPath == NULL)
-	_PyCodec_SearchPath = PyList_New(0);
-    if (_PyCodec_SearchCache == NULL)
-	_PyCodec_SearchCache = PyDict_New();
-    if (_PyCodec_ErrorRegistry == NULL) {
-	int i;
-	_PyCodec_ErrorRegistry = PyDict_New();
-
-	if (_PyCodec_ErrorRegistry) {
-	    for (i = 0; i < sizeof(methods)/sizeof(methods[0]); ++i) {
-		PyObject *func = PyCFunction_New(&methods[i].def, NULL);
-		int res;
-		if (!func)
-		    Py_FatalError("can't initialize codec error registry");
-		res = PyCodec_RegisterError(methods[i].name, func);
-		Py_DECREF(func);
-		if (res)
-		    Py_FatalError("can't initialize codec error registry");
-	    }
+
+    PyInterpreterState *interp = PyThreadState_Get()->interp;
+    PyObject *mod;
+    int i;
+
+    if (interp->codec_search_path != NULL)
+	return 0;
+
+    interp->codec_search_path = PyList_New(0);
+    interp->codec_search_cache = PyDict_New();
+    interp->codec_error_registry = PyDict_New();
+
+    if (interp->codec_error_registry) {
+	for (i = 0; i < sizeof(methods)/sizeof(methods[0]); ++i) {
+	    PyObject *func = PyCFunction_New(&methods[i].def, NULL);
+	    int res;
+	    if (!func)
+		Py_FatalError("can't initialize codec error registry");
+	    res = PyCodec_RegisterError(methods[i].name, func);
+	    Py_DECREF(func);
+	    if (res)
+		Py_FatalError("can't initialize codec error registry");
 	}
     }
-    if (_PyCodec_SearchPath == NULL || 
-	_PyCodec_SearchCache == NULL)
+
+    if (interp->codec_search_path == NULL ||
+	interp->codec_search_cache == NULL ||
+	interp->codec_error_registry == NULL)
 	Py_FatalError("can't initialize codec registry");
-}
 
-void _PyCodecRegistry_Fini(void)
-{
-    Py_XDECREF(_PyCodec_SearchPath);
-    _PyCodec_SearchPath = NULL;
-    Py_XDECREF(_PyCodec_SearchCache);
-    _PyCodec_SearchCache = NULL;
-    Py_XDECREF(_PyCodec_ErrorRegistry);
-    _PyCodec_ErrorRegistry = NULL;
+    mod = PyImport_ImportModuleEx("encodings", NULL, NULL, NULL);
+    if (mod == NULL) {
+	if (PyErr_ExceptionMatches(PyExc_ImportError)) {
+	    /* Ignore ImportErrors... this is done so that
+	       distributions can disable the encodings package. Note
+	       that other errors are not masked, e.g. SystemErrors
+	       raised to inform the user of an error in the Python
+	       configuration are still reported back to the user. */
+	    PyErr_Clear();
+	    return 0;
+	}
+	return -1;
+    }
+    Py_DECREF(mod);
+    return 0;
 }

Python/pystate.c

@@ -49,6 +49,9 @@ PyInterpreterState_New(void)
 		interp->sysdict = NULL;
 		interp->builtins = NULL;
 		interp->tstate_head = NULL;
+		interp->codec_search_path = NULL;
+		interp->codec_search_cache = NULL;
+		interp->codec_error_registry = NULL;
 #ifdef HAVE_DLOPEN
 #ifdef RTLD_NOW
                 interp->dlopenflags = RTLD_NOW;
@@ -75,6 +78,9 @@ PyInterpreterState_Clear(PyInterpreterState *interp)
 	for (p = interp->tstate_head; p != NULL; p = p->next)
 		PyThreadState_Clear(p);
 	HEAD_UNLOCK();
+	ZAP(interp->codec_search_path);
+	ZAP(interp->codec_search_cache);
+	ZAP(interp->codec_error_registry);
 	ZAP(interp->modules);
 	ZAP(interp->sysdict);
 	ZAP(interp->builtins);

Python/pythonrun.c

@@ -49,8 +49,6 @@ static void call_sys_exitfunc(void);
 static void call_ll_exitfuncs(void);
 extern void _PyUnicode_Init(void);
 extern void _PyUnicode_Fini(void);
-extern void _PyCodecRegistry_Init(void);
-extern void _PyCodecRegistry_Fini(void);
 
 int Py_DebugFlag; /* Needed by parser.c */
 int Py_VerboseFlag; /* Needed by import.c */
@@ -144,9 +142,6 @@ Py_Initialize(void)
 	if (interp->modules == NULL)
 		Py_FatalError("Py_Initialize: can't make modules dictionary");
 
-	/* Init codec registry */
-	_PyCodecRegistry_Init();
-
 #ifdef Py_USING_UNICODE
 	/* Init Unicode implementation; relies on the codec registry */
 	_PyUnicode_Init();
@@ -257,9 +252,6 @@ Py_Finalize(void)
 	/* Disable signal handling */
 	PyOS_FiniInterrupts();
 
-	/* Cleanup Codec registry */
-	_PyCodecRegistry_Fini();
-
 	/* drop module references we saved */
 	Py_XDECREF(PyModule_WarningsModule);
 	PyModule_WarningsModule = NULL;