pystate.c 49.1 KB
Newer Older
1 2 3 4

/* Thread and interpreter state structures and their interfaces */

#include "Python.h"
5
#include "pycore_coreconfig.h"
6 7
#include "pycore_pymem.h"
#include "pycore_pystate.h"
8

9 10 11
/* --------------------------------------------------------------------------
CAUTION

12 13 14 15 16
Always use PyMem_RawMalloc() and PyMem_RawFree() directly in this file.  A
number of these functions are advertised as safe to call when the GIL isn't
held, and in a debug build Python redirects (e.g.) PyMem_NEW (etc) to Python's
debugging obmalloc functions.  Those aren't thread-safe (they rely on the GIL
to avoid the expense of doing their own locking).
17 18
-------------------------------------------------------------------------- */

19 20 21 22
#ifdef HAVE_DLOPEN
#ifdef HAVE_DLFCN_H
#include <dlfcn.h>
#endif
23
#if !HAVE_DECL_RTLD_LAZY
24 25 26 27
#define RTLD_LAZY 1
#endif
#endif

28 29 30
#ifdef __cplusplus
extern "C" {
#endif
31

32 33 34 35 36 37 38 39 40 41 42 43
#define _PyRuntimeGILState_GetThreadState(gilstate) \
    ((PyThreadState*)_Py_atomic_load_relaxed(&(gilstate)->tstate_current))
#define _PyRuntimeGILState_SetThreadState(gilstate, value) \
    _Py_atomic_store_relaxed(&(gilstate)->tstate_current, \
                             (uintptr_t)(value))

/* Forward declarations */
static PyThreadState *_PyGILState_GetThisThreadState(struct _gilstate_runtime_state *gilstate);
static void _PyThreadState_Delete(_PyRuntimeState *runtime, PyThreadState *tstate);
static PyThreadState *_PyThreadState_Swap(struct _gilstate_runtime_state *gilstate, PyThreadState *newts);


44 45
static _PyInitError
_PyRuntimeState_Init_impl(_PyRuntimeState *runtime)
46 47
{
    memset(runtime, 0, sizeof(*runtime));
48

49 50
    _PyGC_Initialize(&runtime->gc);
    _PyEval_Initialize(&runtime->ceval);
51
    runtime->preconfig = _PyPreConfig_INIT;
52

53
    runtime->gilstate.check_enabled = 1;
54

55 56
    /* A TSS key must be initialized with Py_tss_NEEDS_INIT
       in accordance with the specification. */
57 58
    Py_tss_t initial = Py_tss_NEEDS_INIT;
    runtime->gilstate.autoTSSkey = initial;
Guido van Rossum's avatar
Guido van Rossum committed
59

60
    runtime->interpreters.mutex = PyThread_allocate_lock();
61 62 63
    if (runtime->interpreters.mutex == NULL) {
        return _Py_INIT_ERR("Can't initialize threads for interpreter");
    }
64
    runtime->interpreters.next_id = -1;
65 66 67 68 69 70

    runtime->xidregistry.mutex = PyThread_allocate_lock();
    if (runtime->xidregistry.mutex == NULL) {
        return _Py_INIT_ERR("Can't initialize threads for cross-interpreter data registry");
    }

71 72
    // Set it to the ID of the main thread of the main interpreter.
    runtime->main_thread = PyThread_get_thread_ident();
73

74
    return _Py_INIT_OK();
75
}
76

77 78 79 80 81 82 83 84 85 86 87 88 89 90
_PyInitError
_PyRuntimeState_Init(_PyRuntimeState *runtime)
{
    /* Force default allocator, since _PyRuntimeState_Fini() must
       use the same allocator than this function. */
    PyMemAllocatorEx old_alloc;
    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);

    _PyInitError err = _PyRuntimeState_Init_impl(runtime);

    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
    return err;
}

91 92 93
void
_PyRuntimeState_Fini(_PyRuntimeState *runtime)
{
94 95 96
    /* Force the allocator used by _PyRuntimeState_Init(). */
    PyMemAllocatorEx old_alloc;
    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
97

98 99 100 101
    if (runtime->interpreters.mutex != NULL) {
        PyThread_free_lock(runtime->interpreters.mutex);
        runtime->interpreters.mutex = NULL;
    }
102

103 104 105 106 107
    if (runtime->xidregistry.mutex != NULL) {
        PyThread_free_lock(runtime->xidregistry.mutex);
        runtime->xidregistry.mutex = NULL;
    }

108 109
    _PyPreConfig_Clear(&runtime->preconfig);

110
    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
111
}
Michael W. Hudson's avatar
Michael W. Hudson committed
112

113 114 115 116 117
/* This function is called from PyOS_AfterFork_Child to ensure that
 * newly created child processes do not share locks with the parent.
 */

void
118
_PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime)
119 120
{
    // This was initially set in _PyRuntimeState_Init().
121 122 123 124 125 126 127 128 129 130 131 132
    runtime->main_thread = PyThread_get_thread_ident();

    /* Force default allocator, since _PyRuntimeState_Fini() must
       use the same allocator than this function. */
    PyMemAllocatorEx old_alloc;
    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);

    runtime->interpreters.mutex = PyThread_allocate_lock();
    runtime->interpreters.main->id_mutex = PyThread_allocate_lock();
    runtime->xidregistry.mutex = PyThread_allocate_lock();

    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
133

134
    if (runtime->interpreters.mutex == NULL) {
135 136 137
        Py_FatalError("Can't initialize lock for runtime interpreters");
    }

138
    if (runtime->interpreters.main->id_mutex == NULL) {
139 140 141
        Py_FatalError("Can't initialize ID lock for main interpreter");
    }

142
    if (runtime->xidregistry.mutex == NULL) {
143 144 145 146
        Py_FatalError("Can't initialize lock for cross-interpreter data registry");
    }
}

147 148 149 150
#define HEAD_LOCK(runtime) \
    PyThread_acquire_lock((runtime)->interpreters.mutex, WAIT_LOCK)
#define HEAD_UNLOCK(runtime) \
    PyThread_release_lock((runtime)->interpreters.mutex)
151

152 153 154
/* Forward declaration */
static void _PyGILState_NoteThreadState(
    struct _gilstate_runtime_state *gilstate, PyThreadState* tstate);
155

156
_PyInitError
157 158
_PyInterpreterState_Enable(_PyRuntimeState *runtime)
{
159 160
    struct pyinterpreters *interpreters = &runtime->interpreters;
    interpreters->next_id = 0;
161 162 163

    /* Py_Finalize() calls _PyRuntimeState_Fini() which clears the mutex.
       Create a new mutex if needed. */
164
    if (interpreters->mutex == NULL) {
165 166 167 168 169
        /* Force default allocator, since _PyRuntimeState_Fini() must
           use the same allocator than this function. */
        PyMemAllocatorEx old_alloc;
        _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);

170
        interpreters->mutex = PyThread_allocate_lock();
171 172 173

        PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);

174
        if (interpreters->mutex == NULL) {
175 176
            return _Py_INIT_ERR("Can't initialize threads for interpreter");
        }
177
    }
178

179
    return _Py_INIT_OK();
180
}
181 182

PyInterpreterState *
183
PyInterpreterState_New(void)
184
{
185
    PyInterpreterState *interp = PyMem_RawMalloc(sizeof(PyInterpreterState));
186 187 188 189
    if (interp == NULL) {
        return NULL;
    }

190
    memset(interp, 0, sizeof(*interp));
191
    interp->id_refcount = -1;
192 193 194
    interp->check_interval = 100;
    interp->core_config = _PyCoreConfig_INIT;
    interp->eval_frame = _PyEval_EvalFrameDefault;
195
#ifdef HAVE_DLOPEN
196
#if HAVE_DECL_RTLD_NOW
197
    interp->dlopenflags = RTLD_NOW;
198
#else
199
    interp->dlopenflags = RTLD_LAZY;
200
#endif
201
#endif
202

203 204 205 206 207
    _PyRuntimeState *runtime = &_PyRuntime;
    struct pyinterpreters *interpreters = &runtime->interpreters;

    HEAD_LOCK(runtime);
    if (interpreters->next_id < 0) {
208 209 210
        /* overflow or Py_Initialize() not called! */
        PyErr_SetString(PyExc_RuntimeError,
                        "failed to get an interpreter ID");
211
        PyMem_RawFree(interp);
212
        interp = NULL;
213 214 215 216 217 218 219
    }
    else {
        interp->id = interpreters->next_id;
        interpreters->next_id += 1;
        interp->next = interpreters->head;
        if (interpreters->main == NULL) {
            interpreters->main = interp;
220
        }
221
        interpreters->head = interp;
222
    }
223
    HEAD_UNLOCK(runtime);
224

225 226 227 228
    if (interp == NULL) {
        return NULL;
    }

229 230
    interp->tstate_next_unique_id = 0;

231
    return interp;
232 233 234
}


235 236
static void
_PyInterpreterState_Clear(_PyRuntimeState *runtime, PyInterpreterState *interp)
237
{
238 239
    HEAD_LOCK(runtime);
    for (PyThreadState *p = interp->tstate_head; p != NULL; p = p->next) {
240
        PyThreadState_Clear(p);
241 242
    }
    HEAD_UNLOCK(runtime);
243
    _PyCoreConfig_Clear(&interp->core_config);
244 245 246
    Py_CLEAR(interp->codec_search_path);
    Py_CLEAR(interp->codec_search_cache);
    Py_CLEAR(interp->codec_error_registry);
247
    Py_CLEAR(interp->modules);
248 249 250
    Py_CLEAR(interp->modules_by_index);
    Py_CLEAR(interp->sysdict);
    Py_CLEAR(interp->builtins);
251
    Py_CLEAR(interp->builtins_copy);
252
    Py_CLEAR(interp->importlib);
253
    Py_CLEAR(interp->import_func);
254
    Py_CLEAR(interp->dict);
255 256 257 258 259
#ifdef HAVE_FORK
    Py_CLEAR(interp->before_forkers);
    Py_CLEAR(interp->after_forkers_parent);
    Py_CLEAR(interp->after_forkers_child);
#endif
260 261 262
    // XXX Once we have one allocator per interpreter (i.e.
    // per-interpreter GC) we must ensure that all of the interpreter's
    // objects have been cleaned up at the point.
263 264
}

265 266 267 268 269 270
void
PyInterpreterState_Clear(PyInterpreterState *interp)
{
    _PyInterpreterState_Clear(&_PyRuntime, interp);
}

271 272

static void
273
zapthreads(_PyRuntimeState *runtime, PyInterpreterState *interp)
274
{
275 276 277 278
    PyThreadState *p;
    /* No need to lock the mutex here because this should only happen
       when the threads are all really dead (XXX famous last words). */
    while ((p = interp->tstate_head) != NULL) {
279
        _PyThreadState_Delete(runtime, p);
280
    }
281
}
282

283

284 285 286
static void
_PyInterpreterState_Delete(_PyRuntimeState *runtime,
                           PyInterpreterState *interp)
287
{
288 289 290
    struct pyinterpreters *interpreters = &runtime->interpreters;
    zapthreads(runtime, interp);
    HEAD_LOCK(runtime);
291
    PyInterpreterState **p;
292 293 294 295 296
    for (p = &interpreters->head; ; p = &(*p)->next) {
        if (*p == NULL) {
            Py_FatalError("PyInterpreterState_Delete: invalid interp");
        }
        if (*p == interp) {
297
            break;
298
        }
299
    }
300
    if (interp->tstate_head != NULL) {
301
        Py_FatalError("PyInterpreterState_Delete: remaining threads");
302
    }
303
    *p = interp->next;
304 305 306
    if (interpreters->main == interp) {
        interpreters->main = NULL;
        if (interpreters->head != NULL) {
307
            Py_FatalError("PyInterpreterState_Delete: remaining subinterpreters");
308
        }
309
    }
310
    HEAD_UNLOCK(runtime);
311 312 313
    if (interp->id_mutex != NULL) {
        PyThread_free_lock(interp->id_mutex);
    }
314
    PyMem_RawFree(interp);
315 316 317
}


318 319 320 321 322 323 324
void
PyInterpreterState_Delete(PyInterpreterState *interp)
{
    _PyInterpreterState_Delete(&_PyRuntime, interp);
}


325 326 327 328 329
/*
 * Delete all interpreter states except the main interpreter.  If there
 * is a current interpreter state, it *must* be the main interpreter.
 */
void
330
_PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime)
331
{
332
    struct _gilstate_runtime_state *gilstate = &runtime->gilstate;
333 334
    struct pyinterpreters *interpreters = &runtime->interpreters;

335
    PyThreadState *tstate = _PyThreadState_Swap(gilstate, NULL);
336
    if (tstate != NULL && tstate->interp != interpreters->main) {
337 338 339
        Py_FatalError("PyInterpreterState_DeleteExceptMain: not main interpreter");
    }

340
    HEAD_LOCK(runtime);
341 342
    PyInterpreterState *interp = interpreters->head;
    interpreters->head = NULL;
343
    while (interp != NULL) {
344 345 346
        if (interp == interpreters->main) {
            interpreters->main->next = NULL;
            interpreters->head = interp;
347
            interp = interp->next;
348 349 350
            continue;
        }

351 352
        _PyInterpreterState_Clear(runtime, interp);  // XXX must activate?
        zapthreads(runtime, interp);
353 354 355
        if (interp->id_mutex != NULL) {
            PyThread_free_lock(interp->id_mutex);
        }
356 357 358
        PyInterpreterState *prev_interp = interp;
        interp = interp->next;
        PyMem_RawFree(prev_interp);
359
    }
360
    HEAD_UNLOCK(runtime);
361

362
    if (interpreters->head == NULL) {
363 364
        Py_FatalError("PyInterpreterState_DeleteExceptMain: missing main");
    }
365
    _PyThreadState_Swap(gilstate, tstate);
366 367 368
}


369 370 371
PyInterpreterState *
_PyInterpreterState_Get(void)
{
372
    PyThreadState *tstate = _PyThreadState_GET();
373 374 375 376 377 378 379 380 381 382 383
    if (tstate == NULL) {
        Py_FatalError("_PyInterpreterState_Get(): no current thread state");
    }
    PyInterpreterState *interp = tstate->interp;
    if (interp == NULL) {
        Py_FatalError("_PyInterpreterState_Get(): no current interpreter");
    }
    return interp;
}


384 385 386 387 388 389 390 391 392 393 394
int64_t
PyInterpreterState_GetID(PyInterpreterState *interp)
{
    if (interp == NULL) {
        PyErr_SetString(PyExc_RuntimeError, "no interpreter provided");
        return -1;
    }
    return interp->id;
}


395
static PyInterpreterState *
396
interp_look_up_id(_PyRuntimeState *runtime, PY_INT64_T requested_id)
397
{
398
    PyInterpreterState *interp = runtime->interpreters.head;
399 400
    while (interp != NULL) {
        PY_INT64_T id = PyInterpreterState_GetID(interp);
401
        if (id < 0) {
402
            return NULL;
403 404
        }
        if (requested_id == id) {
405
            return interp;
406
        }
407 408
        interp = PyInterpreterState_Next(interp);
    }
409
    return NULL;
410 411
}

412 413 414 415 416
PyInterpreterState *
_PyInterpreterState_LookUpID(PY_INT64_T requested_id)
{
    PyInterpreterState *interp = NULL;
    if (requested_id >= 0) {
417 418 419 420
        _PyRuntimeState *runtime = &_PyRuntime;
        HEAD_LOCK(runtime);
        interp = interp_look_up_id(runtime, requested_id);
        HEAD_UNLOCK(runtime);
421 422 423 424 425 426 427 428
    }
    if (interp == NULL && !PyErr_Occurred()) {
        PyErr_Format(PyExc_RuntimeError,
                     "unrecognized interpreter ID %lld", requested_id);
    }
    return interp;
}

429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464

int
_PyInterpreterState_IDInitref(PyInterpreterState *interp)
{
    if (interp->id_mutex != NULL) {
        return 0;
    }
    interp->id_mutex = PyThread_allocate_lock();
    if (interp->id_mutex == NULL) {
        PyErr_SetString(PyExc_RuntimeError,
                        "failed to create init interpreter ID mutex");
        return -1;
    }
    interp->id_refcount = 0;
    return 0;
}


void
_PyInterpreterState_IDIncref(PyInterpreterState *interp)
{
    if (interp->id_mutex == NULL) {
        return;
    }
    PyThread_acquire_lock(interp->id_mutex, WAIT_LOCK);
    interp->id_refcount += 1;
    PyThread_release_lock(interp->id_mutex);
}


void
_PyInterpreterState_IDDecref(PyInterpreterState *interp)
{
    if (interp->id_mutex == NULL) {
        return;
    }
465
    struct _gilstate_runtime_state *gilstate = &_PyRuntime.gilstate;
466 467 468 469 470 471
    PyThread_acquire_lock(interp->id_mutex, WAIT_LOCK);
    assert(interp->id_refcount != 0);
    interp->id_refcount -= 1;
    int64_t refcount = interp->id_refcount;
    PyThread_release_lock(interp->id_mutex);

472
    if (refcount == 0 && interp->requires_idref) {
473 474 475
        // XXX Using the "head" thread isn't strictly correct.
        PyThreadState *tstate = PyInterpreterState_ThreadHead(interp);
        // XXX Possible GILState issues?
476
        PyThreadState *save_tstate = _PyThreadState_Swap(gilstate, tstate);
477
        Py_EndInterpreter(tstate);
478
        _PyThreadState_Swap(gilstate, save_tstate);
479 480 481
    }
}

482 483 484 485 486 487 488 489 490 491 492 493
int
_PyInterpreterState_RequiresIDRef(PyInterpreterState *interp)
{
    return interp->requires_idref;
}

void
_PyInterpreterState_RequireIDRef(PyInterpreterState *interp, int required)
{
    interp->requires_idref = required ? 1 : 0;
}

494 495 496 497 498 499
_PyCoreConfig *
_PyInterpreterState_GetCoreConfig(PyInterpreterState *interp)
{
    return &interp->core_config;
}

500 501 502 503 504 505 506 507 508 509
PyObject *
_PyInterpreterState_GetMainModule(PyInterpreterState *interp)
{
    if (interp->modules == NULL) {
        PyErr_SetString(PyExc_RuntimeError, "interpreter not initialized");
        return NULL;
    }
    return PyMapping_GetItemString(interp->modules, "__main__");
}

510 511 512 513 514 515 516 517 518 519 520 521 522
PyObject *
PyInterpreterState_GetDict(PyInterpreterState *interp)
{
    if (interp->dict == NULL) {
        interp->dict = PyDict_New();
        if (interp->dict == NULL) {
            PyErr_Clear();
        }
    }
    /* Returning NULL means no per-interpreter dict is available. */
    return interp->dict;
}

523 524 525 526
/* Default implementation for _PyThreadState_GetFrame */
static struct _frame *
threadstate_getframe(PyThreadState *self)
{
527
    return self->frame;
528 529
}

530 531
static PyThreadState *
new_threadstate(PyInterpreterState *interp, int init)
532
{
533
    _PyRuntimeState *runtime = &_PyRuntime;
534
    PyThreadState *tstate = (PyThreadState *)PyMem_RawMalloc(sizeof(PyThreadState));
535 536 537
    if (tstate == NULL) {
        return NULL;
    }
538

539
    if (_PyThreadState_GetFrame == NULL) {
540
        _PyThreadState_GetFrame = threadstate_getframe;
541
    }
542

543
    tstate->interp = interp;
544

545 546 547 548 549 550 551 552 553 554
    tstate->frame = NULL;
    tstate->recursion_depth = 0;
    tstate->overflowed = 0;
    tstate->recursion_critical = 0;
    tstate->stackcheck_counter = 0;
    tstate->tracing = 0;
    tstate->use_tracing = 0;
    tstate->gilstate_counter = 0;
    tstate->async_exc = NULL;
    tstate->thread_id = PyThread_get_thread_ident();
555

556
    tstate->dict = NULL;
557

558 559 560
    tstate->curexc_type = NULL;
    tstate->curexc_value = NULL;
    tstate->curexc_traceback = NULL;
561

562 563 564 565 566
    tstate->exc_state.exc_type = NULL;
    tstate->exc_state.exc_value = NULL;
    tstate->exc_state.exc_traceback = NULL;
    tstate->exc_state.previous_item = NULL;
    tstate->exc_info = &tstate->exc_state;
567

568 569 570 571
    tstate->c_profilefunc = NULL;
    tstate->c_tracefunc = NULL;
    tstate->c_profileobj = NULL;
    tstate->c_traceobj = NULL;
572

573 574 575 576
    tstate->trash_delete_nesting = 0;
    tstate->trash_delete_later = NULL;
    tstate->on_delete = NULL;
    tstate->on_delete_data = NULL;
577

578
    tstate->coroutine_origin_tracking_depth = 0;
579

580 581
    tstate->coroutine_wrapper = NULL;
    tstate->in_coroutine_wrapper = 0;
582

583 584
    tstate->async_gen_firstiter = NULL;
    tstate->async_gen_finalizer = NULL;
585

586 587
    tstate->context = NULL;
    tstate->context_ver = 1;
588

589
    tstate->id = ++interp->tstate_next_unique_id;
Michael W. Hudson's avatar
Michael W. Hudson committed
590

591
    if (init) {
592
        _PyThreadState_Init(runtime, tstate);
593
    }
594

595
    HEAD_LOCK(runtime);
596 597 598 599 600
    tstate->prev = NULL;
    tstate->next = interp->tstate_head;
    if (tstate->next)
        tstate->next->prev = tstate;
    interp->tstate_head = tstate;
601
    HEAD_UNLOCK(runtime);
602

603
    return tstate;
604 605
}

606 607 608
PyThreadState *
PyThreadState_New(PyInterpreterState *interp)
{
609
    return new_threadstate(interp, 1);
610 611 612 613 614
}

PyThreadState *
_PyThreadState_Prealloc(PyInterpreterState *interp)
{
615
    return new_threadstate(interp, 0);
616 617 618
}

void
619
_PyThreadState_Init(_PyRuntimeState *runtime, PyThreadState *tstate)
620
{
621
    _PyGILState_NoteThreadState(&runtime->gilstate, tstate);
622 623
}

624
PyObject*
625
PyState_FindModule(struct PyModuleDef* module)
626
{
627
    Py_ssize_t index = module->m_base.m_index;
628
    PyInterpreterState *state = _PyInterpreterState_GET_UNSAFE();
629
    PyObject *res;
630 631 632
    if (module->m_slots) {
        return NULL;
    }
633 634 635 636
    if (index == 0)
        return NULL;
    if (state->modules_by_index == NULL)
        return NULL;
637
    if (index >= PyList_GET_SIZE(state->modules_by_index))
638 639 640
        return NULL;
    res = PyList_GET_ITEM(state->modules_by_index, index);
    return res==Py_None ? NULL : res;
641 642 643 644 645
}

int
_PyState_AddModule(PyObject* module, struct PyModuleDef* def)
{
646
    PyInterpreterState *state;
647 648 649 650
    if (!def) {
        assert(PyErr_Occurred());
        return -1;
    }
651 652 653 654 655
    if (def->m_slots) {
        PyErr_SetString(PyExc_SystemError,
                        "PyState_AddModule called on module with slots");
        return -1;
    }
656
    state = _PyInterpreterState_GET_UNSAFE();
657 658 659 660 661 662 663 664 665 666 667
    if (!state->modules_by_index) {
        state->modules_by_index = PyList_New(0);
        if (!state->modules_by_index)
            return -1;
    }
    while(PyList_GET_SIZE(state->modules_by_index) <= def->m_base.m_index)
        if (PyList_Append(state->modules_by_index, Py_None) < 0)
            return -1;
    Py_INCREF(module);
    return PyList_SetItem(state->modules_by_index,
                          def->m_base.m_index, module);
668
}
669

670 671 672 673
int
PyState_AddModule(PyObject* module, struct PyModuleDef* def)
{
    Py_ssize_t index;
674
    PyInterpreterState *state = _PyInterpreterState_GET_UNSAFE();
675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693
    if (!def) {
        Py_FatalError("PyState_AddModule: Module Definition is NULL");
        return -1;
    }
    index = def->m_base.m_index;
    if (state->modules_by_index) {
        if(PyList_GET_SIZE(state->modules_by_index) >= index) {
            if(module == PyList_GET_ITEM(state->modules_by_index, index)) {
                Py_FatalError("PyState_AddModule: Module already added!");
                return -1;
            }
        }
    }
    return _PyState_AddModule(module, def);
}

int
PyState_RemoveModule(struct PyModuleDef* def)
{
694
    PyInterpreterState *state;
695
    Py_ssize_t index = def->m_base.m_index;
696 697 698 699 700
    if (def->m_slots) {
        PyErr_SetString(PyExc_SystemError,
                        "PyState_RemoveModule called on module with slots");
        return -1;
    }
701
    state = _PyInterpreterState_GET_UNSAFE();
702 703 704 705 706 707 708 709 710 711 712 713
    if (index == 0) {
        Py_FatalError("PyState_RemoveModule: Module index invalid.");
        return -1;
    }
    if (state->modules_by_index == NULL) {
        Py_FatalError("PyState_RemoveModule: Interpreters module-list not acessible.");
        return -1;
    }
    if (index > PyList_GET_SIZE(state->modules_by_index)) {
        Py_FatalError("PyState_RemoveModule: Module index out of bounds.");
        return -1;
    }
714
    Py_INCREF(Py_None);
715 716 717
    return PyList_SetItem(state->modules_by_index, index, Py_None);
}

718 719 720 721
/* used by import.c:PyImport_Cleanup */
void
_PyState_ClearModules(void)
{
722
    PyInterpreterState *state = _PyInterpreterState_GET_UNSAFE();
723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742
    if (state->modules_by_index) {
        Py_ssize_t i;
        for (i = 0; i < PyList_GET_SIZE(state->modules_by_index); i++) {
            PyObject *m = PyList_GET_ITEM(state->modules_by_index, i);
            if (PyModule_Check(m)) {
                /* cleanup the saved copy of module dicts */
                PyModuleDef *md = PyModule_GetDef(m);
                if (md)
                    Py_CLEAR(md->m_base.m_copy);
            }
        }
        /* Setting modules_by_index to NULL could be dangerous, so we
           clear the list instead. */
        if (PyList_SetSlice(state->modules_by_index,
                            0, PyList_GET_SIZE(state->modules_by_index),
                            NULL))
            PyErr_WriteUnraisable(state->modules_by_index);
    }
}

743
void
744
PyThreadState_Clear(PyThreadState *tstate)
745
{
746 747 748
    int verbose = tstate->interp->core_config.verbose;

    if (verbose && tstate->frame != NULL)
749 750
        fprintf(stderr,
          "PyThreadState_Clear: warning: thread still has a frame\n");
751

752
    Py_CLEAR(tstate->frame);
753

754 755
    Py_CLEAR(tstate->dict);
    Py_CLEAR(tstate->async_exc);
756

757 758 759
    Py_CLEAR(tstate->curexc_type);
    Py_CLEAR(tstate->curexc_value);
    Py_CLEAR(tstate->curexc_traceback);
760

761 762 763
    Py_CLEAR(tstate->exc_state.exc_type);
    Py_CLEAR(tstate->exc_state.exc_value);
    Py_CLEAR(tstate->exc_state.exc_traceback);
764

765
    /* The stack of exception states should contain just this thread. */
766
    if (verbose && tstate->exc_info != &tstate->exc_state) {
767 768 769
        fprintf(stderr,
          "PyThreadState_Clear: warning: thread still has a generator\n");
    }
770

771 772 773 774
    tstate->c_profilefunc = NULL;
    tstate->c_tracefunc = NULL;
    Py_CLEAR(tstate->c_profileobj);
    Py_CLEAR(tstate->c_traceobj);
775 776

    Py_CLEAR(tstate->coroutine_wrapper);
777 778
    Py_CLEAR(tstate->async_gen_firstiter);
    Py_CLEAR(tstate->async_gen_finalizer);
779 780

    Py_CLEAR(tstate->context);
781
}
782 783


784 785
/* Common code for PyThreadState_Delete() and PyThreadState_DeleteCurrent() */
static void
786
tstate_delete_common(_PyRuntimeState *runtime, PyThreadState *tstate)
787
{
788
    if (tstate == NULL) {
789
        Py_FatalError("PyThreadState_Delete: NULL tstate");
790 791 792
    }
    PyInterpreterState *interp = tstate->interp;
    if (interp == NULL) {
793
        Py_FatalError("PyThreadState_Delete: NULL interp");
794 795
    }
    HEAD_LOCK(runtime);
796 797 798 799 800 801
    if (tstate->prev)
        tstate->prev->next = tstate->next;
    else
        interp->tstate_head = tstate->next;
    if (tstate->next)
        tstate->next->prev = tstate->prev;
802
    HEAD_UNLOCK(runtime);
803 804 805
    if (tstate->on_delete != NULL) {
        tstate->on_delete(tstate->on_delete_data);
    }
806
    PyMem_RawFree(tstate);
807 808 809
}


810 811
static void
_PyThreadState_Delete(_PyRuntimeState *runtime, PyThreadState *tstate)
812
{
813 814
    struct _gilstate_runtime_state *gilstate = &runtime->gilstate;
    if (tstate == _PyRuntimeGILState_GetThreadState(gilstate)) {
815
        Py_FatalError("PyThreadState_Delete: tstate is still current");
816 817 818
    }
    if (gilstate->autoInterpreterState &&
        PyThread_tss_get(&gilstate->autoTSSkey) == tstate)
819
    {
820
        PyThread_tss_set(&gilstate->autoTSSkey, NULL);
821
    }
822
    tstate_delete_common(runtime, tstate);
823 824 825 826
}


void
827
PyThreadState_Delete(PyThreadState *tstate)
828
{
829
    _PyThreadState_Delete(&_PyRuntime, tstate);
830 831 832 833 834 835 836 837
}


static void
_PyThreadState_DeleteCurrent(_PyRuntimeState *runtime)
{
    struct _gilstate_runtime_state *gilstate = &runtime->gilstate;
    PyThreadState *tstate = _PyRuntimeGILState_GetThreadState(gilstate);
838 839 840
    if (tstate == NULL)
        Py_FatalError(
            "PyThreadState_DeleteCurrent: no current tstate");
841 842 843
    tstate_delete_common(runtime, tstate);
    if (gilstate->autoInterpreterState &&
        PyThread_tss_get(&gilstate->autoTSSkey) == tstate)
844
    {
845
        PyThread_tss_set(&gilstate->autoTSSkey, NULL);
846
    }
847
    _PyRuntimeGILState_SetThreadState(gilstate, NULL);
848
    PyEval_ReleaseLock();
849
}
850

851 852 853 854 855 856
void
PyThreadState_DeleteCurrent()
{
    _PyThreadState_DeleteCurrent(&_PyRuntime);
}

857 858 859 860 861 862 863 864 865 866 867

/*
 * Delete all thread states except the one passed as argument.
 * Note that, if there is a current thread state, it *must* be the one
 * passed as argument.  Also, this won't touch any other interpreters
 * than the current one, since we don't know which thread state should
 * be kept in those other interpreteres.
 */
void
_PyThreadState_DeleteExcept(PyThreadState *tstate)
{
868
    _PyRuntimeState *runtime = &_PyRuntime;
869 870
    PyInterpreterState *interp = tstate->interp;
    PyThreadState *p, *next, *garbage;
871
    HEAD_LOCK(runtime);
872 873
    /* Remove all thread states, except tstate, from the linked list of
       thread states.  This will allow calling PyThreadState_Clear()
874
       without holding the lock. */
875
    garbage = interp->tstate_head;
876 877 878 879 880 881 882
    if (garbage == tstate)
        garbage = tstate->next;
    if (tstate->prev)
        tstate->prev->next = tstate->next;
    if (tstate->next)
        tstate->next->prev = tstate->prev;
    tstate->prev = tstate->next = NULL;
883
    interp->tstate_head = tstate;
884
    HEAD_UNLOCK(runtime);
885 886 887 888 889 890
    /* Clear and deallocate all stale thread states.  Even if this
       executes Python code, we should be safe since it executes
       in the current thread, not one of the stale threads. */
    for (p = garbage; p; p = next) {
        next = p->next;
        PyThreadState_Clear(p);
891
        PyMem_RawFree(p);
892 893
    }
}
894 895


896 897 898
PyThreadState *
_PyThreadState_UncheckedGet(void)
{
899
    return _PyThreadState_GET();
900 901 902
}


903
PyThreadState *
904
PyThreadState_Get(void)
905
{
906
    PyThreadState *tstate = _PyThreadState_GET();
907 908
    if (tstate == NULL)
        Py_FatalError("PyThreadState_Get: no current thread");
909

910
    return tstate;
911 912 913
}


914 915
static PyThreadState *
_PyThreadState_Swap(struct _gilstate_runtime_state *gilstate, PyThreadState *newts)
916
{
917
    PyThreadState *oldts = _PyRuntimeGILState_GetThreadState(gilstate);
918

919
    _PyRuntimeGILState_SetThreadState(gilstate, newts);
920 921 922 923
    /* It should not be possible for more than one thread state
       to be used for a thread.  Check this the best we can in debug
       builds.
    */
924
#if defined(Py_DEBUG)
925 926 927 928 929
    if (newts) {
        /* This can be called from PyEval_RestoreThread(). Similar
           to it, we need to ensure errno doesn't change.
        */
        int err = errno;
930
        PyThreadState *check = _PyGILState_GetThisThreadState(gilstate);
931 932 933 934
        if (check && check->interp == newts->interp && check != newts)
            Py_FatalError("Invalid thread state for this thread");
        errno = err;
    }
935
#endif
936
    return oldts;
937
}
938

939 940 941 942 943 944
PyThreadState *
PyThreadState_Swap(PyThreadState *newts)
{
    return _PyThreadState_Swap(&_PyRuntime.gilstate, newts);
}

945 946 947
/* An extension mechanism to store arbitrary additional per-thread state.
   PyThreadState_GetDict() returns a dictionary that can be used to hold such
   state; the caller should pick a unique key and store its state there.  If
948 949
   PyThreadState_GetDict() returns NULL, an exception has *not* been raised
   and the caller should assume no per-thread state is available. */
950 951

PyObject *
952
PyThreadState_GetDict(void)
953
{
954
    PyThreadState *tstate = _PyThreadState_GET();
955 956
    if (tstate == NULL)
        return NULL;
957

958 959 960 961 962 963 964
    if (tstate->dict == NULL) {
        PyObject *d;
        tstate->dict = d = PyDict_New();
        if (d == NULL)
            PyErr_Clear();
    }
    return tstate->dict;
965
}
966 967


968 969
/* Asynchronously raise an exception in a thread.
   Requested by Just van Rossum and Alex Martelli.
970
   To prevent naive misuse, you must write your own extension
971 972 973 974
   to call this, or use ctypes.  Must be called with the GIL held.
   Returns the number of tstates modified (normally 1, but 0 if `id` didn't
   match any known thread id).  Can be called with exc=NULL to clear an
   existing async exception.  This raises no exceptions. */
975 976

int
977 978
PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc)
{
979
    PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE();
980 981 982 983 984 985 986 987
    PyThreadState *p;

    /* Although the GIL is held, a few C API functions can be called
     * without the GIL held, and in particular some that create and
     * destroy thread and interpreter states.  Those can mutate the
     * list of thread states we're traversing, so to prevent that we lock
     * head_mutex for the duration.
     */
988 989
    _PyRuntimeState *runtime = &_PyRuntime;
    HEAD_LOCK(runtime);
990 991 992 993 994 995 996 997 998 999 1000 1001
    for (p = interp->tstate_head; p != NULL; p = p->next) {
        if (p->thread_id == id) {
            /* Tricky:  we need to decref the current value
             * (if any) in p->async_exc, but that can in turn
             * allow arbitrary Python code to run, including
             * perhaps calls to this function.  To prevent
             * deadlock, we need to release head_mutex before
             * the decref.
             */
            PyObject *old_exc = p->async_exc;
            Py_XINCREF(exc);
            p->async_exc = exc;
1002
            HEAD_UNLOCK(runtime);
1003
            Py_XDECREF(old_exc);
1004
            _PyEval_SignalAsyncExc();
1005 1006 1007
            return 1;
        }
    }
1008
    HEAD_UNLOCK(runtime);
1009
    return 0;
1010 1011 1012
}


1013 1014 1015 1016 1017 1018
/* Routines for advanced debuggers, requested by David Beazley.
   Don't use unless you know what you are doing! */

PyInterpreterState *
PyInterpreterState_Head(void)
{
1019
    return _PyRuntime.interpreters.head;
1020 1021
}

1022 1023 1024
PyInterpreterState *
PyInterpreterState_Main(void)
{
1025
    return _PyRuntime.interpreters.main;
1026 1027
}

1028 1029
PyInterpreterState *
PyInterpreterState_Next(PyInterpreterState *interp) {
1030
    return interp->next;
1031 1032 1033 1034
}

PyThreadState *
PyInterpreterState_ThreadHead(PyInterpreterState *interp) {
1035
    return interp->tstate_head;
1036 1037 1038 1039
}

PyThreadState *
PyThreadState_Next(PyThreadState *tstate) {
1040
    return tstate->next;
1041
}
1042

1043 1044 1045 1046 1047 1048 1049 1050
/* The implementation of sys._current_frames().  This is intended to be
   called with the GIL held, as it will be when called via
   sys._current_frames().  It's possible it would work fine even without
   the GIL held, but haven't thought enough about that.
*/
PyObject *
_PyThread_CurrentFrames(void)
{
1051 1052 1053 1054 1055 1056 1057 1058 1059 1060
    PyObject *result;
    PyInterpreterState *i;

    result = PyDict_New();
    if (result == NULL)
        return NULL;

    /* for i in all interpreters:
     *     for t in all of i's thread states:
     *          if t's frame isn't NULL, map t's id to its frame
1061
     * Because these lists can mutate even when the GIL is held, we
1062 1063
     * need to grab head_mutex for the duration.
     */
1064 1065 1066
    _PyRuntimeState *runtime = &_PyRuntime;
    HEAD_LOCK(runtime);
    for (i = runtime->interpreters.head; i != NULL; i = i->next) {
1067 1068 1069 1070 1071 1072 1073
        PyThreadState *t;
        for (t = i->tstate_head; t != NULL; t = t->next) {
            PyObject *id;
            int stat;
            struct _frame *frame = t->frame;
            if (frame == NULL)
                continue;
1074
            id = PyLong_FromUnsignedLong(t->thread_id);
1075 1076 1077 1078 1079 1080 1081 1082
            if (id == NULL)
                goto Fail;
            stat = PyDict_SetItem(result, id, (PyObject *)frame);
            Py_DECREF(id);
            if (stat < 0)
                goto Fail;
        }
    }
1083
    HEAD_UNLOCK(runtime);
1084
    return result;
1085 1086

 Fail:
1087
    HEAD_UNLOCK(runtime);
1088 1089
    Py_DECREF(result);
    return NULL;
1090
}
1091

1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104
/* Python "auto thread state" API. */

/* Keep this as a static, as it is not reliable!  It can only
   ever be compared to the state for the *current* thread.
   * If not equal, then it doesn't matter that the actual
     value may change immediately after comparison, as it can't
     possibly change to the current thread's state.
   * If equal, then the current thread holds the lock, so the value can't
     change until we yield the lock.
*/
static int
PyThreadState_IsCurrent(PyThreadState *tstate)
{
1105
    /* Must be the tstate for this thread */
1106 1107 1108
    struct _gilstate_runtime_state *gilstate = &_PyRuntime.gilstate;
    assert(_PyGILState_GetThisThreadState(gilstate) == tstate);
    return tstate == _PyRuntimeGILState_GetThreadState(gilstate);
1109 1110
}

Tim Peters's avatar
Tim Peters committed
1111
/* Internal initialization/finalization functions called by
1112
   Py_Initialize/Py_FinalizeEx
1113
*/
1114
void
1115 1116
_PyGILState_Init(_PyRuntimeState *runtime,
                 PyInterpreterState *interp, PyThreadState *tstate)
1117
{
1118 1119 1120 1121 1122 1123 1124
    /* must init with valid states */
    assert(interp != NULL);
    assert(tstate != NULL);

    struct _gilstate_runtime_state *gilstate = &runtime->gilstate;

    if (PyThread_tss_create(&gilstate->autoTSSkey) != 0) {
1125 1126
        Py_FatalError("Could not allocate TSS entry");
    }
1127 1128 1129
    gilstate->autoInterpreterState = interp;
    assert(PyThread_tss_get(&gilstate->autoTSSkey) == NULL);
    assert(tstate->gilstate_counter == 0);
Michael W. Hudson's avatar
Michael W. Hudson committed
1130

1131
    _PyGILState_NoteThreadState(gilstate, tstate);
1132 1133
}

1134 1135 1136
PyInterpreterState *
_PyGILState_GetInterpreterStateUnsafe(void)
{
1137
    return _PyRuntime.gilstate.autoInterpreterState;
1138 1139
}

1140
void
1141
_PyGILState_Fini(_PyRuntimeState *runtime)
1142
{
1143 1144 1145
    struct _gilstate_runtime_state *gilstate = &runtime->gilstate;
    PyThread_tss_delete(&gilstate->autoTSSkey);
    gilstate->autoInterpreterState = NULL;
1146 1147
}

1148
/* Reset the TSS key - called by PyOS_AfterFork_Child().
1149
 * This should not be necessary, but some - buggy - pthread implementations
1150
 * don't reset TSS upon fork(), see issue #10517.
1151 1152
 */
void
1153
_PyGILState_Reinit(_PyRuntimeState *runtime)
1154
{
1155
    struct _gilstate_runtime_state *gilstate = &runtime->gilstate;
1156
    PyThreadState *tstate = _PyGILState_GetThisThreadState(gilstate);
1157 1158 1159

    PyThread_tss_delete(&gilstate->autoTSSkey);
    if (PyThread_tss_create(&gilstate->autoTSSkey) != 0) {
1160 1161
        Py_FatalError("Could not allocate TSS entry");
    }
1162

1163 1164
    /* If the thread had an associated auto thread state, reassociate it with
     * the new key. */
1165
    if (tstate &&
1166
        PyThread_tss_set(&gilstate->autoTSSkey, (void *)tstate) != 0)
1167 1168 1169
    {
        Py_FatalError("Couldn't create autoTSSkey mapping");
    }
1170 1171
}

Michael W. Hudson's avatar
Michael W. Hudson committed
1172 1173 1174 1175 1176
/* When a thread state is created for a thread by some mechanism other than
   PyGILState_Ensure, it's important that the GILState machinery knows about
   it so it doesn't try to create another thread state for the thread (this is
   a better fix for SF bug #1010677 than the first one attempted).
*/
1177
static void
1178
_PyGILState_NoteThreadState(struct _gilstate_runtime_state *gilstate, PyThreadState* tstate)
Michael W. Hudson's avatar
Michael W. Hudson committed
1179
{
1180
    /* If autoTSSkey isn't initialized, this must be the very first
1181 1182
       threadstate created in Py_Initialize().  Don't do anything for now
       (we'll be back here when _PyGILState_Init is called). */
1183
    if (!gilstate->autoInterpreterState) {
1184
        return;
1185
    }
1186

1187
    /* Stick the thread state for this thread in thread specific storage.
Michael W. Hudson's avatar
Michael W. Hudson committed
1188

1189 1190
       The only situation where you can legitimately have more than one
       thread state for an OS level thread is when there are multiple
1191
       interpreters.
1192

1193 1194
       You shouldn't really be using the PyGILState_ APIs anyway (see issues
       #10915 and #15751).
Michael W. Hudson's avatar
Michael W. Hudson committed
1195

1196 1197
       The first thread state created for that given OS level thread will
       "win", which seems reasonable behaviour.
1198
    */
1199 1200
    if (PyThread_tss_get(&gilstate->autoTSSkey) == NULL) {
        if ((PyThread_tss_set(&gilstate->autoTSSkey, (void *)tstate)) != 0) {
1201
            Py_FatalError("Couldn't create autoTSSkey mapping");
1202
        }
1203
    }
Michael W. Hudson's avatar
Michael W. Hudson committed
1204

1205 1206
    /* PyGILState_Release must not try to delete this thread state. */
    tstate->gilstate_counter = 1;
Michael W. Hudson's avatar
Michael W. Hudson committed
1207 1208
}

1209
/* The public functions */
1210 1211 1212 1213 1214 1215 1216 1217
static PyThreadState *
_PyGILState_GetThisThreadState(struct _gilstate_runtime_state *gilstate)
{
    if (gilstate->autoInterpreterState == NULL)
        return NULL;
    return (PyThreadState *)PyThread_tss_get(&gilstate->autoTSSkey);
}

1218 1219
PyThreadState *
PyGILState_GetThisThreadState(void)
1220
{
1221
    return _PyGILState_GetThisThreadState(&_PyRuntime.gilstate);
1222 1223
}

1224 1225 1226
int
PyGILState_Check(void)
{
1227

1228
    if (!_PyGILState_check_enabled) {
1229
        return 1;
1230
    }
1231

1232 1233
    struct _gilstate_runtime_state *gilstate = &_PyRuntime.gilstate;
    if (!PyThread_tss_is_created(&gilstate->autoTSSkey)) {
1234
        return 1;
1235
    }
1236

1237 1238
    PyThreadState *tstate = _PyRuntimeGILState_GetThreadState(gilstate);
    if (tstate == NULL) {
1239
        return 0;
1240
    }
1241

1242
    return (tstate == _PyGILState_GetThisThreadState(gilstate));
1243 1244
}

1245 1246
PyGILState_STATE
PyGILState_Ensure(void)
1247
{
1248
    struct _gilstate_runtime_state *gilstate = &_PyRuntime.gilstate;
1249 1250
    int current;
    PyThreadState *tcur;
1251 1252
    int need_init_threads = 0;

1253 1254 1255 1256 1257
    /* Note that we do not auto-init Python here - apart from
       potential races with 2 threads auto-initializing, pep-311
       spells out other issues.  Embedders are expected to have
       called Py_Initialize() and usually PyEval_InitThreads().
    */
1258
    /* Py_Initialize() hasn't been called! */
1259
    assert(gilstate->autoInterpreterState);
1260

1261
    tcur = (PyThreadState *)PyThread_tss_get(&gilstate->autoTSSkey);
1262
    if (tcur == NULL) {
1263
        need_init_threads = 1;
1264

1265
        /* Create a new thread state for this thread */
1266
        tcur = PyThreadState_New(gilstate->autoInterpreterState);
1267 1268 1269 1270 1271 1272 1273
        if (tcur == NULL)
            Py_FatalError("Couldn't create thread-state for new thread");
        /* This is our thread state!  We'll need to delete it in the
           matching call to PyGILState_Release(). */
        tcur->gilstate_counter = 0;
        current = 0; /* new thread state is never current */
    }
1274
    else {
1275
        current = PyThreadState_IsCurrent(tcur);
1276 1277 1278
    }

    if (current == 0) {
1279
        PyEval_RestoreThread(tcur);
1280 1281
    }

1282 1283 1284 1285 1286 1287
    /* Update our counter in the thread-state - no need for locks:
       - tcur will remain valid as we hold the GIL.
       - the counter is safe as we are the only thread "allowed"
         to modify this value
    */
    ++tcur->gilstate_counter;
1288 1289 1290 1291 1292 1293 1294 1295

    if (need_init_threads) {
        /* At startup, Python has no concrete GIL. If PyGILState_Ensure() is
           called from a new thread for the first time, we need the create the
           GIL. */
        PyEval_InitThreads();
    }

1296
    return current ? PyGILState_LOCKED : PyGILState_UNLOCKED;
1297 1298
}

1299 1300
void
PyGILState_Release(PyGILState_STATE oldstate)
1301
{
1302 1303 1304
    _PyRuntimeState *runtime = &_PyRuntime;
    PyThreadState *tcur = PyThread_tss_get(&runtime->gilstate.autoTSSkey);
    if (tcur == NULL) {
1305 1306
        Py_FatalError("auto-releasing thread-state, "
                      "but no thread-state for this thread");
1307 1308
    }

1309 1310 1311 1312 1313
    /* We must hold the GIL and have our thread state current */
    /* XXX - remove the check - the assert should be fine,
       but while this is very new (April 2003), the extra check
       by release-only users can't hurt.
    */
1314
    if (!PyThreadState_IsCurrent(tcur)) {
1315
        Py_FatalError("This thread state must be current when releasing");
1316
    }
1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332
    assert(PyThreadState_IsCurrent(tcur));
    --tcur->gilstate_counter;
    assert(tcur->gilstate_counter >= 0); /* illegal counter value */

    /* If we're going to destroy this thread-state, we must
     * clear it while the GIL is held, as destructors may run.
     */
    if (tcur->gilstate_counter == 0) {
        /* can't have been locked when we created it */
        assert(oldstate == PyGILState_UNLOCKED);
        PyThreadState_Clear(tcur);
        /* Delete the thread-state.  Note this releases the GIL too!
         * It's vital that the GIL be held here, to avoid shutdown
         * races; see bugs 225673 and 1061968 (that nasty bug has a
         * habit of coming back).
         */
1333
        _PyThreadState_DeleteCurrent(runtime);
1334 1335 1336 1337
    }
    /* Release the lock if necessary */
    else if (oldstate == PyGILState_UNLOCKED)
        PyEval_SaveThread();
1338
}
1339

1340

1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395
/**************************/
/* cross-interpreter data */
/**************************/

/* cross-interpreter data */

crossinterpdatafunc _PyCrossInterpreterData_Lookup(PyObject *);

/* This is a separate func from _PyCrossInterpreterData_Lookup in order
   to keep the registry code separate. */
static crossinterpdatafunc
_lookup_getdata(PyObject *obj)
{
    crossinterpdatafunc getdata = _PyCrossInterpreterData_Lookup(obj);
    if (getdata == NULL && PyErr_Occurred() == 0)
        PyErr_Format(PyExc_ValueError,
                     "%S does not support cross-interpreter data", obj);
    return getdata;
}

int
_PyObject_CheckCrossInterpreterData(PyObject *obj)
{
    crossinterpdatafunc getdata = _lookup_getdata(obj);
    if (getdata == NULL) {
        return -1;
    }
    return 0;
}

static int
_check_xidata(_PyCrossInterpreterData *data)
{
    // data->data can be anything, including NULL, so we don't check it.

    // data->obj may be NULL, so we don't check it.

    if (data->interp < 0) {
        PyErr_SetString(PyExc_SystemError, "missing interp");
        return -1;
    }

    if (data->new_object == NULL) {
        PyErr_SetString(PyExc_SystemError, "missing new_object func");
        return -1;
    }

    // data->free may be NULL, so we don't check it.

    return 0;
}

int
_PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data)
{
1396
    // _PyInterpreterState_Get() aborts if lookup fails, so we don't need
1397
    // to check the result for NULL.
1398
    PyInterpreterState *interp = _PyInterpreterState_Get();
1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426

    // Reset data before re-populating.
    *data = (_PyCrossInterpreterData){0};
    data->free = PyMem_RawFree;  // Set a default that may be overridden.

    // Call the "getdata" func for the object.
    Py_INCREF(obj);
    crossinterpdatafunc getdata = _lookup_getdata(obj);
    if (getdata == NULL) {
        Py_DECREF(obj);
        return -1;
    }
    int res = getdata(obj, data);
    Py_DECREF(obj);
    if (res != 0) {
        return -1;
    }

    // Fill in the blanks and validate the result.
    data->interp = interp->id;
    if (_check_xidata(data) != 0) {
        _PyCrossInterpreterData_Release(data);
        return -1;
    }

    return 0;
}

1427
static void
1428 1429 1430 1431 1432 1433 1434
_release_xidata(void *arg)
{
    _PyCrossInterpreterData *data = (_PyCrossInterpreterData *)arg;
    if (data->free != NULL) {
        data->free(data->data);
    }
    Py_XDECREF(data->obj);
1435 1436 1437
}

static void
1438 1439
_call_in_interpreter(struct _gilstate_runtime_state *gilstate,
                     PyInterpreterState *interp,
1440 1441 1442 1443 1444 1445 1446
                     void (*func)(void *), void *arg)
{
    /* We would use Py_AddPendingCall() if it weren't specific to the
     * main interpreter (see bpo-33608).  In the meantime we take a
     * naive approach.
     */
    PyThreadState *save_tstate = NULL;
1447
    if (interp != _PyRuntimeGILState_GetThreadState(gilstate)->interp) {
1448 1449 1450
        // XXX Using the "head" thread isn't strictly correct.
        PyThreadState *tstate = PyInterpreterState_ThreadHead(interp);
        // XXX Possible GILState issues?
1451
        save_tstate = _PyThreadState_Swap(gilstate, tstate);
1452 1453 1454 1455 1456 1457
    }

    func(arg);

    // Switch back.
    if (save_tstate != NULL) {
1458
        _PyThreadState_Swap(gilstate, save_tstate);
1459
    }
1460 1461
}

1462 1463 1464 1465 1466 1467 1468 1469
void
_PyCrossInterpreterData_Release(_PyCrossInterpreterData *data)
{
    if (data->data == NULL && data->obj == NULL) {
        // Nothing to release!
        return;
    }

1470
    // Switch to the original interpreter.
1471 1472 1473 1474 1475 1476 1477 1478
    PyInterpreterState *interp = _PyInterpreterState_LookUpID(data->interp);
    if (interp == NULL) {
        // The intepreter was already destroyed.
        if (data->free != NULL) {
            // XXX Someone leaked some memory...
        }
        return;
    }
1479

1480
    // "Release" the data and/or the object.
1481 1482
    struct _gilstate_runtime_state *gilstate = &_PyRuntime.gilstate;
    _call_in_interpreter(gilstate, interp, _release_xidata, data);
1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497
}

PyObject *
_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data)
{
    return data->new_object(data);
}

/* registry of {type -> crossinterpdatafunc} */

/* For now we use a global registry of shareable classes.  An
   alternative would be to add a tp_* slot for a class's
   crossinterpdatafunc. It would be simpler and more efficient. */

static int
1498 1499
_register_xidata(struct _xidregistry *xidregistry, PyTypeObject *cls,
                 crossinterpdatafunc getdata)
1500 1501 1502 1503 1504 1505 1506 1507
{
    // Note that we effectively replace already registered classes
    // rather than failing.
    struct _xidregitem *newhead = PyMem_RawMalloc(sizeof(struct _xidregitem));
    if (newhead == NULL)
        return -1;
    newhead->cls = cls;
    newhead->getdata = getdata;
1508 1509
    newhead->next = xidregistry->head;
    xidregistry->head = newhead;
1510 1511 1512
    return 0;
}

1513
static void _register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry);
1514 1515

int
1516
_PyCrossInterpreterData_RegisterClass(PyTypeObject *cls,
1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530
                                       crossinterpdatafunc getdata)
{
    if (!PyType_Check(cls)) {
        PyErr_Format(PyExc_ValueError, "only classes may be registered");
        return -1;
    }
    if (getdata == NULL) {
        PyErr_Format(PyExc_ValueError, "missing 'getdata' func");
        return -1;
    }

    // Make sure the class isn't ever deallocated.
    Py_INCREF((PyObject *)cls);

1531 1532 1533 1534
    struct _xidregistry *xidregistry = &_PyRuntime.xidregistry ;
    PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK);
    if (xidregistry->head == NULL) {
        _register_builtins_for_crossinterpreter_data(xidregistry);
1535
    }
1536 1537
    int res = _register_xidata(xidregistry, cls, getdata);
    PyThread_release_lock(xidregistry->mutex);
1538 1539 1540
    return res;
}

1541 1542 1543 1544
/* Cross-interpreter objects are looked up by exact match on the class.
   We can reassess this policy when we move from a global registry to a
   tp_* slot. */

1545 1546 1547
crossinterpdatafunc
_PyCrossInterpreterData_Lookup(PyObject *obj)
{
1548
    struct _xidregistry *xidregistry = &_PyRuntime.xidregistry ;
1549 1550
    PyObject *cls = PyObject_Type(obj);
    crossinterpdatafunc getdata = NULL;
1551 1552
    PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK);
    struct _xidregitem *cur = xidregistry->head;
1553
    if (cur == NULL) {
1554 1555
        _register_builtins_for_crossinterpreter_data(xidregistry);
        cur = xidregistry->head;
1556 1557 1558 1559 1560 1561 1562
    }
    for(; cur != NULL; cur = cur->next) {
        if (cur->cls == (PyTypeObject *)cls) {
            getdata = cur->getdata;
            break;
        }
    }
1563
    Py_DECREF(cls);
1564
    PyThread_release_lock(xidregistry->mutex);
1565 1566 1567 1568 1569
    return getdata;
}

/* cross-interpreter data for builtin types */

1570 1571 1572 1573 1574
struct _shared_bytes_data {
    char *bytes;
    Py_ssize_t len;
};

1575 1576 1577
static PyObject *
_new_bytes_object(_PyCrossInterpreterData *data)
{
1578 1579
    struct _shared_bytes_data *shared = (struct _shared_bytes_data *)(data->data);
    return PyBytes_FromStringAndSize(shared->bytes, shared->len);
1580 1581 1582 1583 1584
}

static int
_bytes_shared(PyObject *obj, _PyCrossInterpreterData *data)
{
1585 1586 1587 1588 1589
    struct _shared_bytes_data *shared = PyMem_NEW(struct _shared_bytes_data, 1);
    if (PyBytes_AsStringAndSize(obj, &shared->bytes, &shared->len) < 0) {
        return -1;
    }
    data->data = (void *)shared;
1590
    Py_INCREF(obj);
1591 1592
    data->obj = obj;  // Will be "released" (decref'ed) when data released.
    data->new_object = _new_bytes_object;
1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617
    data->free = PyMem_Free;
    return 0;
}

struct _shared_str_data {
    int kind;
    const void *buffer;
    Py_ssize_t len;
};

static PyObject *
_new_str_object(_PyCrossInterpreterData *data)
{
    struct _shared_str_data *shared = (struct _shared_str_data *)(data->data);
    return PyUnicode_FromKindAndData(shared->kind, shared->buffer, shared->len);
}

static int
_str_shared(PyObject *obj, _PyCrossInterpreterData *data)
{
    struct _shared_str_data *shared = PyMem_NEW(struct _shared_str_data, 1);
    shared->kind = PyUnicode_KIND(obj);
    shared->buffer = PyUnicode_DATA(obj);
    shared->len = PyUnicode_GET_LENGTH(obj) - 1;
    data->data = (void *)shared;
1618
    Py_INCREF(obj);
1619 1620 1621 1622 1623 1624 1625 1626 1627
    data->obj = obj;  // Will be "released" (decref'ed) when data released.
    data->new_object = _new_str_object;
    data->free = PyMem_Free;
    return 0;
}

static PyObject *
_new_long_object(_PyCrossInterpreterData *data)
{
1628
    return PyLong_FromSsize_t((Py_ssize_t)(data->data));
1629 1630 1631 1632 1633
}

static int
_long_shared(PyObject *obj, _PyCrossInterpreterData *data)
{
1634 1635 1636 1637 1638
    /* Note that this means the size of shareable ints is bounded by
     * sys.maxsize.  Hence on 32-bit architectures that is half the
     * size of maximum shareable ints on 64-bit.
     */
    Py_ssize_t value = PyLong_AsSsize_t(obj);
1639 1640 1641 1642 1643 1644 1645 1646 1647 1648
    if (value == -1 && PyErr_Occurred()) {
        if (PyErr_ExceptionMatches(PyExc_OverflowError)) {
            PyErr_SetString(PyExc_OverflowError, "try sending as bytes");
        }
        return -1;
    }
    data->data = (void *)value;
    data->obj = NULL;
    data->new_object = _new_long_object;
    data->free = NULL;
1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670
    return 0;
}

static PyObject *
_new_none_object(_PyCrossInterpreterData *data)
{
    // XXX Singleton refcounts are problematic across interpreters...
    Py_INCREF(Py_None);
    return Py_None;
}

static int
_none_shared(PyObject *obj, _PyCrossInterpreterData *data)
{
    data->data = NULL;
    // data->obj remains NULL
    data->new_object = _new_none_object;
    data->free = NULL;  // There is nothing to free.
    return 0;
}

static void
1671
_register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry)
1672 1673
{
    // None
1674
    if (_register_xidata(xidregistry, (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) {
1675 1676 1677
        Py_FatalError("could not register None for cross-interpreter sharing");
    }

1678
    // int
1679
    if (_register_xidata(xidregistry, &PyLong_Type, _long_shared) != 0) {
1680 1681 1682
        Py_FatalError("could not register int for cross-interpreter sharing");
    }

1683
    // bytes
1684
    if (_register_xidata(xidregistry, &PyBytes_Type, _bytes_shared) != 0) {
1685 1686
        Py_FatalError("could not register bytes for cross-interpreter sharing");
    }
1687 1688

    // str
1689
    if (_register_xidata(xidregistry, &PyUnicode_Type, _str_shared) != 0) {
1690 1691
        Py_FatalError("could not register str for cross-interpreter sharing");
    }
1692 1693 1694
}


1695 1696 1697
#ifdef __cplusplus
}
#endif