From 23a6e5943bb2a70b8d6c60afc27aeb6158a238d5 Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Sun, 30 Apr 2017 10:40:10 +0900 Subject: [PATCH 01/41] Implement PEP 539 Thread Specific Storage (TSS) API See PEP 539 for details. highlights of changes: - Add Thread Specific Storage (TSS) API - Add test for transitions of the thread key state. - Mark deprecation to Thread Local Storage (TLS) API - Replace codes that used TLS API with TSS API --- Include/pythread.h | 72 ++++++++++++++++++++--- Modules/_testcapimodule.c | 46 +++++++++++++++ Modules/_tracemalloc.c | 18 +++--- Modules/faulthandler.c | 2 +- Modules/signalmodule.c | 4 +- Python/pystate.c | 53 ++++++++--------- Python/thread.c | 121 ++++++++++++++++++++++++++++++-------- Python/thread_foobar.h | 69 +++++++++++++++++++++- Python/thread_nt.h | 85 ++++++++++++++++++++++++-- Python/thread_pthread.h | 85 +++++++++++++++++++++++++- Python/traceback.c | 2 +- configure | 69 ++++++++++++++++++++++ configure.ac | 19 ++++++ pyconfig.h.in | 6 ++ 14 files changed, 565 insertions(+), 86 deletions(-) diff --git a/Include/pythread.h b/Include/pythread.h index dbacb8bbbcd8eb..c3ad2479b6bd76 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -2,6 +2,8 @@ #ifndef Py_PYTHREAD_H #define Py_PYTHREAD_H +#include /* necessary for TSS key */ + typedef void *PyThread_type_lock; typedef void *PyThread_type_sema; @@ -29,8 +31,8 @@ PyAPI_FUNC(unsigned long) PyThread_get_thread_ident(void); PyAPI_FUNC(PyThread_type_lock) PyThread_allocate_lock(void); PyAPI_FUNC(void) PyThread_free_lock(PyThread_type_lock); PyAPI_FUNC(int) PyThread_acquire_lock(PyThread_type_lock, int); -#define WAIT_LOCK 1 -#define NOWAIT_LOCK 0 +#define WAIT_LOCK 1 +#define NOWAIT_LOCK 0 /* PY_TIMEOUT_T is the integral type used to specify timeouts when waiting on a lock (see PyThread_acquire_lock_timed() below). @@ -77,15 +79,67 @@ PyAPI_FUNC(int) PyThread_set_stacksize(size_t); PyAPI_FUNC(PyObject*) PyThread_GetInfo(void); #endif -/* Thread Local Storage (TLS) API */ -PyAPI_FUNC(int) PyThread_create_key(void); -PyAPI_FUNC(void) PyThread_delete_key(int); -PyAPI_FUNC(int) PyThread_set_key_value(int, void *); -PyAPI_FUNC(void *) PyThread_get_key_value(int); -PyAPI_FUNC(void) PyThread_delete_key_value(int key); +/* Thread Local Storage (TLS) API + TLS API is DEPRECATED. Use Thread Specific Storage API. +*/ +PyAPI_FUNC(int) PyThread_create_key(void) Py_DEPRECATED(3.7); +PyAPI_FUNC(void) PyThread_delete_key(int key) Py_DEPRECATED(3.7); +PyAPI_FUNC(int) PyThread_set_key_value(int key, void *value) Py_DEPRECATED(3.7); +PyAPI_FUNC(void *) PyThread_get_key_value(int key) Py_DEPRECATED(3.7); +PyAPI_FUNC(void) PyThread_delete_key_value(int key) Py_DEPRECATED(3.7); + +/* Cleanup after a fork */ +PyAPI_FUNC(void) PyThread_ReInitTLS(void) Py_DEPRECATED(3.7); + +/* Thread Specific Storage (TSS) API + + POSIX hasn't defined that pthread_key_t is compatible with int + (for details, see PEP 539). Therefore, TSS API uses opaque type to cover + the key details. +*/ + +#if defined(_POSIX_THREADS) +# define NATIVE_TLS_KEY_T pthread_key_t +#elif defined(NT_THREADS) +# include +# define NATIVE_TLS_KEY_T DWORD +#else /* For the platform that has not supplied native TLS */ +# define NATIVE_TLS_KEY_T int +#endif + +/* Py_tss_t is opaque type and you *must not* directly read and write. + When you'd check whether the key is created, use PyThread_tss_is_created. +*/ +typedef struct { + bool _is_initialized; + NATIVE_TLS_KEY_T _key; +} Py_tss_t; + +#undef NATIVE_TLS_KEY_T + +static inline bool +PyThread_tss_is_created(Py_tss_t key) +{ + return key._is_initialized; +} + +/* Py_tss_NEEDS_INIT is the defined invalid value, and you *must* initialize + the Py_tss_t variable by this value to use TSS API. + + For example: + static Py_tss_t thekey = Py_tss_NEEDS_INIT; + int fail = PyThread_tss_create(&thekey); +*/ +#define Py_tss_NEEDS_INIT {._is_initialized = false} + +PyAPI_FUNC(int) PyThread_tss_create(Py_tss_t *key); +PyAPI_FUNC(void) PyThread_tss_delete(Py_tss_t *key); +PyAPI_FUNC(int) PyThread_tss_set(Py_tss_t key, void *value); +PyAPI_FUNC(void *) PyThread_tss_get(Py_tss_t key); +PyAPI_FUNC(void) PyThread_tss_delete_value(Py_tss_t key); /* Cleanup after a fork */ -PyAPI_FUNC(void) PyThread_ReInitTLS(void); +PyAPI_FUNC(void) PyThread_ReInitTSS(void); #ifdef __cplusplus } diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 54605872304ba5..4e985020d5e2cb 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4027,6 +4027,49 @@ dict_get_version(PyObject *self, PyObject *args) return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)version); } +#ifdef WITH_THREAD +static PyObject * +test_pythread_tss_key_state(PyObject *self, PyObject *args) +{ + Py_tss_t tss_key = Py_tss_NEEDS_INIT; + if (PyThread_tss_is_created(tss_key)) { + return raiseTestError("test_pythread_tss_key_state", + "TSS key not in an uninitialized state at " + "creation time"); + } + if (PyThread_tss_create(&tss_key) != 0) { + PyErr_SetString(PyExc_RuntimeError, "PyThread_tss_create failed"); + return NULL; + } + if (!PyThread_tss_is_created(tss_key)) { + return raiseTestError("test_pythread_tss_key_state", + "PyThread_tss_create succeeded, " + "but with TSS key in an uninitialized state"); + } + if (PyThread_tss_create(&tss_key) != 0) { + return raiseTestError("test_pythread_tss_key_state", + "PyThread_tss_create unsuccessful with " + "an already initialized key"); + } +#define CHECK_TSS_API(expr) \ + (void)(expr); \ + if (!PyThread_tss_is_created(tss_key)) { \ + return raiseTestError("test_pythread_tss_key_state", \ + "TSS key initialization state was not " \ + "preserved after calling " #expr); } + CHECK_TSS_API(PyThread_tss_set(tss_key, NULL)); + CHECK_TSS_API(PyThread_tss_get(tss_key)); + CHECK_TSS_API(PyThread_tss_delete_value(tss_key)); +#undef CHECK_TSS_API + PyThread_tss_delete(&tss_key); + if (PyThread_tss_is_created(tss_key)) { + return raiseTestError("test_pythread_tss_key_state", + "PyThread_tss_delete called, but did not " + "set the key state to uninitialized"); + } + Py_RETURN_NONE; +} +#endif static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, @@ -4232,6 +4275,9 @@ static PyMethodDef TestMethods[] = { {"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS}, {"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS}, {"dict_get_version", dict_get_version, METH_VARARGS}, +#ifdef WITH_THREAD + {"test_pythread_tss_key_state", test_pythread_tss_key_state, METH_VARARGS}, +#endif {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 950789ba53da6e..58bdd9a24ef145 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -171,7 +171,7 @@ tracemalloc_error(const char *format, ...) # error "need native thread local storage (TLS)" #endif -static int tracemalloc_reentrant_key = -1; +static Py_tss_t tracemalloc_reentrant_key = Py_tss_NEEDS_INIT; /* Any non-NULL pointer can be used */ #define REENTRANT Py_True @@ -181,8 +181,8 @@ get_reentrant(void) { void *ptr; - assert(tracemalloc_reentrant_key != -1); - ptr = PyThread_get_key_value(tracemalloc_reentrant_key); + assert(PyThread_tss_is_created(tracemalloc_reentrant_key)); + ptr = PyThread_tss_get(tracemalloc_reentrant_key); if (ptr != NULL) { assert(ptr == REENTRANT); return 1; @@ -195,15 +195,15 @@ static void set_reentrant(int reentrant) { assert(reentrant == 0 || reentrant == 1); - assert(tracemalloc_reentrant_key != -1); + assert(PyThread_tss_is_created(tracemalloc_reentrant_key)); if (reentrant) { assert(!get_reentrant()); - PyThread_set_key_value(tracemalloc_reentrant_key, REENTRANT); + PyThread_tss_set(tracemalloc_reentrant_key, REENTRANT); } else { assert(get_reentrant()); - PyThread_set_key_value(tracemalloc_reentrant_key, NULL); + PyThread_tss_set(tracemalloc_reentrant_key, NULL); } } @@ -993,8 +993,7 @@ tracemalloc_init(void) PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); #ifdef REENTRANT_THREADLOCAL - tracemalloc_reentrant_key = PyThread_create_key(); - if (tracemalloc_reentrant_key == -1) { + if (PyThread_tss_create(&tracemalloc_reentrant_key) != 0) { #ifdef MS_WINDOWS PyErr_SetFromWindowsErr(0); #else @@ -1079,8 +1078,7 @@ tracemalloc_deinit(void) #endif #ifdef REENTRANT_THREADLOCAL - PyThread_delete_key(tracemalloc_reentrant_key); - tracemalloc_reentrant_key = -1; + PyThread_tss_delete(&tracemalloc_reentrant_key); #endif Py_XDECREF(unknown_filename); diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index dcfebf2781f6c9..89023fdcee873a 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -235,7 +235,7 @@ faulthandler_dump_traceback(int fd, int all_threads, PyThreadState_Get() doesn't give the state of the thread that caused the fault if the thread released the GIL, and so this function cannot be - used. Read the thread local storage (TLS) instead: call + used. Read the thread specific storage (TSS) instead: call PyGILState_GetThisThreadState(). */ tstate = PyGILState_GetThisThreadState(); #else diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 7982139822a598..11d5d5e1864736 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -1606,9 +1606,9 @@ PyOS_AfterFork(void) * the interpreter had an opportunity to call the handlers. issue9535. */ _clear_pending_signals(); #ifdef WITH_THREAD - /* PyThread_ReInitTLS() must be called early, to make sure that the TLS API + /* PyThread_ReInitTSS() must be called early, to make sure that the TSS API * can be called safely. */ - PyThread_ReInitTLS(); + PyThread_ReInitTSS(); _PyGILState_Reinit(); PyEval_ReInitThreads(); main_thread = PyThread_get_thread_ident(); diff --git a/Python/pystate.c b/Python/pystate.c index 8e81707c7cc559..ceca6c4448703c 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -47,7 +47,7 @@ static PyThread_type_lock head_mutex = NULL; /* Protects interp->tstate_head */ GILState implementation */ static PyInterpreterState *autoInterpreterState = NULL; -static int autoTLSkey = -1; +static Py_tss_t autoTSSkey = Py_tss_NEEDS_INIT; #else #define HEAD_INIT() /* Nothing */ #define HEAD_LOCK() /* Nothing */ @@ -444,8 +444,8 @@ PyThreadState_Delete(PyThreadState *tstate) if (tstate == GET_TSTATE()) Py_FatalError("PyThreadState_Delete: tstate is still current"); #ifdef WITH_THREAD - if (autoInterpreterState && PyThread_get_key_value(autoTLSkey) == tstate) - PyThread_delete_key_value(autoTLSkey); + if (autoInterpreterState && PyThread_tss_get(autoTSSkey) == tstate) + PyThread_tss_delete_value(autoTSSkey); #endif /* WITH_THREAD */ tstate_delete_common(tstate); } @@ -460,8 +460,8 @@ PyThreadState_DeleteCurrent() Py_FatalError( "PyThreadState_DeleteCurrent: no current tstate"); tstate_delete_common(tstate); - if (autoInterpreterState && PyThread_get_key_value(autoTLSkey) == tstate) - PyThread_delete_key_value(autoTLSkey); + if (autoInterpreterState && PyThread_tss_get(autoTSSkey) == tstate) + PyThread_tss_delete_value(autoTSSkey); SET_TSTATE(NULL); PyEval_ReleaseLock(); } @@ -713,11 +713,10 @@ void _PyGILState_Init(PyInterpreterState *i, PyThreadState *t) { assert(i && t); /* must init with valid states */ - autoTLSkey = PyThread_create_key(); - if (autoTLSkey == -1) - Py_FatalError("Could not allocate TLS entry"); + if (PyThread_tss_create(&autoTSSkey) != 0) + Py_FatalError("Could not allocate TSS entry"); autoInterpreterState = i; - assert(PyThread_get_key_value(autoTLSkey) == NULL); + assert(PyThread_tss_get(autoTSSkey) == NULL); assert(t->gilstate_counter == 0); _PyGILState_NoteThreadState(t); @@ -732,27 +731,26 @@ _PyGILState_GetInterpreterStateUnsafe(void) void _PyGILState_Fini(void) { - PyThread_delete_key(autoTLSkey); - autoTLSkey = -1; + PyThread_tss_delete(&autoTSSkey); autoInterpreterState = NULL; } -/* Reset the TLS key - called by PyOS_AfterFork(). +/* Reset the TSS key - called by PyOS_AfterFork(). * This should not be necessary, but some - buggy - pthread implementations - * don't reset TLS upon fork(), see issue #10517. + * don't reset TSS upon fork(), see issue #10517. */ void _PyGILState_Reinit(void) { PyThreadState *tstate = PyGILState_GetThisThreadState(); - PyThread_delete_key(autoTLSkey); - if ((autoTLSkey = PyThread_create_key()) == -1) - Py_FatalError("Could not allocate TLS entry"); + PyThread_tss_delete(&autoTSSkey); + if (PyThread_tss_create(&autoTSSkey) != 0) + Py_FatalError("Could not allocate TSS entry"); /* If the thread had an associated auto thread state, reassociate it with * the new key. */ - if (tstate && PyThread_set_key_value(autoTLSkey, (void *)tstate) < 0) - Py_FatalError("Couldn't create autoTLSkey mapping"); + if (tstate && PyThread_tss_set(autoTSSkey, (void *)tstate) != 0) + Py_FatalError("Couldn't create autoTSSkey mapping"); } /* When a thread state is created for a thread by some mechanism other than @@ -763,13 +761,13 @@ _PyGILState_Reinit(void) static void _PyGILState_NoteThreadState(PyThreadState* tstate) { - /* If autoTLSkey isn't initialized, this must be the very first + /* If autoTSSkey isn't initialized, this must be the very first threadstate created in Py_Initialize(). Don't do anything for now (we'll be back here when _PyGILState_Init is called). */ if (!autoInterpreterState) return; - /* Stick the thread state for this thread in thread local storage. + /* Stick the thread state for this thread in thread specific storage. The only situation where you can legitimately have more than one thread state for an OS level thread is when there are multiple @@ -781,9 +779,9 @@ _PyGILState_NoteThreadState(PyThreadState* tstate) The first thread state created for that given OS level thread will "win", which seems reasonable behaviour. */ - if (PyThread_get_key_value(autoTLSkey) == NULL) { - if (PyThread_set_key_value(autoTLSkey, (void *)tstate) < 0) - Py_FatalError("Couldn't create autoTLSkey mapping"); + if (PyThread_tss_get(autoTSSkey) == NULL) { + if (PyThread_tss_set(autoTSSkey, (void *)tstate) != 0) + Py_FatalError("Couldn't create autoTSSkey mapping"); } /* PyGILState_Release must not try to delete this thread state. */ @@ -796,7 +794,7 @@ PyGILState_GetThisThreadState(void) { if (autoInterpreterState == NULL) return NULL; - return (PyThreadState *)PyThread_get_key_value(autoTLSkey); + return (PyThreadState *)PyThread_tss_get(autoTSSkey); } int @@ -807,7 +805,7 @@ PyGILState_Check(void) if (!_PyGILState_check_enabled) return 1; - if (autoTLSkey == -1) + if (!PyThread_tss_is_created(autoTSSkey)) return 1; tstate = GET_TSTATE(); @@ -828,7 +826,7 @@ PyGILState_Ensure(void) called Py_Initialize() and usually PyEval_InitThreads(). */ assert(autoInterpreterState); /* Py_Initialize() hasn't been called! */ - tcur = (PyThreadState *)PyThread_get_key_value(autoTLSkey); + tcur = (PyThreadState *)PyThread_tss_get(autoTSSkey); if (tcur == NULL) { /* At startup, Python has no concrete GIL. If PyGILState_Ensure() is called from a new thread for the first time, we need the create the @@ -860,8 +858,7 @@ PyGILState_Ensure(void) void PyGILState_Release(PyGILState_STATE oldstate) { - PyThreadState *tcur = (PyThreadState *)PyThread_get_key_value( - autoTLSkey); + PyThreadState *tcur = (PyThreadState *)PyThread_tss_get(autoTSSkey); if (tcur == NULL) Py_FatalError("auto-releasing thread-state, " "but no thread-state for this thread"); diff --git a/Python/thread.c b/Python/thread.c index 3a52e1e89237d7..6ec67b25fb12cd 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -20,6 +20,7 @@ #include #endif +#include /* necessary for TSS key */ #include #include "pythread.h" @@ -125,25 +126,25 @@ PyThread_set_stacksize(size_t size) TLS implementation, provide our own. This code stolen from "thread_sgi.h", where it was the only - implementation of an existing Python TLS API. + implementation of a Python TLS API that has been deprecated. */ /* ------------------------------------------------------------------------ Per-thread data ("key") support. -Use PyThread_create_key() to create a new key. This is typically shared +Use PyThread_tss_create(&thekey) to create a new key. This is typically shared across threads. -Use PyThread_set_key_value(thekey, value) to associate void* value with +Use PyThread_tss_set(thekey, value) to associate void* value with thekey in the current thread. Each thread has a distinct mapping of thekey to a void* value. Caution: if the current thread already has a mapping for thekey, value is ignored. -Use PyThread_get_key_value(thekey) to retrieve the void* value associated +Use PyThread_tss_get(thekey) to retrieve the void* value associated with thekey in the current thread. This returns NULL if no value is associated with thekey in the current thread. -Use PyThread_delete_key_value(thekey) to forget the current thread's associated -value for thekey. PyThread_delete_key(thekey) forgets the values associated +Use PyThread_tss_delete_value(thekey) to forget the current thread's associated +value for thekey. PyThread_tss_delete(&thekey) forgets the values associated with thekey across *all* threads. While some of these functions have error-return values, none set any @@ -155,12 +156,12 @@ happen to be PyObject*, these functions don't do refcount operations on them either. The GIL does not need to be held when calling these functions; they supply -their own locking. This isn't true of PyThread_create_key(), though (see +their own locking. This isn't true of PyThread_tss_create(), though (see next paragraph). -There's a hidden assumption that PyThread_create_key() will be called before +There's a hidden assumption that PyThread_tss_create() will be called before any of the other functions are called. There's also a hidden assumption -that calls to PyThread_create_key() are serialized externally. +that calls to PyThread_tss_create() are serialized externally. ------------------------------------------------------------------------ */ /* A singly-linked list of struct key objects remembers all the key->value @@ -181,7 +182,7 @@ struct key { static struct key *keyhead = NULL; static PyThread_type_lock keymutex = NULL; -static int nkeys = 0; /* PyThread_create_key() hands out nkeys+1 next */ +static int nkeys = 0; /* PyThread_tss_create() hands out nkeys+1 next */ /* Internal helper. * If the current thread has a mapping for key, the appropriate struct key* @@ -200,7 +201,7 @@ static int nkeys = 0; /* PyThread_create_key() hands out nkeys+1 next */ * another thread to mutate the list, via key deletion, concurrent with * find_key() crawling over the list. Hilarity ensued. For example, when * the for-loop here does "p = p->next", p could end up pointing at a - * record that PyThread_delete_key_value() was concurrently free()'ing. + * record that PyThread_tss_delete_value() was concurrently free()'ing. * That could lead to anything, from failing to find a key that exists, to * segfaults. Now we lock the whole routine. */ @@ -225,10 +226,10 @@ find_key(int set_value, int key, void *value) * in a tight loop with the lock held. A similar check is done * in pystate.c tstate_delete_common(). */ if (p == prev_p) - Py_FatalError("tls find_key: small circular list(!)"); + Py_FatalError("tss find_key: small circular list(!)"); prev_p = p; if (p->next == keyhead) - Py_FatalError("tls find_key: circular list(!)"); + Py_FatalError("tss find_key: circular list(!)"); } if (!set_value && value == NULL) { assert(p == NULL); @@ -247,31 +248,97 @@ find_key(int set_value, int key, void *value) return p; } -/* Return a new key. This must be called before any other functions in + +/* Thread Local Storage (TLS) API, DEPRECATED since Python 3.7 + + In the own implementation, TLS API has been changed to call TSS API. +*/ + +int +PyThread_create_key(void) +{ + Py_tss_t proxy = Py_tss_NEEDS_INIT; + if (PyThread_tss_create(&proxy) != 0) + return -1; + /* In the own implementation, platform-specific key type is int. */ + return proxy._key; +} + +void +PyThread_delete_key(int key) +{ + Py_tss_t proxy = {._is_initialized = true, ._key = key}; + PyThread_tss_delete(&proxy); +} + +int +PyThread_set_key_value(int key, void *value) +{ + Py_tss_t proxy = {._is_initialized = true, ._key = key}; + return PyThread_tss_set(proxy, value); +} + +void * +PyThread_get_key_value(int key) +{ + Py_tss_t proxy = {._is_initialized = true, ._key = key}; + return PyThread_tss_get(proxy); +} + +void +PyThread_delete_key_value(int key) +{ + Py_tss_t proxy = {._is_initialized = true, ._key = key}; + PyThread_tss_delete_value(proxy); +} + + +void +PyThread_ReInitTLS(void) +{ + PyThread_ReInitTSS(); +} + + +/* Thread Specific Storage (TSS) API */ + +/* Assign a new key. This must be called before any other functions in * this family, and callers must arrange to serialize calls to this * function. No violations are detected. */ int -PyThread_create_key(void) +PyThread_tss_create(Py_tss_t *key) { /* All parts of this function are wrong if it's called by multiple * threads simultaneously. */ + assert(key != NULL); + /* If the key has been created, function is silently skipped. */ + if (key->_is_initialized) + return 0; + if (keymutex == NULL) keymutex = PyThread_allocate_lock(); - return ++nkeys; + key->_key = ++nkeys; + key->_is_initialized = true; + return 0; } /* Forget the associations for key across *all* threads. */ void -PyThread_delete_key(int key) +PyThread_tss_delete(Py_tss_t *key) { + assert(key != NULL); + /* If the key has not been created, function is silently skipped. */ + if (!key->_is_initialized) + return; + struct key *p, **q; PyThread_acquire_lock(keymutex, 1); q = &keyhead; while ((p = *q) != NULL) { - if (p->key == key) { + if (p->key == key->_key) { *q = p->next; PyMem_RawFree((void *)p); /* NB This does *not* free p->value! */ @@ -279,15 +346,16 @@ PyThread_delete_key(int key) else q = &p->next; } + key->_key = -1; + key->_is_initialized = false; PyThread_release_lock(keymutex); } int -PyThread_set_key_value(int key, void *value) +PyThread_tss_set(Py_tss_t key, void *value) { - struct key *p; + struct key *p = find_key(1, key._key, value); - p = find_key(1, key, value); if (p == NULL) return -1; else @@ -298,9 +366,9 @@ PyThread_set_key_value(int key, void *value) * if the current thread doesn't have an association for key. */ void * -PyThread_get_key_value(int key) +PyThread_tss_get(Py_tss_t key) { - struct key *p = find_key(0, key, NULL); + struct key *p = find_key(0, key._key, NULL); if (p == NULL) return NULL; @@ -310,7 +378,7 @@ PyThread_get_key_value(int key) /* Forget the current thread's association for key, if any. */ void -PyThread_delete_key_value(int key) +PyThread_tss_delete_value(Py_tss_t key) { unsigned long id = PyThread_get_thread_ident(); struct key *p, **q; @@ -318,7 +386,7 @@ PyThread_delete_key_value(int key) PyThread_acquire_lock(keymutex, 1); q = &keyhead; while ((p = *q) != NULL) { - if (p->key == key && p->id == id) { + if (p->key == key._key && p->id == id) { *q = p->next; PyMem_RawFree((void *)p); /* NB This does *not* free p->value! */ @@ -330,13 +398,14 @@ PyThread_delete_key_value(int key) PyThread_release_lock(keymutex); } + /* Forget everything not associated with the current thread id. * This function is called from PyOS_AfterFork(). It is necessary * because other thread ids which were in use at the time of the fork * may be reused for new threads created in the forked process. */ void -PyThread_ReInitTLS(void) +PyThread_ReInitTSS(void) { unsigned long id = PyThread_get_thread_ident(); struct key *p, **q; diff --git a/Python/thread_foobar.h b/Python/thread_foobar.h index 45cfd1bb6fd79e..8bb4c8fa9c15a0 100644 --- a/Python/thread_foobar.h +++ b/Python/thread_foobar.h @@ -71,7 +71,7 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, dprintf(("PyThread_acquire_lock_timed(%p, %lld, %d) called\n", lock, microseconds, intr_flag)); dprintf(("PyThread_acquire_lock_timed(%p, %lld, %d) -> %d\n", - lock, microseconds, intr_flag, success)); + lock, microseconds, intr_flag, success)); return success; } @@ -85,6 +85,9 @@ PyThread_release_lock(PyThread_type_lock lock) #define Py_HAVE_NATIVE_TLS #ifdef Py_HAVE_NATIVE_TLS + +/* Thread Local Storage (TLS) API, DEPRECATED since Python 3.7 */ + int PyThread_create_key(void) { @@ -123,10 +126,74 @@ PyThread_delete_key_value(int key) } + void PyThread_ReInitTLS(void) { } + +/* Thread Specific Storage (TSS) API */ + +int +PyThread_tss_create(Py_tss_t *key) +{ + assert(key != NULL); + /* If the key has been created, function is silently skipped. */ + if (key->_is_initialized) + return 0; + + Py_tss_t new_key; + /* A failure in this case returns -1 */ + if (!new_key._key) + return -1; + key->_key = new_key._key; + key->_is_initialized = true; + return 0; +} + +void +PyThread_tss_delete(Py_tss_t *key) +{ + assert(key != NULL); + /* If the key has not been created, function is silently skipped. */ + if (!key->_is_initialized) + return; + + key->_is_initialized = false; +} + +int +PyThread_tss_set(Py_tss_t key, void *value) +{ + int ok; + + /* A failure in this case returns -1 */ + if (!ok) + return -1; + return 0; +} + +void * +PyThread_tss_get(Py_tss_t key) +{ + void *result; + + return result; +} + +void +PyThread_tss_delete_value(Py_tss_t key) +{ + +} + + +void +PyThread_ReInitTSS(void) +{ + +} + #endif diff --git a/Python/thread_nt.h b/Python/thread_nt.h index 8158ff82487a28..ca32a9bcba33f2 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -8,6 +8,7 @@ #ifdef HAVE_PROCESS_H #include #endif +#include /* necessary for TSS key */ /* options */ #ifndef _PY_USE_CV_LOCKS @@ -352,6 +353,9 @@ _pythread_nt_set_stacksize(size_t size) #define Py_HAVE_NATIVE_TLS #ifdef Py_HAVE_NATIVE_TLS + +/* Thread Local Storage (TLS) API, DEPRECATED since Python 3.7 */ + int PyThread_create_key(void) { @@ -370,12 +374,8 @@ PyThread_delete_key(int key) int PyThread_set_key_value(int key, void *value) { - BOOL ok; - - ok = TlsSetValue(key, value); - if (!ok) - return -1; - return 0; + BOOL ok = TlsSetValue(key, value); + return ok ? 0 : -1; } void * @@ -402,6 +402,7 @@ PyThread_delete_key_value(int key) TlsSetValue(key, NULL); } + /* reinitialization of TLS is not necessary after fork when using * the native TLS functions. And forking isn't supported on Windows either. */ @@ -409,4 +410,76 @@ void PyThread_ReInitTLS(void) {} + +/* Thread Specific Storage (TSS) API */ + +int +PyThread_tss_create(Py_tss_t *key) +{ + assert(key != NULL); + /* If the key has been created, function is silently skipped. */ + if (key->_is_initialized) + return 0; + + DWORD result = TlsAlloc(); + if (result == TLS_OUT_OF_INDEXES) + return -1; + /* In Windows, platform-specific key type is DWORD. */ + key->_key = result; + key->_is_initialized = true; + return 0; +} + +void +PyThread_tss_delete(Py_tss_t *key) +{ + assert(key != NULL); + /* If the key has not been created, function is silently skipped. */ + if (!key->_is_initialized) + return; + + TlsFree(key->_key); + key->_key = TLS_OUT_OF_INDEXES; + key->_is_initialized = false; +} + +int +PyThread_tss_set(Py_tss_t key, void *value) +{ + BOOL ok = TlsSetValue(key._key, value); + return ok ? 0 : -1; +} + +void * +PyThread_tss_get(Py_tss_t key) +{ + /* because TSS is used in the Py_END_ALLOW_THREAD macro, + * it is necessary to preserve the windows error state, because + * it is assumed to be preserved across the call to the macro. + * Ideally, the macro should be fixed, but it is simpler to + * do it here. + */ + DWORD error = GetLastError(); + void *result = TlsGetValue(key._key); + SetLastError(error); + return result; +} + +void +PyThread_tss_delete_value(Py_tss_t key) +{ + /* NULL is used as "key missing", and it is also the default + * given by TlsGetValue() if nothing has been set yet. + */ + TlsSetValue(key._key, NULL); +} + + +/* reinitialization of TSS is not necessary after fork when using + * the native TLS functions. And forking isn't supported on Windows either. + */ +void +PyThread_ReInitTSS(void) +{} + #endif diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index b95840ce2d76ac..1d5a4f90949784 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -1,6 +1,7 @@ /* Posix threads interface */ +#include /* necessary for TSS key */ #include #include #if defined(__APPLE__) || defined(HAVE_PTHREAD_DESTRUCTOR) @@ -605,9 +606,19 @@ _pythread_pthread_set_stacksize(size_t size) #define Py_HAVE_NATIVE_TLS +/* Thread Local Storage (TLS) API, DEPRECATED since Python 3.7 + + Issue #25658: POSIX hasn't defined that pthread_key_t is compatible + with int. Note that if incompatible with int, PyThread_create_key + returns immediately a failure status and the other TLS functions + all are no-ops. Moreover, PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT will + be unnecessary after removing this API. +*/ + int PyThread_create_key(void) { +#ifdef PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT pthread_key_t key; int fail = pthread_key_create(&key, NULL); if (fail) @@ -619,34 +630,104 @@ PyThread_create_key(void) return -1; } return (int)key; +#else + return -1; /* never return valid key value. */ +#endif } void PyThread_delete_key(int key) { +#ifdef PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT pthread_key_delete(key); +#endif } void PyThread_delete_key_value(int key) { +#ifdef PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT pthread_setspecific(key, NULL); +#endif } int PyThread_set_key_value(int key, void *value) { - int fail; - fail = pthread_setspecific(key, value); +#ifdef PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT + int fail = pthread_setspecific(key, value); return fail ? -1 : 0; +#else + return -1; +#endif } void * PyThread_get_key_value(int key) { +#ifdef PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT return pthread_getspecific(key); +#else + return NULL; +#endif } + void PyThread_ReInitTLS(void) {} + + +/* Thread Specific Storage (TSS) API */ + +int +PyThread_tss_create(Py_tss_t *key) +{ + assert(key != NULL); + /* If the key has been created, function is silently skipped. */ + if (key->_is_initialized) + return 0; + + int fail = pthread_key_create(&(key->_key), NULL); + if (fail) + return -1; + key->_is_initialized = true; + return 0; +} + +void +PyThread_tss_delete(Py_tss_t *key) +{ + assert(key != NULL); + /* If the key has not been created, function is silently skipped. */ + if (!key->_is_initialized) + return; + + pthread_key_delete(key->_key); + /* pthread has not provided the defined invalid value for the key. */ + key->_is_initialized = false; +} + +int +PyThread_tss_set(Py_tss_t key, void *value) +{ + int fail = pthread_setspecific(key._key, value); + return fail ? -1 : 0; +} + +void * +PyThread_tss_get(Py_tss_t key) +{ + return pthread_getspecific(key._key); +} + +void +PyThread_tss_delete_value(Py_tss_t key) +{ + pthread_setspecific(key._key, NULL); +} + + +void +PyThread_ReInitTSS(void) +{} diff --git a/Python/traceback.c b/Python/traceback.c index b52385ef12f3c9..9154f76dab28c9 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -760,7 +760,7 @@ _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp, PyThreadState_Get() doesn't give the state of the thread that caused the fault if the thread released the GIL, and so this function - cannot be used. Read the thread local storage (TLS) instead: call + cannot be used. Read the thread specific storage (TSS) instead: call PyGILState_GetThisThreadState(). */ current_tstate = PyGILState_GetThisThreadState(); } diff --git a/configure b/configure index 3e89bfabc3a519..59af7659e4fe57 100755 --- a/configure +++ b/configure @@ -9000,6 +9000,75 @@ cat >>confdefs.h <<_ACEOF _ACEOF +fi + +# Issue #25658: POSIX hasn't defined that pthread_key_t is compatible with int. +# This checking will be unnecessary after removing deprecated TLS API. +# The cast to long int works around a bug in the HP C Compiler +# version HP92453-01 B.11.11.23709.GP, which incorrectly rejects +# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# This bug is HP SR number 8606223364. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking size of pthread_key_t" >&5 +$as_echo_n "checking size of pthread_key_t... " >&6; } +if ${ac_cv_sizeof_pthread_key_t+:} false; then : + $as_echo_n "(cached) " >&6 +else + if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (pthread_key_t))" "ac_cv_sizeof_pthread_key_t" "#include +"; then : + +else + if test "$ac_cv_type_pthread_key_t" = yes; then + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error 77 "cannot compute sizeof (pthread_key_t) +See \`config.log' for more details" "$LINENO" 5; } + else + ac_cv_sizeof_pthread_key_t=0 + fi +fi + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_pthread_key_t" >&5 +$as_echo "$ac_cv_sizeof_pthread_key_t" >&6; } + + + +cat >>confdefs.h <<_ACEOF +#define SIZEOF_PTHREAD_KEY_T $ac_cv_sizeof_pthread_key_t +_ACEOF + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether pthread_key_t is compatible with int" >&5 +$as_echo_n "checking whether pthread_key_t is compatible with int... " >&6; } +if test "$ac_cv_sizeof_pthread_key_t" -eq "$ac_cv_sizeof_int" ; then + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +pthread_key_t k; k * 1; + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_pthread_key_t_is_arithmetic_type=yes +else + ac_pthread_key_t_is_arithmetic_type=no + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_pthread_key_t_is_arithmetic_type" >&5 +$as_echo "$ac_pthread_key_t_is_arithmetic_type" >&6; } + if test "$ac_pthread_key_t_is_arithmetic_type" = yes ; then + +$as_echo "#define PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT 1" >>confdefs.h + + fi +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } fi CC="$ac_save_cc" diff --git a/configure.ac b/configure.ac index c58322dc8c7c07..bc0cff5dddff75 100644 --- a/configure.ac +++ b/configure.ac @@ -2291,6 +2291,25 @@ if test "$have_pthread_t" = yes ; then #endif ]) fi + +# Issue #25658: POSIX hasn't defined that pthread_key_t is compatible with int. +# This checking will be unnecessary after removing deprecated TLS API. +AC_CHECK_SIZEOF(pthread_key_t, [], [[#include ]]) +AC_MSG_CHECKING(whether pthread_key_t is compatible with int) +if test "$ac_cv_sizeof_pthread_key_t" -eq "$ac_cv_sizeof_int" ; then + AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM([[#include ]], [[pthread_key_t k; k * 1;]])], + [ac_pthread_key_t_is_arithmetic_type=yes], + [ac_pthread_key_t_is_arithmetic_type=no] + ) + AC_MSG_RESULT($ac_pthread_key_t_is_arithmetic_type) + if test "$ac_pthread_key_t_is_arithmetic_type" = yes ; then + AC_DEFINE(PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT, 1, + [Define if pthread_key_t is compatible with int.]) + fi +else + AC_MSG_RESULT(no) +fi CC="$ac_save_cc" AC_SUBST(OTHER_LIBTOOL_OPT) diff --git a/pyconfig.h.in b/pyconfig.h.in index 0a3d59ef9ae602..c976674e5d2187 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1241,6 +1241,9 @@ /* Define if POSIX semaphores aren't enabled on your system */ #undef POSIX_SEMAPHORES_NOT_ENABLED +/* Define if pthread_key_t is compatible with int. */ +#undef PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT + /* Defined if PTHREAD_SCOPE_SYSTEM supported. */ #undef PTHREAD_SYSTEM_SCHED_SUPPORTED @@ -1296,6 +1299,9 @@ /* The size of `pid_t', as computed by sizeof. */ #undef SIZEOF_PID_T +/* The size of `pthread_key_t', as computed by sizeof. */ +#undef SIZEOF_PTHREAD_KEY_T + /* The size of `pthread_t', as computed by sizeof. */ #undef SIZEOF_PTHREAD_T From 17b1be71c3a2c4dd7a5135fe92114652634eb77e Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Tue, 13 Jun 2017 23:40:04 +0900 Subject: [PATCH 02/41] bpo-25658: Rename ReInitTLS function in Modules/posixmodule.c The code has been moved from Modules/signalmodule.c. (346cbd3) --- Modules/posixmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index be8a66dd5028e8..80ec5672a368f2 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -450,9 +450,9 @@ void PyOS_AfterFork_Child(void) { #ifdef WITH_THREAD - /* PyThread_ReInitTLS() must be called early, to make sure that the TLS API + /* PyThread_ReInitTSS() must be called early, to make sure that the TSS API * can be called safely. */ - PyThread_ReInitTLS(); + PyThread_ReInitTSS(); _PyGILState_Reinit(); PyEval_ReInitThreads(); _PyImport_ReInitLock(); From 2f57cc0e69cd1ff807668b792a9d58ab3bf7ebc4 Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Wed, 3 May 2017 17:08:52 +0900 Subject: [PATCH 03/41] bpo-25658: Modify comment about wrapper function https://github.com/python/cpython/pull/1362#discussion_r114079176 --- Python/thread.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Python/thread.c b/Python/thread.c index 44638482f50210..85bd5099bd6906 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -245,7 +245,9 @@ find_key(int set_value, int key, void *value) /* Thread Local Storage (TLS) API, DEPRECATED since Python 3.7 - In the own implementation, TLS API has been changed to call TSS API. + While the native thread-local storage implementations have been retained, + this fallback implementation is now just a compatibility wrapper around + the TSS API. */ int From f827c452c32043b1e18f32daff9283f631b4fb6f Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Sat, 6 May 2017 21:32:28 +0900 Subject: [PATCH 04/41] bpo-25658: Rename native TSS key type macro https://github.com/python/cpython/pull/1362#discussion_r114473591 --- Include/pythread.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Include/pythread.h b/Include/pythread.h index c3ad2479b6bd76..4839ff720edd8f 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -99,12 +99,12 @@ PyAPI_FUNC(void) PyThread_ReInitTLS(void) Py_DEPRECATED(3.7); */ #if defined(_POSIX_THREADS) -# define NATIVE_TLS_KEY_T pthread_key_t +# define NATIVE_TSS_KEY_T pthread_key_t #elif defined(NT_THREADS) # include -# define NATIVE_TLS_KEY_T DWORD -#else /* For the platform that has not supplied native TLS */ -# define NATIVE_TLS_KEY_T int +# define NATIVE_TSS_KEY_T DWORD +#else /* For the platform that has not supplied native TSS */ +# define NATIVE_TSS_KEY_T int #endif /* Py_tss_t is opaque type and you *must not* directly read and write. @@ -112,10 +112,10 @@ PyAPI_FUNC(void) PyThread_ReInitTLS(void) Py_DEPRECATED(3.7); */ typedef struct { bool _is_initialized; - NATIVE_TLS_KEY_T _key; + NATIVE_TSS_KEY_T _key; } Py_tss_t; -#undef NATIVE_TLS_KEY_T +#undef NATIVE_TSS_KEY_T static inline bool PyThread_tss_is_created(Py_tss_t key) From b0e5ce1b2225719616cc04c31f9dfb2d8d6a0a94 Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Tue, 13 Jun 2017 06:46:46 +0900 Subject: [PATCH 05/41] bpo-25658: Format empty function definitions https://github.com/python/cpython/pull/1362#discussion_r114079269 --- Python/thread_nt.h | 6 ++++-- Python/thread_pthread.h | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Python/thread_nt.h b/Python/thread_nt.h index ca32a9bcba33f2..8fdbc12631a7de 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -408,7 +408,8 @@ PyThread_delete_key_value(int key) */ void PyThread_ReInitTLS(void) -{} +{ +} /* Thread Specific Storage (TSS) API */ @@ -480,6 +481,7 @@ PyThread_tss_delete_value(Py_tss_t key) */ void PyThread_ReInitTSS(void) -{} +{ +} #endif diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 1d5a4f90949784..d4e2d284e9c767 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -675,7 +675,8 @@ PyThread_get_key_value(int key) void PyThread_ReInitTLS(void) -{} +{ +} /* Thread Specific Storage (TSS) API */ @@ -730,4 +731,5 @@ PyThread_tss_delete_value(Py_tss_t key) void PyThread_ReInitTSS(void) -{} +{ +} From eb0e4a951ae21f1c091dd5ac065a3c433e81ad65 Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Sun, 7 May 2017 06:20:07 +0900 Subject: [PATCH 06/41] bpo-25658: Change API interfaces to provide even in the limited API https://github.com/python/cpython/pull/1362#discussion_r114079006 --- Include/pythread.h | 35 +++++++++++++-------- Modules/_testcapimodule.c | 27 ++++++++++++----- Modules/_tracemalloc.c | 10 +++--- Python/pystate.c | 24 +++++++-------- Python/thread.c | 64 +++++++++++++++++++++++++++++++-------- Python/thread_nt.h | 19 +++++++----- Python/thread_pthread.h | 17 ++++++----- 7 files changed, 132 insertions(+), 64 deletions(-) diff --git a/Include/pythread.h b/Include/pythread.h index 4839ff720edd8f..9ee3e871d57585 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -79,6 +79,7 @@ PyAPI_FUNC(int) PyThread_set_stacksize(size_t); PyAPI_FUNC(PyObject*) PyThread_GetInfo(void); #endif + /* Thread Local Storage (TLS) API TLS API is DEPRECATED. Use Thread Specific Storage API. */ @@ -91,18 +92,26 @@ PyAPI_FUNC(void) PyThread_delete_key_value(int key) Py_DEPRECATED(3.7); /* Cleanup after a fork */ PyAPI_FUNC(void) PyThread_ReInitTLS(void) Py_DEPRECATED(3.7); + /* Thread Specific Storage (TSS) API POSIX hasn't defined that pthread_key_t is compatible with int (for details, see PEP 539). Therefore, TSS API uses opaque type to cover the key details. */ +#ifdef Py_LIMITED_API +typedef struct _py_tss_t Py_tss_t; +#else #if defined(_POSIX_THREADS) + /* Darwin needs pthread.h to know type name the pthread_key_t. */ +# include # define NATIVE_TSS_KEY_T pthread_key_t #elif defined(NT_THREADS) -# include -# define NATIVE_TSS_KEY_T DWORD + /* In Windows, native TSS key type is DWORD, + but hardcode the unsigned long to avoid errors for include directive. + */ +# define NATIVE_TSS_KEY_T unsigned long #else /* For the platform that has not supplied native TSS */ # define NATIVE_TSS_KEY_T int #endif @@ -110,19 +119,13 @@ PyAPI_FUNC(void) PyThread_ReInitTLS(void) Py_DEPRECATED(3.7); /* Py_tss_t is opaque type and you *must not* directly read and write. When you'd check whether the key is created, use PyThread_tss_is_created. */ -typedef struct { +typedef struct _py_tss_t { bool _is_initialized; NATIVE_TSS_KEY_T _key; } Py_tss_t; #undef NATIVE_TSS_KEY_T -static inline bool -PyThread_tss_is_created(Py_tss_t key) -{ - return key._is_initialized; -} - /* Py_tss_NEEDS_INIT is the defined invalid value, and you *must* initialize the Py_tss_t variable by this value to use TSS API. @@ -131,15 +134,23 @@ PyThread_tss_is_created(Py_tss_t key) int fail = PyThread_tss_create(&thekey); */ #define Py_tss_NEEDS_INIT {._is_initialized = false} +#endif PyAPI_FUNC(int) PyThread_tss_create(Py_tss_t *key); PyAPI_FUNC(void) PyThread_tss_delete(Py_tss_t *key); -PyAPI_FUNC(int) PyThread_tss_set(Py_tss_t key, void *value); -PyAPI_FUNC(void *) PyThread_tss_get(Py_tss_t key); -PyAPI_FUNC(void) PyThread_tss_delete_value(Py_tss_t key); +PyAPI_FUNC(int) PyThread_tss_set(Py_tss_t *key, void *value); +PyAPI_FUNC(void *) PyThread_tss_get(Py_tss_t *key); +PyAPI_FUNC(void) PyThread_tss_delete_value(Py_tss_t *key); +#ifndef Py_LIMITED_API /* Cleanup after a fork */ PyAPI_FUNC(void) PyThread_ReInitTSS(void); +#endif + +PyAPI_FUNC(Py_tss_t *) PyThread_tss_alloc(void); +PyAPI_FUNC(void) PyThread_tss_free(Py_tss_t *key); + +PyAPI_FUNC(bool) PyThread_tss_is_created(Py_tss_t *key); #ifdef __cplusplus } diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 30a6764201f78b..6deba4a327bd03 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4032,7 +4032,7 @@ static PyObject * test_pythread_tss_key_state(PyObject *self, PyObject *args) { Py_tss_t tss_key = Py_tss_NEEDS_INIT; - if (PyThread_tss_is_created(tss_key)) { + if (PyThread_tss_is_created(&tss_key)) { return raiseTestError("test_pythread_tss_key_state", "TSS key not in an uninitialized state at " "creation time"); @@ -4041,7 +4041,7 @@ test_pythread_tss_key_state(PyObject *self, PyObject *args) PyErr_SetString(PyExc_RuntimeError, "PyThread_tss_create failed"); return NULL; } - if (!PyThread_tss_is_created(tss_key)) { + if (!PyThread_tss_is_created(&tss_key)) { return raiseTestError("test_pythread_tss_key_state", "PyThread_tss_create succeeded, " "but with TSS key in an uninitialized state"); @@ -4053,20 +4053,33 @@ test_pythread_tss_key_state(PyObject *self, PyObject *args) } #define CHECK_TSS_API(expr) \ (void)(expr); \ - if (!PyThread_tss_is_created(tss_key)) { \ + if (!PyThread_tss_is_created(&tss_key)) { \ return raiseTestError("test_pythread_tss_key_state", \ "TSS key initialization state was not " \ "preserved after calling " #expr); } - CHECK_TSS_API(PyThread_tss_set(tss_key, NULL)); - CHECK_TSS_API(PyThread_tss_get(tss_key)); - CHECK_TSS_API(PyThread_tss_delete_value(tss_key)); + CHECK_TSS_API(PyThread_tss_set(&tss_key, NULL)); + CHECK_TSS_API(PyThread_tss_get(&tss_key)); + CHECK_TSS_API(PyThread_tss_delete_value(&tss_key)); #undef CHECK_TSS_API PyThread_tss_delete(&tss_key); - if (PyThread_tss_is_created(tss_key)) { + if (PyThread_tss_is_created(&tss_key)) { return raiseTestError("test_pythread_tss_key_state", "PyThread_tss_delete called, but did not " "set the key state to uninitialized"); } + + Py_tss_t *ptr_key = PyThread_tss_alloc(); + if (ptr_key == NULL) { + PyErr_SetString(PyExc_RuntimeError, "PyThread_tss_alloc failed"); + return NULL; + } + if (PyThread_tss_is_created(ptr_key)) { + return raiseTestError("test_pythread_tss_key_state", + "TSS key not in an uninitialized state at " + "allocation time"); + } + PyThread_tss_free(ptr_key); + ptr_key = NULL; Py_RETURN_NONE; } #endif diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 58bdd9a24ef145..41ee491003a718 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -181,8 +181,8 @@ get_reentrant(void) { void *ptr; - assert(PyThread_tss_is_created(tracemalloc_reentrant_key)); - ptr = PyThread_tss_get(tracemalloc_reentrant_key); + assert(PyThread_tss_is_created(&tracemalloc_reentrant_key)); + ptr = PyThread_tss_get(&tracemalloc_reentrant_key); if (ptr != NULL) { assert(ptr == REENTRANT); return 1; @@ -195,15 +195,15 @@ static void set_reentrant(int reentrant) { assert(reentrant == 0 || reentrant == 1); - assert(PyThread_tss_is_created(tracemalloc_reentrant_key)); + assert(PyThread_tss_is_created(&tracemalloc_reentrant_key)); if (reentrant) { assert(!get_reentrant()); - PyThread_tss_set(tracemalloc_reentrant_key, REENTRANT); + PyThread_tss_set(&tracemalloc_reentrant_key, REENTRANT); } else { assert(get_reentrant()); - PyThread_tss_set(tracemalloc_reentrant_key, NULL); + PyThread_tss_set(&tracemalloc_reentrant_key, NULL); } } diff --git a/Python/pystate.c b/Python/pystate.c index 73b2d8a50c6dad..468b3d7efbe299 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -501,8 +501,8 @@ PyThreadState_Delete(PyThreadState *tstate) if (tstate == GET_TSTATE()) Py_FatalError("PyThreadState_Delete: tstate is still current"); #ifdef WITH_THREAD - if (autoInterpreterState && PyThread_tss_get(autoTSSkey) == tstate) - PyThread_tss_delete_value(autoTSSkey); + if (autoInterpreterState && PyThread_tss_get(&autoTSSkey) == tstate) + PyThread_tss_delete_value(&autoTSSkey); #endif /* WITH_THREAD */ tstate_delete_common(tstate); } @@ -517,8 +517,8 @@ PyThreadState_DeleteCurrent() Py_FatalError( "PyThreadState_DeleteCurrent: no current tstate"); tstate_delete_common(tstate); - if (autoInterpreterState && PyThread_tss_get(autoTSSkey) == tstate) - PyThread_tss_delete_value(autoTSSkey); + if (autoInterpreterState && PyThread_tss_get(&autoTSSkey) == tstate) + PyThread_tss_delete_value(&autoTSSkey); SET_TSTATE(NULL); PyEval_ReleaseLock(); } @@ -779,7 +779,7 @@ _PyGILState_Init(PyInterpreterState *i, PyThreadState *t) if (PyThread_tss_create(&autoTSSkey) != 0) Py_FatalError("Could not allocate TSS entry"); autoInterpreterState = i; - assert(PyThread_tss_get(autoTSSkey) == NULL); + assert(PyThread_tss_get(&autoTSSkey) == NULL); assert(t->gilstate_counter == 0); _PyGILState_NoteThreadState(t); @@ -816,7 +816,7 @@ _PyGILState_Reinit(void) /* If the thread had an associated auto thread state, reassociate it with * the new key. */ - if (tstate && PyThread_tss_set(autoTSSkey, (void *)tstate) != 0) + if (tstate && PyThread_tss_set(&autoTSSkey, (void *)tstate) != 0) Py_FatalError("Couldn't create autoTSSkey mapping"); } @@ -846,8 +846,8 @@ _PyGILState_NoteThreadState(PyThreadState* tstate) The first thread state created for that given OS level thread will "win", which seems reasonable behaviour. */ - if (PyThread_tss_get(autoTSSkey) == NULL) { - if (PyThread_tss_set(autoTSSkey, (void *)tstate) != 0) + if (PyThread_tss_get(&autoTSSkey) == NULL) { + if (PyThread_tss_set(&autoTSSkey, (void *)tstate) != 0) Py_FatalError("Couldn't create autoTSSkey mapping"); } @@ -861,7 +861,7 @@ PyGILState_GetThisThreadState(void) { if (autoInterpreterState == NULL) return NULL; - return (PyThreadState *)PyThread_tss_get(autoTSSkey); + return (PyThreadState *)PyThread_tss_get(&autoTSSkey); } int @@ -872,7 +872,7 @@ PyGILState_Check(void) if (!_PyGILState_check_enabled) return 1; - if (!PyThread_tss_is_created(autoTSSkey)) + if (!PyThread_tss_is_created(&autoTSSkey)) return 1; tstate = GET_TSTATE(); @@ -893,7 +893,7 @@ PyGILState_Ensure(void) called Py_Initialize() and usually PyEval_InitThreads(). */ assert(autoInterpreterState); /* Py_Initialize() hasn't been called! */ - tcur = (PyThreadState *)PyThread_tss_get(autoTSSkey); + tcur = (PyThreadState *)PyThread_tss_get(&autoTSSkey); if (tcur == NULL) { /* At startup, Python has no concrete GIL. If PyGILState_Ensure() is called from a new thread for the first time, we need the create the @@ -925,7 +925,7 @@ PyGILState_Ensure(void) void PyGILState_Release(PyGILState_STATE oldstate) { - PyThreadState *tcur = (PyThreadState *)PyThread_tss_get(autoTSSkey); + PyThreadState *tcur = (PyThreadState *)PyThread_tss_get(&autoTSSkey); if (tcur == NULL) Py_FatalError("auto-releasing thread-state, " "but no thread-state for this thread"); diff --git a/Python/thread.c b/Python/thread.c index 85bd5099bd6906..6496450f408082 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -115,6 +115,7 @@ PyThread_set_stacksize(size_t size) #endif } + #ifndef Py_HAVE_NATIVE_TLS /* If the platform has not supplied a platform specific TLS implementation, provide our own. @@ -128,16 +129,16 @@ Per-thread data ("key") support. Use PyThread_tss_create(&thekey) to create a new key. This is typically shared across threads. -Use PyThread_tss_set(thekey, value) to associate void* value with +Use PyThread_tss_set(&thekey, value) to associate void* value with thekey in the current thread. Each thread has a distinct mapping of thekey to a void* value. Caution: if the current thread already has a mapping for thekey, value is ignored. -Use PyThread_tss_get(thekey) to retrieve the void* value associated +Use PyThread_tss_get(&thekey) to retrieve the void* value associated with thekey in the current thread. This returns NULL if no value is associated with thekey in the current thread. -Use PyThread_tss_delete_value(thekey) to forget the current thread's associated +Use PyThread_tss_delete_value(&thekey) to forget the current thread's associated value for thekey. PyThread_tss_delete(&thekey) forgets the values associated with thekey across *all* threads. @@ -271,21 +272,21 @@ int PyThread_set_key_value(int key, void *value) { Py_tss_t proxy = {._is_initialized = true, ._key = key}; - return PyThread_tss_set(proxy, value); + return PyThread_tss_set(&proxy, value); } void * PyThread_get_key_value(int key) { Py_tss_t proxy = {._is_initialized = true, ._key = key}; - return PyThread_tss_get(proxy); + return PyThread_tss_get(&proxy); } void PyThread_delete_key_value(int key) { Py_tss_t proxy = {._is_initialized = true, ._key = key}; - PyThread_tss_delete_value(proxy); + PyThread_tss_delete_value(&proxy); } @@ -296,7 +297,10 @@ PyThread_ReInitTLS(void) } -/* Thread Specific Storage (TSS) API */ +/* Thread Specific Storage (TSS) API + + Implementation part of platform-specific +*/ /* Assign a new key. This must be called before any other functions in * this family, and callers must arrange to serialize calls to this @@ -348,9 +352,9 @@ PyThread_tss_delete(Py_tss_t *key) } int -PyThread_tss_set(Py_tss_t key, void *value) +PyThread_tss_set(Py_tss_t *key, void *value) { - struct key *p = find_key(1, key._key, value); + struct key *p = find_key(1, key->_key, value); if (p == NULL) return -1; @@ -362,9 +366,9 @@ PyThread_tss_set(Py_tss_t key, void *value) * if the current thread doesn't have an association for key. */ void * -PyThread_tss_get(Py_tss_t key) +PyThread_tss_get(Py_tss_t *key) { - struct key *p = find_key(0, key._key, NULL); + struct key *p = find_key(0, key->_key, NULL); if (p == NULL) return NULL; @@ -374,7 +378,7 @@ PyThread_tss_get(Py_tss_t key) /* Forget the current thread's association for key, if any. */ void -PyThread_tss_delete_value(Py_tss_t key) +PyThread_tss_delete_value(Py_tss_t *key) { unsigned long id = PyThread_get_thread_ident(); struct key *p, **q; @@ -382,7 +386,7 @@ PyThread_tss_delete_value(Py_tss_t key) PyThread_acquire_lock(keymutex, 1); q = &keyhead; while ((p = *q) != NULL) { - if (p->key == key._key && p->id == id) { + if (p->key == key->_key && p->id == id) { *q = p->next; PyMem_RawFree((void *)p); /* NB This does *not* free p->value! */ @@ -428,6 +432,40 @@ PyThread_ReInitTSS(void) #endif /* Py_HAVE_NATIVE_TLS */ + +/* Thread Specific Storage (TSS) API + + Implementation part of common +*/ + +Py_tss_t * +PyThread_tss_alloc(void) +{ + Py_tss_t *new_key = (Py_tss_t *)PyMem_RawMalloc(sizeof(Py_tss_t)); + if (new_key == NULL) { + return NULL; + } + new_key->_is_initialized = false; + return new_key; +} + +void +PyThread_tss_free(Py_tss_t *key) +{ + if (key != NULL) { + PyThread_tss_delete(key); + } + PyMem_RawFree((void *)key); +} + +bool +PyThread_tss_is_created(Py_tss_t *key) +{ + assert(key != NULL); + return key->_is_initialized; +} + + PyDoc_STRVAR(threadinfo__doc__, "sys.thread_info\n\ \n\ diff --git a/Python/thread_nt.h b/Python/thread_nt.h index 8fdbc12631a7de..979cdf65989313 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -359,7 +359,7 @@ _pythread_nt_set_stacksize(size_t size) int PyThread_create_key(void) { - DWORD result= TlsAlloc(); + DWORD result = TlsAlloc(); if (result == TLS_OUT_OF_INDEXES) return -1; return (int)result; @@ -412,7 +412,10 @@ PyThread_ReInitTLS(void) } -/* Thread Specific Storage (TSS) API */ +/* Thread Specific Storage (TSS) API + + Implementation part of platform-specific +*/ int PyThread_tss_create(Py_tss_t *key) @@ -445,14 +448,14 @@ PyThread_tss_delete(Py_tss_t *key) } int -PyThread_tss_set(Py_tss_t key, void *value) +PyThread_tss_set(Py_tss_t *key, void *value) { - BOOL ok = TlsSetValue(key._key, value); + BOOL ok = TlsSetValue(key->_key, value); return ok ? 0 : -1; } void * -PyThread_tss_get(Py_tss_t key) +PyThread_tss_get(Py_tss_t *key) { /* because TSS is used in the Py_END_ALLOW_THREAD macro, * it is necessary to preserve the windows error state, because @@ -461,18 +464,18 @@ PyThread_tss_get(Py_tss_t key) * do it here. */ DWORD error = GetLastError(); - void *result = TlsGetValue(key._key); + void *result = TlsGetValue(key->_key); SetLastError(error); return result; } void -PyThread_tss_delete_value(Py_tss_t key) +PyThread_tss_delete_value(Py_tss_t *key) { /* NULL is used as "key missing", and it is also the default * given by TlsGetValue() if nothing has been set yet. */ - TlsSetValue(key._key, NULL); + TlsSetValue(key->_key, NULL); } diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index d4e2d284e9c767..2cb51a55c3e7d6 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -679,7 +679,10 @@ PyThread_ReInitTLS(void) } -/* Thread Specific Storage (TSS) API */ +/* Thread Specific Storage (TSS) API + + Implementation part of platform-specific +*/ int PyThread_tss_create(Py_tss_t *key) @@ -710,22 +713,22 @@ PyThread_tss_delete(Py_tss_t *key) } int -PyThread_tss_set(Py_tss_t key, void *value) +PyThread_tss_set(Py_tss_t *key, void *value) { - int fail = pthread_setspecific(key._key, value); + int fail = pthread_setspecific(key->_key, value); return fail ? -1 : 0; } void * -PyThread_tss_get(Py_tss_t key) +PyThread_tss_get(Py_tss_t *key) { - return pthread_getspecific(key._key); + return pthread_getspecific(key->_key); } void -PyThread_tss_delete_value(Py_tss_t key) +PyThread_tss_delete_value(Py_tss_t *key) { - pthread_setspecific(key._key, NULL); + pthread_setspecific(key->_key, NULL); } From 7532acb58b8727b84c90c62109e94b3bacce92c4 Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Tue, 13 Jun 2017 11:40:34 +0900 Subject: [PATCH 07/41] bpo-25658: Clarify comments for Py_tss_t type explanations https://github.com/python/cpython/pull/1362#discussion_r114078743 --- Include/pythread.h | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/Include/pythread.h b/Include/pythread.h index 9ee3e871d57585..9267359cc9b2c6 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -81,7 +81,11 @@ PyAPI_FUNC(PyObject*) PyThread_GetInfo(void); /* Thread Local Storage (TLS) API - TLS API is DEPRECATED. Use Thread Specific Storage API. + TLS API is DEPRECATED. Use Thread Specific Storage (TSS) API. + + Since existing TLS API has assumed the thread key type to int, but it is + not compatible with POSIX (see PEP 539). Therefore, new TSS API uses + opaque data type for the thread key to ensure cross-platform. */ PyAPI_FUNC(int) PyThread_create_key(void) Py_DEPRECATED(3.7); PyAPI_FUNC(void) PyThread_delete_key(int key) Py_DEPRECATED(3.7); @@ -93,11 +97,10 @@ PyAPI_FUNC(void) PyThread_delete_key_value(int key) Py_DEPRECATED(3.7); PyAPI_FUNC(void) PyThread_ReInitTLS(void) Py_DEPRECATED(3.7); -/* Thread Specific Storage (TSS) API +/* Thread Specific Storage (TSS) API */ - POSIX hasn't defined that pthread_key_t is compatible with int - (for details, see PEP 539). Therefore, TSS API uses opaque type to cover - the key details. +/* Py_tss_t is an opaque data type the definition of which depends on the + underlying TSS implementation. */ #ifdef Py_LIMITED_API typedef struct _py_tss_t Py_tss_t; @@ -112,12 +115,14 @@ typedef struct _py_tss_t Py_tss_t; but hardcode the unsigned long to avoid errors for include directive. */ # define NATIVE_TSS_KEY_T unsigned long -#else /* For the platform that has not supplied native TSS */ +#else + /* For the platform that has not supplied native TSS */ # define NATIVE_TSS_KEY_T int #endif -/* Py_tss_t is opaque type and you *must not* directly read and write. - When you'd check whether the key is created, use PyThread_tss_is_created. +/* When Py_LIMITED_API is not defined, the type layout of Py_tss_t is in + public to allow for static declarations in API clients. Even in this case, + you must handle TSS key through API functions due to compatibility. */ typedef struct _py_tss_t { bool _is_initialized; @@ -126,13 +131,7 @@ typedef struct _py_tss_t { #undef NATIVE_TSS_KEY_T -/* Py_tss_NEEDS_INIT is the defined invalid value, and you *must* initialize - the Py_tss_t variable by this value to use TSS API. - - For example: - static Py_tss_t thekey = Py_tss_NEEDS_INIT; - int fail = PyThread_tss_create(&thekey); -*/ +/* In static declaration, you must initialize with Py_tss_NEEDS_INIT. */ #define Py_tss_NEEDS_INIT {._is_initialized = false} #endif @@ -147,9 +146,14 @@ PyAPI_FUNC(void) PyThread_tss_delete_value(Py_tss_t *key); PyAPI_FUNC(void) PyThread_ReInitTSS(void); #endif +/* In the limited API, Py_tss_t value must be allocated through a pointer by + PyThread_tss_alloc, and free by PyThread_tss_free at the life cycle end of + the CPython interpreter. +*/ PyAPI_FUNC(Py_tss_t *) PyThread_tss_alloc(void); PyAPI_FUNC(void) PyThread_tss_free(Py_tss_t *key); +/* When you'd check whether the key is created, use PyThread_tss_is_created. */ PyAPI_FUNC(bool) PyThread_tss_is_created(Py_tss_t *key); #ifdef __cplusplus From d25f441e6ac8eef69e93881c03044139213abf6b Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Tue, 13 Jun 2017 05:09:18 +0900 Subject: [PATCH 08/41] bpo-25658: Add an example function to xxlimited module Show a concrete example using TSS API under the limited API. --- Modules/xxlimited.c | 70 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 5041ac8284a3c9..d3139810fc3123 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -15,6 +15,9 @@ /* Xxo objects */ #include "Python.h" +#ifdef WITH_THREAD +#include "pythread.h" +#endif static PyObject *ErrorObject; @@ -215,6 +218,69 @@ static PyType_Spec Null_Type_spec = { /* ---------- */ +#ifdef WITH_THREAD +PyDoc_STRVAR(xx_tss_doc, +"tss(a) -> a\n\ +\n\ +TSS API test in the limited API."); + +/* In the limited API, Py_tss_t value cannot be statically initialized. +static Py_tss_t tss_key = Py_tss_NEEDS_INIT; +*/ +static Py_tss_t *tss_key; + +static PyObject * +xx_tss(PyObject *self, PyObject *args) +{ + PyObject *a, *b; + if (!PyArg_ParseTuple(args, "O:tss", &a)) { + return NULL; + } + + tss_key = PyThread_tss_alloc(); + if (tss_key == NULL) { + PyErr_SetString(PyExc_RuntimeError, "PyThread_tss_alloc failed"); + return NULL; + } + if (PyThread_tss_create(tss_key) != 0) { + PyThread_tss_free(tss_key); + PyErr_SetString(PyExc_RuntimeError, "PyThread_tss_create failed"); + return NULL; + } + if (!PyThread_tss_is_created(tss_key)) { + PyThread_tss_free(tss_key); + PyErr_SetString(PyExc_RuntimeError, + "TSS key in an uninitialized state"); + return NULL; + } + if (PyThread_tss_set(tss_key, (void *)a) != 0) { + PyThread_tss_free(tss_key); + PyErr_SetString(PyExc_RuntimeError, "PyThread_tss_set failed"); + return NULL; + } + b = (PyObject *)PyThread_tss_get(tss_key); + if (a != b) { + PyThread_tss_free(tss_key); + PyErr_SetString(PyExc_RuntimeError, "PyThread_tss_get failed"); + return NULL; + } + PyThread_tss_delete_value(tss_key); + if (PyThread_tss_get(tss_key) != NULL) { + PyThread_tss_free(tss_key); + PyErr_SetString(PyExc_RuntimeError, "TSS value is not deleted"); + return NULL; + } + PyThread_tss_delete(tss_key); + PyThread_tss_free(tss_key); + tss_key = NULL; + + Py_INCREF(b); + return b; +} +#endif /* WITH_THREAD */ + +/* ---------- */ + /* List of functions defined in the module */ static PyMethodDef xx_methods[] = { @@ -224,6 +290,10 @@ static PyMethodDef xx_methods[] = { xx_foo_doc}, {"new", xx_new, METH_VARARGS, PyDoc_STR("new() -> new Xx object")}, +#ifdef WITH_THREAD + {"tss", xx_tss, METH_VARARGS, + xx_tss_doc}, +#endif {NULL, NULL} /* sentinel */ }; From 722ea6868672c20dbb520c2bfbef52a6d2d80beb Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Tue, 13 Jun 2017 06:09:08 +0900 Subject: [PATCH 09/41] bpo-25658: Add braces to follow updated PEP 7 format https://github.com/python/peps/issues/279 --- Python/pystate.c | 21 ++++++++++++++------- Python/thread.c | 18 ++++++++++++------ Python/thread_nt.h | 9 ++++++--- Python/thread_pthread.h | 9 ++++++--- 4 files changed, 38 insertions(+), 19 deletions(-) diff --git a/Python/pystate.c b/Python/pystate.c index 468b3d7efbe299..076db6779a58ef 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -501,8 +501,9 @@ PyThreadState_Delete(PyThreadState *tstate) if (tstate == GET_TSTATE()) Py_FatalError("PyThreadState_Delete: tstate is still current"); #ifdef WITH_THREAD - if (autoInterpreterState && PyThread_tss_get(&autoTSSkey) == tstate) + if (autoInterpreterState && PyThread_tss_get(&autoTSSkey) == tstate) { PyThread_tss_delete_value(&autoTSSkey); + } #endif /* WITH_THREAD */ tstate_delete_common(tstate); } @@ -517,8 +518,9 @@ PyThreadState_DeleteCurrent() Py_FatalError( "PyThreadState_DeleteCurrent: no current tstate"); tstate_delete_common(tstate); - if (autoInterpreterState && PyThread_tss_get(&autoTSSkey) == tstate) + if (autoInterpreterState && PyThread_tss_get(&autoTSSkey) == tstate) { PyThread_tss_delete_value(&autoTSSkey); + } SET_TSTATE(NULL); PyEval_ReleaseLock(); } @@ -776,8 +778,9 @@ void _PyGILState_Init(PyInterpreterState *i, PyThreadState *t) { assert(i && t); /* must init with valid states */ - if (PyThread_tss_create(&autoTSSkey) != 0) + if (PyThread_tss_create(&autoTSSkey) != 0) { Py_FatalError("Could not allocate TSS entry"); + } autoInterpreterState = i; assert(PyThread_tss_get(&autoTSSkey) == NULL); assert(t->gilstate_counter == 0); @@ -811,13 +814,15 @@ _PyGILState_Reinit(void) #endif PyThreadState *tstate = PyGILState_GetThisThreadState(); PyThread_tss_delete(&autoTSSkey); - if (PyThread_tss_create(&autoTSSkey) != 0) + if (PyThread_tss_create(&autoTSSkey) != 0) { Py_FatalError("Could not allocate TSS entry"); + } /* If the thread had an associated auto thread state, reassociate it with * the new key. */ - if (tstate && PyThread_tss_set(&autoTSSkey, (void *)tstate) != 0) + if (tstate && PyThread_tss_set(&autoTSSkey, (void *)tstate) != 0) { Py_FatalError("Couldn't create autoTSSkey mapping"); + } } /* When a thread state is created for a thread by some mechanism other than @@ -847,8 +852,9 @@ _PyGILState_NoteThreadState(PyThreadState* tstate) "win", which seems reasonable behaviour. */ if (PyThread_tss_get(&autoTSSkey) == NULL) { - if (PyThread_tss_set(&autoTSSkey, (void *)tstate) != 0) + if (PyThread_tss_set(&autoTSSkey, (void *)tstate) != 0) { Py_FatalError("Couldn't create autoTSSkey mapping"); + } } /* PyGILState_Release must not try to delete this thread state. */ @@ -872,8 +878,9 @@ PyGILState_Check(void) if (!_PyGILState_check_enabled) return 1; - if (!PyThread_tss_is_created(&autoTSSkey)) + if (!PyThread_tss_is_created(&autoTSSkey)) { return 1; + } tstate = GET_TSTATE(); if (tstate == NULL) diff --git a/Python/thread.c b/Python/thread.c index 6496450f408082..4686b7f588aa55 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -220,11 +220,13 @@ find_key(int set_value, int key, void *value) * they do we must abort. Otherwise we'll end up spinning * in a tight loop with the lock held. A similar check is done * in pystate.c tstate_delete_common(). */ - if (p == prev_p) + if (p == prev_p) { Py_FatalError("tss find_key: small circular list(!)"); + } prev_p = p; - if (p->next == keyhead) + if (p->next == keyhead) { Py_FatalError("tss find_key: circular list(!)"); + } } if (!set_value && value == NULL) { assert(p == NULL); @@ -255,8 +257,9 @@ int PyThread_create_key(void) { Py_tss_t proxy = Py_tss_NEEDS_INIT; - if (PyThread_tss_create(&proxy) != 0) + if (PyThread_tss_create(&proxy) != 0) { return -1; + } /* In the own implementation, platform-specific key type is int. */ return proxy._key; } @@ -314,11 +317,13 @@ PyThread_tss_create(Py_tss_t *key) */ assert(key != NULL); /* If the key has been created, function is silently skipped. */ - if (key->_is_initialized) + if (key->_is_initialized) { return 0; + } - if (keymutex == NULL) + if (keymutex == NULL) { keymutex = PyThread_allocate_lock(); + } key->_key = ++nkeys; key->_is_initialized = true; return 0; @@ -330,8 +335,9 @@ PyThread_tss_delete(Py_tss_t *key) { assert(key != NULL); /* If the key has not been created, function is silently skipped. */ - if (!key->_is_initialized) + if (!key->_is_initialized) { return; + } struct key *p, **q; diff --git a/Python/thread_nt.h b/Python/thread_nt.h index 979cdf65989313..92a5c90f035214 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -422,12 +422,14 @@ PyThread_tss_create(Py_tss_t *key) { assert(key != NULL); /* If the key has been created, function is silently skipped. */ - if (key->_is_initialized) + if (key->_is_initialized) { return 0; + } DWORD result = TlsAlloc(); - if (result == TLS_OUT_OF_INDEXES) + if (result == TLS_OUT_OF_INDEXES) { return -1; + } /* In Windows, platform-specific key type is DWORD. */ key->_key = result; key->_is_initialized = true; @@ -439,8 +441,9 @@ PyThread_tss_delete(Py_tss_t *key) { assert(key != NULL); /* If the key has not been created, function is silently skipped. */ - if (!key->_is_initialized) + if (!key->_is_initialized) { return; + } TlsFree(key->_key); key->_key = TLS_OUT_OF_INDEXES; diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 2cb51a55c3e7d6..f5e089a30ccdfd 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -689,12 +689,14 @@ PyThread_tss_create(Py_tss_t *key) { assert(key != NULL); /* If the key has been created, function is silently skipped. */ - if (key->_is_initialized) + if (key->_is_initialized) { return 0; + } int fail = pthread_key_create(&(key->_key), NULL); - if (fail) + if (fail) { return -1; + } key->_is_initialized = true; return 0; } @@ -704,8 +706,9 @@ PyThread_tss_delete(Py_tss_t *key) { assert(key != NULL); /* If the key has not been created, function is silently skipped. */ - if (!key->_is_initialized) + if (!key->_is_initialized) { return; + } pthread_key_delete(key->_key); /* pthread has not provided the defined invalid value for the key. */ From 5204c6b1c6dc9445a9fb7f95597640928571d5c2 Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Tue, 6 Jun 2017 20:57:13 +0900 Subject: [PATCH 10/41] bpo-25658: Add NEWS and What's New entries --- Doc/whatsnew/3.7.rst | 7 +++++++ Misc/NEWS | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 602db8c0ac8e68..b53bc791cd0d94 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -238,6 +238,13 @@ Optimizations Build and C API Changes ======================= +* Added Thread Specific Storage (TSS) API which would supersede use of the + existing Thread Local Storage (TLS) API within the CPython interpreter, while + deprecating the existing API. The change is motivated primarily by the fact + that the old API uses int to represent TLS keys across all platforms, which + is neither POSIX-compliant, nor portable in any practical sense. For more + information, see :pep:`539` and :issue:`25658`. + * A full copy of libffi is no longer bundled for use when building the :mod:`_ctypes ` module on non-OSX UNIX platforms. An installed copy of libffi is now required when building ``_ctypes`` on such platforms. diff --git a/Misc/NEWS b/Misc/NEWS index 7d31839e6734d8..8efaef93a5ac4c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -1081,6 +1081,11 @@ Windows C API ----- +- bpo-25658: Implement PEP 539 for Thread Specific Strage (TSS) API: it is a + new Thread Local Storage (TLS) API to CPython which would supersede use of + the existing TLS API within the CPython interpreter, while deprecating the + existing API. + - bpo-16500: Deprecate PyOS_AfterFork() and add PyOS_BeforeFork(), PyOS_AfterFork_Parent() and PyOS_AfterFork_Child(). From 594ec29a07794f7f1748c5923362ad1f77e14dc5 Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Mon, 8 May 2017 17:15:45 +0900 Subject: [PATCH 11/41] bpo-25658: Add TSS API to Windows stable ABI https://ci.appveyor.com/project/python/cpython/build/3.7.0a0.1850 --- PC/python3.def | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/PC/python3.def b/PC/python3.def index ad65294045f677..9d938aa8ef41de 100644 --- a/PC/python3.def +++ b/PC/python3.def @@ -570,6 +570,14 @@ EXPORTS PyThreadState_New=python37.PyThreadState_New PyThreadState_SetAsyncExc=python37.PyThreadState_SetAsyncExc PyThreadState_Swap=python37.PyThreadState_Swap + PyThread_tss_alloc=python37.PyThread_tss_alloc + PyThread_tss_create=python37.PyThread_tss_create + PyThread_tss_delete=python37.PyThread_tss_delete + PyThread_tss_delete_value=python37.PyThread_tss_delete_value + PyThread_tss_free=python37.PyThread_tss_free + PyThread_tss_get=python37.PyThread_tss_get + PyThread_tss_is_created=python37.PyThread_tss_is_created + PyThread_tss_set=python37.PyThread_tss_set PyTraceBack_Here=python37.PyTraceBack_Here PyTraceBack_Print=python37.PyTraceBack_Print PyTraceBack_Type=python37.PyTraceBack_Type DATA From ab5f7ea1463327cac2c2aa475d15b3e2fdecbac6 Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Sat, 24 Jun 2017 14:37:47 +0900 Subject: [PATCH 12/41] bpo-25658: Add news using blurb New workflow was announced on Python-Dev. --- Misc/NEWS | 5 ----- .../next/C API/2017-06-24-14-30-44.bpo-25658.vm8vGE.rst | 3 +++ 2 files changed, 3 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2017-06-24-14-30-44.bpo-25658.vm8vGE.rst diff --git a/Misc/NEWS b/Misc/NEWS index f49553e251269e..f33589d0085a5c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -1242,11 +1242,6 @@ Windows C API ----- -- bpo-25658: Implement PEP 539 for Thread Specific Strage (TSS) API: it is a - new Thread Local Storage (TLS) API to CPython which would supersede use of - the existing TLS API within the CPython interpreter, while deprecating the - existing API. - - bpo-16500: Deprecate PyOS_AfterFork() and add PyOS_BeforeFork(), PyOS_AfterFork_Parent() and PyOS_AfterFork_Child(). diff --git a/Misc/NEWS.d/next/C API/2017-06-24-14-30-44.bpo-25658.vm8vGE.rst b/Misc/NEWS.d/next/C API/2017-06-24-14-30-44.bpo-25658.vm8vGE.rst new file mode 100644 index 00000000000000..99a682f1227aec --- /dev/null +++ b/Misc/NEWS.d/next/C API/2017-06-24-14-30-44.bpo-25658.vm8vGE.rst @@ -0,0 +1,3 @@ +Implement PEP 539 for Thread Specific Stroage (TSS) API: it is a new Thread +Local Storage (TLS) API to CPython which would supersede use of the existing +TLS API within the CPython interpreter, while deprecating the existing API. From b1be3e0a0c35b8ce905bc56c84a6ff342b0a7f6f Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Wed, 12 Jul 2017 04:09:26 +0900 Subject: [PATCH 13/41] bpo-25658: Set error directive on pythread.h to require native thread --- Include/pythread.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Include/pythread.h b/Include/pythread.h index 9267359cc9b2c6..3b29269e6c90b0 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -2,6 +2,10 @@ #ifndef Py_PYTHREAD_H #define Py_PYTHREAD_H +#if !(defined(_POSIX_THREADS) || defined(NT_THREADS)) +# error "Require native thread feature. See https://bugs.python.org/issue30832" +#endif + #include /* necessary for TSS key */ typedef void *PyThread_type_lock; @@ -115,9 +119,6 @@ typedef struct _py_tss_t Py_tss_t; but hardcode the unsigned long to avoid errors for include directive. */ # define NATIVE_TSS_KEY_T unsigned long -#else - /* For the platform that has not supplied native TSS */ -# define NATIVE_TSS_KEY_T int #endif /* When Py_LIMITED_API is not defined, the type layout of Py_tss_t is in From 44519c38d0036cc31294eb8663aa44fddf25d3fd Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Tue, 4 Jul 2017 13:21:44 +0900 Subject: [PATCH 14/41] bpo-25658: Omit PyThread_ReInitTSS function After aa0aa04, the own implementation for TLS is gone and PyThread_ReInitTSS function has made to be no-op for all supported platforms (Windows and pthreads). --- Include/pythread.h | 5 ----- Modules/posixmodule.c | 3 --- Python/thread_nt.h | 9 --------- Python/thread_pthread.h | 6 ------ 4 files changed, 23 deletions(-) diff --git a/Include/pythread.h b/Include/pythread.h index 3b29269e6c90b0..e2d0f63b73d9b3 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -142,11 +142,6 @@ PyAPI_FUNC(int) PyThread_tss_set(Py_tss_t *key, void *value); PyAPI_FUNC(void *) PyThread_tss_get(Py_tss_t *key); PyAPI_FUNC(void) PyThread_tss_delete_value(Py_tss_t *key); -#ifndef Py_LIMITED_API -/* Cleanup after a fork */ -PyAPI_FUNC(void) PyThread_ReInitTSS(void); -#endif - /* In the limited API, Py_tss_t value must be allocated through a pointer by PyThread_tss_alloc, and free by PyThread_tss_free at the life cycle end of the CPython interpreter. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 31e0ecde762769..677fbdc20f613c 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -450,9 +450,6 @@ void PyOS_AfterFork_Child(void) { #ifdef WITH_THREAD - /* PyThread_ReInitTSS() must be called early, to make sure that the TSS API - * can be called safely. */ - PyThread_ReInitTSS(); _PyGILState_Reinit(); PyEval_ReInitThreads(); _PyImport_ReInitLock(); diff --git a/Python/thread_nt.h b/Python/thread_nt.h index f997139fc1c7b9..4019889c7cf08b 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -475,12 +475,3 @@ PyThread_tss_delete_value(Py_tss_t *key) */ TlsSetValue(key->_key, NULL); } - - -/* reinitialization of TSS is not necessary after fork when using - * the native TLS functions. And forking isn't supported on Windows either. - */ -void -PyThread_ReInitTSS(void) -{ -} diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 89e87561b7c553..484a9193a5dfc2 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -737,9 +737,3 @@ PyThread_tss_delete_value(Py_tss_t *key) { pthread_setspecific(key->_key, NULL); } - - -void -PyThread_ReInitTSS(void) -{ -} From ba04cefe3baa3b20dba891b9d6ecfa4a18f2ecaa Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Wed, 12 Jul 2017 05:36:27 +0900 Subject: [PATCH 15/41] bpo-25658: Omit PyThread_tss_delete_value function After aa0aa04, the own implementation for TLS is gone and PyThread_tss_delete_value function just does to empty the storage (i.e. "PyThread_tss_delete_value(key)" has made to be equal to "PyThread_tss_set(key, NULL)"). --- Include/pythread.h | 1 - Modules/_testcapimodule.c | 1 - Modules/xxlimited.c | 6 ------ PC/python3.def | 1 - Python/pystate.c | 4 ++-- Python/thread.c | 2 +- Python/thread_nt.h | 9 --------- Python/thread_pthread.h | 6 ------ 8 files changed, 3 insertions(+), 27 deletions(-) diff --git a/Include/pythread.h b/Include/pythread.h index e2d0f63b73d9b3..19f40d72d24d01 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -140,7 +140,6 @@ PyAPI_FUNC(int) PyThread_tss_create(Py_tss_t *key); PyAPI_FUNC(void) PyThread_tss_delete(Py_tss_t *key); PyAPI_FUNC(int) PyThread_tss_set(Py_tss_t *key, void *value); PyAPI_FUNC(void *) PyThread_tss_get(Py_tss_t *key); -PyAPI_FUNC(void) PyThread_tss_delete_value(Py_tss_t *key); /* In the limited API, Py_tss_t value must be allocated through a pointer by PyThread_tss_alloc, and free by PyThread_tss_free at the life cycle end of diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 5838fdadb1fe74..3911f8fc38213e 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4183,7 +4183,6 @@ test_pythread_tss_key_state(PyObject *self, PyObject *args) "preserved after calling " #expr); } CHECK_TSS_API(PyThread_tss_set(&tss_key, NULL)); CHECK_TSS_API(PyThread_tss_get(&tss_key)); - CHECK_TSS_API(PyThread_tss_delete_value(&tss_key)); #undef CHECK_TSS_API PyThread_tss_delete(&tss_key); if (PyThread_tss_is_created(&tss_key)) { diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index d3139810fc3123..87d328ff952978 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -264,12 +264,6 @@ xx_tss(PyObject *self, PyObject *args) PyErr_SetString(PyExc_RuntimeError, "PyThread_tss_get failed"); return NULL; } - PyThread_tss_delete_value(tss_key); - if (PyThread_tss_get(tss_key) != NULL) { - PyThread_tss_free(tss_key); - PyErr_SetString(PyExc_RuntimeError, "TSS value is not deleted"); - return NULL; - } PyThread_tss_delete(tss_key); PyThread_tss_free(tss_key); tss_key = NULL; diff --git a/PC/python3.def b/PC/python3.def index 9d938aa8ef41de..1d089ec28b1b02 100644 --- a/PC/python3.def +++ b/PC/python3.def @@ -573,7 +573,6 @@ EXPORTS PyThread_tss_alloc=python37.PyThread_tss_alloc PyThread_tss_create=python37.PyThread_tss_create PyThread_tss_delete=python37.PyThread_tss_delete - PyThread_tss_delete_value=python37.PyThread_tss_delete_value PyThread_tss_free=python37.PyThread_tss_free PyThread_tss_get=python37.PyThread_tss_get PyThread_tss_is_created=python37.PyThread_tss_is_created diff --git a/Python/pystate.c b/Python/pystate.c index f8a1c21b161b1e..ea70c40cb04db6 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -502,7 +502,7 @@ PyThreadState_Delete(PyThreadState *tstate) Py_FatalError("PyThreadState_Delete: tstate is still current"); #ifdef WITH_THREAD if (autoInterpreterState && PyThread_tss_get(&autoTSSkey) == tstate) { - PyThread_tss_delete_value(&autoTSSkey); + PyThread_tss_set(&autoTSSkey, NULL); } #endif /* WITH_THREAD */ tstate_delete_common(tstate); @@ -519,7 +519,7 @@ PyThreadState_DeleteCurrent() "PyThreadState_DeleteCurrent: no current tstate"); tstate_delete_common(tstate); if (autoInterpreterState && PyThread_tss_get(&autoTSSkey) == tstate) { - PyThread_tss_delete_value(&autoTSSkey); + PyThread_tss_set(&autoTSSkey, NULL); } SET_TSTATE(NULL); PyEval_ReleaseLock(); diff --git a/Python/thread.c b/Python/thread.c index eb68075a302007..4f11239be18c77 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -131,7 +131,7 @@ Use PyThread_tss_get(&thekey) to retrieve the void* value associated with thekey in the current thread. This returns NULL if no value is associated with thekey in the current thread. -Use PyThread_tss_delete_value(&thekey) to forget the current thread's associated +Use PyThread_tss_set(&thekey, NULL) to forget the current thread's associated value for thekey. PyThread_tss_delete(&thekey) forgets the values associated with thekey across *all* threads. diff --git a/Python/thread_nt.h b/Python/thread_nt.h index 4019889c7cf08b..718a70f11dd606 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -466,12 +466,3 @@ PyThread_tss_get(Py_tss_t *key) SetLastError(error); return result; } - -void -PyThread_tss_delete_value(Py_tss_t *key) -{ - /* NULL is used as "key missing", and it is also the default - * given by TlsGetValue() if nothing has been set yet. - */ - TlsSetValue(key->_key, NULL); -} diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 484a9193a5dfc2..049a1b46fb574a 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -731,9 +731,3 @@ PyThread_tss_get(Py_tss_t *key) { return pthread_getspecific(key->_key); } - -void -PyThread_tss_delete_value(Py_tss_t *key) -{ - pthread_setspecific(key->_key, NULL); -} From 8cb57afd33b58f777dde668d0c0070b98b382640 Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Sun, 30 Jul 2017 16:04:36 +0900 Subject: [PATCH 16/41] bpo-25658: Reorder the test function --- Modules/_testcapimodule.c | 112 +++++++++++++++++++------------------- 1 file changed, 57 insertions(+), 55 deletions(-) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 3911f8fc38213e..427e34aefd01af 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4151,61 +4151,6 @@ dict_get_version(PyObject *self, PyObject *args) return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)version); } -#ifdef WITH_THREAD -static PyObject * -test_pythread_tss_key_state(PyObject *self, PyObject *args) -{ - Py_tss_t tss_key = Py_tss_NEEDS_INIT; - if (PyThread_tss_is_created(&tss_key)) { - return raiseTestError("test_pythread_tss_key_state", - "TSS key not in an uninitialized state at " - "creation time"); - } - if (PyThread_tss_create(&tss_key) != 0) { - PyErr_SetString(PyExc_RuntimeError, "PyThread_tss_create failed"); - return NULL; - } - if (!PyThread_tss_is_created(&tss_key)) { - return raiseTestError("test_pythread_tss_key_state", - "PyThread_tss_create succeeded, " - "but with TSS key in an uninitialized state"); - } - if (PyThread_tss_create(&tss_key) != 0) { - return raiseTestError("test_pythread_tss_key_state", - "PyThread_tss_create unsuccessful with " - "an already initialized key"); - } -#define CHECK_TSS_API(expr) \ - (void)(expr); \ - if (!PyThread_tss_is_created(&tss_key)) { \ - return raiseTestError("test_pythread_tss_key_state", \ - "TSS key initialization state was not " \ - "preserved after calling " #expr); } - CHECK_TSS_API(PyThread_tss_set(&tss_key, NULL)); - CHECK_TSS_API(PyThread_tss_get(&tss_key)); -#undef CHECK_TSS_API - PyThread_tss_delete(&tss_key); - if (PyThread_tss_is_created(&tss_key)) { - return raiseTestError("test_pythread_tss_key_state", - "PyThread_tss_delete called, but did not " - "set the key state to uninitialized"); - } - - Py_tss_t *ptr_key = PyThread_tss_alloc(); - if (ptr_key == NULL) { - PyErr_SetString(PyExc_RuntimeError, "PyThread_tss_alloc failed"); - return NULL; - } - if (PyThread_tss_is_created(ptr_key)) { - return raiseTestError("test_pythread_tss_key_state", - "TSS key not in an uninitialized state at " - "allocation time"); - } - PyThread_tss_free(ptr_key); - ptr_key = NULL; - Py_RETURN_NONE; -} -#endif static PyObject * raise_SIGINT_then_send_None(PyObject *self, PyObject *args) @@ -4335,6 +4280,63 @@ stack_pointer(PyObject *self, PyObject *args) } +#ifdef WITH_THREAD +static PyObject * +test_pythread_tss_key_state(PyObject *self, PyObject *args) +{ + Py_tss_t tss_key = Py_tss_NEEDS_INIT; + if (PyThread_tss_is_created(&tss_key)) { + return raiseTestError("test_pythread_tss_key_state", + "TSS key not in an uninitialized state at " + "creation time"); + } + if (PyThread_tss_create(&tss_key) != 0) { + PyErr_SetString(PyExc_RuntimeError, "PyThread_tss_create failed"); + return NULL; + } + if (!PyThread_tss_is_created(&tss_key)) { + return raiseTestError("test_pythread_tss_key_state", + "PyThread_tss_create succeeded, " + "but with TSS key in an uninitialized state"); + } + if (PyThread_tss_create(&tss_key) != 0) { + return raiseTestError("test_pythread_tss_key_state", + "PyThread_tss_create unsuccessful with " + "an already initialized key"); + } +#define CHECK_TSS_API(expr) \ + (void)(expr); \ + if (!PyThread_tss_is_created(&tss_key)) { \ + return raiseTestError("test_pythread_tss_key_state", \ + "TSS key initialization state was not " \ + "preserved after calling " #expr); } + CHECK_TSS_API(PyThread_tss_set(&tss_key, NULL)); + CHECK_TSS_API(PyThread_tss_get(&tss_key)); +#undef CHECK_TSS_API + PyThread_tss_delete(&tss_key); + if (PyThread_tss_is_created(&tss_key)) { + return raiseTestError("test_pythread_tss_key_state", + "PyThread_tss_delete called, but did not " + "set the key state to uninitialized"); + } + + Py_tss_t *ptr_key = PyThread_tss_alloc(); + if (ptr_key == NULL) { + PyErr_SetString(PyExc_RuntimeError, "PyThread_tss_alloc failed"); + return NULL; + } + if (PyThread_tss_is_created(ptr_key)) { + return raiseTestError("test_pythread_tss_key_state", + "TSS key not in an uninitialized state at " + "allocation time"); + } + PyThread_tss_free(ptr_key); + ptr_key = NULL; + Py_RETURN_NONE; +} +#endif + + static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, {"raise_memoryerror", (PyCFunction)raise_memoryerror, METH_NOARGS}, From cd35d5b9da88bfc86ec3d91da8023f290ecff429 Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Sun, 30 Jul 2017 16:04:47 +0900 Subject: [PATCH 17/41] bpo-25658: Update comments --- Include/pythread.h | 17 ++++++++--------- Modules/_tracemalloc.c | 5 ++--- Modules/xxlimited.c | 2 +- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/Include/pythread.h b/Include/pythread.h index 19f40d72d24d01..5f868e0449ac5c 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -87,9 +87,9 @@ PyAPI_FUNC(PyObject*) PyThread_GetInfo(void); /* Thread Local Storage (TLS) API TLS API is DEPRECATED. Use Thread Specific Storage (TSS) API. - Since existing TLS API has assumed the thread key type to int, but it is - not compatible with POSIX (see PEP 539). Therefore, new TSS API uses - opaque data type for the thread key to ensure cross-platform. + The existing TLS API has used int to represent TLS keys across all + platforms, but it is not POSIX-compliant. Therefore, the new TSS API uses + opaque data type to represent TSS keys to be compatible (see PEP 539). */ PyAPI_FUNC(int) PyThread_create_key(void) Py_DEPRECATED(3.7); PyAPI_FUNC(void) PyThread_delete_key(int key) Py_DEPRECATED(3.7); @@ -122,7 +122,7 @@ typedef struct _py_tss_t Py_tss_t; #endif /* When Py_LIMITED_API is not defined, the type layout of Py_tss_t is in - public to allow for static declarations in API clients. Even in this case, + public to allow static allocation in the API clients. Even in this case, you must handle TSS key through API functions due to compatibility. */ typedef struct _py_tss_t { @@ -132,7 +132,7 @@ typedef struct _py_tss_t { #undef NATIVE_TSS_KEY_T -/* In static declaration, you must initialize with Py_tss_NEEDS_INIT. */ +/* When static allocation, you must initialize with Py_tss_NEEDS_INIT. */ #define Py_tss_NEEDS_INIT {._is_initialized = false} #endif @@ -141,14 +141,13 @@ PyAPI_FUNC(void) PyThread_tss_delete(Py_tss_t *key); PyAPI_FUNC(int) PyThread_tss_set(Py_tss_t *key, void *value); PyAPI_FUNC(void *) PyThread_tss_get(Py_tss_t *key); -/* In the limited API, Py_tss_t value must be allocated through a pointer by - PyThread_tss_alloc, and free by PyThread_tss_free at the life cycle end of - the CPython interpreter. +/* In the limited API, Py_tss_t value must be allocated by PyThread_tss_alloc, + and free by PyThread_tss_free at the life cycle end of the CPython + interpreter. */ PyAPI_FUNC(Py_tss_t *) PyThread_tss_alloc(void); PyAPI_FUNC(void) PyThread_tss_free(Py_tss_t *key); -/* When you'd check whether the key is created, use PyThread_tss_is_created. */ PyAPI_FUNC(bool) PyThread_tss_is_created(Py_tss_t *key); #ifdef __cplusplus diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 7746f72c2578fb..04666d5ba9ff8e 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -167,9 +167,8 @@ tracemalloc_error(const char *format, ...) #if defined(WITH_THREAD) && defined(TRACE_RAW_MALLOC) #define REENTRANT_THREADLOCAL -/* If your OS does not provide native thread local storage, you can implement - it manually using a lock. Functions of thread.c cannot be used because - they use PyMem_RawMalloc() which leads to a reentrant call. */ +/* If your OS does not provide native thread local storage, CPython + which is compiled with thread support does not provide tracemalloc. */ #if !(defined(_POSIX_THREADS) || defined(NT_THREADS)) # error "need native thread local storage (TLS)" #endif diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 87d328ff952978..2b82419cd05c27 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -224,7 +224,7 @@ PyDoc_STRVAR(xx_tss_doc, \n\ TSS API test in the limited API."); -/* In the limited API, Py_tss_t value cannot be statically initialized. +/* In the limited API, Py_tss_t value cannot be statically allocated. static Py_tss_t tss_key = Py_tss_NEEDS_INIT; */ static Py_tss_t *tss_key; From a810b799167aec37884fe48dd081885448692f66 Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Sun, 30 Jul 2017 16:05:04 +0900 Subject: [PATCH 18/41] bpo-25658: Enclose Py_tss_t definition with the Py_LIMITED_API macro Make more clearly the switch of show/hide implementation detail by the macro definition --- Include/pythread.h | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Include/pythread.h b/Include/pythread.h index 5f868e0449ac5c..a3c377b1b55f58 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -106,10 +106,9 @@ PyAPI_FUNC(void) PyThread_ReInitTLS(void) Py_DEPRECATED(3.7); /* Py_tss_t is an opaque data type the definition of which depends on the underlying TSS implementation. */ -#ifdef Py_LIMITED_API -typedef struct _py_tss_t Py_tss_t; -#else +typedef struct _Py_tss_t Py_tss_t; +#ifndef Py_LIMITED_API #if defined(_POSIX_THREADS) /* Darwin needs pthread.h to know type name the pthread_key_t. */ # include @@ -125,16 +124,16 @@ typedef struct _py_tss_t Py_tss_t; public to allow static allocation in the API clients. Even in this case, you must handle TSS key through API functions due to compatibility. */ -typedef struct _py_tss_t { +struct _Py_tss_t { bool _is_initialized; NATIVE_TSS_KEY_T _key; -} Py_tss_t; +}; #undef NATIVE_TSS_KEY_T /* When static allocation, you must initialize with Py_tss_NEEDS_INIT. */ #define Py_tss_NEEDS_INIT {._is_initialized = false} -#endif +#endif /* !Py_LIMITED_API */ PyAPI_FUNC(int) PyThread_tss_create(Py_tss_t *key); PyAPI_FUNC(void) PyThread_tss_delete(Py_tss_t *key); From 99379e0f1ec3b52448231e2bc434434a3fe5a524 Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Thu, 31 Aug 2017 16:21:43 +0900 Subject: [PATCH 19/41] bpo-25658: follow the draft specification --- Python/thread.c | 2 +- Python/thread_nt.h | 2 ++ Python/thread_pthread.h | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Python/thread.c b/Python/thread.c index 4f11239be18c77..5e32d4fa88dd8b 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -173,8 +173,8 @@ PyThread_tss_free(Py_tss_t *key) { if (key != NULL) { PyThread_tss_delete(key); + PyMem_RawFree((void *)key); } - PyMem_RawFree((void *)key); } bool diff --git a/Python/thread_nt.h b/Python/thread_nt.h index 718a70f11dd606..a1402aa5a1710b 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -448,6 +448,7 @@ PyThread_tss_delete(Py_tss_t *key) int PyThread_tss_set(Py_tss_t *key, void *value) { + assert(key != NULL); BOOL ok = TlsSetValue(key->_key, value); return ok ? 0 : -1; } @@ -455,6 +456,7 @@ PyThread_tss_set(Py_tss_t *key, void *value) void * PyThread_tss_get(Py_tss_t *key) { + assert(key != NULL); /* because TSS is used in the Py_END_ALLOW_THREAD macro, * it is necessary to preserve the windows error state, because * it is assumed to be preserved across the call to the macro. diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 049a1b46fb574a..8051528c6a76cb 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -722,6 +722,7 @@ PyThread_tss_delete(Py_tss_t *key) int PyThread_tss_set(Py_tss_t *key, void *value) { + assert(key != NULL); int fail = pthread_setspecific(key->_key, value); return fail ? -1 : 0; } @@ -729,5 +730,6 @@ PyThread_tss_set(Py_tss_t *key, void *value) void * PyThread_tss_get(Py_tss_t *key) { + assert(key != NULL); return pthread_getspecific(key->_key); } From 739ed55f73cb21e077eae3701e759a061514cea4 Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Sat, 2 Sep 2017 07:15:18 +0900 Subject: [PATCH 20/41] bpo-25658: change the declaration of PyThread_tss_is_created Use an int for the initialized flag and avoid the extra dependency. https://github.com/python/cpython/pull/1362#pullrequestreview-59884901 --- Include/pythread.h | 8 +++----- Python/thread.c | 2 +- Python/thread_nt.h | 5 ++--- Python/thread_pthread.h | 5 ++--- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/Include/pythread.h b/Include/pythread.h index a3c377b1b55f58..5552236cb2f47c 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -6,8 +6,6 @@ # error "Require native thread feature. See https://bugs.python.org/issue30832" #endif -#include /* necessary for TSS key */ - typedef void *PyThread_type_lock; typedef void *PyThread_type_sema; @@ -125,14 +123,14 @@ typedef struct _Py_tss_t Py_tss_t; you must handle TSS key through API functions due to compatibility. */ struct _Py_tss_t { - bool _is_initialized; + int _is_initialized; NATIVE_TSS_KEY_T _key; }; #undef NATIVE_TSS_KEY_T /* When static allocation, you must initialize with Py_tss_NEEDS_INIT. */ -#define Py_tss_NEEDS_INIT {._is_initialized = false} +#define Py_tss_NEEDS_INIT {._is_initialized = 0} #endif /* !Py_LIMITED_API */ PyAPI_FUNC(int) PyThread_tss_create(Py_tss_t *key); @@ -147,7 +145,7 @@ PyAPI_FUNC(void *) PyThread_tss_get(Py_tss_t *key); PyAPI_FUNC(Py_tss_t *) PyThread_tss_alloc(void); PyAPI_FUNC(void) PyThread_tss_free(Py_tss_t *key); -PyAPI_FUNC(bool) PyThread_tss_is_created(Py_tss_t *key); +PyAPI_FUNC(int) PyThread_tss_is_created(Py_tss_t *key); #ifdef __cplusplus } diff --git a/Python/thread.c b/Python/thread.c index 5e32d4fa88dd8b..749d66ffd31720 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -177,7 +177,7 @@ PyThread_tss_free(Py_tss_t *key) } } -bool +int PyThread_tss_is_created(Py_tss_t *key) { assert(key != NULL); diff --git a/Python/thread_nt.h b/Python/thread_nt.h index a1402aa5a1710b..9042650b7b403d 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -8,7 +8,6 @@ #ifdef HAVE_PROCESS_H #include #endif -#include /* necessary for TSS key */ /* options */ #ifndef _PY_USE_CV_LOCKS @@ -427,7 +426,7 @@ PyThread_tss_create(Py_tss_t *key) } /* In Windows, platform-specific key type is DWORD. */ key->_key = result; - key->_is_initialized = true; + key->_is_initialized = 1; return 0; } @@ -442,7 +441,7 @@ PyThread_tss_delete(Py_tss_t *key) TlsFree(key->_key); key->_key = TLS_OUT_OF_INDEXES; - key->_is_initialized = false; + key->_is_initialized = 0; } int diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 8051528c6a76cb..e80d94329ba737 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -1,7 +1,6 @@ /* Posix threads interface */ -#include /* necessary for TSS key */ #include #include #if defined(__APPLE__) || defined(HAVE_PTHREAD_DESTRUCTOR) @@ -701,7 +700,7 @@ PyThread_tss_create(Py_tss_t *key) if (fail) { return -1; } - key->_is_initialized = true; + key->_is_initialized = 1; return 0; } @@ -716,7 +715,7 @@ PyThread_tss_delete(Py_tss_t *key) pthread_key_delete(key->_key); /* pthread has not provided the defined invalid value for the key. */ - key->_is_initialized = false; + key->_is_initialized = 0; } int From 4c38b093e55cc8fdbe351093a9a4f2afee2fd510 Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Sat, 2 Sep 2017 07:18:00 +0900 Subject: [PATCH 21/41] bpo-25658: use a partial positional initializer for Py_tss_NEEDS_INIT Using designated initializers in header files prevents us using Python in C++. https://github.com/python/cpython/pull/1362#pullrequestreview-59884205 --- Include/pythread.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/pythread.h b/Include/pythread.h index 5552236cb2f47c..8f03d5f9c74c40 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -130,7 +130,7 @@ struct _Py_tss_t { #undef NATIVE_TSS_KEY_T /* When static allocation, you must initialize with Py_tss_NEEDS_INIT. */ -#define Py_tss_NEEDS_INIT {._is_initialized = 0} +#define Py_tss_NEEDS_INIT {0} #endif /* !Py_LIMITED_API */ PyAPI_FUNC(int) PyThread_tss_create(Py_tss_t *key); From 8a0036c286556e040e81a20d0906cb00d00fd856 Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Sat, 2 Sep 2017 10:16:20 +0900 Subject: [PATCH 22/41] bpo-25658: add the version check for the limited API TSS API is not in the limited API prior to CPython 3.7. https://github.com/python/cpython/pull/1362#pullrequestreview-59885908 --- Include/pythread.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Include/pythread.h b/Include/pythread.h index 8f03d5f9c74c40..bacd3c28469817 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -99,6 +99,8 @@ PyAPI_FUNC(void) PyThread_delete_key_value(int key) Py_DEPRECATED(3.7); PyAPI_FUNC(void) PyThread_ReInitTLS(void) Py_DEPRECATED(3.7); +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03070000 +/* New in 3.7 */ /* Thread Specific Storage (TSS) API */ /* Py_tss_t is an opaque data type the definition of which depends on the @@ -146,6 +148,7 @@ PyAPI_FUNC(Py_tss_t *) PyThread_tss_alloc(void); PyAPI_FUNC(void) PyThread_tss_free(Py_tss_t *key); PyAPI_FUNC(int) PyThread_tss_is_created(Py_tss_t *key); +#endif /* New in 3.7 */ #ifdef __cplusplus } From a46b75f59ec1a5c4fac746c1be795d7f0d115859 Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Sat, 2 Sep 2017 10:28:00 +0900 Subject: [PATCH 23/41] Revert "bpo-25658: Add an example function to xxlimited module" This reverts commit d25f441e6ac8eef69e93881c03044139213abf6b. Since the specification is almost fixed, revert a temporary commit which builds xxlimited extension module to use the TSS API in the limited API. --- Modules/xxlimited.c | 64 --------------------------------------------- 1 file changed, 64 deletions(-) diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 2b82419cd05c27..5041ac8284a3c9 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -15,9 +15,6 @@ /* Xxo objects */ #include "Python.h" -#ifdef WITH_THREAD -#include "pythread.h" -#endif static PyObject *ErrorObject; @@ -218,63 +215,6 @@ static PyType_Spec Null_Type_spec = { /* ---------- */ -#ifdef WITH_THREAD -PyDoc_STRVAR(xx_tss_doc, -"tss(a) -> a\n\ -\n\ -TSS API test in the limited API."); - -/* In the limited API, Py_tss_t value cannot be statically allocated. -static Py_tss_t tss_key = Py_tss_NEEDS_INIT; -*/ -static Py_tss_t *tss_key; - -static PyObject * -xx_tss(PyObject *self, PyObject *args) -{ - PyObject *a, *b; - if (!PyArg_ParseTuple(args, "O:tss", &a)) { - return NULL; - } - - tss_key = PyThread_tss_alloc(); - if (tss_key == NULL) { - PyErr_SetString(PyExc_RuntimeError, "PyThread_tss_alloc failed"); - return NULL; - } - if (PyThread_tss_create(tss_key) != 0) { - PyThread_tss_free(tss_key); - PyErr_SetString(PyExc_RuntimeError, "PyThread_tss_create failed"); - return NULL; - } - if (!PyThread_tss_is_created(tss_key)) { - PyThread_tss_free(tss_key); - PyErr_SetString(PyExc_RuntimeError, - "TSS key in an uninitialized state"); - return NULL; - } - if (PyThread_tss_set(tss_key, (void *)a) != 0) { - PyThread_tss_free(tss_key); - PyErr_SetString(PyExc_RuntimeError, "PyThread_tss_set failed"); - return NULL; - } - b = (PyObject *)PyThread_tss_get(tss_key); - if (a != b) { - PyThread_tss_free(tss_key); - PyErr_SetString(PyExc_RuntimeError, "PyThread_tss_get failed"); - return NULL; - } - PyThread_tss_delete(tss_key); - PyThread_tss_free(tss_key); - tss_key = NULL; - - Py_INCREF(b); - return b; -} -#endif /* WITH_THREAD */ - -/* ---------- */ - /* List of functions defined in the module */ static PyMethodDef xx_methods[] = { @@ -284,10 +224,6 @@ static PyMethodDef xx_methods[] = { xx_foo_doc}, {"new", xx_new, METH_VARARGS, PyDoc_STR("new() -> new Xx object")}, -#ifdef WITH_THREAD - {"tss", xx_tss, METH_VARARGS, - xx_tss_doc}, -#endif {NULL, NULL} /* sentinel */ }; From 29658c7c36b79e8363de10efa4cb48940579969a Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Sat, 2 Sep 2017 11:30:29 +0900 Subject: [PATCH 24/41] Revert "bpo-25658: Add TSS API to Windows stable ABI" This reverts commit 594ec29a07794f7f1748c5923362ad1f77e14dc5. Revert a temporary commit which supports TSS API in Windows stable ABI, because the existing TLS API is not provided in Windows stable ABI. TSS API support goes to maintain the status quo. --- PC/python3.def | 7 ------- 1 file changed, 7 deletions(-) diff --git a/PC/python3.def b/PC/python3.def index 1d089ec28b1b02..ad65294045f677 100644 --- a/PC/python3.def +++ b/PC/python3.def @@ -570,13 +570,6 @@ EXPORTS PyThreadState_New=python37.PyThreadState_New PyThreadState_SetAsyncExc=python37.PyThreadState_SetAsyncExc PyThreadState_Swap=python37.PyThreadState_Swap - PyThread_tss_alloc=python37.PyThread_tss_alloc - PyThread_tss_create=python37.PyThread_tss_create - PyThread_tss_delete=python37.PyThread_tss_delete - PyThread_tss_free=python37.PyThread_tss_free - PyThread_tss_get=python37.PyThread_tss_get - PyThread_tss_is_created=python37.PyThread_tss_is_created - PyThread_tss_set=python37.PyThread_tss_set PyTraceBack_Here=python37.PyTraceBack_Here PyTraceBack_Print=python37.PyTraceBack_Print PyTraceBack_Type=python37.PyTraceBack_Type DATA From 5e8c1bd48088bfede3d7cd1ae59a154a57c7544b Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Sat, 2 Sep 2017 12:14:12 +0900 Subject: [PATCH 25/41] bpo-25658: fix an assignment to the _is_initialized field in PyThread_tss_alloc --- Python/thread.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Python/thread.c b/Python/thread.c index 749d66ffd31720..c22126f7035d68 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -20,7 +20,6 @@ #include #endif -#include /* necessary for TSS key */ #include #include "pythread.h" @@ -164,7 +163,7 @@ PyThread_tss_alloc(void) if (new_key == NULL) { return NULL; } - new_key->_is_initialized = false; + new_key->_is_initialized = 0; return new_key; } From 61dcde48b67c1c35e5661a9a03e4814529893c4a Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Fri, 8 Sep 2017 17:51:33 +0900 Subject: [PATCH 26/41] bpo-25658: remove WITH_THREAD guard --- Modules/_testcapimodule.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 68f3e2131f0606..b512c05a3feb29 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4306,7 +4306,6 @@ py_w_stopcode(PyObject *self, PyObject *args) #endif -#ifdef WITH_THREAD static PyObject * test_pythread_tss_key_state(PyObject *self, PyObject *args) { @@ -4360,7 +4359,6 @@ test_pythread_tss_key_state(PyObject *self, PyObject *args) ptr_key = NULL; Py_RETURN_NONE; } -#endif static PyMethodDef TestMethods[] = { @@ -4575,9 +4573,7 @@ static PyMethodDef TestMethods[] = { #ifdef W_STOPCODE {"W_STOPCODE", py_w_stopcode, METH_VARARGS}, #endif -#ifdef WITH_THREAD {"test_pythread_tss_key_state", test_pythread_tss_key_state, METH_VARARGS}, -#endif {NULL, NULL} /* sentinel */ }; From 6a93856d0d0b297e7cd0996535e860fc9ffdf96c Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Sun, 10 Sep 2017 00:00:10 +0900 Subject: [PATCH 27/41] bpo-25658: add contiributor names to news and whatsnew --- Doc/whatsnew/3.7.rst | 3 ++- .../NEWS.d/next/C API/2017-06-24-14-30-44.bpo-25658.vm8vGE.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 5c39cd777088f9..97eeda2782bc5d 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -320,7 +320,8 @@ Build and C API Changes deprecating the existing API. The change is motivated primarily by the fact that the old API uses int to represent TLS keys across all platforms, which is neither POSIX-compliant, nor portable in any practical sense. For more - information, see :pep:`539` and :issue:`25658`. + information, see :pep:`539`. (Contributed by Erik M. Bray and Masayuki + Yamamoto in :issue:`25658`.) * A full copy of libffi is no longer bundled for use when building the :mod:`_ctypes ` module on non-OSX UNIX platforms. An installed copy diff --git a/Misc/NEWS.d/next/C API/2017-06-24-14-30-44.bpo-25658.vm8vGE.rst b/Misc/NEWS.d/next/C API/2017-06-24-14-30-44.bpo-25658.vm8vGE.rst index 99a682f1227aec..d4bb19acc4b335 100644 --- a/Misc/NEWS.d/next/C API/2017-06-24-14-30-44.bpo-25658.vm8vGE.rst +++ b/Misc/NEWS.d/next/C API/2017-06-24-14-30-44.bpo-25658.vm8vGE.rst @@ -1,3 +1,4 @@ Implement PEP 539 for Thread Specific Stroage (TSS) API: it is a new Thread Local Storage (TLS) API to CPython which would supersede use of the existing TLS API within the CPython interpreter, while deprecating the existing API. +PEP written by Erik M. Bray, patch by Masayuki Yamamoto. From fa82f9c31e7cdc3bed88c4407279563da5079400 Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Sun, 10 Sep 2017 01:00:08 +0900 Subject: [PATCH 28/41] bpo-25658: update compile error message in thread-less builds Remove tracemalloc's message that has made no longer sensible for since all platforms require native threads. --- Include/pythread.h | 2 +- Modules/_tracemalloc.c | 6 ------ Python/thread.c | 2 +- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/Include/pythread.h b/Include/pythread.h index bacd3c28469817..763c416b3bf976 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -3,7 +3,7 @@ #define Py_PYTHREAD_H #if !(defined(_POSIX_THREADS) || defined(NT_THREADS)) -# error "Require native thread feature. See https://bugs.python.org/issue30832" +# error "Require native threads. See https://bugs.python.org/issue31370" #endif typedef void *PyThread_type_lock; diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 01f9e8f17d8a61..3f43e845f04399 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -167,12 +167,6 @@ tracemalloc_error(const char *format, ...) #if defined(TRACE_RAW_MALLOC) #define REENTRANT_THREADLOCAL -/* If your OS does not provide native thread local storage, CPython - which is compiled with thread support does not provide tracemalloc. */ -#if !(defined(_POSIX_THREADS) || defined(NT_THREADS)) -# error "need native thread local storage (TLS)" -#endif - static Py_tss_t tracemalloc_reentrant_key = Py_tss_NEEDS_INIT; /* Any non-NULL pointer can be used */ diff --git a/Python/thread.c b/Python/thread.c index b7b38fdbc030d3..3cfe0f2b4ab871 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -84,7 +84,7 @@ PyThread_init_thread(void) # define PYTHREAD_NAME "nt" # include "thread_nt.h" #else -# error "Require native thread feature. See https://bugs.python.org/issue30832" +# error "Require native threads. See https://bugs.python.org/issue31370" #endif From 12727a6240a9656beacc697eeca4ca8020f27c21 Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Sun, 10 Sep 2017 03:41:21 +0900 Subject: [PATCH 29/41] bpo-25658: create PEP 539 topic for New Features in whatsnew --- Doc/whatsnew/3.7.rst | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 97eeda2782bc5d..370042ccbb1eda 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -108,6 +108,37 @@ locale remains active when the core interpreter is initialized. PEP written and implemented by Nick Coghlan. +.. _whatsnew37-pep539: + +PEP 539: A New C-API for Thread-Local Storage in CPython +-------------------------------------------------------- + +While CPython provides C API for the thread-local storage internally, the +existing Thread Local Storage (TLS) API uses ``int`` to represent TLS keys +across all platforms. This has not generally been a problem for +officially-support platforms, but that is neither POSIX-compliant, nor portable +in any practical sense. + +:pep:`539` changes this by providing the new Thread Specific Storage (TSS) API +to CPython which would supersede use of the existing TLS API within the CPython +interpreter, while deprecating the existing API. The TSS API uses a new type +``Py_tss_t`` to conceal the native TLS key via opaque data type, therefore, +this will allow to build CPython on platforms where the native TLS key is +defined in a way that cannot be safely cast to ``int``. + +Moreover, on platforms where the native TLS key is defined in a way that cannot +be safely cast to ``int``, ``PyThread_create_key`` will return immediately with +a failure status, and the other TLS functions will all be no-ops on such +platforms. This indicates clearly that the old API is not supported on +platforms where it cannot be used reliably, and that no effort will be made to +add such support. + +.. seealso:: + + :pep:`539` -- A New C-API for Thread-Local Storage in CPython + PEP written by Erik M. Bray; implementation by Masayuki Yamamoto. + + Other Language Changes ====================== @@ -315,14 +346,6 @@ Optimizations Build and C API Changes ======================= -* Added Thread Specific Storage (TSS) API which would supersede use of the - existing Thread Local Storage (TLS) API within the CPython interpreter, while - deprecating the existing API. The change is motivated primarily by the fact - that the old API uses int to represent TLS keys across all platforms, which - is neither POSIX-compliant, nor portable in any practical sense. For more - information, see :pep:`539`. (Contributed by Erik M. Bray and Masayuki - Yamamoto in :issue:`25658`.) - * A full copy of libffi is no longer bundled for use when building the :mod:`_ctypes ` module on non-OSX UNIX platforms. An installed copy of libffi is now required when building ``_ctypes`` on such platforms. From 37d6794bbf1635ca9d6f1d4e3ad443ca86e9df97 Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Mon, 11 Sep 2017 02:44:17 +0900 Subject: [PATCH 30/41] bpo-25658: put header's error directive into Py_LIMITED_API guard Code smell breaking backward compatibility in the limited API. --- Include/pythread.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Include/pythread.h b/Include/pythread.h index 763c416b3bf976..9580a45dad01c7 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -2,10 +2,6 @@ #ifndef Py_PYTHREAD_H #define Py_PYTHREAD_H -#if !(defined(_POSIX_THREADS) || defined(NT_THREADS)) -# error "Require native threads. See https://bugs.python.org/issue31370" -#endif - typedef void *PyThread_type_lock; typedef void *PyThread_type_sema; @@ -118,6 +114,8 @@ typedef struct _Py_tss_t Py_tss_t; but hardcode the unsigned long to avoid errors for include directive. */ # define NATIVE_TSS_KEY_T unsigned long +#else +# error "Require native threads. See https://bugs.python.org/issue30832" #endif /* When Py_LIMITED_API is not defined, the type layout of Py_tss_t is in From 02a165b8b8e5b88578f2c4b78e4d1f6b430b9ba8 Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Wed, 13 Sep 2017 21:59:52 +0900 Subject: [PATCH 31/41] bpo-25658: show an API difference on whatsnew by initialization codes --- Doc/whatsnew/3.7.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 370042ccbb1eda..1fa7c76fd4202c 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -126,6 +126,20 @@ interpreter, while deprecating the existing API. The TSS API uses a new type this will allow to build CPython on platforms where the native TLS key is defined in a way that cannot be safely cast to ``int``. +A TLS key has been initialized previously like: + +.. code-block:: c + + int key = PyThread_create_key(); + int fail = key == -1; + +With the change, a TSS key is initialized like: + +.. code-block:: c + + Py_tss_t key = Py_tss_NEEDS_INIT; + int fail = PyThread_tss_create(&key); + Moreover, on platforms where the native TLS key is defined in a way that cannot be safely cast to ``int``, ``PyThread_create_key`` will return immediately with a failure status, and the other TLS functions will all be no-ops on such From e0c7fd62824fb736445d163c49eafae586842c28 Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Mon, 18 Sep 2017 00:05:58 +0900 Subject: [PATCH 32/41] bpo-25658: fix an issue number in the error directive https://github.com/python/cpython/pull/1362#discussion_r139301168 --- Include/pythread.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/pythread.h b/Include/pythread.h index 9580a45dad01c7..9e7f9b01395f4e 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -115,7 +115,7 @@ typedef struct _Py_tss_t Py_tss_t; */ # define NATIVE_TSS_KEY_T unsigned long #else -# error "Require native threads. See https://bugs.python.org/issue30832" +# error "Require native threads. See https://bugs.python.org/issue31370" #endif /* When Py_LIMITED_API is not defined, the type layout of Py_tss_t is in From 35a8fa50d18c42600060a3047308f74c687aa384 Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Mon, 18 Sep 2017 00:06:26 +0900 Subject: [PATCH 33/41] bpo-25658: fix comments https://github.com/python/cpython/pull/1362#discussion_r139301183 https://github.com/python/cpython/pull/1362#discussion_r139301195 https://github.com/python/cpython/pull/1362#discussion_r139301215 --- Python/thread.c | 2 +- Python/thread_nt.h | 7 +++++-- Python/thread_pthread.h | 6 ++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Python/thread.c b/Python/thread.c index 3cfe0f2b4ab871..55ace55687b412 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -149,7 +149,7 @@ that calls to PyThread_tss_create() are serialized externally. /* Thread Specific Storage (TSS) API - Implementation part of common + Cross-platform components of TSS API implementation. */ Py_tss_t * diff --git a/Python/thread_nt.h b/Python/thread_nt.h index 99afe1cad1c7ee..bae8bcc35669fd 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -349,7 +349,10 @@ _pythread_nt_set_stacksize(size_t size) #define THREAD_SET_STACKSIZE(x) _pythread_nt_set_stacksize(x) -/* Thread Local Storage (TLS) API, DEPRECATED since Python 3.7 */ +/* Thread Local Storage (TLS) API + + This API is DEPRECATED since Python 3.7. See PEP 539 for details. +*/ int PyThread_create_key(void) @@ -409,7 +412,7 @@ PyThread_ReInitTLS(void) /* Thread Specific Storage (TSS) API - Implementation part of platform-specific + Platform-specific components of TSS API implementation. */ int diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 6a1d5e5f4dd2ca..3ae934fab0ec3f 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -610,7 +610,9 @@ _pythread_pthread_set_stacksize(size_t size) #define THREAD_SET_STACKSIZE(x) _pythread_pthread_set_stacksize(x) -/* Thread Local Storage (TLS) API, DEPRECATED since Python 3.7 +/* Thread Local Storage (TLS) API + + This API is DEPRECATED since Python 3.7. See PEP 539 for details. Issue #25658: POSIX hasn't defined that pthread_key_t is compatible with int. Note that if incompatible with int, PyThread_create_key @@ -685,7 +687,7 @@ PyThread_ReInitTLS(void) /* Thread Specific Storage (TSS) API - Implementation part of platform-specific + Platform-specific components of TSS API implementation. */ int From 45af047291fc2f9861989de960febe7bfe2a8b9c Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Mon, 18 Sep 2017 00:07:02 +0900 Subject: [PATCH 34/41] bpo-25658: update a comment the pthreads for TLS implementation --- Python/thread_pthread.h | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 3ae934fab0ec3f..f065146a6896b3 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -613,12 +613,16 @@ _pythread_pthread_set_stacksize(size_t size) /* Thread Local Storage (TLS) API This API is DEPRECATED since Python 3.7. See PEP 539 for details. +*/ + +/* Issue #25658: On platforms where native TLS key is defined in a way that + cannot be safely cast to int, PyThread_create_key returns immediately a + failure status and other TLS functions all are no-ops. This indicates + clearly that the old API is not supported on platforms where it cannot be + used reliably, and that no effort will be made to add such support. - Issue #25658: POSIX hasn't defined that pthread_key_t is compatible - with int. Note that if incompatible with int, PyThread_create_key - returns immediately a failure status and the other TLS functions - all are no-ops. Moreover, PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT will - be unnecessary after removing this API. + Note: PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT will be unnecessary after + removing this API. */ int From a0d742a43e31bc0405f6e278a3120d3d51be47dd Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Mon, 8 May 2017 17:15:45 +0900 Subject: [PATCH 35/41] bpo-25658: Add the TSS API to Windows stable ABI --- PC/python3.def | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/PC/python3.def b/PC/python3.def index ad65294045f677..1d089ec28b1b02 100644 --- a/PC/python3.def +++ b/PC/python3.def @@ -570,6 +570,13 @@ EXPORTS PyThreadState_New=python37.PyThreadState_New PyThreadState_SetAsyncExc=python37.PyThreadState_SetAsyncExc PyThreadState_Swap=python37.PyThreadState_Swap + PyThread_tss_alloc=python37.PyThread_tss_alloc + PyThread_tss_create=python37.PyThread_tss_create + PyThread_tss_delete=python37.PyThread_tss_delete + PyThread_tss_free=python37.PyThread_tss_free + PyThread_tss_get=python37.PyThread_tss_get + PyThread_tss_is_created=python37.PyThread_tss_is_created + PyThread_tss_set=python37.PyThread_tss_set PyTraceBack_Here=python37.PyTraceBack_Here PyTraceBack_Print=python37.PyTraceBack_Print PyTraceBack_Type=python37.PyTraceBack_Type DATA From 5073d6688a37977d08176fe80644ef28fcfd8422 Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Fri, 6 Oct 2017 16:56:13 +0900 Subject: [PATCH 36/41] bpo-25658: remove API description on the code --- Python/thread.c | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/Python/thread.c b/Python/thread.c index 55ace55687b412..7eac836dc16f3d 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -111,42 +111,6 @@ PyThread_set_stacksize(size_t size) } -/* ------------------------------------------------------------------------ -Per-thread data ("key") support. - -Use PyThread_tss_create(&thekey) to create a new key. This is typically shared -across threads. - -Use PyThread_tss_set(&thekey, value) to associate void* value with -thekey in the current thread. Each thread has a distinct mapping of thekey -to a void* value. Caution: if the current thread already has a mapping -for thekey, value is ignored. - -Use PyThread_tss_get(&thekey) to retrieve the void* value associated -with thekey in the current thread. This returns NULL if no value is -associated with thekey in the current thread. - -Use PyThread_tss_set(&thekey, NULL) to forget the current thread's associated -value for thekey. PyThread_tss_delete(&thekey) forgets the values associated -with thekey across *all* threads. - -While some of these functions have error-return values, none set any -Python exception. - -None of the functions does memory management on behalf of the void* values. -You need to allocate and deallocate them yourself. If the void* values -happen to be PyObject*, these functions don't do refcount operations on -them either. - -The GIL does not need to be held when calling these functions; they supply -their own locking. This isn't true of PyThread_tss_create(), though (see -next paragraph). - -There's a hidden assumption that PyThread_tss_create() will be called before -any of the other functions are called. There's also a hidden assumption -that calls to PyThread_tss_create() are serialized externally. ------------------------------------------------------------------------- */ - /* Thread Specific Storage (TSS) API Cross-platform components of TSS API implementation. From c9f75361a3a6759447de61543c9bb13b3ff20734 Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Fri, 6 Oct 2017 16:57:03 +0900 Subject: [PATCH 37/41] bpo-25658: add API document - overview - the existing TLS API (without descripiton) - the new TSS API - type and macro - dynamic allocation - methods --- Doc/c-api/init.rst | 158 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index c12e1c7fba3cb8..f8e6fb4d17a18e 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1192,3 +1192,161 @@ These functions are only intended to be used by advanced debugging tools. Return the next thread state object after *tstate* from the list of all such objects belonging to the same :c:type:`PyInterpreterState` object. + +.. _thread-local-storage: + +Thread Local Storage Support +============================ + +.. sectionauthor:: Masayuki Yamamoto + + +The Python interpreter provides low-level support for thread-local storage +(TLS) which wrapped underlying TLS implementation to be compatible, besides the +Python-level support (:class:`threading.local`). CPython TLS APIs are similar +to pthreads and Windows API: use a thread key and functions to associate a +:c:type:`void\*` value per thread, a thread key is typically shared across +threads and no destructor support like `pthread_key_create +`_. +The GIL does not need to be held when calling these functions; they supply +their own locking. + +Note that :file:`Python.h` does not include the declaration of the TLS APIs, +you need to include :file:`pythread.h` to use thread-local storage. + +.. note:: + None of the API functions does memory management on behalf of the + :c:type:`void\*` values. You need to allocate and deallocate them yourself. + If the :c:type:`void\*` values happen to be :c:type:`PyObject\*`, these + functions don't do refcount operations on them either. + + +.. _thread-local-storage-api: + +Thread Local Storage (TLS) API +------------------------------ + +.. deprecated:: 3.7 + This API is superseded by + :ref:`Thread Specific Storage (TSS) API `. + +.. note:: + No support for platforms where the native TLS key is defined in a way that + cannot be safely cast to ``int``. On such platforms, + :c:func:`PyThread_create_key` will return immediately with a failure status, + and the other TLS functions will all be no-ops on such platforms. + +There is no description because prevents us from use of this API. + + +.. c:function:: int PyThread_create_key() +.. c:function:: void PyThread_delete_key(int key) +.. c:function:: int PyThread_set_key_value(int key, void *value) +.. c:function:: void* PyThread_get_key_value(int key) +.. c:function:: void PyThread_delete_key_value(int key) +.. c:function:: void PyThread_ReInitTLS() + + +.. _thread-specific-storage-api: + +Thread Specific Storage (TSS) API +--------------------------------- + +TSS API is introduced to supersede the use of the existing TLS API within the +CPython interpreter. This API uses a new type :c:type:`Py_tss_t` instead of +:c:type:`int` to represent thread keys. + +.. versionadded:: 3.7 + +.. seealso:: "A New C-API for Thread-Local Storage in CPython" (:pep:`539`) + + +.. c:type:: Py_tss_t + + This data structure represents the state of a thread key, the definition of + which may depend on the underlying TLS implementation, and it has an + internal field representing the key's initialization state. There are no + public members in this structure. + + When :ref:`Py_LIMITED_API ` is not defined, static allocation of + this type by :c:macro:`Py_tss_NEEDS_INIT` is allowed. + + +.. c:macro:: Py_tss_NEEDS_INIT + + This macro expands to the default value for :c:type:`Py_tss_t` variables. + Note that this macro won't be defined with :ref:`Py_LIMITED_API `. + + +Dynamic Allocation +~~~~~~~~~~~~~~~~~~ + +Dynamic allocation of the :c:type:`Py_tss_t`, particularly in extension modules +built with :ref:`Py_LIMITED_API `, where static allocation of this type +is not possible due to its implementation being opaque at build time. + + +.. c:function:: Py_tss_t* PyThread_tss_alloc() + + Return a value which is the same state as a value initialized with + :c:macro:`Py_tss_NEEDS_INIT`, or *NULL* in the case of dynamic allocation + failure. + + +.. c:function:: void PyThread_tss_free(Py_tss_t *key) + + Free a TSS key value allocated by :c:func:`PyThread_tss_alloc`, or does + no-op if the value pointed to by the *key* argument is *NULL*. In the case + of freeing, this behavior involves calling :c:func:`PyThread_tss_delete` + preventively. + + .. warning:: + A freed key becomes a dangling pointer, you should reset the key to + *NULL*. + + +Methods +~~~~~~~ + +The parameter *key* of these functions must not be *NULL*. Moreover, the +behaviors of :c:func:`PyThread_tss_set` and :c:func:`PyThread_tss_get` are +undefined if the given :c:type:`Py_tss_t` has not been initialized by +:c:func:`PyThread_tss_create`. + + +.. c:function:: int PyThread_tss_is_created(Py_tss_t *key) + + Return a non-zero value if the given :c:type:`Py_tss_t` has been initialized + by :c:func:`PyThread_tss_create`. + + +.. c:function:: int PyThread_tss_create(Py_tss_t *key) + + Return a zero value on success an initialization of a TSS key. The behavior + is undefined if the value pointed to by the *key* argument is not + initialized by :c:macro:`Py_tss_NEEDS_INIT`. This function can be called + repeatedly on the same key--calling it on an already initialized key is a + no-op and immediately returns success. + + +.. c:function:: void PyThread_tss_delete(Py_tss_t *key) + + Destroy a TSS key to forget the values associated with the key across all + threads, and change the key's initialization state to uninitialized. A + destroyed key is able to be initialized again by + :c:func:`PyThread_tss_create`. This function can be called repeatedly on + the same key--calling it on an already destroyed key is a no-op. + + +.. c:function:: int PyThread_tss_set(Py_tss_t *key, void *value) + + Return a zero value on success associated a :c:type:`void\*` value with a + TSS key in the current thread. Each thread has a distinct mapping of the + key to a :c:type:`void\*` value. + + +.. c:function:: void* PyThread_tss_get(Py_tss_t *key) + + Return the :c:type:`void\*` value associated with a TSS key in the current + thread. This returns *NULL* if no value is associated with the key in the + current thread. From a9a9df6089e42f7983c7ca28b99e3baa8ef3853b Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Fri, 6 Oct 2017 16:58:25 +0900 Subject: [PATCH 38/41] bpo-25658: modify whatsnew document --- Doc/whatsnew/3.7.rst | 53 +++++++++++++++++--------------------------- 1 file changed, 20 insertions(+), 33 deletions(-) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 547ff499ac30db..ecdd2fe2713338 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -132,39 +132,26 @@ built-in ``breakpoint()``. PEP 539: A New C-API for Thread-Local Storage in CPython -------------------------------------------------------- -While CPython provides C API for the thread-local storage internally, the -existing Thread Local Storage (TLS) API uses ``int`` to represent TLS keys -across all platforms. This has not generally been a problem for -officially-support platforms, but that is neither POSIX-compliant, nor portable -in any practical sense. - -:pep:`539` changes this by providing the new Thread Specific Storage (TSS) API -to CPython which would supersede use of the existing TLS API within the CPython -interpreter, while deprecating the existing API. The TSS API uses a new type -``Py_tss_t`` to conceal the native TLS key via opaque data type, therefore, -this will allow to build CPython on platforms where the native TLS key is -defined in a way that cannot be safely cast to ``int``. - -A TLS key has been initialized previously like: - -.. code-block:: c - - int key = PyThread_create_key(); - int fail = key == -1; - -With the change, a TSS key is initialized like: - -.. code-block:: c - - Py_tss_t key = Py_tss_NEEDS_INIT; - int fail = PyThread_tss_create(&key); - -Moreover, on platforms where the native TLS key is defined in a way that cannot -be safely cast to ``int``, ``PyThread_create_key`` will return immediately with -a failure status, and the other TLS functions will all be no-ops on such -platforms. This indicates clearly that the old API is not supported on -platforms where it cannot be used reliably, and that no effort will be made to -add such support. +While Python provides a C API for thread-local storage support; the existing +:ref:`Thread Local Storage (TLS) API ` has used +:c:type:`int` to represent TLS keys across all platforms. This has not +generally been a problem for officially-support platforms, but that is neither +POSIX-compliant, nor portable in any practical sense. + +:pep:`539` changes this by providing a new :ref:`Thread Specific Storage (TSS) +API ` to CPython which supersedes use of the +existing TLS API within the CPython interpreter, while deprecating the existing +API. The TSS API uses a new type :c:type:`Py_tss_t` instead of :c:type:`int` +to represent TSS keys--an opaque type the definition of which may depend on +the underlying TLS implementation. Therefore, this will allow to build CPython +on platforms where the native TLS key is defined in a way that cannot be safely +cast to :c:type:`int`. + +Note that on platforms where the native TLS key is defined in a way that cannot +be safely cast to :c:type:`int`, all functions of the existing TLS API will be +no-op and immediately return failure. This indicates clearly that the old API +is not supported on platforms where it cannot be used reliably, and that no +effort will be made to add such support. .. seealso:: From 60828d858cdbab38688d5b080ff1cdbc270b13eb Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Fri, 6 Oct 2017 16:59:14 +0900 Subject: [PATCH 39/41] bpo-25658: modify header comments Several comments is no longer necessary by API document. --- Include/pythread.h | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/Include/pythread.h b/Include/pythread.h index 9e7f9b01395f4e..e5ec9035795a1e 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -99,10 +99,7 @@ PyAPI_FUNC(void) PyThread_ReInitTLS(void) Py_DEPRECATED(3.7); /* New in 3.7 */ /* Thread Specific Storage (TSS) API */ -/* Py_tss_t is an opaque data type the definition of which depends on the - underlying TSS implementation. -*/ -typedef struct _Py_tss_t Py_tss_t; +typedef struct _Py_tss_t Py_tss_t; /* opaque */ #ifndef Py_LIMITED_API #if defined(_POSIX_THREADS) @@ -118,9 +115,9 @@ typedef struct _Py_tss_t Py_tss_t; # error "Require native threads. See https://bugs.python.org/issue31370" #endif -/* When Py_LIMITED_API is not defined, the type layout of Py_tss_t is in - public to allow static allocation in the API clients. Even in this case, - you must handle TSS key through API functions due to compatibility. +/* When Py_LIMITED_API is not defined, the type layout of Py_tss_t is + exposed to allow static allocation in the API clients. Even in this case, + you must handle TSS keys through API functions due to compatibility. */ struct _Py_tss_t { int _is_initialized; @@ -138,10 +135,6 @@ PyAPI_FUNC(void) PyThread_tss_delete(Py_tss_t *key); PyAPI_FUNC(int) PyThread_tss_set(Py_tss_t *key, void *value); PyAPI_FUNC(void *) PyThread_tss_get(Py_tss_t *key); -/* In the limited API, Py_tss_t value must be allocated by PyThread_tss_alloc, - and free by PyThread_tss_free at the life cycle end of the CPython - interpreter. -*/ PyAPI_FUNC(Py_tss_t *) PyThread_tss_alloc(void); PyAPI_FUNC(void) PyThread_tss_free(Py_tss_t *key); From f1717b4eef9ace1d80291f8f2e5de4dfb5068a51 Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Fri, 6 Oct 2017 17:00:13 +0900 Subject: [PATCH 40/41] bpo-25658: update order of the declaration New order is along API document. --- Include/pythread.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Include/pythread.h b/Include/pythread.h index e5ec9035795a1e..d6674685f289ff 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -130,15 +130,15 @@ struct _Py_tss_t { #define Py_tss_NEEDS_INIT {0} #endif /* !Py_LIMITED_API */ -PyAPI_FUNC(int) PyThread_tss_create(Py_tss_t *key); -PyAPI_FUNC(void) PyThread_tss_delete(Py_tss_t *key); -PyAPI_FUNC(int) PyThread_tss_set(Py_tss_t *key, void *value); -PyAPI_FUNC(void *) PyThread_tss_get(Py_tss_t *key); - PyAPI_FUNC(Py_tss_t *) PyThread_tss_alloc(void); PyAPI_FUNC(void) PyThread_tss_free(Py_tss_t *key); +/* The parameter key must not be NULL. */ PyAPI_FUNC(int) PyThread_tss_is_created(Py_tss_t *key); +PyAPI_FUNC(int) PyThread_tss_create(Py_tss_t *key); +PyAPI_FUNC(void) PyThread_tss_delete(Py_tss_t *key); +PyAPI_FUNC(int) PyThread_tss_set(Py_tss_t *key, void *value); +PyAPI_FUNC(void *) PyThread_tss_get(Py_tss_t *key); #endif /* New in 3.7 */ #ifdef __cplusplus From 80e768415f0f458d42e847af810e179b82c41c27 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Fri, 6 Oct 2017 20:25:00 +1000 Subject: [PATCH 41/41] Adjust some details of documentation wording --- Doc/c-api/init.rst | 99 +++++++++++++++++++++++----------------------- 1 file changed, 49 insertions(+), 50 deletions(-) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index f8e6fb4d17a18e..7792058683da0c 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1200,53 +1200,25 @@ Thread Local Storage Support .. sectionauthor:: Masayuki Yamamoto - The Python interpreter provides low-level support for thread-local storage -(TLS) which wrapped underlying TLS implementation to be compatible, besides the -Python-level support (:class:`threading.local`). CPython TLS APIs are similar -to pthreads and Windows API: use a thread key and functions to associate a -:c:type:`void\*` value per thread, a thread key is typically shared across -threads and no destructor support like `pthread_key_create -`_. -The GIL does not need to be held when calling these functions; they supply +(TLS) which wraps the underlying native TLS implementation to support the +Python-level thread local storage API (:class:`threading.local`). The +CPython C level APIs are similar to those offered by pthreads and Windows: +use a thread key and functions to associate a :c:type:`void\*` value per +thread. + +The GIL does *not* need to be held when calling these functions; they supply their own locking. Note that :file:`Python.h` does not include the declaration of the TLS APIs, you need to include :file:`pythread.h` to use thread-local storage. .. note:: - None of the API functions does memory management on behalf of the + None of these API functions handle memory management on behalf of the :c:type:`void\*` values. You need to allocate and deallocate them yourself. If the :c:type:`void\*` values happen to be :c:type:`PyObject\*`, these functions don't do refcount operations on them either. - -.. _thread-local-storage-api: - -Thread Local Storage (TLS) API ------------------------------- - -.. deprecated:: 3.7 - This API is superseded by - :ref:`Thread Specific Storage (TSS) API `. - -.. note:: - No support for platforms where the native TLS key is defined in a way that - cannot be safely cast to ``int``. On such platforms, - :c:func:`PyThread_create_key` will return immediately with a failure status, - and the other TLS functions will all be no-ops on such platforms. - -There is no description because prevents us from use of this API. - - -.. c:function:: int PyThread_create_key() -.. c:function:: void PyThread_delete_key(int key) -.. c:function:: int PyThread_set_key_value(int key, void *value) -.. c:function:: void* PyThread_get_key_value(int key) -.. c:function:: void PyThread_delete_key_value(int key) -.. c:function:: void PyThread_ReInitTLS() - - .. _thread-specific-storage-api: Thread Specific Storage (TSS) API @@ -1281,7 +1253,7 @@ CPython interpreter. This API uses a new type :c:type:`Py_tss_t` instead of Dynamic Allocation ~~~~~~~~~~~~~~~~~~ -Dynamic allocation of the :c:type:`Py_tss_t`, particularly in extension modules +Dynamic allocation of the :c:type:`Py_tss_t`, required in extension modules built with :ref:`Py_LIMITED_API `, where static allocation of this type is not possible due to its implementation being opaque at build time. @@ -1295,14 +1267,14 @@ is not possible due to its implementation being opaque at build time. .. c:function:: void PyThread_tss_free(Py_tss_t *key) - Free a TSS key value allocated by :c:func:`PyThread_tss_alloc`, or does - no-op if the value pointed to by the *key* argument is *NULL*. In the case - of freeing, this behavior involves calling :c:func:`PyThread_tss_delete` - preventively. + Free the given *key* allocated by :c:func:`PyThread_tss_alloc`, after + first calling :c:func:`PyThread_tss_delete` to ensure any associated + thread locals have been unassigned. This is a no-op if the *key* + argument is `NULL`. - .. warning:: + .. note:: A freed key becomes a dangling pointer, you should reset the key to - *NULL*. + `NULL`. Methods @@ -1322,10 +1294,10 @@ undefined if the given :c:type:`Py_tss_t` has not been initialized by .. c:function:: int PyThread_tss_create(Py_tss_t *key) - Return a zero value on success an initialization of a TSS key. The behavior + Return a zero value on successful initialization of a TSS key. The behavior is undefined if the value pointed to by the *key* argument is not initialized by :c:macro:`Py_tss_NEEDS_INIT`. This function can be called - repeatedly on the same key--calling it on an already initialized key is a + repeatedly on the same key -- calling it on an already initialized key is a no-op and immediately returns success. @@ -1334,15 +1306,15 @@ undefined if the given :c:type:`Py_tss_t` has not been initialized by Destroy a TSS key to forget the values associated with the key across all threads, and change the key's initialization state to uninitialized. A destroyed key is able to be initialized again by - :c:func:`PyThread_tss_create`. This function can be called repeatedly on - the same key--calling it on an already destroyed key is a no-op. + :c:func:`PyThread_tss_create`. This function can be called repeatedly on + the same key -- calling it on an already destroyed key is a no-op. .. c:function:: int PyThread_tss_set(Py_tss_t *key, void *value) - Return a zero value on success associated a :c:type:`void\*` value with a - TSS key in the current thread. Each thread has a distinct mapping of the - key to a :c:type:`void\*` value. + Return a zero value to indicate successfully associating a :c:type:`void\*` + value with a TSS key in the current thread. Each thread has a distinct + mapping of the key to a :c:type:`void\*` value. .. c:function:: void* PyThread_tss_get(Py_tss_t *key) @@ -1350,3 +1322,30 @@ undefined if the given :c:type:`Py_tss_t` has not been initialized by Return the :c:type:`void\*` value associated with a TSS key in the current thread. This returns *NULL* if no value is associated with the key in the current thread. + + +.. _thread-local-storage-api: + +Thread Local Storage (TLS) API +------------------------------ + +.. deprecated:: 3.7 + This API is superseded by + :ref:`Thread Specific Storage (TSS) API `. + +.. note:: + This version of the API does not support platforms where the native TLS key + is defined in a way that cannot be safely cast to ``int``. On such platforms, + :c:func:`PyThread_create_key` will return immediately with a failure status, + and the other TLS functions will all be no-ops on such platforms. + +Due to the compatibility problem noted above, this version of the API should not +be used in new code. + +.. c:function:: int PyThread_create_key() +.. c:function:: void PyThread_delete_key(int key) +.. c:function:: int PyThread_set_key_value(int key, void *value) +.. c:function:: void* PyThread_get_key_value(int key) +.. c:function:: void PyThread_delete_key_value(int key) +.. c:function:: void PyThread_ReInitTLS() +