TLS thread local storage refresh

Signed-off-by: Daniel A. Steffen <dsteffen@apple.com>

Author: frankeh

configure.ac

@@ -108,6 +108,27 @@ AC_ARG_WITH([swift-toolchain],
 AM_CONDITIONAL(HAVE_SWIFT, $have_swift)
 AC_SUBST([SWIFTC])
 
+#
+# Enable __thread based TSD on platforms where it is efficient
+# Allow override based on command line argument to configure
+#
+AC_CANONICAL_TARGET
+AC_ARG_ENABLE([thread-local-storage],
+  [AS_HELP_STRING([--enable-thread-local-storage],
+    [Enable usage of thread local storage via __thread])],,
+  [case $target_os in
+      linux*)
+        enable_thread_local_storage=yes
+	    ;;
+      *)
+        enable_thread_local_storage=no
+   esac]
+)
+AS_IF([test "x$enable_thread_local_storage" = "xyes"],
+  [AC_DEFINE(DISPATCH_USE_THREAD_LOCAL_STORAGE, 1,
+    [Enable usage of thread local storage via __thread])]
+)
+
 AC_USE_SYSTEM_EXTENSIONS
 AM_INIT_AUTOMAKE([foreign no-dependencies subdir-objects])
 LT_INIT([disable-static])

src/init.c

@@ -67,7 +67,10 @@ void *(*_dispatch_begin_NSAutoReleasePool)(void);
 void (*_dispatch_end_NSAutoReleasePool)(void *);
 #endif
 
-#if !DISPATCH_USE_DIRECT_TSD
+#if DISPATCH_USE_THREAD_LOCAL_STORAGE
+__thread struct dispatch_tsd __dispatch_tsd;
+pthread_key_t __dispatch_tsd_key;
+#elif !DISPATCH_USE_DIRECT_TSD
 pthread_key_t dispatch_queue_key;
 pthread_key_t dispatch_frame_key;
 pthread_key_t dispatch_cache_key;
@@ -82,7 +85,7 @@ pthread_key_t dispatch_bcounter_key;
 pthread_key_t dispatch_sema4_key;
 pthread_key_t dispatch_voucher_key;
 pthread_key_t dispatch_deferred_items_key;
-#endif // !DISPATCH_USE_DIRECT_TSD
+#endif // !DISPATCH_USE_DIRECT_TSD && !DISPATCH_USE_THREAD_LOCAL_STORAGE
 
 #if VOUCHER_USE_MACH_VOUCHER
 dispatch_once_t _voucher_task_mach_voucher_pred;
@@ -184,10 +187,6 @@ const struct dispatch_tsd_indexes_s dispatch_tsd_indexes = {
 	.dti_voucher_index = dispatch_voucher_key,
 	.dti_qos_class_index = dispatch_priority_key,
 };
-#else // DISPATCH_USE_DIRECT_TSD
-#ifndef __LINUX_PORT_HDD__
-#error Not implemented on this platform
-#endif
 #endif // DISPATCH_USE_DIRECT_TSD
 
 // 6618342 Contact the team that owns the Instrument DTrace probe before

src/queue.c

@@ -894,6 +894,9 @@ libdispatch_init(void)
 #endif
 #endif
 
+#if DISPATCH_USE_THREAD_LOCAL_STORAGE
+	_dispatch_thread_key_create(&__dispatch_tsd_key, _libdispatch_tsd_cleanup);
+#else
 	_dispatch_thread_key_create(&dispatch_queue_key, _dispatch_queue_cleanup);
 	_dispatch_thread_key_create(&dispatch_deferred_items_key,
 			_dispatch_deferred_items_cleanup);
@@ -913,6 +916,7 @@ libdispatch_init(void)
 				_dispatch_thread_semaphore_dispose);
 	}
 #endif
+#endif
 
 #if DISPATCH_USE_RESOLVERS // rdar://problem/8541707
 	_dispatch_main_q.do_targetq = &_dispatch_root_queues[
@@ -967,6 +971,59 @@ _dispatch_get_mach_host_port(void)
 }
 #endif
 
+#if DISPATCH_USE_THREAD_LOCAL_STORAGE
+#include <unistd.h>
+#include <sys/syscall.h>
+
+#ifdef SYS_gettid
+DISPATCH_ALWAYS_INLINE
+static inline pid_t
+gettid(void)
+{
+	return (pid_t) syscall(SYS_gettid);
+}
+#else
+#error "SYS_gettid unavailable on this system"
+#endif
+
+#define _tsd_call_cleanup(k, f)  do { \
+		if ((f) && tsd->k) ((void(*)(void*))(f))(tsd->k); \
+    } while (0)
+
+void
+_libdispatch_tsd_cleanup(void *ctx)
+{
+	struct dispatch_tsd *tsd = (struct dispatch_tsd*) ctx;
+
+	_tsd_call_cleanup(dispatch_queue_key, _dispatch_queue_cleanup);
+	_tsd_call_cleanup(dispatch_frame_key, _dispatch_frame_cleanup);
+	_tsd_call_cleanup(dispatch_cache_key, _dispatch_cache_cleanup);
+	_tsd_call_cleanup(dispatch_context_key, _dispatch_context_cleanup);
+	_tsd_call_cleanup(dispatch_pthread_root_queue_observer_hooks_key,
+			NULL);
+	_tsd_call_cleanup(dispatch_defaultpriority_key, NULL);
+#if DISPATCH_PERF_MON && !DISPATCH_INTROSPECTION
+	_tsd_call_cleanup(dispatch_bcounter_key, NULL);
+#endif
+#if DISPATCH_LOCK_USE_SEMAPHORE_FALLBACK
+	_tsd_call_cleanup(dispatch_sema4_key, _dispatch_thread_semaphore_dispose);
+#endif
+	_tsd_call_cleanup(dispatch_priority_key, NULL);
+	_tsd_call_cleanup(dispatch_voucher_key, _voucher_thread_cleanup);
+	_tsd_call_cleanup(dispatch_deferred_items_key,
+			_dispatch_deferred_items_cleanup);
+	tsd->tid = 0;
+}
+
+DISPATCH_NOINLINE
+void
+libdispatch_tsd_init(void)
+{
+	pthread_setspecific(__dispatch_tsd_key, &__dispatch_tsd);
+	__dispatch_tsd.tid = gettid();
+}
+#endif
+
 DISPATCH_EXPORT DISPATCH_NOTHROW
 void
 dispatch_atfork_child(void)

src/shims/linux_stubs.c

@@ -57,3 +57,4 @@ unsigned short dispatch_callout__return_semaphore;
 unsigned short dispatch_queue__push_semaphore;
 void (*_dispatch_block_special_invoke)(void*);
 struct dispatch_queue_attr_s _dispatch_queue_attr_concurrent;
+

src/shims/lock.h

@@ -98,7 +98,8 @@ typedef pid_t dispatch_lock_owner;
 #define DLOCK_OWNER_MASK			((dispatch_lock)FUTEX_TID_MASK)
 #define DLOCK_WAITERS_BIT			((dispatch_lock)FUTEX_WAITERS)
 #define DLOCK_FAILED_TRYLOCK_BIT	((dispatch_lock)FUTEX_OWNER_DIED)
-#define _dispatch_tid_self()		syscall(SYS_gettid) /* FIXME: should be cached in TSD instead of doing syscall each time */
+#define _dispatch_tid_self() \
+		((dispatch_lock_owner)(_dispatch_get_tsd_base()->tid))
 
 DISPATCH_ALWAYS_INLINE
 static inline bool

src/shims/tsd.h

@@ -92,10 +92,76 @@ _dispatch_thread_key_create(const unsigned long *k, void (*d)(void *))
 	if (!*k || !d) return;
 	dispatch_assert_zero(pthread_key_init_np((int)*k, d));
 }
+#elif DISPATCH_USE_THREAD_LOCAL_STORAGE
+
+DISPATCH_TSD_INLINE
+static inline void
+_dispatch_thread_key_create(pthread_key_t *k, void (*d)(void *))
+{
+	dispatch_assert_zero(pthread_key_create(k, d));
+}
+
+struct dispatch_tsd {
+	pid_t tid;
+	void *dispatch_queue_key;
+	void *dispatch_frame_key;
+	void *dispatch_cache_key;
+	void *dispatch_context_key;
+	void *dispatch_pthread_root_queue_observer_hooks_key;
+	void *dispatch_defaultpriority_key;
+#if DISPATCH_INTROSPECTION
+	void *dispatch_introspection_key;
+#elif DISPATCH_PERF_MON
+	void *dispatch_bcounter_key;
+#endif
+#if DISPATCH_LOCK_USE_SEMAPHORE_FALLBACK
+	void *dispatch_sema4_key;
+#endif
+	void *dispatch_priority_key;
+	void *dispatch_voucher_key;
+	void *dispatch_deferred_items_key;
+};
+
+extern __thread struct dispatch_tsd __dispatch_tsd;
+extern pthread_key_t __dispatch_tsd_key;
+extern void libdispatch_tsd_init(void);
+extern void _libdispatch_tsd_cleanup(void *ctx);
+
+DISPATCH_ALWAYS_INLINE
+static inline struct dispatch_tsd *
+_dispatch_get_tsd_base(void)
+{
+	if (unlikely(__dispatch_tsd.tid == 0)) {
+		libdispatch_tsd_init();
+	}
+	OS_COMPILER_CAN_ASSUME(__dispatch_tsd.tid != 0);
+	return &__dispatch_tsd;
+}
+
+#define _dispatch_thread_getspecific(key) \
+	(_dispatch_get_tsd_base()->key)
+#define _dispatch_thread_setspecific(key, value) \
+	(void)(_dispatch_get_tsd_base()->key = (value))
+
+#define _dispatch_thread_getspecific_pair(k1, p1, k2, p2) \
+	( *(p1) = _dispatch_thread_getspecific(k1), \
+	  *(p2) = _dispatch_thread_getspecific(k2) )
+
+#define _dispatch_thread_getspecific_packed_pair(k1, k2, p) \
+	( (p)[0] = _dispatch_thread_getspecific(k1), \
+	  (p)[1] = _dispatch_thread_getspecific(k2) )
+
+#define _dispatch_thread_setspecific_pair(k1, p1, k2, p2) \
+	( _dispatch_thread_setspecific(k1,p1), \
+	  _dispatch_thread_setspecific(k2,p2) )
+
+#define _dispatch_thread_setspecific_packed_pair(k1, k2, p) \
+	( _dispatch_thread_setspecific(k1,(p)[0]), \
+	  _dispatch_thread_setspecific(k2,(p)[1]) )
+
 #else
 extern pthread_key_t dispatch_queue_key;
 extern pthread_key_t dispatch_frame_key;
-extern pthread_key_t dispatch_sema4_key;
 extern pthread_key_t dispatch_cache_key;
 extern pthread_key_t dispatch_context_key;
 extern pthread_key_t dispatch_pthread_root_queue_observer_hooks_key;
@@ -105,7 +171,8 @@ extern pthread_key_t dispatch_introspection_key;
 #elif DISPATCH_PERF_MON
 extern pthread_key_t dispatch_bcounter_key;
 #endif
-extern pthread_key_t dispatch_cache_key;
+extern pthread_key_t dispatch_sema4_key;
+extern pthread_key_t dispatch_priority_key;
 extern pthread_key_t dispatch_voucher_key;
 extern pthread_key_t dispatch_deferred_items_key;
 
@@ -117,6 +184,7 @@ _dispatch_thread_key_create(pthread_key_t *k, void (*d)(void *))
 }
 #endif
 
+#ifndef DISPATCH_USE_THREAD_LOCAL_STORAGE
 DISPATCH_TSD_INLINE
 static inline void
 _dispatch_thread_setspecific(pthread_key_t k, void *v)
@@ -210,6 +278,7 @@ _dispatch_thread_setspecific_packed_pair(pthread_key_t k1, pthread_key_t k2,
 	_dispatch_thread_setspecific(k1, p[0]);
 	_dispatch_thread_setspecific(k2, p[1]);
 }
+#endif 
 
 #if TARGET_OS_WIN32
 #define _dispatch_thread_self() ((uintptr_t)GetCurrentThreadId())
@@ -228,6 +297,8 @@ _dispatch_thread_setspecific_packed_pair(pthread_key_t k1, pthread_key_t k2,
 #if DISPATCH_USE_DIRECT_TSD
 #define _dispatch_thread_port() ((mach_port_t)(uintptr_t)\
 		_dispatch_thread_getspecific(_PTHREAD_TSD_SLOT_MACH_THREAD_SELF))
+#elif DISPATCH_USE_THREAD_LOCAL_STORAGE
+#define _dispatch_thread_port() ((mach_port_t)(_dispatch_get_tsd_base()->tid))
 #else
 #define _dispatch_thread_port() pthread_mach_thread_np(_dispatch_thread_self())
 #endif