From 1c4fec1d9171b8d27e1a41c6bc605bba7c4ce42a Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 23 Apr 2026 16:55:38 +0200 Subject: [PATCH 1/8] Initial improvements --- .../CompilerServices/AsyncHelpers.CoreCLR.cs | 151 ++++++++++++------ src/coreclr/vm/asyncthunks.cpp | 18 ++- src/coreclr/vm/corelib.h | 5 +- 3 files changed, 122 insertions(+), 52 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs index 0eeea7e127dfd0..2023658c0ab00c 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs @@ -22,31 +22,29 @@ internal struct ExecutionAndSyncBlockStore // so that they won't "leak" out of the first await. public ExecutionContext? _previousExecutionCtx; public SynchronizationContext? _previousSyncCtx; - public Thread _thread; - public void Push() + public void Push(Thread thread) { - _thread = Thread.CurrentThread; // Here we get the execution context for synchronous restoring, // not for flowing across suspension to potentially another thread. // Therefore we do not need to worry about IsFlowSuppressed - _previousExecutionCtx = _thread._executionContext; - _previousSyncCtx = _thread._synchronizationContext; + _previousExecutionCtx = thread._executionContext; + _previousSyncCtx = thread._synchronizationContext; } - public void Pop() + public void Pop(Thread thread) { // The common case is that these have not changed, so avoid the cost of a write barrier if not needed. - if (_previousSyncCtx != _thread._synchronizationContext) + if (_previousSyncCtx != thread._synchronizationContext) { // Restore changed SynchronizationContext back to previous - _thread._synchronizationContext = _previousSyncCtx; + thread._synchronizationContext = _previousSyncCtx; } - ExecutionContext? currentExecutionCtx = _thread._executionContext; + ExecutionContext? currentExecutionCtx = thread._executionContext; if (_previousExecutionCtx != currentExecutionCtx) { - ExecutionContext.RestoreChangedContextToThread(_thread, _previousExecutionCtx, currentExecutionCtx); + ExecutionContext.RestoreChangedContextToThread(thread, _previousExecutionCtx, currentExecutionCtx); } } } @@ -208,10 +206,14 @@ public static partial class AsyncHelpers // Used during suspensions to hold the continuation chain and on what we are waiting. // Methods like FinalizeTaskReturningThunk will unlink the state and wrap into a Task. - private struct RuntimeAsyncAwaitState + private unsafe struct RuntimeAsyncAwaitState { public Continuation? SentinelContinuation; + // We cache the other TLS members here to avoid unnecessary repeated TLS lookups. + public Thread CurrentThread; + public AsyncDispatcherInfo** CurrentDispatcherInfo; + // The following are the possible introducers of asynchrony into a chain of awaits. // In other words - when we build a chain of continuations it would be logicaly attached // to one of these notifiers. @@ -225,12 +227,44 @@ private struct RuntimeAsyncAwaitState public void CaptureContexts() { - Thread curThread = Thread.CurrentThreadAssumedInitialized; + // CaptureContext is called from leaf await helpers. We either just started a runtime async chain + // (from a thunk), or we came from DispatchContinuations (on resumption). + // Both cases have already initialized Thread.t_currentThread. + Thread curThread = CurrentThread ??= Thread.CurrentThreadAssumedInitialized; + Debug.Assert(curThread != null); + // Here we get the execution context for presenting to the notifier, // not for flowing across suspension to potentially another thread. // Therefore we do not need to worry about IsFlowSuppressed - ExecutionContext = curThread._executionContext; - SynchronizationContext = curThread._synchronizationContext; + Debug.Assert(ExecutionContext is null && SynchronizationContext is null); + ExecutionContext? execContext = curThread._executionContext; + if (execContext != null) + { + ExecutionContext = execContext; + } + + SynchronizationContext? syncContext = curThread._synchronizationContext; + if (syncContext != null) + { + SynchronizationContext = syncContext; + } + } + + public Thread GetOrInitCurrentThread() + { + return CurrentThread ??= Thread.CurrentThread; + } + + public AsyncDispatcherInfo** GetDispatcherInfoPointer() + { + if (CurrentDispatcherInfo != null) + return CurrentDispatcherInfo; + + // This relies on coreclr and NAOT runtimes always storing ThreadStatic pointer fields in a pinned fashion. + fixed (AsyncDispatcherInfo** pCurrentDispatcherInfo = &AsyncDispatcherInfo.t_current) + { + return CurrentDispatcherInfo = pCurrentDispatcherInfo; + } } } @@ -343,13 +377,19 @@ void ITaskCompletionAction.Invoke(Task completingTask) bool ITaskCompletionAction.InvokeMayRunArbitraryCode => true; - private Action GetContinuationAction() => (Action)m_action!; + private Action GetContinuationAction() + { + object? action = m_action; + Debug.Assert(action is Action); + return Unsafe.As(action); + } private Continuation MoveContinuationState() { - Continuation continuation = (Continuation)m_stateObject!; + object? stateObject = m_stateObject; + Debug.Assert(stateObject is Continuation); m_stateObject = null; - return continuation; + return Unsafe.As(stateObject); } private void SetContinuationState(Continuation value) @@ -362,7 +402,20 @@ internal bool HandleSuspended() { ref RuntimeAsyncAwaitState state = ref t_runtimeAsyncAwaitState; - RestoreContextsOnSuspension(false, state.ExecutionContext, state.SynchronizationContext); + Thread currentThread = state.CurrentThread; + Debug.Assert(currentThread != null); + + ExecutionContext? suspendingExecutionContext = state.ExecutionContext; + SynchronizationContext? suspendingSyncContext = state.SynchronizationContext; + if (suspendingExecutionContext != currentThread._executionContext) + { + currentThread._executionContext = suspendingExecutionContext; + } + + if (suspendingSyncContext != currentThread._synchronizationContext) + { + currentThread._synchronizationContext = suspendingSyncContext; + } ICriticalNotifyCompletion? critNotifier = state.CriticalNotifier; INotifyCompletion? notifier = state.Notifier; @@ -500,13 +553,18 @@ private unsafe void DispatchContinuations() } } + ref RuntimeAsyncAwaitState awaitState = ref t_runtimeAsyncAwaitState; + Thread currentThread = awaitState.GetOrInitCurrentThread(); + ExecutionAndSyncBlockStore contexts = default; - contexts.Push(); + contexts.Push(currentThread); + + AsyncDispatcherInfo** pDispatcherInfo = awaitState.GetDispatcherInfoPointer(); AsyncDispatcherInfo asyncDispatcherInfo; - asyncDispatcherInfo.Next = AsyncDispatcherInfo.t_current; + asyncDispatcherInfo.Next = *pDispatcherInfo; asyncDispatcherInfo.NextContinuation = MoveContinuationState(); - AsyncDispatcherInfo.t_current = &asyncDispatcherInfo; + *pDispatcherInfo = &asyncDispatcherInfo; while (true) { @@ -526,8 +584,8 @@ private unsafe void DispatchContinuations() newContinuation.Next = nextContinuation; HandleSuspended(); - contexts.Pop(); - AsyncDispatcherInfo.t_current = asyncDispatcherInfo.Next; + contexts.Pop(currentThread); + *pDispatcherInfo = asyncDispatcherInfo.Next; return; } } @@ -542,9 +600,8 @@ private unsafe void DispatchContinuations() TrySetCanceled(oce.CancellationToken, oce) : TrySetException(ex); - contexts.Pop(); - - AsyncDispatcherInfo.t_current = asyncDispatcherInfo.Next; + contexts.Pop(currentThread); + *pDispatcherInfo = asyncDispatcherInfo.Next; if (!successfullySet) { @@ -562,9 +619,8 @@ private unsafe void DispatchContinuations() { bool successfullySet = TrySetResult(m_result); - contexts.Pop(); - - AsyncDispatcherInfo.t_current = asyncDispatcherInfo.Next; + contexts.Pop(currentThread); + *pDispatcherInfo = asyncDispatcherInfo.Next; if (!successfullySet) { @@ -576,8 +632,8 @@ private unsafe void DispatchContinuations() if (QueueContinuationFollowUpActionIfNecessary(asyncDispatcherInfo.NextContinuation)) { - contexts.Pop(); - AsyncDispatcherInfo.t_current = asyncDispatcherInfo.Next; + contexts.Pop(currentThread); + *pDispatcherInfo = asyncDispatcherInfo.Next; return; } @@ -585,8 +641,8 @@ private unsafe void DispatchContinuations() { SetContinuationState(asyncDispatcherInfo.NextContinuation); - contexts.Pop(); - AsyncDispatcherInfo.t_current = asyncDispatcherInfo.Next; + contexts.Pop(currentThread); + *pDispatcherInfo = asyncDispatcherInfo.Next; InstrumentedDispatchContinuations(AsyncInstrumentation.ActiveFlags); return; @@ -597,13 +653,18 @@ private unsafe void DispatchContinuations() [StackTraceHidden] private unsafe void InstrumentedDispatchContinuations(AsyncInstrumentation.Flags flags) { + ref RuntimeAsyncAwaitState awaitState = ref t_runtimeAsyncAwaitState; + Thread currentThread = awaitState.GetOrInitCurrentThread(); + ExecutionAndSyncBlockStore contexts = default; - contexts.Push(); + contexts.Push(currentThread); + + AsyncDispatcherInfo** pDispatcherInfo = awaitState.GetDispatcherInfoPointer(); AsyncDispatcherInfo asyncDispatcherInfo; - asyncDispatcherInfo.Next = AsyncDispatcherInfo.t_current; + asyncDispatcherInfo.Next = *pDispatcherInfo; asyncDispatcherInfo.NextContinuation = MoveContinuationState(); - AsyncDispatcherInfo.t_current = &asyncDispatcherInfo; + *pDispatcherInfo = &asyncDispatcherInfo; RuntimeAsyncInstrumentationHelpers.ResumeRuntimeAsyncContext(this, ref asyncDispatcherInfo, flags); @@ -627,8 +688,8 @@ private unsafe void InstrumentedDispatchContinuations(AsyncInstrumentation.Flags RuntimeAsyncInstrumentationHelpers.SuspendRuntimeAsyncContext(flags, curContinuation, newContinuation); InstrumentedHandleSuspended(flags, newContinuation); - contexts.Pop(); - AsyncDispatcherInfo.t_current = asyncDispatcherInfo.Next; + contexts.Pop(currentThread); + *pDispatcherInfo = asyncDispatcherInfo.Next; return; } @@ -647,9 +708,8 @@ private unsafe void InstrumentedDispatchContinuations(AsyncInstrumentation.Flags TrySetCanceled(oce.CancellationToken, oce) : TrySetException(ex); - contexts.Pop(); - - AsyncDispatcherInfo.t_current = asyncDispatcherInfo.Next; + contexts.Pop(currentThread); + *pDispatcherInfo = asyncDispatcherInfo.Next; if (!successfullySet) { @@ -671,9 +731,8 @@ private unsafe void InstrumentedDispatchContinuations(AsyncInstrumentation.Flags bool successfullySet = TrySetResult(m_result); - contexts.Pop(); - - AsyncDispatcherInfo.t_current = asyncDispatcherInfo.Next; + contexts.Pop(currentThread); + *pDispatcherInfo = asyncDispatcherInfo.Next; if (!successfullySet) { @@ -687,8 +746,8 @@ private unsafe void InstrumentedDispatchContinuations(AsyncInstrumentation.Flags { RuntimeAsyncInstrumentationHelpers.SuspendRuntimeAsyncContext(ref asyncDispatcherInfo, flags, curContinuation); - contexts.Pop(); - AsyncDispatcherInfo.t_current = asyncDispatcherInfo.Next; + contexts.Pop(currentThread); + *pDispatcherInfo = asyncDispatcherInfo.Next; return; } diff --git a/src/coreclr/vm/asyncthunks.cpp b/src/coreclr/vm/asyncthunks.cpp index 3fa631527cf29c..f26f8aa2966bd4 100644 --- a/src/coreclr/vm/asyncthunks.cpp +++ b/src/coreclr/vm/asyncthunks.cpp @@ -93,8 +93,9 @@ void MethodDesc::EmitTaskReturningThunk(MethodDesc* pAsyncCallVariant, MetaSig& // Emits roughly the following code: // + // Thread currentThread = Thread.CurrentThread; // ExecutionAndSyncBlockStore store = default; - // store.Push(); + // store.Push(currentThread); // try // { // try @@ -113,7 +114,7 @@ void MethodDesc::EmitTaskReturningThunk(MethodDesc* pAsyncCallVariant, MetaSig& // } // finally // { - // store.Pop(); + // store.Pop(currentThread); // } ILCodeStream* pCode = pSL->NewCodeStream(ILStubLinker::kDispatch); @@ -132,6 +133,10 @@ void MethodDesc::EmitTaskReturningThunk(MethodDesc* pAsyncCallVariant, MetaSig& LocalDesc returnLocalDesc(thTaskRet); DWORD returnTaskLocal = pCode->NewLocal(returnLocalDesc); + + LocalDesc threadLocalDesc(CoreLibBinder::GetClass(CLASS__THREAD)); + DWORD threadLocal = pCode->NewLocal(threadLocalDesc); + LocalDesc executionAndSyncBlockStoreLocalDesc(CoreLibBinder::GetClass(CLASS__EXECUTIONANDSYNCBLOCKSTORE)); DWORD executionAndSyncBlockStoreLocal = pCode->NewLocal(executionAndSyncBlockStoreLocalDesc); @@ -139,8 +144,12 @@ void MethodDesc::EmitTaskReturningThunk(MethodDesc* pAsyncCallVariant, MetaSig& ILCodeLabel* suspendedLabel = pCode->NewCodeLabel(); ILCodeLabel* finishedLabel = pCode->NewCodeLabel(); + pCode->EmitCALL(METHOD__THREAD__GET_CURRENTTHREAD, 0, 1); + pCode->EmitSTLOC(threadLocal); + pCode->EmitLDLOCA(executionAndSyncBlockStoreLocal); - pCode->EmitCALL(pCode->GetToken(CoreLibBinder::GetMethod(METHOD__EXECUTIONANDSYNCBLOCKSTORE__PUSH)), 1, 0); + pCode->EmitLDLOC(threadLocal); + pCode->EmitCALL(METHOD__EXECUTIONANDSYNCBLOCKSTORE__PUSH, 2, 0); { pCode->BeginTryBlock(); @@ -262,7 +271,8 @@ void MethodDesc::EmitTaskReturningThunk(MethodDesc* pAsyncCallVariant, MetaSig& { pCode->BeginFinallyBlock(); pCode->EmitLDLOCA(executionAndSyncBlockStoreLocal); - pCode->EmitCALL(pCode->GetToken(CoreLibBinder::GetMethod(METHOD__EXECUTIONANDSYNCBLOCKSTORE__POP)), 1, 0); + pCode->EmitLDLOC(threadLocal); + pCode->EmitCALL(METHOD__EXECUTIONANDSYNCBLOCKSTORE__POP, 2, 0); pCode->EmitENDFINALLY(); pCode->EndFinallyBlock(); } diff --git a/src/coreclr/vm/corelib.h b/src/coreclr/vm/corelib.h index f75fdcce0c832d..59ef47bd8303d1 100644 --- a/src/coreclr/vm/corelib.h +++ b/src/coreclr/vm/corelib.h @@ -978,11 +978,12 @@ DEFINE_FIELD(EXECUTIONCONTEXT, DEFAULT_FLOW_SUPPRESSED, DefaultFlowSu DEFINE_CLASS(DIRECTONTHREADLOCALDATA, Threading, Thread+DirectOnThreadLocalData) DEFINE_CLASS(THREAD, Threading, Thread) -DEFINE_METHOD(THREAD, START_CALLBACK, StartCallback, SM_PtrThread_RetVoid) +DEFINE_METHOD(THREAD, START_CALLBACK, StartCallback, SM_PtrThread_RetVoid) DEFINE_METHOD(THREAD, POLLGC, PollGC, NoSig) DEFINE_METHOD(THREAD, ON_THREAD_EXITING, OnThreadExited, SM_PtrThread_PtrException_RetVoid) +DEFINE_METHOD(THREAD, GET_CURRENTTHREAD, get_CurrentThread, NoSig) #ifdef FOR_ILLINK -DEFINE_METHOD(THREAD, CTOR, .ctor, IM_RetVoid) +DEFINE_METHOD(THREAD, CTOR, .ctor, IM_RetVoid) #endif // FOR_ILLINK #ifdef FEATURE_OBJCMARSHAL From 12d2817d2751acdc80d4ef3414922eaafdf7475e Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 23 Apr 2026 20:26:17 +0200 Subject: [PATCH 2/8] Optimize runtime async suspend/resume machinery Several optimizations around suspension/resumption: - Reduce number of TLS accesses by storing `Thread.CurrentThread` and `&AsyncDispatcherInfo.t_current` inside `RuntimeAsyncAwaitState`, and only accessing `RuntimeAsyncAwaitState` - Remove a number of write barriers by moving TLS object fields into a `ref struct`. Allocate this ref struct on the stack in the two places that initiate runtime async chains: task-returning thunks and `DispatchContinuations`. Keep a pointer to this in the TLS. - Use `Unsafe` in a couple of places to avoid unnecessary cast checks on the hot path For a suspension heavy benchmark this improves performance by around 25%. --- .../CompilerServices/AsyncHelpers.CoreCLR.cs | 234 ++++++++---------- .../Common/TypeSystem/IL/Stubs/AsyncThunks.cs | 40 +-- src/coreclr/vm/asyncthunks.cpp | 36 +-- src/coreclr/vm/corelib.h | 19 +- src/coreclr/vm/metasig.h | 6 +- .../Runtime/CompilerServices/AsyncHelpers.cs | 18 +- 6 files changed, 171 insertions(+), 182 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs index 2023658c0ab00c..a82e93d5dc3ba8 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs @@ -15,40 +15,6 @@ namespace System.Runtime.CompilerServices { - internal struct ExecutionAndSyncBlockStore - { - // Store current ExecutionContext and SynchronizationContext as "previousXxx". - // This allows us to restore them and undo any Context changes made in stateMachine.MoveNext - // so that they won't "leak" out of the first await. - public ExecutionContext? _previousExecutionCtx; - public SynchronizationContext? _previousSyncCtx; - - public void Push(Thread thread) - { - // Here we get the execution context for synchronous restoring, - // not for flowing across suspension to potentially another thread. - // Therefore we do not need to worry about IsFlowSuppressed - _previousExecutionCtx = thread._executionContext; - _previousSyncCtx = thread._synchronizationContext; - } - - public void Pop(Thread thread) - { - // The common case is that these have not changed, so avoid the cost of a write barrier if not needed. - if (_previousSyncCtx != thread._synchronizationContext) - { - // Restore changed SynchronizationContext back to previous - thread._synchronizationContext = _previousSyncCtx; - } - - ExecutionContext? currentExecutionCtx = thread._executionContext; - if (_previousExecutionCtx != currentExecutionCtx) - { - ExecutionContext.RestoreChangedContextToThread(thread, _previousExecutionCtx, currentExecutionCtx); - } - } - } - [Flags] // Keep in sync with CORINFO_CONTINUATION_FLAGS internal enum ContinuationFlags @@ -204,16 +170,8 @@ public static partial class AsyncHelpers [Intrinsic] private static void TailAwait() => throw new UnreachableException(); - // Used during suspensions to hold the continuation chain and on what we are waiting. - // Methods like FinalizeTaskReturningThunk will unlink the state and wrap into a Task. - private unsafe struct RuntimeAsyncAwaitState + private ref struct RuntimeAsyncStackState { - public Continuation? SentinelContinuation; - - // We cache the other TLS members here to avoid unnecessary repeated TLS lookups. - public Thread CurrentThread; - public AsyncDispatcherInfo** CurrentDispatcherInfo; - // The following are the possible introducers of asynchrony into a chain of awaits. // In other words - when we build a chain of continuations it would be logicaly attached // to one of these notifiers. @@ -222,40 +180,67 @@ private unsafe struct RuntimeAsyncAwaitState public ValueTaskSourceNotifier? ValueTaskSourceNotifier; public Task? TaskNotifier; - public ExecutionContext? ExecutionContext; - public SynchronizationContext? SynchronizationContext; + // When we suspend in the leaf, the contexts are captured into these fields. + public ExecutionContext? LeafExecutionContext; + public SynchronizationContext? LeafSynchronizationContext; - public void CaptureContexts() + // When we enter the root of the async chain (either an async thunk + // or DispatchContinuations), the contexts are captured into these + // fields. + public ExecutionContext? RootExecutionContext; + public SynchronizationContext? RootSynchronizationContext; + + public void Push(Thread thread) { - // CaptureContext is called from leaf await helpers. We either just started a runtime async chain - // (from a thunk), or we came from DispatchContinuations (on resumption). - // Both cases have already initialized Thread.t_currentThread. - Thread curThread = CurrentThread ??= Thread.CurrentThreadAssumedInitialized; - Debug.Assert(curThread != null); + RootExecutionContext = thread._executionContext; + RootSynchronizationContext = thread._synchronizationContext; + } - // Here we get the execution context for presenting to the notifier, - // not for flowing across suspension to potentially another thread. - // Therefore we do not need to worry about IsFlowSuppressed - Debug.Assert(ExecutionContext is null && SynchronizationContext is null); - ExecutionContext? execContext = curThread._executionContext; - if (execContext != null) + public void Pop(Thread thread) + { + // The common case is that these have not changed, so avoid the cost of a write barrier if not needed. + if (RootSynchronizationContext != thread._synchronizationContext) { - ExecutionContext = execContext; + // Restore changed SynchronizationContext back to previous + thread._synchronizationContext = RootSynchronizationContext; } - SynchronizationContext? syncContext = curThread._synchronizationContext; - if (syncContext != null) + ExecutionContext? currentExecutionCtx = thread._executionContext; + if (RootExecutionContext != currentExecutionCtx) { - SynchronizationContext = syncContext; + ExecutionContext.RestoreChangedContextToThread(thread, RootExecutionContext, currentExecutionCtx); } } + } + + // Used during suspensions to hold the continuation chain and on what we are waiting. + // Methods like FinalizeTaskReturningThunk will unlink the state and wrap into a Task. + private unsafe struct RuntimeAsyncAwaitState + { + public Continuation? SentinelContinuation; - public Thread GetOrInitCurrentThread() + // We cache the other TLS members here to avoid unnecessary repeated TLS lookups. + public Thread CurrentThread; + public AsyncDispatcherInfo** CurrentDispatcherInfo; + + public RuntimeAsyncStackState* StackState; + + public void CaptureContexts() { - return CurrentThread ??= Thread.CurrentThread; + // CaptureContext is called from leaf await helpers. We either just started a runtime async chain + // (from a thunk), or we came from DispatchContinuations (on resumption). + // Both cases have already initialized Thread.t_currentThread. + Thread curThread = CurrentThread; + Debug.Assert(curThread != null); + Debug.Assert(StackState != null); + // Here we get the execution context for presenting to the notifier, + // not for flowing across suspension to potentially another thread. + // Therefore we do not need to worry about IsFlowSuppressed + StackState->LeafExecutionContext = curThread._executionContext; + StackState->LeafSynchronizationContext = curThread._synchronizationContext; } - public AsyncDispatcherInfo** GetDispatcherInfoPointer() + public AsyncDispatcherInfo** GetOrInitDispatcherInfoPointer() { if (CurrentDispatcherInfo != null) return CurrentDispatcherInfo; @@ -266,10 +251,24 @@ public Thread GetOrInitCurrentThread() return CurrentDispatcherInfo = pCurrentDispatcherInfo; } } + + // At the start of an async chain (task-returning thunk or DispatchContinuations) this function + // is called + public void Push(RuntimeAsyncStackState* stackState) + { + StackState = stackState; + stackState->Push(CurrentThread ??= Thread.CurrentThread); + } + + // This function is called at the end of an async chain + public void Pop() + { + StackState->Pop(CurrentThread); + } } [ThreadStatic] - private static RuntimeAsyncAwaitState t_runtimeAsyncAwaitState; + private static unsafe RuntimeAsyncAwaitState t_runtimeAsyncAwaitState; #if !NATIVEAOT [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "AsyncHelpers_AddContinuationToExInternal")] @@ -331,20 +330,18 @@ private static unsafe Continuation AllocContinuationClass(Continuation prevConti /// Task or a ValueTaskNotifier whose completion we are awaiting. [BypassReadyToRun] [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.Async)] - private static void TransparentAwait(object o) + private static unsafe void TransparentAwait(object o) { ref RuntimeAsyncAwaitState state = ref t_runtimeAsyncAwaitState; - Continuation? sentinelContinuation = state.SentinelContinuation; - if (sentinelContinuation == null) - state.SentinelContinuation = sentinelContinuation = new Continuation(); + Continuation? sentinelContinuation = state.SentinelContinuation ??= new Continuation(); if (o is Task t) { - state.TaskNotifier = t; + state.StackState->TaskNotifier = t; } else { - state.ValueTaskSourceNotifier = (ValueTaskSourceNotifier)o; + state.StackState->ValueTaskSourceNotifier = (ValueTaskSourceNotifier)o; } state.CaptureContexts(); @@ -398,15 +395,13 @@ private void SetContinuationState(Continuation value) m_stateObject = value; } - internal bool HandleSuspended() + internal unsafe bool HandleSuspended(ref RuntimeAsyncAwaitState state) { - ref RuntimeAsyncAwaitState state = ref t_runtimeAsyncAwaitState; - Thread currentThread = state.CurrentThread; Debug.Assert(currentThread != null); - ExecutionContext? suspendingExecutionContext = state.ExecutionContext; - SynchronizationContext? suspendingSyncContext = state.SynchronizationContext; + ExecutionContext? suspendingExecutionContext = state.StackState->LeafExecutionContext; + SynchronizationContext? suspendingSyncContext = state.StackState->LeafSynchronizationContext; if (suspendingExecutionContext != currentThread._executionContext) { currentThread._executionContext = suspendingExecutionContext; @@ -417,17 +412,10 @@ internal bool HandleSuspended() currentThread._synchronizationContext = suspendingSyncContext; } - ICriticalNotifyCompletion? critNotifier = state.CriticalNotifier; - INotifyCompletion? notifier = state.Notifier; - ValueTaskSourceNotifier? vtsNotifier = state.ValueTaskSourceNotifier; - Task? taskNotifier = state.TaskNotifier; - - state.CriticalNotifier = null; - state.Notifier = null; - state.ValueTaskSourceNotifier = null; - state.TaskNotifier = null; - state.ExecutionContext = null; - state.SynchronizationContext = null; + ICriticalNotifyCompletion? critNotifier = state.StackState->CriticalNotifier; + INotifyCompletion? notifier = state.StackState->Notifier; + ValueTaskSourceNotifier? vtsNotifier = state.StackState->ValueTaskSourceNotifier; + Task? taskNotifier = state.StackState->TaskNotifier; Continuation sentinelContinuation = state.SentinelContinuation!; Continuation headContinuation = sentinelContinuation.Next!; @@ -513,7 +501,7 @@ internal bool HandleSuspended() return false; } - internal void InstrumentedHandleSuspended(AsyncInstrumentation.Flags flags, Continuation? newContinuation = null) + internal void InstrumentedHandleSuspended(AsyncInstrumentation.Flags flags, ref RuntimeAsyncAwaitState state, Continuation? newContinuation = null) { if (AsyncInstrumentation.IsEnabled.AsyncDebugger(flags)) { @@ -521,7 +509,7 @@ internal void InstrumentedHandleSuspended(AsyncInstrumentation.Flags flags, Cont AsyncDebugger.HandleSuspended(nextContinuation, newContinuation); - if (!HandleSuspended()) + if (!HandleSuspended(ref state)) { AsyncDebugger.HandleSuspendedFailed(this, nextContinuation); } @@ -529,7 +517,7 @@ internal void InstrumentedHandleSuspended(AsyncInstrumentation.Flags flags, Cont return; } - HandleSuspended(); + HandleSuspended(ref state); } #pragma warning disable CA1822 // Mark members as static @@ -553,13 +541,12 @@ private unsafe void DispatchContinuations() } } - ref RuntimeAsyncAwaitState awaitState = ref t_runtimeAsyncAwaitState; - Thread currentThread = awaitState.GetOrInitCurrentThread(); + RuntimeAsyncStackState stackState = default; - ExecutionAndSyncBlockStore contexts = default; - contexts.Push(currentThread); + ref RuntimeAsyncAwaitState awaitState = ref t_runtimeAsyncAwaitState; + awaitState.Push(&stackState); - AsyncDispatcherInfo** pDispatcherInfo = awaitState.GetDispatcherInfoPointer(); + AsyncDispatcherInfo** pDispatcherInfo = awaitState.GetOrInitDispatcherInfoPointer(); AsyncDispatcherInfo asyncDispatcherInfo; asyncDispatcherInfo.Next = *pDispatcherInfo; @@ -582,9 +569,9 @@ private unsafe void DispatchContinuations() if (newContinuation != null) { newContinuation.Next = nextContinuation; - HandleSuspended(); + HandleSuspended(ref awaitState); - contexts.Pop(currentThread); + awaitState.Pop(); *pDispatcherInfo = asyncDispatcherInfo.Next; return; } @@ -600,7 +587,7 @@ private unsafe void DispatchContinuations() TrySetCanceled(oce.CancellationToken, oce) : TrySetException(ex); - contexts.Pop(currentThread); + awaitState.Pop(); *pDispatcherInfo = asyncDispatcherInfo.Next; if (!successfullySet) @@ -619,7 +606,7 @@ private unsafe void DispatchContinuations() { bool successfullySet = TrySetResult(m_result); - contexts.Pop(currentThread); + awaitState.Pop(); *pDispatcherInfo = asyncDispatcherInfo.Next; if (!successfullySet) @@ -632,7 +619,7 @@ private unsafe void DispatchContinuations() if (QueueContinuationFollowUpActionIfNecessary(asyncDispatcherInfo.NextContinuation)) { - contexts.Pop(currentThread); + awaitState.Pop(); *pDispatcherInfo = asyncDispatcherInfo.Next; return; } @@ -641,7 +628,7 @@ private unsafe void DispatchContinuations() { SetContinuationState(asyncDispatcherInfo.NextContinuation); - contexts.Pop(currentThread); + awaitState.Pop(); *pDispatcherInfo = asyncDispatcherInfo.Next; InstrumentedDispatchContinuations(AsyncInstrumentation.ActiveFlags); @@ -653,13 +640,12 @@ private unsafe void DispatchContinuations() [StackTraceHidden] private unsafe void InstrumentedDispatchContinuations(AsyncInstrumentation.Flags flags) { - ref RuntimeAsyncAwaitState awaitState = ref t_runtimeAsyncAwaitState; - Thread currentThread = awaitState.GetOrInitCurrentThread(); + RuntimeAsyncStackState stackState = default; - ExecutionAndSyncBlockStore contexts = default; - contexts.Push(currentThread); + ref RuntimeAsyncAwaitState awaitState = ref t_runtimeAsyncAwaitState; + awaitState.Push(&stackState); - AsyncDispatcherInfo** pDispatcherInfo = awaitState.GetDispatcherInfoPointer(); + AsyncDispatcherInfo** pDispatcherInfo = awaitState.GetOrInitDispatcherInfoPointer(); AsyncDispatcherInfo asyncDispatcherInfo; asyncDispatcherInfo.Next = *pDispatcherInfo; @@ -686,9 +672,9 @@ private unsafe void InstrumentedDispatchContinuations(AsyncInstrumentation.Flags { newContinuation.Next = nextContinuation; RuntimeAsyncInstrumentationHelpers.SuspendRuntimeAsyncContext(flags, curContinuation, newContinuation); - InstrumentedHandleSuspended(flags, newContinuation); + InstrumentedHandleSuspended(flags, ref awaitState, newContinuation); - contexts.Pop(currentThread); + awaitState.Pop(); *pDispatcherInfo = asyncDispatcherInfo.Next; return; } @@ -708,7 +694,7 @@ private unsafe void InstrumentedDispatchContinuations(AsyncInstrumentation.Flags TrySetCanceled(oce.CancellationToken, oce) : TrySetException(ex); - contexts.Pop(currentThread); + awaitState.Pop(); *pDispatcherInfo = asyncDispatcherInfo.Next; if (!successfullySet) @@ -731,7 +717,7 @@ private unsafe void InstrumentedDispatchContinuations(AsyncInstrumentation.Flags bool successfullySet = TrySetResult(m_result); - contexts.Pop(currentThread); + awaitState.Pop(); *pDispatcherInfo = asyncDispatcherInfo.Next; if (!successfullySet) @@ -746,7 +732,7 @@ private unsafe void InstrumentedDispatchContinuations(AsyncInstrumentation.Flags { RuntimeAsyncInstrumentationHelpers.SuspendRuntimeAsyncContext(ref asyncDispatcherInfo, flags, curContinuation); - contexts.Pop(currentThread); + awaitState.Pop(); *pDispatcherInfo = asyncDispatcherInfo.Next; return; } @@ -854,7 +840,7 @@ private bool QueueContinuationFollowUpActionIfNecessary(Continuation continuatio }; } - private static void InstrumentedFinalizeRuntimeAsyncTask(RuntimeAsyncTask task, AsyncInstrumentation.Flags flags) + private static void InstrumentedFinalizeRuntimeAsyncTask(RuntimeAsyncTask task, ref RuntimeAsyncAwaitState state, AsyncInstrumentation.Flags flags) { if (AsyncInstrumentation.IsEnabled.CreateAsyncContext(flags)) { @@ -865,54 +851,54 @@ private static void InstrumentedFinalizeRuntimeAsyncTask(RuntimeAsyncTask } } - task.InstrumentedHandleSuspended(flags); + task.InstrumentedHandleSuspended(flags, ref state); return; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void FinalizeRuntimeAsyncTask(RuntimeAsyncTask task) + private static void FinalizeRuntimeAsyncTask(ref RuntimeAsyncAwaitState state, RuntimeAsyncTask task) { if (RuntimeAsyncInstrumentationHelpers.InstrumentCheckPoint) { AsyncInstrumentation.Flags flags = AsyncInstrumentation.SyncActiveFlags(); if (flags != AsyncInstrumentation.Flags.Disabled) { - InstrumentedFinalizeRuntimeAsyncTask(task, flags); + InstrumentedFinalizeRuntimeAsyncTask(task, ref state, flags); return; } } - task.HandleSuspended(); + task.HandleSuspended(ref state); } // Change return type to RuntimeAsyncTask -- no benefit since this is used for Task returning thunks only #pragma warning disable CA1859 // When a Task-returning thunk gets a continuation result // it calls here to make a Task that awaits on the current async state. - private static Task FinalizeTaskReturningThunk() + private static Task FinalizeTaskReturningThunk(ref RuntimeAsyncAwaitState state) { RuntimeAsyncTask result = new(); - FinalizeRuntimeAsyncTask(result!); + FinalizeRuntimeAsyncTask(ref state, result!); return result; } - private static Task FinalizeTaskReturningThunk() + private static Task FinalizeTaskReturningThunk(ref RuntimeAsyncAwaitState state) { RuntimeAsyncTask result = new(); - FinalizeRuntimeAsyncTask(result!); + FinalizeRuntimeAsyncTask(ref state, result!); return result; } - private static ValueTask FinalizeValueTaskReturningThunk() + private static ValueTask FinalizeValueTaskReturningThunk(ref RuntimeAsyncAwaitState state) { // We only come to these methods in the expensive case (already // suspended), so ValueTask optimization here is not relevant. - return new ValueTask(FinalizeTaskReturningThunk()); + return new ValueTask(FinalizeTaskReturningThunk(ref state)); } - private static ValueTask FinalizeValueTaskReturningThunk() + private static ValueTask FinalizeValueTaskReturningThunk(ref RuntimeAsyncAwaitState state) { - return new ValueTask(FinalizeTaskReturningThunk()); + return new ValueTask(FinalizeTaskReturningThunk(ref state)); } private static Task TaskFromException(Exception ex) diff --git a/src/coreclr/tools/Common/TypeSystem/IL/Stubs/AsyncThunks.cs b/src/coreclr/tools/Common/TypeSystem/IL/Stubs/AsyncThunks.cs index 2d9891ef57b874..d5f367c6498b0b 100644 --- a/src/coreclr/tools/Common/TypeSystem/IL/Stubs/AsyncThunks.cs +++ b/src/coreclr/tools/Common/TypeSystem/IL/Stubs/AsyncThunks.cs @@ -38,15 +38,22 @@ public static MethodIL EmitTaskReturningThunk(MethodDesc taskReturningMethod, Me ILLocalVariable returnTaskLocal = emitter.NewLocal(returnType); - TypeDesc executionAndSyncBlockStoreType = context.SystemModule.GetKnownType("System.Runtime.CompilerServices"u8, "ExecutionAndSyncBlockStore"u8); - ILLocalVariable executionAndSyncBlockStoreLocal = emitter.NewLocal(executionAndSyncBlockStoreType); + MetadataType asyncHelpersType = context.SystemModule.GetKnownType("System.Runtime.CompilerServices"u8, "AsyncHelpers"u8); + TypeDesc stackStateType = asyncHelpersType.GetKnownNestedType("RuntimeAsyncStackState"u8); + ILLocalVariable stackStateLocal = emitter.NewLocal(stackStateType); + TypeDesc awaitStateType = asyncHelpersType.GetKnownNestedType("RuntimeAsyncAwaitState"u8); + ILLocalVariable refAwaitStateLocal = emitter.NewLocal(awaitStateType.MakeByRefType()); ILCodeLabel returnTaskLabel = emitter.NewCodeLabel(); ILCodeLabel suspendedLabel = emitter.NewCodeLabel(); ILCodeLabel finishedLabel = emitter.NewCodeLabel(); - codestream.EmitLdLoca(executionAndSyncBlockStoreLocal); - codestream.Emit(ILOpcode.call, emitter.NewToken(executionAndSyncBlockStoreType.GetKnownMethod("Push"u8, null))); + codestream.Emit(ILOpcode.ldsflda, emitter.NewToken(asyncHelpersType.GetKnownField("t_runtimeAsyncAwaitState"u8))); + codestream.EmitStLoc(refAwaitStateLocal); + + codestream.EmitLdLoc(refAwaitStateLocal); + codestream.EmitLdLoca(stackStateLocal); + codestream.Emit(ILOpcode.call, emitter.NewToken(awaitStateType.GetKnownMethod("Push"u8, null))); ILExceptionRegionBuilder tryFinallyRegion = emitter.NewFinallyRegion(); { @@ -90,9 +97,7 @@ public static MethodIL EmitTaskReturningThunk(MethodDesc taskReturningMethod, Me codestream.EmitStLoc(logicalResultLocal); } - MethodDesc asyncCallContinuationMd = context.SystemModule - .GetKnownType("System.Runtime.CompilerServices"u8, "AsyncHelpers"u8) - .GetKnownMethod("AsyncCallContinuation"u8, null); + MethodDesc asyncCallContinuationMd = asyncHelpersType.GetKnownMethod("AsyncCallContinuation"u8, null); codestream.Emit(ILOpcode.call, emitter.NewToken(asyncCallContinuationMd)); @@ -161,8 +166,7 @@ public static MethodIL EmitTaskReturningThunk(MethodDesc taskReturningMethod, Me parameters: new[] { exceptionType } ); - fromExceptionMd = context.SystemModule - .GetKnownType("System.Runtime.CompilerServices"u8, "AsyncHelpers"u8) + fromExceptionMd = asyncHelpersType .GetKnownMethod(isValueTask ? "ValueTaskFromException"u8 : "TaskFromException"u8, fromExceptionSignature) .MakeInstantiatedMethod(new Instantiation(logicalReturnType)); } @@ -175,8 +179,7 @@ public static MethodIL EmitTaskReturningThunk(MethodDesc taskReturningMethod, Me parameters: new[] { exceptionType } ); - fromExceptionMd = context.SystemModule - .GetKnownType("System.Runtime.CompilerServices"u8, "AsyncHelpers"u8) + fromExceptionMd = asyncHelpersType .GetKnownMethod(isValueTask ? "ValueTaskFromException"u8 : "TaskFromException"u8, fromExceptionSignature); } @@ -195,11 +198,10 @@ public static MethodIL EmitTaskReturningThunk(MethodDesc taskReturningMethod, Me MethodSignatureFlags.Static, genericParameterCount: 1, returnType: ((MetadataType)returnType.GetTypeDefinition()).MakeInstantiatedType(context.GetSignatureVariable(0, true)), - parameters: Array.Empty() + parameters: [awaitStateType.MakeByRefType()] ); - finalizeTaskReturningThunkMd = context.SystemModule - .GetKnownType("System.Runtime.CompilerServices"u8, "AsyncHelpers"u8) + finalizeTaskReturningThunkMd = asyncHelpersType .GetKnownMethod(isValueTask ? "FinalizeValueTaskReturningThunk"u8 : "FinalizeTaskReturningThunk"u8, finalizeReturningThunkSignature) .MakeInstantiatedMethod(new Instantiation(logicalReturnType)); } @@ -209,14 +211,14 @@ public static MethodIL EmitTaskReturningThunk(MethodDesc taskReturningMethod, Me MethodSignatureFlags.Static, genericParameterCount: 0, returnType: returnType, - parameters: Array.Empty() + parameters: [awaitStateType.MakeByRefType()] ); - finalizeTaskReturningThunkMd = context.SystemModule - .GetKnownType("System.Runtime.CompilerServices"u8, "AsyncHelpers"u8) + finalizeTaskReturningThunkMd = asyncHelpersType .GetKnownMethod(isValueTask ? "FinalizeValueTaskReturningThunk"u8 : "FinalizeTaskReturningThunk"u8, finalizeReturningThunkSignature); } + codestream.EmitLdLoc(refAwaitStateLocal); codestream.Emit(ILOpcode.call, emitter.NewToken(finalizeTaskReturningThunkMd)); codestream.EmitStLoc(returnTaskLocal); codestream.Emit(ILOpcode.leave, returnTaskLabel); @@ -227,8 +229,8 @@ public static MethodIL EmitTaskReturningThunk(MethodDesc taskReturningMethod, Me { codestream.BeginHandler(tryFinallyRegion); - codestream.EmitLdLoca(executionAndSyncBlockStoreLocal); - codestream.Emit(ILOpcode.call, emitter.NewToken(executionAndSyncBlockStoreType.GetKnownMethod("Pop"u8, null))); + codestream.EmitLdLoc(refAwaitStateLocal); + codestream.Emit(ILOpcode.call, emitter.NewToken(awaitStateType.GetKnownMethod("Pop"u8, null))); codestream.Emit(ILOpcode.endfinally); codestream.EndHandler(tryFinallyRegion); } diff --git a/src/coreclr/vm/asyncthunks.cpp b/src/coreclr/vm/asyncthunks.cpp index f26f8aa2966bd4..68da2bb0239f5a 100644 --- a/src/coreclr/vm/asyncthunks.cpp +++ b/src/coreclr/vm/asyncthunks.cpp @@ -93,9 +93,10 @@ void MethodDesc::EmitTaskReturningThunk(MethodDesc* pAsyncCallVariant, MetaSig& // Emits roughly the following code: // - // Thread currentThread = Thread.CurrentThread; - // ExecutionAndSyncBlockStore store = default; - // store.Push(currentThread); + // RuntimeAsyncStackState stackState; + // ref RuntimeAsyncAwaitState awaitState = ref AsyncHelpers.t_runtimeAsyncAwaitState; + // awaitState.Push(&stackState); + // // try // { // try @@ -114,7 +115,7 @@ void MethodDesc::EmitTaskReturningThunk(MethodDesc* pAsyncCallVariant, MetaSig& // } // finally // { - // store.Pop(currentThread); + // awaitState.Pop(); // } ILCodeStream* pCode = pSL->NewCodeStream(ILStubLinker::kDispatch); @@ -134,22 +135,23 @@ void MethodDesc::EmitTaskReturningThunk(MethodDesc* pAsyncCallVariant, MetaSig& LocalDesc returnLocalDesc(thTaskRet); DWORD returnTaskLocal = pCode->NewLocal(returnLocalDesc); - LocalDesc threadLocalDesc(CoreLibBinder::GetClass(CLASS__THREAD)); - DWORD threadLocal = pCode->NewLocal(threadLocalDesc); + LocalDesc stackStateLocalDesc(TypeHandle(CoreLibBinder::GetClass(CLASS__RUNTIME_ASYNC_STACK_STATE))); + DWORD stackStateLocal = pCode->NewLocal(stackStateLocalDesc); - LocalDesc executionAndSyncBlockStoreLocalDesc(CoreLibBinder::GetClass(CLASS__EXECUTIONANDSYNCBLOCKSTORE)); - DWORD executionAndSyncBlockStoreLocal = pCode->NewLocal(executionAndSyncBlockStoreLocalDesc); + LocalDesc refAwaitStateLocalDesc(TypeHandle(CoreLibBinder::GetClass(CLASS__RUNTIME_ASYNC_AWAIT_STATE))); + refAwaitStateLocalDesc.MakeByRef(); + DWORD refAwaitStateLocal = pCode->NewLocal(refAwaitStateLocalDesc); ILCodeLabel* returnTaskLabel = pCode->NewCodeLabel(); ILCodeLabel* suspendedLabel = pCode->NewCodeLabel(); ILCodeLabel* finishedLabel = pCode->NewCodeLabel(); - pCode->EmitCALL(METHOD__THREAD__GET_CURRENTTHREAD, 0, 1); - pCode->EmitSTLOC(threadLocal); + pCode->EmitLDSFLDA(pCode->GetToken(CoreLibBinder::GetField(FIELD__ASYNC_HELPERS__TLS_RUNTIME_ASYNC_AWAIT_STATE))); + pCode->EmitSTLOC(refAwaitStateLocal); - pCode->EmitLDLOCA(executionAndSyncBlockStoreLocal); - pCode->EmitLDLOC(threadLocal); - pCode->EmitCALL(METHOD__EXECUTIONANDSYNCBLOCKSTORE__PUSH, 2, 0); + pCode->EmitLDLOC(refAwaitStateLocal); + pCode->EmitLDLOCA(stackStateLocal); + pCode->EmitCALL(METHOD__RUNTIME_ASYNC_AWAIT_STATE__PUSH, 2, 0); { pCode->BeginTryBlock(); @@ -261,7 +263,8 @@ void MethodDesc::EmitTaskReturningThunk(MethodDesc* pAsyncCallVariant, MetaSig& md = CoreLibBinder::GetMethod(METHOD__ASYNC_HELPERS__FINALIZE_TASK_RETURNING_THUNK); finalizeTaskReturningThunkToken = pCode->GetToken(md); } - pCode->EmitCALL(finalizeTaskReturningThunkToken, 0, 1); + pCode->EmitLDLOC(refAwaitStateLocal); + pCode->EmitCALL(finalizeTaskReturningThunkToken, 1, 1); pCode->EmitSTLOC(returnTaskLocal); pCode->EmitLEAVE(returnTaskLabel); @@ -270,9 +273,8 @@ void MethodDesc::EmitTaskReturningThunk(MethodDesc* pAsyncCallVariant, MetaSig& // { pCode->BeginFinallyBlock(); - pCode->EmitLDLOCA(executionAndSyncBlockStoreLocal); - pCode->EmitLDLOC(threadLocal); - pCode->EmitCALL(METHOD__EXECUTIONANDSYNCBLOCKSTORE__POP, 2, 0); + pCode->EmitLDLOC(refAwaitStateLocal); + pCode->EmitCALL(METHOD__RUNTIME_ASYNC_AWAIT_STATE__POP, 1, 0); pCode->EmitENDFINALLY(); pCode->EndFinallyBlock(); } diff --git a/src/coreclr/vm/corelib.h b/src/coreclr/vm/corelib.h index 59ef47bd8303d1..4dbfc3d185a2ab 100644 --- a/src/coreclr/vm/corelib.h +++ b/src/coreclr/vm/corelib.h @@ -683,10 +683,6 @@ DEFINE_CLASS(RESOURCE_MANAGER, Resources, ResourceManager) DEFINE_CLASS(RTFIELD, Reflection, RtFieldInfo) DEFINE_METHOD(RTFIELD, GET_FIELDESC, GetFieldDesc, IM_RetIntPtr) -DEFINE_CLASS(EXECUTIONANDSYNCBLOCKSTORE, CompilerServices, ExecutionAndSyncBlockStore) -DEFINE_METHOD(EXECUTIONANDSYNCBLOCKSTORE, PUSH, Push, NoSig) -DEFINE_METHOD(EXECUTIONANDSYNCBLOCKSTORE, POP, Pop, NoSig) - DEFINE_CLASS(RUNTIME_HELPERS, CompilerServices, RuntimeHelpers) DEFINE_METHOD(RUNTIME_HELPERS, IS_BITWISE_EQUATABLE, IsBitwiseEquatable, NoSig) DEFINE_METHOD(RUNTIME_HELPERS, GET_RAW_DATA, GetRawData, NoSig) @@ -711,10 +707,10 @@ DEFINE_METHOD(ASYNC_HELPERS, ALLOC_CONTINUATION, AllocContinuation, DEFINE_METHOD(ASYNC_HELPERS, ALLOC_CONTINUATION_METHOD, AllocContinuationMethod, NoSig) DEFINE_METHOD(ASYNC_HELPERS, ALLOC_CONTINUATION_CLASS, AllocContinuationClass, NoSig) -DEFINE_METHOD(ASYNC_HELPERS, FINALIZE_TASK_RETURNING_THUNK, FinalizeTaskReturningThunk, SM_RetTask) -DEFINE_METHOD(ASYNC_HELPERS, FINALIZE_TASK_RETURNING_THUNK_1, FinalizeTaskReturningThunk, GM_RetTaskOfT) -DEFINE_METHOD(ASYNC_HELPERS, FINALIZE_VALUETASK_RETURNING_THUNK, FinalizeValueTaskReturningThunk, SM_RetValueTask) -DEFINE_METHOD(ASYNC_HELPERS, FINALIZE_VALUETASK_RETURNING_THUNK_1, FinalizeValueTaskReturningThunk, GM_RetValueTaskOfT) +DEFINE_METHOD(ASYNC_HELPERS, FINALIZE_TASK_RETURNING_THUNK, FinalizeTaskReturningThunk, SM_RefRuntimeAsyncAwaitState_RetTask) +DEFINE_METHOD(ASYNC_HELPERS, FINALIZE_TASK_RETURNING_THUNK_1, FinalizeTaskReturningThunk, GM_RefRuntimeAsyncAwaitState_RetTaskOfT) +DEFINE_METHOD(ASYNC_HELPERS, FINALIZE_VALUETASK_RETURNING_THUNK, FinalizeValueTaskReturningThunk, SM_RefRuntimeAsyncAwaitState_RetValueTask) +DEFINE_METHOD(ASYNC_HELPERS, FINALIZE_VALUETASK_RETURNING_THUNK_1, FinalizeValueTaskReturningThunk, GM_RefRuntimeAsyncAwaitState_RetValueTaskOfT) DEFINE_METHOD(ASYNC_HELPERS, TASK_FROM_EXCEPTION, TaskFromException, SM_Exception_RetTask) DEFINE_METHOD(ASYNC_HELPERS, TASK_FROM_EXCEPTION_1, TaskFromException, GM_Exception_RetTaskOfT) @@ -732,11 +728,18 @@ DEFINE_METHOD(ASYNC_HELPERS, RESTORE_CONTEXTS, RestoreContexts, No DEFINE_METHOD(ASYNC_HELPERS, RESTORE_CONTEXTS_ON_SUSPENSION, RestoreContextsOnSuspension, NoSig) DEFINE_METHOD(ASYNC_HELPERS, ASYNC_CALL_CONTINUATION, AsyncCallContinuation, NoSig) DEFINE_METHOD(ASYNC_HELPERS, TAIL_AWAIT, TailAwait, NoSig) +DEFINE_FIELD(ASYNC_HELPERS, TLS_RUNTIME_ASYNC_AWAIT_STATE, t_runtimeAsyncAwaitState) #ifdef FEATURE_INTERPRETER DEFINE_METHOD(ASYNC_HELPERS, RESUME_INTERPRETER_CONTINUATION, ResumeInterpreterContinuation, NoSig) #endif +DEFINE_CLASS(RUNTIME_ASYNC_AWAIT_STATE, CompilerServices, AsyncHelpers+RuntimeAsyncAwaitState) +DEFINE_METHOD(RUNTIME_ASYNC_AWAIT_STATE, PUSH, Push, NoSig) +DEFINE_METHOD(RUNTIME_ASYNC_AWAIT_STATE, POP, Pop, NoSig) + +DEFINE_CLASS(RUNTIME_ASYNC_STACK_STATE, CompilerServices, AsyncHelpers+RuntimeAsyncStackState) + DEFINE_CLASS_U(CompilerServices, Continuation, ContinuationObject) DEFINE_FIELD_U(Next, ContinuationObject, Next) DEFINE_FIELD_U(ResumeInfo, ContinuationObject, ResumeInfo) diff --git a/src/coreclr/vm/metasig.h b/src/coreclr/vm/metasig.h index 0d631bf34b4db0..d304d63ff9d908 100644 --- a/src/coreclr/vm/metasig.h +++ b/src/coreclr/vm/metasig.h @@ -547,8 +547,10 @@ DEFINE_METASIG(SM(PtrByte_RetStr, P(b), s)) DEFINE_METASIG(SM(Str_RetPtrByte, s, P(b))) DEFINE_METASIG(SM(PtrByte_RetVoid, P(b), v)) -DEFINE_METASIG_T(GM(RetTaskOfT, IMAGE_CEE_CS_CALLCONV_DEFAULT, 1, , GI(C(TASK_1), 1, M(0)))) -DEFINE_METASIG_T(GM(RetValueTaskOfT, IMAGE_CEE_CS_CALLCONV_DEFAULT, 1, , GI(g(VALUETASK_1), 1, M(0)))) +DEFINE_METASIG_T(SM(RefRuntimeAsyncAwaitState_RetTask, r(g(RUNTIME_ASYNC_AWAIT_STATE)), C(TASK))) +DEFINE_METASIG_T(SM(RefRuntimeAsyncAwaitState_RetValueTask, r(g(RUNTIME_ASYNC_AWAIT_STATE)), g(VALUETASK))) +DEFINE_METASIG_T(GM(RefRuntimeAsyncAwaitState_RetTaskOfT, IMAGE_CEE_CS_CALLCONV_DEFAULT, 1, r(g(RUNTIME_ASYNC_AWAIT_STATE)), GI(C(TASK_1), 1, M(0)))) +DEFINE_METASIG_T(GM(RefRuntimeAsyncAwaitState_RetValueTaskOfT, IMAGE_CEE_CS_CALLCONV_DEFAULT, 1, r(g(RUNTIME_ASYNC_AWAIT_STATE)), GI(g(VALUETASK_1), 1, M(0)))) // Undefine macros in case we include the file again in the compilation unit diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.cs index 0d5ff2b7ca8143..510b863d4dda8e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.cs @@ -25,14 +25,11 @@ public static partial class AsyncHelpers [BypassReadyToRun] [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.Async)] [StackTraceHidden] - public static void AwaitAwaiter(TAwaiter awaiter) where TAwaiter : INotifyCompletion + public static unsafe void AwaitAwaiter(TAwaiter awaiter) where TAwaiter : INotifyCompletion { ref RuntimeAsyncAwaitState state = ref t_runtimeAsyncAwaitState; - Continuation? sentinelContinuation = state.SentinelContinuation; - if (sentinelContinuation == null) - state.SentinelContinuation = sentinelContinuation = new Continuation(); - - state.Notifier = awaiter; + Continuation? sentinelContinuation = state.SentinelContinuation ??= new Continuation(); + state.StackState->Notifier = awaiter; state.CaptureContexts(); AsyncSuspend(sentinelContinuation); } @@ -48,14 +45,11 @@ public static void AwaitAwaiter(TAwaiter awaiter) where TAwaiter : INo [BypassReadyToRun] [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.Async)] [StackTraceHidden] - public static void UnsafeAwaitAwaiter(TAwaiter awaiter) where TAwaiter : ICriticalNotifyCompletion + public static unsafe void UnsafeAwaitAwaiter(TAwaiter awaiter) where TAwaiter : ICriticalNotifyCompletion { ref RuntimeAsyncAwaitState state = ref t_runtimeAsyncAwaitState; - Continuation? sentinelContinuation = state.SentinelContinuation; - if (sentinelContinuation == null) - state.SentinelContinuation = sentinelContinuation = new Continuation(); - - state.CriticalNotifier = awaiter; + Continuation? sentinelContinuation = state.SentinelContinuation ??= new Continuation(); + state.StackState->CriticalNotifier = awaiter; state.CaptureContexts(); AsyncSuspend(sentinelContinuation); } From 1c04f22c19e4d0b46079b94eb67d90423e4dbf1e Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 23 Apr 2026 20:48:53 +0200 Subject: [PATCH 3/8] Clean up --- .../System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs | 4 ++-- src/coreclr/vm/asyncthunks.cpp | 2 +- src/coreclr/vm/corelib.h | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs index a82e93d5dc3ba8..61b12c33bb3064 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs @@ -245,7 +245,7 @@ public void CaptureContexts() if (CurrentDispatcherInfo != null) return CurrentDispatcherInfo; - // This relies on coreclr and NAOT runtimes always storing ThreadStatic pointer fields in a pinned fashion. + // This relies on coreclr and NativeAOT runtimes always storing ThreadStatic pointer fields in a pinned fashion. fixed (AsyncDispatcherInfo** pCurrentDispatcherInfo = &AsyncDispatcherInfo.t_current) { return CurrentDispatcherInfo = pCurrentDispatcherInfo; @@ -268,7 +268,7 @@ public void Pop() } [ThreadStatic] - private static unsafe RuntimeAsyncAwaitState t_runtimeAsyncAwaitState; + private static RuntimeAsyncAwaitState t_runtimeAsyncAwaitState; #if !NATIVEAOT [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "AsyncHelpers_AddContinuationToExInternal")] diff --git a/src/coreclr/vm/asyncthunks.cpp b/src/coreclr/vm/asyncthunks.cpp index 68da2bb0239f5a..ef45a4a41c715d 100644 --- a/src/coreclr/vm/asyncthunks.cpp +++ b/src/coreclr/vm/asyncthunks.cpp @@ -106,7 +106,7 @@ void MethodDesc::EmitTaskReturningThunk(MethodDesc* pAsyncCallVariant, MetaSig& // if (AsyncHelpers.AsyncCallContinuation() == null) // return Task.FromResult(result); // - // return FinalizeTaskReturningThunk(); + // return FinalizeTaskReturningThunk(ref awaitState); // } // catch (Exception ex) // { diff --git a/src/coreclr/vm/corelib.h b/src/coreclr/vm/corelib.h index 4dbfc3d185a2ab..674942ce365a61 100644 --- a/src/coreclr/vm/corelib.h +++ b/src/coreclr/vm/corelib.h @@ -984,7 +984,6 @@ DEFINE_CLASS(THREAD, Threading, Thread) DEFINE_METHOD(THREAD, START_CALLBACK, StartCallback, SM_PtrThread_RetVoid) DEFINE_METHOD(THREAD, POLLGC, PollGC, NoSig) DEFINE_METHOD(THREAD, ON_THREAD_EXITING, OnThreadExited, SM_PtrThread_PtrException_RetVoid) -DEFINE_METHOD(THREAD, GET_CURRENTTHREAD, get_CurrentThread, NoSig) #ifdef FOR_ILLINK DEFINE_METHOD(THREAD, CTOR, .ctor, IM_RetVoid) #endif // FOR_ILLINK From c6cada346c89df7717bde62518882fc6530e962f Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 23 Apr 2026 20:51:29 +0200 Subject: [PATCH 4/8] Feedback --- .../src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs index 61b12c33bb3064..5200395c93469f 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs @@ -264,6 +264,7 @@ public void Push(RuntimeAsyncStackState* stackState) public void Pop() { StackState->Pop(CurrentThread); + StackState = null; } } From 4eb15e726e5e58debfff9318275f11b1d53898c2 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 24 Apr 2026 01:09:21 +0200 Subject: [PATCH 5/8] Proper linking --- .../System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs index 5200395c93469f..01875c1c1c9e87 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs @@ -190,6 +190,8 @@ private ref struct RuntimeAsyncStackState public ExecutionContext? RootExecutionContext; public SynchronizationContext? RootSynchronizationContext; + public unsafe RuntimeAsyncStackState* Next; + public void Push(Thread thread) { RootExecutionContext = thread._executionContext; @@ -256,6 +258,7 @@ public void CaptureContexts() // is called public void Push(RuntimeAsyncStackState* stackState) { + stackState->Next = StackState; StackState = stackState; stackState->Push(CurrentThread ??= Thread.CurrentThread); } @@ -264,7 +267,7 @@ public void Push(RuntimeAsyncStackState* stackState) public void Pop() { StackState->Pop(CurrentThread); - StackState = null; + StackState = StackState->Next; } } From 4b4569818dd646f496b6ebd7db25d77c2bd239ed Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 24 Apr 2026 10:21:12 +0200 Subject: [PATCH 6/8] Remove dispatcher info TLS optimization --- .../CompilerServices/AsyncHelpers.CoreCLR.cs | 43 +++++++------------ 1 file changed, 15 insertions(+), 28 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs index 01875c1c1c9e87..c38757e80a653a 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs @@ -223,7 +223,6 @@ private unsafe struct RuntimeAsyncAwaitState // We cache the other TLS members here to avoid unnecessary repeated TLS lookups. public Thread CurrentThread; - public AsyncDispatcherInfo** CurrentDispatcherInfo; public RuntimeAsyncStackState* StackState; @@ -242,18 +241,6 @@ public void CaptureContexts() StackState->LeafSynchronizationContext = curThread._synchronizationContext; } - public AsyncDispatcherInfo** GetOrInitDispatcherInfoPointer() - { - if (CurrentDispatcherInfo != null) - return CurrentDispatcherInfo; - - // This relies on coreclr and NativeAOT runtimes always storing ThreadStatic pointer fields in a pinned fashion. - fixed (AsyncDispatcherInfo** pCurrentDispatcherInfo = &AsyncDispatcherInfo.t_current) - { - return CurrentDispatcherInfo = pCurrentDispatcherInfo; - } - } - // At the start of an async chain (task-returning thunk or DispatchContinuations) this function // is called public void Push(RuntimeAsyncStackState* stackState) @@ -550,12 +537,12 @@ private unsafe void DispatchContinuations() ref RuntimeAsyncAwaitState awaitState = ref t_runtimeAsyncAwaitState; awaitState.Push(&stackState); - AsyncDispatcherInfo** pDispatcherInfo = awaitState.GetOrInitDispatcherInfoPointer(); + ref AsyncDispatcherInfo* refDispatcherInfo = ref AsyncDispatcherInfo.t_current; AsyncDispatcherInfo asyncDispatcherInfo; - asyncDispatcherInfo.Next = *pDispatcherInfo; + asyncDispatcherInfo.Next = refDispatcherInfo; asyncDispatcherInfo.NextContinuation = MoveContinuationState(); - *pDispatcherInfo = &asyncDispatcherInfo; + refDispatcherInfo = &asyncDispatcherInfo; while (true) { @@ -576,7 +563,7 @@ private unsafe void DispatchContinuations() HandleSuspended(ref awaitState); awaitState.Pop(); - *pDispatcherInfo = asyncDispatcherInfo.Next; + refDispatcherInfo = asyncDispatcherInfo.Next; return; } } @@ -592,7 +579,7 @@ private unsafe void DispatchContinuations() TrySetException(ex); awaitState.Pop(); - *pDispatcherInfo = asyncDispatcherInfo.Next; + refDispatcherInfo = asyncDispatcherInfo.Next; if (!successfullySet) { @@ -611,7 +598,7 @@ private unsafe void DispatchContinuations() bool successfullySet = TrySetResult(m_result); awaitState.Pop(); - *pDispatcherInfo = asyncDispatcherInfo.Next; + refDispatcherInfo = asyncDispatcherInfo.Next; if (!successfullySet) { @@ -624,7 +611,7 @@ private unsafe void DispatchContinuations() if (QueueContinuationFollowUpActionIfNecessary(asyncDispatcherInfo.NextContinuation)) { awaitState.Pop(); - *pDispatcherInfo = asyncDispatcherInfo.Next; + refDispatcherInfo = asyncDispatcherInfo.Next; return; } @@ -633,7 +620,7 @@ private unsafe void DispatchContinuations() SetContinuationState(asyncDispatcherInfo.NextContinuation); awaitState.Pop(); - *pDispatcherInfo = asyncDispatcherInfo.Next; + refDispatcherInfo = asyncDispatcherInfo.Next; InstrumentedDispatchContinuations(AsyncInstrumentation.ActiveFlags); return; @@ -649,12 +636,12 @@ private unsafe void InstrumentedDispatchContinuations(AsyncInstrumentation.Flags ref RuntimeAsyncAwaitState awaitState = ref t_runtimeAsyncAwaitState; awaitState.Push(&stackState); - AsyncDispatcherInfo** pDispatcherInfo = awaitState.GetOrInitDispatcherInfoPointer(); + ref AsyncDispatcherInfo* refDispatcherInfo = ref AsyncDispatcherInfo.t_current; AsyncDispatcherInfo asyncDispatcherInfo; - asyncDispatcherInfo.Next = *pDispatcherInfo; + asyncDispatcherInfo.Next = refDispatcherInfo; asyncDispatcherInfo.NextContinuation = MoveContinuationState(); - *pDispatcherInfo = &asyncDispatcherInfo; + refDispatcherInfo = &asyncDispatcherInfo; RuntimeAsyncInstrumentationHelpers.ResumeRuntimeAsyncContext(this, ref asyncDispatcherInfo, flags); @@ -679,7 +666,7 @@ private unsafe void InstrumentedDispatchContinuations(AsyncInstrumentation.Flags InstrumentedHandleSuspended(flags, ref awaitState, newContinuation); awaitState.Pop(); - *pDispatcherInfo = asyncDispatcherInfo.Next; + refDispatcherInfo = asyncDispatcherInfo.Next; return; } @@ -699,7 +686,7 @@ private unsafe void InstrumentedDispatchContinuations(AsyncInstrumentation.Flags TrySetException(ex); awaitState.Pop(); - *pDispatcherInfo = asyncDispatcherInfo.Next; + refDispatcherInfo = asyncDispatcherInfo.Next; if (!successfullySet) { @@ -722,7 +709,7 @@ private unsafe void InstrumentedDispatchContinuations(AsyncInstrumentation.Flags bool successfullySet = TrySetResult(m_result); awaitState.Pop(); - *pDispatcherInfo = asyncDispatcherInfo.Next; + refDispatcherInfo = asyncDispatcherInfo.Next; if (!successfullySet) { @@ -737,7 +724,7 @@ private unsafe void InstrumentedDispatchContinuations(AsyncInstrumentation.Flags RuntimeAsyncInstrumentationHelpers.SuspendRuntimeAsyncContext(ref asyncDispatcherInfo, flags, curContinuation); awaitState.Pop(); - *pDispatcherInfo = asyncDispatcherInfo.Next; + refDispatcherInfo = asyncDispatcherInfo.Next; return; } From a72203c468eeb4c1bc6d47747a8a16646602dfa2 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sat, 25 Apr 2026 01:02:20 +0200 Subject: [PATCH 7/8] Comments, feedback --- .../CompilerServices/AsyncHelpers.CoreCLR.cs | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs index c38757e80a653a..1e13a9441596d7 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs @@ -170,6 +170,11 @@ public static partial class AsyncHelpers [Intrinsic] private static void TailAwait() => throw new UnreachableException(); + // This is state used by suspension/resumption machinery and stored in + // the two places that initiate runtime async chains: either a + // task-returning thunk, or DispatchContinuations. A pointer to this + // state is kept in the runtime async TLS. This storage method avoids + // costly write barriers on the hot path of suspension/resumption. private ref struct RuntimeAsyncStackState { // The following are the possible introducers of asynchrony into a chain of awaits. @@ -221,7 +226,7 @@ private unsafe struct RuntimeAsyncAwaitState { public Continuation? SentinelContinuation; - // We cache the other TLS members here to avoid unnecessary repeated TLS lookups. + // We cache the thread here to avoid unnecessary repeated TLS lookups. public Thread CurrentThread; public RuntimeAsyncStackState* StackState; @@ -230,7 +235,7 @@ public void CaptureContexts() { // CaptureContext is called from leaf await helpers. We either just started a runtime async chain // (from a thunk), or we came from DispatchContinuations (on resumption). - // Both cases have already initialized Thread.t_currentThread. + // Both cases have already initialized CurrentThread. Thread curThread = CurrentThread; Debug.Assert(curThread != null); Debug.Assert(StackState != null); @@ -253,6 +258,7 @@ public void Push(RuntimeAsyncStackState* stackState) // This function is called at the end of an async chain public void Pop() { + Debug.Assert(CurrentThread != null); StackState->Pop(CurrentThread); StackState = StackState->Next; } @@ -391,8 +397,9 @@ internal unsafe bool HandleSuspended(ref RuntimeAsyncAwaitState state) Thread currentThread = state.CurrentThread; Debug.Assert(currentThread != null); - ExecutionContext? suspendingExecutionContext = state.StackState->LeafExecutionContext; - SynchronizationContext? suspendingSyncContext = state.StackState->LeafSynchronizationContext; + RuntimeAsyncStackState* stackState = state.StackState; + ExecutionContext? suspendingExecutionContext = stackState->LeafExecutionContext; + SynchronizationContext? suspendingSyncContext = stackState->LeafSynchronizationContext; if (suspendingExecutionContext != currentThread._executionContext) { currentThread._executionContext = suspendingExecutionContext; @@ -403,11 +410,6 @@ internal unsafe bool HandleSuspended(ref RuntimeAsyncAwaitState state) currentThread._synchronizationContext = suspendingSyncContext; } - ICriticalNotifyCompletion? critNotifier = state.StackState->CriticalNotifier; - INotifyCompletion? notifier = state.StackState->Notifier; - ValueTaskSourceNotifier? vtsNotifier = state.StackState->ValueTaskSourceNotifier; - Task? taskNotifier = state.StackState->TaskNotifier; - Continuation sentinelContinuation = state.SentinelContinuation!; Continuation headContinuation = sentinelContinuation.Next!; sentinelContinuation.Next = null; @@ -425,21 +427,21 @@ internal unsafe bool HandleSuspended(ref RuntimeAsyncAwaitState state) try { - if (critNotifier != null) + if (stackState->CriticalNotifier != null) { - critNotifier.UnsafeOnCompleted(GetContinuationAction()); + stackState->CriticalNotifier!.UnsafeOnCompleted(GetContinuationAction()); } - else if (taskNotifier != null) + else if (stackState->TaskNotifier != null) { // Runtime async callable wrapper for task returning // method. This implements the context transparent // forwarding and makes these wrappers minimal cost. - if (!taskNotifier.TryAddCompletionAction(this)) + if (!stackState->TaskNotifier!.TryAddCompletionAction(this)) { ThreadPool.UnsafeQueueUserWorkItemInternal(this, preferLocal: true); } } - else if (vtsNotifier != null) + else if (stackState->ValueTaskSourceNotifier != null) { // The awaiter must inform the ValueTaskSource on whether the continuation // wants to run on a context, although the source may decide to ignore the suggestion. @@ -474,12 +476,12 @@ internal unsafe bool HandleSuspended(ref RuntimeAsyncAwaitState state) // Clear continuation flags, so that continuation runs transparently nextUserContinuation.Flags &= ~continueFlags; - vtsNotifier.OnCompleted(s_runContinuationAction, this, configFlags); + stackState->ValueTaskSourceNotifier!.OnCompleted(s_runContinuationAction, this, configFlags); } else { - Debug.Assert(notifier != null); - notifier.OnCompleted(GetContinuationAction()); + Debug.Assert(stackState->Notifier != null); + stackState->Notifier!.OnCompleted(GetContinuationAction()); } return true; From d9e5d077ade4c8ce7d4d69487671333c0acd9fe3 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sat, 25 Apr 2026 15:02:39 +0200 Subject: [PATCH 8/8] CSE to avoid a few null-forgiving operators --- .../Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs index 1e13a9441596d7..81d9f982c4e7ea 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs @@ -427,21 +427,21 @@ internal unsafe bool HandleSuspended(ref RuntimeAsyncAwaitState state) try { - if (stackState->CriticalNotifier != null) + if (stackState->CriticalNotifier is { } critNotifier) { - stackState->CriticalNotifier!.UnsafeOnCompleted(GetContinuationAction()); + critNotifier.UnsafeOnCompleted(GetContinuationAction()); } - else if (stackState->TaskNotifier != null) + else if (stackState->TaskNotifier is { } taskNotifier) { // Runtime async callable wrapper for task returning // method. This implements the context transparent // forwarding and makes these wrappers minimal cost. - if (!stackState->TaskNotifier!.TryAddCompletionAction(this)) + if (!taskNotifier.TryAddCompletionAction(this)) { ThreadPool.UnsafeQueueUserWorkItemInternal(this, preferLocal: true); } } - else if (stackState->ValueTaskSourceNotifier != null) + else if (stackState->ValueTaskSourceNotifier is { } valueTaskSourceNotifier) { // The awaiter must inform the ValueTaskSource on whether the continuation // wants to run on a context, although the source may decide to ignore the suggestion. @@ -476,7 +476,7 @@ internal unsafe bool HandleSuspended(ref RuntimeAsyncAwaitState state) // Clear continuation flags, so that continuation runs transparently nextUserContinuation.Flags &= ~continueFlags; - stackState->ValueTaskSourceNotifier!.OnCompleted(s_runContinuationAction, this, configFlags); + valueTaskSourceNotifier.OnCompleted(s_runContinuationAction, this, configFlags); } else {