From c2dc33e26aeb7bcbcc608a638e70522186536113 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 11 Mar 2026 18:01:12 +0100 Subject: [PATCH 01/28] WIP --- src/coreclr/jit/async.cpp | 144 ++++++++++++++++++++++++------ src/coreclr/jit/async.h | 81 +++++++++++++++-- src/coreclr/jit/jitconfigvalues.h | 1 + 3 files changed, 193 insertions(+), 33 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 032f3860c6c4d4..af0bf99a83dd45 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -438,6 +438,17 @@ BasicBlock* Compiler::CreateReturnBB(unsigned* mergedReturnLcl) return newReturnBB; } +void ContinuationLayoutBuilder::SetReturn(var_types type, ClassLayout* layout) +{ + ReturnType = type; + ReturnLayout = layout; +} + +void ContinuationLayoutBuilder::AddLocal(unsigned lclNum) +{ + m_locals.push_back(lclNum); +} + //------------------------------------------------------------------------ // DefaultValueAnalysis: // Computes which tracked locals have their default (zero) value at each @@ -810,7 +821,7 @@ class AsyncLiveness void Update(GenTree* node); bool IsLive(unsigned lclNum); template - void GetLiveLocals(jitstd::vector& liveLocals, Functor includeLocal); + void GetLiveLocals(ContinuationLayoutBuilder* layoutBuilder, Functor includeLocal); private: bool IsLocalCaptureUnnecessary(unsigned lclNum); @@ -999,17 +1010,17 @@ bool AsyncLiveness::IsLive(unsigned lclNum) // Get live locals that should be captured at this point. // // Parameters: -// liveLocals - Vector to add live local information into +// builder - Layout builder to add local into // includeLocal - Functor to check if a local should be included // template -void AsyncLiveness::GetLiveLocals(jitstd::vector& liveLocals, Functor includeLocal) +void AsyncLiveness::GetLiveLocals(ContinuationLayoutBuilder* builder, Functor includeLocal) { for (unsigned lclNum = 0; lclNum < m_numVars; lclNum++) { if (includeLocal(lclNum) && IsLive(lclNum)) { - liveLocals.push_back(LiveLocalInfo(lclNum)); + builder->AddLocal(lclNum); } } } @@ -1313,16 +1324,16 @@ void AsyncTransformation::Transform( } #endif - m_liveLocalsScratch.clear(); - jitstd::vector& liveLocals = m_liveLocalsScratch; + ContinuationLayoutBuilder* layoutBuilder = new (m_compiler, CMK_Async) ContinuationLayoutBuilder(m_compiler); - CreateLiveSetForSuspension(block, call, defs, life, liveLocals); + CreateLiveSetForSuspension(block, call, defs, life, layoutBuilder); - ContinuationLayout layout = LayOutContinuation(block, call, ContinuationNeedsKeepAlive(life), liveLocals); + BuildContinuation(block, call, ContinuationNeedsKeepAlive(life), layoutBuilder); + //ContinuationLayout layout = LayOutContinuation(block, call, ContinuationNeedsKeepAlive(life), liveLocals); CallDefinitionInfo callDefInfo = CanonicalizeCallDefinition(block, call, &life); - unsigned stateNum = (unsigned)m_resumptionBBs.size(); + unsigned stateNum = (unsigned)m_states.size(); JITDUMP(" Assigned state %u\n", stateNum); BasicBlock* suspendBB = CreateSuspension(block, call, stateNum, life, layout); @@ -1331,7 +1342,7 @@ void AsyncTransformation::Transform( BasicBlock* resumeBB = CreateResumption(block, *remainder, call, callDefInfo, stateNum, layout); - m_resumptionBBs.push_back(resumeBB); + m_states.push_back(AsyncState(layoutBuilder, suspendBB, resumeBB)); CreateDebugInfoForSuspensionPoint(layout); } @@ -1342,17 +1353,17 @@ void AsyncTransformation::Transform( // specified call. // // Parameters: -// block - The block containing the async call -// call - The async call -// defs - Current live LIR edges -// life - Liveness information about live locals -// liveLocals - Information about each live local. +// block - The block containing the async call +// call - The async call +// defs - Current live LIR edges +// life - Liveness information about live locals +// layoutBuilder - Layout being built // void AsyncTransformation::CreateLiveSetForSuspension(BasicBlock* block, GenTreeCall* call, const jitstd::vector& defs, AsyncLiveness& life, - jitstd::vector& liveLocals) + ContinuationLayoutBuilder* layoutBuilder) { SmallHashTable excludedLocals(m_compiler->getAllocator(CMK_Async)); @@ -1387,22 +1398,22 @@ void AsyncTransformation::CreateLiveSetForSuspension(BasicBlock* excludedLocals.AddOrUpdate(m_compiler->lvaAsyncExecutionContextVar, true); } - life.GetLiveLocals(liveLocals, [&](unsigned lclNum) { + life.GetLiveLocals(layoutBuilder, [&](unsigned lclNum) { return !excludedLocals.Contains(lclNum); }); - LiftLIREdges(block, defs, liveLocals); + LiftLIREdges(block, defs, layoutBuilder); #ifdef DEBUG if (m_compiler->verbose) { - printf(" %zu live locals\n", liveLocals.size()); + printf(" %zu live locals\n", layoutBuilder->Locals().size()); - if (liveLocals.size() > 0) + if (layoutBuilder->Locals().size() > 0) { const char* sep = " "; - for (LiveLocalInfo& inf : liveLocals) + for (unsigned lclNum : layoutBuilder->Locals()) { - printf("%sV%02u (%s)", sep, inf.LclNum, varTypeName(m_compiler->lvaGetDesc(inf.LclNum)->TypeGet())); + printf("%sV%02u (%s)", sep, lclNum, varTypeName(m_compiler->lvaGetDesc(lclNum)->TypeGet())); sep = ", "; } @@ -1441,13 +1452,13 @@ bool AsyncTransformation::HasNonContextRestoreExceptionalFlow(BasicBlock* block) // indicating that these locals are live. // // Parameters: -// block - The block containing the definitions of the LIR edges -// defs - Current outstanding LIR edges -// liveLocals - [out] Vector to add new live local information into +// block - The block containing the definitions of the LIR edges +// defs - Current outstanding LIR edges +// layoutBuilder - Continuation layout builder to add new locals to // void AsyncTransformation::LiftLIREdges(BasicBlock* block, const jitstd::vector& defs, - jitstd::vector& liveLocals) + ContinuationLayoutBuilder* layoutBuilder) { if (defs.size() <= 0) { @@ -1475,7 +1486,7 @@ void AsyncTransformation::LiftLIREdges(BasicBlock* block, assert(gotUse); // Defs list should not contain unused values. unsigned newLclNum = use.ReplaceWithLclVar(m_compiler); - liveLocals.push_back(LiveLocalInfo(newLclNum)); + layoutBuilder->AddLocal(newLclNum); GenTree* newUse = use.Def(); LIR::AsRange(block).Remove(newUse); LIR::AsRange(block).InsertBefore(use.User(), newUse); @@ -1564,6 +1575,53 @@ class GCPointerBitMapBuilder } }; +void AsyncTransformation::BuildContinuation(BasicBlock* block, GenTreeCall* call, bool needsKeepAlive, ContinuationLayoutBuilder* layoutBuilder) +{ + layoutBuilder->SetReturn(call->gtReturnType, m_compiler->typGetObjLayout(call->gtRetClsHnd)); + if (call->gtReturnType != TYP_VOID) + { + JITDUMP("Call has return; continuation will have return value\n"); + } + + // For OSR, we store the IL offset that inspired the OSR method at the + // beginning of the data (and store -1 in the tier0 version). This must be + // at the beginning because the tier0 and OSR versions need to agree on + // this. + if (m_compiler->doesMethodHavePatchpoints() || m_compiler->opts.IsOSR()) + { + JITDUMP(" Method %s; keeping IL offset that inspired OSR method at the beginning of non-GC data\n", + m_compiler->doesMethodHavePatchpoints() ? "has patchpoints" : "is an OSR method"); + // Must be pointer sized for compatibility with Continuation methods that access fields + layoutBuilder->SetNeedsOSRILOffset(); + } + + if (HasNonContextRestoreExceptionalFlow(block)) + { + // If we are enclosed in any try region that isn't our special "context + // restore" try region then we need to rethrow an exception. For our + // special "context restore" try region we know that it is a no-op on + // the resumption path. + layoutBuilder->SetNeedsException(); + JITDUMP(" " FMT_BB " is in try region %u; continuation will have exception\n", block->bbNum, + block->getTryIndex()); + } + + if (call->GetAsyncInfo().ContinuationContextHandling == ContinuationContextHandling::ContinueOnCapturedContext) + { + layoutBuilder->SetNeedsContinuationContext(); + JITDUMP(" Continuation continues on captured context; continuation will have context\n"); + } + + if (needsKeepAlive) + { + layoutBuilder->SetNeedsKeepAlive(); + JITDUMP(" Continuation will have keep alive object\n"); + } + + layoutBuilder->SetNeedsExecutionContext(); + JITDUMP(" Call has async-only save and restore of ExecutionContext; continuation will have ExecutionContext\n"); +} + //------------------------------------------------------------------------ // AsyncTransformation::LayOutContinuation: // Create the layout of the GC pointer and data arrays in the continuation @@ -1906,6 +1964,38 @@ CallDefinitionInfo AsyncTransformation::CanonicalizeCallDefinition(BasicBlock* return callDefInfo; } +BasicBlock* AsyncTransformation::CreateSuspensionBlock(BasicBlock* block, GenTreeCall* call, unsigned stateNum) +{ + if (m_lastSuspensionBB == nullptr) + { + m_lastSuspensionBB = m_compiler->fgLastBBInMainFunction(); + } + + BasicBlock* suspendBB = m_compiler->fgNewBBafter(BBJ_RETURN, m_lastSuspensionBB, false); + suspendBB->clearTryIndex(); + suspendBB->clearHndIndex(); + suspendBB->inheritWeightPercentage(block, 0); + m_lastSuspensionBB = suspendBB; + + if (m_sharedReturnBB != nullptr) + { + suspendBB->SetKindAndTargetEdge(BBJ_ALWAYS, m_compiler->fgAddRefPred(m_sharedReturnBB, suspendBB)); + } + + JITDUMP(" Creating suspension " FMT_BB " for state %u\n", suspendBB->bbNum, stateNum); + + GenTreeILOffset* ilOffsetNode = + m_compiler->gtNewILOffsetNode(call->GetAsyncInfo().CallAsyncDebugInfo DEBUGARG(BAD_IL_OFFSET)); + + LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_compiler, ilOffsetNode)); + + GenTree* recordOffset = + new (m_compiler, GT_RECORD_ASYNC_RESUME) GenTreeVal(GT_RECORD_ASYNC_RESUME, TYP_VOID, stateNum); + LIR::AsRange(suspendBB).InsertAtEnd(recordOffset); + + return suspendBB; +} + //------------------------------------------------------------------------ // AsyncTransformation::CreateSuspension: // Create the basic block that when branched to suspends execution after the diff --git a/src/coreclr/jit/async.h b/src/coreclr/jit/async.h index b07348a0e42141..ec35d6098db24a 100644 --- a/src/coreclr/jit/async.h +++ b/src/coreclr/jit/async.h @@ -19,6 +19,57 @@ struct LiveLocalInfo } }; +struct ContinuationLayoutBuilder +{ +private: + Compiler* m_compiler; + bool m_needsOSRILOffset = false; + bool m_needsException = false; + bool m_needsContinuationContext = false; + bool m_needsKeepAlive = false; + bool m_needsExecutionContext = false; + + var_types ReturnType = TYP_VOID; + ClassLayout* ReturnLayout = nullptr; + + jitstd::vector m_locals; + +public: + ContinuationLayoutBuilder(Compiler* compiler) + : m_compiler(compiler) + , m_locals(compiler->getAllocator(CMK_Async)) + { + } + + void SetNeedsOSRILOffset() + { + m_needsOSRILOffset = true; + } + void SetNeedsException() + { + m_needsException = true; + } + void SetNeedsContinuationContext() + { + m_needsContinuationContext = true; + } + void SetNeedsKeepAlive() + { + m_needsKeepAlive = true; + } + void SetNeedsExecutionContext() + { + m_needsExecutionContext = true; + } + void SetReturn(var_types type, ClassLayout* layout); + void AddLocal(unsigned lclNum); + + const jitstd::vector Locals() const + { + return m_locals; + } +}; + struct ContinuationLayout { unsigned Size = 0; @@ -54,14 +105,27 @@ struct CallDefinitionInfo GenTree* InsertAfter = nullptr; }; +struct AsyncState +{ + AsyncState(ContinuationLayoutBuilder* layout, BasicBlock* suspensionBB, BasicBlock* resumptionBB) + : Layout(layout) + , SuspensionBB(suspensionBB) + , ResumptionBB(resumptionBB) + { + } + + ContinuationLayoutBuilder* Layout; + BasicBlock* SuspensionBB; + BasicBlock* ResumptionBB; +}; + class AsyncTransformation { friend class AsyncLiveness; Compiler* m_compiler; - jitstd::vector m_liveLocalsScratch; CORINFO_ASYNC_INFO* m_asyncInfo; - jitstd::vector m_resumptionBBs; + jitstd::vector m_states; unsigned m_returnedContinuationVar = BAD_VAR_NUM; unsigned m_newContinuationVar = BAD_VAR_NUM; unsigned m_dataArrayVar = BAD_VAR_NUM; @@ -86,16 +150,21 @@ class AsyncTransformation GenTreeCall* call, const jitstd::vector& defs, AsyncLiveness& life, - jitstd::vector& liveLocals); + ContinuationLayoutBuilder* layoutBuilder); bool HasNonContextRestoreExceptionalFlow(BasicBlock* block); void LiftLIREdges(BasicBlock* block, const jitstd::vector& defs, - jitstd::vector& liveLocals); + ContinuationLayoutBuilder* layoutBuilder); bool ContinuationNeedsKeepAlive(class AsyncLiveness& life); + void BuildContinuation(BasicBlock* block, + GenTreeCall* call, + bool needsKeepAlive, + ContinuationLayoutBuilder* layoutBuilder); + ContinuationLayout LayOutContinuation(BasicBlock* block, GenTreeCall* call, bool needsKeepAlive, @@ -103,6 +172,7 @@ class AsyncTransformation CallDefinitionInfo CanonicalizeCallDefinition(BasicBlock* block, GenTreeCall* call, AsyncLiveness* life); + BasicBlock* CreateSuspensionBlock(BasicBlock* block, GenTreeCall* call, unsigned stateNum); BasicBlock* CreateSuspension( BasicBlock* block, GenTreeCall* call, unsigned stateNum, AsyncLiveness& life, const ContinuationLayout& layout); GenTreeCall* CreateAllocContinuationCall(AsyncLiveness& life, @@ -151,8 +221,7 @@ class AsyncTransformation public: AsyncTransformation(Compiler* comp) : m_compiler(comp) - , m_liveLocalsScratch(comp->getAllocator(CMK_Async)) - , m_resumptionBBs(comp->getAllocator(CMK_Async)) + , m_states(comp->getAllocator(CMK_Async)) { } diff --git a/src/coreclr/jit/jitconfigvalues.h b/src/coreclr/jit/jitconfigvalues.h index 7741e0f6b8fe19..7031d3c6851a91 100644 --- a/src/coreclr/jit/jitconfigvalues.h +++ b/src/coreclr/jit/jitconfigvalues.h @@ -592,6 +592,7 @@ OPT_CONFIG_INTEGER(JitDoOptimizeMaskConversions, "JitDoOptimizeMaskConversions", OPT_CONFIG_INTEGER(JitOptimizeAwait, "JitOptimizeAwait", 1) // Perform optimization of Await intrinsics OPT_CONFIG_STRING(JitAsyncDefaultValueAnalysisRange, "JitAsyncDefaultValueAnalysisRange") // Enable async default value analysis based on method hash range +OPT_CONFIG_INTEGER(JitAsyncReuseContinuations, "JitAsyncReuseContinuations", 1) // Save and reuse continuations in runtime async functions RELEASE_CONFIG_INTEGER(JitEnableOptRepeat, "JitEnableOptRepeat", 1) // If zero, do not allow JitOptRepeat RELEASE_CONFIG_METHODSET(JitOptRepeat, "JitOptRepeat") // Runs optimizer multiple times on specified methods From 20f71de677f73cc38e357be8feafecd6ae68b6f6 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 12 Mar 2026 15:08:02 +0100 Subject: [PATCH 02/28] Create suspensions/resumptions separately from processing IR in async transformation --- src/coreclr/jit/async.cpp | 515 +++++++++++++++++------------- src/coreclr/jit/async.h | 146 ++++++--- src/coreclr/jit/jitconfigvalues.h | 1 - src/coreclr/jit/jitstd/vector.h | 2 + 4 files changed, 408 insertions(+), 256 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index af0bf99a83dd45..2e1d1e770a26db 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -438,17 +438,31 @@ BasicBlock* Compiler::CreateReturnBB(unsigned* mergedReturnLcl) return newReturnBB; } -void ContinuationLayoutBuilder::SetReturn(var_types type, ClassLayout* layout) +void ContinuationLayoutBuilder::AddReturn(const ReturnTypeInfo& info) { - ReturnType = type; - ReturnLayout = layout; + for (const ReturnTypeInfo& ret : m_returns) + { + if ((ret.ReturnType == info.ReturnType) && (ret.ReturnLayout == info.ReturnLayout)) + { + // This return type is already in the layout, no need to add another slot for it. + return; + } + } + + m_returns.push_back(info); } void ContinuationLayoutBuilder::AddLocal(unsigned lclNum) { + assert(m_locals.empty() || (lclNum > m_locals[m_locals.size() - 1])); m_locals.push_back(lclNum); } +bool ContinuationLayoutBuilder::ContainsLocal(unsigned lclNum) const +{ + return BinarySearch(m_locals.data(), (int)m_locals.size(), lclNum) != nullptr; +} + //------------------------------------------------------------------------ // DefaultValueAnalysis: // Computes which tracked locals have their default (zero) value at each @@ -1189,6 +1203,8 @@ PhaseStatus AsyncTransformation::Run() } while (any); } + CreateResumptionsAndSuspensions(); + // After transforming all async calls we have created resumption blocks; // create the resumption switch. CreateResumptionSwitch(); @@ -1329,22 +1345,21 @@ void AsyncTransformation::Transform( CreateLiveSetForSuspension(block, call, defs, life, layoutBuilder); BuildContinuation(block, call, ContinuationNeedsKeepAlive(life), layoutBuilder); - //ContinuationLayout layout = LayOutContinuation(block, call, ContinuationNeedsKeepAlive(life), liveLocals); CallDefinitionInfo callDefInfo = CanonicalizeCallDefinition(block, call, &life); unsigned stateNum = (unsigned)m_states.size(); JITDUMP(" Assigned state %u\n", stateNum); - BasicBlock* suspendBB = CreateSuspension(block, call, stateNum, life, layout); + BasicBlock* suspendBB = CreateSuspensionBlock(block, call, stateNum); CreateCheckAndSuspendAfterCall(block, call, callDefInfo, suspendBB, remainder); - BasicBlock* resumeBB = CreateResumption(block, *remainder, call, callDefInfo, stateNum, layout); + BasicBlock* resumeBB = CreateResumptionBlock(*remainder, call, stateNum, layoutBuilder); - m_states.push_back(AsyncState(layoutBuilder, suspendBB, resumeBB)); + m_states.push_back(AsyncState(stateNum, layoutBuilder, block, call, callDefInfo, suspendBB, resumeBB)); - CreateDebugInfoForSuspensionPoint(layout); + JITDUMP("\n"); } //------------------------------------------------------------------------ @@ -1577,10 +1592,11 @@ class GCPointerBitMapBuilder void AsyncTransformation::BuildContinuation(BasicBlock* block, GenTreeCall* call, bool needsKeepAlive, ContinuationLayoutBuilder* layoutBuilder) { - layoutBuilder->SetReturn(call->gtReturnType, m_compiler->typGetObjLayout(call->gtRetClsHnd)); if (call->gtReturnType != TYP_VOID) { - JITDUMP("Call has return; continuation will have return value\n"); + ClassLayout* layout = call->gtReturnType == TYP_STRUCT ? m_compiler->typGetObjLayout(call->gtRetClsHnd) : nullptr; + layoutBuilder->AddReturn(ReturnTypeInfo(call->gtReturnType, layout)); + JITDUMP(" Call has return; continuation will have return value\n"); } // For OSR, we store the IL offset that inspired the OSR method at the @@ -1622,31 +1638,70 @@ void AsyncTransformation::BuildContinuation(BasicBlock* block, GenTreeCall* call JITDUMP(" Call has async-only save and restore of ExecutionContext; continuation will have ExecutionContext\n"); } -//------------------------------------------------------------------------ -// AsyncTransformation::LayOutContinuation: -// Create the layout of the GC pointer and data arrays in the continuation -// object. -// -// Parameters: -// block - The block containing the async call -// call - The async call -// needsKeepAlive - Whether the layout needs a "keep alive" field allocated -// liveLocals - [in, out] Information about each live local. Size/alignment -// information is read and offset/index information is written. -// -// Returns: -// Layout information. -// -ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* block, - GenTreeCall* call, - bool needsKeepAlive, - jitstd::vector& liveLocals) +#ifdef DEBUG +void ContinuationLayout::Dump(int indent) { - ContinuationLayout layout(liveLocals); + printf("%*sContinuation layout (%u bytes):\n", indent, "", Size); + if (OSRILOffset != UINT_MAX) + { + printf("%*s +%03u OSR IL offset\n", indent, "", OSRILOffset); + } - for (LiveLocalInfo& inf : liveLocals) + if (ExceptionOffset != UINT_MAX) { - LclVarDsc* dsc = m_compiler->lvaGetDesc(inf.LclNum); + printf("%*s +%03u Exception\n", indent, "", ExceptionOffset); + } + + if (ContinuationContextOffset != UINT_MAX) + { + printf("%*s +%03u Continuation context\n", indent, "", ContinuationContextOffset); + } + + if (KeepAliveOffset != UINT_MAX) + { + printf("%*s +%03u Keep alive object\n", indent, "", KeepAliveOffset); + } + + if (ExecutionContextOffset) + { + printf("%*s +%03u Execution context\n", indent, "", ExecutionContextOffset); + } + + for (const LiveLocalInfo& inf : Locals) + { + printf("%*s +%03u V%02u: %u bytes\n", indent, "", inf.Offset, inf.LclNum, inf.Size); + } + + for (const ReturnInfo& ret : Returns) + { + printf("%*s +%03u %u bytes for %s return\n", indent, "", ret.Offset, ret.Size, ret.Type.ReturnType == TYP_STRUCT ? ret.Type.ReturnLayout->GetClassName() : varTypeName(ret.Type.ReturnType)); + } +} +#endif + +const ReturnInfo* ContinuationLayout::FindReturn(GenTreeCall* call) const +{ + for (const ReturnInfo& ret : Returns) + { + if ((ret.Type.ReturnType == call->gtReturnType) && ((call->gtReturnType != TYP_STRUCT) || (ret.Type.ReturnLayout->GetClassHandle() == call->gtRetClsHnd))) + { + return &ret; + } + } + + assert(!"Could not find return for call"); + return nullptr; +} + +ContinuationLayout* ContinuationLayoutBuilder::Create() +{ + ContinuationLayout* layout = new (m_compiler, CMK_Async) ContinuationLayout(m_compiler); + layout->Locals.reserve(m_locals.size()); + + for (unsigned lclNum : m_locals) + { + LclVarDsc* dsc = m_compiler->lvaGetDesc(lclNum); + LiveLocalInfo inf(lclNum); if (dsc->TypeIs(TYP_STRUCT) || dsc->IsImplicitByRef()) { @@ -1677,13 +1732,10 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* inf.Size = genTypeSize(dsc); } - // Saving/storing of longs here may be the first place we introduce - // long IR. We need to potentially decompose this on x86, so indicate - // that to the backend. - m_compiler->compLongUsed |= dsc->TypeIs(TYP_LONG); + layout->Locals.push_back(inf); } - jitstd::sort(liveLocals.begin(), liveLocals.end(), [=](const LiveLocalInfo& lhs, const LiveLocalInfo& rhs) { + jitstd::sort(layout->Locals.begin(), layout->Locals.end(), [=](const LiveLocalInfo& lhs, const LiveLocalInfo& rhs) { bool lhsIsRef = m_compiler->lvaGetDesc(lhs.LclNum)->TypeIs(TYP_REF); bool rhsIsRef = m_compiler->lvaGetDesc(rhs.LclNum)->TypeIs(TYP_REF); @@ -1703,112 +1755,84 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* return lhs.LclNum < rhs.LclNum; }); - if (call->gtReturnType == TYP_STRUCT) + for (const ReturnTypeInfo& ret : m_returns) { - layout.ReturnStructLayout = m_compiler->typGetObjLayout(call->gtRetClsHnd); - layout.ReturnSize = layout.ReturnStructLayout->GetSize(); - layout.ReturnAlignment = m_compiler->info.compCompHnd->getClassAlignmentRequirement(call->gtRetClsHnd); - } - else - { - layout.ReturnSize = genTypeSize(call->gtReturnType); - layout.ReturnAlignment = layout.ReturnSize; - } + ReturnInfo retInfo(ret); - assert((layout.ReturnSize > 0) == (call->gtReturnType != TYP_VOID)); + if (ret.ReturnType == TYP_STRUCT) + { + retInfo.Size = ret.ReturnLayout->GetSize(); + retInfo.Alignment = m_compiler->info.compCompHnd->getClassAlignmentRequirement(ret.ReturnLayout->GetClassHandle()); + } + else + { + retInfo.Size = genTypeSize(ret.ReturnType); + retInfo.Alignment = retInfo.Size; + } + + layout->Returns.push_back(retInfo); + } - auto allocLayout = [&layout](unsigned align, unsigned size) { - layout.Size = roundUp(layout.Size, align); - unsigned offset = layout.Size; - layout.Size += size; + auto allocLayout = [layout](unsigned align, unsigned size) { + layout->Size = roundUp(layout->Size, align); + unsigned offset = layout->Size; + layout->Size += size; return offset; }; - // For OSR, we store the IL offset that inspired the OSR method at the - // beginning of the data (and store -1 in the tier0 version). This must be - // at the beginning because the tier0 and OSR versions need to agree on - // this. - if (m_compiler->doesMethodHavePatchpoints() || m_compiler->opts.IsOSR()) + if (m_needsOSRILOffset) { - JITDUMP(" Method %s; keeping IL offset that inspired OSR method at the beginning of non-GC data\n", - m_compiler->doesMethodHavePatchpoints() ? "has patchpoints" : "is an OSR method"); // Must be pointer sized for compatibility with Continuation methods that access fields - layout.OSRILOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); + layout->OSRILOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); } - if (HasNonContextRestoreExceptionalFlow(block)) + if (m_needsException) { - // If we are enclosed in any try region that isn't our special "context - // restore" try region then we need to rethrow an exception. For our - // special "context restore" try region we know that it is a no-op on - // the resumption path. - layout.ExceptionOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); - JITDUMP(" " FMT_BB " is in try region %u; exception will be at offset %u\n", block->bbNum, - block->getTryIndex(), layout.ExceptionOffset); + layout->ExceptionOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); } - if (call->GetAsyncInfo().ContinuationContextHandling == ContinuationContextHandling::ContinueOnCapturedContext) + if (m_needsContinuationContext) { - layout.ContinuationContextOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); - JITDUMP(" Continuation continues on captured context; context will be at offset %u\n", - layout.ContinuationContextOffset); + layout->ContinuationContextOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); } - if (layout.ReturnSize > 0) + // Now allocate all returns + for (ReturnInfo& ret : layout->Returns) { - layout.ReturnValOffset = allocLayout(layout.ReturnHeapAlignment(), layout.ReturnSize); - - JITDUMP(" Will store return of type %s, size %u at offset %u\n", - call->gtReturnType == TYP_STRUCT ? layout.ReturnStructLayout->GetClassName() - : varTypeName(call->gtReturnType), - layout.ReturnSize, layout.ReturnValOffset); + ret.Offset = allocLayout(ret.Alignment, ret.Size); } - if (needsKeepAlive) + if (m_needsKeepAlive) { - layout.KeepAliveOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); - JITDUMP(" Continuation needs keep alive object; will be at offset %u\n", layout.KeepAliveOffset); + layout->KeepAliveOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); } - layout.ExecutionContextOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); - JITDUMP(" Call has async-only save and restore of ExecutionContext; ExecutionContext will be at offset %u\n", - layout.ExecutionContextOffset); - - for (LiveLocalInfo& inf : liveLocals) + if (m_needsExecutionContext) { - inf.Offset = allocLayout(inf.HeapAlignment(), inf.Size); + layout->ExecutionContextOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); } - layout.Size = roundUp(layout.Size, TARGET_POINTER_SIZE); - -#ifdef DEBUG - if (m_compiler->verbose) + // Then all locals + for (LiveLocalInfo& inf : layout->Locals) { - printf(" Continuation layout (%u bytes):\n", layout.Size); - for (LiveLocalInfo& inf : liveLocals) - { - printf(" +%03u V%02u: %u bytes\n", inf.Offset, inf.LclNum, inf.Size); - } + inf.Offset = allocLayout(inf.HeapAlignment(), inf.Size); } -#endif + layout->Size = roundUp(layout->Size, TARGET_POINTER_SIZE); + + JITDUMPEXEC(layout->Dump(2)); // Now create continuation type. First create bitmap of object refs. - bool* objRefs = layout.Size < TARGET_POINTER_SIZE + bool* objRefs = layout->Size < TARGET_POINTER_SIZE ? nullptr - : new (m_compiler, CMK_Async) bool[layout.Size / TARGET_POINTER_SIZE]{}; + : new (m_compiler, CMK_Async) bool[layout->Size / TARGET_POINTER_SIZE]{}; - GCPointerBitMapBuilder bitmapBuilder(objRefs, layout.Size); - bitmapBuilder.SetIfNotMax(layout.ExceptionOffset); - bitmapBuilder.SetIfNotMax(layout.ContinuationContextOffset); - bitmapBuilder.SetIfNotMax(layout.KeepAliveOffset); - bitmapBuilder.SetIfNotMax(layout.ExecutionContextOffset); + GCPointerBitMapBuilder bitmapBuilder(objRefs, layout->Size); + bitmapBuilder.SetIfNotMax(layout->ExceptionOffset); + bitmapBuilder.SetIfNotMax(layout->ContinuationContextOffset); + bitmapBuilder.SetIfNotMax(layout->KeepAliveOffset); + bitmapBuilder.SetIfNotMax(layout->ExecutionContextOffset); - if (layout.ReturnSize > 0) - { - bitmapBuilder.SetType(layout.ReturnValOffset, call->gtReturnType, layout.ReturnStructLayout); - } - - for (LiveLocalInfo& inf : liveLocals) + for (LiveLocalInfo& inf : layout->Locals) { LclVarDsc* dsc = m_compiler->lvaGetDesc(inf.LclNum); var_types storedType; @@ -1826,12 +1850,17 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* bitmapBuilder.SetType(inf.Offset, storedType, layout); } + for (ReturnInfo& ret : layout->Returns) + { + bitmapBuilder.SetType(ret.Offset, ret.Type.ReturnType, ret.Type.ReturnLayout); + } + #ifdef DEBUG if (m_compiler->verbose) { - printf("Getting continuation layout size = %u, numGCRefs = %u\n", layout.Size, bitmapBuilder.NumObjRefs); + printf(" Getting continuation layout size = %u, numGCRefs = %u\n", layout->Size, bitmapBuilder.NumObjRefs); bool* start = objRefs; - bool* endOfObjRefs = objRefs + layout.Size / TARGET_POINTER_SIZE; + bool* endOfObjRefs = objRefs + layout->Size / TARGET_POINTER_SIZE; while (start < endOfObjRefs) { while (start < endOfObjRefs && !*start) @@ -1844,19 +1873,21 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* while (end < endOfObjRefs && *end) end++; - printf(" [%3u..%3u) obj refs\n", (start - objRefs) * TARGET_POINTER_SIZE, + printf(" [%3u..%3u) obj refs\n", (start - objRefs) * TARGET_POINTER_SIZE, (end - objRefs) * TARGET_POINTER_SIZE); start = end; } } #endif - layout.ClassHnd = - m_compiler->info.compCompHnd->getContinuationType(layout.Size, objRefs, layout.Size / TARGET_POINTER_SIZE); + // Then request the new type from the VM. + + layout->ClassHnd = + m_compiler->info.compCompHnd->getContinuationType(layout->Size, objRefs, layout->Size / TARGET_POINTER_SIZE); #ifdef DEBUG char buffer[256]; - JITDUMP(" Result = %s\n", m_compiler->eeGetClassName(layout.ClassHnd, buffer, ArrLen(buffer))); + JITDUMP(" Result = %s\n", m_compiler->eeGetClassName(layout->ClassHnd, buffer, ArrLen(buffer))); #endif return layout; @@ -1984,15 +2015,6 @@ BasicBlock* AsyncTransformation::CreateSuspensionBlock(BasicBlock* block, GenTre JITDUMP(" Creating suspension " FMT_BB " for state %u\n", suspendBB->bbNum, stateNum); - GenTreeILOffset* ilOffsetNode = - m_compiler->gtNewILOffsetNode(call->GetAsyncInfo().CallAsyncDebugInfo DEBUGARG(BAD_IL_OFFSET)); - - LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_compiler, ilOffsetNode)); - - GenTree* recordOffset = - new (m_compiler, GT_RECORD_ASYNC_RESUME) GenTreeVal(GT_RECORD_ASYNC_RESUME, TYP_VOID, stateNum); - LIR::AsRange(suspendBB).InsertAtEnd(recordOffset); - return suspendBB; } @@ -2011,27 +2033,9 @@ BasicBlock* AsyncTransformation::CreateSuspensionBlock(BasicBlock* block, GenTre // Returns: // The new basic block that was created. // -BasicBlock* AsyncTransformation::CreateSuspension( - BasicBlock* block, GenTreeCall* call, unsigned stateNum, AsyncLiveness& life, const ContinuationLayout& layout) +void AsyncTransformation::CreateSuspension( + BasicBlock* callBlock, GenTreeCall* call, BasicBlock* suspendBB, unsigned stateNum, const ContinuationLayout& layout, const ContinuationLayoutBuilder& subLayout) { - if (m_lastSuspensionBB == nullptr) - { - m_lastSuspensionBB = m_compiler->fgLastBBInMainFunction(); - } - - BasicBlock* suspendBB = m_compiler->fgNewBBafter(BBJ_RETURN, m_lastSuspensionBB, false); - suspendBB->clearTryIndex(); - suspendBB->clearHndIndex(); - suspendBB->inheritWeightPercentage(block, 0); - m_lastSuspensionBB = suspendBB; - - if (m_sharedReturnBB != nullptr) - { - suspendBB->SetKindAndTargetEdge(BBJ_ALWAYS, m_compiler->fgAddRefPred(m_sharedReturnBB, suspendBB)); - } - - JITDUMP(" Creating suspension " FMT_BB " for state %u\n", suspendBB->bbNum, stateNum); - GenTreeILOffset* ilOffsetNode = m_compiler->gtNewILOffsetNode(call->GetAsyncInfo().CallAsyncDebugInfo DEBUGARG(BAD_IL_OFFSET)); @@ -2044,7 +2048,7 @@ BasicBlock* AsyncTransformation::CreateSuspension( // Allocate continuation GenTree* returnedContinuation = m_compiler->gtNewLclvNode(GetReturnedContinuationVar(), TYP_REF); - GenTreeCall* allocContinuation = CreateAllocContinuationCall(life, returnedContinuation, layout); + GenTreeCall* allocContinuation = CreateAllocContinuationCall(subLayout.NeedsKeepAlive(), returnedContinuation, layout); m_compiler->compCurBB = suspendBB; m_compiler->fgMorphTree(allocContinuation); @@ -2072,13 +2076,13 @@ BasicBlock* AsyncTransformation::CreateSuspension( // Fill in 'flags' const AsyncCallInfo& callInfo = call->GetAsyncInfo(); unsigned continuationFlags = 0; - if (layout.OSRILOffset != UINT_MAX) + if (subLayout.NeedsOSRILOffset()) continuationFlags |= CORINFO_CONTINUATION_HAS_OSR_ILOFFSET; - if (layout.ExceptionOffset != UINT_MAX) + if (subLayout.NeedsException()) continuationFlags |= CORINFO_CONTINUATION_HAS_EXCEPTION; - if (layout.ContinuationContextOffset != UINT_MAX) + if (subLayout.NeedsContinuationContext()) continuationFlags |= CORINFO_CONTINUATION_HAS_CONTINUATION_CONTEXT; - if (layout.ReturnValOffset != UINT_MAX) + if (call->gtReturnType != TYP_VOID) continuationFlags |= CORINFO_CONTINUATION_HAS_RESULT; if (callInfo.ContinuationContextHandling == ContinuationContextHandling::ContinueOnThreadPool) continuationFlags |= CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL; @@ -2091,10 +2095,10 @@ BasicBlock* AsyncTransformation::CreateSuspension( if (layout.Size > 0) { - FillInDataOnSuspension(call, layout, suspendBB); + FillInDataOnSuspension(call, layout, subLayout, suspendBB); } - RestoreContexts(block, call, suspendBB); + RestoreContexts(callBlock, call, suspendBB); if (suspendBB->KindIs(BBJ_RETURN)) { @@ -2102,8 +2106,6 @@ BasicBlock* AsyncTransformation::CreateSuspension( GenTree* ret = m_compiler->gtNewOperNode(GT_RETURN_SUSPEND, TYP_VOID, newContinuation); LIR::AsRange(suspendBB).InsertAtEnd(newContinuation, ret); } - - return suspendBB; } //------------------------------------------------------------------------ @@ -2118,14 +2120,14 @@ BasicBlock* AsyncTransformation::CreateSuspension( // Returns: // IR node representing the allocation. // -GenTreeCall* AsyncTransformation::CreateAllocContinuationCall(AsyncLiveness& life, +GenTreeCall* AsyncTransformation::CreateAllocContinuationCall(bool hasKeepAlive, GenTree* prevContinuation, const ContinuationLayout& layout) { GenTree* contClassHndNode = m_compiler->gtNewIconEmbClsHndNode(layout.ClassHnd); // If we need to keep the loader alive, use a different helper. - if (ContinuationNeedsKeepAlive(life)) + if (hasKeepAlive) { assert(layout.KeepAliveOffset != UINT_MAX); GenTree* handleArg = m_compiler->gtNewLclvNode(m_compiler->info.compTypeCtxtArg, TYP_I_IMPL); @@ -2155,6 +2157,7 @@ GenTreeCall* AsyncTransformation::CreateAllocContinuationCall(AsyncLiveness& // void AsyncTransformation::FillInDataOnSuspension(GenTreeCall* call, const ContinuationLayout& layout, + const ContinuationLayoutBuilder& subLayout, BasicBlock* suspendBB) { if (m_compiler->doesMethodHavePatchpoints() || m_compiler->opts.IsOSR()) @@ -2177,7 +2180,10 @@ void AsyncTransformation::FillInDataOnSuspension(GenTreeCall* call, // Fill in data for (const LiveLocalInfo& inf : layout.Locals) { - assert(inf.Size > 0); + if (!subLayout.ContainsLocal(inf.LclNum)) + { + continue; + } LclVarDsc* dsc = m_compiler->lvaGetDesc(inf.LclNum); @@ -2211,9 +2217,14 @@ void AsyncTransformation::FillInDataOnSuspension(GenTreeCall* call, } LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_compiler, store)); + + // Saving/storing of longs here may be the first place we introduce + // long IR. We need to potentially decompose this on x86, so indicate + // that to the backend. + m_compiler->compLongUsed |= dsc->TypeIs(TYP_LONG); } - if (layout.ContinuationContextOffset != UINT_MAX) + if (subLayout.NeedsContinuationContext()) { // Insert call // AsyncHelpers.CaptureContinuationContext( @@ -2264,7 +2275,7 @@ void AsyncTransformation::FillInDataOnSuspension(GenTreeCall* call, LIR::AsRange(suspendBB).Remove(flagsPlaceholder); } - if (layout.ExecutionContextOffset != UINT_MAX) + if (subLayout.NeedsExecutionContext()) { GenTreeCall* captureExecContext = m_compiler->gtNewCallNode(CT_USER_FUNC, m_asyncInfo->captureExecutionContextMethHnd, TYP_REF); @@ -2433,28 +2444,7 @@ void AsyncTransformation::CreateCheckAndSuspendAfterCall(BasicBlock* block->GetFalseEdge()->setLikelihood(1); } -//------------------------------------------------------------------------ -// AsyncTransformation::CreateResumption: -// Create the basic block that when branched to resumes execution on entry to -// the function. -// -// Parameters: -// block - The block containing the async call -// remainder - The block that contains the IR after the (split) async call -// call - The async call -// callDefInfo - Information about the async call's definition -// stateNum - State number assigned to this suspension point -// layout - Layout information for the continuation object -// -// Returns: -// The new basic block that was created. -// -BasicBlock* AsyncTransformation::CreateResumption(BasicBlock* block, - BasicBlock* remainder, - GenTreeCall* call, - const CallDefinitionInfo& callDefInfo, - unsigned stateNum, - const ContinuationLayout& layout) +BasicBlock* AsyncTransformation::CreateResumptionBlock(BasicBlock* remainder, GenTreeCall* call, unsigned stateNum, ContinuationLayoutBuilder* layoutBuilder) { if (m_lastResumptionBB == nullptr) { @@ -2474,6 +2464,32 @@ BasicBlock* AsyncTransformation::CreateResumption(BasicBlock* bloc JITDUMP(" Creating resumption " FMT_BB " for state %u\n", resumeBB->bbNum, stateNum); + return resumeBB; +} + +//------------------------------------------------------------------------ +// AsyncTransformation::CreateResumption: +// Create the basic block that when branched to resumes execution on entry to +// the function. +// +// Parameters: +// block - The block containing the async call +// remainder - The block that contains the IR after the (split) async call +// call - The async call +// callDefInfo - Information about the async call's definition +// stateNum - State number assigned to this suspension point +// layout - Layout information for the continuation object +// +// Returns: +// The new basic block that was created. +// +void AsyncTransformation::CreateResumption(BasicBlock* callBlock, + GenTreeCall* call, + BasicBlock* resumeBB, + const CallDefinitionInfo& callDefInfo, + const ContinuationLayout& layout, + const ContinuationLayoutBuilder& subLayout) +{ GenTreeILOffset* ilOffsetNode = m_compiler->gtNewILOffsetNode(call->GetAsyncInfo().CallAsyncDebugInfo DEBUGARG(BAD_IL_OFFSET)); @@ -2481,22 +2497,20 @@ BasicBlock* AsyncTransformation::CreateResumption(BasicBlock* bloc if (layout.Size > 0) { - RestoreFromDataOnResumption(layout, resumeBB); + RestoreFromDataOnResumption(layout, subLayout, resumeBB); } BasicBlock* storeResultBB = resumeBB; - if (layout.ExceptionOffset != UINT_MAX) + if (subLayout.NeedsException()) { - storeResultBB = RethrowExceptionOnResumption(block, layout, resumeBB); + storeResultBB = RethrowExceptionOnResumption(callBlock, layout, resumeBB); } - if ((layout.ReturnSize > 0) && (callDefInfo.DefinitionNode != nullptr)) + if ((call->gtReturnType != TYP_VOID) && (callDefInfo.DefinitionNode != nullptr)) { CopyReturnValueOnResumption(call, callDefInfo, layout, storeResultBB); } - - return resumeBB; } //------------------------------------------------------------------------ @@ -2508,9 +2522,9 @@ BasicBlock* AsyncTransformation::CreateResumption(BasicBlock* bloc // layout - Information about the continuation layout // resumeBB - Basic block to append IR to // -void AsyncTransformation::RestoreFromDataOnResumption(const ContinuationLayout& layout, BasicBlock* resumeBB) +void AsyncTransformation::RestoreFromDataOnResumption(const ContinuationLayout& layout, const ContinuationLayoutBuilder& subLayout, BasicBlock* resumeBB) { - if (layout.ExecutionContextOffset != BAD_VAR_NUM) + if (subLayout.NeedsExecutionContext()) { GenTree* valuePlaceholder = m_compiler->gtNewZeroConNode(TYP_REF); GenTreeCall* restoreCall = @@ -2540,6 +2554,11 @@ void AsyncTransformation::RestoreFromDataOnResumption(const ContinuationLayout& // Copy data for (const LiveLocalInfo& inf : layout.Locals) { + if (!subLayout.ContainsLocal(inf.LclNum)) + { + continue; + } + LclVarDsc* dsc = m_compiler->lvaGetDesc(inf.LclNum); GenTree* continuation = m_compiler->gtNewLclvNode(m_compiler->lvaAsyncContinuationArg, TYP_REF); @@ -2574,7 +2593,7 @@ void AsyncTransformation::RestoreFromDataOnResumption(const ContinuationLayout& LIR::AsRange(resumeBB).InsertAtEnd(LIR::SeqTree(m_compiler, store)); } - if (layout.KeepAliveOffset != UINT_MAX) + if (subLayout.NeedsKeepAlive()) { // Ensure that the continuation remains alive until we finished loading the generic context GenTree* continuation = m_compiler->gtNewLclvNode(m_compiler->lvaAsyncContinuationArg, TYP_REF); @@ -2685,14 +2704,16 @@ void AsyncTransformation::CopyReturnValueOnResumption(GenTreeCall* const ContinuationLayout& layout, BasicBlock* storeResultBB) { + const ReturnInfo* retInfo = layout.FindReturn(call); + assert(retInfo != nullptr); GenTree* resultBase = m_compiler->gtNewLclvNode(m_compiler->lvaAsyncContinuationArg, TYP_REF); - unsigned resultOffset = OFFSETOF__CORINFO_Continuation__data + layout.ReturnValOffset; + unsigned resultOffset = OFFSETOF__CORINFO_Continuation__data + retInfo->Offset; assert(callDefInfo.DefinitionNode != nullptr); LclVarDsc* resultLcl = m_compiler->lvaGetDesc(callDefInfo.DefinitionNode); GenTreeFlags indirFlags = - GTF_IND_NONFAULTING | (layout.ReturnHeapAlignment() < layout.ReturnAlignment ? GTF_IND_UNALIGNED : GTF_EMPTY); + GTF_IND_NONFAULTING | (retInfo->HeapAlignment() < retInfo->Alignment ? GTF_IND_UNALIGNED : GTF_EMPTY); // TODO-TP: We can use liveness to avoid generating a lot of this IR. if (call->gtReturnType == TYP_STRUCT) @@ -2701,17 +2722,17 @@ void AsyncTransformation::CopyReturnValueOnResumption(GenTreeCall* { GenTree* resultOffsetNode = m_compiler->gtNewIconNode((ssize_t)resultOffset, TYP_I_IMPL); GenTree* resultAddr = m_compiler->gtNewOperNode(GT_ADD, TYP_BYREF, resultBase, resultOffsetNode); - GenTree* resultData = m_compiler->gtNewLoadValueNode(layout.ReturnStructLayout, resultAddr, indirFlags); + GenTree* resultData = m_compiler->gtNewLoadValueNode(retInfo->Type.ReturnLayout, resultAddr, indirFlags); GenTree* storeResult; if ((callDefInfo.DefinitionNode->GetLclOffs() == 0) && - ClassLayout::AreCompatible(resultLcl->GetLayout(), layout.ReturnStructLayout)) + ClassLayout::AreCompatible(resultLcl->GetLayout(), retInfo->Type.ReturnLayout)) { storeResult = m_compiler->gtNewStoreLclVarNode(callDefInfo.DefinitionNode->GetLclNum(), resultData); } else { storeResult = m_compiler->gtNewStoreLclFldNode(callDefInfo.DefinitionNode->GetLclNum(), TYP_STRUCT, - layout.ReturnStructLayout, + retInfo->Type.ReturnLayout, callDefInfo.DefinitionNode->GetLclOffs(), resultData); } @@ -2829,12 +2850,17 @@ GenTreeStoreInd* AsyncTransformation::StoreAtOffset( // Parameters: // layout - Layout of continuation // -void AsyncTransformation::CreateDebugInfoForSuspensionPoint(const ContinuationLayout& layout) +void AsyncTransformation::CreateDebugInfoForSuspensionPoint(const ContinuationLayout& layout, const ContinuationLayoutBuilder& subLayout) { uint32_t numLocals = 0; - for (const LiveLocalInfo& local : layout.Locals) + for (const LiveLocalInfo& inf : layout.Locals) { - unsigned ilVarNum = m_compiler->compMap2ILvarNum(local.LclNum); + if (!subLayout.ContainsLocal(inf.LclNum)) + { + continue; + } + + unsigned ilVarNum = m_compiler->compMap2ILvarNum(inf.LclNum); if (ilVarNum == (unsigned)ICorDebugInfo::UNKNOWN_ILNUM) { continue; @@ -2842,7 +2868,7 @@ void AsyncTransformation::CreateDebugInfoForSuspensionPoint(const ContinuationLa ICorDebugInfo::AsyncContinuationVarInfo varInf; varInf.VarNumber = ilVarNum; - varInf.Offset = OFFSETOF__CORINFO_Continuation__data + local.Offset; + varInf.Offset = OFFSETOF__CORINFO_Continuation__data + inf.Offset; m_compiler->compAsyncVars->push_back(varInf); numLocals++; } @@ -2971,6 +2997,66 @@ BasicBlock* AsyncTransformation::GetSharedReturnBB() #endif } +void AsyncTransformation::CreateResumptionsAndSuspensions() +{ + JITDUMP("Creating suspensions and resumptions for %zu states\n", m_states.size()); + for (const AsyncState& state : m_states) + { + JITDUMP("State %u suspend @ " FMT_BB ", resume @ " FMT_BB "\n", state.Number, state.SuspensionBB->bbNum, state.ResumptionBB->bbNum); + ContinuationLayout* layout = state.Layout->Create(); + CreateSuspension(state.CallBlock, state.Call, state.SuspensionBB, state.Number, *layout, *state.Layout); + CreateResumption(state.CallBlock, state.Call, state.ResumptionBB, state.CallDefInfo, *layout, *state.Layout); + CreateDebugInfoForSuspensionPoint(*layout, *state.Layout); + + JITDUMP("\n"); + } +} + +ContinuationLayoutBuilder* ContinuationLayoutBuilder::CreateSharedLayout(Compiler* comp, const jitstd::vector& states) +{ + unsigned maxLocalStored = 0; + for (const AsyncState& state : states) + { + jitstd::vector& locals = state.Layout->m_locals; + if (locals.size() > 0) + { + maxLocalStored = std::max(maxLocalStored, locals[locals.size() - 1]); + } + } + + ContinuationLayoutBuilder* sharedLayout = new (comp, CMK_Async) ContinuationLayoutBuilder(comp); + BitVecTraits traits(maxLocalStored + 1, comp); + BitVec locals(BitVecOps::MakeEmpty(&traits)); + + for (const AsyncState& state : states) + { + ContinuationLayoutBuilder* layout = state.Layout; + sharedLayout->m_needsOSRILOffset |= layout->m_needsOSRILOffset; + sharedLayout->m_needsException |= layout->m_needsException; + sharedLayout->m_needsContinuationContext |= layout->m_needsContinuationContext; + sharedLayout->m_needsKeepAlive |= layout->m_needsKeepAlive; + sharedLayout->m_needsExecutionContext |= layout->m_needsExecutionContext; + + for (unsigned local : layout->m_locals) + { + assert(local <= maxLocalStored); + BitVecOps::AddElemD(&traits, locals, local); + } + + for (const ReturnTypeInfo& ret : layout->m_returns) + { + sharedLayout->AddReturn(ret); + } + } + + BitVecOps::VisitBits(&traits, locals, [=](unsigned localNum) { + sharedLayout->AddLocal(localNum); + return true; + }); + + return sharedLayout; +} + //------------------------------------------------------------------------ // AsyncTransformation::CreateResumptionSwitch: // Create the IR for the entry of the function that checks the continuation @@ -2989,19 +3075,19 @@ void AsyncTransformation::CreateResumptionSwitch() FlowEdge* resumingEdge; - if (m_resumptionBBs.size() == 1) + if (m_states.size() == 1) { JITDUMP(" Redirecting entry " FMT_BB " directly to " FMT_BB " as it is the only resumption block\n", - newEntryBB->bbNum, m_resumptionBBs[0]->bbNum); - resumingEdge = m_compiler->fgAddRefPred(m_resumptionBBs[0], newEntryBB); + newEntryBB->bbNum, m_states[0].ResumptionBB->bbNum); + resumingEdge = m_compiler->fgAddRefPred(m_states[0].ResumptionBB, newEntryBB); } - else if (m_resumptionBBs.size() == 2) + else if (m_states.size() == 2) { - BasicBlock* condBB = m_compiler->fgNewBBbefore(BBJ_COND, m_resumptionBBs[0], true); + BasicBlock* condBB = m_compiler->fgNewBBbefore(BBJ_COND, m_states[0].ResumptionBB, true); condBB->inheritWeightPercentage(newEntryBB, 0); - FlowEdge* to0 = m_compiler->fgAddRefPred(m_resumptionBBs[0], condBB); - FlowEdge* to1 = m_compiler->fgAddRefPred(m_resumptionBBs[1], condBB); + FlowEdge* to0 = m_compiler->fgAddRefPred(m_states[0].ResumptionBB, condBB); + FlowEdge* to1 = m_compiler->fgAddRefPred(m_states[1].ResumptionBB, condBB); condBB->SetCond(to1, to0); to1->setLikelihood(0.5); to0->setLikelihood(0.5); @@ -3025,13 +3111,13 @@ void AsyncTransformation::CreateResumptionSwitch() } else { - BasicBlock* switchBB = m_compiler->fgNewBBbefore(BBJ_SWITCH, m_resumptionBBs[0], true); + BasicBlock* switchBB = m_compiler->fgNewBBbefore(BBJ_SWITCH, m_states[0].ResumptionBB, true); switchBB->inheritWeightPercentage(newEntryBB, 0); resumingEdge = m_compiler->fgAddRefPred(switchBB, newEntryBB); JITDUMP(" Redirecting entry " FMT_BB " to BBJ_SWITCH " FMT_BB " for resumption with %zu states\n", - newEntryBB->bbNum, switchBB->bbNum, m_resumptionBBs.size()); + newEntryBB->bbNum, switchBB->bbNum, m_states.size()); continuationArg = m_compiler->gtNewLclvNode(m_compiler->lvaAsyncContinuationArg, TYP_REF); unsigned stateOffset = m_compiler->info.compCompHnd->getFieldOffset(m_asyncInfo->continuationStateFldHnd); @@ -3044,17 +3130,18 @@ void AsyncTransformation::CreateResumptionSwitch() m_compiler->fgHasSwitch = true; - // Default case. TODO-CQ: Support bbsHasDefault = false before lowering. - m_resumptionBBs.push_back(m_resumptionBBs[0]); - const size_t numCases = m_resumptionBBs.size(); + // Add 1 for default case + const size_t numCases = m_states.size() + 1; FlowEdge** const cases = new (m_compiler, CMK_FlowEdge) FlowEdge*[numCases * 2]; FlowEdge** const succs = cases + numCases; unsigned numUniqueSuccs = 0; - const weight_t stateLikelihood = 1.0 / m_resumptionBBs.size(); + const weight_t stateLikelihood = 1.0 / numCases; for (size_t i = 0; i < numCases; i++) { - FlowEdge* const edge = m_compiler->fgAddRefPred(m_resumptionBBs[i], switchBB); + // Wrap around and use first resumption BB as default case + BasicBlock* resumptionBB = m_states[i % m_states.size()].ResumptionBB; + FlowEdge* const edge = m_compiler->fgAddRefPred(resumptionBB, switchBB); edge->setLikelihood(stateLikelihood); cases[i] = edge; diff --git a/src/coreclr/jit/async.h b/src/coreclr/jit/async.h index ec35d6098db24a..81ec4f64599e1c 100644 --- a/src/coreclr/jit/async.h +++ b/src/coreclr/jit/async.h @@ -1,6 +1,36 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +struct ReturnTypeInfo +{ + var_types ReturnType = TYP_UNDEF; + ClassLayout* ReturnLayout = nullptr; + + ReturnTypeInfo(var_types returnType, ClassLayout* returnLayout) + : ReturnType(returnType) + , ReturnLayout(returnLayout) + { + } +}; + +struct ReturnInfo +{ + ReturnTypeInfo Type; + unsigned Alignment; + unsigned Offset; + unsigned Size; + + ReturnInfo(ReturnTypeInfo type) + : Type(type) + { + } + + unsigned HeapAlignment() const + { + return std::min(Alignment, (unsigned)TARGET_POINTER_SIZE); + } +}; + struct LiveLocalInfo { unsigned LclNum; @@ -29,14 +59,13 @@ struct ContinuationLayoutBuilder bool m_needsKeepAlive = false; bool m_needsExecutionContext = false; - var_types ReturnType = TYP_VOID; - ClassLayout* ReturnLayout = nullptr; - + jitstd::vector m_returns; jitstd::vector m_locals; public: ContinuationLayoutBuilder(Compiler* compiler) : m_compiler(compiler) + , m_returns(compiler->getAllocator(CMK_Async)) , m_locals(compiler->getAllocator(CMK_Async)) { } @@ -45,55 +74,78 @@ struct ContinuationLayoutBuilder { m_needsOSRILOffset = true; } + bool NeedsOSRILOffset() const + { + return m_needsOSRILOffset; + } void SetNeedsException() { m_needsException = true; } + bool NeedsException() const + { + return m_needsException; + } void SetNeedsContinuationContext() { m_needsContinuationContext = true; } + bool NeedsContinuationContext() const + { + return m_needsContinuationContext; + } void SetNeedsKeepAlive() { m_needsKeepAlive = true; } + bool NeedsKeepAlive() const + { + return m_needsKeepAlive; + } void SetNeedsExecutionContext() { m_needsExecutionContext = true; } - void SetReturn(var_types type, ClassLayout* layout); + bool NeedsExecutionContext() const + { + return m_needsExecutionContext; + } + void AddReturn(const ReturnTypeInfo& info); void AddLocal(unsigned lclNum); + bool ContainsLocal(unsigned lclNum) const; - const jitstd::vector Locals() const + const jitstd::vector& Locals() const { return m_locals; } + + struct ContinuationLayout* Create(); + + static ContinuationLayoutBuilder* CreateSharedLayout(Compiler* comp, const jitstd::vector& states); }; struct ContinuationLayout { - unsigned Size = 0; - unsigned OSRILOffset = UINT_MAX; - unsigned ExceptionOffset = UINT_MAX; - unsigned ContinuationContextOffset = UINT_MAX; - unsigned KeepAliveOffset = UINT_MAX; - ClassLayout* ReturnStructLayout = nullptr; - unsigned ReturnAlignment = 0; - unsigned ReturnSize = 0; - unsigned ReturnValOffset = UINT_MAX; - unsigned ExecutionContextOffset = UINT_MAX; - const jitstd::vector& Locals; - CORINFO_CLASS_HANDLE ClassHnd = NO_CLASS_HANDLE; - - explicit ContinuationLayout(const jitstd::vector& locals) - : Locals(locals) + unsigned Size = 0; + unsigned OSRILOffset = UINT_MAX; + unsigned ExceptionOffset = UINT_MAX; + unsigned ContinuationContextOffset = UINT_MAX; + unsigned KeepAliveOffset = UINT_MAX; + unsigned ExecutionContextOffset = UINT_MAX; + jitstd::vector Locals; + jitstd::vector Returns; + CORINFO_CLASS_HANDLE ClassHnd = NO_CLASS_HANDLE; + + ContinuationLayout(Compiler* comp) + : Locals(comp->getAllocator(CMK_Async)) + , Returns(comp->getAllocator(CMK_Async)) { } - unsigned ReturnHeapAlignment() const - { - return std::min(ReturnAlignment, (unsigned)TARGET_POINTER_SIZE); - } + const ReturnInfo* FindReturn(GenTreeCall* call) const; +#ifdef DEBUG + void Dump(int indent = 0); +#endif }; struct CallDefinitionInfo @@ -107,14 +159,26 @@ struct CallDefinitionInfo struct AsyncState { - AsyncState(ContinuationLayoutBuilder* layout, BasicBlock* suspensionBB, BasicBlock* resumptionBB) - : Layout(layout) + AsyncState( + unsigned number, + ContinuationLayoutBuilder* layout, + BasicBlock* callBlock, GenTreeCall* call, CallDefinitionInfo callDefInfo, + BasicBlock* suspensionBB, BasicBlock* resumptionBB) + : Number(number) + , Layout(layout) + , CallBlock(callBlock) + , Call(call) + , CallDefInfo(callDefInfo) , SuspensionBB(suspensionBB) , ResumptionBB(resumptionBB) { } + unsigned Number; ContinuationLayoutBuilder* Layout; + BasicBlock* CallBlock; + GenTreeCall* Call; + CallDefinitionInfo CallDefInfo; BasicBlock* SuspensionBB; BasicBlock* ResumptionBB; }; @@ -165,34 +229,33 @@ class AsyncTransformation bool needsKeepAlive, ContinuationLayoutBuilder* layoutBuilder); - ContinuationLayout LayOutContinuation(BasicBlock* block, - GenTreeCall* call, - bool needsKeepAlive, - jitstd::vector& liveLocals); + ContinuationLayout LayOutContinuation(const ContinuationLayoutBuilder* layoutBuilder); CallDefinitionInfo CanonicalizeCallDefinition(BasicBlock* block, GenTreeCall* call, AsyncLiveness* life); BasicBlock* CreateSuspensionBlock(BasicBlock* block, GenTreeCall* call, unsigned stateNum); - BasicBlock* CreateSuspension( - BasicBlock* block, GenTreeCall* call, unsigned stateNum, AsyncLiveness& life, const ContinuationLayout& layout); - GenTreeCall* CreateAllocContinuationCall(AsyncLiveness& life, + void CreateSuspension( + BasicBlock* callBlock, GenTreeCall* call, BasicBlock* suspendBB, unsigned stateNum, const ContinuationLayout& layout, const ContinuationLayoutBuilder& subLayout); + + GenTreeCall* CreateAllocContinuationCall(bool hasKeepAlive, GenTree* prevContinuation, const ContinuationLayout& layout); - void FillInDataOnSuspension(GenTreeCall* call, const ContinuationLayout& layout, BasicBlock* suspendBB); + void FillInDataOnSuspension(GenTreeCall* call, const ContinuationLayout& layout, const ContinuationLayoutBuilder& subLayout, BasicBlock* suspendBB); void RestoreContexts(BasicBlock* block, GenTreeCall* call, BasicBlock* suspendBB); void CreateCheckAndSuspendAfterCall(BasicBlock* block, GenTreeCall* call, const CallDefinitionInfo& callDefInfo, BasicBlock* suspendBB, BasicBlock** remainder); - BasicBlock* CreateResumption(BasicBlock* block, - BasicBlock* remainder, + BasicBlock* CreateResumptionBlock(BasicBlock* remainder, GenTreeCall* call, unsigned stateNum, ContinuationLayoutBuilder* layoutBuilder); + void CreateResumption(BasicBlock* callBlock, GenTreeCall* call, + BasicBlock* resumeBB, const CallDefinitionInfo& callDefInfo, - unsigned stateNum, - const ContinuationLayout& layout); - void SetSuspendedIndicator(BasicBlock* block, BasicBlock* callBlock, GenTreeCall* call); - void RestoreFromDataOnResumption(const ContinuationLayout& layout, BasicBlock* resumeBB); + const ContinuationLayout& layout, + const ContinuationLayoutBuilder& subLayout); + + void RestoreFromDataOnResumption(const ContinuationLayout& layout, const ContinuationLayoutBuilder& subLayout, BasicBlock* resumeBB); BasicBlock* RethrowExceptionOnResumption(BasicBlock* block, const ContinuationLayout& layout, BasicBlock* resumeBB); void CopyReturnValueOnResumption(GenTreeCall* call, const CallDefinitionInfo& callDefInfo, @@ -209,13 +272,14 @@ class AsyncTransformation var_types storeType, GenTreeFlags indirFlags = GTF_IND_NONFAULTING); - void CreateDebugInfoForSuspensionPoint(const ContinuationLayout& layout); + void CreateDebugInfoForSuspensionPoint(const ContinuationLayout& layout, const ContinuationLayoutBuilder& subLayout); unsigned GetReturnedContinuationVar(); unsigned GetNewContinuationVar(); unsigned GetResultBaseVar(); unsigned GetExceptionVar(); BasicBlock* GetSharedReturnBB(); + void CreateResumptionsAndSuspensions(); void CreateResumptionSwitch(); public: diff --git a/src/coreclr/jit/jitconfigvalues.h b/src/coreclr/jit/jitconfigvalues.h index 7031d3c6851a91..7741e0f6b8fe19 100644 --- a/src/coreclr/jit/jitconfigvalues.h +++ b/src/coreclr/jit/jitconfigvalues.h @@ -592,7 +592,6 @@ OPT_CONFIG_INTEGER(JitDoOptimizeMaskConversions, "JitDoOptimizeMaskConversions", OPT_CONFIG_INTEGER(JitOptimizeAwait, "JitOptimizeAwait", 1) // Perform optimization of Await intrinsics OPT_CONFIG_STRING(JitAsyncDefaultValueAnalysisRange, "JitAsyncDefaultValueAnalysisRange") // Enable async default value analysis based on method hash range -OPT_CONFIG_INTEGER(JitAsyncReuseContinuations, "JitAsyncReuseContinuations", 1) // Save and reuse continuations in runtime async functions RELEASE_CONFIG_INTEGER(JitEnableOptRepeat, "JitEnableOptRepeat", 1) // If zero, do not allow JitOptRepeat RELEASE_CONFIG_METHODSET(JitOptRepeat, "JitOptRepeat") // Runs optimizer multiple times on specified methods diff --git a/src/coreclr/jit/jitstd/vector.h b/src/coreclr/jit/jitstd/vector.h index 78dadd3651e87a..2d0a91210ecc95 100644 --- a/src/coreclr/jit/jitstd/vector.h +++ b/src/coreclr/jit/jitstd/vector.h @@ -218,6 +218,8 @@ class vector T* data() { return m_pArray; } + const T* data() const { return m_pArray; } + void swap(vector& vec); private: From 395395eeb42a5f982884780d9a4d224a92b8497b Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 12 Mar 2026 15:23:43 +0100 Subject: [PATCH 03/28] Run jit-format --- src/coreclr/jit/async.cpp | 83 +++++++++++++++--------- src/coreclr/jit/async.h | 132 +++++++++++++++++++++----------------- 2 files changed, 126 insertions(+), 89 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 2e1d1e770a26db..ebbc10dccad6bc 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -1590,11 +1590,15 @@ class GCPointerBitMapBuilder } }; -void AsyncTransformation::BuildContinuation(BasicBlock* block, GenTreeCall* call, bool needsKeepAlive, ContinuationLayoutBuilder* layoutBuilder) +void AsyncTransformation::BuildContinuation(BasicBlock* block, + GenTreeCall* call, + bool needsKeepAlive, + ContinuationLayoutBuilder* layoutBuilder) { if (call->gtReturnType != TYP_VOID) { - ClassLayout* layout = call->gtReturnType == TYP_STRUCT ? m_compiler->typGetObjLayout(call->gtRetClsHnd) : nullptr; + ClassLayout* layout = + call->gtReturnType == TYP_STRUCT ? m_compiler->typGetObjLayout(call->gtRetClsHnd) : nullptr; layoutBuilder->AddReturn(ReturnTypeInfo(call->gtReturnType, layout)); JITDUMP(" Call has return; continuation will have return value\n"); } @@ -1674,7 +1678,9 @@ void ContinuationLayout::Dump(int indent) for (const ReturnInfo& ret : Returns) { - printf("%*s +%03u %u bytes for %s return\n", indent, "", ret.Offset, ret.Size, ret.Type.ReturnType == TYP_STRUCT ? ret.Type.ReturnLayout->GetClassName() : varTypeName(ret.Type.ReturnType)); + printf("%*s +%03u %u bytes for %s return\n", indent, "", ret.Offset, ret.Size, + ret.Type.ReturnType == TYP_STRUCT ? ret.Type.ReturnLayout->GetClassName() + : varTypeName(ret.Type.ReturnType)); } } #endif @@ -1683,7 +1689,8 @@ const ReturnInfo* ContinuationLayout::FindReturn(GenTreeCall* call) const { for (const ReturnInfo& ret : Returns) { - if ((ret.Type.ReturnType == call->gtReturnType) && ((call->gtReturnType != TYP_STRUCT) || (ret.Type.ReturnLayout->GetClassHandle() == call->gtRetClsHnd))) + if ((ret.Type.ReturnType == call->gtReturnType) && + ((call->gtReturnType != TYP_STRUCT) || (ret.Type.ReturnLayout->GetClassHandle() == call->gtRetClsHnd))) { return &ret; } @@ -1700,7 +1707,7 @@ ContinuationLayout* ContinuationLayoutBuilder::Create() for (unsigned lclNum : m_locals) { - LclVarDsc* dsc = m_compiler->lvaGetDesc(lclNum); + LclVarDsc* dsc = m_compiler->lvaGetDesc(lclNum); LiveLocalInfo inf(lclNum); if (dsc->TypeIs(TYP_STRUCT) || dsc->IsImplicitByRef()) @@ -1762,11 +1769,12 @@ ContinuationLayout* ContinuationLayoutBuilder::Create() if (ret.ReturnType == TYP_STRUCT) { retInfo.Size = ret.ReturnLayout->GetSize(); - retInfo.Alignment = m_compiler->info.compCompHnd->getClassAlignmentRequirement(ret.ReturnLayout->GetClassHandle()); + retInfo.Alignment = + m_compiler->info.compCompHnd->getClassAlignmentRequirement(ret.ReturnLayout->GetClassHandle()); } else { - retInfo.Size = genTypeSize(ret.ReturnType); + retInfo.Size = genTypeSize(ret.ReturnType); retInfo.Alignment = retInfo.Size; } @@ -1774,7 +1782,7 @@ ContinuationLayout* ContinuationLayoutBuilder::Create() } auto allocLayout = [layout](unsigned align, unsigned size) { - layout->Size = roundUp(layout->Size, align); + layout->Size = roundUp(layout->Size, align); unsigned offset = layout->Size; layout->Size += size; return offset; @@ -2033,8 +2041,12 @@ BasicBlock* AsyncTransformation::CreateSuspensionBlock(BasicBlock* block, GenTre // Returns: // The new basic block that was created. // -void AsyncTransformation::CreateSuspension( - BasicBlock* callBlock, GenTreeCall* call, BasicBlock* suspendBB, unsigned stateNum, const ContinuationLayout& layout, const ContinuationLayoutBuilder& subLayout) +void AsyncTransformation::CreateSuspension(BasicBlock* callBlock, + GenTreeCall* call, + BasicBlock* suspendBB, + unsigned stateNum, + const ContinuationLayout& layout, + const ContinuationLayoutBuilder& subLayout) { GenTreeILOffset* ilOffsetNode = m_compiler->gtNewILOffsetNode(call->GetAsyncInfo().CallAsyncDebugInfo DEBUGARG(BAD_IL_OFFSET)); @@ -2048,7 +2060,8 @@ void AsyncTransformation::CreateSuspension( // Allocate continuation GenTree* returnedContinuation = m_compiler->gtNewLclvNode(GetReturnedContinuationVar(), TYP_REF); - GenTreeCall* allocContinuation = CreateAllocContinuationCall(subLayout.NeedsKeepAlive(), returnedContinuation, layout); + GenTreeCall* allocContinuation = + CreateAllocContinuationCall(subLayout.NeedsKeepAlive(), returnedContinuation, layout); m_compiler->compCurBB = suspendBB; m_compiler->fgMorphTree(allocContinuation); @@ -2120,7 +2133,7 @@ void AsyncTransformation::CreateSuspension( // Returns: // IR node representing the allocation. // -GenTreeCall* AsyncTransformation::CreateAllocContinuationCall(bool hasKeepAlive, +GenTreeCall* AsyncTransformation::CreateAllocContinuationCall(bool hasKeepAlive, GenTree* prevContinuation, const ContinuationLayout& layout) { @@ -2155,10 +2168,10 @@ GenTreeCall* AsyncTransformation::CreateAllocContinuationCall(bool hasKeepAlive, // layout - Information about the continuation layout // suspendBB - Basic block to add IR to. // -void AsyncTransformation::FillInDataOnSuspension(GenTreeCall* call, - const ContinuationLayout& layout, +void AsyncTransformation::FillInDataOnSuspension(GenTreeCall* call, + const ContinuationLayout& layout, const ContinuationLayoutBuilder& subLayout, - BasicBlock* suspendBB) + BasicBlock* suspendBB) { if (m_compiler->doesMethodHavePatchpoints() || m_compiler->opts.IsOSR()) { @@ -2444,7 +2457,10 @@ void AsyncTransformation::CreateCheckAndSuspendAfterCall(BasicBlock* block->GetFalseEdge()->setLikelihood(1); } -BasicBlock* AsyncTransformation::CreateResumptionBlock(BasicBlock* remainder, GenTreeCall* call, unsigned stateNum, ContinuationLayoutBuilder* layoutBuilder) +BasicBlock* AsyncTransformation::CreateResumptionBlock(BasicBlock* remainder, + GenTreeCall* call, + unsigned stateNum, + ContinuationLayoutBuilder* layoutBuilder) { if (m_lastResumptionBB == nullptr) { @@ -2483,12 +2499,12 @@ BasicBlock* AsyncTransformation::CreateResumptionBlock(BasicBlock* remainder, Ge // Returns: // The new basic block that was created. // -void AsyncTransformation::CreateResumption(BasicBlock* callBlock, - GenTreeCall* call, - BasicBlock* resumeBB, - const CallDefinitionInfo& callDefInfo, - const ContinuationLayout& layout, - const ContinuationLayoutBuilder& subLayout) +void AsyncTransformation::CreateResumption(BasicBlock* callBlock, + GenTreeCall* call, + BasicBlock* resumeBB, + const CallDefinitionInfo& callDefInfo, + const ContinuationLayout& layout, + const ContinuationLayoutBuilder& subLayout) { GenTreeILOffset* ilOffsetNode = m_compiler->gtNewILOffsetNode(call->GetAsyncInfo().CallAsyncDebugInfo DEBUGARG(BAD_IL_OFFSET)); @@ -2522,7 +2538,9 @@ void AsyncTransformation::CreateResumption(BasicBlock* callBlock, // layout - Information about the continuation layout // resumeBB - Basic block to append IR to // -void AsyncTransformation::RestoreFromDataOnResumption(const ContinuationLayout& layout, const ContinuationLayoutBuilder& subLayout, BasicBlock* resumeBB) +void AsyncTransformation::RestoreFromDataOnResumption(const ContinuationLayout& layout, + const ContinuationLayoutBuilder& subLayout, + BasicBlock* resumeBB) { if (subLayout.NeedsExecutionContext()) { @@ -2850,7 +2868,8 @@ GenTreeStoreInd* AsyncTransformation::StoreAtOffset( // Parameters: // layout - Layout of continuation // -void AsyncTransformation::CreateDebugInfoForSuspensionPoint(const ContinuationLayout& layout, const ContinuationLayoutBuilder& subLayout) +void AsyncTransformation::CreateDebugInfoForSuspensionPoint(const ContinuationLayout& layout, + const ContinuationLayoutBuilder& subLayout) { uint32_t numLocals = 0; for (const LiveLocalInfo& inf : layout.Locals) @@ -3002,7 +3021,8 @@ void AsyncTransformation::CreateResumptionsAndSuspensions() JITDUMP("Creating suspensions and resumptions for %zu states\n", m_states.size()); for (const AsyncState& state : m_states) { - JITDUMP("State %u suspend @ " FMT_BB ", resume @ " FMT_BB "\n", state.Number, state.SuspensionBB->bbNum, state.ResumptionBB->bbNum); + JITDUMP("State %u suspend @ " FMT_BB ", resume @ " FMT_BB "\n", state.Number, state.SuspensionBB->bbNum, + state.ResumptionBB->bbNum); ContinuationLayout* layout = state.Layout->Create(); CreateSuspension(state.CallBlock, state.Call, state.SuspensionBB, state.Number, *layout, *state.Layout); CreateResumption(state.CallBlock, state.Call, state.ResumptionBB, state.CallDefInfo, *layout, *state.Layout); @@ -3012,7 +3032,8 @@ void AsyncTransformation::CreateResumptionsAndSuspensions() } } -ContinuationLayoutBuilder* ContinuationLayoutBuilder::CreateSharedLayout(Compiler* comp, const jitstd::vector& states) +ContinuationLayoutBuilder* ContinuationLayoutBuilder::CreateSharedLayout(Compiler* comp, + const jitstd::vector& states) { unsigned maxLocalStored = 0; for (const AsyncState& state : states) @@ -3025,8 +3046,8 @@ ContinuationLayoutBuilder* ContinuationLayoutBuilder::CreateSharedLayout(Compile } ContinuationLayoutBuilder* sharedLayout = new (comp, CMK_Async) ContinuationLayoutBuilder(comp); - BitVecTraits traits(maxLocalStored + 1, comp); - BitVec locals(BitVecOps::MakeEmpty(&traits)); + BitVecTraits traits(maxLocalStored + 1, comp); + BitVec locals(BitVecOps::MakeEmpty(&traits)); for (const AsyncState& state : states) { @@ -3052,7 +3073,7 @@ ContinuationLayoutBuilder* ContinuationLayoutBuilder::CreateSharedLayout(Compile BitVecOps::VisitBits(&traits, locals, [=](unsigned localNum) { sharedLayout->AddLocal(localNum); return true; - }); + }); return sharedLayout; } @@ -3140,8 +3161,8 @@ void AsyncTransformation::CreateResumptionSwitch() for (size_t i = 0; i < numCases; i++) { // Wrap around and use first resumption BB as default case - BasicBlock* resumptionBB = m_states[i % m_states.size()].ResumptionBB; - FlowEdge* const edge = m_compiler->fgAddRefPred(resumptionBB, switchBB); + BasicBlock* resumptionBB = m_states[i % m_states.size()].ResumptionBB; + FlowEdge* const edge = m_compiler->fgAddRefPred(resumptionBB, switchBB); edge->setLikelihood(stateLikelihood); cases[i] = edge; diff --git a/src/coreclr/jit/async.h b/src/coreclr/jit/async.h index 81ec4f64599e1c..36392f9b8c6fd5 100644 --- a/src/coreclr/jit/async.h +++ b/src/coreclr/jit/async.h @@ -3,7 +3,7 @@ struct ReturnTypeInfo { - var_types ReturnType = TYP_UNDEF; + var_types ReturnType = TYP_UNDEF; ClassLayout* ReturnLayout = nullptr; ReturnTypeInfo(var_types returnType, ClassLayout* returnLayout) @@ -16,9 +16,9 @@ struct ReturnTypeInfo struct ReturnInfo { ReturnTypeInfo Type; - unsigned Alignment; - unsigned Offset; - unsigned Size; + unsigned Alignment; + unsigned Offset; + unsigned Size; ReturnInfo(ReturnTypeInfo type) : Type(type) @@ -53,14 +53,14 @@ struct ContinuationLayoutBuilder { private: Compiler* m_compiler; - bool m_needsOSRILOffset = false; - bool m_needsException = false; - bool m_needsContinuationContext = false; - bool m_needsKeepAlive = false; - bool m_needsExecutionContext = false; + bool m_needsOSRILOffset = false; + bool m_needsException = false; + bool m_needsContinuationContext = false; + bool m_needsKeepAlive = false; + bool m_needsExecutionContext = false; jitstd::vector m_returns; - jitstd::vector m_locals; + jitstd::vector m_locals; public: ContinuationLayoutBuilder(Compiler* compiler) @@ -121,20 +121,21 @@ struct ContinuationLayoutBuilder struct ContinuationLayout* Create(); - static ContinuationLayoutBuilder* CreateSharedLayout(Compiler* comp, const jitstd::vector& states); + static ContinuationLayoutBuilder* CreateSharedLayout(Compiler* comp, + const jitstd::vector& states); }; struct ContinuationLayout { - unsigned Size = 0; - unsigned OSRILOffset = UINT_MAX; - unsigned ExceptionOffset = UINT_MAX; - unsigned ContinuationContextOffset = UINT_MAX; - unsigned KeepAliveOffset = UINT_MAX; - unsigned ExecutionContextOffset = UINT_MAX; + unsigned Size = 0; + unsigned OSRILOffset = UINT_MAX; + unsigned ExceptionOffset = UINT_MAX; + unsigned ContinuationContextOffset = UINT_MAX; + unsigned KeepAliveOffset = UINT_MAX; + unsigned ExecutionContextOffset = UINT_MAX; jitstd::vector Locals; - jitstd::vector Returns; - CORINFO_CLASS_HANDLE ClassHnd = NO_CLASS_HANDLE; + jitstd::vector Returns; + CORINFO_CLASS_HANDLE ClassHnd = NO_CLASS_HANDLE; ContinuationLayout(Compiler* comp) : Locals(comp->getAllocator(CMK_Async)) @@ -159,11 +160,13 @@ struct CallDefinitionInfo struct AsyncState { - AsyncState( - unsigned number, - ContinuationLayoutBuilder* layout, - BasicBlock* callBlock, GenTreeCall* call, CallDefinitionInfo callDefInfo, - BasicBlock* suspensionBB, BasicBlock* resumptionBB) + AsyncState(unsigned number, + ContinuationLayoutBuilder* layout, + BasicBlock* callBlock, + GenTreeCall* call, + CallDefinitionInfo callDefInfo, + BasicBlock* suspensionBB, + BasicBlock* resumptionBB) : Number(number) , Layout(layout) , CallBlock(callBlock) @@ -174,31 +177,31 @@ struct AsyncState { } - unsigned Number; + unsigned Number; ContinuationLayoutBuilder* Layout; - BasicBlock* CallBlock; - GenTreeCall* Call; - CallDefinitionInfo CallDefInfo; - BasicBlock* SuspensionBB; - BasicBlock* ResumptionBB; + BasicBlock* CallBlock; + GenTreeCall* Call; + CallDefinitionInfo CallDefInfo; + BasicBlock* SuspensionBB; + BasicBlock* ResumptionBB; }; class AsyncTransformation { friend class AsyncLiveness; - Compiler* m_compiler; - CORINFO_ASYNC_INFO* m_asyncInfo; - jitstd::vector m_states; - unsigned m_returnedContinuationVar = BAD_VAR_NUM; - unsigned m_newContinuationVar = BAD_VAR_NUM; - unsigned m_dataArrayVar = BAD_VAR_NUM; - unsigned m_gcDataArrayVar = BAD_VAR_NUM; - unsigned m_resultBaseVar = BAD_VAR_NUM; - unsigned m_exceptionVar = BAD_VAR_NUM; - BasicBlock* m_lastSuspensionBB = nullptr; - BasicBlock* m_lastResumptionBB = nullptr; - BasicBlock* m_sharedReturnBB = nullptr; + Compiler* m_compiler; + CORINFO_ASYNC_INFO* m_asyncInfo; + jitstd::vector m_states; + unsigned m_returnedContinuationVar = BAD_VAR_NUM; + unsigned m_newContinuationVar = BAD_VAR_NUM; + unsigned m_dataArrayVar = BAD_VAR_NUM; + unsigned m_gcDataArrayVar = BAD_VAR_NUM; + unsigned m_resultBaseVar = BAD_VAR_NUM; + unsigned m_exceptionVar = BAD_VAR_NUM; + BasicBlock* m_lastSuspensionBB = nullptr; + BasicBlock* m_lastResumptionBB = nullptr; + BasicBlock* m_sharedReturnBB = nullptr; void TransformTailAwait(BasicBlock* block, GenTreeCall* call, BasicBlock** remainder); BasicBlock* CreateTailAwaitSuspension(BasicBlock* block, GenTreeCall* call); @@ -224,9 +227,9 @@ class AsyncTransformation bool ContinuationNeedsKeepAlive(class AsyncLiveness& life); - void BuildContinuation(BasicBlock* block, - GenTreeCall* call, - bool needsKeepAlive, + void BuildContinuation(BasicBlock* block, + GenTreeCall* call, + bool needsKeepAlive, ContinuationLayoutBuilder* layoutBuilder); ContinuationLayout LayOutContinuation(const ContinuationLayoutBuilder* layoutBuilder); @@ -234,28 +237,40 @@ class AsyncTransformation CallDefinitionInfo CanonicalizeCallDefinition(BasicBlock* block, GenTreeCall* call, AsyncLiveness* life); BasicBlock* CreateSuspensionBlock(BasicBlock* block, GenTreeCall* call, unsigned stateNum); - void CreateSuspension( - BasicBlock* callBlock, GenTreeCall* call, BasicBlock* suspendBB, unsigned stateNum, const ContinuationLayout& layout, const ContinuationLayoutBuilder& subLayout); - - GenTreeCall* CreateAllocContinuationCall(bool hasKeepAlive, + void CreateSuspension(BasicBlock* callBlock, + GenTreeCall* call, + BasicBlock* suspendBB, + unsigned stateNum, + const ContinuationLayout& layout, + const ContinuationLayoutBuilder& subLayout); + + GenTreeCall* CreateAllocContinuationCall(bool hasKeepAlive, GenTree* prevContinuation, const ContinuationLayout& layout); - void FillInDataOnSuspension(GenTreeCall* call, const ContinuationLayout& layout, const ContinuationLayoutBuilder& subLayout, BasicBlock* suspendBB); + void FillInDataOnSuspension(GenTreeCall* call, + const ContinuationLayout& layout, + const ContinuationLayoutBuilder& subLayout, + BasicBlock* suspendBB); void RestoreContexts(BasicBlock* block, GenTreeCall* call, BasicBlock* suspendBB); void CreateCheckAndSuspendAfterCall(BasicBlock* block, GenTreeCall* call, const CallDefinitionInfo& callDefInfo, BasicBlock* suspendBB, BasicBlock** remainder); - BasicBlock* CreateResumptionBlock(BasicBlock* remainder, GenTreeCall* call, unsigned stateNum, ContinuationLayoutBuilder* layoutBuilder); - void CreateResumption(BasicBlock* callBlock, - GenTreeCall* call, - BasicBlock* resumeBB, - const CallDefinitionInfo& callDefInfo, - const ContinuationLayout& layout, + BasicBlock* CreateResumptionBlock(BasicBlock* remainder, + GenTreeCall* call, + unsigned stateNum, + ContinuationLayoutBuilder* layoutBuilder); + void CreateResumption(BasicBlock* callBlock, + GenTreeCall* call, + BasicBlock* resumeBB, + const CallDefinitionInfo& callDefInfo, + const ContinuationLayout& layout, const ContinuationLayoutBuilder& subLayout); - void RestoreFromDataOnResumption(const ContinuationLayout& layout, const ContinuationLayoutBuilder& subLayout, BasicBlock* resumeBB); + void RestoreFromDataOnResumption(const ContinuationLayout& layout, + const ContinuationLayoutBuilder& subLayout, + BasicBlock* resumeBB); BasicBlock* RethrowExceptionOnResumption(BasicBlock* block, const ContinuationLayout& layout, BasicBlock* resumeBB); void CopyReturnValueOnResumption(GenTreeCall* call, const CallDefinitionInfo& callDefInfo, @@ -272,7 +287,8 @@ class AsyncTransformation var_types storeType, GenTreeFlags indirFlags = GTF_IND_NONFAULTING); - void CreateDebugInfoForSuspensionPoint(const ContinuationLayout& layout, const ContinuationLayoutBuilder& subLayout); + void CreateDebugInfoForSuspensionPoint(const ContinuationLayout& layout, + const ContinuationLayoutBuilder& subLayout); unsigned GetReturnedContinuationVar(); unsigned GetNewContinuationVar(); unsigned GetResultBaseVar(); From d9a8e7f3df8b56a759df15b715f44f8c15229583 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 12 Mar 2026 15:34:06 +0100 Subject: [PATCH 04/28] Add function headers --- src/coreclr/jit/async.cpp | 194 ++++++++++++++++++++++++++++++-------- 1 file changed, 157 insertions(+), 37 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index ebbc10dccad6bc..03670b2d47d9da 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -45,6 +45,15 @@ #include "jitstd/algorithm.h" #include "async.h" +//------------------------------------------------------------------------ +// SetCallEntrypointForR2R: +// Set the entrypoint for a call when compiling for Ready-to-Run. +// +// Parameters: +// call - The call node to set the entrypoint on. +// compiler - The compiler instance. +// handle - The method handle to look up the entrypoint for. +// static void SetCallEntrypointForR2R(GenTreeCall* call, Compiler* compiler, CORINFO_METHOD_HANDLE handle) { #ifdef FEATURE_READYTORUN @@ -438,6 +447,14 @@ BasicBlock* Compiler::CreateReturnBB(unsigned* mergedReturnLcl) return newReturnBB; } +//------------------------------------------------------------------------ +// ContinuationLayoutBuilder::AddReturn: +// Add a return type to the continuation layout, if it is not already +// present. +// +// Parameters: +// info - The return type information to add. +// void ContinuationLayoutBuilder::AddReturn(const ReturnTypeInfo& info) { for (const ReturnTypeInfo& ret : m_returns) @@ -452,12 +469,30 @@ void ContinuationLayoutBuilder::AddReturn(const ReturnTypeInfo& info) m_returns.push_back(info); } +//------------------------------------------------------------------------ +// ContinuationLayoutBuilder::AddLocal: +// Add a local to the continuation layout. Locals must be added in +// ascending order by local number. +// +// Parameters: +// lclNum - The local number to add. +// void ContinuationLayoutBuilder::AddLocal(unsigned lclNum) { assert(m_locals.empty() || (lclNum > m_locals[m_locals.size() - 1])); m_locals.push_back(lclNum); } +//------------------------------------------------------------------------ +// ContinuationLayoutBuilder::ContainsLocal: +// Check if the specified local is in this layout. +// +// Parameters: +// lclNum - The local number to check. +// +// Returns: +// True if the local is contained in this layout. +// bool ContinuationLayoutBuilder::ContainsLocal(unsigned lclNum) const { return BinarySearch(m_locals.data(), (int)m_locals.size(), lclNum) != nullptr; @@ -616,7 +651,6 @@ static bool IsDefaultValue(GenTree* node) return node->IsIntegralConst(0) || node->IsFloatPositiveZero() || node->IsVectorZero(); } -//------------------------------------------------------------------------ //------------------------------------------------------------------------ // UpdateMutatedLocal: // If the given node is a local store or LCL_ADDR, and the local is tracked, @@ -1590,6 +1624,19 @@ class GCPointerBitMapBuilder } }; +//------------------------------------------------------------------------ +// AsyncTransformation::BuildContinuation: +// Determine what fields the continuation object needs for the specified +// async call, including return values, exception, context, and OSR +// support, and configure the layout builder accordingly. +// +// Parameters: +// block - The block containing the async call. +// call - The async call. +// needsKeepAlive - Whether a KeepAlive field is needed to keep a +// LoaderAllocator alive. +// layoutBuilder - The continuation layout builder to configure. +// void AsyncTransformation::BuildContinuation(BasicBlock* block, GenTreeCall* call, bool needsKeepAlive, @@ -1643,6 +1690,14 @@ void AsyncTransformation::BuildContinuation(BasicBlock* block, } #ifdef DEBUG +//------------------------------------------------------------------------ +// ContinuationLayout::Dump: +// Debug helper to print the continuation layout including offsets of all +// fields and locals. +// +// Parameters: +// indent - Number of spaces to indent the output. +// void ContinuationLayout::Dump(int indent) { printf("%*sContinuation layout (%u bytes):\n", indent, "", Size); @@ -1685,6 +1740,16 @@ void ContinuationLayout::Dump(int indent) } #endif +//------------------------------------------------------------------------ +// ContinuationLayout::FindReturn: +// Find the return info entry matching the specified call's return type. +// +// Parameters: +// call - The async call whose return type to look up. +// +// Returns: +// Pointer to the matching ReturnInfo entry. +// const ReturnInfo* ContinuationLayout::FindReturn(GenTreeCall* call) const { for (const ReturnInfo& ret : Returns) @@ -1700,6 +1765,15 @@ const ReturnInfo* ContinuationLayout::FindReturn(GenTreeCall* call) const return nullptr; } +//------------------------------------------------------------------------ +// ContinuationLayoutBuilder::Create: +// Finalize the layout by computing offsets for all fields, locals, and +// return values. Allocates the continuation type from the VM. +// +// Returns: +// The finalized ContinuationLayout with computed offsets and a class +// handle for the continuation type. +// ContinuationLayout* ContinuationLayoutBuilder::Create() { ContinuationLayout* layout = new (m_compiler, CMK_Async) ContinuationLayout(m_compiler); @@ -2003,6 +2077,19 @@ CallDefinitionInfo AsyncTransformation::CanonicalizeCallDefinition(BasicBlock* return callDefInfo; } +//------------------------------------------------------------------------ +// AsyncTransformation::CreateSuspensionBlock: +// Create an empty basic block that will hold the suspension IR for the +// specified async call. +// +// Parameters: +// block - The block containing the async call. +// call - The async call. +// stateNum - State number assigned to this suspension point. +// +// Returns: +// The new basic block. +// BasicBlock* AsyncTransformation::CreateSuspensionBlock(BasicBlock* block, GenTreeCall* call, unsigned stateNum) { if (m_lastSuspensionBB == nullptr) @@ -2028,18 +2115,17 @@ BasicBlock* AsyncTransformation::CreateSuspensionBlock(BasicBlock* block, GenTre //------------------------------------------------------------------------ // AsyncTransformation::CreateSuspension: -// Create the basic block that when branched to suspends execution after the -// specified async call. +// Populate the suspension block with IR that allocates a continuation +// object, fills in its state, flags, resume info, data, and restores +// contexts. // // Parameters: -// block - The block containing the async call -// call - The async call -// stateNum - State number assigned to this suspension point -// life - Liveness information about live locals -// layout - Layout information for the continuation object -// -// Returns: -// The new basic block that was created. +// callBlock - The block containing the async call. +// call - The async call. +// suspendBB - The suspension basic block to add IR to. +// stateNum - State number assigned to this suspension point. +// layout - Layout information for the continuation object. +// subLayout - Per-call layout builder indicating which fields are needed. // void AsyncTransformation::CreateSuspension(BasicBlock* callBlock, GenTreeCall* call, @@ -2126,9 +2212,9 @@ void AsyncTransformation::CreateSuspension(BasicBlock* call // Create a call to the JIT helper that allocates a continuation. // // Parameters: -// life - Liveness information about live locals -// prevContinuation - IR node that has the value of the previous continuation object -// layout - Layout information +// hasKeepAlive - Whether the continuation needs a KeepAlive field. +// prevContinuation - IR node that has the value of the previous continuation object. +// layout - Layout information for the continuation. // // Returns: // IR node representing the allocation. @@ -2161,11 +2247,14 @@ GenTreeCall* AsyncTransformation::CreateAllocContinuationCall(bool //------------------------------------------------------------------------ // AsyncTransformation::FillInDataOnSuspension: -// Create IR that fills the data array of the continuation object. +// Create IR that fills the data array of the continuation object with +// live local values, OSR IL offset, continuation context, and execution +// context. // // Parameters: -// call - The async call -// layout - Information about the continuation layout +// call - The async call. +// layout - Information about the continuation layout. +// subLayout - Per-call layout builder indicating which fields are needed. // suspendBB - Basic block to add IR to. // void AsyncTransformation::FillInDataOnSuspension(GenTreeCall* call, @@ -2457,6 +2546,20 @@ void AsyncTransformation::CreateCheckAndSuspendAfterCall(BasicBlock* block->GetFalseEdge()->setLikelihood(1); } +//------------------------------------------------------------------------ +// AsyncTransformation::CreateResumptionBlock: +// Create an empty basic block that will hold the resumption IR for the +// specified async call. +// +// Parameters: +// remainder - The block that contains the IR after the async call. +// call - The async call. +// stateNum - State number assigned to this suspension point. +// layoutBuilder - The continuation layout builder for this call. +// +// Returns: +// The new basic block. +// BasicBlock* AsyncTransformation::CreateResumptionBlock(BasicBlock* remainder, GenTreeCall* call, unsigned stateNum, @@ -2485,19 +2588,17 @@ BasicBlock* AsyncTransformation::CreateResumptionBlock(BasicBlock* //------------------------------------------------------------------------ // AsyncTransformation::CreateResumption: -// Create the basic block that when branched to resumes execution on entry to -// the function. +// Populate the resumption block with IR that restores live state from +// the continuation object, rethrows exceptions if necessary, and copies +// the return value. // // Parameters: -// block - The block containing the async call -// remainder - The block that contains the IR after the (split) async call -// call - The async call -// callDefInfo - Information about the async call's definition -// stateNum - State number assigned to this suspension point -// layout - Layout information for the continuation object -// -// Returns: -// The new basic block that was created. +// callBlock - The block containing the async call. +// call - The async call. +// resumeBB - The resumption basic block to add IR to. +// callDefInfo - Information about the async call's definition. +// layout - Layout information for the continuation object. +// subLayout - Per-call layout builder indicating which fields are needed. // void AsyncTransformation::CreateResumption(BasicBlock* callBlock, GenTreeCall* call, @@ -2532,11 +2633,12 @@ void AsyncTransformation::CreateResumption(BasicBlock* call //------------------------------------------------------------------------ // AsyncTransformation::RestoreFromDataOnResumption: // Create IR that restores locals from the data array of the continuation -// object. +// object, including execution context restoration. // // Parameters: -// layout - Information about the continuation layout -// resumeBB - Basic block to append IR to +// layout - Information about the continuation layout. +// subLayout - Per-call layout builder indicating which fields are needed. +// resumeBB - Basic block to append IR to. // void AsyncTransformation::RestoreFromDataOnResumption(const ContinuationLayout& layout, const ContinuationLayoutBuilder& subLayout, @@ -2711,11 +2813,10 @@ BasicBlock* AsyncTransformation::RethrowExceptionOnResumption(BasicBlock* // right local. // // Parameters: -// call - The async call -// callDefInfo - Information about the async call's definition -// block - The block containing the async call -// layout - Layout information for the continuation object -// storeResultBB - Basic block to append IR to +// call - The async call. +// callDefInfo - Information about the async call's definition. +// layout - Layout information for the continuation object. +// storeResultBB - Basic block to append IR to. // void AsyncTransformation::CopyReturnValueOnResumption(GenTreeCall* call, const CallDefinitionInfo& callDefInfo, @@ -2866,7 +2967,8 @@ GenTreeStoreInd* AsyncTransformation::StoreAtOffset( // Create debug info for the specific suspension point we just created. // // Parameters: -// layout - Layout of continuation +// layout - Layout of continuation. +// subLayout - Per-call layout builder indicating which locals are present. // void AsyncTransformation::CreateDebugInfoForSuspensionPoint(const ContinuationLayout& layout, const ContinuationLayoutBuilder& subLayout) @@ -3016,6 +3118,11 @@ BasicBlock* AsyncTransformation::GetSharedReturnBB() #endif } +//------------------------------------------------------------------------ +// AsyncTransformation::CreateResumptionsAndSuspensions: +// Walk all recorded async states and create the suspension and resumption +// IR, continuation layouts, and debug info for each one. +// void AsyncTransformation::CreateResumptionsAndSuspensions() { JITDUMP("Creating suspensions and resumptions for %zu states\n", m_states.size()); @@ -3032,6 +3139,19 @@ void AsyncTransformation::CreateResumptionsAndSuspensions() } } +//------------------------------------------------------------------------ +// ContinuationLayoutBuilder::CreateSharedLayout: +// Create a shared continuation layout that is the union of all per-call +// layouts. The shared layout contains every local, return type, and +// optional field needed by any individual suspension point. +// +// Parameters: +// comp - The compiler instance. +// states - The vector of async states to merge. +// +// Returns: +// A new ContinuationLayoutBuilder representing the merged layout. +// ContinuationLayoutBuilder* ContinuationLayoutBuilder::CreateSharedLayout(Compiler* comp, const jitstd::vector& states) { From 8e54df169a4459e7272387faa70ec75c7c4eabc8 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 12 Mar 2026 15:41:40 +0100 Subject: [PATCH 05/28] Remove unused CreateSharedLayout --- src/coreclr/jit/async.cpp | 59 --------------------------------------- src/coreclr/jit/async.h | 3 -- 2 files changed, 62 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 03670b2d47d9da..2ad5fb24e66a9e 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -3139,65 +3139,6 @@ void AsyncTransformation::CreateResumptionsAndSuspensions() } } -//------------------------------------------------------------------------ -// ContinuationLayoutBuilder::CreateSharedLayout: -// Create a shared continuation layout that is the union of all per-call -// layouts. The shared layout contains every local, return type, and -// optional field needed by any individual suspension point. -// -// Parameters: -// comp - The compiler instance. -// states - The vector of async states to merge. -// -// Returns: -// A new ContinuationLayoutBuilder representing the merged layout. -// -ContinuationLayoutBuilder* ContinuationLayoutBuilder::CreateSharedLayout(Compiler* comp, - const jitstd::vector& states) -{ - unsigned maxLocalStored = 0; - for (const AsyncState& state : states) - { - jitstd::vector& locals = state.Layout->m_locals; - if (locals.size() > 0) - { - maxLocalStored = std::max(maxLocalStored, locals[locals.size() - 1]); - } - } - - ContinuationLayoutBuilder* sharedLayout = new (comp, CMK_Async) ContinuationLayoutBuilder(comp); - BitVecTraits traits(maxLocalStored + 1, comp); - BitVec locals(BitVecOps::MakeEmpty(&traits)); - - for (const AsyncState& state : states) - { - ContinuationLayoutBuilder* layout = state.Layout; - sharedLayout->m_needsOSRILOffset |= layout->m_needsOSRILOffset; - sharedLayout->m_needsException |= layout->m_needsException; - sharedLayout->m_needsContinuationContext |= layout->m_needsContinuationContext; - sharedLayout->m_needsKeepAlive |= layout->m_needsKeepAlive; - sharedLayout->m_needsExecutionContext |= layout->m_needsExecutionContext; - - for (unsigned local : layout->m_locals) - { - assert(local <= maxLocalStored); - BitVecOps::AddElemD(&traits, locals, local); - } - - for (const ReturnTypeInfo& ret : layout->m_returns) - { - sharedLayout->AddReturn(ret); - } - } - - BitVecOps::VisitBits(&traits, locals, [=](unsigned localNum) { - sharedLayout->AddLocal(localNum); - return true; - }); - - return sharedLayout; -} - //------------------------------------------------------------------------ // AsyncTransformation::CreateResumptionSwitch: // Create the IR for the entry of the function that checks the continuation diff --git a/src/coreclr/jit/async.h b/src/coreclr/jit/async.h index 36392f9b8c6fd5..b14f4bd2866bad 100644 --- a/src/coreclr/jit/async.h +++ b/src/coreclr/jit/async.h @@ -120,9 +120,6 @@ struct ContinuationLayoutBuilder } struct ContinuationLayout* Create(); - - static ContinuationLayoutBuilder* CreateSharedLayout(Compiler* comp, - const jitstd::vector& states); }; struct ContinuationLayout From e7987f47e9d6d85723267c0cb1c87b284f3145c7 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 12 Mar 2026 15:46:24 +0100 Subject: [PATCH 06/28] Feedback --- src/coreclr/jit/async.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 2ad5fb24e66a9e..6ddd9f67ee64c1 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -1881,7 +1881,7 @@ ContinuationLayout* ContinuationLayoutBuilder::Create() // Now allocate all returns for (ReturnInfo& ret : layout->Returns) { - ret.Offset = allocLayout(ret.Alignment, ret.Size); + ret.Offset = allocLayout(ret.HeapAlignment(), ret.Size); } if (m_needsKeepAlive) From 76334fea138f9bffce080e2223d35258770a4fce Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 12 Mar 2026 15:47:50 +0100 Subject: [PATCH 07/28] Feedback --- src/coreclr/jit/async.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 6ddd9f67ee64c1..d5ba26ca712b9a 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -1721,7 +1721,7 @@ void ContinuationLayout::Dump(int indent) printf("%*s +%03u Keep alive object\n", indent, "", KeepAliveOffset); } - if (ExecutionContextOffset) + if (ExecutionContextOffset != UINT_MAX) { printf("%*s +%03u Execution context\n", indent, "", ExecutionContextOffset); } From d2f3ed77181f43f824ac49489c9dc6a9a58f3cbd Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 12 Mar 2026 15:54:38 +0100 Subject: [PATCH 08/28] Revert "Remove unused CreateSharedLayout" This reverts commit 8e54df169a4459e7272387faa70ec75c7c4eabc8. --- src/coreclr/jit/async.cpp | 59 +++++++++++++++++++++++++++++++++++++++ src/coreclr/jit/async.h | 3 ++ 2 files changed, 62 insertions(+) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index d5ba26ca712b9a..4966bcf03e9d3c 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -3139,6 +3139,65 @@ void AsyncTransformation::CreateResumptionsAndSuspensions() } } +//------------------------------------------------------------------------ +// ContinuationLayoutBuilder::CreateSharedLayout: +// Create a shared continuation layout that is the union of all per-call +// layouts. The shared layout contains every local, return type, and +// optional field needed by any individual suspension point. +// +// Parameters: +// comp - The compiler instance. +// states - The vector of async states to merge. +// +// Returns: +// A new ContinuationLayoutBuilder representing the merged layout. +// +ContinuationLayoutBuilder* ContinuationLayoutBuilder::CreateSharedLayout(Compiler* comp, + const jitstd::vector& states) +{ + unsigned maxLocalStored = 0; + for (const AsyncState& state : states) + { + jitstd::vector& locals = state.Layout->m_locals; + if (locals.size() > 0) + { + maxLocalStored = std::max(maxLocalStored, locals[locals.size() - 1]); + } + } + + ContinuationLayoutBuilder* sharedLayout = new (comp, CMK_Async) ContinuationLayoutBuilder(comp); + BitVecTraits traits(maxLocalStored + 1, comp); + BitVec locals(BitVecOps::MakeEmpty(&traits)); + + for (const AsyncState& state : states) + { + ContinuationLayoutBuilder* layout = state.Layout; + sharedLayout->m_needsOSRILOffset |= layout->m_needsOSRILOffset; + sharedLayout->m_needsException |= layout->m_needsException; + sharedLayout->m_needsContinuationContext |= layout->m_needsContinuationContext; + sharedLayout->m_needsKeepAlive |= layout->m_needsKeepAlive; + sharedLayout->m_needsExecutionContext |= layout->m_needsExecutionContext; + + for (unsigned local : layout->m_locals) + { + assert(local <= maxLocalStored); + BitVecOps::AddElemD(&traits, locals, local); + } + + for (const ReturnTypeInfo& ret : layout->m_returns) + { + sharedLayout->AddReturn(ret); + } + } + + BitVecOps::VisitBits(&traits, locals, [=](unsigned localNum) { + sharedLayout->AddLocal(localNum); + return true; + }); + + return sharedLayout; +} + //------------------------------------------------------------------------ // AsyncTransformation::CreateResumptionSwitch: // Create the IR for the entry of the function that checks the continuation diff --git a/src/coreclr/jit/async.h b/src/coreclr/jit/async.h index b14f4bd2866bad..36392f9b8c6fd5 100644 --- a/src/coreclr/jit/async.h +++ b/src/coreclr/jit/async.h @@ -120,6 +120,9 @@ struct ContinuationLayoutBuilder } struct ContinuationLayout* Create(); + + static ContinuationLayoutBuilder* CreateSharedLayout(Compiler* comp, + const jitstd::vector& states); }; struct ContinuationLayout From 18e4d8d3091fc28b192b43d533e85c301b853a17 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 13 Mar 2026 09:23:49 +0100 Subject: [PATCH 09/28] WIP --- .../CompilerServices/AsyncHelpers.CoreCLR.cs | 70 +++++++++---------- src/coreclr/inc/corinfo.h | 31 ++++---- src/coreclr/interpreter/compiler.cpp | 6 +- src/coreclr/jit/async.cpp | 41 +++++++++-- src/coreclr/jit/jitconfigvalues.h | 1 + src/coreclr/vm/interpexec.cpp | 6 +- src/coreclr/vm/object.h | 34 ++++----- 7 files changed, 104 insertions(+), 85 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs index 40bbcdc14db299..05f86aeadf0846 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs @@ -57,33 +57,20 @@ public void Pop() [Flags] // Keep in sync with CORINFO_CONTINUATION_FLAGS - internal enum ContinuationFlags + internal enum ContinuationFlags : uint { - // Note: the following 'Has' members determine the members present at - // the beginning of the continuation's data chunk. Each field is - // pointer sized when present, apart from the result that has variable - // size. - - // Whether or not the continuation starts with an OSR IL offset. - HasOsrILOffset = 1, - // If this bit is set the continuation resumes inside a try block and - // thus if an exception is being propagated, needs to be resumed. - HasException = 2, - // If this bit is set the continuation has space for a continuation - // context. - HasContinuationContext = 4, - // If this bit is set the continuation has space to store a result - // returned by the callee. - HasResult = 8, - // If this bit is set the continuation should continue on the thread - // pool. - ContinueOnThreadPool = 16, - // If this bit is set the continuation context is a - // SynchronizationContext that we should continue on. - ContinueOnCapturedSynchronizationContext = 32, - // If this bit is set the continuation context is a TaskScheduler that - // we should continue on. - ContinueOnCapturedTaskScheduler = 64, + ContinueOnThreadPool = 1u << 0, + ContinueOnCapturedSynchronizationContext = 1u << 1, + ContinueOnCapturedTaskScheduler = 1u << 2, + + ExceptionIndexFirstBit = 3u, + ExceptionIndexNumBits = 2u, + + ContinuationContextIndexFirstBit = 5u, + ContinuationContextIndexNumBits = 2u, + + ResultIndexFirstBit = 7u, + ResultIndexBits = 26u, } // Keep in sync with CORINFO_AsyncResumeInfo in corinfo.h @@ -120,28 +107,37 @@ internal unsafe class Continuation public unsafe object GetContinuationContext() { - Debug.Assert((Flags & ContinuationFlags.HasContinuationContext) != 0); - uint contIndex = (uint)BitOperations.PopCount((uint)Flags & ((uint)ContinuationFlags.HasContinuationContext - 1)); + const uint mask = (1u << (int)ContinuationFlags.ContinuationContextIndexNumBits) - 1; + uint index = ((uint)Flags >> (int)ContinuationFlags.ContinuationContextIndexFirstBit) & mask; + Debug.Assert(index != mask); ref byte data = ref RuntimeHelpers.GetRawData(this); - return Unsafe.As(ref Unsafe.Add(ref data, DataOffset + contIndex * PointerSize)); + return Unsafe.As(ref Unsafe.Add(ref data, DataOffset + index * PointerSize)); + } + + public bool HasException() + { + const uint mask = (1u << (int)ContinuationFlags.ExceptionIndexNumBits) - 1; + uint index = ((uint)Flags >> (int)ContinuationFlags.ExceptionIndexFirstBit) & mask; + return index != mask; } public void SetException(Exception ex) { - Debug.Assert((Flags & ContinuationFlags.HasException) != 0); - uint contIndex = (uint)BitOperations.PopCount((uint)Flags & ((uint)ContinuationFlags.HasException - 1)); + const uint mask = (1u << (int)ContinuationFlags.ExceptionIndexNumBits) - 1; + uint index = ((uint)Flags >> (int)ContinuationFlags.ExceptionIndexFirstBit) & mask; + Debug.Assert(index != mask); ref byte data = ref RuntimeHelpers.GetRawData(this); - Unsafe.As(ref Unsafe.Add(ref data, DataOffset + contIndex * PointerSize)) = ex; + Unsafe.As(ref Unsafe.Add(ref data, DataOffset + index * PointerSize)) = ex; } public ref byte GetResultStorageOrNull() { - if ((Flags & ContinuationFlags.HasResult) == 0) + const uint mask = (1u << (int)ContinuationFlags.ResultIndexBits) - 1; + uint index = ((uint)Flags >> (int)ContinuationFlags.ResultIndexFirstBit) & mask; + if (index == mask) return ref Unsafe.NullRef(); - - uint contIndex = (uint)BitOperations.PopCount((uint)Flags & ((uint)ContinuationFlags.HasResult - 1)); ref byte data = ref RuntimeHelpers.GetRawData(this); - return ref Unsafe.Add(ref data, DataOffset + contIndex * PointerSize); + return ref Unsafe.Add(ref data, DataOffset + index * PointerSize); } } @@ -627,7 +623,7 @@ private unsafe void DispatchContinuations() System.Exception.AppendExceptionStackFrame(ex, ip, 0); #endif } - if (continuation == null || (continuation.Flags & ContinuationFlags.HasException) != 0) + if (continuation == null || continuation.HasException()) return continuation; if (Task.s_asyncDebuggingEnabled) { diff --git a/src/coreclr/inc/corinfo.h b/src/coreclr/inc/corinfo.h index 61ac918ffda3f3..ce52aaba028211 100644 --- a/src/coreclr/inc/corinfo.h +++ b/src/coreclr/inc/corinfo.h @@ -1792,31 +1792,24 @@ struct CORINFO_EE_INFO // Keep in sync with ContinuationFlags enum in BCL sources enum CorInfoContinuationFlags { - // Note: the following 'Has' members determine the members present at - // the beginning of the continuation's data chunk. Each field is - // pointer sized when present, apart from the result that has variable - // size. - - // Whether or not the continuation starts with an OSR IL offset. - CORINFO_CONTINUATION_HAS_OSR_ILOFFSET = 1, - // If this bit is set the continuation resumes inside a try block and - // thus if an exception is being propagated, needs to be resumed. - CORINFO_CONTINUATION_HAS_EXCEPTION = 2, - // If this bit is set the continuation has space for a continuation - // context. - CORINFO_CONTINUATION_HAS_CONTINUATION_CONTEXT = 4, - // If this bit is set the continuation has space to store a result - // returned by the callee. - CORINFO_CONTINUATION_HAS_RESULT = 8, // If this bit is set the continuation should continue on the thread // pool. - CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL = 16, + CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL = 1 << 0, // If this bit is set the continuation context is a // SynchronizationContext that we should continue on. - CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_SYNCHRONIZATION_CONTEXT = 32, + CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_SYNCHRONIZATION_CONTEXT = 1 << 1, // If this bit is set the continuation context is a TaskScheduler that // we should continue on. - CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_TASK_SCHEDULER = 64, + CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_TASK_SCHEDULER = 1 << 2, + + CORINFO_CONTINUATION_EXCEPTION_INDEX_FIRST_BIT = 3u, + CORINFO_CONTINUATION_EXCEPTION_INDEX_NUM_BITS = 2u, + + CORINFO_CONTINUATION_CONTEXT_INDEX_FIRST_BIT = 5u, + CORINFO_CONTINUATION_CONTEXT_INDEX_NUM_BITS = 2u, + + CORINFO_CONTINUATION_RESULT_INDEX_FIRST_BIT = 7u, + CORINFO_CONTINUATION_RESULT_INDEX_NUM_BITS = 26u, }; struct CORINFO_ASYNC_INFO diff --git a/src/coreclr/interpreter/compiler.cpp b/src/coreclr/interpreter/compiler.cpp index 10c649fa958f13..e22ac769b1c4b3 100644 --- a/src/coreclr/interpreter/compiler.cpp +++ b/src/coreclr/interpreter/compiler.cpp @@ -5934,12 +5934,12 @@ void InterpCompiler::EmitSuspend(const CORINFO_CALL_INFO &callInfo, Continuation int32_t flags = 0; if (returnValueVar != -1) { - flags |= CORINFO_CONTINUATION_HAS_RESULT; + //flags |= CORINFO_CONTINUATION_HAS_RESULT; } if (captureContinuationContext) { - flags |= CORINFO_CONTINUATION_HAS_CONTINUATION_CONTEXT; + //flags |= CORINFO_CONTINUATION_HAS_CONTINUATION_CONTEXT; } if (continuationContextHandling == ContinuationContextHandling::ContinueOnThreadPool) @@ -5949,7 +5949,7 @@ void InterpCompiler::EmitSuspend(const CORINFO_CALL_INFO &callInfo, Continuation if (needsEHHandling) { - flags |= CORINFO_CONTINUATION_HAS_EXCEPTION; + //flags |= CORINFO_CONTINUATION_HAS_EXCEPTION; } suspendData->flags = (CorInfoContinuationFlags)flags; diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 4966bcf03e9d3c..48e39922bd22b3 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -1881,6 +1881,8 @@ ContinuationLayout* ContinuationLayoutBuilder::Create() // Now allocate all returns for (ReturnInfo& ret : layout->Returns) { + // All returns must be pointer aligned because of the offset encoding in Continuation::Flags. + layout->Size = roundUp(layout->Size, TARGET_POINTER_SIZE); ret.Offset = allocLayout(ret.HeapAlignment(), ret.Size); } @@ -2175,14 +2177,30 @@ void AsyncTransformation::CreateSuspension(BasicBlock* call // Fill in 'flags' const AsyncCallInfo& callInfo = call->GetAsyncInfo(); unsigned continuationFlags = 0; - if (subLayout.NeedsOSRILOffset()) - continuationFlags |= CORINFO_CONTINUATION_HAS_OSR_ILOFFSET; + auto encodeIndex = [&continuationFlags](unsigned offset, unsigned firstBit, unsigned numBits) { + assert(numBits < 32); + assert((offset % TARGET_POINTER_SIZE) == 0); + unsigned index = offset / TARGET_POINTER_SIZE; + unsigned mask = (1u << numBits) - 1; + + if ((index & mask) != index) + { + IMPL_LIMITATION("Cannot encode continuation offset in flags"); + } + + continuationFlags |= index << firstBit; + }; + if (subLayout.NeedsException()) - continuationFlags |= CORINFO_CONTINUATION_HAS_EXCEPTION; + encodeIndex(layout.ExceptionOffset, CORINFO_CONTINUATION_EXCEPTION_INDEX_FIRST_BIT, CORINFO_CONTINUATION_EXCEPTION_INDEX_NUM_BITS); if (subLayout.NeedsContinuationContext()) - continuationFlags |= CORINFO_CONTINUATION_HAS_CONTINUATION_CONTEXT; + encodeIndex(layout.ContinuationContextOffset, CORINFO_CONTINUATION_CONTEXT_INDEX_FIRST_BIT, CORINFO_CONTINUATION_CONTEXT_INDEX_NUM_BITS); if (call->gtReturnType != TYP_VOID) - continuationFlags |= CORINFO_CONTINUATION_HAS_RESULT; + { + const ReturnInfo* returnInfo = layout.FindReturn(call); + assert(returnInfo != nullptr); + encodeIndex(returnInfo->Offset, CORINFO_CONTINUATION_RESULT_INDEX_FIRST_BIT, CORINFO_CONTINUATION_RESULT_INDEX_NUM_BITS); + } if (callInfo.ContinuationContextHandling == ContinuationContextHandling::ContinueOnThreadPool) continuationFlags |= CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL; @@ -3125,12 +3143,23 @@ BasicBlock* AsyncTransformation::GetSharedReturnBB() // void AsyncTransformation::CreateResumptionsAndSuspensions() { + bool useSharedLayout = m_states.size() > 1; + INDEBUG(useSharedLayout &= JitConfig.JitAsyncReuseContinuations() != 0); + + ContinuationLayout* sharedLayout = nullptr; + if (useSharedLayout) + { + JITDUMP("Creating shared layout:\n"); + ContinuationLayoutBuilder* sharedLayoutBuilder = ContinuationLayoutBuilder::CreateSharedLayout(m_compiler, m_states); + sharedLayout = sharedLayoutBuilder->Create(); + } + JITDUMP("Creating suspensions and resumptions for %zu states\n", m_states.size()); for (const AsyncState& state : m_states) { JITDUMP("State %u suspend @ " FMT_BB ", resume @ " FMT_BB "\n", state.Number, state.SuspensionBB->bbNum, state.ResumptionBB->bbNum); - ContinuationLayout* layout = state.Layout->Create(); + ContinuationLayout* layout = sharedLayout == nullptr ? state.Layout->Create() : sharedLayout; CreateSuspension(state.CallBlock, state.Call, state.SuspensionBB, state.Number, *layout, *state.Layout); CreateResumption(state.CallBlock, state.Call, state.ResumptionBB, state.CallDefInfo, *layout, *state.Layout); CreateDebugInfoForSuspensionPoint(*layout, *state.Layout); diff --git a/src/coreclr/jit/jitconfigvalues.h b/src/coreclr/jit/jitconfigvalues.h index 7741e0f6b8fe19..c6508173e361c4 100644 --- a/src/coreclr/jit/jitconfigvalues.h +++ b/src/coreclr/jit/jitconfigvalues.h @@ -592,6 +592,7 @@ OPT_CONFIG_INTEGER(JitDoOptimizeMaskConversions, "JitDoOptimizeMaskConversions", OPT_CONFIG_INTEGER(JitOptimizeAwait, "JitOptimizeAwait", 1) // Perform optimization of Await intrinsics OPT_CONFIG_STRING(JitAsyncDefaultValueAnalysisRange, "JitAsyncDefaultValueAnalysisRange") // Enable async default value analysis based on method hash range +OPT_CONFIG_INTEGER(JitAsyncReuseContinuations, "JitAsyncReuseContinuations", 1) // Perform optimization of Await intrinsics RELEASE_CONFIG_INTEGER(JitEnableOptRepeat, "JitEnableOptRepeat", 1) // If zero, do not allow JitOptRepeat RELEASE_CONFIG_METHODSET(JitOptRepeat, "JitOptRepeat") // Runs optimizer multiple times on specified methods diff --git a/src/coreclr/vm/interpexec.cpp b/src/coreclr/vm/interpexec.cpp index 8be761a468a125..19ad5fa7a63ecf 100644 --- a/src/coreclr/vm/interpexec.cpp +++ b/src/coreclr/vm/interpexec.cpp @@ -4248,13 +4248,13 @@ do \ SetObjectReference((OBJECTREF *)((uint8_t*)(OBJECTREFToObject(continuation)) + pAsyncSuspendData->offsetIntoContinuationTypeForExecutionContext), executionContext); continuation->SetFlags(pAsyncSuspendData->flags); - if (pAsyncSuspendData->flags & CORINFO_CONTINUATION_HAS_CONTINUATION_CONTEXT) + if (pAsyncSuspendData->flags & /*CORINFO_CONTINUATION_HAS_CONTINUATION_CONTEXT*/ 123) { MethodDesc *captureSyncContextMethod = pAsyncSuspendData->captureSyncContextMethod; int32_t *flagsAddress = continuation->GetFlagsAddress(); size_t continuationOffset = OFFSETOF__CORINFO_Continuation__data; uint8_t *pContinuationData = (uint8_t*)OBJECTREFToObject(continuation) + continuationOffset; - if (pAsyncSuspendData->flags & CORINFO_CONTINUATION_HAS_EXCEPTION) + if (pAsyncSuspendData->flags & /*CORINFO_CONTINUATION_HAS_EXCEPTION */ 456) { pContinuationData += sizeof(OBJECTREF); } @@ -4421,7 +4421,7 @@ do \ pCopyEntry++; } - if (pAsyncSuspendData->flags & CORINFO_CONTINUATION_HAS_EXCEPTION) + if (pAsyncSuspendData->flags & /*CORINFO_CONTINUATION_HAS_EXCEPTION */ 567) { // Throw exception if needed OBJECTREF exception = *continuation->GetExceptionObjectStorage(); diff --git a/src/coreclr/vm/object.h b/src/coreclr/vm/object.h index 24c76bc438526d..e8674c419d8143 100644 --- a/src/coreclr/vm/object.h +++ b/src/coreclr/vm/object.h @@ -2146,31 +2146,31 @@ class ContinuationObject : public Object { LIMITED_METHOD_CONTRACT; PTR_BYTE dataAddress = dac_cast((dac_cast(this) + OFFSETOF__CORINFO_Continuation__data)); - if (GetFlags() & CORINFO_CONTINUATION_HAS_OSR_ILOFFSET) - { - dataAddress += sizeof(void*); - } - if (GetFlags() & CORINFO_CONTINUATION_HAS_EXCEPTION) - { - dataAddress += sizeof(void*); - } - if (GetFlags() & CORINFO_CONTINUATION_HAS_CONTINUATION_CONTEXT) - { - dataAddress += sizeof(void*); - } + //if (GetFlags() & CORINFO_CONTINUATION_HAS_OSR_ILOFFSET) + //{ + // dataAddress += sizeof(void*); + //} + //if (GetFlags() & CORINFO_CONTINUATION_HAS_EXCEPTION) + //{ + // dataAddress += sizeof(void*); + //} + //if (GetFlags() & CORINFO_CONTINUATION_HAS_CONTINUATION_CONTEXT) + //{ + // dataAddress += sizeof(void*); + //} return dataAddress; } PTR_OBJECTREF GetExceptionObjectStorage() { LIMITED_METHOD_CONTRACT; - _ASSERTE((GetFlags() & CORINFO_CONTINUATION_HAS_EXCEPTION)); + //_ASSERTE((GetFlags() & CORINFO_CONTINUATION_HAS_EXCEPTION)); PTR_BYTE dataAddress = dac_cast((dac_cast(this) + OFFSETOF__CORINFO_Continuation__data)); - if (GetFlags() & CORINFO_CONTINUATION_HAS_OSR_ILOFFSET) - { - dataAddress += sizeof(void*); - } + //if (GetFlags() & CORINFO_CONTINUATION_HAS_OSR_ILOFFSET) + //{ + // dataAddress += sizeof(void*); + //} return dac_cast(dataAddress); } From dba08cb4b489ebcb4e78959d5f025d4b78afb5f6 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 13 Mar 2026 21:58:23 +0100 Subject: [PATCH 10/28] WIP --- src/coreclr/jit/async.cpp | 77 ++++++++++++++++++++++++++++++++++----- src/coreclr/jit/async.h | 3 +- 2 files changed, 69 insertions(+), 11 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 48e39922bd22b3..095cc2cde81370 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -2156,11 +2156,53 @@ void AsyncTransformation::CreateSuspension(BasicBlock* call LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_compiler, allocContinuation)); - GenTree* storeNewContinuation = m_compiler->gtNewStoreLclVarNode(GetNewContinuationVar(), allocContinuation); + unsigned newContinuationVar = GetNewContinuationVar(); + GenTree* storeNewContinuation = m_compiler->gtNewStoreLclVarNode(newContinuationVar, allocContinuation); LIR::AsRange(suspendBB).InsertAtEnd(storeNewContinuation); + if (ReuseContinuations()) + { + // Split suspendBB into suspendBB -> [reuse continuation with Next store] -> [allocNewBlock with allocation call] -> suspendBBTail [empty] + BasicBlock* allocNewBB = m_compiler->fgSplitBlockAfterNode(suspendBB, recordOffset); + BasicBlock* reuseContinuationBB = m_compiler->fgSplitBlockAtEnd(suspendBB); + BasicBlock* suspendTailBB = m_compiler->fgSplitBlockAtEnd(allocNewBB); + + m_compiler->fgRemoveRefPred(reuseContinuationBB->GetTargetEdge()); + FlowEdge* toSuspendTail = m_compiler->fgAddRefPred(suspendTailBB, reuseContinuationBB); + reuseContinuationBB->SetTargetEdge(toSuspendTail); + + FlowEdge* toAllocNew = m_compiler->fgAddRefPred(allocNewBB, suspendBB); + suspendBB->SetCond(suspendBB->GetTargetEdge(), toAllocNew); + suspendBB->GetTrueEdge()->setLikelihood(1.0); + suspendBB->GetFalseEdge()->setLikelihood(0.0); + + JITDUMP("Continuation reuse is active. Split suspendBB into suspendBB " FMT_BB " -> reuseContinuationBlock " FMT_BB " -> allocNewBlock " FMT_BB " -> suspendBBTail " FMT_BB "\n", + suspendBB->bbNum, reuseContinuationBB->bbNum, allocNewBB->bbNum, suspendTailBB->bbNum); + + // Store newContinuationVar = asyncContinuationArg in suspendBB + GenTree* continuationParam = m_compiler->gtNewLclvNode(m_compiler->lvaAsyncContinuationArg, TYP_REF); + GenTree* storeContinuationParam = m_compiler->gtNewStoreLclVarNode(newContinuationVar, continuationParam); + LIR::AsRange(suspendBB).InsertAtEnd(continuationParam, storeContinuationParam); + + // Check if newContinuationVar != null, jump to reuseContinuationBlock + GenTree* reusedContinuation = m_compiler->gtNewLclvNode(newContinuationVar, TYP_REF); + GenTree* null = m_compiler->gtNewNull(); + GenTree* neNull = m_compiler->gtNewOperNode(GT_NE, TYP_INT, reusedContinuation, null); + GenTree* jtrue = m_compiler->gtNewOperNode(GT_JTRUE, TYP_VOID, neNull); + LIR::AsRange(suspendBB).InsertAtEnd(reusedContinuation, null, neNull, jtrue); + + // Fill in 'Next' in reuseContinuationBB + GenTree* newContinuation = m_compiler->gtNewLclvNode(newContinuationVar, TYP_REF); + unsigned nextOffset = m_compiler->info.compCompHnd->getFieldOffset(m_asyncInfo->continuationNextFldHnd); + returnedContinuation = m_compiler->gtNewLclvNode(GetReturnedContinuationVar(), TYP_REF); + GenTree* storeNext = StoreAtOffset(returnedContinuation, nextOffset, newContinuation, TYP_REF); + LIR::AsRange(reuseContinuationBB).InsertAtEnd(LIR::SeqTree(m_compiler, storeNext)); + + suspendBB = suspendTailBB; + } + // Fill in 'ResumeInfo' - GenTree* newContinuation = m_compiler->gtNewLclvNode(GetNewContinuationVar(), TYP_REF); + GenTree* newContinuation = m_compiler->gtNewLclvNode(newContinuationVar, TYP_REF); unsigned resumeInfoOffset = m_compiler->info.compCompHnd->getFieldOffset(m_asyncInfo->continuationResumeInfoFldHnd); GenTree* resumeInfoAddr = new (m_compiler, GT_ASYNC_RESUME_INFO) GenTreeVal(GT_ASYNC_RESUME_INFO, TYP_I_IMPL, (ssize_t)stateNum); @@ -2168,7 +2210,7 @@ void AsyncTransformation::CreateSuspension(BasicBlock* call LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_compiler, storeResume)); // Fill in 'state' - newContinuation = m_compiler->gtNewLclvNode(GetNewContinuationVar(), TYP_REF); + newContinuation = m_compiler->gtNewLclvNode(newContinuationVar, TYP_REF); unsigned stateOffset = m_compiler->info.compCompHnd->getFieldOffset(m_asyncInfo->continuationStateFldHnd); GenTree* stateNumNode = m_compiler->gtNewIconNode((ssize_t)stateNum, TYP_INT); GenTree* storeState = StoreAtOffset(newContinuation, stateOffset, stateNumNode, TYP_INT); @@ -2204,7 +2246,7 @@ void AsyncTransformation::CreateSuspension(BasicBlock* call if (callInfo.ContinuationContextHandling == ContinuationContextHandling::ContinueOnThreadPool) continuationFlags |= CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL; - newContinuation = m_compiler->gtNewLclvNode(GetNewContinuationVar(), TYP_REF); + newContinuation = m_compiler->gtNewLclvNode(newContinuationVar, TYP_REF); unsigned flagsOffset = m_compiler->info.compCompHnd->getFieldOffset(m_asyncInfo->continuationFlagsFldHnd); GenTree* flagsNode = m_compiler->gtNewIconNode((ssize_t)continuationFlags, TYP_INT); GenTree* storeFlags = StoreAtOffset(newContinuation, flagsOffset, flagsNode, TYP_INT); @@ -2219,7 +2261,7 @@ void AsyncTransformation::CreateSuspension(BasicBlock* call if (suspendBB->KindIs(BBJ_RETURN)) { - newContinuation = m_compiler->gtNewLclvNode(GetNewContinuationVar(), TYP_REF); + newContinuation = m_compiler->gtNewLclvNode(newContinuationVar, TYP_REF); GenTree* ret = m_compiler->gtNewOperNode(GT_RETURN_SUSPEND, TYP_VOID, newContinuation); LIR::AsRange(suspendBB).InsertAtEnd(newContinuation, ret); } @@ -2460,10 +2502,10 @@ void AsyncTransformation::RestoreContexts(BasicBlock* block, GenTreeCall* call, GenTree* continuation = m_compiler->gtNewLclvNode(m_compiler->lvaAsyncContinuationArg, TYP_REF); GenTree* null = m_compiler->gtNewNull(); - GenTree* started = m_compiler->gtNewOperNode(GT_NE, TYP_INT, continuation, null); + GenTree* resumed = m_compiler->gtNewOperNode(GT_NE, TYP_INT, continuation, null); - LIR::AsRange(suspendBB).InsertBefore(resumedPlaceholder, LIR::SeqTree(m_compiler, started)); - use.ReplaceWith(started); + LIR::AsRange(suspendBB).InsertBefore(resumedPlaceholder, LIR::SeqTree(m_compiler, resumed)); + use.ReplaceWith(resumed); LIR::AsRange(suspendBB).Remove(resumedPlaceholder); // Replace execContextPlaceholder with actual value @@ -3143,8 +3185,7 @@ BasicBlock* AsyncTransformation::GetSharedReturnBB() // void AsyncTransformation::CreateResumptionsAndSuspensions() { - bool useSharedLayout = m_states.size() > 1; - INDEBUG(useSharedLayout &= JitConfig.JitAsyncReuseContinuations() != 0); + bool useSharedLayout = (m_states.size() > 1) && ReuseContinuations(); ContinuationLayout* sharedLayout = nullptr; if (useSharedLayout) @@ -3168,6 +3209,22 @@ void AsyncTransformation::CreateResumptionsAndSuspensions() } } +//------------------------------------------------------------------------ +// AsyncTransformation::ReuseContinuations: +// Returns true if continuation reuse is enabled. +// +// Returns: +// True if so. +// +bool AsyncTransformation::ReuseContinuations() +{ +#ifdef DEBUG + return JitConfig.JitAsyncReuseContinuations() != 0; +#else + return false; +#endif +} + //------------------------------------------------------------------------ // ContinuationLayoutBuilder::CreateSharedLayout: // Create a shared continuation layout that is the union of all per-call diff --git a/src/coreclr/jit/async.h b/src/coreclr/jit/async.h index 36392f9b8c6fd5..be34459cbbcb2d 100644 --- a/src/coreclr/jit/async.h +++ b/src/coreclr/jit/async.h @@ -251,7 +251,7 @@ class AsyncTransformation const ContinuationLayout& layout, const ContinuationLayoutBuilder& subLayout, BasicBlock* suspendBB); - void RestoreContexts(BasicBlock* block, GenTreeCall* call, BasicBlock* suspendBB); + void RestoreContexts(BasicBlock* block, GenTreeCall* call, BasicBlock* insertionBB); void CreateCheckAndSuspendAfterCall(BasicBlock* block, GenTreeCall* call, const CallDefinitionInfo& callDefInfo, @@ -295,6 +295,7 @@ class AsyncTransformation unsigned GetExceptionVar(); BasicBlock* GetSharedReturnBB(); + bool ReuseContinuations(); void CreateResumptionsAndSuspensions(); void CreateResumptionSwitch(); From ddb08038ca60ddf1abdf4a8275f6655ee0f435e8 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 13 Mar 2026 22:45:07 +0100 Subject: [PATCH 11/28] Make continuation reuse a release config --- src/coreclr/jit/async.cpp | 4 ---- src/coreclr/jit/jitconfigvalues.h | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 095cc2cde81370..87e7f378ca5dad 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -3218,11 +3218,7 @@ void AsyncTransformation::CreateResumptionsAndSuspensions() // bool AsyncTransformation::ReuseContinuations() { -#ifdef DEBUG return JitConfig.JitAsyncReuseContinuations() != 0; -#else - return false; -#endif } //------------------------------------------------------------------------ diff --git a/src/coreclr/jit/jitconfigvalues.h b/src/coreclr/jit/jitconfigvalues.h index c6508173e361c4..209375e1b3b4c4 100644 --- a/src/coreclr/jit/jitconfigvalues.h +++ b/src/coreclr/jit/jitconfigvalues.h @@ -592,7 +592,7 @@ OPT_CONFIG_INTEGER(JitDoOptimizeMaskConversions, "JitDoOptimizeMaskConversions", OPT_CONFIG_INTEGER(JitOptimizeAwait, "JitOptimizeAwait", 1) // Perform optimization of Await intrinsics OPT_CONFIG_STRING(JitAsyncDefaultValueAnalysisRange, "JitAsyncDefaultValueAnalysisRange") // Enable async default value analysis based on method hash range -OPT_CONFIG_INTEGER(JitAsyncReuseContinuations, "JitAsyncReuseContinuations", 1) // Perform optimization of Await intrinsics +RELEASE_CONFIG_INTEGER(JitAsyncReuseContinuations, "JitAsyncReuseContinuations", 1) // Perform optimization of Await intrinsics RELEASE_CONFIG_INTEGER(JitEnableOptRepeat, "JitEnableOptRepeat", 1) // If zero, do not allow JitOptRepeat RELEASE_CONFIG_METHODSET(JitOptRepeat, "JitOptRepeat") // Runs optimizer multiple times on specified methods From 17e04b519482346a632620731035c091f8d883ca Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sat, 14 Mar 2026 14:29:56 +0100 Subject: [PATCH 12/28] Fix OSR bug --- src/coreclr/jit/async.cpp | 33 +++++++++++++++++++++++++++++---- src/coreclr/jit/async.h | 1 + 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 87e7f378ca5dad..fb7271a98c429a 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -1237,6 +1237,23 @@ PhaseStatus AsyncTransformation::Run() } while (any); } + if (ReuseContinuations()) + { + // Set up the local containing the continuation we can reuse. For OSR + // things are special: we can transition to the OSR method after having + // resumed in the tier0 method. In that case we end up with the tier0 + // continuation in the OSR method, but we cannot reuse it. + if (m_compiler->opts.IsOSR()) + { + m_reuseContinuationVar = m_compiler->lvaGrabTemp(false DEBUGARG("OSR reusable continuation")); + m_compiler->lvaGetDesc(m_reuseContinuationVar)->lvType = TYP_REF; + } + else + { + m_reuseContinuationVar = m_compiler->lvaAsyncContinuationArg; + } + } + CreateResumptionsAndSuspensions(); // After transforming all async calls we have created resumption blocks; @@ -2179,10 +2196,10 @@ void AsyncTransformation::CreateSuspension(BasicBlock* call JITDUMP("Continuation reuse is active. Split suspendBB into suspendBB " FMT_BB " -> reuseContinuationBlock " FMT_BB " -> allocNewBlock " FMT_BB " -> suspendBBTail " FMT_BB "\n", suspendBB->bbNum, reuseContinuationBB->bbNum, allocNewBB->bbNum, suspendTailBB->bbNum); - // Store newContinuationVar = asyncContinuationArg in suspendBB - GenTree* continuationParam = m_compiler->gtNewLclvNode(m_compiler->lvaAsyncContinuationArg, TYP_REF); - GenTree* storeContinuationParam = m_compiler->gtNewStoreLclVarNode(newContinuationVar, continuationParam); - LIR::AsRange(suspendBB).InsertAtEnd(continuationParam, storeContinuationParam); + // Store newContinuationVar = reusableContinuation in suspendBB + GenTree* reusableContinuation = m_compiler->gtNewLclvNode(m_reuseContinuationVar, TYP_REF); + GenTree* storeContinuationParam = m_compiler->gtNewStoreLclVarNode(newContinuationVar, reusableContinuation); + LIR::AsRange(suspendBB).InsertAtEnd(reusableContinuation, storeContinuationParam); // Check if newContinuationVar != null, jump to reuseContinuationBlock GenTree* reusedContinuation = m_compiler->gtNewLclvNode(newContinuationVar, TYP_REF); @@ -3471,5 +3488,13 @@ void AsyncTransformation::CreateResumptionSwitch() GenTree* ltZero = m_compiler->gtNewOperNode(GT_LT, TYP_INT, ilOffset, zero); GenTree* jtrue = m_compiler->gtNewOperNode(GT_JTRUE, TYP_VOID, ltZero); LIR::AsRange(checkILOffsetBB).InsertAtEnd(LIR::SeqTree(m_compiler, jtrue)); + + if (ReuseContinuations()) + { + // Also, save the fact that we have a reusable continuation + continuationArg = m_compiler->gtNewLclvNode(m_compiler->lvaAsyncContinuationArg, TYP_REF); + GenTree* storeReusable = m_compiler->gtNewStoreLclVarNode(m_reuseContinuationVar, continuationArg); + LIR::AsRange(onContinuationBB).InsertAtBeginning(continuationArg, storeReusable); + } } } diff --git a/src/coreclr/jit/async.h b/src/coreclr/jit/async.h index be34459cbbcb2d..2534b5bcd3913b 100644 --- a/src/coreclr/jit/async.h +++ b/src/coreclr/jit/async.h @@ -195,6 +195,7 @@ class AsyncTransformation jitstd::vector m_states; unsigned m_returnedContinuationVar = BAD_VAR_NUM; unsigned m_newContinuationVar = BAD_VAR_NUM; + unsigned m_reuseContinuationVar = BAD_VAR_NUM; unsigned m_dataArrayVar = BAD_VAR_NUM; unsigned m_gcDataArrayVar = BAD_VAR_NUM; unsigned m_resultBaseVar = BAD_VAR_NUM; From 120329c81ae77972cdfd9725e3eb93436058e278 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sat, 14 Mar 2026 14:32:02 +0100 Subject: [PATCH 13/28] Run jit-format --- src/coreclr/jit/async.cpp | 54 +++++++++++++++++-------------- src/coreclr/jit/jitconfigvalues.h | 4 ++- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index fb7271a98c429a..3b7939007efff6 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -1900,7 +1900,7 @@ ContinuationLayout* ContinuationLayoutBuilder::Create() { // All returns must be pointer aligned because of the offset encoding in Continuation::Flags. layout->Size = roundUp(layout->Size, TARGET_POINTER_SIZE); - ret.Offset = allocLayout(ret.HeapAlignment(), ret.Size); + ret.Offset = allocLayout(ret.HeapAlignment(), ret.Size); } if (m_needsKeepAlive) @@ -2173,16 +2173,17 @@ void AsyncTransformation::CreateSuspension(BasicBlock* call LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_compiler, allocContinuation)); - unsigned newContinuationVar = GetNewContinuationVar(); + unsigned newContinuationVar = GetNewContinuationVar(); GenTree* storeNewContinuation = m_compiler->gtNewStoreLclVarNode(newContinuationVar, allocContinuation); LIR::AsRange(suspendBB).InsertAtEnd(storeNewContinuation); if (ReuseContinuations()) { - // Split suspendBB into suspendBB -> [reuse continuation with Next store] -> [allocNewBlock with allocation call] -> suspendBBTail [empty] - BasicBlock* allocNewBB = m_compiler->fgSplitBlockAfterNode(suspendBB, recordOffset); + // Split suspendBB into suspendBB -> [reuse continuation with Next store] -> [allocNewBlock with allocation + // call] -> suspendBBTail [empty] + BasicBlock* allocNewBB = m_compiler->fgSplitBlockAfterNode(suspendBB, recordOffset); BasicBlock* reuseContinuationBB = m_compiler->fgSplitBlockAtEnd(suspendBB); - BasicBlock* suspendTailBB = m_compiler->fgSplitBlockAtEnd(allocNewBB); + BasicBlock* suspendTailBB = m_compiler->fgSplitBlockAtEnd(allocNewBB); m_compiler->fgRemoveRefPred(reuseContinuationBB->GetTargetEdge()); FlowEdge* toSuspendTail = m_compiler->fgAddRefPred(suspendTailBB, reuseContinuationBB); @@ -2193,26 +2194,27 @@ void AsyncTransformation::CreateSuspension(BasicBlock* call suspendBB->GetTrueEdge()->setLikelihood(1.0); suspendBB->GetFalseEdge()->setLikelihood(0.0); - JITDUMP("Continuation reuse is active. Split suspendBB into suspendBB " FMT_BB " -> reuseContinuationBlock " FMT_BB " -> allocNewBlock " FMT_BB " -> suspendBBTail " FMT_BB "\n", - suspendBB->bbNum, reuseContinuationBB->bbNum, allocNewBB->bbNum, suspendTailBB->bbNum); + JITDUMP("Continuation reuse is active. Split suspendBB into suspendBB " FMT_BB + " -> reuseContinuationBlock " FMT_BB " -> allocNewBlock " FMT_BB " -> suspendBBTail " FMT_BB "\n", + suspendBB->bbNum, reuseContinuationBB->bbNum, allocNewBB->bbNum, suspendTailBB->bbNum); // Store newContinuationVar = reusableContinuation in suspendBB - GenTree* reusableContinuation = m_compiler->gtNewLclvNode(m_reuseContinuationVar, TYP_REF); + GenTree* reusableContinuation = m_compiler->gtNewLclvNode(m_reuseContinuationVar, TYP_REF); GenTree* storeContinuationParam = m_compiler->gtNewStoreLclVarNode(newContinuationVar, reusableContinuation); LIR::AsRange(suspendBB).InsertAtEnd(reusableContinuation, storeContinuationParam); // Check if newContinuationVar != null, jump to reuseContinuationBlock GenTree* reusedContinuation = m_compiler->gtNewLclvNode(newContinuationVar, TYP_REF); - GenTree* null = m_compiler->gtNewNull(); - GenTree* neNull = m_compiler->gtNewOperNode(GT_NE, TYP_INT, reusedContinuation, null); - GenTree* jtrue = m_compiler->gtNewOperNode(GT_JTRUE, TYP_VOID, neNull); + GenTree* null = m_compiler->gtNewNull(); + GenTree* neNull = m_compiler->gtNewOperNode(GT_NE, TYP_INT, reusedContinuation, null); + GenTree* jtrue = m_compiler->gtNewOperNode(GT_JTRUE, TYP_VOID, neNull); LIR::AsRange(suspendBB).InsertAtEnd(reusedContinuation, null, neNull, jtrue); // Fill in 'Next' in reuseContinuationBB - GenTree* newContinuation = m_compiler->gtNewLclvNode(newContinuationVar, TYP_REF); - unsigned nextOffset = m_compiler->info.compCompHnd->getFieldOffset(m_asyncInfo->continuationNextFldHnd); - returnedContinuation = m_compiler->gtNewLclvNode(GetReturnedContinuationVar(), TYP_REF); - GenTree* storeNext = StoreAtOffset(returnedContinuation, nextOffset, newContinuation, TYP_REF); + GenTree* newContinuation = m_compiler->gtNewLclvNode(newContinuationVar, TYP_REF); + unsigned nextOffset = m_compiler->info.compCompHnd->getFieldOffset(m_asyncInfo->continuationNextFldHnd); + returnedContinuation = m_compiler->gtNewLclvNode(GetReturnedContinuationVar(), TYP_REF); + GenTree* storeNext = StoreAtOffset(returnedContinuation, nextOffset, newContinuation, TYP_REF); LIR::AsRange(reuseContinuationBB).InsertAtEnd(LIR::SeqTree(m_compiler, storeNext)); suspendBB = suspendTailBB; @@ -2236,11 +2238,11 @@ void AsyncTransformation::CreateSuspension(BasicBlock* call // Fill in 'flags' const AsyncCallInfo& callInfo = call->GetAsyncInfo(); unsigned continuationFlags = 0; - auto encodeIndex = [&continuationFlags](unsigned offset, unsigned firstBit, unsigned numBits) { + auto encodeIndex = [&continuationFlags](unsigned offset, unsigned firstBit, unsigned numBits) { assert(numBits < 32); assert((offset % TARGET_POINTER_SIZE) == 0); unsigned index = offset / TARGET_POINTER_SIZE; - unsigned mask = (1u << numBits) - 1; + unsigned mask = (1u << numBits) - 1; if ((index & mask) != index) { @@ -2248,17 +2250,20 @@ void AsyncTransformation::CreateSuspension(BasicBlock* call } continuationFlags |= index << firstBit; - }; + }; if (subLayout.NeedsException()) - encodeIndex(layout.ExceptionOffset, CORINFO_CONTINUATION_EXCEPTION_INDEX_FIRST_BIT, CORINFO_CONTINUATION_EXCEPTION_INDEX_NUM_BITS); + encodeIndex(layout.ExceptionOffset, CORINFO_CONTINUATION_EXCEPTION_INDEX_FIRST_BIT, + CORINFO_CONTINUATION_EXCEPTION_INDEX_NUM_BITS); if (subLayout.NeedsContinuationContext()) - encodeIndex(layout.ContinuationContextOffset, CORINFO_CONTINUATION_CONTEXT_INDEX_FIRST_BIT, CORINFO_CONTINUATION_CONTEXT_INDEX_NUM_BITS); + encodeIndex(layout.ContinuationContextOffset, CORINFO_CONTINUATION_CONTEXT_INDEX_FIRST_BIT, + CORINFO_CONTINUATION_CONTEXT_INDEX_NUM_BITS); if (call->gtReturnType != TYP_VOID) { const ReturnInfo* returnInfo = layout.FindReturn(call); assert(returnInfo != nullptr); - encodeIndex(returnInfo->Offset, CORINFO_CONTINUATION_RESULT_INDEX_FIRST_BIT, CORINFO_CONTINUATION_RESULT_INDEX_NUM_BITS); + encodeIndex(returnInfo->Offset, CORINFO_CONTINUATION_RESULT_INDEX_FIRST_BIT, + CORINFO_CONTINUATION_RESULT_INDEX_NUM_BITS); } if (callInfo.ContinuationContextHandling == ContinuationContextHandling::ContinueOnThreadPool) continuationFlags |= CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL; @@ -3203,12 +3208,13 @@ BasicBlock* AsyncTransformation::GetSharedReturnBB() void AsyncTransformation::CreateResumptionsAndSuspensions() { bool useSharedLayout = (m_states.size() > 1) && ReuseContinuations(); - + ContinuationLayout* sharedLayout = nullptr; if (useSharedLayout) { JITDUMP("Creating shared layout:\n"); - ContinuationLayoutBuilder* sharedLayoutBuilder = ContinuationLayoutBuilder::CreateSharedLayout(m_compiler, m_states); + ContinuationLayoutBuilder* sharedLayoutBuilder = + ContinuationLayoutBuilder::CreateSharedLayout(m_compiler, m_states); sharedLayout = sharedLayoutBuilder->Create(); } @@ -3492,7 +3498,7 @@ void AsyncTransformation::CreateResumptionSwitch() if (ReuseContinuations()) { // Also, save the fact that we have a reusable continuation - continuationArg = m_compiler->gtNewLclvNode(m_compiler->lvaAsyncContinuationArg, TYP_REF); + continuationArg = m_compiler->gtNewLclvNode(m_compiler->lvaAsyncContinuationArg, TYP_REF); GenTree* storeReusable = m_compiler->gtNewStoreLclVarNode(m_reuseContinuationVar, continuationArg); LIR::AsRange(onContinuationBB).InsertAtBeginning(continuationArg, storeReusable); } diff --git a/src/coreclr/jit/jitconfigvalues.h b/src/coreclr/jit/jitconfigvalues.h index 209375e1b3b4c4..7c7dbfe17fc1d0 100644 --- a/src/coreclr/jit/jitconfigvalues.h +++ b/src/coreclr/jit/jitconfigvalues.h @@ -592,7 +592,9 @@ OPT_CONFIG_INTEGER(JitDoOptimizeMaskConversions, "JitDoOptimizeMaskConversions", OPT_CONFIG_INTEGER(JitOptimizeAwait, "JitOptimizeAwait", 1) // Perform optimization of Await intrinsics OPT_CONFIG_STRING(JitAsyncDefaultValueAnalysisRange, "JitAsyncDefaultValueAnalysisRange") // Enable async default value analysis based on method hash range -RELEASE_CONFIG_INTEGER(JitAsyncReuseContinuations, "JitAsyncReuseContinuations", 1) // Perform optimization of Await intrinsics +// Save and reuse continuation instances in runtime async functions. Also +// implies use of shared continuation layouts for all suspension points. +RELEASE_CONFIG_INTEGER(JitAsyncReuseContinuations, "JitAsyncReuseContinuations", 1) RELEASE_CONFIG_INTEGER(JitEnableOptRepeat, "JitEnableOptRepeat", 1) // If zero, do not allow JitOptRepeat RELEASE_CONFIG_METHODSET(JitOptRepeat, "JitOptRepeat") // Runs optimizer multiple times on specified methods From 2510fe8912d4204127f3136bd99dbc85acabad85 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sat, 14 Mar 2026 14:41:17 +0100 Subject: [PATCH 14/28] Nits --- .../CompilerServices/AsyncHelpers.CoreCLR.cs | 20 +++++++++---------- src/coreclr/inc/corinfo.h | 12 +++++------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs index 05f86aeadf0846..1906fc709f1702 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs @@ -57,20 +57,20 @@ public void Pop() [Flags] // Keep in sync with CORINFO_CONTINUATION_FLAGS - internal enum ContinuationFlags : uint + internal enum ContinuationFlags { - ContinueOnThreadPool = 1u << 0, - ContinueOnCapturedSynchronizationContext = 1u << 1, - ContinueOnCapturedTaskScheduler = 1u << 2, + ContinueOnThreadPool = 1 << 0, + ContinueOnCapturedSynchronizationContext = 1 << 1, + ContinueOnCapturedTaskScheduler = 1 << 2, - ExceptionIndexFirstBit = 3u, - ExceptionIndexNumBits = 2u, + ExceptionIndexFirstBit = 3, + ExceptionIndexNumBits = 2, - ContinuationContextIndexFirstBit = 5u, - ContinuationContextIndexNumBits = 2u, + ContinuationContextIndexFirstBit = 5, + ContinuationContextIndexNumBits = 2, - ResultIndexFirstBit = 7u, - ResultIndexBits = 26u, + ResultIndexFirstBit = 7, + ResultIndexBits = 26, } // Keep in sync with CORINFO_AsyncResumeInfo in corinfo.h diff --git a/src/coreclr/inc/corinfo.h b/src/coreclr/inc/corinfo.h index ce52aaba028211..f5ac9628d40524 100644 --- a/src/coreclr/inc/corinfo.h +++ b/src/coreclr/inc/corinfo.h @@ -1802,14 +1802,14 @@ enum CorInfoContinuationFlags // we should continue on. CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_TASK_SCHEDULER = 1 << 2, - CORINFO_CONTINUATION_EXCEPTION_INDEX_FIRST_BIT = 3u, - CORINFO_CONTINUATION_EXCEPTION_INDEX_NUM_BITS = 2u, + CORINFO_CONTINUATION_EXCEPTION_INDEX_FIRST_BIT = 3, + CORINFO_CONTINUATION_EXCEPTION_INDEX_NUM_BITS = 2, - CORINFO_CONTINUATION_CONTEXT_INDEX_FIRST_BIT = 5u, - CORINFO_CONTINUATION_CONTEXT_INDEX_NUM_BITS = 2u, + CORINFO_CONTINUATION_CONTEXT_INDEX_FIRST_BIT = 5, + CORINFO_CONTINUATION_CONTEXT_INDEX_NUM_BITS = 2, - CORINFO_CONTINUATION_RESULT_INDEX_FIRST_BIT = 7u, - CORINFO_CONTINUATION_RESULT_INDEX_NUM_BITS = 26u, + CORINFO_CONTINUATION_RESULT_INDEX_FIRST_BIT = 7, + CORINFO_CONTINUATION_RESULT_INDEX_NUM_BITS = 26, }; struct CORINFO_ASYNC_INFO From bbcf76396d0459aae77ff5bf4e1e92979fd0c4f0 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sat, 14 Mar 2026 14:49:33 +0100 Subject: [PATCH 15/28] Fix bits --- .../src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs | 2 +- src/coreclr/inc/corinfo.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs index 1906fc709f1702..87d7a201e5ece4 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs @@ -70,7 +70,7 @@ internal enum ContinuationFlags ContinuationContextIndexNumBits = 2, ResultIndexFirstBit = 7, - ResultIndexBits = 26, + ResultIndexBits = 25, } // Keep in sync with CORINFO_AsyncResumeInfo in corinfo.h diff --git a/src/coreclr/inc/corinfo.h b/src/coreclr/inc/corinfo.h index f5ac9628d40524..650ad98fd8565e 100644 --- a/src/coreclr/inc/corinfo.h +++ b/src/coreclr/inc/corinfo.h @@ -1809,7 +1809,7 @@ enum CorInfoContinuationFlags CORINFO_CONTINUATION_CONTEXT_INDEX_NUM_BITS = 2, CORINFO_CONTINUATION_RESULT_INDEX_FIRST_BIT = 7, - CORINFO_CONTINUATION_RESULT_INDEX_NUM_BITS = 26, + CORINFO_CONTINUATION_RESULT_INDEX_NUM_BITS = 25, }; struct CORINFO_ASYNC_INFO From 9738d9ea768bcd75bf90c66af7bd9b61fb9fccc8 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sat, 14 Mar 2026 15:27:46 +0100 Subject: [PATCH 16/28] Fix interpreter --- .../CompilerServices/AsyncHelpers.CoreCLR.cs | 22 +++++++----- src/coreclr/interpreter/compiler.cpp | 31 ++++++++-------- src/coreclr/jit/async.cpp | 2 +- src/coreclr/vm/interpexec.cpp | 15 ++++---- src/coreclr/vm/object.h | 35 ++++++++----------- 5 files changed, 52 insertions(+), 53 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs index 87d7a201e5ece4..29125a21d07bce 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs @@ -63,6 +63,11 @@ internal enum ContinuationFlags ContinueOnCapturedSynchronizationContext = 1 << 1, ContinueOnCapturedTaskScheduler = 1 << 2, + // The flags encode where in the continuation various members are stored. + // If the encoded index is 0, it means no such member is present. + // Otherwise the exact offset of the member is computed as + // DataOffset + (index - 1) * PointerSize + // ExceptionIndexFirstBit = 3, ExceptionIndexNumBits = 2, @@ -105,39 +110,40 @@ internal unsafe class Continuation private const int DataOffset = PointerSize /* Next */ + PointerSize /* Resume */ + 8 /* Flags + State */; + // See note in ContinuationFlags above for the computation of these offsets. + public unsafe object GetContinuationContext() { const uint mask = (1u << (int)ContinuationFlags.ContinuationContextIndexNumBits) - 1; uint index = ((uint)Flags >> (int)ContinuationFlags.ContinuationContextIndexFirstBit) & mask; - Debug.Assert(index != mask); + Debug.Assert(index != 0); ref byte data = ref RuntimeHelpers.GetRawData(this); - return Unsafe.As(ref Unsafe.Add(ref data, DataOffset + index * PointerSize)); + return Unsafe.As(ref Unsafe.Add(ref data, (DataOffset - PointerSize) + index * PointerSize)); } public bool HasException() { const uint mask = (1u << (int)ContinuationFlags.ExceptionIndexNumBits) - 1; - uint index = ((uint)Flags >> (int)ContinuationFlags.ExceptionIndexFirstBit) & mask; - return index != mask; + return ((uint)Flags & (mask << (int)ContinuationFlags.ExceptionIndexFirstBit)) != 0; } public void SetException(Exception ex) { const uint mask = (1u << (int)ContinuationFlags.ExceptionIndexNumBits) - 1; uint index = ((uint)Flags >> (int)ContinuationFlags.ExceptionIndexFirstBit) & mask; - Debug.Assert(index != mask); + Debug.Assert(index != 0); ref byte data = ref RuntimeHelpers.GetRawData(this); - Unsafe.As(ref Unsafe.Add(ref data, DataOffset + index * PointerSize)) = ex; + Unsafe.As(ref Unsafe.Add(ref data, (DataOffset - PointerSize) + index * PointerSize)) = ex; } public ref byte GetResultStorageOrNull() { const uint mask = (1u << (int)ContinuationFlags.ResultIndexBits) - 1; uint index = ((uint)Flags >> (int)ContinuationFlags.ResultIndexFirstBit) & mask; - if (index == mask) + if (index == 0) return ref Unsafe.NullRef(); ref byte data = ref RuntimeHelpers.GetRawData(this); - return ref Unsafe.Add(ref data, DataOffset + index * PointerSize); + return ref Unsafe.Add(ref data, (DataOffset - PointerSize) + index * PointerSize); } } diff --git a/src/coreclr/interpreter/compiler.cpp b/src/coreclr/interpreter/compiler.cpp index e22ac769b1c4b3..23d2e58386397f 100644 --- a/src/coreclr/interpreter/compiler.cpp +++ b/src/coreclr/interpreter/compiler.cpp @@ -5817,6 +5817,18 @@ void InterpCompiler::EmitSuspend(const CORINFO_CALL_INFO &callInfo, Continuation // Fill in the GC reference map int32_t currentOffset = 0; int32_t returnValueDataStartOffset = 0; + + uint32_t flags = 0; + auto encodeIndex = [&flags](unsigned offset, unsigned firstBit, unsigned numBits) { + assert(numBits < 32); + assert((offset % TARGET_POINTER_SIZE) == 0); + unsigned index = 1 + offset / TARGET_POINTER_SIZE; + unsigned mask = (1u << numBits) - 1; + + assert((index & mask) == index); + flags |= index << firstBit; + }; + for (int32_t i = -3; i < liveVars.GetSize(); i++) { int32_t var; @@ -5827,6 +5839,7 @@ void InterpCompiler::EmitSuspend(const CORINFO_CALL_INFO &callInfo, Continuation continue; INTERP_DUMP("Allocate EH at offset %d\n", currentOffset); SetSlotToTrue(objRefSlots, currentOffset); + encodeIndex(currentOffset, CORINFO_CONTINUATION_EXCEPTION_INDEX_FIRST_BIT, CORINFO_CONTINUATION_EXCEPTION_INDEX_NUM_BITS); currentOffset += sizeof(void*); // Align to pointer size to match the expected layout continue; } @@ -5836,12 +5849,14 @@ void InterpCompiler::EmitSuspend(const CORINFO_CALL_INFO &callInfo, Continuation continue; INTERP_DUMP("Allocate ContinuationContext at offset %d\n", currentOffset); SetSlotToTrue(objRefSlots, currentOffset); + encodeIndex(currentOffset, CORINFO_CONTINUATION_CONTEXT_INDEX_FIRST_BIT, CORINFO_CONTINUATION_CONTEXT_INDEX_NUM_BITS); currentOffset += sizeof(void*); // Align to pointer size to match the expected layout continue; } if (i == -1) { returnValueDataStartOffset = currentOffset; + encodeIndex(currentOffset, CORINFO_CONTINUATION_RESULT_INDEX_FIRST_BIT, CORINFO_CONTINUATION_RESULT_INDEX_NUM_BITS); // Handle return value first if (returnValueVar == -1) continue; @@ -5931,27 +5946,11 @@ void InterpCompiler::EmitSuspend(const CORINFO_CALL_INFO &callInfo, Continuation AllocateIntervalMapData_ForVars(&suspendData->liveLocalsIntervals, liveVars); AllocateIntervalMapData_ForVars(&suspendData->zeroedLocalsIntervals, varsToZero); - int32_t flags = 0; - if (returnValueVar != -1) - { - //flags |= CORINFO_CONTINUATION_HAS_RESULT; - } - - if (captureContinuationContext) - { - //flags |= CORINFO_CONTINUATION_HAS_CONTINUATION_CONTEXT; - } - if (continuationContextHandling == ContinuationContextHandling::ContinueOnThreadPool) { flags |= CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL; } - if (needsEHHandling) - { - //flags |= CORINFO_CONTINUATION_HAS_EXCEPTION; - } - suspendData->flags = (CorInfoContinuationFlags)flags; suspendData->offsetIntoContinuationTypeForExecutionContext = execContextOffset + OFFSETOF__CORINFO_Continuation__data; diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 3b7939007efff6..3e54b50e13a99a 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -2241,7 +2241,7 @@ void AsyncTransformation::CreateSuspension(BasicBlock* call auto encodeIndex = [&continuationFlags](unsigned offset, unsigned firstBit, unsigned numBits) { assert(numBits < 32); assert((offset % TARGET_POINTER_SIZE) == 0); - unsigned index = offset / TARGET_POINTER_SIZE; + unsigned index = 1 + offset / TARGET_POINTER_SIZE; unsigned mask = (1u << numBits) - 1; if ((index & mask) != index) diff --git a/src/coreclr/vm/interpexec.cpp b/src/coreclr/vm/interpexec.cpp index 19ad5fa7a63ecf..f24f550e0920d6 100644 --- a/src/coreclr/vm/interpexec.cpp +++ b/src/coreclr/vm/interpexec.cpp @@ -4248,16 +4248,14 @@ do \ SetObjectReference((OBJECTREF *)((uint8_t*)(OBJECTREFToObject(continuation)) + pAsyncSuspendData->offsetIntoContinuationTypeForExecutionContext), executionContext); continuation->SetFlags(pAsyncSuspendData->flags); - if (pAsyncSuspendData->flags & /*CORINFO_CONTINUATION_HAS_CONTINUATION_CONTEXT*/ 123) + uint32_t continuationContextMask = (1u << CORINFO_CONTINUATION_CONTEXT_INDEX_NUM_BITS) - 1; + uint32_t continuationContextIndex = ((uint32_t)pAsyncSuspendData->flags >> CORINFO_CONTINUATION_CONTEXT_INDEX_FIRST_BIT) & continuationContextMask; + if (continuationContextIndex != 0) { MethodDesc *captureSyncContextMethod = pAsyncSuspendData->captureSyncContextMethod; int32_t *flagsAddress = continuation->GetFlagsAddress(); - size_t continuationOffset = OFFSETOF__CORINFO_Continuation__data; + size_t continuationOffset = OFFSETOF__CORINFO_Continuation__data + (continuationContextIndex - 1) * TARGET_POINTER_SIZE; uint8_t *pContinuationData = (uint8_t*)OBJECTREFToObject(continuation) + continuationOffset; - if (pAsyncSuspendData->flags & /*CORINFO_CONTINUATION_HAS_EXCEPTION */ 456) - { - pContinuationData += sizeof(OBJECTREF); - } returnOffset = ip[1]; callArgsOffset = pMethod->allocaSize; @@ -4421,10 +4419,11 @@ do \ pCopyEntry++; } - if (pAsyncSuspendData->flags & /*CORINFO_CONTINUATION_HAS_EXCEPTION */ 567) + PTR_OBJECTREF pException = continuation->GetExceptionObjectStorageOrNull(); + if (pException != NULL) { // Throw exception if needed - OBJECTREF exception = *continuation->GetExceptionObjectStorage(); + OBJECTREF exception = *pException; if (exception != NULL) { diff --git a/src/coreclr/vm/object.h b/src/coreclr/vm/object.h index e8674c419d8143..d841dcb042fc85 100644 --- a/src/coreclr/vm/object.h +++ b/src/coreclr/vm/object.h @@ -2145,32 +2145,27 @@ class ContinuationObject : public Object PTR_BYTE GetResultStorage() { LIMITED_METHOD_CONTRACT; - PTR_BYTE dataAddress = dac_cast((dac_cast(this) + OFFSETOF__CORINFO_Continuation__data)); - //if (GetFlags() & CORINFO_CONTINUATION_HAS_OSR_ILOFFSET) - //{ - // dataAddress += sizeof(void*); - //} - //if (GetFlags() & CORINFO_CONTINUATION_HAS_EXCEPTION) - //{ - // dataAddress += sizeof(void*); - //} - //if (GetFlags() & CORINFO_CONTINUATION_HAS_CONTINUATION_CONTEXT) - //{ - // dataAddress += sizeof(void*); - //} + + uint32_t mask = (1u << CORINFO_CONTINUATION_RESULT_INDEX_NUM_BITS) - 1; + uint32_t index = ((uint32_t)Flags >> CORINFO_CONTINUATION_RESULT_INDEX_FIRST_BIT) & mask; + _ASSERTE(index != 0); + + uint32_t offset = OFFSETOF__CORINFO_Continuation__data + (index - 1) * TARGET_POINTER_SIZE; + PTR_BYTE dataAddress = dac_cast((dac_cast(this) + offset)); return dataAddress; } - PTR_OBJECTREF GetExceptionObjectStorage() + PTR_OBJECTREF GetExceptionObjectStorageOrNull() { LIMITED_METHOD_CONTRACT; - //_ASSERTE((GetFlags() & CORINFO_CONTINUATION_HAS_EXCEPTION)); - PTR_BYTE dataAddress = dac_cast((dac_cast(this) + OFFSETOF__CORINFO_Continuation__data)); - //if (GetFlags() & CORINFO_CONTINUATION_HAS_OSR_ILOFFSET) - //{ - // dataAddress += sizeof(void*); - //} + uint32_t mask = (1u << CORINFO_CONTINUATION_EXCEPTION_INDEX_NUM_BITS) - 1; + uint32_t index = ((uint32_t)Flags >> CORINFO_CONTINUATION_EXCEPTION_INDEX_FIRST_BIT) & mask; + if (index == 0) + return NULL; + + uint32_t offset = OFFSETOF__CORINFO_Continuation__data + (index - 1) * TARGET_POINTER_SIZE; + PTR_BYTE dataAddress = dac_cast(dac_cast(this) + offset); return dac_cast(dataAddress); } From a06b698c1ef70a37acbf4f569a4ed1884ae5544e Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sat, 14 Mar 2026 15:47:29 +0100 Subject: [PATCH 17/28] Use layout compatibility for returns --- src/coreclr/jit/async.cpp | 23 ++++++++++++++++------- src/coreclr/jit/async.h | 2 +- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 3e54b50e13a99a..7917c2f020c998 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -459,11 +459,18 @@ void ContinuationLayoutBuilder::AddReturn(const ReturnTypeInfo& info) { for (const ReturnTypeInfo& ret : m_returns) { - if ((ret.ReturnType == info.ReturnType) && (ret.ReturnLayout == info.ReturnLayout)) + if (ret.ReturnType != info.ReturnType) { - // This return type is already in the layout, no need to add another slot for it. - return; + continue; + } + + if ((ret.ReturnType == TYP_STRUCT) && !ClassLayout::AreCompatible(ret.ReturnLayout, info.ReturnLayout)) + { + continue; } + + // This return type is already in the layout, no need to add another slot for it. + return; } m_returns.push_back(info); @@ -1762,17 +1769,19 @@ void ContinuationLayout::Dump(int indent) // Find the return info entry matching the specified call's return type. // // Parameters: +// comp - Compiler instance to use for looking up struct return layouts. // call - The async call whose return type to look up. // // Returns: // Pointer to the matching ReturnInfo entry. // -const ReturnInfo* ContinuationLayout::FindReturn(GenTreeCall* call) const +const ReturnInfo* ContinuationLayout::FindReturn(Compiler* comp, GenTreeCall* call) const { + ClassLayout* layout = call->gtReturnType == TYP_STRUCT ? comp->typGetObjLayout(call->gtRetClsHnd) : nullptr; for (const ReturnInfo& ret : Returns) { if ((ret.Type.ReturnType == call->gtReturnType) && - ((call->gtReturnType != TYP_STRUCT) || (ret.Type.ReturnLayout->GetClassHandle() == call->gtRetClsHnd))) + ((call->gtReturnType != TYP_STRUCT) || ClassLayout::AreCompatible(ret.Type.ReturnLayout, layout))) { return &ret; } @@ -2260,7 +2269,7 @@ void AsyncTransformation::CreateSuspension(BasicBlock* call CORINFO_CONTINUATION_CONTEXT_INDEX_NUM_BITS); if (call->gtReturnType != TYP_VOID) { - const ReturnInfo* returnInfo = layout.FindReturn(call); + const ReturnInfo* returnInfo = layout.FindReturn(m_compiler, call); assert(returnInfo != nullptr); encodeIndex(returnInfo->Offset, CORINFO_CONTINUATION_RESULT_INDEX_FIRST_BIT, CORINFO_CONTINUATION_RESULT_INDEX_NUM_BITS); @@ -2905,7 +2914,7 @@ void AsyncTransformation::CopyReturnValueOnResumption(GenTreeCall* const ContinuationLayout& layout, BasicBlock* storeResultBB) { - const ReturnInfo* retInfo = layout.FindReturn(call); + const ReturnInfo* retInfo = layout.FindReturn(m_compiler, call); assert(retInfo != nullptr); GenTree* resultBase = m_compiler->gtNewLclvNode(m_compiler->lvaAsyncContinuationArg, TYP_REF); unsigned resultOffset = OFFSETOF__CORINFO_Continuation__data + retInfo->Offset; diff --git a/src/coreclr/jit/async.h b/src/coreclr/jit/async.h index 2534b5bcd3913b..0d443563421472 100644 --- a/src/coreclr/jit/async.h +++ b/src/coreclr/jit/async.h @@ -143,7 +143,7 @@ struct ContinuationLayout { } - const ReturnInfo* FindReturn(GenTreeCall* call) const; + const ReturnInfo* FindReturn(Compiler* comp, GenTreeCall* call) const; #ifdef DEBUG void Dump(int indent = 0); #endif From 69cb47b159c7f9c820ae0e4138ee8b27ebf6dbd0 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sat, 14 Mar 2026 20:32:44 +0100 Subject: [PATCH 18/28] WIP --- src/coreclr/jit/async.cpp | 110 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 7917c2f020c998..bc37fcf492b339 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -854,6 +854,116 @@ void DefaultValueAnalysis::DumpMutatedVarsIn() } #endif +class PreservedValueAnalysis +{ + Compiler* m_compiler; + // Blocks that have awaits in them + BitVec m_awaitBlocks; + // Blocks that may be entered after we resumed + BitVec m_resumeReachableBlocks; + VARSET_TP* m_mutatedVars; // Per-block set of locals mutated to non-default. + VARSET_TP* m_mutatedVarsIn; // Per-block set of locals mutated. + + // DataFlow::ForwardAnalysis callback used in Phase 2. + class DataFlowCallback + { + DefaultValueAnalysis& m_analysis; + Compiler* m_compiler; + VARSET_TP m_preMergeIn; + + public: + DataFlowCallback(DefaultValueAnalysis& analysis, Compiler* compiler) + : m_analysis(analysis) + , m_compiler(compiler) + , m_preMergeIn(VarSetOps::UninitVal()) + { + } + + void StartMerge(BasicBlock* block) + { + // Save the current in set for change detection later. + VarSetOps::Assign(m_compiler, m_preMergeIn, m_analysis.m_mutatedVarsIn[block->bbNum]); + } + + void Merge(BasicBlock* block, BasicBlock* predBlock, unsigned dupCount) + { + // The out set of a predecessor is its in set plus the locals + // mutated in that block: mutatedOut = mutatedIn | mutated. + VarSetOps::UnionD(m_compiler, m_analysis.m_mutatedVarsIn[block->bbNum], + m_analysis.m_mutatedVarsIn[predBlock->bbNum]); + VarSetOps::UnionD(m_compiler, m_analysis.m_mutatedVarsIn[block->bbNum], + m_analysis.m_mutatedVars[predBlock->bbNum]); + } + + void MergeHandler(BasicBlock* block, BasicBlock* firstTryBlock, BasicBlock* lastTryBlock) + { + // A handler can be reached from any point in the try region. + // A local is mutated at handler entry if it was mutated at try + // entry or mutated anywhere within the try region. + for (BasicBlock* tryBlock = firstTryBlock; tryBlock != lastTryBlock->Next(); tryBlock = tryBlock->Next()) + { + VarSetOps::UnionD(m_compiler, m_analysis.m_mutatedVarsIn[block->bbNum], + m_analysis.m_mutatedVarsIn[tryBlock->bbNum]); + VarSetOps::UnionD(m_compiler, m_analysis.m_mutatedVarsIn[block->bbNum], + m_analysis.m_mutatedVars[tryBlock->bbNum]); + } + } + + bool EndMerge(BasicBlock* block) + { + return !VarSetOps::Equal(m_compiler, m_preMergeIn, m_analysis.m_mutatedVarsIn[block->bbNum]); + } + }; + +public: + PreservedValueAnalysis(Compiler* compiler) + : m_compiler(compiler) + , m_mutatedVars(nullptr) + , m_mutatedVarsIn(nullptr) + { + } + + void Run(); + const VARSET_TP& GetMutatedVarsIn(BasicBlock* block) const; + +private: + void ComputePerBlockMutatedVars(); + void ComputeInterBlockDefaultValues(); + +#ifdef DEBUG + void DumpMutatedVars(); + void DumpMutatedVarsIn(); +#endif +}; + +//------------------------------------------------------------------------ +// PreservedValueAnalysis::Run: +// Run the default value analysis: compute per-block mutation sets, then +// propagate default value information forward through the flow graph. +// +void PreservedValueAnalysis::Run() +{ +#ifdef DEBUG + static ConfigMethodRange s_range; + s_range.EnsureInit(JitConfig.JitAsyncDefaultValueAnalysisRange()); + + if (!s_range.Contains(m_compiler->info.compMethodHash())) + { + JITDUMP("Default value analysis disabled because of method range\n"); + m_mutatedVarsIn = m_compiler->fgAllocateTypeForEachBlk(CMK_Async); + for (BasicBlock* block : m_compiler->Blocks()) + { + VarSetOps::AssignNoCopy(m_compiler, m_mutatedVarsIn[block->bbNum], VarSetOps::MakeFull(m_compiler)); + } + + return; + } + +#endif + ComputePerBlockMutatedVars(); + ComputeInterBlockDefaultValues(); +} + class AsyncLiveness { Compiler* m_compiler; From b1c94515ab0290be7e2dd26e4cc72a6f1800a51f Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 16 Mar 2026 14:40:26 +0100 Subject: [PATCH 19/28] Finish --- src/coreclr/jit/async.cpp | 491 +++++++++++++++++++++++++++--- src/coreclr/jit/async.h | 64 ++-- src/coreclr/jit/jitconfigvalues.h | 12 +- 3 files changed, 499 insertions(+), 68 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index bc37fcf492b339..b90e396d7c8a26 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -618,8 +618,8 @@ void DefaultValueAnalysis::Run() return; } - #endif + ComputePerBlockMutatedVars(); ComputeInterBlockDefaultValues(); } @@ -854,25 +854,93 @@ void DefaultValueAnalysis::DumpMutatedVarsIn() } #endif +//------------------------------------------------------------------------ +// MarkMutatedLocal: +// If the given node is a local store or LCL_ADDR, and the local is tracked, +// mark it as mutated in the provided set. Unlike UpdateMutatedLocal, all +// stores count as mutations (including stores of default values). +// +// Parameters: +// compiler - The compiler instance. +// node - The IR node to check. +// mutated - [in/out] The set to update. +// +static void MarkMutatedLocal(Compiler* compiler, GenTree* node, VARSET_TP& mutated) +{ + if (!node->OperIsLocalStore() && !node->OperIs(GT_LCL_ADDR)) + { + return; + } + + LclVarDsc* varDsc = compiler->lvaGetDesc(node->AsLclVarCommon()); + + if (varDsc->lvTracked) + { + VarSetOps::AddElemD(compiler, mutated, varDsc->lvVarIndex); + return; + } + + if (varDsc->lvPromoted) + { + for (unsigned i = 0; i < varDsc->lvFieldCnt; i++) + { + LclVarDsc* fieldDsc = compiler->lvaGetDesc(varDsc->lvFieldLclStart + i); + if (fieldDsc->lvTracked) + { + VarSetOps::AddElemD(compiler, mutated, fieldDsc->lvVarIndex); + } + } + } +} + +//------------------------------------------------------------------------ +// PreservedValueAnalysis: +// Computes which tracked locals may have been mutated since the previous +// resumption point. A local that has not been mutated since the last +// resumption does not need to be stored into a reused continuation, because +// the continuation already holds the correct value. +// +// The analysis proceeds in several steps: +// +// 0. Identify blocks that contain awaits and compute which blocks are +// reachable after resumption. +// +// 1. Per-block: compute which tracked locals are mutated (assigned any +// value or have their address taken) in each block. Only mutations +// reachable after resumption need to be taken into account. +// +// 2. Inter-block: forward dataflow to compute for each block the set of +// tracked locals that have been mutated since the previous resumption. +// At merge points the sets are unioned (a local is mutated if it is +// mutated on any incoming path). +// class PreservedValueAnalysis { - Compiler* m_compiler; - // Blocks that have awaits in them + Compiler* m_compiler; + + BitVecTraits m_blockTraits; + + // Blocks that have awaits in them. BitVec m_awaitBlocks; - // Blocks that may be entered after we resumed + + // Blocks that may be entered after we resumed. BitVec m_resumeReachableBlocks; - VARSET_TP* m_mutatedVars; // Per-block set of locals mutated to non-default. - VARSET_TP* m_mutatedVarsIn; // Per-block set of locals mutated. + + // Per-block set of locals that may be mutated by each block after a resumption. + VARSET_TP* m_mutatedVars; + + // Per-block incoming set of locals possibly mutated since previous resumption. + VARSET_TP* m_mutatedVarsIn; // DataFlow::ForwardAnalysis callback used in Phase 2. class DataFlowCallback { - DefaultValueAnalysis& m_analysis; - Compiler* m_compiler; - VARSET_TP m_preMergeIn; + PreservedValueAnalysis& m_analysis; + Compiler* m_compiler; + VARSET_TP m_preMergeIn; public: - DataFlowCallback(DefaultValueAnalysis& analysis, Compiler* compiler) + DataFlowCallback(PreservedValueAnalysis& analysis, Compiler* compiler) : m_analysis(analysis) , m_compiler(compiler) , m_preMergeIn(VarSetOps::UninitVal()) @@ -881,14 +949,12 @@ class PreservedValueAnalysis void StartMerge(BasicBlock* block) { - // Save the current in set for change detection later. VarSetOps::Assign(m_compiler, m_preMergeIn, m_analysis.m_mutatedVarsIn[block->bbNum]); } void Merge(BasicBlock* block, BasicBlock* predBlock, unsigned dupCount) { - // The out set of a predecessor is its in set plus the locals - // mutated in that block: mutatedOut = mutatedIn | mutated. + // Normal predecessor: mutatedOut = mutatedIn | mutated. VarSetOps::UnionD(m_compiler, m_analysis.m_mutatedVarsIn[block->bbNum], m_analysis.m_mutatedVarsIn[predBlock->bbNum]); VarSetOps::UnionD(m_compiler, m_analysis.m_mutatedVarsIn[block->bbNum], @@ -897,9 +963,6 @@ class PreservedValueAnalysis void MergeHandler(BasicBlock* block, BasicBlock* firstTryBlock, BasicBlock* lastTryBlock) { - // A handler can be reached from any point in the try region. - // A local is mutated at handler entry if it was mutated at try - // entry or mutated anywhere within the try region. for (BasicBlock* tryBlock = firstTryBlock; tryBlock != lastTryBlock->Next(); tryBlock = tryBlock->Next()) { VarSetOps::UnionD(m_compiler, m_analysis.m_mutatedVarsIn[block->bbNum], @@ -918,19 +981,26 @@ class PreservedValueAnalysis public: PreservedValueAnalysis(Compiler* compiler) : m_compiler(compiler) + , m_blockTraits(compiler->fgBBNumMax + 1, compiler) + , m_awaitBlocks(BitVecOps::UninitVal()) + , m_resumeReachableBlocks(BitVecOps::UninitVal()) , m_mutatedVars(nullptr) , m_mutatedVarsIn(nullptr) { } - void Run(); + void Run(ArrayStack& awaitBlocks); const VARSET_TP& GetMutatedVarsIn(BasicBlock* block) const; + bool IsResumeReachable(BasicBlock* block); private: + void ComputeResumeReachableBlocks(ArrayStack& awaitBlocks); void ComputePerBlockMutatedVars(); - void ComputeInterBlockDefaultValues(); + void ComputeInterBlockMutatedVars(); #ifdef DEBUG + void DumpAwaitBlocks(); + void DumpResumeReachableBlocks(); void DumpMutatedVars(); void DumpMutatedVarsIn(); #endif @@ -938,18 +1008,19 @@ class PreservedValueAnalysis //------------------------------------------------------------------------ // PreservedValueAnalysis::Run: -// Run the default value analysis: compute per-block mutation sets, then -// propagate default value information forward through the flow graph. +// Run the preserved value analysis: identify await blocks, compute +// resume-reachable blocks, then compute per-block and inter-block +// mutation sets relative to resumption points. // -void PreservedValueAnalysis::Run() +void PreservedValueAnalysis::Run(ArrayStack& awaitBlocks) { #ifdef DEBUG static ConfigMethodRange s_range; - s_range.EnsureInit(JitConfig.JitAsyncDefaultValueAnalysisRange()); + s_range.EnsureInit(JitConfig.JitAsyncPreservedValueAnalysisRange()); if (!s_range.Contains(m_compiler->info.compMethodHash())) { - JITDUMP("Default value analysis disabled because of method range\n"); + JITDUMP("Preserved value analysis disabled because of method range\n"); m_mutatedVarsIn = m_compiler->fgAllocateTypeForEachBlk(CMK_Async); for (BasicBlock* block : m_compiler->Blocks()) { @@ -958,27 +1029,288 @@ void PreservedValueAnalysis::Run() return; } - #endif + + ComputeResumeReachableBlocks(awaitBlocks); ComputePerBlockMutatedVars(); - ComputeInterBlockDefaultValues(); + ComputeInterBlockMutatedVars(); +} + +//------------------------------------------------------------------------ +// PreservedValueAnalysis::GetMutatedVarsIn: +// Get the set of tracked locals that may have been mutated since the +// previous resumption on entry to the specified block. +// +// Parameters: +// block - The basic block. +// +// Returns: +// The VARSET_TP of tracked locals mutated since last resumption. A local +// NOT in this set has a preserved value in the continuation and does not +// need to be re-stored when reusing it. +// +const VARSET_TP& PreservedValueAnalysis::GetMutatedVarsIn(BasicBlock* block) const +{ + assert(m_mutatedVarsIn != nullptr); + return m_mutatedVarsIn[block->bbNum]; +} + +bool PreservedValueAnalysis::IsResumeReachable(BasicBlock* block) +{ + return BitVecOps::IsMember(&m_blockTraits, m_resumeReachableBlocks, block->bbNum); +} + +//------------------------------------------------------------------------ +// PreservedValueAnalysis::ComputeResumeReachableBlocks: +// Step 0: Identify blocks containing awaits, then compute the set of +// blocks reachable after any resumption via a forward fixpoint from +// the successors of await blocks. +// +void PreservedValueAnalysis::ComputeResumeReachableBlocks(ArrayStack& awaitBlocks) +{ + m_awaitBlocks = BitVecOps::MakeEmpty(&m_blockTraits); + m_resumeReachableBlocks = BitVecOps::MakeEmpty(&m_blockTraits); + + ArrayStack worklist(m_compiler->getAllocator(CMK_Async)); + // Find all blocks that contain awaits. + for (BasicBlock* awaitBlock : awaitBlocks.BottomUpOrder()) + { + BitVecOps::AddElemD(&m_blockTraits, m_awaitBlocks, awaitBlock->bbNum); + worklist.Push(awaitBlock); + } + + JITDUMP("Preserved value analysis: blocks containing awaits\n"); + DBEXEC(m_compiler->verbose, DumpAwaitBlocks()); + + while (!worklist.Empty()) + { + BasicBlock* block = worklist.Pop(); + + block->VisitAllSuccs(m_compiler, [&](BasicBlock* succ) { + if (BitVecOps::TryAddElemD(&m_blockTraits, m_resumeReachableBlocks, succ->bbNum)) + { + worklist.Push(succ); + } + return BasicBlockVisit::Continue; + }); + } + + JITDUMP("Preserved value analysis: blocks reachable after resuming\n"); + DBEXEC(m_compiler->verbose, DumpResumeReachableBlocks()); +} + +//------------------------------------------------------------------------ +// PreservedValueAnalysis::ComputePerBlockMutatedVars: +// Phase 1: For each reachable basic block compute the set of tracked locals +// that are mutated. +// +// For blocks that are reachable after resumption the full set of mutations +// in the block is recorded. +// +// For blocks that are NOT reachable after resumption but contain an await, +// only mutations that occur after the first suspension point are recorded, +// because mutations before the first suspension are not relevant to +// preserved values (the continuation did not exist yet or was not being +// reused at that point). +// +void PreservedValueAnalysis::ComputePerBlockMutatedVars() +{ + m_mutatedVars = m_compiler->fgAllocateTypeForEachBlk(CMK_Async); + + for (unsigned i = 0; i <= m_compiler->fgBBNumMax; i++) + { + VarSetOps::AssignNoCopy(m_compiler, m_mutatedVars[i], VarSetOps::MakeEmpty(m_compiler)); + } + + for (BasicBlock* block : m_compiler->Blocks()) + { + VARSET_TP& mutated = m_mutatedVars[block->bbNum]; + + bool isAwaitBlock = BitVecOps::IsMember(&m_blockTraits, m_awaitBlocks, block->bbNum); + bool isResumeReachable = BitVecOps::IsMember(&m_blockTraits, m_resumeReachableBlocks, block->bbNum); + + if (!isResumeReachable && !isAwaitBlock) + { + continue; + } + + GenTree* node = block->GetFirstLIRNode(); + + if (!isResumeReachable) + { + while (!node->IsCall() || !node->AsCall()->IsAsync()) + { + node = node->gtNext; + assert(node != nullptr); + } + + node = node->gtNext; + } + + while (node != nullptr) + { + MarkMutatedLocal(m_compiler, node, mutated); + node = node->gtNext; + } + } + + JITDUMP("Preserved value analysis: per-block mutated vars after resumption\n"); + DBEXEC(m_compiler->verbose, DumpMutatedVars()); +} + +//------------------------------------------------------------------------ +// PreservedValueAnalysis::ComputeInterBlockMutatedVars: +// Phase 2: Forward dataflow to compute for each block the set of tracked +// locals that have been mutated since the previous resumption on entry. +// +// Transfer function: mutatedOut[B] = mutatedIn[B] | mutated[B] +// Merge: mutatedIn[B] = union of mutatedOut[pred] for all preds +// +// At method entry all locals are considered mutated (no previous resumption +// to preserve from). +// +void PreservedValueAnalysis::ComputeInterBlockMutatedVars() +{ + m_mutatedVarsIn = m_compiler->fgAllocateTypeForEachBlk(CMK_Async); + + for (unsigned i = 0; i <= m_compiler->fgBBNumMax; i++) + { + VarSetOps::AssignNoCopy(m_compiler, m_mutatedVarsIn[i], VarSetOps::MakeEmpty(m_compiler)); + } + + DataFlowCallback callback(*this, m_compiler); + DataFlow flow(m_compiler); + flow.ForwardAnalysis(callback); + + JITDUMP("Preserved value analysis: per-block mutated vars on entry\n"); + DBEXEC(m_compiler->verbose, DumpMutatedVarsIn()); } +#ifdef DEBUG +//------------------------------------------------------------------------ +// PreservedValueAnalysis::DumpAwaitBlocks: +// Debug helper to print the set of blocks containing awaits. +// +void PreservedValueAnalysis::DumpAwaitBlocks() +{ + printf(" Await blocks:"); + const char* sep = " "; + for (BasicBlock* block : m_compiler->Blocks()) + { + if (BitVecOps::IsMember(&m_blockTraits, m_awaitBlocks, block->bbNum)) + { + printf("%s" FMT_BB, sep, block->bbNum); + sep = ", "; + } + } + printf("\n"); +} + +//------------------------------------------------------------------------ +// PreservedValueAnalysis::DumpResumeReachableBlocks: +// Debug helper to print the set of resume-reachable blocks. +// +void PreservedValueAnalysis::DumpResumeReachableBlocks() +{ + printf(" Resume-reachable blocks:"); + const char* sep = " "; + for (BasicBlock* block : m_compiler->Blocks()) + { + if (BitVecOps::IsMember(&m_blockTraits, m_resumeReachableBlocks, block->bbNum)) + { + printf("%s" FMT_BB, sep, block->bbNum); + sep = ", "; + } + } + printf("\n"); +} + +//------------------------------------------------------------------------ +// PreservedValueAnalysis::DumpMutatedVars: +// Debug helper to print the per-block mutated variable sets. +// +void PreservedValueAnalysis::DumpMutatedVars() +{ + const FlowGraphDfsTree* dfsTree = m_compiler->m_dfsTree; + for (unsigned i = 0; i < dfsTree->GetPostOrderCount(); i++) + { + BasicBlock* block = dfsTree->GetPostOrder(i); + if (!VarSetOps::IsEmpty(m_compiler, m_mutatedVars[block->bbNum])) + { + printf(" " FMT_BB " mutated: ", block->bbNum); + VarSetOps::Iter iter(m_compiler, m_mutatedVars[block->bbNum]); + unsigned varIndex = 0; + const char* sep = ""; + while (iter.NextElem(&varIndex)) + { + unsigned lclNum = m_compiler->lvaTrackedToVarNum[varIndex]; + printf("%sV%02u", sep, lclNum); + sep = " "; + } + printf("\n"); + } + } +} + +//------------------------------------------------------------------------ +// PreservedValueAnalysis::DumpMutatedVarsIn: +// Debug helper to print the per-block mutated-on-entry variable sets. +// +void PreservedValueAnalysis::DumpMutatedVarsIn() +{ + const FlowGraphDfsTree* dfsTree = m_compiler->m_dfsTree; + for (unsigned i = dfsTree->GetPostOrderCount(); i > 0; i--) + { + BasicBlock* block = dfsTree->GetPostOrder(i - 1); + printf(" " FMT_BB " mutated since resumption on entry: ", block->bbNum); + + if (VarSetOps::IsEmpty(m_compiler, m_mutatedVarsIn[block->bbNum])) + { + printf(""); + } + else if (VarSetOps::Equal(m_compiler, m_mutatedVarsIn[block->bbNum], VarSetOps::MakeFull(m_compiler))) + { + printf(""); + } + else + { + VarSetOps::Iter iter(m_compiler, m_mutatedVarsIn[block->bbNum]); + unsigned varIndex = 0; + const char* sep = ""; + while (iter.NextElem(&varIndex)) + { + unsigned lclNum = m_compiler->lvaTrackedToVarNum[varIndex]; + printf("%sV%02u", sep, lclNum); + sep = " "; + } + } + printf("\n"); + } +} +#endif + class AsyncLiveness { - Compiler* m_compiler; - TreeLifeUpdater m_updater; - unsigned m_numVars; - DefaultValueAnalysis& m_defaultValueAnalysis; - VARSET_TP m_mutatedValues; + Compiler* m_compiler; + TreeLifeUpdater m_updater; + unsigned m_numVars; + DefaultValueAnalysis& m_defaultValueAnalysis; + PreservedValueAnalysis& m_preservedValueAnalysis; + VARSET_TP m_mutatedValues; + bool m_resumeReachable = false; + VARSET_TP m_mutatedSinceResumption; public: - AsyncLiveness(Compiler* comp, DefaultValueAnalysis& defaultValueAnalysis) + AsyncLiveness(Compiler* comp, + DefaultValueAnalysis& defaultValueAnalysis, + PreservedValueAnalysis& preservedValueAnalysis) : m_compiler(comp) , m_updater(comp) , m_numVars(comp->lvaCount) , m_defaultValueAnalysis(defaultValueAnalysis) + , m_preservedValueAnalysis(preservedValueAnalysis) , m_mutatedValues(VarSetOps::MakeEmpty(comp)) + , m_mutatedSinceResumption(VarSetOps::MakeEmpty(comp)) { } @@ -988,6 +1320,15 @@ class AsyncLiveness template void GetLiveLocals(ContinuationLayoutBuilder* layoutBuilder, Functor includeLocal); + bool IsResumeReachable() const + { + return m_resumeReachable; + } + + VARSET_TP GetMutatedSinceResumption() const + { + return m_mutatedSinceResumption; + } private: bool IsLocalCaptureUnnecessary(unsigned lclNum); }; @@ -1004,6 +1345,8 @@ void AsyncLiveness::StartBlock(BasicBlock* block) { VarSetOps::Assign(m_compiler, m_compiler->compCurLife, block->bbLiveIn); VarSetOps::Assign(m_compiler, m_mutatedValues, m_defaultValueAnalysis.GetMutatedVarsIn(block)); + VarSetOps::Assign(m_compiler, m_mutatedSinceResumption, m_preservedValueAnalysis.GetMutatedVarsIn(block)); + m_resumeReachable = m_preservedValueAnalysis.IsResumeReachable(block); } //------------------------------------------------------------------------ @@ -1018,6 +1361,11 @@ void AsyncLiveness::Update(GenTree* node) { m_updater.UpdateLife(node); UpdateMutatedLocal(m_compiler, node, m_mutatedValues); + if (m_resumeReachable) + { + MarkMutatedLocal(m_compiler, node, m_mutatedSinceResumption); + } + m_resumeReachable |= node->IsCall() && node->AsCall()->IsAsync(); } //------------------------------------------------------------------------ @@ -1292,7 +1640,9 @@ PhaseStatus AsyncTransformation::Run() DefaultValueAnalysis defaultValues(m_compiler); defaultValues.Run(); - AsyncLiveness liveness(m_compiler, defaultValues); + PreservedValueAnalysis preservedValues(m_compiler); + preservedValues.Run(worklist); + AsyncLiveness liveness(m_compiler, defaultValues, preservedValues); // Now walk the IR for all the blocks that contain async calls. Keep track // of liveness and outstanding LIR edges as we go; the LIR edges that cross @@ -1508,6 +1858,9 @@ void AsyncTransformation::Transform( } #endif + bool resumeReachable = life.IsResumeReachable(); + VARSET_TP mutatedSinceResumption(VarSetOps::MakeCopy(m_compiler, life.GetMutatedSinceResumption())); + ContinuationLayoutBuilder* layoutBuilder = new (m_compiler, CMK_Async) ContinuationLayoutBuilder(m_compiler); CreateLiveSetForSuspension(block, call, defs, life, layoutBuilder); @@ -1525,7 +1878,8 @@ void AsyncTransformation::Transform( BasicBlock* resumeBB = CreateResumptionBlock(*remainder, call, stateNum, layoutBuilder); - m_states.push_back(AsyncState(stateNum, layoutBuilder, block, call, callDefInfo, suspendBB, resumeBB)); + m_states.push_back(AsyncState(stateNum, layoutBuilder, block, call, callDefInfo, suspendBB, resumeBB, + resumeReachable, mutatedSinceResumption)); JITDUMP("\n"); } @@ -2264,13 +2618,18 @@ BasicBlock* AsyncTransformation::CreateSuspensionBlock(BasicBlock* block, GenTre // stateNum - State number assigned to this suspension point. // layout - Layout information for the continuation object. // subLayout - Per-call layout builder indicating which fields are needed. +// resumeReachable - Whether or not this suspension point is reachable from a previous resumption +// mutatedSinceResumption - If this suspension point is reachable from a previous resumption +// then this indicates the set of tracked variables that may have been mutated since then. // void AsyncTransformation::CreateSuspension(BasicBlock* callBlock, GenTreeCall* call, BasicBlock* suspendBB, unsigned stateNum, const ContinuationLayout& layout, - const ContinuationLayoutBuilder& subLayout) + const ContinuationLayoutBuilder& subLayout, + bool resumeReachable, + VARSET_VALARG_TP mutatedSinceResumption) { GenTreeILOffset* ilOffsetNode = m_compiler->gtNewILOffsetNode(call->GetAsyncInfo().CallAsyncDebugInfo DEBUGARG(BAD_IL_OFFSET)); @@ -2296,7 +2655,8 @@ void AsyncTransformation::CreateSuspension(BasicBlock* call GenTree* storeNewContinuation = m_compiler->gtNewStoreLclVarNode(newContinuationVar, allocContinuation); LIR::AsRange(suspendBB).InsertAtEnd(storeNewContinuation); - if (ReuseContinuations()) + SaveSet tailSaveSet = SaveSet::All; + if (ReuseContinuations() && resumeReachable) { // Split suspendBB into suspendBB -> [reuse continuation with Next store] -> [allocNewBlock with allocation // call] -> suspendBBTail [empty] @@ -2336,6 +2696,14 @@ void AsyncTransformation::CreateSuspension(BasicBlock* call GenTree* storeNext = StoreAtOffset(returnedContinuation, nextOffset, newContinuation, TYP_REF); LIR::AsRange(reuseContinuationBB).InsertAtEnd(LIR::SeqTree(m_compiler, storeNext)); + // In the path where we allocated a new continuation we save only locals that we know to be unmutated since the + // last resumption. + FillInDataOnSuspension(call, layout, subLayout, allocNewBB, mutatedSinceResumption, SaveSet::UnmutatedLocals); + + // We can skip saving unmutated locals in the shared path -- we only need to save locals that may have been + // mutated since the last resumption. + tailSaveSet = SaveSet::MutatedLocals; + suspendBB = suspendTailBB; } @@ -2393,10 +2761,7 @@ void AsyncTransformation::CreateSuspension(BasicBlock* call GenTree* storeFlags = StoreAtOffset(newContinuation, flagsOffset, flagsNode, TYP_INT); LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_compiler, storeFlags)); - if (layout.Size > 0) - { - FillInDataOnSuspension(call, layout, subLayout, suspendBB); - } + FillInDataOnSuspension(call, layout, subLayout, suspendBB, mutatedSinceResumption, tailSaveSet); RestoreContexts(callBlock, call, suspendBB); @@ -2461,9 +2826,11 @@ GenTreeCall* AsyncTransformation::CreateAllocContinuationCall(bool void AsyncTransformation::FillInDataOnSuspension(GenTreeCall* call, const ContinuationLayout& layout, const ContinuationLayoutBuilder& subLayout, - BasicBlock* suspendBB) + BasicBlock* suspendBB, + VARSET_VALARG_TP mutatedSinceResumption, + SaveSet saveSet) { - if (m_compiler->doesMethodHavePatchpoints() || m_compiler->opts.IsOSR()) + if ((saveSet != SaveSet::MutatedLocals) && (m_compiler->doesMethodHavePatchpoints() || m_compiler->opts.IsOSR())) { GenTree* ilOffsetToStore; if (m_compiler->doesMethodHavePatchpoints()) @@ -2490,6 +2857,12 @@ void AsyncTransformation::FillInDataOnSuspension(GenTreeCall* LclVarDsc* dsc = m_compiler->lvaGetDesc(inf.LclNum); + if ((saveSet != SaveSet::All) && + ((saveSet == SaveSet::UnmutatedLocals) != IsLocalUnmutatedSinceLastResumption(dsc, mutatedSinceResumption))) + { + continue; + } + GenTree* newContinuation = m_compiler->gtNewLclvNode(GetNewContinuationVar(), TYP_REF); unsigned offset = OFFSETOF__CORINFO_Continuation__data + inf.Offset; @@ -2527,7 +2900,7 @@ void AsyncTransformation::FillInDataOnSuspension(GenTreeCall* m_compiler->compLongUsed |= dsc->TypeIs(TYP_LONG); } - if (subLayout.NeedsContinuationContext()) + if ((saveSet != SaveSet::UnmutatedLocals) && subLayout.NeedsContinuationContext()) { // Insert call // AsyncHelpers.CaptureContinuationContext( @@ -2578,7 +2951,7 @@ void AsyncTransformation::FillInDataOnSuspension(GenTreeCall* LIR::AsRange(suspendBB).Remove(flagsPlaceholder); } - if (subLayout.NeedsExecutionContext()) + if ((saveSet != SaveSet::UnmutatedLocals) && subLayout.NeedsExecutionContext()) { GenTreeCall* captureExecContext = m_compiler->gtNewCallNode(CT_USER_FUNC, m_asyncInfo->captureExecutionContextMethHnd, TYP_REF); @@ -2594,6 +2967,35 @@ void AsyncTransformation::FillInDataOnSuspension(GenTreeCall* } } +bool AsyncTransformation::IsLocalUnmutatedSinceLastResumption(const LclVarDsc* dsc, + VARSET_VALARG_TP mutatedSinceResumption) +{ + if (dsc->lvPromoted) + { + for (unsigned i = 0; i < dsc->lvFieldCnt; i++) + { + LclVarDsc* fieldDsc = m_compiler->lvaGetDesc(dsc->lvFieldLclStart + i); + if (!fieldDsc->lvTracked || VarSetOps::IsMember(m_compiler, mutatedSinceResumption, fieldDsc->lvVarIndex)) + { + return false; + } + } + + return true; + } + + // We should only see struct fields for independently promoted structs + assert(!dsc->lvIsStructField || + (m_compiler->lvaGetPromotionType(dsc->lvParentLcl) == Compiler::PROMOTION_TYPE_INDEPENDENT)); + + if (!dsc->lvTracked) + { + return false; + } + + return !VarSetOps::IsMember(m_compiler, mutatedSinceResumption, dsc->lvVarIndex); +} + //------------------------------------------------------------------------ // AsyncTransformation::RestoreContexts: // Create IR to restore contexts on suspension. @@ -3343,7 +3745,8 @@ void AsyncTransformation::CreateResumptionsAndSuspensions() JITDUMP("State %u suspend @ " FMT_BB ", resume @ " FMT_BB "\n", state.Number, state.SuspensionBB->bbNum, state.ResumptionBB->bbNum); ContinuationLayout* layout = sharedLayout == nullptr ? state.Layout->Create() : sharedLayout; - CreateSuspension(state.CallBlock, state.Call, state.SuspensionBB, state.Number, *layout, *state.Layout); + CreateSuspension(state.CallBlock, state.Call, state.SuspensionBB, state.Number, *layout, *state.Layout, + state.ResumeReachable, state.MutatedSincePreviousResumption); CreateResumption(state.CallBlock, state.Call, state.ResumptionBB, state.CallDefInfo, *layout, *state.Layout); CreateDebugInfoForSuspensionPoint(*layout, *state.Layout); diff --git a/src/coreclr/jit/async.h b/src/coreclr/jit/async.h index 0d443563421472..ede78c71b59aab 100644 --- a/src/coreclr/jit/async.h +++ b/src/coreclr/jit/async.h @@ -166,7 +166,9 @@ struct AsyncState GenTreeCall* call, CallDefinitionInfo callDefInfo, BasicBlock* suspensionBB, - BasicBlock* resumptionBB) + BasicBlock* resumptionBB, + bool resumeReachable, + VARSET_TP mutatedSincePreviousResumption) : Number(number) , Layout(layout) , CallBlock(callBlock) @@ -174,6 +176,7 @@ struct AsyncState , CallDefInfo(callDefInfo) , SuspensionBB(suspensionBB) , ResumptionBB(resumptionBB) + , MutatedSincePreviousResumption(mutatedSincePreviousResumption) { } @@ -184,6 +187,17 @@ struct AsyncState CallDefinitionInfo CallDefInfo; BasicBlock* SuspensionBB; BasicBlock* ResumptionBB; + // Is this suspension point reachable after a previous resumption? + bool ResumeReachable; + // Set of variables that may have been mutated since the previous resumption. + VARSET_TP MutatedSincePreviousResumption; +}; + +enum class SaveSet +{ + All, + UnmutatedLocals, + MutatedLocals, }; class AsyncTransformation @@ -243,31 +257,37 @@ class AsyncTransformation BasicBlock* suspendBB, unsigned stateNum, const ContinuationLayout& layout, - const ContinuationLayoutBuilder& subLayout); + const ContinuationLayoutBuilder& subLayout, + bool resumeReachable, + VARSET_VALARG_TP mutatedSinceResumption); GenTreeCall* CreateAllocContinuationCall(bool hasKeepAlive, GenTree* prevContinuation, const ContinuationLayout& layout); - void FillInDataOnSuspension(GenTreeCall* call, - const ContinuationLayout& layout, - const ContinuationLayoutBuilder& subLayout, - BasicBlock* suspendBB); - void RestoreContexts(BasicBlock* block, GenTreeCall* call, BasicBlock* insertionBB); - void CreateCheckAndSuspendAfterCall(BasicBlock* block, - GenTreeCall* call, - const CallDefinitionInfo& callDefInfo, - BasicBlock* suspendBB, - BasicBlock** remainder); - BasicBlock* CreateResumptionBlock(BasicBlock* remainder, - GenTreeCall* call, - unsigned stateNum, - ContinuationLayoutBuilder* layoutBuilder); - void CreateResumption(BasicBlock* callBlock, - GenTreeCall* call, - BasicBlock* resumeBB, - const CallDefinitionInfo& callDefInfo, - const ContinuationLayout& layout, - const ContinuationLayoutBuilder& subLayout); + + void FillInDataOnSuspension(GenTreeCall* call, + const ContinuationLayout& layout, + const ContinuationLayoutBuilder& subLayout, + BasicBlock* suspendBB, + VARSET_VALARG_TP mutatedSinceResumption, + SaveSet saveSet); + bool IsLocalUnmutatedSinceLastResumption(const LclVarDsc* dsc, VARSET_VALARG_TP mutatedSinceResumption); + void RestoreContexts(BasicBlock* block, GenTreeCall* call, BasicBlock* insertionBB); + void CreateCheckAndSuspendAfterCall(BasicBlock* block, + GenTreeCall* call, + const CallDefinitionInfo& callDefInfo, + BasicBlock* suspendBB, + BasicBlock** remainder); + BasicBlock* CreateResumptionBlock(BasicBlock* remainder, + GenTreeCall* call, + unsigned stateNum, + ContinuationLayoutBuilder* layoutBuilder); + void CreateResumption(BasicBlock* callBlock, + GenTreeCall* call, + BasicBlock* resumeBB, + const CallDefinitionInfo& callDefInfo, + const ContinuationLayout& layout, + const ContinuationLayoutBuilder& subLayout); void RestoreFromDataOnResumption(const ContinuationLayout& layout, const ContinuationLayoutBuilder& subLayout, diff --git a/src/coreclr/jit/jitconfigvalues.h b/src/coreclr/jit/jitconfigvalues.h index 7c7dbfe17fc1d0..1513ffdc09c63f 100644 --- a/src/coreclr/jit/jitconfigvalues.h +++ b/src/coreclr/jit/jitconfigvalues.h @@ -590,11 +590,19 @@ OPT_CONFIG_INTEGER(JitDoOptimizeMaskConversions, "JitDoOptimizeMaskConversions", // conversions OPT_CONFIG_INTEGER(JitOptimizeAwait, "JitOptimizeAwait", 1) // Perform optimization of Await intrinsics -OPT_CONFIG_STRING(JitAsyncDefaultValueAnalysisRange, - "JitAsyncDefaultValueAnalysisRange") // Enable async default value analysis based on method hash range + +// Enable async default value analysis based on method hash range. This +// optimization skips saving locals in continuations when they are known to +// have their default value. +OPT_CONFIG_STRING(JitAsyncDefaultValueAnalysisRange, "JitAsyncDefaultValueAnalysisRange") // Save and reuse continuation instances in runtime async functions. Also // implies use of shared continuation layouts for all suspension points. RELEASE_CONFIG_INTEGER(JitAsyncReuseContinuations, "JitAsyncReuseContinuations", 1) +// Enable async preserved value analysis based on method hash range. This +// analysis computes state that is guaranteed to not have been changed since +// the last time suspension happened, and skips storing them in the case where +// a continuation is being reused. +OPT_CONFIG_STRING(JitAsyncPreservedValueAnalysisRange, "JitAsyncPreservedValueAnalysisRange") RELEASE_CONFIG_INTEGER(JitEnableOptRepeat, "JitEnableOptRepeat", 1) // If zero, do not allow JitOptRepeat RELEASE_CONFIG_METHODSET(JitOptRepeat, "JitOptRepeat") // Runs optimizer multiple times on specified methods From 89571af394eecc76df6acb8a09b59e02e47358dc Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 16 Mar 2026 15:33:02 +0100 Subject: [PATCH 20/28] Feedback --- src/coreclr/jit/async.cpp | 2 +- src/coreclr/jit/async.h | 1 + src/coreclr/jit/jitconfigvalues.h | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index b90e396d7c8a26..4180d9008198bf 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -2368,7 +2368,7 @@ ContinuationLayout* ContinuationLayoutBuilder::Create() layout->ContinuationContextOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); } - // Now allocate all returns + // Now allocate all returns for (ReturnInfo& ret : layout->Returns) { // All returns must be pointer aligned because of the offset encoding in Continuation::Flags. diff --git a/src/coreclr/jit/async.h b/src/coreclr/jit/async.h index ede78c71b59aab..61e61a61a88fb7 100644 --- a/src/coreclr/jit/async.h +++ b/src/coreclr/jit/async.h @@ -176,6 +176,7 @@ struct AsyncState , CallDefInfo(callDefInfo) , SuspensionBB(suspensionBB) , ResumptionBB(resumptionBB) + , ResumeReachable(resumeReachable) , MutatedSincePreviousResumption(mutatedSincePreviousResumption) { } diff --git a/src/coreclr/jit/jitconfigvalues.h b/src/coreclr/jit/jitconfigvalues.h index 1513ffdc09c63f..e519f1cc5ab6d1 100644 --- a/src/coreclr/jit/jitconfigvalues.h +++ b/src/coreclr/jit/jitconfigvalues.h @@ -594,7 +594,7 @@ OPT_CONFIG_INTEGER(JitOptimizeAwait, "JitOptimizeAwait", 1) // Perform optimizat // Enable async default value analysis based on method hash range. This // optimization skips saving locals in continuations when they are known to // have their default value. -OPT_CONFIG_STRING(JitAsyncDefaultValueAnalysisRange, "JitAsyncDefaultValueAnalysisRange") +OPT_CONFIG_STRING(JitAsyncDefaultValueAnalysisRange, "JitAsyncDefaultValueAnalysisRange") // Save and reuse continuation instances in runtime async functions. Also // implies use of shared continuation layouts for all suspension points. RELEASE_CONFIG_INTEGER(JitAsyncReuseContinuations, "JitAsyncReuseContinuations", 1) From 32f249208303e3ccec4bf7d0abed35631b2e2038 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 19 Mar 2026 13:50:13 +0100 Subject: [PATCH 21/28] Fix after merge, run jit-format --- src/coreclr/jit/async.cpp | 12 +++--------- src/coreclr/jit/async.h | 7 ++----- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 4180d9008198bf..cbab306b3b31fc 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -1872,7 +1872,7 @@ void AsyncTransformation::Transform( unsigned stateNum = (unsigned)m_states.size(); JITDUMP(" Assigned state %u\n", stateNum); - BasicBlock* suspendBB = CreateSuspensionBlock(block, call, stateNum); + BasicBlock* suspendBB = CreateSuspensionBlock(block, stateNum); CreateCheckAndSuspendAfterCall(block, call, callDefInfo, suspendBB, remainder); @@ -2576,13 +2576,12 @@ CallDefinitionInfo AsyncTransformation::CanonicalizeCallDefinition(BasicBlock* // // Parameters: // block - The block containing the async call. -// call - The async call. // stateNum - State number assigned to this suspension point. // // Returns: // The new basic block. // -BasicBlock* AsyncTransformation::CreateSuspensionBlock(BasicBlock* block, GenTreeCall* call, unsigned stateNum) +BasicBlock* AsyncTransformation::CreateSuspensionBlock(BasicBlock* block, unsigned stateNum) { if (m_lastSuspensionBB == nullptr) { @@ -3156,17 +3155,12 @@ void AsyncTransformation::CreateCheckAndSuspendAfterCall(BasicBlock* // // Parameters: // remainder - The block that contains the IR after the async call. -// call - The async call. // stateNum - State number assigned to this suspension point. -// layoutBuilder - The continuation layout builder for this call. // // Returns: // The new basic block. // -BasicBlock* AsyncTransformation::CreateResumptionBlock(BasicBlock* remainder, - GenTreeCall* call, - unsigned stateNum, - ContinuationLayoutBuilder* layoutBuilder) +BasicBlock* AsyncTransformation::CreateResumptionBlock(BasicBlock* remainder, unsigned stateNum) { if (m_lastResumptionBB == nullptr) { diff --git a/src/coreclr/jit/async.h b/src/coreclr/jit/async.h index 9a5531dbf6318b..b0651e3c3e17c6 100644 --- a/src/coreclr/jit/async.h +++ b/src/coreclr/jit/async.h @@ -250,7 +250,7 @@ class AsyncTransformation CallDefinitionInfo CanonicalizeCallDefinition(BasicBlock* block, GenTreeCall* call, AsyncLiveness* life); - BasicBlock* CreateSuspensionBlock(BasicBlock* block, GenTreeCall* call, unsigned stateNum); + BasicBlock* CreateSuspensionBlock(BasicBlock* block, unsigned stateNum); void CreateSuspension(BasicBlock* callBlock, GenTreeCall* call, BasicBlock* suspendBB, @@ -277,10 +277,7 @@ class AsyncTransformation const CallDefinitionInfo& callDefInfo, BasicBlock* suspendBB, BasicBlock** remainder); - BasicBlock* CreateResumptionBlock(BasicBlock* remainder, - GenTreeCall* call, - unsigned stateNum, - ContinuationLayoutBuilder* layoutBuilder); + BasicBlock* CreateResumptionBlock(BasicBlock* remainder, unsigned stateNum); void CreateResumption(BasicBlock* callBlock, GenTreeCall* call, BasicBlock* resumeBB, From a2c3f4e709249d2f8e011c387d4d9a86bd8f4aaa Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 19 Mar 2026 14:11:29 +0100 Subject: [PATCH 22/28] Fix --- src/coreclr/jit/async.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index cbab306b3b31fc..de3752de270652 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -1062,9 +1062,8 @@ bool PreservedValueAnalysis::IsResumeReachable(BasicBlock* block) //------------------------------------------------------------------------ // PreservedValueAnalysis::ComputeResumeReachableBlocks: -// Step 0: Identify blocks containing awaits, then compute the set of -// blocks reachable after any resumption via a forward fixpoint from -// the successors of await blocks. +// Phase 0: Identify blocks containing awaits, then compute the set of +// blocks reachable after any resumption via a DFS starting from await blocks. // void PreservedValueAnalysis::ComputeResumeReachableBlocks(ArrayStack& awaitBlocks) { @@ -1082,6 +1081,7 @@ void PreservedValueAnalysis::ComputeResumeReachableBlocks(ArrayStackverbose, DumpAwaitBlocks()); + // DFS from those blocks. while (!worklist.Empty()) { BasicBlock* block = worklist.Pop(); @@ -1166,8 +1166,7 @@ void PreservedValueAnalysis::ComputePerBlockMutatedVars() // Transfer function: mutatedOut[B] = mutatedIn[B] | mutated[B] // Merge: mutatedIn[B] = union of mutatedOut[pred] for all preds // -// At method entry all locals are considered mutated (no previous resumption -// to preserve from). +// At method entry no locals are considered mutated (not reachable from a resumption). // void PreservedValueAnalysis::ComputeInterBlockMutatedVars() { @@ -1876,7 +1875,7 @@ void AsyncTransformation::Transform( CreateCheckAndSuspendAfterCall(block, call, callDefInfo, suspendBB, remainder); - BasicBlock* resumeBB = CreateResumptionBlock(*remainder, call, stateNum, layoutBuilder); + BasicBlock* resumeBB = CreateResumptionBlock(*remainder, stateNum); m_states.push_back(AsyncState(stateNum, layoutBuilder, block, call, callDefInfo, suspendBB, resumeBB, resumeReachable, mutatedSinceResumption)); From b4dd46bc7cc987fa92cfd9bdfabdfb0a9c7c05ba Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 20 Mar 2026 11:55:23 +0100 Subject: [PATCH 23/28] Refactor, fix bug not considering call defs in async calls as mutated --- src/coreclr/jit/async.cpp | 162 +++++++++++++++++++------------------- 1 file changed, 80 insertions(+), 82 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index de3752de270652..1fc9daa7eb3bfa 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -658,6 +658,36 @@ static bool IsDefaultValue(GenTree* node) return node->IsIntegralConst(0) || node->IsFloatPositiveZero() || node->IsVectorZero(); } +//------------------------------------------------------------------------ +// MarkMutatedVarDsc: +// Mark a VarDsc (or its promoted fields) in the specified varset. +// +// Parameters: +// compiler - The compiler instance. +// varDsc - The var. +// mutated - [in/out] The set to update. +// +static void MarkMutatedVarDsc(Compiler* compiler, LclVarDsc* varDsc, VARSET_TP& mutated) +{ + if (varDsc->lvTracked) + { + VarSetOps::AddElemD(compiler, mutated, varDsc->lvVarIndex); + return; + } + + if (varDsc->lvPromoted) + { + for (unsigned i = 0; i < varDsc->lvFieldCnt; i++) + { + LclVarDsc* fieldDsc = compiler->lvaGetDesc(varDsc->lvFieldLclStart + i); + if (fieldDsc->lvTracked) + { + VarSetOps::AddElemD(compiler, mutated, fieldDsc->lvVarIndex); + } + } + } +} + //------------------------------------------------------------------------ // UpdateMutatedLocal: // If the given node is a local store or LCL_ADDR, and the local is tracked, @@ -696,28 +726,32 @@ static void UpdateMutatedLocal(Compiler* compiler, GenTree* node, VARSET_TP& mut } LclVarDsc* varDsc = compiler->lvaGetDesc(node->AsLclVarCommon()); + MarkMutatedVarDsc(compiler, varDsc, mutated); +} - if (varDsc->lvTracked) - { - VarSetOps::AddElemD(compiler, mutated, varDsc->lvVarIndex); - return; - } - - // For promoted structs the parent may not be tracked but the field locals - // are. When the parent is mutated, all tracked fields must be marked as - // mutated as well. - if (varDsc->lvPromoted) +#ifdef DEBUG +//------------------------------------------------------------------------ +// PrintVarSet: +// Print a varset as a space separate list of locals. +// +// Parameters: +// comp - Compiler instance +// set - The varset to print. +// +static void PrintVarSet(Compiler* comp, VARSET_VALARG_TP set) +{ + VarSetOps::Iter iter(comp, set); + unsigned varIndex = 0; + const char* sep = ""; + while (iter.NextElem(&varIndex)) { - for (unsigned i = 0; i < varDsc->lvFieldCnt; i++) - { - LclVarDsc* fieldDsc = compiler->lvaGetDesc(varDsc->lvFieldLclStart + i); - if (fieldDsc->lvTracked) - { - VarSetOps::AddElemD(compiler, mutated, fieldDsc->lvVarIndex); - } - } + unsigned lclNum = comp->lvaTrackedToVarNum[varIndex]; + printf("%sV%02u", sep, lclNum); + sep = " "; } + printf("\n"); } +#endif //------------------------------------------------------------------------ // DefaultValueAnalysis::ComputePerBlockMutatedVars: @@ -807,16 +841,7 @@ void DefaultValueAnalysis::DumpMutatedVars() if (!VarSetOps::IsEmpty(m_compiler, m_mutatedVars[block->bbNum])) { printf(" " FMT_BB " mutated: ", block->bbNum); - VarSetOps::Iter iter(m_compiler, m_mutatedVars[block->bbNum]); - unsigned varIndex = 0; - const char* sep = ""; - while (iter.NextElem(&varIndex)) - { - unsigned lclNum = m_compiler->lvaTrackedToVarNum[varIndex]; - printf("%sV%02u", sep, lclNum); - sep = " "; - } - printf("\n"); + PrintVarSet(m_compiler, m_mutatedVars[block->bbNum]); } } } @@ -835,21 +860,12 @@ void DefaultValueAnalysis::DumpMutatedVarsIn() if (VarSetOps::IsEmpty(m_compiler, m_mutatedVarsIn[block->bbNum])) { - printf(""); + printf("\n"); } else { - VarSetOps::Iter iter(m_compiler, m_mutatedVarsIn[block->bbNum]); - unsigned varIndex = 0; - const char* sep = ""; - while (iter.NextElem(&varIndex)) - { - unsigned lclNum = m_compiler->lvaTrackedToVarNum[varIndex]; - printf("%sV%02u", sep, lclNum); - sep = " "; - } + PrintVarSet(m_compiler, m_mutatedVarsIn[block->bbNum]); } - printf("\n"); } } #endif @@ -867,29 +883,21 @@ void DefaultValueAnalysis::DumpMutatedVarsIn() // static void MarkMutatedLocal(Compiler* compiler, GenTree* node, VARSET_TP& mutated) { - if (!node->OperIsLocalStore() && !node->OperIs(GT_LCL_ADDR)) + if (node->IsCall()) { - return; + auto visitDef = [&](GenTreeLclVarCommon* lcl) { + MarkMutatedVarDsc(compiler, compiler->lvaGetDesc(lcl), mutated); + return GenTree::VisitResult::Continue; + }; + node->VisitLocalDefNodes(compiler, visitDef); } - - LclVarDsc* varDsc = compiler->lvaGetDesc(node->AsLclVarCommon()); - - if (varDsc->lvTracked) + else if (node->OperIsLocalStore() || node->OperIs(GT_LCL_ADDR)) { - VarSetOps::AddElemD(compiler, mutated, varDsc->lvVarIndex); - return; + MarkMutatedVarDsc(compiler, compiler->lvaGetDesc(node->AsLclVarCommon()), mutated); } - - if (varDsc->lvPromoted) + else { - for (unsigned i = 0; i < varDsc->lvFieldCnt; i++) - { - LclVarDsc* fieldDsc = compiler->lvaGetDesc(varDsc->lvFieldLclStart + i); - if (fieldDsc->lvTracked) - { - VarSetOps::AddElemD(compiler, mutated, fieldDsc->lvVarIndex); - } - } + return; } } @@ -1143,8 +1151,6 @@ void PreservedValueAnalysis::ComputePerBlockMutatedVars() node = node->gtNext; assert(node != nullptr); } - - node = node->gtNext; } while (node != nullptr) @@ -1237,16 +1243,7 @@ void PreservedValueAnalysis::DumpMutatedVars() if (!VarSetOps::IsEmpty(m_compiler, m_mutatedVars[block->bbNum])) { printf(" " FMT_BB " mutated: ", block->bbNum); - VarSetOps::Iter iter(m_compiler, m_mutatedVars[block->bbNum]); - unsigned varIndex = 0; - const char* sep = ""; - while (iter.NextElem(&varIndex)) - { - unsigned lclNum = m_compiler->lvaTrackedToVarNum[varIndex]; - printf("%sV%02u", sep, lclNum); - sep = " "; - } - printf("\n"); + PrintVarSet(m_compiler, m_mutatedVars[block->bbNum]); } } } @@ -1265,25 +1262,16 @@ void PreservedValueAnalysis::DumpMutatedVarsIn() if (VarSetOps::IsEmpty(m_compiler, m_mutatedVarsIn[block->bbNum])) { - printf(""); + printf("\n"); } else if (VarSetOps::Equal(m_compiler, m_mutatedVarsIn[block->bbNum], VarSetOps::MakeFull(m_compiler))) { - printf(""); + printf("\n"); } else { - VarSetOps::Iter iter(m_compiler, m_mutatedVarsIn[block->bbNum]); - unsigned varIndex = 0; - const char* sep = ""; - while (iter.NextElem(&varIndex)) - { - unsigned lclNum = m_compiler->lvaTrackedToVarNum[varIndex]; - printf("%sV%02u", sep, lclNum); - sep = " "; - } + PrintVarSet(m_compiler, m_mutatedVarsIn[block->bbNum]); } - printf("\n"); } } #endif @@ -1360,11 +1348,14 @@ void AsyncLiveness::Update(GenTree* node) { m_updater.UpdateLife(node); UpdateMutatedLocal(m_compiler, node, m_mutatedValues); + + // If this is an async call then we can reach defs after resumption now. + // Make sure defs happening as part of the call are included as mutated since resumption. + m_resumeReachable |= node->IsCall() && node->AsCall()->IsAsync(); if (m_resumeReachable) { MarkMutatedLocal(m_compiler, node, m_mutatedSinceResumption); } - m_resumeReachable |= node->IsCall() && node->AsCall()->IsAsync(); } //------------------------------------------------------------------------ @@ -1860,6 +1851,13 @@ void AsyncTransformation::Transform( bool resumeReachable = life.IsResumeReachable(); VARSET_TP mutatedSinceResumption(VarSetOps::MakeCopy(m_compiler, life.GetMutatedSinceResumption())); + JITDUMP(" This suspension point is%s resume-reachable\n", resumeReachable ? "" : " NOT"); + if (resumeReachable) + { + JITDUMP(" Locals mutated since previous resumption: "); + DBEXEC(VERBOSE, PrintVarSet(m_compiler, mutatedSinceResumption)); + } + ContinuationLayoutBuilder* layoutBuilder = new (m_compiler, CMK_Async) ContinuationLayoutBuilder(m_compiler); CreateLiveSetForSuspension(block, call, defs, life, layoutBuilder); From 5ba2cbf12d20f920c56bbac2a110ffbd75afc085 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 20 Mar 2026 20:51:20 +0100 Subject: [PATCH 24/28] Simplify --- src/coreclr/jit/async.cpp | 27 ++++++++++++++++++--------- src/coreclr/jit/async.h | 2 +- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 1fc9daa7eb3bfa..bd57d61b3df08f 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -2853,8 +2853,7 @@ void AsyncTransformation::FillInDataOnSuspension(GenTreeCall* LclVarDsc* dsc = m_compiler->lvaGetDesc(inf.LclNum); - if ((saveSet != SaveSet::All) && - ((saveSet == SaveSet::UnmutatedLocals) != IsLocalUnmutatedSinceLastResumption(dsc, mutatedSinceResumption))) + if ((saveSet != SaveSet::All) && (GetLocalSaveSet(dsc, mutatedSinceResumption) != saveSet)) { continue; } @@ -2963,8 +2962,18 @@ void AsyncTransformation::FillInDataOnSuspension(GenTreeCall* } } -bool AsyncTransformation::IsLocalUnmutatedSinceLastResumption(const LclVarDsc* dsc, - VARSET_VALARG_TP mutatedSinceResumption) +//------------------------------------------------------------------------ +// AsyncTransformation::GetLocalSaveSet: +// Get the save set that a local should be saved as part of. +// +// Parameters: +// dsc - The local +// mutatedSinceResumption - Set of locals that may have been mutated since a resumption +// +// Returns: +// The set to save the local as part of. +// +SaveSet AsyncTransformation::GetLocalSaveSet(const LclVarDsc* dsc, VARSET_VALARG_TP mutatedSinceResumption) { if (dsc->lvPromoted) { @@ -2973,23 +2982,23 @@ bool AsyncTransformation::IsLocalUnmutatedSinceLastResumption(const LclVarDsc* d LclVarDsc* fieldDsc = m_compiler->lvaGetDesc(dsc->lvFieldLclStart + i); if (!fieldDsc->lvTracked || VarSetOps::IsMember(m_compiler, mutatedSinceResumption, fieldDsc->lvVarIndex)) { - return false; + return SaveSet::MutatedLocals; } } - return true; + return SaveSet::UnmutatedLocals; } // We should only see struct fields for independently promoted structs assert(!dsc->lvIsStructField || (m_compiler->lvaGetPromotionType(dsc->lvParentLcl) == Compiler::PROMOTION_TYPE_INDEPENDENT)); - if (!dsc->lvTracked) + if (!dsc->lvTracked || VarSetOps::IsMember(m_compiler, mutatedSinceResumption, dsc->lvVarIndex)) { - return false; + return SaveSet::MutatedLocals; } - return !VarSetOps::IsMember(m_compiler, mutatedSinceResumption, dsc->lvVarIndex); + return SaveSet::UnmutatedLocals; } //------------------------------------------------------------------------ diff --git a/src/coreclr/jit/async.h b/src/coreclr/jit/async.h index b0651e3c3e17c6..15f84777b6c0e6 100644 --- a/src/coreclr/jit/async.h +++ b/src/coreclr/jit/async.h @@ -270,7 +270,7 @@ class AsyncTransformation BasicBlock* suspendBB, VARSET_VALARG_TP mutatedSinceResumption, SaveSet saveSet); - bool IsLocalUnmutatedSinceLastResumption(const LclVarDsc* dsc, VARSET_VALARG_TP mutatedSinceResumption); + SaveSet GetLocalSaveSet(const LclVarDsc* dsc, VARSET_VALARG_TP mutatedSinceResumption); void RestoreContexts(BasicBlock* block, GenTreeCall* call, BasicBlock* insertionBB); void CreateCheckAndSuspendAfterCall(BasicBlock* block, GenTreeCall* call, From a8cb378d524bdd861201607a68fd5db32edff696 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 20 Mar 2026 23:45:57 +0100 Subject: [PATCH 25/28] Apply suggestion from @jakobbotsch --- src/coreclr/jit/async.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index bd57d61b3df08f..bc365dc195ec58 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -1855,7 +1855,7 @@ void AsyncTransformation::Transform( if (resumeReachable) { JITDUMP(" Locals mutated since previous resumption: "); - DBEXEC(VERBOSE, PrintVarSet(m_compiler, mutatedSinceResumption)); + JITDUMPEXEC(PrintVarSet(m_compiler, mutatedSinceResumption)); } ContinuationLayoutBuilder* layoutBuilder = new (m_compiler, CMK_Async) ContinuationLayoutBuilder(m_compiler); From 4bc6dcff0ce5484cac526b26acb7c3be8a169ec8 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 20 Mar 2026 23:47:41 +0100 Subject: [PATCH 26/28] Use JITDUMPEXEC --- src/coreclr/jit/async.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index bc365dc195ec58..39891fa2f3168b 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -785,7 +785,7 @@ void DefaultValueAnalysis::ComputePerBlockMutatedVars() } JITDUMP("Default value analysis: per-block mutated vars\n"); - DBEXEC(m_compiler->verbose, DumpMutatedVars()); + JITDUMPEXEC(DumpMutatedVars()); } //------------------------------------------------------------------------ @@ -824,7 +824,7 @@ void DefaultValueAnalysis::ComputeInterBlockDefaultValues() flow.ForwardAnalysis(callback); JITDUMP("Default value analysis: per-block mutated vars on entry\n"); - DBEXEC(m_compiler->verbose, DumpMutatedVarsIn()); + JITDUMPEXEC(DumpMutatedVarsIn()); } #ifdef DEBUG @@ -1087,7 +1087,7 @@ void PreservedValueAnalysis::ComputeResumeReachableBlocks(ArrayStackverbose, DumpAwaitBlocks()); + JITDUMPEXEC(DumpAwaitBlocks()); // DFS from those blocks. while (!worklist.Empty()) @@ -1104,7 +1104,7 @@ void PreservedValueAnalysis::ComputeResumeReachableBlocks(ArrayStackverbose, DumpResumeReachableBlocks()); + JITDUMPEXEC(DumpResumeReachableBlocks()); } //------------------------------------------------------------------------ @@ -1161,7 +1161,7 @@ void PreservedValueAnalysis::ComputePerBlockMutatedVars() } JITDUMP("Preserved value analysis: per-block mutated vars after resumption\n"); - DBEXEC(m_compiler->verbose, DumpMutatedVars()); + JITDUMPEXEC(DumpMutatedVars()); } //------------------------------------------------------------------------ @@ -1188,7 +1188,7 @@ void PreservedValueAnalysis::ComputeInterBlockMutatedVars() flow.ForwardAnalysis(callback); JITDUMP("Preserved value analysis: per-block mutated vars on entry\n"); - DBEXEC(m_compiler->verbose, DumpMutatedVarsIn()); + JITDUMPEXEC(DumpMutatedVarsIn()); } #ifdef DEBUG From 777895d81b4c5566babfb6f35e66d92c2cca1f4a Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 20 Mar 2026 23:59:00 +0100 Subject: [PATCH 27/28] Fix function headers --- src/coreclr/jit/async.cpp | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 39891fa2f3168b..062cd7a2e408bb 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -1020,6 +1020,9 @@ class PreservedValueAnalysis // resume-reachable blocks, then compute per-block and inter-block // mutation sets relative to resumption points. // +// Parameters: +// awaitblocks - Blocks containing async calls +// void PreservedValueAnalysis::Run(ArrayStack& awaitBlocks) { #ifdef DEBUG @@ -1053,9 +1056,9 @@ void PreservedValueAnalysis::Run(ArrayStack& awaitBlocks) // block - The basic block. // // Returns: -// The VARSET_TP of tracked locals mutated since last resumption. A local -// NOT in this set has a preserved value in the continuation and does not -// need to be re-stored when reusing it. +// The VARSET_TP of tracked locals that may have been mutated since last +// resumption. A tracked local NOT in this set has a preserved value in the +// continuation and does not need to be re-stored when reusing it. // const VARSET_TP& PreservedValueAnalysis::GetMutatedVarsIn(BasicBlock* block) const { @@ -1063,6 +1066,19 @@ const VARSET_TP& PreservedValueAnalysis::GetMutatedVarsIn(BasicBlock* block) con return m_mutatedVarsIn[block->bbNum]; } +//------------------------------------------------------------------------ +// PreservedValueAnalysis::IsResumeReachable: +// Check if the specified basic block is reachable after a previous resumption. +// +// Parameters: +// block - The basic block. +// +// Returns: +// True if so. Blocks that are not resume-reachable will never be able to +// reuse a continuation. Also, mutations of locals that are not +// resume-reachable do not need to be considered for preserved value +// analysis. +// bool PreservedValueAnalysis::IsResumeReachable(BasicBlock* block) { return BitVecOps::IsMember(&m_blockTraits, m_resumeReachableBlocks, block->bbNum); @@ -1323,7 +1339,7 @@ class AsyncLiveness //------------------------------------------------------------------------ // AsyncLiveness::StartBlock: // Indicate that we are now starting a new block, and do relevant liveness -// updates for it. +// and other analysis updates for it. // // Parameters: // block - The block that we are starting. From d0c1d4b26971564131d89490c8d2717ef86b1d2d Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sat, 21 Mar 2026 00:21:28 +0100 Subject: [PATCH 28/28] Feedback --- src/coreclr/jit/async.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 062cd7a2e408bb..f0fcce4dace184 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -732,7 +732,7 @@ static void UpdateMutatedLocal(Compiler* compiler, GenTree* node, VARSET_TP& mut #ifdef DEBUG //------------------------------------------------------------------------ // PrintVarSet: -// Print a varset as a space separate list of locals. +// Print a varset as a space-separated list of locals. // // Parameters: // comp - Compiler instance @@ -1033,11 +1033,16 @@ void PreservedValueAnalysis::Run(ArrayStack& awaitBlocks) { JITDUMP("Preserved value analysis disabled because of method range\n"); m_mutatedVarsIn = m_compiler->fgAllocateTypeForEachBlk(CMK_Async); + m_mutatedVars = m_compiler->fgAllocateTypeForEachBlk(CMK_Async); for (BasicBlock* block : m_compiler->Blocks()) { VarSetOps::AssignNoCopy(m_compiler, m_mutatedVarsIn[block->bbNum], VarSetOps::MakeFull(m_compiler)); + VarSetOps::AssignNoCopy(m_compiler, m_mutatedVars[block->bbNum], VarSetOps::MakeFull(m_compiler)); } + m_resumeReachableBlocks = BitVecOps::MakeFull(&m_blockTraits); + m_awaitBlocks = BitVecOps::MakeFull(&m_blockTraits); + return; } #endif