From 2b114f6f654ca949ae4bc8d0bd47c18ad9e1cb32 Mon Sep 17 00:00:00 2001 From: Andy Ayers Date: Thu, 23 Apr 2026 15:36:25 -0700 Subject: [PATCH 1/4] JIT: Fix profile inconsistency asserts in flow graph optimization Fix three sources of profile weight inconsistency in fgopt.cpp that cause assert failures under PGO stress modes (JitRandomEdgeCounts, JitRandomGuardedDevirtualization): 1. fgCompactBlock: When compacting a block with a target that had other predecessors removed by earlier transforms, the inherited weight may not match incoming edge weights. Detect this and mark fgPgoConsistent false. 2. fgOptimizeBranchToEmptyUnconditional: When fgRedirectEdge merges two edges of a BBJ_COND (TrueEdge == FalseEdge), the merged edge retains the old likelihood instead of 1.0. Fix by setting likelihood to 1.0 after merge. 3. fgOptimizeBranchToEmptyUnconditional: When decreaseBBProfileWeight would clamp a block's weight to zero, mark fgPgoConsistent false. Also fix three pre-existing misleading JITDUMP messages that said 'Data %s consistent' when marking data as inconsistent. Fixes dotnet/runtime#126381. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/fgopt.cpp | 55 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/fgopt.cpp b/src/coreclr/jit/fgopt.cpp index fa79f661f4e2f9..afb5b1d5a4f5d0 100644 --- a/src/coreclr/jit/fgopt.cpp +++ b/src/coreclr/jit/fgopt.cpp @@ -881,7 +881,9 @@ void Compiler::fgCompactBlock(BasicBlock* block) JITDUMP("\nCompacting " FMT_BB " into " FMT_BB ":\n", target->bbNum, block->bbNum); fgRemoveRefPred(block->GetTargetEdge()); - if (target->countOfInEdges() > 0) + const bool targetHadOtherPreds = (target->countOfInEdges() > 0); + + if (targetHadOtherPreds) { JITDUMP("Second block has %u other incoming edges\n", target->countOfInEdges()); assert(block->isEmpty()); @@ -1028,6 +1030,32 @@ void Compiler::fgCompactBlock(BasicBlock* block) block->SetFlags(BBF_PROF_WEIGHT); } + // When the target had other incoming edges that were retargeted to block, + // the inherited weight may not precisely match block's actual incoming flow + // due to accumulated rounding from prior weight adjustments (e.g., + // decreaseBBProfileWeight called during earlier optimizations in the same + // iteration of fgUpdateFlowGraph). Mark profile as potentially inconsistent. + if (targetHadOtherPreds && hasProfileWeight && fgPgoConsistent) + { + weight_t incomingLikelyWeight = 0; + for (FlowEdge* const predEdge : block->PredEdges()) + { + if (predEdge->hasLikelihood()) + { + incomingLikelyWeight += predEdge->getLikelyWeight(); + } + } + + if (!fgProfileWeightsConsistentOrSmall(block->bbWeight, incomingLikelyWeight)) + { + JITDUMP("fgCompactBlock: " FMT_BB " weight " FMT_WT " inconsistent with incoming " FMT_WT + " after compaction. Data %s inconsistent.\n", + block->bbNum, block->bbWeight, incomingLikelyWeight, + fgPgoConsistent ? "is now" : "was already"); + fgPgoConsistent = false; + } + } + VarSetOps::AssignAllowUninitRhs(this, block->bbLiveOut, target->bbLiveOut); // Update the beginning and ending IL offsets (bbCodeOffs and bbCodeOffsEnd). @@ -1331,12 +1359,28 @@ bool Compiler::fgOptimizeBranchToEmptyUnconditional(BasicBlock* block, BasicBloc assert(!block->FalseTargetIs(bDest)); removedWeight = block->GetTrueEdge()->getLikelyWeight(); fgRedirectEdge(block->TrueEdgeRef(), bDest->GetTarget()); + + // If the redirect caused an edge merge (both edges now point + // to the same target), fix the likelihood. fgRedirectEdge + // keeps the existing edge's likelihood and drops the redirected + // edge's. Since this is a BBJ_COND whose outgoing likelihoods + // must sum to 1.0, and both edges now go to the same block, + // the merged edge must carry likelihood 1.0. + if (block->GetTrueEdge() == block->GetFalseEdge()) + { + block->GetTrueEdge()->setLikelihood(1.0); + } } else { assert(block->FalseTargetIs(bDest)); removedWeight = block->GetFalseEdge()->getLikelyWeight(); fgRedirectEdge(block->FalseEdgeRef(), bDest->GetTarget()); + + if (block->GetTrueEdge() == block->GetFalseEdge()) + { + block->GetFalseEdge()->setLikelihood(1.0); + } } break; @@ -1350,6 +1394,11 @@ bool Compiler::fgOptimizeBranchToEmptyUnconditional(BasicBlock* block, BasicBloc // if (bDest->hasProfileWeight()) { + if (fgPgoConsistent && (bDest->bbWeight < removedWeight)) + { + JITDUMP("Clamping " FMT_BB " weight in fgOptimizeBranchToEmptyUnconditional\n", bDest->bbNum); + fgPgoConsistent = false; + } bDest->decreaseBBProfileWeight(removedWeight); } @@ -1616,7 +1665,7 @@ bool Compiler::fgOptimizeSwitchBranches(BasicBlock* block) if (modified) { JITDUMP( - "fgOptimizeSwitchBranches: Optimized switch flow. Profile needs to be re-propagated. Data %s consistent.\n", + "fgOptimizeSwitchBranches: Optimized switch flow. Profile needs to be re-propagated. Data %s inconsistent.\n", fgPgoConsistent ? "is now" : "was already"); fgPgoConsistent = false; } @@ -4780,7 +4829,7 @@ bool Compiler::fgUpdateFlowGraph(bool doTailDuplication /* = false */, bool isPh // Mark the profile as inconsistent if we might have propagated the OSR entry weight. if (modified && opts.IsOSR()) { - JITDUMP("fgUpdateFlowGraph: Inconsistent OSR entry weight may have been propagated. Data %s consistent.\n", + JITDUMP("fgUpdateFlowGraph: Inconsistent OSR entry weight may have been propagated. Data %s inconsistent.\n", fgPgoConsistent ? "is now" : "was already"); fgPgoConsistent = false; } From d450544f7a6e9279335d55ecff0a764c5c182689 Mon Sep 17 00:00:00 2001 From: Andy Ayers Date: Thu, 23 Apr 2026 15:56:57 -0700 Subject: [PATCH 2/4] Fix formatting Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/fgopt.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/coreclr/jit/fgopt.cpp b/src/coreclr/jit/fgopt.cpp index afb5b1d5a4f5d0..0288550ecbd99b 100644 --- a/src/coreclr/jit/fgopt.cpp +++ b/src/coreclr/jit/fgopt.cpp @@ -1050,8 +1050,7 @@ void Compiler::fgCompactBlock(BasicBlock* block) { JITDUMP("fgCompactBlock: " FMT_BB " weight " FMT_WT " inconsistent with incoming " FMT_WT " after compaction. Data %s inconsistent.\n", - block->bbNum, block->bbWeight, incomingLikelyWeight, - fgPgoConsistent ? "is now" : "was already"); + block->bbNum, block->bbWeight, incomingLikelyWeight, fgPgoConsistent ? "is now" : "was already"); fgPgoConsistent = false; } } From 1e010dc164e3eba7bec16bb736a1602771eb4b42 Mon Sep 17 00:00:00 2001 From: Andy Ayers Date: Thu, 23 Apr 2026 16:05:57 -0700 Subject: [PATCH 3/4] Address review: gate consistency check on post-inheritance profile state Use block->hasProfileWeight() (post-inheritWeight) instead of the cached pre-inheritance value, so the check also fires when an unprofiled block inherits profile weight from a profiled target. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/fgopt.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/fgopt.cpp b/src/coreclr/jit/fgopt.cpp index 0288550ecbd99b..1dba199a100e24 100644 --- a/src/coreclr/jit/fgopt.cpp +++ b/src/coreclr/jit/fgopt.cpp @@ -1035,7 +1035,7 @@ void Compiler::fgCompactBlock(BasicBlock* block) // due to accumulated rounding from prior weight adjustments (e.g., // decreaseBBProfileWeight called during earlier optimizations in the same // iteration of fgUpdateFlowGraph). Mark profile as potentially inconsistent. - if (targetHadOtherPreds && hasProfileWeight && fgPgoConsistent) + if (targetHadOtherPreds && block->hasProfileWeight() && fgPgoConsistent) { weight_t incomingLikelyWeight = 0; for (FlowEdge* const predEdge : block->PredEdges()) From 7937104d5b62a50bcc54b0a872d45da907ec7855 Mon Sep 17 00:00:00 2001 From: Andy Ayers Date: Thu, 23 Apr 2026 16:27:11 -0700 Subject: [PATCH 4/4] Address review: remove DEBUG-only hasLikelihood() guard hasLikelihood() is #ifdef DEBUG only and would break Release builds. Remove the guard and sum getLikelyWeight() unconditionally, since all edges have likelihoods set by this point in compilation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/fgopt.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/coreclr/jit/fgopt.cpp b/src/coreclr/jit/fgopt.cpp index 1dba199a100e24..5e1756aae995e6 100644 --- a/src/coreclr/jit/fgopt.cpp +++ b/src/coreclr/jit/fgopt.cpp @@ -1040,10 +1040,7 @@ void Compiler::fgCompactBlock(BasicBlock* block) weight_t incomingLikelyWeight = 0; for (FlowEdge* const predEdge : block->PredEdges()) { - if (predEdge->hasLikelihood()) - { - incomingLikelyWeight += predEdge->getLikelyWeight(); - } + incomingLikelyWeight += predEdge->getLikelyWeight(); } if (!fgProfileWeightsConsistentOrSmall(block->bbWeight, incomingLikelyWeight))