Skip to content

JIT: Generalize pattern-based IV analysis #119537

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jakobbotsch merged 2 commits into from
Sep 12, 2025
Merged

JIT: Generalize pattern-based IV analysis #119537

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e5df810
JIT: Generalize pattern-based IV analysis
jakobbotsch Sep 10, 2025
aba3333
Merge branch 'main' of github.com:dotnet/runtime into improve-pattern…
jakobbotsch Sep 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions src/coreclr/jit/compiler.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1996,11 +1996,12 @@ class FlowGraphNaturalLoop

GenTreeLclVarCommon* FindDef(unsigned lclNum);

void MatchInit(NaturalLoopIterInfo* info, BasicBlock* initBlock, GenTree* init);
bool MatchLimit(unsigned iterVar, GenTree* test, NaturalLoopIterInfo* info);
bool FindConstInit(BasicBlock* preheader, NaturalLoopIterInfo* info);
bool CheckLoopConditionBaseCase(BasicBlock* initBlock, NaturalLoopIterInfo* info);
bool IsZeroTripTest(BasicBlock* initBlock, NaturalLoopIterInfo* info);
bool InitBlockEntersLoopOnTrue(BasicBlock* initBlock);
bool HasZeroTripTest(BasicBlock* preheader, NaturalLoopIterInfo* info);
bool IsZeroTripTest(BasicBlock* guardBlock, bool entersOnTrue, NaturalLoopIterInfo* info);

template<typename T>
static bool EvaluateRelop(T op1, T op2, genTreeOps oper);
public:
Expand Down Expand Up @@ -3645,6 +3646,7 @@ class Compiler
unsigned gtSetEvalOrderMinOpts(GenTree* tree);
bool gtMayHaveStoreInterference(GenTree* treeWithStores, GenTree* tree);
bool gtTreeHasLocalRead(GenTree* tree, unsigned lclNum);
bool gtTreeHasLocalStore(GenTree* tree, unsigned lclNum);

void gtSetStmtInfo(Statement* stmt);

Expand Down Expand Up @@ -7063,8 +7065,8 @@ class Compiler

bool optIsLoopTestEvalIntoTemp(Statement* testStmt, Statement** newTestStmt);
unsigned optIsLoopIncrTree(GenTree* incr);
bool optExtractInitTestIncr(
BasicBlock** pInitBlock, BasicBlock* bottom, BasicBlock* top, GenTree** ppInit, GenTree** ppTest, GenTree** ppIncr);
bool optExtractTestIncr(
BasicBlock* bottom, BasicBlock* top, GenTree** ppTest, GenTree** ppIncr);

void optSetMappedBlockTargets(BasicBlock* blk,
BasicBlock* newBlk,
Expand Down
220 changes: 112 additions & 108 deletions src/coreclr/jit/flowgraph.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5504,9 +5504,7 @@ bool FlowGraphNaturalLoop::AnalyzeIteration(NaturalLoopIterInfo* info)

JITDUMP(" Preheader = " FMT_BB "\n", preheader->bbNum);

BasicBlock* initBlock = nullptr;
GenTree* init = nullptr;
GenTree* test = nullptr;
GenTree* test = nullptr;

info->IterVar = BAD_VAR_NUM;

Expand All @@ -5521,8 +5519,7 @@ bool FlowGraphNaturalLoop::AnalyzeIteration(NaturalLoopIterInfo* info)
}

GenTree* iterTree = nullptr;
initBlock = preheader;
if (!comp->optExtractInitTestIncr(&initBlock, cond, m_header, &init, &test, &iterTree))
if (!comp->optExtractTestIncr(cond, m_header, &test, &iterTree))
{
JITDUMP(" Could not extract an IV\n");
continue;
Expand Down Expand Up @@ -5580,19 +5577,17 @@ bool FlowGraphNaturalLoop::AnalyzeIteration(NaturalLoopIterInfo* info)
return false;
}

if (init == nullptr)
if (FindConstInit(preheader, info))
{
JITDUMP(" Init = <none>, test = [%06u], incr = [%06u]\n", Compiler::dspTreeID(test),
Compiler::dspTreeID(info->IterTree));
JITDUMP(" Init = [%06u], test = [%06u], incr = [%06u]\n", Compiler::dspTreeID(info->InitTree),
Compiler::dspTreeID(test), Compiler::dspTreeID(info->IterTree));
}
else
{
JITDUMP(" Init = [%06u], test = [%06u], incr = [%06u]\n", Compiler::dspTreeID(init), Compiler::dspTreeID(test),
JITDUMP(" Init = <none>, test = [%06u], incr = [%06u]\n", Compiler::dspTreeID(test),
Compiler::dspTreeID(info->IterTree));
}

MatchInit(info, initBlock, init);

bool result = VisitDefs([=](GenTreeLclVarCommon* def) {
if ((def->GetLclNum() != info->IterVar) || (def == info->IterTree))
return true;
Expand All @@ -5606,7 +5601,7 @@ bool FlowGraphNaturalLoop::AnalyzeIteration(NaturalLoopIterInfo* info)
return false;
}

if (!CheckLoopConditionBaseCase(initBlock, info))
if (!CheckLoopConditionBaseCase(preheader, info))
{
JITDUMP(" Loop condition may not be true on the first iteration\n");
return false;
Expand Down Expand Up @@ -5637,33 +5632,6 @@ bool FlowGraphNaturalLoop::AnalyzeIteration(NaturalLoopIterInfo* info)
return true;
}

//------------------------------------------------------------------------
// FlowGraphNaturalLoop::MatchInit: Try to pattern match the initialization of
// an induction variable.
//
// Parameters:
// info - [in, out] Info structure to query and fill out
// initBlock - Block containing the initialization tree
// init - Initialization tree
//
// Remarks:
// We do not necessarily guarantee or require to be able to find any
// initialization.
//
void FlowGraphNaturalLoop::MatchInit(NaturalLoopIterInfo* info, BasicBlock* initBlock, GenTree* init)
{
if ((init == nullptr) || !init->OperIs(GT_STORE_LCL_VAR) || (init->AsLclVarCommon()->GetLclNum() != info->IterVar))
return;

GenTree* initValue = init->AsLclVar()->Data();
if (!initValue->IsCnsIntOrI() || !initValue->TypeIs(TYP_INT))
return;

info->HasConstInit = true;
info->ConstInitValue = (int)initValue->AsIntCon()->IconValue();
INDEBUG(info->InitTree = init);
}

//------------------------------------------------------------------------
// FlowGraphNaturalLoop::MatchLimit: Try to pattern match the loop test of an
// induction variable.
Expand Down Expand Up @@ -5801,6 +5769,62 @@ bool FlowGraphNaturalLoop::MatchLimit(unsigned iterVar, GenTree* test, NaturalLo
return true;
}

//------------------------------------------------------------------------
// FlowGraphNaturalLoop::FindConstInit:
// Find an unconditional constant initialization of the iteration variable,
// recording and returning its information.
//
// Parameters:
// preheader - Preheader of the loop, to start the search from
// info - [in, out] Loop information
//
// Returns:
// True if a constant init of the iteration variable was found; otherwise false.
//
bool FlowGraphNaturalLoop::FindConstInit(BasicBlock* preheader, NaturalLoopIterInfo* info)
{
BasicBlock* curBlock = preheader;
do
{
Statement* stmt = curBlock->lastStmt();
if (stmt != nullptr)
{
while (true)
{
GenTree* tree = stmt->GetRootNode();
if (tree->OperIs(GT_STORE_LCL_VAR))
{
GenTreeLclVarCommon* store = tree->AsLclVarCommon();
GenTree* data = store->Data();
if ((store->GetLclNum() == info->IterVar) && data->IsCnsIntOrI() && data->TypeIs(TYP_INT))
{
info->HasConstInit = true;
info->ConstInitValue = (int)data->AsIntCon()->IconValue();
INDEBUG(info->InitTree = tree);
return true;
}
}

if (GetDfsTree()->GetCompiler()->gtTreeHasLocalStore(tree, info->IterVar))
{
return false;
}

if (stmt == curBlock->firstStmt())
{
break;
}

stmt = stmt->GetPrevStmt();
}
}

curBlock = curBlock->GetUniquePred(GetDfsTree()->GetCompiler());
} while (curBlock != nullptr);

return false;
}

//------------------------------------------------------------------------
// EvaluateRelop: Evaluate a relational operator with constant arguments.
//
Expand Down Expand Up @@ -5852,7 +5876,7 @@ bool FlowGraphNaturalLoop::EvaluateRelop(T op1, T op2, genTreeOps oper)
// * The condition being trivially true in the first iteration (e.g. for (int i = 0; i < 3; i++))
// * The condition is checked before entry (often due to loop inversion)
//
bool FlowGraphNaturalLoop::CheckLoopConditionBaseCase(BasicBlock* initBlock, NaturalLoopIterInfo* info)
bool FlowGraphNaturalLoop::CheckLoopConditionBaseCase(BasicBlock* preheader, NaturalLoopIterInfo* info)
{
// TODO: A common loop idiom is to enter the loop at the test, with the
// unique in-loop predecessor of the header block being the increment. We
Expand Down Expand Up @@ -5883,8 +5907,7 @@ bool FlowGraphNaturalLoop::CheckLoopConditionBaseCase(BasicBlock* initBlock, Nat
}
}

// Do we have a zero-trip test?
if (initBlock->KindIs(BBJ_COND) && IsZeroTripTest(initBlock, info))
if (HasZeroTripTest(preheader, info))
{
return true;
}
Expand All @@ -5893,33 +5916,64 @@ bool FlowGraphNaturalLoop::CheckLoopConditionBaseCase(BasicBlock* initBlock, Nat
}

//------------------------------------------------------------------------
// IsZeroTripTest: Check whether `initBlock`, a BBJ_COND block that enters the
// loop in one case and not in the other, implies that the loop invariant is
// true on entry.
// HasZeroTripTest: Check whether the loop has a zero trip test guarding it
// from being entered.
//
// Parameters:
// preheader - The preheader block
// info - Iteration information
//
// Returns:
// True if we could prove that the loop invariant is true on entry.
//
bool FlowGraphNaturalLoop::HasZeroTripTest(BasicBlock* preheader, NaturalLoopIterInfo* info)
{
assert(!preheader->KindIs(BBJ_COND));
BasicBlock* curBlock = preheader;
while (true)
{
BasicBlock* prevBlock = curBlock;
curBlock = curBlock->GetUniquePred(GetDfsTree()->GetCompiler());

if (curBlock == nullptr)
{
return false;
}

if (curBlock->KindIs(BBJ_COND) && (curBlock->GetFalseTarget() != curBlock->GetTrueTarget()) &&
IsZeroTripTest(curBlock, curBlock->TrueTargetIs(prevBlock), info))
{
return true;
}
}
}

//------------------------------------------------------------------------
// IsZeroTripTest: Check whether the loop has a zero trip test guarding it
// from being entered.
//
// Parameters:
// guardBlock - The preheader block
// entersWhenTrue - Whether the loop is entered on true or false of the guard block.
// info - Iteration information
//
// Returns:
// True if we could prove that the loop invariant is true on entry through
// "initBlock".
// "guardBlock".
//
bool FlowGraphNaturalLoop::IsZeroTripTest(BasicBlock* initBlock, NaturalLoopIterInfo* info)
bool FlowGraphNaturalLoop::IsZeroTripTest(BasicBlock* guardBlock, bool entersWhenTrue, NaturalLoopIterInfo* info)
{
assert(initBlock->KindIs(BBJ_COND));
GenTree* enteringJTrue = initBlock->lastStmt()->GetRootNode();
assert(guardBlock->KindIs(BBJ_COND));
GenTree* enteringJTrue = guardBlock->lastStmt()->GetRootNode();
assert(enteringJTrue->OperIs(GT_JTRUE));
GenTree* relop = enteringJTrue->gtGetOp1();
if (!relop->OperIsCmpCompare())
{
return false;
}

// Technically optExtractInitTestIncr only handles the "false"
// entry case, and preheader creation should ensure that that's the
// only time we'll see a BBJ_COND init block. However, it does not
// hurt to let this logic be correct by construction.
bool enterOnTrue = InitBlockEntersLoopOnTrue(initBlock);

JITDUMP(" Init block " FMT_BB " enters the loop when condition [%06u] evaluates to %s\n", initBlock->bbNum,
Compiler::dspTreeID(relop), enterOnTrue ? "true" : "false");
JITDUMP(" Guard block " FMT_BB " enters the loop when condition [%06u] evaluates to %s\n", guardBlock->bbNum,
Compiler::dspTreeID(relop), entersWhenTrue ? "true" : "false");

GenTree* limitCandidate;
genTreeOps oper;
Expand All @@ -5943,7 +5997,7 @@ bool FlowGraphNaturalLoop::IsZeroTripTest(BasicBlock* initBlock, NaturalLoopIter
return false;
}

if (!enterOnTrue)
if (!entersWhenTrue)
{
oper = GenTree::ReverseRelop(oper);
}
Expand All @@ -5968,56 +6022,6 @@ bool FlowGraphNaturalLoop::IsZeroTripTest(BasicBlock* initBlock, NaturalLoopIter
return true;
}

//------------------------------------------------------------------------
// InitBlockEntersLoopOnTrue: Determine whether a BBJ_COND init block enters the
// loop in the false or true case.
//
// Parameters:
// initBlock - A BBJ_COND block that is assumed to dominate the loop, and
// only enter the loop in one of the two cases.
//
// Returns:
// True if the loop is entered if the condition evaluates to true; otherwise false.
//
// Remarks:
// Handles only limited cases (optExtractInitTestIncr ensures that we see
// only limited cases).
//
bool FlowGraphNaturalLoop::InitBlockEntersLoopOnTrue(BasicBlock* initBlock)
{
assert(initBlock->KindIs(BBJ_COND));

if (initBlock->FalseTargetIs(GetHeader()))
{
return false;
}

if (initBlock->TrueTargetIs(GetHeader()))
{
return true;
}

// `optExtractInitTestIncr` may look at preds of preds to find an init
// block, so try a little bit harder. Today this always happens since we
// always have preheaders created in the places we call
// FlowGraphNaturalLoop::AnalyzeIteration.
for (FlowEdge* enterEdge : EntryEdges())
{
BasicBlock* entering = enterEdge->getSourceBlock();
if (initBlock->FalseTargetIs(entering))
{
return false;
}
if (initBlock->TrueTargetIs(entering))
{
return true;
}
}

assert(!"Could not find init block enter side");
return false;
}

//------------------------------------------------------------------------
// GetLexicallyTopMostBlock: Get the lexically top-most block contained within
// the loop.
Expand Down
Loading
Loading