diff --git a/src/coreclr/jit/assertionprop.cpp b/src/coreclr/jit/assertionprop.cpp index 71d299ee7c0e75..5d299d10ede2f9 100644 --- a/src/coreclr/jit/assertionprop.cpp +++ b/src/coreclr/jit/assertionprop.cpp @@ -1649,7 +1649,7 @@ AssertionIndex Compiler::optCreateJtrueAssertions(GenTree* op1, GenTree* op2, bo return assertionIndex; } -AssertionInfo Compiler::optCreateJTrueBoundsAssertion(GenTree* tree) +AssertionInfo Compiler::optCreateJTrueBoundsAssertion(GenTree* tree, bool createComplementary) { // These assertions are VN based, so not relevant for local prop // @@ -1705,7 +1705,10 @@ AssertionInfo Compiler::optCreateJTrueBoundsAssertion(GenTree* tree) relopFunc = ValueNumStore::SwapRelop(relopFunc); AssertionDsc dsc = AssertionDsc::CreateCompareCheckedBound(this, relopFunc, op2VN, op1VN, 0); AssertionIndex idx = optAddAssertion(dsc); - optCreateComplementaryAssertion(idx); + if (createComplementary) + { + optCreateComplementaryAssertion(idx); + } return idx; } @@ -1714,7 +1717,10 @@ AssertionInfo Compiler::optCreateJTrueBoundsAssertion(GenTree* tree) { AssertionDsc dsc = AssertionDsc::CreateCompareCheckedBound(this, relopFunc, op1VN, op2VN, 0); AssertionIndex idx = optAddAssertion(dsc); - optCreateComplementaryAssertion(idx); + if (createComplementary) + { + optCreateComplementaryAssertion(idx); + } return idx; } @@ -1727,7 +1733,10 @@ AssertionInfo Compiler::optCreateJTrueBoundsAssertion(GenTree* tree) relopFunc = ValueNumStore::SwapRelop(relopFunc); AssertionDsc dsc = AssertionDsc::CreateCompareCheckedBound(this, relopFunc, op2VN, checkedBnd, checkedBndCns); AssertionIndex idx = optAddAssertion(dsc); - optCreateComplementaryAssertion(idx); + if (createComplementary) + { + optCreateComplementaryAssertion(idx); + } return idx; } @@ -1736,7 +1745,10 @@ AssertionInfo Compiler::optCreateJTrueBoundsAssertion(GenTree* tree) { AssertionDsc dsc = AssertionDsc::CreateCompareCheckedBound(this, relopFunc, op1VN, checkedBnd, checkedBndCns); AssertionIndex idx = optAddAssertion(dsc); - optCreateComplementaryAssertion(idx); + if (createComplementary) + { + optCreateComplementaryAssertion(idx); + } return idx; } @@ -1745,6 +1757,14 @@ AssertionInfo Compiler::optCreateJTrueBoundsAssertion(GenTree* tree) ValueNumStore::UnsignedCompareCheckedBoundInfo unsignedCompareBnd; if (vnStore->IsVNUnsignedCompareCheckedBound(relopVN, &unsignedCompareBnd)) { + if (!createComplementary && (unsignedCompareBnd.cmpOper == VNF_GE_UN)) + { + // The no-throw assertion only holds on the FALSE edge of "i >= bnd". A one-sided + // caller guarantees the relop is TRUE at the use site, so the assertion does not + // hold there - skip it. + return NO_ASSERTION_INDEX; + } + ValueNum idxVN = vnStore->VNNormalValue(unsignedCompareBnd.vnIdx); ValueNum lenVN = vnStore->VNNormalValue(unsignedCompareBnd.vnBound); @@ -1767,7 +1787,10 @@ AssertionInfo Compiler::optCreateJTrueBoundsAssertion(GenTree* tree) relopFunc = ValueNumStore::SwapRelop(relopFunc); AssertionDsc dsc = AssertionDsc::CreateConstantBound(this, relopFunc, op2VN, op1VN); AssertionIndex idx = optAddAssertion(dsc); - optCreateComplementaryAssertion(idx); + if (createComplementary) + { + optCreateComplementaryAssertion(idx); + } return idx; } @@ -1775,7 +1798,10 @@ AssertionInfo Compiler::optCreateJTrueBoundsAssertion(GenTree* tree) { AssertionDsc dsc = AssertionDsc::CreateConstantBound(this, relopFunc, op1VN, op2VN); AssertionIndex idx = optAddAssertion(dsc); - optCreateComplementaryAssertion(idx); + if (createComplementary) + { + optCreateComplementaryAssertion(idx); + } return idx; } @@ -2163,6 +2189,13 @@ void Compiler::optAssertionGen(GenTree* tree) assertionInfo = optAssertionGenJtrue(tree); break; + case GT_ASSUME: + // The relop wrapped by GT_ASSUME is known to be true at this point. + // Generate the same assertion we would for `JTRUE(relop)` taking the true edge, + // but skip the complementary assertion since this is a one-sided fact. + assertionInfo = optCreateJTrueBoundsAssertion(tree, /* createComplementary */ false); + break; + default: // All other gtOper node kinds, leave 'assertionIndex' = NO_ASSERTION_INDEX break; diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index e794aee7840842..f753149cb3e2bd 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -7615,6 +7615,7 @@ class Compiler #define OMF_HAS_STACK_ARRAY 0x00100000 // Method contains stack allocated arrays #define OMF_HAS_BOUNDS_CHECKS 0x00200000 // Method contains bounds checks #define OMF_HAS_EARLY_QMARKS 0x00400000 // Method contains early expandable QMARKs +#define OMF_HAS_ASSUMPTIONS 0x00800000 // Method contains GT_ASSUME nodes // clang-format on @@ -7665,6 +7666,16 @@ class Compiler optMethodFlags |= OMF_HAS_EXPANDABLE_CAST; } + bool doesMethodHaveAssumptions() const + { + return (optMethodFlags & OMF_HAS_ASSUMPTIONS) != 0; + } + + void setMethodHasAssumptions() + { + optMethodFlags |= OMF_HAS_ASSUMPTIONS; + } + bool doesMethodHaveGuardedDevirtualization() const { return (optMethodFlags & OMF_HAS_GUARDEDDEVIRT) != 0; @@ -8726,7 +8737,7 @@ class Compiler // Assertion Gen functions. void optAssertionGen(GenTree* tree); AssertionIndex optAssertionGenCast(GenTreeCast* cast); - AssertionInfo optCreateJTrueBoundsAssertion(GenTree* tree); + AssertionInfo optCreateJTrueBoundsAssertion(GenTree* tree, bool createComplementary = true); AssertionInfo optAssertionGenJtrue(GenTree* tree); AssertionIndex optCreateJtrueAssertions(GenTree* op1, GenTree* op2, bool equals); AssertionIndex optFindComplementary(AssertionIndex assertionIndex); @@ -12490,6 +12501,7 @@ class GenTreeVisitor case GT_RUNTIMELOOKUP: case GT_ARR_ADDR: case GT_KEEPALIVE: + case GT_ASSUME: case GT_INC_SATURATE: { GenTreeUnOp* const unOp = node->AsUnOp(); diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 0ff1ab40625d97..a5e31a6cef356d 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -4476,6 +4476,7 @@ GenTree::VisitResult GenTree::VisitOperandUses(TVisitor visitor) case GT_PUTARG_STK: case GT_RETURNTRAP: case GT_KEEPALIVE: + case GT_ASSUME: case GT_INC_SATURATE: case GT_RETURN_SUSPEND: return visitor(&this->AsUnOp()->gtOp1); diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 4745ab497f07ff..49d37246701a23 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -6900,6 +6900,7 @@ bool GenTree::TryGetUse(GenTree* operand, GenTree*** pUse) case GT_BSWAP: case GT_BSWAP16: case GT_KEEPALIVE: + case GT_ASSUME: case GT_INC_SATURATE: if (operand == this->AsUnOp()->gtOp1) { @@ -10547,6 +10548,7 @@ GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node) case GT_BSWAP: case GT_BSWAP16: case GT_KEEPALIVE: + case GT_ASSUME: case GT_INC_SATURATE: case GT_RETURNTRAP: case GT_RETURN_SUSPEND: diff --git a/src/coreclr/jit/gtlist.h b/src/coreclr/jit/gtlist.h index d14790d89fb622..9672fed5bd93f8 100644 --- a/src/coreclr/jit/gtlist.h +++ b/src/coreclr/jit/gtlist.h @@ -65,6 +65,7 @@ GTNODE(NEG , GenTreeOp ,0,0,GTK_UNOP) GTNODE(INTRINSIC , GenTreeIntrinsic ,0,0,GTK_BINOP|GTK_EXOP) GTNODE(KEEPALIVE , GenTree ,0,0,GTK_UNOP|GTK_NOVALUE) // keep operand alive, generate no code, produce no result +GTNODE(ASSUME , GenTreeOp ,0,0,GTK_UNOP|GTK_NOVALUE|DBK_NOTLIR) // op1 is non-zero GTNODE(CAST , GenTreeCast ,0,0,GTK_UNOP|GTK_EXOP) // conversion to another type GTNODE(BITCAST , GenTreeOp ,0,1,GTK_UNOP) // reinterpretation of bits as another type diff --git a/src/coreclr/jit/liveness.cpp b/src/coreclr/jit/liveness.cpp index 6e1aca212fb5c1..774b67c6835958 100644 --- a/src/coreclr/jit/liveness.cpp +++ b/src/coreclr/jit/liveness.cpp @@ -2567,6 +2567,7 @@ void Liveness::ComputeLifeLIR(VARSET_TP& life, BasicBlock* block, VAR case GT_IL_OFFSET: case GT_RECORD_ASYNC_RESUME: case GT_KEEPALIVE: + case GT_ASSUME: case GT_SWIFT_ERROR_RET: case GT_GCPOLL: case GT_WASM_JEXCEPT: diff --git a/src/coreclr/jit/optimizer.cpp b/src/coreclr/jit/optimizer.cpp index 636d588bb82d2a..cefa0bf1708237 100644 --- a/src/coreclr/jit/optimizer.cpp +++ b/src/coreclr/jit/optimizer.cpp @@ -2126,6 +2126,40 @@ bool Compiler::optTryInvertWhileLoop(FlowGraphNaturalLoop* loop) #endif // DEBUG Metrics.LoopsInverted++; + + // Synthesize a GT_ASSUME at the start of `stayInLoopSucc` recording that the + // loop guard is known to be true on every entry into the loop body. This compensates + // for the cyclic value numbers that loop inversion creates on phi merges, which can + // otherwise prevent later phases (assertion prop, range check) from proving that + // the guard still holds inside the loop body. + // + auto isSuitableRelopOp = [this](GenTree* node) -> bool { + return node->IsInvariant() || (node->OperIs(GT_LCL_VAR) && !lvaVarAddrExposed(node->AsLclVar()->GetLclNum())); + }; + + GenTree* jtrue = condBlock->lastStmt()->GetRootNode(); + if (jtrue->OperIs(GT_JTRUE)) + { + GenTree* origRelop = condBlock->lastStmt()->GetRootNode()->gtGetOp1(); + if (origRelop->OperIsCmpCompare() && isSuitableRelopOp(origRelop->gtGetOp1()) && + isSuitableRelopOp(origRelop->gtGetOp2())) + { + GenTree* relopClone = gtCloneExpr(origRelop); + if (trueExits) + { + relopClone = gtReverseCond(relopClone); + } + relopClone->gtFlags &= ~GTF_RELOP_JMP_USED; + relopClone->gtFlags |= GTF_DONT_CSE; + + GenTree* assertion = gtNewOperNode(GT_ASSUME, TYP_VOID, relopClone); + Statement* stmt = fgNewStmtFromTree(assertion, condBlock->lastStmt()->GetDebugInfo()); + fgInsertStmtAtBeg(stayInLoopSucc, stmt); + setMethodHasAssumptions(); + JITDUMP("Inserted GT_ASSUME at start of " FMT_BB " to record loop guard\n", stayInLoopSucc->bbNum); + } + } + return true; } diff --git a/src/coreclr/jit/rangecheck.cpp b/src/coreclr/jit/rangecheck.cpp index dca63fb6efdb2a..5f61ccdcddf1a5 100644 --- a/src/coreclr/jit/rangecheck.cpp +++ b/src/coreclr/jit/rangecheck.cpp @@ -14,12 +14,32 @@ // PhaseStatus Compiler::rangeCheckPhase() { - if (!doesMethodHaveBoundsChecks() || (fgSsaPassesCompleted == 0)) + bool madeChanges = false; + if (doesMethodHaveBoundsChecks() && (fgSsaPassesCompleted > 0)) { - return PhaseStatus::MODIFIED_NOTHING; + madeChanges = GetRangeCheck()->OptimizeRangeChecks(); + } + + // GT_ASSUME nodes are no longer needed. Remove all side-effect-free instances + // to avoid pessimizing subsequent phases + if (doesMethodHaveAssumptions()) + { + for (BasicBlock* const block : Blocks()) + { + for (Statement* stmt = block->firstStmt(); stmt != nullptr;) + { + Statement* nextStmt = stmt->GetNextStmt(); + GenTree* root = stmt->GetRootNode(); + if (root->OperIs(GT_ASSUME) && ((root->gtGetOp1()->gtFlags & GTF_SIDE_EFFECT) == 0)) + { + fgRemoveStmt(block, stmt); + madeChanges = true; + } + stmt = nextStmt; + } + } } - const bool madeChanges = GetRangeCheck()->OptimizeRangeChecks(); return madeChanges ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING; } diff --git a/src/coreclr/jit/rationalize.cpp b/src/coreclr/jit/rationalize.cpp index e32a1b13ef6b52..4e897447b72bdf 100644 --- a/src/coreclr/jit/rationalize.cpp +++ b/src/coreclr/jit/rationalize.cpp @@ -1716,7 +1716,8 @@ Compiler::fgWalkResult Rationalizer::RewriteNode(GenTree** useEdge, Compiler::Ge case GT_BOX: case GT_ARR_ADDR: - // BOX/ARR_ADDR are "passthrough" nodes, + case GT_ASSUME: + // BOX/ARR_ADDR/ASSERTION are "passthrough" nodes, // and at this point we no longer need them. if (node->gtGetOp1() != nullptr) { diff --git a/src/coreclr/jit/valuenum.cpp b/src/coreclr/jit/valuenum.cpp index 3d76fd6333b04f..01d811cf5ac315 100644 --- a/src/coreclr/jit/valuenum.cpp +++ b/src/coreclr/jit/valuenum.cpp @@ -12956,6 +12956,7 @@ void Compiler::fgValueNumberTree(GenTree* tree) case GT_RETFILT: case GT_RETURN_SUSPEND: case GT_NULLCHECK: + case GT_ASSUME: if (tree->gtGetOp1() != nullptr) { tree->gtVNPair = vnStore->VNPWithExc(vnStore->VNPForVoid(),