diff --git a/src/coreclr/jit/assertionprop.cpp b/src/coreclr/jit/assertionprop.cpp index c15cb42786ed7e..a066b23403609a 100644 --- a/src/coreclr/jit/assertionprop.cpp +++ b/src/coreclr/jit/assertionprop.cpp @@ -1636,8 +1636,35 @@ AssertionIndex Compiler::optCreateAssertion(GenTree* op1, // // Copy Assertions // + case GT_OBJ: + case GT_BLK: + { + // TODO-ADDR: delete once local morph folds SIMD-typed indirections. + // + GenTree* const addr = op2->AsIndir()->Addr(); + + if (addr->OperIs(GT_ADDR)) + { + GenTree* const base = addr->AsOp()->gtOp1; + + if (base->OperIs(GT_LCL_VAR) && varTypeIsStruct(base)) + { + ClassLayout* const varLayout = base->GetLayout(this); + ClassLayout* const objLayout = op2->GetLayout(this); + if (ClassLayout::AreCompatible(varLayout, objLayout)) + { + op2 = base; + goto IS_COPY; + } + } + } + + goto DONE_ASSERTION; + } + case GT_LCL_VAR: { + IS_COPY: // // Must either be an OAK_EQUAL or an OAK_NOT_EQUAL assertion // @@ -3416,7 +3443,14 @@ bool Compiler::optZeroObjAssertionProp(GenTree* tree, ASSERT_VALARG_TP assertion return false; } - unsigned lclNum = tree->AsLclVar()->GetLclNum(); + // No ZEROOBJ assertions for simd. + // + if (varTypeIsSIMD(tree)) + { + return false; + } + + const unsigned lclNum = tree->AsLclVar()->GetLclNum(); AssertionIndex assertionIndex = optLocalAssertionIsEqualOrNotEqual(O1K_LCLVAR, lclNum, O2K_ZEROOBJ, 0, assertions); if (assertionIndex == NO_ASSERTION_INDEX) { @@ -3568,6 +3602,20 @@ GenTree* Compiler::optCopyAssertionProp(AssertionDsc* curAssertion, return nullptr; } + // Heuristic: for LclFld prop, don't force the copy or its promoted fields to be in memory. + // + if (tree->OperIs(GT_LCL_FLD)) + { + if (copyVarDsc->IsEnregisterableLcl() || copyVarDsc->lvPromotedStruct()) + { + return nullptr; + } + else + { + lvaSetVarDoNotEnregister(copyLclNum DEBUGARG(DoNotEnregisterReason::LocalField)); + } + } + tree->SetLclNum(copyLclNum); tree->SetSsaNum(copySsaNum); @@ -3689,6 +3737,71 @@ GenTree* Compiler::optAssertionProp_LclVar(ASSERT_VALARG_TP assertions, GenTreeL return nullptr; } +//------------------------------------------------------------------------ +// optAssertionProp_LclFld: try and optimize a local field use via assertions +// +// Arguments: +// assertions - set of live assertions +// tree - local field use to optimize +// stmt - statement containing the tree +// +// Returns: +// Updated tree, or nullptr +// +// Notes: +// stmt may be nullptr during local assertion prop +// +GenTree* Compiler::optAssertionProp_LclFld(ASSERT_VALARG_TP assertions, GenTreeLclVarCommon* tree, Statement* stmt) +{ + // If we have a var definition then bail or + // If this is the address of the var then it will have the GTF_DONT_CSE + // flag set and we don't want to to assertion prop on it. + if (tree->gtFlags & (GTF_VAR_DEF | GTF_DONT_CSE)) + { + return nullptr; + } + + // Only run during local prop and if copies are available. + // + if (!optLocalAssertionProp || !optCanPropLclVar) + { + return nullptr; + } + + BitVecOps::Iter iter(apTraits, assertions); + unsigned index = 0; + while (iter.NextElem(&index)) + { + AssertionIndex assertionIndex = GetAssertionIndex(index); + if (assertionIndex > optAssertionCount) + { + break; + } + + // See if the variable is equal to another variable. + AssertionDsc* curAssertion = optGetAssertion(assertionIndex); + if (!curAssertion->CanPropLclVar()) + { + continue; + } + + // Copy prop. + if (curAssertion->op2.kind == O2K_LCLVAR_COPY) + { + // Perform copy assertion prop. + GenTree* newTree = optCopyAssertionProp(curAssertion, tree, stmt DEBUGARG(assertionIndex)); + if (newTree != nullptr) + { + return newTree; + } + } + + continue; + } + + return nullptr; +} + //------------------------------------------------------------------------ // optAssertionProp_Asg: Try and optimize an assignment via assertions. // @@ -4916,6 +5029,9 @@ GenTree* Compiler::optAssertionProp(ASSERT_VALARG_TP assertions, GenTree* tree, case GT_LCL_VAR: return optAssertionProp_LclVar(assertions, tree->AsLclVarCommon(), stmt); + case GT_LCL_FLD: + return optAssertionProp_LclFld(assertions, tree->AsLclVarCommon(), stmt); + case GT_ASG: return optAssertionProp_Asg(assertions, tree->AsOp(), stmt); diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index cc69470cbb9af7..5b4ce674f4a2ba 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -5767,7 +5767,7 @@ class Compiler GenTree* fgMorphCopyBlock(GenTree* tree); GenTree* fgMorphStoreDynBlock(GenTreeStoreDynBlk* tree); GenTree* fgMorphForRegisterFP(GenTree* tree); - GenTree* fgMorphSmpOp(GenTree* tree, MorphAddrContext* mac = nullptr); + GenTree* fgMorphSmpOp(GenTree* tree, MorphAddrContext* mac, bool* optAssertionPropDone = nullptr); GenTree* fgOptimizeCast(GenTreeCast* cast); GenTree* fgOptimizeCastOnAssignment(GenTreeOp* asg); GenTree* fgOptimizeEqualityComparisonWithConst(GenTreeOp* cmp); @@ -5786,7 +5786,7 @@ class Compiler GenTree* fgMorphRetInd(GenTreeUnOp* tree); GenTree* fgMorphModToSubMulDiv(GenTreeOp* tree); GenTree* fgMorphUModToAndSub(GenTreeOp* tree); - GenTree* fgMorphSmpOpOptional(GenTreeOp* tree); + GenTree* fgMorphSmpOpOptional(GenTreeOp* tree, bool* optAssertionPropDone); GenTree* fgMorphMultiOp(GenTreeMultiOp* multiOp); GenTree* fgMorphConst(GenTree* tree); @@ -5802,7 +5802,8 @@ class Compiler private: void fgKillDependentAssertionsSingle(unsigned lclNum DEBUGARG(GenTree* tree)); void fgKillDependentAssertions(unsigned lclNum DEBUGARG(GenTree* tree)); - void fgMorphTreeDone(GenTree* tree, GenTree* oldTree = nullptr DEBUGARG(int morphNum = 0)); + void fgMorphTreeDone(GenTree* tree); + void fgMorphTreeDone(GenTree* tree, bool optAssertionPropDone, bool isMorphedTree DEBUGARG(int morphNum = 0)); Statement* fgMorphStmt; @@ -7368,6 +7369,7 @@ class Compiler // Assertion propagation functions. GenTree* optAssertionProp(ASSERT_VALARG_TP assertions, GenTree* tree, Statement* stmt, BasicBlock* block); GenTree* optAssertionProp_LclVar(ASSERT_VALARG_TP assertions, GenTreeLclVarCommon* tree, Statement* stmt); + GenTree* optAssertionProp_LclFld(ASSERT_VALARG_TP assertions, GenTreeLclVarCommon* tree, Statement* stmt); GenTree* optAssertionProp_Asg(ASSERT_VALARG_TP assertions, GenTreeOp* asg, Statement* stmt); GenTree* optAssertionProp_Return(ASSERT_VALARG_TP assertions, GenTreeUnOp* ret, Statement* stmt); GenTree* optAssertionProp_Ind(ASSERT_VALARG_TP assertions, GenTree* tree, Statement* stmt); diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 2e638b22f3c9e5..abe66b5de931e3 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -5500,7 +5500,7 @@ GenTree* Compiler::fgMorphField(GenTree* tree, MorphAddrContext* mac) tree->gtFlags |= (GTF_IND_INVARIANT | GTF_IND_NONFAULTING | GTF_IND_NONNULL); } - return fgMorphSmpOp(tree); + return fgMorphSmpOp(tree, /* mac */ nullptr); } } @@ -9771,16 +9771,23 @@ GenTree* Compiler::fgMorphCastedBitwiseOp(GenTreeOp* tree) return nullptr; } -/***************************************************************************** - * - * Transform the given GTK_SMPOP tree for code generation. - */ - +//------------------------------------------------------------------------ +// fgMorphSmpOp: morph a GTK_SMPOP tree +// +// Arguments: +// tree - tree to morph +// mac - address context for morphing +// optAssertionPropDone - [out, optional] set true if local assertions +// were killed/genned while morphing this tree +// +// Returns: +// Tree, possibly updated +// #ifdef _PREFAST_ #pragma warning(push) #pragma warning(disable : 21000) // Suppress PREFast warning about overly large function #endif -GenTree* Compiler::fgMorphSmpOp(GenTree* tree, MorphAddrContext* mac) +GenTree* Compiler::fgMorphSmpOp(GenTree* tree, MorphAddrContext* mac, bool* optAssertionPropDone) { ALLOCA_CHECK(); assert(tree->OperKind() & GTK_SMPOP); @@ -11713,7 +11720,7 @@ GenTree* Compiler::fgMorphSmpOp(GenTree* tree, MorphAddrContext* mac) return tree; } - tree = fgMorphSmpOpOptional(tree->AsOp()); + tree = fgMorphSmpOpOptional(tree->AsOp(), optAssertionPropDone); return tree; } @@ -13099,8 +13106,18 @@ GenTree* Compiler::fgMorphRetInd(GenTreeUnOp* ret) #ifdef _PREFAST_ #pragma warning(pop) #endif - -GenTree* Compiler::fgMorphSmpOpOptional(GenTreeOp* tree) +//------------------------------------------------------------- +// fgMorphSmpOpOptional: optional post-order morping of some SMP trees +// +// Arguments: +// tree - tree to morph +// optAssertionPropDone - [out, optional] set true if local assertions were +// killed/genned by the optional morphing +// +// Returns: +// Tree, possibly updated +// +GenTree* Compiler::fgMorphSmpOpOptional(GenTreeOp* tree, bool* optAssertionPropDone) { genTreeOps oper = tree->gtOper; GenTree* op1 = tree->gtOp1; @@ -13199,6 +13216,14 @@ GenTree* Compiler::fgMorphSmpOpOptional(GenTreeOp* tree) if (varTypeIsStruct(typ) && !tree->IsPhiDefn()) { + // Block ops handle assertion kill/gen specially. + // See PrepareDst and PropagateAssertions + // + if (optAssertionPropDone != nullptr) + { + *optAssertionPropDone = true; + } + if (tree->OperIsCopyBlkOp()) { return fgMorphCopyBlock(tree); @@ -14019,6 +14044,8 @@ GenTree* Compiler::fgMorphTree(GenTree* tree, MorphAddrContext* mac) } #endif + bool optAssertionPropDone = false; + /*------------------------------------------------------------------------- * fgMorphTree() can potentially replace a tree with another, and the * caller has to store the return value correctly. @@ -14083,11 +14110,11 @@ GenTree* Compiler::fgMorphTree(GenTree* tree, MorphAddrContext* mac) /* Save the original un-morphed tree for fgMorphTreeDone */ - GenTree* oldTree = tree; + GenTree* const oldTree = tree; /* Figure out what kind of a node we have */ - unsigned kind = tree->OperKind(); + unsigned const kind = tree->OperKind(); /* Is this a constant node? */ @@ -14109,7 +14136,7 @@ GenTree* Compiler::fgMorphTree(GenTree* tree, MorphAddrContext* mac) if (kind & GTK_SMPOP) { - tree = fgMorphSmpOp(tree, mac); + tree = fgMorphSmpOp(tree, mac, &optAssertionPropDone); goto DONE; } @@ -14221,7 +14248,8 @@ GenTree* Compiler::fgMorphTree(GenTree* tree, MorphAddrContext* mac) } DONE: - fgMorphTreeDone(tree, oldTree DEBUGARG(thisMorphNum)); + const bool isNewTree = (oldTree != tree); + fgMorphTreeDone(tree, optAssertionPropDone, isNewTree DEBUGARG(thisMorphNum)); return tree; } @@ -14314,20 +14342,40 @@ void Compiler::fgKillDependentAssertions(unsigned lclNum DEBUGARG(GenTree* tree) } } -/***************************************************************************** - * - * This function is called to complete the morphing of a tree node - * It should only be called once for each node. - * If DEBUG is defined the flag GTF_DEBUG_NODE_MORPHED is checked and updated, - * to enforce the invariant that each node is only morphed once. - * If local assertion prop is enabled the result tree may be replaced - * by an equivalent tree. - * - */ +//------------------------------------------------------------------------ +// fgMorphTreeDone: complete the morphing of a tree node +// +// Arguments: +// tree - the tree after morphing +// +// Notes: +// Simple version where the tree has not been marked +// as morphed, and where assertion kill/gen has not yet been done. +// +void Compiler::fgMorphTreeDone(GenTree* tree) +{ + fgMorphTreeDone(tree, false, false); +} -void Compiler::fgMorphTreeDone(GenTree* tree, - GenTree* oldTree /* == NULL */ - DEBUGARG(int morphNum)) +//------------------------------------------------------------------------ +// fgMorphTreeDone: complete the morphing of a tree node +// +// Arguments: +// tree - the tree after morphing +// optAssertionPropDone - true if local assertion prop was done already +// isMorphedTree - true if caller should have marked tree as morphed +// morphNum - counts invocations of fgMorphTree +// +// Notes: +// This function is called to complete the morphing of a tree node +// It should only be called once for each node. +// If DEBUG is defined the flag GTF_DEBUG_NODE_MORPHED is checked and updated, +// to enforce the invariant that each node is only morphed once. +// +// When local assertion prop is active assertions are killed and generated +// based on tree (unless optAssertionPropDone is true). +// +void Compiler::fgMorphTreeDone(GenTree* tree, bool optAssertionPropDone, bool isMorphedTree DEBUGARG(int morphNum)) { #ifdef DEBUG if (verbose && treesBeforeAfterMorph) @@ -14343,36 +14391,33 @@ void Compiler::fgMorphTreeDone(GenTree* tree, return; } - if ((oldTree != nullptr) && (oldTree != tree)) + if (isMorphedTree) { - /* Ensure that we have morphed this node */ + // caller should have set the morphed flag + // assert((tree->gtDebugFlags & GTF_DEBUG_NODE_MORPHED) && "ERROR: Did not morph this node!"); - -#ifdef DEBUG - TransferTestDataToNode(oldTree, tree); -#endif } else { - // Ensure that we haven't morphed this node already + // caller should not have set the morphed flag + // assert(((tree->gtDebugFlags & GTF_DEBUG_NODE_MORPHED) == 0) && "ERROR: Already morphed this node!"); + INDEBUG(tree->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED); } - if (tree->OperIsConst()) - { - goto DONE; - } - - if (!optLocalAssertionProp) + // Note "tree" may generate new assertions that we + // miss if we did them early... perhaps we should skip + // kills but rerun gens. + // + if (tree->OperIsConst() || !optLocalAssertionProp || optAssertionPropDone) { - goto DONE; + return; } - /* Do we have any active assertions? */ - + // Kill active assertions + // if (optAssertionCount > 0) { - /* Is this an assignment to a local variable */ GenTreeLclVarCommon* lclVarTree = nullptr; // The check below will miss LIR-style assignments. @@ -14383,23 +14428,18 @@ void Compiler::fgMorphTreeDone(GenTree* tree, // DefinesLocal can return true for some BLK op uses, so // check what gets assigned only when we're at an assignment. + // if (tree->OperIsSsaDef() && tree->DefinesLocal(this, &lclVarTree)) { - unsigned lclNum = lclVarTree->GetLclNum(); + const unsigned lclNum = lclVarTree->GetLclNum(); noway_assert(lclNum < lvaCount); fgKillDependentAssertions(lclNum DEBUGARG(tree)); } } - /* If this tree makes a new assertion - make it available */ + // Generate new assertions + // optAssertionGen(tree); - -DONE:; - -#ifdef DEBUG - /* Mark this node as being morphed */ - tree->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED; -#endif } //------------------------------------------------------------------------ diff --git a/src/coreclr/jit/morphblock.cpp b/src/coreclr/jit/morphblock.cpp index a976ded54d7f83..d3db65e1785c37 100644 --- a/src/coreclr/jit/morphblock.cpp +++ b/src/coreclr/jit/morphblock.cpp @@ -19,6 +19,8 @@ class MorphInitBlockHelper virtual void TrySpecialCases(); virtual void MorphStructCases(); + void PropagateAssertions(); + virtual const char* GetHelperName() const { return "MorphInitBlock"; @@ -125,7 +127,7 @@ GenTree* MorphInitBlockHelper::Morph() PrepareDst(); PrepareSrc(); - + PropagateAssertions(); TrySpecialCases(); if (m_transformationDecision == BlockTransformation::Undefined) @@ -275,6 +277,23 @@ void MorphInitBlockHelper::PrepareDst() #endif // DEBUG } +//------------------------------------------------------------------------ +// PropagateAssertions: propagate assertions based on the original tree +// +// Notes: +// Once the init or copy tree is morphed, assertion gen can no +// longer recognize what it means. +// +// So we generate assertions based on the original tree. +// +void MorphInitBlockHelper::PropagateAssertions() +{ + if (m_comp->optLocalAssertionProp) + { + m_comp->optAssertionGen(m_asg); + } +} + //------------------------------------------------------------------------ // PrepareSrc: Transform the asg src to an appropriate form and initialize member fields // with information about it.