diff --git a/src/coreclr/jit/assertionprop.cpp b/src/coreclr/jit/assertionprop.cpp index 9d4731ac02f6ab..108229ed7c93b2 100644 --- a/src/coreclr/jit/assertionprop.cpp +++ b/src/coreclr/jit/assertionprop.cpp @@ -2741,6 +2741,63 @@ GenTree* Compiler::optVNBasedFoldExpr_Call(BasicBlock* block, GenTree* parent, G return nullptr; } +//------------------------------------------------------------------------------ +// optVNBasedFoldExpr_Call: Folds given ADD/SUB operation using VN to a simpler MUL tree. +// +// Arguments: +// block - The block containing the tree. +// parent - The parent node of the tree. +// tree - The ADD/SUB tree to fold +// +// Return Value: +// Returns a new tree or nullptr if nothing is changed. +// +GenTree* Compiler::optVNBasedFoldExpr_AddSub(BasicBlock* block, GenTree* parent, GenTree* tree) +{ + assert(tree->OperIs(GT_ADD, GT_SUB)); + + ValueNumPair vnPair = tree->gtVNPair; + ValueNum vnCnv = vnStore->VNConservativeNormalValue(vnPair); + VNFuncApp vnFuncApp; + + if (!vnStore->GetVNFunc(vnCnv, &vnFuncApp) || !vnFuncApp.FuncIs(VNF_ADD, VNF_SUB) || + vnStore->TypeOfVN(vnCnv) != TYP_INT) + { + return nullptr; + } + + ValueNum vnOp1 = vnFuncApp.m_args[0]; + ValueNum vnOp2 = vnFuncApp.m_args[1]; + VNFuncApp vnFuncAppOp1; + VNFuncApp vnFuncAppOp2; + int cns = 0; + + if (vnStore->IsVNIntegralConstant(vnOp1, &cns) && cns == 0) + { + return tree->OperIs(GT_ADD) ? tree->gtGetOp2() : tree; + } + else if (vnStore->IsVNIntegralConstant(vnOp2, &cns) && cns == 0) + { + return tree->gtGetOp1(); + } + else if (!vnStore->GetVNFunc(vnOp1, &vnFuncAppOp1) || + !vnFuncAppOp1.FuncIs(VNF_InitVal, VNF_MemOpaque, VNF_ADD, VNF_SUB, VNF_LSH, VNF_XOR, VNF_MUL) || + !vnStore->GetVNFunc(vnOp2, &vnFuncAppOp2) || + !vnFuncAppOp2.FuncIs(VNF_InitVal, VNF_MemOpaque, VNF_ADD, VNF_SUB, VNF_LSH, VNF_XOR, VNF_MUL)) + { + return nullptr; + } + + GenTree* foldedTree = fgMorphReduceAddOrSubOps(tree); + + if (foldedTree == tree) + { + return nullptr; + } + + return foldedTree; +} + //------------------------------------------------------------------------------ // optVNBasedFoldExpr: Folds given tree using VN to a constant or a simpler tree. // @@ -2765,7 +2822,9 @@ GenTree* Compiler::optVNBasedFoldExpr(BasicBlock* block, GenTree* parent, GenTre { case GT_CALL: return optVNBasedFoldExpr_Call(block, parent, tree->AsCall()); - + case GT_ADD: + case GT_SUB: + return optVNBasedFoldExpr_AddSub(block, parent, tree); // We can add more VN-based foldings here. default: diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 35be79978cc0e3..c9526a4c838e03 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -6934,7 +6934,8 @@ class Compiler GenTreeOp* fgMorphCommutative(GenTreeOp* tree); - GenTree* fgMorphReduceAddOps(GenTree* tree); + GenTree* fgMorphReduceAddOrSubOps(GenTree* tree); + bool fgMorphOperIsIntScalarMulLclVar(GenTree* op, ssize_t* scalar, unsigned int* lclNum); public: GenTree* fgMorphTree(GenTree* tree, MorphAddrContext* mac = nullptr); @@ -8776,6 +8777,7 @@ class Compiler GenTree* optVNBasedFoldExpr_Call_Memmove(GenTreeCall* call); GenTree* optVNBasedFoldExpr_Call_Memset(GenTreeCall* call); GenTree* optVNBasedFoldExpr_Call_Memcmp(GenTreeCall* call); + GenTree* optVNBasedFoldExpr_AddSub(BasicBlock* block, GenTree* parent, GenTree* tree); AssertionIndex GetAssertionCount() { diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 7caba07734be6d..d443c4fc95e4bb 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -7507,13 +7507,6 @@ GenTree* Compiler::fgMorphSmpOp(GenTree* tree, MorphAddrContext* mac, bool* optA break; } - if (opts.OptimizationEnabled() && fgGlobalMorph) - { - GenTree* morphed = fgMorphReduceAddOps(tree); - if (morphed != tree) - return fgMorphTree(morphed); - } - /*------------------------------------------------------------------------- * Process the first operand, if any */ @@ -15523,8 +15516,155 @@ bool Compiler::fgCanTailCallViaJitHelper(GenTreeCall* call) } //------------------------------------------------------------------------ -// fgMorphReduceAddOps: reduce successive variable adds into a single multiply, -// e.g., i + i + i + i => i * 4. +// fgMorphOperIsIntScalarMulLclVar: detect if op is a variable or multiple of a variable, +// e.g., i => true. +// e.g., -i => true. +// e.g., -2 * i => true. +// e.g., 5 * i + 1 => false. +// +// Arguments: +// op - Operation to analyze +// scalar - the scalar to which variable is multiplied +// lclNum - lclNum of the variable +// +// Return Value: +// true if pattern matches and false otherwise +// +bool Compiler::fgMorphOperIsIntScalarMulLclVar(GenTree* op, ssize_t* scalar, unsigned int* lclNum) +{ + if (op->OperIs(GT_LCL_VAR)) + { + *scalar = 1; + *lclNum = op->AsLclVar()->GetLclNum(); + return true; + } + else if (op->OperIs(GT_COMMA) && op->gtGetOp1()->OperIs(GT_STORE_LCL_VAR) && + op->gtGetOp1()->gtGetOp1()->OperIs(GT_IND)) + { + *scalar = 1; + *lclNum = op->gtGetOp1()->AsLclVar()->GetLclNum(); + return true; + } + else if (op->OperIs(GT_SUB) && op->gtGetOp1()->IsIntegralConst(0)) + { + if (op->gtGetOp2()->OperIs(GT_LCL_VAR)) + { + *scalar = 1; + *lclNum = op->gtGetOp2()->AsLclVar()->GetLclNum(); + return true; + } + else if (op->gtGetOp2()->OperIs(GT_COMMA) && op->gtGetOp2()->gtGetOp1()->OperIs(GT_STORE_LCL_VAR) && + op->gtGetOp2()->gtGetOp1()->gtGetOp1()->OperIs(GT_IND)) + { + *scalar = 1; + *lclNum = op->gtGetOp2()->gtGetOp1()->AsLclVar()->GetLclNum(); + return true; + } + else + { + return false; + } + } + + if (op->OperIs(GT_LSH) && op->gtGetOp2()->IsIntegralConst()) + { + GenTree* op1 = op->gtGetOp1(); + ssize_t temp_scalar = static_cast(pow(2, op->gtGetOp2()->AsIntConCommon()->IconValue())); + + if (op1->OperIs(GT_LCL_VAR)) + { + *scalar = temp_scalar; + *lclNum = op1->AsLclVar()->GetLclNum(); + } + else if (op1->OperIs(GT_NEG) && op1->gtGetOp1()->OperIs(GT_LCL_VAR)) + { + *scalar = -temp_scalar; + *lclNum = op1->gtGetOp1()->AsLclVar()->GetLclNum(); + } + else if (op1->OperIs(GT_COMMA) && op1->gtGetOp1()->OperIs(GT_STORE_LCL_VAR) && + op1->gtGetOp1()->gtGetOp1()->OperIs(GT_IND)) + { + *scalar = temp_scalar; + *lclNum = op1->gtGetOp1()->AsLclVar()->GetLclNum(); + } + else if (op1->OperIs(GT_NEG) && op1->gtGetOp1()->OperIs(GT_COMMA) && + op1->gtGetOp1()->gtGetOp1()->OperIs(GT_STORE_LCL_VAR) && + op1->gtGetOp1()->gtGetOp1()->gtGetOp1()->OperIs(GT_IND)) + { + *scalar = -temp_scalar; + *lclNum = op1->gtGetOp1()->gtGetOp1()->AsLclVar()->GetLclNum(); + } + else + { + return false; + } + } + else if (op->OperIs(GT_MUL)) + { + GenTree* op1 = op->gtGetOp1(); + GenTree* op2 = op->gtGetOp2(); + + if (op1->IsIntegralConst() && op2->OperIs(GT_LCL_VAR)) + { + *scalar = op1->AsIntConCommon()->IconValue(); + *lclNum = op2->AsLclVar()->GetLclNum(); + } + else if (op2->IsIntegralConst() && op1->OperIs(GT_LCL_VAR)) + { + *scalar = op2->AsIntConCommon()->IconValue(); + *lclNum = op1->AsLclVar()->GetLclNum(); + } + else if (op1->IsIntegralConst() && op2->OperIs(GT_COMMA) && op2->gtGetOp1()->OperIs(GT_STORE_LCL_VAR) && + op2->gtGetOp1()->gtGetOp1()->OperIs(GT_IND)) + { + *scalar = op1->AsIntConCommon()->IconValue(); + *lclNum = op2->gtGetOp1()->AsLclVar()->GetLclNum(); + } + else if (op2->IsIntegralConst() && op1->OperIs(GT_COMMA) && op1->gtGetOp1()->OperIs(GT_STORE_LCL_VAR) && + op1->gtGetOp1()->gtGetOp1()->OperIs(GT_IND)) + { + *scalar = op2->AsIntConCommon()->IconValue(); + *lclNum = op1->gtGetOp1()->AsLclVar()->GetLclNum(); + } + else + { + return false; + } + } + else if (op->OperIs(GT_XOR)) + { + GenTree* op1 = op->gtGetOp1(); + GenTree* op2 = op->gtGetOp2(); + + if (op1->OperIs(GT_LCL_VAR) && op2->OperIs(GT_LCL_VAR) && + op1->AsLclVar()->GetLclNum() == op2->AsLclVar()->GetLclNum()) + { + *scalar = 0; + *lclNum = op2->AsLclVar()->GetLclNum(); + } + else if (op1->OperIs(GT_COMMA) && op1->gtGetOp1()->OperIs(GT_STORE_LCL_VAR) && + op1->gtGetOp1()->gtGetOp1()->OperIs(GT_IND) && op2->OperIs(GT_LCL_VAR) && + op1->gtGetOp1()->AsLclVar()->GetLclNum() == op2->AsLclVar()->GetLclNum()) + { + *scalar = 0; + *lclNum = op1->gtGetOp1()->AsLclVar()->GetLclNum(); + } + else + { + return false; + } + } + else + { + return false; + } + + return true; +} + +//------------------------------------------------------------------------ +// fgMorphReduceAddOrSubOps: reduce successive variable adds/subs into a single multiply, +// e.g., i + 3 * i + i - 4 * i + i => i * 2. // // Arguments: // tree - tree for reduction @@ -15532,54 +15672,74 @@ bool Compiler::fgCanTailCallViaJitHelper(GenTreeCall* call) // Return Value: // reduced tree if pattern matches, original tree otherwise // -GenTree* Compiler::fgMorphReduceAddOps(GenTree* tree) +GenTree* Compiler::fgMorphReduceAddOrSubOps(GenTree* tree) { - // ADD(_, V0) starts the pattern match. - if (!tree->OperIs(GT_ADD) || tree->gtOverflow()) + // ADD(_, V0) OR SUB(_, V0) starts the pattern match. + if (!tree->OperIs(GT_ADD, GT_SUB) || tree->gtOverflow()) { return tree; } #if !defined(TARGET_64BIT) && !defined(TARGET_WASM) - // Transforming 64-bit ADD to 64-bit MUL on 32-bit system results in replacing - // ADD ops with a helper function call. Don't apply optimization in that case. + // Transforming 64-bit ADD/SUB to 64-bit MUL on 32-bit system results in replacing + // ADD/SUB ops with a helper function call. Don't apply optimization in that case. if (tree->TypeIs(TYP_LONG)) { return tree; } #endif // !defined(TARGET_64BIT) && !defined(TARGET_WASM) - GenTree* lclVarTree = tree->AsOp()->gtOp2; - GenTree* consTree = tree->AsOp()->gtOp1; + genTreeOps targetOp = tree->OperGet(); + GenTree* lclVarTree = tree->AsOp()->gtOp2; + GenTree* consTree = tree->AsOp()->gtOp1; - GenTree* op1 = consTree; - GenTree* op2 = lclVarTree; + GenTree* op1 = consTree; + ssize_t op1Scalar = 0; + unsigned int op1lclNum = 0; + GenTree* op2 = lclVarTree; + ssize_t op2Scalar = 0; + unsigned int op2lclNum = 0; - if (!op2->OperIs(GT_LCL_VAR) || !varTypeIsIntegral(op2)) + if (!fgMorphOperIsIntScalarMulLclVar(op2, &op2Scalar, &op2lclNum) || !varTypeIsIntegral(op2)) { return tree; } - int foldCount = 0; - unsigned lclNum = op2->AsLclVarCommon()->GetLclNum(); + ssize_t foldCount = 0; + unsigned int lclNum = op2lclNum; - // Search for pattern of shape ADD(ADD(ADD(lclNum, lclNum), lclNum), lclNum). + // Search for pattern of shape ADD(SUB(ADD(lclNum, lclNum), lclNum), lclNum). while (true) { - // ADD(lclNum, lclNum), end of tree - if (op1->OperIs(GT_LCL_VAR) && op1->AsLclVarCommon()->GetLclNum() == lclNum && op2->OperIs(GT_LCL_VAR) && - op2->AsLclVarCommon()->GetLclNum() == lclNum) + // ADD(lclNum, lclNum) OR SUB(lclNum, lclNum), end of tree + if ((fgMorphOperIsIntScalarMulLclVar(op1, &op1Scalar, &op1lclNum) && op1lclNum == lclNum) && + (fgMorphOperIsIntScalarMulLclVar(op2, &op2Scalar, &op2lclNum) && op2lclNum == lclNum)) { - foldCount += 2; + if (targetOp == GT_ADD) + { + foldCount += op1Scalar + op2Scalar; + } + else + { + foldCount += op1Scalar - op2Scalar; + } break; } - // ADD(ADD(X, Y), lclNum), keep descending - else if (op1->OperIs(GT_ADD) && !op1->gtOverflow() && op2->OperIs(GT_LCL_VAR) && - op2->AsLclVarCommon()->GetLclNum() == lclNum) + // ADD(SUB(X, Y), lclNum), keep descending + else if (op1->OperIs(GT_ADD, GT_SUB) && !op1->gtOverflow() && + (fgMorphOperIsIntScalarMulLclVar(op2, &op2Scalar, &op2lclNum) && op2lclNum == lclNum)) { - foldCount++; - op2 = op1->AsOp()->gtOp2; - op1 = op1->AsOp()->gtOp1; + if (targetOp == GT_ADD) + { + foldCount += op2Scalar; + } + else + { + foldCount -= op2Scalar; + } + targetOp = op1->OperGet(); + op2 = op1->AsOp()->gtOp2; + op1 = op1->AsOp()->gtOp1; } // Any other case is a pattern we won't attempt to fold for now. else @@ -15588,14 +15748,22 @@ GenTree* Compiler::fgMorphReduceAddOps(GenTree* tree) } } - // V0 + V0 ... + V0 becomes V0 * foldCount, where postorder transform will optimize + // V0 + V0 ... + V0 becomes V0 * foldCount, + // V0 - V0 ... - V0 becomes V0 * (- foldCount + 2), where postorder transform will optimize // accordingly - consTree->BashToConst(foldCount, tree->TypeGet()); - - GenTree* morphed = gtNewOperNode(GT_MUL, tree->TypeGet(), lclVarTree, consTree); - DEBUG_DESTROY_NODE(tree); + if (foldCount == 0) + { + return gtNewOperNode(GT_XOR, tree->TypeGet(), op1, + gtCloneExpr(op1->OperIs(GT_LCL_VAR) ? op1 : op1->gtEffectiveVal())); + } + else if (foldCount == -1) + { + return gtNewOperNode(GT_SUB, tree->TypeGet(), gtNewZeroConNode(tree->TypeGet()), + op1->OperIs(GT_MUL, GT_LSH, GT_XOR) ? op1->gtGetOp1() : op1); + } - return morphed; + return gtNewOperNode(GT_MUL, tree->TypeGet(), op1->OperIs(GT_MUL, GT_LSH, GT_XOR) ? op1->gtGetOp1() : op1, + gtNewIconNode(foldCount, tree->TypeGet())); } //------------------------------------------------------------------------