diff --git a/src/coreclr/jit/ifconversion.cpp b/src/coreclr/jit/ifconversion.cpp index 0b9e11c962e8fb..8a40d777d53fc3 100644 --- a/src/coreclr/jit/ifconversion.cpp +++ b/src/coreclr/jit/ifconversion.cpp @@ -45,16 +45,12 @@ class OptIfConversionDsc IfConvertOperation m_thenOperation; // The single operation in the Then case. IfConvertOperation m_elseOperation; // The single operation in the Else case. - int m_checkLimit = 4; // Max number of chained blocks to allow in both the True and Else cases. - genTreeOps m_mainOper = GT_COUNT; // The main oper of the if conversion. bool m_doElseConversion = false; // Does the If conversion have an else statement. - bool m_flowFound = false; // Has a valid flow been found. - bool IfConvertCheckInnerBlockFlow(BasicBlock* block); - bool IfConvertCheckThenFlow(); - void IfConvertFindFlow(); - bool IfConvertCheckStmts(BasicBlock* fromBlock, IfConvertOperation* foundOperation); + bool IfConvertCheck(); + bool IfConvertCheckFlow(); + bool IfConvertCheckStmts(BasicBlock* block, IfConvertOperation* foundOperation); GenTree* TryTransformSelectOperOrLocal(GenTree* oper, GenTree* lcl); GenTree* TryTransformSelectOperOrZero(GenTree* oper, GenTree* lcl); @@ -68,290 +64,200 @@ class OptIfConversionDsc }; //----------------------------------------------------------------------------- -// IfConvertCheckInnerBlockFlow -// -// Check if the flow of a block is valid for use as an inner block (either a Then or Else block) -// in an If Conversion. +// IfConvertCheck // -// Assumptions: -// m_startBlock and m_doElseConversion are set. -// -// Arguments: -// block -- Block to check. -// -// Returns: -// True if Checks are ok, else false. +// Check whether the JTRUE block and its successors can be expressed as a SELECT. +// In the process, get the data required to perform the transformation. +// Notes: +// Sets m_finalBlock, m_doElseConversion, m_thenOperation, m_elseOperation and m_mainOper // -bool OptIfConversionDsc::IfConvertCheckInnerBlockFlow(BasicBlock* block) +bool OptIfConversionDsc::IfConvertCheck() { - // Block should have a single successor or be a return. - if (!(block->GetUniqueSucc() != nullptr || (m_doElseConversion && (block->KindIs(BBJ_RETURN))))) + if (!IfConvertCheckFlow()) { return false; } - // Check that we have linear flow and are still in the same EH region - - if (block->GetUniquePred(m_compiler) == nullptr) - { + if (!IfConvertCheckStmts(m_startBlock->GetFalseTarget(), &m_thenOperation)) + { return false; } + + m_mainOper = m_thenOperation.node->OperGet(); + assert(m_mainOper == GT_RETURN || m_mainOper == GT_STORE_LCL_VAR); - if (!BasicBlock::sameEHRegion(block, m_startBlock)) + if (m_doElseConversion) { - return false; + if (!IfConvertCheckStmts(m_startBlock->GetTrueTarget(), &m_elseOperation)) + { + return false; + } } + else + { + assert(m_mainOper == GT_STORE_LCL_VAR); - return true; -} + GenTreeLclVar* thenStore = m_thenOperation.node->AsLclVar(); + unsigned lclNum = thenStore->GetLclNum(); -//----------------------------------------------------------------------------- -// IfConvertCheckThenFlow -// -// Check all the Then blocks between m_startBlock and m_finalBlock are valid. -// -// Assumptions: -// m_startBlock, m_finalBlock and m_doElseConversion are set. -// -// Returns: -// If a conversion is found, then set m_flowFound and return true. -// If a conversion is not found, and it's ok to keep searching, return true. -// Otherwise, return false. -// -// Notes: -// Sets m_flowFound and m_mainOper. -// -bool OptIfConversionDsc::IfConvertCheckThenFlow() -{ - m_flowFound = false; - BasicBlock* thenBlock = m_startBlock->GetFalseTarget(); - - for (int thenLimit = 0; thenLimit < m_checkLimit; thenLimit++) - { - if (!IfConvertCheckInnerBlockFlow(thenBlock)) + bool lclVarUsedInThen = m_compiler->gtHasRef(thenStore->Data(), lclNum); + if (!lclVarUsedInThen) { - // Then block is not in a valid flow. - return true; + Statement* last = m_startBlock->lastStmt(); + Statement* stmt = last; + do + { + GenTree* tree = stmt->GetRootNode(); + + if (tree->OperIs(GT_STORE_LCL_VAR)) + { + GenTreeLclVar* prevStore = tree->AsLclVar(); + if (prevStore->GetLclNum() == lclNum) + { + m_doElseConversion = true; + m_elseOperation.block = m_startBlock; + m_elseOperation.stmt = stmt; + m_elseOperation.node = tree; + + break; + } + } + + if (m_compiler->gtHasRef(tree, lclNum)) + { + break; + } + + stmt = stmt->GetPrevStmt(); + } while (stmt != last); } - BasicBlock* thenBlockNext = thenBlock->GetUniqueSucc(); + } + + if (m_doElseConversion) + { + // Both operations are the same node type. + assert(m_thenOperation.node->OperGet() == m_elseOperation.node->OperGet()); - if (thenBlockNext == m_finalBlock) + // Currently can only support Else Store Blocks that have the same destination as the Then block. + if (m_mainOper == GT_STORE_LCL_VAR) { - // All the Then blocks up to m_finalBlock are in a valid flow. - m_flowFound = true; - if (thenBlock->KindIs(BBJ_RETURN)) - { - assert(m_finalBlock == nullptr); - m_mainOper = GT_RETURN; - } - else + unsigned lclNumThen = m_thenOperation.node->AsLclVarCommon()->GetLclNum(); + unsigned lclNumElse = m_elseOperation.node->AsLclVarCommon()->GetLclNum(); + + if (lclNumThen != lclNumElse) { - m_mainOper = GT_STORE_LCL_VAR; + return false; } - return true; } - - if (thenBlockNext == nullptr) - { - // Invalid Then and Else combination. - return false; - } - - thenBlock = thenBlockNext; } - // Nothing found. Still valid to continue. return true; } //----------------------------------------------------------------------------- -// IfConvertFindFlow +// IfConvertCheckFlow // -// Find a valid if conversion flow from m_startBlock to a final block. -// There might be multiple Then and Else blocks in the flow - use m_checkLimit to limit this. +// Check if there is a valid flow from m_startBlock to a final block. // // Notes: -// Sets m_flowFound, m_finalBlock, m_doElseConversion and m_mainOper. +// Sets m_finalBlock and m_doElseConversion. // -void OptIfConversionDsc::IfConvertFindFlow() +bool OptIfConversionDsc::IfConvertCheckFlow() { - // First check for flow with no else case. The final block is the destination of the jump. - m_doElseConversion = false; - m_finalBlock = m_startBlock->GetTrueTarget(); - assert(m_finalBlock != nullptr); - if (!IfConvertCheckThenFlow() || m_flowFound) - { - // Either the flow is invalid, or a flow was found. - return; - } + BasicBlock* falseBb = m_startBlock->GetFalseTarget(); + BasicBlock* trueBb = m_startBlock->GetTrueTarget(); - // Look for flows with else blocks. The final block is the block after the else block. - m_doElseConversion = true; - for (int elseLimit = 0; elseLimit < m_checkLimit; elseLimit++) + if (falseBb->GetUniquePred(m_compiler) == nullptr) { - BasicBlock* elseBlock = m_finalBlock; - if (elseBlock == nullptr || !IfConvertCheckInnerBlockFlow(elseBlock)) - { - // Need a valid else block in a valid flow . - return; - } + return false; + } - m_finalBlock = elseBlock->GetUniqueSucc(); + m_doElseConversion = trueBb->GetUniquePred(m_compiler) != nullptr; + m_finalBlock = m_doElseConversion ? trueBb->GetUniqueSucc() : trueBb; - if (!IfConvertCheckThenFlow() || m_flowFound) - { - // Either the flow is invalid, or a flow was found. - return; - } + // m_finalBlock is only allowed to be null if both return. + // E.g: Then block exits by throwing an exception => we bail here. + if (m_finalBlock == nullptr && (!falseBb->KindIs(BBJ_RETURN) || !trueBb->KindIs(BBJ_RETURN))) + { + return false; } + + return falseBb->GetUniqueSucc() == m_finalBlock; } //----------------------------------------------------------------------------- // IfConvertCheckStmts // -// From the given block to the final block, check all the statements and nodes are -// valid for an If conversion. Chain of blocks must contain only a single local -// store and no other operations. +// Check whether the statements in the block are valid for an If conversion. +// It must contain only a single RETURN or STORE node. NOPs are ignored. // // Arguments: -// fromBlock - Block inside the if statement to start from (Either Then or Else path). -// foundOperation - Returns the found operation. +// block - Block to check +// foundOperation - The found operation // // Returns: -// If everything is valid, then set foundOperation to the store and return true. -// Otherwise return false. +// True if the statements are valid for an If conversion. In that case foundOperation is also set. // -bool OptIfConversionDsc::IfConvertCheckStmts(BasicBlock* fromBlock, IfConvertOperation* foundOperation) +bool OptIfConversionDsc::IfConvertCheckStmts(BasicBlock* block, IfConvertOperation* foundOperation) { bool found = false; - for (BasicBlock* block = fromBlock; block != m_finalBlock; block = block->GetUniqueSucc()) + // Can all the nodes within the block be made to conditionally execute? + for (Statement* stmt : block->Statements()) { - assert(block != nullptr); - - // Can all the nodes within the block be made to conditionally execute? - for (Statement* const stmt : block->Statements()) + GenTree* tree = stmt->GetRootNode(); + if (tree->OperIs(GT_STORE_LCL_VAR, GT_RETURN)) { - GenTree* tree = stmt->GetRootNode(); - switch (tree->OperGet()) + // Operation has multiple statements which is not supported by SELECT + if (found) { - case GT_STORE_LCL_VAR: - { - // Only one per operation per block can be conditionally executed. - if (found) - { - return false; - } - - // Ensure the local has integer type. - if (!varTypeIsIntegralOrI(tree)) - { - return false; - } - -#ifndef TARGET_64BIT - // Disallow 64-bit operands on 32-bit targets as the backend currently cannot - // handle contained relops efficiently after decomposition. - if (varTypeIsLong(tree)) - { - return false; - } -#endif - GenTree* op1 = tree->AsLclVar()->Data(); - - // Ensure it won't cause any additional side effects. - if ((op1->gtFlags & (GTF_SIDE_EFFECT | GTF_ORDER_SIDEEFF)) != 0) - { - return false; - } - - // Ensure the source isn't a phi. - if (op1->OperIs(GT_PHI)) - { - return false; - } - - // Evaluating unconditionally effectively has the same effect as reordering - // with the condition (for example, the condition could be an explicit bounds - // check and the operand could read an array element). Disallow this except - // for some common cases that we know are always side effect free. - if (((m_cond->gtFlags & GTF_ORDER_SIDEEFF) != 0) && !op1->IsInvariant() && !op1->OperIsLocal()) - { - return false; - } - - found = true; - foundOperation->block = block; - foundOperation->stmt = stmt; - foundOperation->node = tree; - break; - } - - case GT_RETURN: - { - // GT_SWIFT_ERROR_RET not supported - GenTree* const retVal = tree->gtGetOp1(); - - // Only allow RETURNs if else conversion is being used. - if (!m_doElseConversion) - { - return false; - } - - // Only one per operation per block can be conditionally executed. - if (found || retVal == nullptr) - { - return false; - } + return false; + } - // Ensure the operation has integer type. - if (!varTypeIsIntegralOrI(tree)) - { - return false; - } + // Ensure the operation has integer type. + if (!varTypeIsIntegralOrI(tree)) + { + return false; + } #ifndef TARGET_64BIT - // Disallow 64-bit operands on 32-bit targets as the backend currently cannot - // handle contained relops efficiently after decomposition. - if (varTypeIsLong(tree)) - { - return false; - } + // Disallow 64-bit operands on 32-bit targets as the backend currently cannot + // handle contained relops efficiently after decomposition. + if (varTypeIsLong(tree)) + { + return false; + } #endif - // Ensure it won't cause any additional side effects. - if ((retVal->gtFlags & (GTF_SIDE_EFFECT | GTF_ORDER_SIDEEFF)) != 0) - { - return false; - } - - // Evaluating unconditionally effectively has the same effect as reordering - // with the condition (for example, the condition could be an explicit bounds - // check and the operand could read an array element). Disallow this except - // for some common cases that we know are always side effect free. - if (((m_cond->gtFlags & GTF_ORDER_SIDEEFF) != 0) && !retVal->IsInvariant() && - !retVal->OperIsLocal()) - { - return false; - } + GenTree* op1 = tree->gtGetOp1(); - found = true; - foundOperation->block = block; - foundOperation->stmt = stmt; - foundOperation->node = tree; - break; - } - - // These do not need conditional execution. - case GT_NOP: - break; + // Ensure it won't cause any additional side effects. + if ((op1->gtFlags & (GTF_SIDE_EFFECT | GTF_ORDER_SIDEEFF)) != 0) + { + return false; + } - // Cannot optimise this block. - default: - return false; + // Evaluating unconditionally effectively has the same effect as reordering + // with the condition (for example, the condition could be an explicit bounds + // check and the operand could read an array element). Disallow this except + // for some common cases that we know are always side effect free. + if (((m_cond->gtFlags & GTF_ORDER_SIDEEFF) != 0) && !op1->IsInvariant() && !op1->OperIsLocal()) + { + return false; } + + found = true; + foundOperation->block = block; + foundOperation->stmt = stmt; + foundOperation->node = tree; + } + else if (!tree->OperIs(GT_NOP)) + { + // Cannot optimise this block. + return false; } } + return found; } @@ -365,22 +271,15 @@ void OptIfConversionDsc::IfConvertDump() { m_compiler->fgDumpBlock(m_startBlock); - bool beforeTransformation = m_startBlock->KindIs(BBJ_COND); - if (beforeTransformation) + // Then & Else only exist before the transformation + if (m_startBlock->KindIs(BBJ_COND)) { - // Dump all Then blocks - for (BasicBlock* bb = m_startBlock->GetFalseTarget(); bb != m_finalBlock; bb = bb->GetUniqueSucc()) - { - m_compiler->fgDumpBlock(bb); - } - + JITDUMP("\nThen case:"); + m_compiler->fgDumpStmtTree(m_thenOperation.block, m_thenOperation.stmt); if (m_doElseConversion) { - // Dump all Else blocks - for (BasicBlock* bb = m_startBlock->GetTrueTarget(); bb != m_finalBlock; bb = bb->GetUniqueSucc()) - { - m_compiler->fgDumpBlock(bb); - } + JITDUMP("\nElse case:"); + m_compiler->fgDumpStmtTree(m_elseOperation.block, m_elseOperation.stmt); } } } @@ -438,9 +337,6 @@ void OptIfConversionDsc::IfConvertDump() // the existing value of local var 0 is used: // // ------------ BB03 [009..00D) -> BB05 (always), preds={BB02} succs={BB05} -// STMT00004 -// * NOP void -// // STMT00005 // * STORE_LCL_VAR int V00 arg0 // \--* SELECT int @@ -450,9 +346,6 @@ void OptIfConversionDsc::IfConvertDump() // +--* CNS_INT int 5 $47 // \--* LCL_VAR int V00 // -// ------------ BB04 [00D..010), preds={} succs={BB05} -// -// // Example of simple if conversion with an else condition // // This is similar to the simple if conversion above, but with an else statement @@ -479,9 +372,6 @@ void OptIfConversionDsc::IfConvertDump() // Again this is squashed into a single block, with the SELECT node handling both cases. // // ------------ BB03 [009..00D) -> BB05 (always), preds={BB02} succs={BB05} -// STMT00004 -// * NOP void -// // STMT00005 // * STORE_LCL_VAR int V00 arg0 // \--* SELECT int @@ -491,12 +381,6 @@ void OptIfConversionDsc::IfConvertDump() // +--* CNS_INT int 5 $47 // +--* CNS_INT int 9 $48 // -// STMT00006 -// * NOP void -// -// ------------ BB04 [00D..010), preds={} succs={BB06} -// ------------ BB05 [00D..010), preds={} succs={BB06} -// // Alternatively, an if conversion with an else condition may use RETURNs. // return (x < 7) ? 5 : 9; // @@ -507,22 +391,19 @@ void OptIfConversionDsc::IfConvertDump() // +--* LCL_VAR int V02 // \--* CNS_INT int 7 $46 // -// ------------ BB04 [00D..010), preds={BB03} succs={BB06} +// ------------ BB04 [00D..010) (return), preds={BB03} succs={} // STMT00005 // * RETURN int $VN.Void // +--* CNS_INT int 5 $41 // -// ------------ BB05 [00D..010), preds={BB03} succs={BB06} +// ------------ BB05 [00D..010) (return), preds={BB03} succs={} // STMT00006 // * RETURN int $VN.Void // +--* CNS_INT int 9 $43 // // becomes: // -// ------------ BB03 [009..00D) -> BB05 (always), preds={BB02} succs={BB05} -// STMT00004 -// * NOP void -// +// ------------ BB03 [009..00D) (return), preds={BB02} succs={} // STMT00005 // * RETURN int $VN.Void // \--* SELECT int @@ -532,12 +413,6 @@ void OptIfConversionDsc::IfConvertDump() // +--* CNS_INT int 5 $41 // +--* CNS_INT int 9 $43 // -// STMT00006 -// * NOP void -// -// ------------ BB04 [00D..010), preds={} succs={BB06} -// ------------ BB05 [00D..010), preds={} succs={BB06} -// bool OptIfConversionDsc::optIfConvert(int* pReachabilityBudget) { if ((*pReachabilityBudget) <= 0) @@ -545,68 +420,35 @@ bool OptIfConversionDsc::optIfConvert(int* pReachabilityBudget) return false; } - // Does the block end by branching via a JTRUE after a compare? - if (!m_startBlock->KindIs(BBJ_COND) || (m_startBlock->NumSucc() != 2)) + if (m_startBlock->firstStmt() == nullptr) { return false; } - // Verify the test block ends with a condition that we can manipulate. GenTree* last = m_startBlock->lastStmt()->GetRootNode(); - noway_assert(last->OperIs(GT_JTRUE)); - m_cond = last->gtGetOp1(); - if (!m_cond->OperIsCompare()) - { - return false; - } - - // Look for valid flow of Then and Else blocks. - IfConvertFindFlow(); - if (!m_flowFound) + if (!last->OperIs(GT_JTRUE)) { return false; } + + m_cond = last->gtGetOp1(); + assert(m_cond->OperIsCompare()); - // Check the Then and Else blocks have a single operation each. - if (!IfConvertCheckStmts(m_startBlock->GetFalseTarget(), &m_thenOperation)) + if (!IfConvertCheck()) { return false; } - assert(m_thenOperation.node->OperIs(GT_STORE_LCL_VAR, GT_RETURN)); - if (m_doElseConversion) - { - if (!IfConvertCheckStmts(m_startBlock->GetTrueTarget(), &m_elseOperation)) - { - return false; - } - - // Both operations must be the same node type. - if (m_thenOperation.node->OperGet() != m_elseOperation.node->OperGet()) - { - return false; - } - - // Currently can only support Else Store Blocks that have the same destination as the Then block. - if (m_thenOperation.node->OperIs(GT_STORE_LCL_VAR)) - { - unsigned lclNumThen = m_thenOperation.node->AsLclVarCommon()->GetLclNum(); - unsigned lclNumElse = m_elseOperation.node->AsLclVarCommon()->GetLclNum(); - if (lclNumThen != lclNumElse) - { - return false; - } - } - } #ifdef DEBUG if (m_compiler->verbose) { - JITDUMP("\nConditionally executing " FMT_BB, m_thenOperation.block->bbNum); + JITDUMP("JTRUE block is " FMT_BB ". ", m_startBlock->bbNum); + JITDUMP("Using " FMT_STMT " (Then) ", m_thenOperation.stmt->GetID()); if (m_doElseConversion) { - JITDUMP(" and " FMT_BB, m_elseOperation.block->bbNum); + JITDUMP("and " FMT_STMT " (Else) ", m_elseOperation.stmt->GetID()); } - JITDUMP(" inside " FMT_BB "\n", m_startBlock->bbNum); + JITDUMP("in the conversion.\n"); IfConvertDump(); } #endif @@ -622,7 +464,7 @@ bool OptIfConversionDsc::optIfConvert(int* pReachabilityBudget) { thenCost = m_thenOperation.node->AsLclVar()->Data()->GetCostEx() + (m_compiler->gtIsLikelyRegVar(m_thenOperation.node) ? 0 : 2); - if (m_doElseConversion) + if (m_doElseConversion && m_elseOperation.block != m_startBlock) { elseCost = m_elseOperation.node->AsLclVar()->Data()->GetCostEx() + (m_compiler->gtIsLikelyRegVar(m_elseOperation.node) ? 0 : 2); @@ -729,6 +571,10 @@ bool OptIfConversionDsc::optIfConvert(int* pReachabilityBudget) m_compiler->fgSetStmtSeq(m_thenOperation.stmt); // Replace JTRUE with STORE(SELECT)/RETURN(SELECT) statement + if (m_doElseConversion && m_elseOperation.block == m_startBlock) + { + m_compiler->fgRemoveStmt(m_startBlock, m_elseOperation.stmt); + } m_compiler->fgInsertStmtBefore(m_startBlock, m_startBlock->lastStmt(), m_thenOperation.stmt); m_compiler->fgRemoveStmt(m_startBlock, m_startBlock->lastStmt()); m_thenOperation.block->SetFirstStmt(nullptr); @@ -745,28 +591,21 @@ bool OptIfConversionDsc::optIfConvert(int* pReachabilityBudget) else { FlowEdge* newEdge = - m_doElseConversion ? m_compiler->fgAddRefPred(m_finalBlock, m_startBlock) : m_startBlock->GetTrueEdge(); + (m_doElseConversion && m_elseOperation.block != m_startBlock) ? m_compiler->fgAddRefPred(m_finalBlock, m_startBlock) : m_startBlock->GetTrueEdge(); m_startBlock->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); } assert(m_startBlock->GetUniqueSucc() == m_finalBlock); - // Remove all Then/Else blocks - auto removeBlocks = [&](BasicBlock* start) { - m_compiler->fgRemoveAllRefPreds(start, m_startBlock); + // Remove Then/Else block + auto removeBlock = [&](BasicBlock* start) { start->bbWeight = BB_ZERO_WEIGHT; - assert(start->bbPreds == nullptr); - - for (BasicBlock* bb = start; bb != m_finalBlock;) - { - BasicBlock* next = bb->GetUniqueSucc(); - m_compiler->fgRemoveBlock(bb, true); - bb = next; - } + m_compiler->fgRemoveAllRefPreds(start, m_startBlock); + m_compiler->fgRemoveBlock(start, true); }; - removeBlocks(falseBb); - if (m_doElseConversion) + removeBlock(falseBb); + if (m_doElseConversion && m_elseOperation.block != m_startBlock) { - removeBlocks(trueBb); + removeBlock(trueBb); } #ifdef DEBUG