diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 3758c56da45f53..75ec6bce3cf5bd 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -4835,7 +4835,8 @@ class Compiler unsigned methAttr, CORINFO_CONTEXT_HANDLE exactContextHnd, InlineCandidateInfo** ppInlineCandidateInfo, - InlineResult* inlineResult); + InlineResult* inlineResult, + UINT8 candidateId); void impInlineRecordArgInfo(InlineInfo* pInlineInfo, GenTree* curArgVal, @@ -4860,9 +4861,10 @@ class Compiler bool exactContextNeedsRuntimeLookup, CORINFO_CALL_INFO* callInfo); - void impMarkInlineCandidateHelper(GenTreeCall* call, + bool impMarkInlineCandidateHelper(GenTreeCall* call, CORINFO_CONTEXT_HANDLE exactContextHnd, bool exactContextNeedsRuntimeLookup, + UINT8 candidateIndex, CORINFO_CALL_INFO* callInfo); bool impTailCallRetTypeCompatible(bool allowWidening, @@ -7377,8 +7379,6 @@ class Compiler void addGuardedDevirtualizationCandidate(GenTreeCall* call, CORINFO_METHOD_HANDLE methodHandle, CORINFO_CLASS_HANDLE classHandle, - unsigned methodAttr, - unsigned classAttr, unsigned likelihood); bool doesMethodHaveExpRuntimeLookup() diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 9b2e4080a62a2d..f3677a504d4f0f 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -888,7 +888,8 @@ void Compiler::fgInvokeInlineeCompiler(GenTreeCall* call, InlineResult* inlineRe inlineInfo.hasSIMDTypeArgLocalOrReturn = false; #endif // FEATURE_SIMD - InlineCandidateInfo* inlineCandidateInfo = call->gtInlineCandidateInfo; + assert(call->GetGDVCandidatesCount() <= 1); + InlineCandidateInfo* inlineCandidateInfo = call->GetInlineCandidateInfo(); noway_assert(inlineCandidateInfo); // Store the link to inlineCandidateInfo into inlineInfo inlineInfo.inlineCandidateInfo = inlineCandidateInfo; @@ -1430,10 +1431,10 @@ void Compiler::fgInsertInlineeBlocks(InlineInfo* pInlineInfo) // but may still be referenced from a GT_RET_EXPR node. We will replace GT_RET_EXPR node // in fgUpdateInlineReturnExpressionPlaceHolder. At that time we will also update the flags // on the basic block of GT_RET_EXPR node. - if (iciCall->gtInlineCandidateInfo->retExpr->OperGet() == GT_RET_EXPR) + if (iciCall->GetInlineCandidateInfo()->retExpr->OperGet() == GT_RET_EXPR) { // Save the basic block flags from the retExpr basic block. - iciCall->gtInlineCandidateInfo->retExpr->AsRetExpr()->bbFlags = pInlineInfo->retBB->bbFlags; + iciCall->GetInlineCandidateInfo()->retExpr->AsRetExpr()->bbFlags = pInlineInfo->retBB->bbFlags; } iciCall->ReplaceWith(pInlineInfo->retExpr, this); } diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index a9f5402b50852c..46c80956230a24 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -1043,6 +1043,76 @@ bool GenTreeCall::IsHelperCall(Compiler* compiler, unsigned helper) const return IsHelperCall(compiler->eeFindHelper(helper)); } +//------------------------------------------------------------------------- +// SetSingleInlineCandidate: Sets (copies) inline info to the current call. +// +// Arguments: +// compiler - the compiler instance for allocations +// info - inline candidate info to set +// +// Return Value: +// Returns pointer to the copied inline info +// +InlineCandidateInfo* GenTreeCall::SetSingleInlineCandidate(Compiler* comp, const InlineCandidateInfo* info) +{ + assert(info != nullptr); + assert(GetGDVCandidatesCount() == 0); + gtInlineCandidateInfo = new (comp, CMK_Inlining) InlineCandidateInfo[1]{*info}; + return gtInlineCandidateInfo; +} + +//------------------------------------------------------------------------- +// AddGDVInlineCandidate: Adds (copies) given inline candidate info to the list +// of GDV candidates. +// +// Arguments: +// compiler - the compiler instance for allocations +// info - inline candidate info to add +// +// Return Value: +// Returns pointer to the copied inline info +// +InlineCandidateInfo* GenTreeCall::AddGDVInlineCandidate(Compiler* comp, const InlineCandidateInfo* info) +{ + assert(IsGuardedDevirtualizationCandidate()); + const UINT8 maxCandidates = (UINT8)JitConfig.JitGuardedDevirtualizationCheckCount(); + if (GetGDVCandidatesCount() == 0) + { + // In >90% cases we deal with monomorphic calls so let's avoid allocating too much. + SetSingleInlineCandidate(comp, info); + SetGDVCandidatesCount(1); + } + else if (GetGDVCandidatesCount() == 1) + { + // Non-monomorphic case - re-alloc + const InlineCandidateInfo firstInfo = gtInlineCandidateInfo[0]; + gtInlineCandidateInfo = new (comp, CMK_Inlining) InlineCandidateInfo[maxCandidates]{firstInfo, *info}; + SetGDVCandidatesCount(2); + } + else + { + gtInlineCandidateInfo[GetGDVCandidatesCount()] = *info; + SetGDVCandidatesCount(GetGDVCandidatesCount() + 1); + } + assert(GetGDVCandidatesCount() <= maxCandidates); + return GetInlineCandidateInfo(GetGDVCandidatesCount() - 1); +} + +//------------------------------------------------------------------------- +// GetInlineCandidateInfo: Returns an inline candidate for given index. +// +// Arguments: +// index - 0 or index of a candidate for GDV +// +// Return Value: +// Returns an inline candidate for given index. +// +InlineCandidateInfo* GenTreeCall::GetInlineCandidateInfo(UINT8 index) const +{ + assert(max(1, GetGDVCandidatesCount()) > index); + return gtInlineCandidateInfo + index; +} + //------------------------------------------------------------------------ // GenTreeCall::ReplaceCallOperand: // Replaces a given operand to a call node and updates the call @@ -6415,6 +6485,9 @@ GenTreeCall* Compiler::gtNewCallNode( node->gtRetClsHnd = nullptr; node->gtControlExpr = nullptr; node->gtCallMoreFlags = GTF_CALL_M_EMPTY; + node->gtCallLateArgs = nullptr; + node->gtReturnType = type; + node->SetGDVCandidatesCount(0); if (callType == CT_INDIRECT) { @@ -6422,10 +6495,8 @@ GenTreeCall* Compiler::gtNewCallNode( } else { - node->gtInlineCandidateInfo = nullptr; + node->ClearInlineInfo(); } - node->gtCallLateArgs = nullptr; - node->gtReturnType = type; #ifdef FEATURE_READYTORUN node->gtEntryPoint.addr = nullptr; @@ -8308,8 +8379,8 @@ GenTreeCall* Compiler::gtCloneExprCallHelper(GenTreeCall* tree, } else { - copy->gtCallMethHnd = tree->gtCallMethHnd; - copy->gtInlineCandidateInfo = nullptr; + copy->gtCallMethHnd = tree->gtCallMethHnd; + copy->ClearInlineInfo(); } if (tree->fgArgInfo) @@ -12122,10 +12193,10 @@ void Compiler::gtDispTree(GenTree* tree, printf(" (FramesRoot last use)"); } - if (((call->gtFlags & GTF_CALL_INLINE_CANDIDATE) != 0) && (call->gtInlineCandidateInfo != nullptr) && - (call->gtInlineCandidateInfo->exactContextHnd != nullptr)) + if (((call->gtFlags & GTF_CALL_INLINE_CANDIDATE) != 0) && (call->GetInlineCandidateInfo() != nullptr) && + (call->GetInlineCandidateInfo()->exactContextHnd != nullptr)) { - printf(" (exactContextHnd=0x%p)", dspPtr(call->gtInlineCandidateInfo->exactContextHnd)); + printf(" (exactContextHnd=0x%p)", dspPtr(call->GetInlineCandidateInfo()->exactContextHnd)); } gtDispCommonEndLine(tree); @@ -17774,7 +17845,7 @@ CORINFO_CLASS_HANDLE Compiler::gtGetClassHandle(GenTree* tree, bool* pIsExact, b { // For inline candidates, we've already cached the return // type class handle in the inline info. - InlineCandidateInfo* inlInfo = call->gtInlineCandidateInfo; + InlineCandidateInfo* inlInfo = call->GetInlineCandidateInfo(); assert(inlInfo != nullptr); // Grab it as our first cut at a return type. diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 53f73dbf7d1906..3818118fd094f0 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -4710,6 +4710,7 @@ struct GenTreeCall final : public GenTree void ClearGuardedDevirtualizationCandidate() { gtCallMoreFlags &= ~GTF_CALL_M_GUARDED_DEVIRT; + SetGDVCandidatesCount(0); } void SetGuardedDevirtualizationCandidate() @@ -4765,14 +4766,25 @@ struct GenTreeCall final : public GenTree // only used for CALLI unmanaged calls (CT_INDIRECT) GenTree* gtCallCookie; // gtInlineCandidateInfo is only used when inlining methods - InlineCandidateInfo* gtInlineCandidateInfo; - GuardedDevirtualizationCandidateInfo* gtGuardedDevirtualizationCandidateInfo; - ClassProfileCandidateInfo* gtClassProfileCandidateInfo; - void* gtStubCallStubAddr; // GTF_CALL_VIRT_STUB - these are never inlined + InlineCandidateInfo* gtInlineCandidateInfo; + ClassProfileCandidateInfo* gtClassProfileCandidateInfo; + void* gtStubCallStubAddr; // GTF_CALL_VIRT_STUB - these are never inlined CORINFO_GENERIC_HANDLE compileTimeHelperArgumentHandle; // Used to track type handle argument of dynamic helpers void* gtDirectCallAddress; // Used to pass direct call address between lower and codegen }; + void ClearInlineInfo() + { + SetGDVCandidatesCount(0); + gtInlineCandidateInfo = nullptr; + } + + InlineCandidateInfo* SetSingleInlineCandidate(Compiler* comp, const InlineCandidateInfo* info); + + InlineCandidateInfo* AddGDVInlineCandidate(Compiler* comp, const InlineCandidateInfo* info); + + InlineCandidateInfo* GetInlineCandidateInfo(UINT8 index = 0) const; + // expression evaluated after args are placed which determines the control target GenTree* gtControlExpr; @@ -4795,6 +4807,18 @@ struct GenTreeCall final : public GenTree IL_OFFSET gtRawILOffset; #endif // defined(DEBUG) || defined(INLINE_DATA) + UINT8 gtGDVCandidatesCount; + + UINT8 GetGDVCandidatesCount() const + { + return gtGDVCandidatesCount; + } + + void SetGDVCandidatesCount(UINT8 count) + { + gtGDVCandidatesCount = count; + } + bool IsHelperCall() const { return gtCallType == CT_HELPER; diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 6601bdcc56c104..54bc3cc2299b18 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -2528,7 +2528,7 @@ bool Compiler::impSpillStackEntry(unsigned level, { JITDUMP("\n*** see V%02u = GT_RET_EXPR, noting temp\n", tnum); GenTree* call = tree->AsRetExpr()->gtInlineCandidate; - InlineCandidateInfo* ici = call->AsCall()->gtInlineCandidateInfo; + InlineCandidateInfo* ici = call->AsCall()->GetInlineCandidateInfo(); ici->preexistingSpillTemp = tnum; } } @@ -6912,8 +6912,8 @@ void Compiler::impImportAndPushBox(CORINFO_RESOLVED_TOKEN* pResolvedToken) if (call->IsVirtualStub()) { JITDUMP("Restoring stub addr %p from guarded devirt candidate info\n", - dspPtr(call->gtGuardedDevirtualizationCandidateInfo->stubAddr)); - call->gtStubCallStubAddr = call->gtGuardedDevirtualizationCandidateInfo->stubAddr; + dspPtr(call->GetInlineCandidateInfo()->stubAddr)); + call->gtStubCallStubAddr = call->GetInlineCandidateInfo()->stubAddr; } } } @@ -9557,14 +9557,14 @@ var_types Compiler::impImportCall(OPCODE opcode, // Make the call its own tree (spill the stack if needed). impAppendTree(call, (unsigned)CHECK_SPILL_ALL, impCurStmtOffs); - // TODO: Still using the widened type. - GenTree* retExpr = gtNewInlineCandidateReturnExpr(call, genActualType(callRetTyp), compCurBB->bbFlags); - - // Link the retExpr to the call so if necessary we can manipulate it later. - origCall->gtInlineCandidateInfo->retExpr = retExpr; - // Propagate retExpr as the placeholder for the call. - call = retExpr; + // TODO: Still using the widened type. + call = gtNewInlineCandidateReturnExpr(call, genActualType(callRetTyp), compCurBB->bbFlags); + for (UINT8 i = 0; i < max(1, origCall->GetGDVCandidatesCount()); i++) + { + // Save link to retExpr to all gdv candidates just in case. + origCall->GetInlineCandidateInfo(i)->retExpr = call; + } } else { @@ -19354,7 +19354,8 @@ void Compiler::impCheckCanInline(GenTreeCall* call, unsigned methAttr, CORINFO_CONTEXT_HANDLE exactContextHnd, InlineCandidateInfo** ppInlineCandidateInfo, - InlineResult* inlineResult) + InlineResult* inlineResult, + UINT8 candidateId) { // Either EE or JIT might throw exceptions below. // If that happens, just don't inline the method. @@ -19368,6 +19369,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call, CORINFO_CONTEXT_HANDLE exactContextHnd; InlineResult* result; InlineCandidateInfo** ppInlineCandidateInfo; + UINT8 candidateId; } param; memset(¶m, 0, sizeof(param)); @@ -19378,6 +19380,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call, param.exactContextHnd = (exactContextHnd != nullptr) ? exactContextHnd : MAKE_METHODCONTEXT(fncHandle); param.result = inlineResult; param.ppInlineCandidateInfo = ppInlineCandidateInfo; + param.candidateId = candidateId; bool success = eeRunWithErrorTrap( [](Param* pParam) { @@ -19503,7 +19506,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call, if (pParam->call->IsGuardedDevirtualizationCandidate()) { - pInfo = pParam->call->gtInlineCandidateInfo; + pInfo = pParam->call->GetInlineCandidateInfo(pParam->candidateId); } else { @@ -20527,8 +20530,26 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode, { GenTreeCall* call = callNode->AsCall(); - // Do the actual evaluation - impMarkInlineCandidateHelper(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo); + if (call->IsGuardedDevirtualizationCandidate()) + { + UINT8 inlineableGdvCount = 0; + for (UINT8 candidateId = 0; candidateId < call->GetGDVCandidatesCount(); candidateId++) + { + if (!impMarkInlineCandidateHelper(call, exactContextHnd, exactContextNeedsRuntimeLookup, candidateId, + callInfo)) + { + // TODO: remove only non-inlineable candidates here. + // Currently we just ignore all candidates after first non-inlineable one + break; + } + inlineableGdvCount++; + } + call->SetGDVCandidatesCount(inlineableGdvCount); + } + else + { + impMarkInlineCandidateHelper(call, exactContextHnd, exactContextNeedsRuntimeLookup, 0, callInfo); + } // If this call is an inline candidate or is not a guarded devirtualization // candidate, we're done. @@ -20554,8 +20575,8 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode, if (call->IsVirtualStub()) { JITDUMP("Restoring stub addr %p from guarded devirt candidate info\n", - dspPtr(call->gtGuardedDevirtualizationCandidateInfo->stubAddr)); - call->gtStubCallStubAddr = call->gtGuardedDevirtualizationCandidateInfo->stubAddr; + dspPtr(call->GetInlineCandidateInfo()->stubAddr)); + call->gtStubCallStubAddr = call->GetInlineCandidateInfo()->stubAddr; } } @@ -20569,6 +20590,9 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode, // exactContextNeedsRuntimeLookup -- true if context required runtime lookup // callInfo -- call info from VM // +// Returns: +// True if the given call is inlineable. +// // Notes: // If callNode is an inline candidate, this method sets the flag // GTF_CALL_INLINE_CANDIDATE, and ensures that helper methods have @@ -20579,9 +20603,10 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode, // method may be marked as "noinline" to short-circuit any // future assessments of calls to this method. -void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, +bool Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, CORINFO_CONTEXT_HANDLE exactContextHnd, bool exactContextNeedsRuntimeLookup, + UINT8 candidateIndex, CORINFO_CALL_INFO* callInfo) { // Let the strategy know there's another call @@ -20596,7 +20621,7 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, * figure out why we did not set MAXOPT for this compile. */ assert(!compIsForInlining()); - return; + return false; } if (compIsForImportOnly()) @@ -20604,7 +20629,7 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, // Don't bother creating the inline candidate during verification. // Otherwise the call to info.compCompHnd->canInline will trigger a recursive verification // that leads to the creation of multiple instances of Compiler. - return; + return false; } InlineResult inlineResult(this, call, nullptr, "impMarkInlineCandidate"); @@ -20613,21 +20638,21 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, if (opts.compDbgCode) { inlineResult.NoteFatal(InlineObservation::CALLER_DEBUG_CODEGEN); - return; + return false; } // Don't inline if inlining into this method is disabled. if (impInlineRoot()->m_inlineStrategy->IsInliningDisabled()) { inlineResult.NoteFatal(InlineObservation::CALLER_IS_JIT_NOINLINE); - return; + return false; } // Don't inline into callers that use the NextCallReturnAddress intrinsic. if (info.compHasNextCallRetAddr) { inlineResult.NoteFatal(InlineObservation::CALLER_USES_NEXT_CALL_RET_ADDR); - return; + return false; } // Inlining candidate determination needs to honor only IL tail prefix. @@ -20635,7 +20660,7 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, if (call->IsTailPrefixedCall()) { inlineResult.NoteFatal(InlineObservation::CALLSITE_EXPLICIT_TAIL_PREFIX); - return; + return false; } // Tail recursion elimination takes precedence over inlining. @@ -20645,7 +20670,7 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, if (gtIsRecursiveCall(call) && call->IsImplicitTailCall()) { inlineResult.NoteFatal(InlineObservation::CALLSITE_IMPLICIT_REC_TAIL_CALL); - return; + return false; } if (call->IsVirtual()) @@ -20655,7 +20680,7 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, if (!call->IsGuardedDevirtualizationCandidate()) { inlineResult.NoteFatal(InlineObservation::CALLSITE_IS_NOT_DIRECT); - return; + return false; } } @@ -20665,16 +20690,19 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, { assert(!call->IsGuardedDevirtualizationCandidate()); inlineResult.NoteFatal(InlineObservation::CALLSITE_IS_CALL_TO_HELPER); - return; + return false; } /* Ignore indirect calls */ if (call->gtCallType == CT_INDIRECT) { inlineResult.NoteFatal(InlineObservation::CALLSITE_IS_NOT_DIRECT_MANAGED); - return; + return false; } + assert(max(1, call->GetGDVCandidatesCount()) > candidateIndex); + InlineCandidateInfo* inlineInfo = call->GetInlineCandidateInfo(); + /* I removed the check for BBJ_THROW. BBJ_THROW is usually marked as rarely run. This more or less * restricts the inliner to non-expanding inlines. I removed the check to allow for non-expanding * inlining in throw blocks. I should consider the same thing for catch and filter regions. */ @@ -20684,13 +20712,14 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, if (call->IsGuardedDevirtualizationCandidate()) { - if (call->gtGuardedDevirtualizationCandidateInfo->guardedMethodUnboxedEntryHandle != nullptr) + inlineInfo = call->GetInlineCandidateInfo(candidateIndex); + if (inlineInfo->guardedMethodUnboxedEntryHandle != nullptr) { - fncHandle = call->gtGuardedDevirtualizationCandidateInfo->guardedMethodUnboxedEntryHandle; + fncHandle = inlineInfo->guardedMethodUnboxedEntryHandle; } else { - fncHandle = call->gtGuardedDevirtualizationCandidateInfo->guardedMethodHandle; + fncHandle = inlineInfo->guardedMethodHandle; } methAttr = info.compCompHnd->getMethodAttribs(fncHandle); } @@ -20736,7 +20765,7 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, #endif inlineResult.NoteFatal(InlineObservation::CALLSITE_IS_WITHIN_CATCH); - return; + return false; } if (bbInFilterILRange(compCurBB)) @@ -20749,7 +20778,7 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, #endif inlineResult.NoteFatal(InlineObservation::CALLSITE_IS_WITHIN_FILTER); - return; + return false; } } @@ -20758,7 +20787,7 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, if (methAttr & CORINFO_FLG_DONT_INLINE) { inlineResult.NoteFatal(InlineObservation::CALLEE_IS_NOINLINE); - return; + return false; } /* Cannot inline synchronized methods */ @@ -20766,7 +20795,7 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, if (methAttr & CORINFO_FLG_SYNCH) { inlineResult.NoteFatal(InlineObservation::CALLEE_IS_SYNCHRONIZED); - return; + return false; } /* Check legality of PInvoke callsite (for inlining of marshalling code) */ @@ -20778,25 +20807,24 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, if (!impCanPInvokeInlineCallSite(block)) { inlineResult.NoteFatal(InlineObservation::CALLSITE_PINVOKE_EH); - return; + return false; } } InlineCandidateInfo* inlineCandidateInfo = nullptr; - impCheckCanInline(call, fncHandle, methAttr, exactContextHnd, &inlineCandidateInfo, &inlineResult); + impCheckCanInline(call, fncHandle, methAttr, exactContextHnd, &inlineCandidateInfo, &inlineResult, candidateIndex); if (inlineResult.IsFailure()) { - return; + return false; } // The old value should be null OR this call should be a guarded devirtualization candidate. - assert((call->gtInlineCandidateInfo == nullptr) || call->IsGuardedDevirtualizationCandidate()); + assert((inlineInfo == nullptr) || call->IsGuardedDevirtualizationCandidate()); // The new value should not be null. assert(inlineCandidateInfo != nullptr); inlineCandidateInfo->exactContextNeedsRuntimeLookup = exactContextNeedsRuntimeLookup; - call->gtInlineCandidateInfo = inlineCandidateInfo; // If we're in an inlinee compiler, and have a return spill temp, and this inline candidate // is also a tail call candidate, it can use the same return spill temp. @@ -20809,6 +20837,17 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, inlineCandidateInfo->preexistingSpillTemp); } + if (call->IsGuardedDevirtualizationCandidate()) + { + assert(call->GetInlineCandidateInfo() != nullptr); + *call->GetInlineCandidateInfo(candidateIndex) = *inlineCandidateInfo; + } + else + { + call->SetSingleInlineCandidate(this, inlineCandidateInfo); + } + inlineCandidateInfo = nullptr; + // Mark the call node as inline candidate. call->gtFlags |= GTF_CALL_INLINE_CANDIDATE; @@ -20818,6 +20857,7 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, // Since we're not actually inlining yet, and this call site is // still just an inline candidate, there's nothing to report. inlineResult.SetReported(); + return true; } /******************************************************************************/ @@ -21334,7 +21374,7 @@ void Compiler::impDevirtualizeCall(GenTreeCall* call, // Clear the inline candidate info (may be non-null since // it's a union field used for other things by virtual // stubs) - call->gtInlineCandidateInfo = nullptr; + call->ClearInlineInfo(); #if defined(DEBUG) if (verbose) @@ -21924,11 +21964,9 @@ void Compiler::considerGuardedDevirtualization( // See if there's a likely guess for the class. // - const unsigned likelihoodThreshold = isInterface ? 25 : 30; - unsigned likelihood = 0; - unsigned numberOfClasses = 0; - - CORINFO_CLASS_HANDLE likelyClass = NO_CLASS_HANDLE; + const unsigned primaryLikelihoodThreshold = isInterface ? 25 : 30; + const unsigned secondaryLikelihoodThreshold = isInterface ? 10 : 15; + unsigned numberOfClasses = 0; bool doRandomDevirt = false; @@ -21963,17 +22001,12 @@ void Compiler::considerGuardedDevirtualization( // For now we only use the most popular type - likelihood = likelyClasses[0].likelihood; - likelyClass = likelyClasses[0].clsHandle; - if (numberOfClasses < 1) { JITDUMP("No likely class, sorry\n"); return; } - assert(likelyClass != NO_CLASS_HANDLE); - // Print all likely classes JITDUMP("%s classes for %p (%s):\n", doRandomDevirt ? "Random" : "Likely", dspPtr(objClass), objClassName) for (UINT32 i = 0; i < numberOfClasses; i++) @@ -21982,44 +22015,60 @@ void Compiler::considerGuardedDevirtualization( eeGetClassName(likelyClasses[i].clsHandle), likelyClasses[i].likelihood); } - // Todo: a more advanced heuristic using likelihood, number of - // classes, and the profile count for this block. - // - // For now we will guess if the likelihood is at least 25%/30% (intfc/virt), as studies - // have shown this transformation should pay off even if we guess wrong sometimes. - // - if (likelihood < likelihoodThreshold) + assert(call->GetGDVCandidatesCount() == 0); + + for (UINT i = 0; i < numberOfClasses; i++) { - JITDUMP("Not guessing for class; likelihood is below %s call threshold %u\n", callKind, likelihoodThreshold); - return; - } + const UINT8 maxCandidates = (UINT8)JitConfig.JitGuardedDevirtualizationCheckCount(); + if (call->GetGDVCandidatesCount() >= maxCandidates) + { + JITDUMP("Reaching the JitGuardedDevirtualizationCheckCount=%u limit, other candidates are ignored.", + maxCandidates); + break; + } - // Figure out which method will be called. - // - CORINFO_DEVIRTUALIZATION_INFO dvInfo; - dvInfo.virtualMethod = baseMethod; - dvInfo.objClass = likelyClass; - dvInfo.context = *pContextHandle; - dvInfo.exactContext = *pContextHandle; - dvInfo.pResolvedTokenVirtualMethod = nullptr; + assert(likelyClasses[i].clsHandle != NO_CLASS_HANDLE); - const bool canResolve = info.compCompHnd->resolveVirtualMethod(&dvInfo); + // Todo: a more advanced heuristic using likelihood, number of + // classes, and the profile count for this block. + // + // For now we will guess if the likelihood is at least 25%/30% (intfc/virt), as studies + // have shown this transformation should pay off even if we guess wrong sometimes. + // And smaller likelihoods for secondary guesses - 10%/15% (intfc/virt). + // + const UINT32 currentThreshold = + call->GetGDVCandidatesCount() == 0 ? primaryLikelihoodThreshold : secondaryLikelihoodThreshold; + if ((likelyClasses[i].likelihood < currentThreshold)) + { + JITDUMP("Not guessing for class; likelihood is below %s call threshold %u\n", callKind, currentThreshold); + break; + } - if (!canResolve) - { - JITDUMP("Can't figure out which method would be invoked, sorry\n"); - return; - } + // Figure out which method will be called. + // + CORINFO_DEVIRTUALIZATION_INFO dvInfo = {}; + dvInfo.virtualMethod = baseMethod; + dvInfo.objClass = likelyClasses[i].clsHandle; + dvInfo.context = *pContextHandle; + dvInfo.exactContext = *pContextHandle; + dvInfo.pResolvedTokenVirtualMethod = nullptr; - CORINFO_METHOD_HANDLE likelyMethod = dvInfo.devirtualizedMethod; - JITDUMP("%s call would invoke method %s\n", callKind, eeGetMethodName(likelyMethod, nullptr)); + const bool canResolve = info.compCompHnd->resolveVirtualMethod(&dvInfo); - // Add this as a potential candidate. - // - uint32_t const likelyMethodAttribs = info.compCompHnd->getMethodAttribs(likelyMethod); - uint32_t const likelyClassAttribs = info.compCompHnd->getClassAttribs(likelyClass); - addGuardedDevirtualizationCandidate(call, likelyMethod, likelyClass, likelyMethodAttribs, likelyClassAttribs, - likelihood); + if (!canResolve) + { + JITDUMP("Can't figure out which method would be invoked, sorry\n"); + continue; + } + + CORINFO_METHOD_HANDLE likelyMethod = dvInfo.devirtualizedMethod; + JITDUMP("%s call would invoke method %s\n", callKind, eeGetMethodName(likelyMethod, nullptr)); + + // Add this as a potential candidate. + // + addGuardedDevirtualizationCandidate(call, likelyMethod, likelyClasses[i].clsHandle, + likelyClasses[i].likelihood); + } } //------------------------------------------------------------------------ @@ -22039,15 +22088,11 @@ void Compiler::considerGuardedDevirtualization( // call - potential guarded devirtualization candidate // methodHandle - method that will be invoked if the class test succeeds // classHandle - class that will be tested for at runtime -// methodAttr - attributes of the method -// classAttr - attributes of the class // likelihood - odds that this class is the class seen at runtime // void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call, CORINFO_METHOD_HANDLE methodHandle, CORINFO_CLASS_HANDLE classHandle, - unsigned methodAttr, - unsigned classAttr, unsigned likelihood) { // This transformation only makes sense for virtual calls @@ -22114,17 +22159,17 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call, // Gather some information for later. Note we actually allocate InlineCandidateInfo // here, as the devirtualized half of this call will likely become an inline candidate. // - GuardedDevirtualizationCandidateInfo* pInfo = new (this, CMK_Inlining) InlineCandidateInfo; + InlineCandidateInfo pInfo = {}; - pInfo->guardedMethodHandle = methodHandle; - pInfo->guardedMethodUnboxedEntryHandle = nullptr; - pInfo->guardedClassHandle = classHandle; - pInfo->likelihood = likelihood; - pInfo->requiresInstMethodTableArg = false; + pInfo.guardedMethodHandle = methodHandle; + pInfo.guardedMethodUnboxedEntryHandle = nullptr; + pInfo.guardedClassHandle = classHandle; + pInfo.likelihood = likelihood; + pInfo.requiresInstMethodTableArg = false; // If the guarded class is a value class, look for an unboxed entry point. // - if ((classAttr & CORINFO_FLG_VALUECLASS) != 0) + if (info.compCompHnd->isValueClass(classHandle)) { JITDUMP(" ... class is a value class, looking for unboxed entry\n"); bool requiresInstMethodTableArg = false; @@ -22134,8 +22179,8 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call, if (unboxedEntryMethodHandle != nullptr) { JITDUMP(" ... updating GDV candidate with unboxed entry info\n"); - pInfo->guardedMethodUnboxedEntryHandle = unboxedEntryMethodHandle; - pInfo->requiresInstMethodTableArg = requiresInstMethodTableArg; + pInfo.guardedMethodUnboxedEntryHandle = unboxedEntryMethodHandle; + pInfo.requiresInstMethodTableArg = requiresInstMethodTableArg; } } @@ -22144,14 +22189,14 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call, if (call->IsVirtualStub()) { JITDUMP("Saving stub addr %p in candidate info\n", dspPtr(call->gtStubCallStubAddr)); - pInfo->stubAddr = call->gtStubCallStubAddr; + pInfo.stubAddr = call->gtStubCallStubAddr; } else { - pInfo->stubAddr = nullptr; + pInfo.stubAddr = nullptr; } - call->gtGuardedDevirtualizationCandidateInfo = pInfo; + call->AddGDVInlineCandidate(this, &pInfo); } void Compiler::addExpRuntimeLookupCandidate(GenTreeCall* call) diff --git a/src/coreclr/jit/indirectcalltransformer.cpp b/src/coreclr/jit/indirectcalltransformer.cpp index 6be64a86b6ebbb..2863c7288211ed 100644 --- a/src/coreclr/jit/indirectcalltransformer.cpp +++ b/src/coreclr/jit/indirectcalltransformer.cpp @@ -209,8 +209,12 @@ class IndirectCallTransformer FixupRetExpr(); ClearFlag(); CreateRemainder(); - CreateCheck(); - CreateThen(); + assert(GetChecksCount() > 0); + for (UINT8 i = 0; i < GetChecksCount(); i++) + { + CreateCheck(i); + CreateThen(i); + } CreateElse(); RemoveOldStatement(); SetWeights(); @@ -233,7 +237,7 @@ class IndirectCallTransformer remainderBlock->bbFlags |= BBF_INTERNAL; } - virtual void CreateCheck() = 0; + virtual void CreateCheck(UINT8 checkIdx) = 0; //------------------------------------------------------------------------ // CreateAndInsertBasicBlock: ask compiler to create new basic block. @@ -252,8 +256,8 @@ class IndirectCallTransformer return block; } - virtual void CreateThen() = 0; - virtual void CreateElse() = 0; + virtual void CreateThen(UINT8 checkIdx) = 0; + virtual void CreateElse() = 0; //------------------------------------------------------------------------ // RemoveOldStatement: remove original stmt from current block. @@ -274,6 +278,14 @@ class IndirectCallTransformer elseBlock->inheritWeightPercentage(currBlock, 100 - likelihood); } + //------------------------------------------------------------------------ + // GetChecksCount: Get number of Check-Then nodes + // + virtual UINT8 GetChecksCount() + { + return 1; + } + //------------------------------------------------------------------------ // ChainFlow: link new blocks into correct cfg. // @@ -355,7 +367,7 @@ class IndirectCallTransformer //------------------------------------------------------------------------ // CreateCheck: create check block, that checks fat pointer bit set. // - virtual void CreateCheck() + virtual void CreateCheck(UINT8 checkIdx) { checkBlock = CreateAndInsertBasicBlock(BBJ_COND, currBlock); GenTree* fatPointerMask = new (compiler, GT_CNS_INT) GenTreeIntCon(TYP_I_IMPL, FAT_POINTER_MASK); @@ -372,7 +384,7 @@ class IndirectCallTransformer // CreateThen: create then block, that is executed if the check succeeds. // This simply executes the original call. // - virtual void CreateThen() + virtual void CreateThen(UINT8 checkIdx) { thenBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, checkBlock); Statement* copyOfOriginalStmt = compiler->gtCloneStmt(stmt); @@ -507,7 +519,7 @@ class IndirectCallTransformer //------------------------------------------------------------------------ // Run: transform the statement as described above. // - virtual void Run() + virtual void Run() override { origCall = GetCall(stmt); @@ -519,15 +531,36 @@ class IndirectCallTransformer if (!origCall->IsInlineCandidate()) { JITDUMP("*** %s Bailing on [%06u] -- not an inline candidate\n", Name(), compiler->dspTreeID(origCall)); - ClearFlag(); + origCall->ClearGuardedDevirtualizationCandidate(); return; } - likelihood = origCall->gtGuardedDevirtualizationCandidateInfo->likelihood; - assert((likelihood >= 0) && (likelihood <= 100)); - JITDUMP("Likelihood of correct guess is %u\n", likelihood); + assert(origCall->GetGDVCandidatesCount() > 0); + + // likelihood field is only used by ScoutForChainedGdv that only supports monomorphic calls for now, + // hence, we only take likelihood of the first candidate here + likelihood = origCall->GetInlineCandidateInfo(0)->likelihood; + + JITDUMP("\nLikelihoods of GDV candidates:"); + assert(likelihood <= 100); + for (UINT8 candidateId = 0; candidateId < origCall->GetGDVCandidatesCount(); candidateId++) + { + const UINT32 currentLikelihood = origCall->GetInlineCandidateInfo(candidateId)->likelihood; + JITDUMP("\n\tCandidate %u - %u%%", candidateId, currentLikelihood); + if (candidateId > 0) + { + const UINT32 prevLikelihood = origCall->GetInlineCandidateInfo(candidateId - 1)->likelihood; + assert(prevLikelihood >= currentLikelihood); + JITDUMP(" (relative likelihood %0.1f%%)", currentLikelihood * 100.0 / prevLikelihood); + } + } + JITDUMP("\n\n"); + + const bool multipleCandidates = origCall->GetGDVCandidatesCount() > 1; + const bool isChainedGdv = + ((origCall->gtCallMoreFlags & GTF_CALL_M_GUARDED_DEVIRT_CHAIN) != 0) && !multipleCandidates; - const bool isChainedGdv = (origCall->gtCallMoreFlags & GTF_CALL_M_GUARDED_DEVIRT_CHAIN) != 0; + // NOTE: calls with multiple GDV candidates don't support this opt yet. if (isChainedGdv) { @@ -535,19 +568,23 @@ class IndirectCallTransformer } Transform(); + origCall->ClearGuardedDevirtualizationCandidate(); if (isChainedGdv) { TransformForChainedGdv(); } - // Look ahead and see if there's another Gdv we might chain to this one. - // - ScoutForChainedGdv(); + if (!multipleCandidates) + { + // Look ahead and see if there's another Gdv we might chain to this one. + // + ScoutForChainedGdv(); + } } protected: - virtual const char* Name() + virtual const char* Name() override { return "GuardedDevirtualization"; } @@ -560,7 +597,7 @@ class IndirectCallTransformer // // Return Value: // call tree node pointer. - virtual GenTreeCall* GetCall(Statement* callStmt) + virtual GenTreeCall* GetCall(Statement* callStmt) override { GenTree* tree = callStmt->GetRootNode(); assert(tree->IsCall()); @@ -569,22 +606,33 @@ class IndirectCallTransformer } //------------------------------------------------------------------------ - // ClearFlag: clear guarded devirtualization candidate flag from the original call. + // ClearFlag: not used // - virtual void ClearFlag() + virtual void ClearFlag() override { - origCall->ClearGuardedDevirtualizationCandidate(); } //------------------------------------------------------------------------ // CreateCheck: create check block and check method table // - virtual void CreateCheck() + virtual void CreateCheck(UINT8 checkIdx) override { - // There's no need for a new block here. We can just append to currBlock. - // - checkBlock = currBlock; - checkBlock->bbJumpKind = BBJ_COND; + if (checkIdx == 0) + { + // There's no need for a new block here. We can just append to currBlock. + checkBlock = currBlock; + checkBlock->bbJumpKind = BBJ_COND; + checkBlock->inheritWeight(currBlock); + } + else + { + // Append to the previous "Then" block + BasicBlock* prevCheckBlock = checkBlock; + checkBlock = CreateAndInsertBasicBlock(BBJ_COND, thenBlock); + checkBlock->bbFlags |= currBlock->bbFlags & BBF_SPLIT_GAINED; + prevCheckBlock->bbJumpDest = checkBlock; + checkBlock->setBBProfileWeight(prevCheckBlock->bbWeight - thenBlock->bbWeight); + } // Fetch method table from object arg to call. GenTree* thisTree = compiler->gtCloneExpr(origCall->gtCallThisArg->GetNode()); @@ -615,10 +663,9 @@ class IndirectCallTransformer // Find target method table // - GenTree* methodTable = compiler->gtNewMethodTableLookup(thisTree); - GuardedDevirtualizationCandidateInfo* guardedInfo = origCall->gtGuardedDevirtualizationCandidateInfo; - CORINFO_CLASS_HANDLE clsHnd = guardedInfo->guardedClassHandle; - GenTree* targetMethodTable = compiler->gtNewIconEmbClsHndNode(clsHnd); + GenTree* methodTable = compiler->gtNewMethodTableLookup(thisTree); + CORINFO_CLASS_HANDLE clsHnd = origCall->GetInlineCandidateInfo(checkIdx)->guardedClassHandle; + GenTree* targetMethodTable = compiler->gtNewIconEmbClsHndNode(clsHnd); // Compare and jump to else (which does the indirect call) if NOT equal // @@ -631,7 +678,7 @@ class IndirectCallTransformer //------------------------------------------------------------------------ // FixupRetExpr: set up to repair return value placeholder from call // - virtual void FixupRetExpr() + virtual void FixupRetExpr() override { // If call returns a value, we need to copy it to a temp, and // bash the associated GT_RET_EXPR to refer to the temp instead @@ -639,7 +686,9 @@ class IndirectCallTransformer // // Note implicit by-ref returns should have already been converted // so any struct copy we induce here should be cheap. - InlineCandidateInfo* const inlineInfo = origCall->gtInlineCandidateInfo; + + assert(origCall->GetGDVCandidatesCount() > 0); + InlineCandidateInfo* const inlineInfo = origCall->GetInlineCandidateInfo(0); GenTree* const retExpr = inlineInfo->retExpr; // Sanity check the ret expr if non-null: it should refer to the original call. @@ -653,7 +702,11 @@ class IndirectCallTransformer // If there's a spill temp already associated with this inline candidate, // use that instead of allocating a new temp. // - returnTemp = inlineInfo->preexistingSpillTemp; + + if (returnTemp == BAD_VAR_NUM) + { + returnTemp = inlineInfo->preexistingSpillTemp; + } if (returnTemp != BAD_VAR_NUM) { @@ -719,15 +772,25 @@ class IndirectCallTransformer } } + //------------------------------------------------------------------------ + // GetChecksCount: Get number of Check-Then + // + virtual UINT8 GetChecksCount() override + { + return origCall->GetGDVCandidatesCount(); + } + //------------------------------------------------------------------------ // CreateThen: create then block with direct call to method // - virtual void CreateThen() + virtual void CreateThen(UINT8 checkIdx) override { thenBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, checkBlock); thenBlock->bbFlags |= currBlock->bbFlags & BBF_SPLIT_GAINED; + thenBlock->bbJumpDest = remainderBlock; + thenBlock->inheritWeightPercentage(currBlock, origCall->GetInlineCandidateInfo(checkIdx)->likelihood); - InlineCandidateInfo* inlineInfo = origCall->gtInlineCandidateInfo; + InlineCandidateInfo* inlineInfo = origCall->GetInlineCandidateInfo(checkIdx); CORINFO_CLASS_HANDLE clsHnd = inlineInfo->guardedClassHandle; // copy 'this' to temp with exact type. @@ -741,6 +804,7 @@ class IndirectCallTransformer GenTreeCall* call = compiler->gtCloneCandidateCall(origCall); call->gtCallThisArg = compiler->gtNewCallArgs(compiler->gtNewLclvNode(thisTemp, TYP_REF)); call->SetIsGuarded(); + call->ClearGuardedDevirtualizationCandidate(); JITDUMP("Direct call [%06u] in block " FMT_BB "\n", compiler->dspTreeID(call), thenBlock->bbNum); @@ -772,7 +836,9 @@ class IndirectCallTransformer "inlineable\n"); call->gtFlags &= ~GTF_CALL_INLINE_CANDIDATE; - call->gtInlineCandidateInfo = nullptr; + + if (!call->IsVirtualStub()) + call->ClearInlineInfo(); if (returnTemp != BAD_VAR_NUM) { @@ -796,7 +862,7 @@ class IndirectCallTransformer inlineInfo->clsHandle = compiler->info.compCompHnd->getMethodClass(methodHnd); inlineInfo->exactContextHnd = context; inlineInfo->preexistingSpillTemp = returnTemp; - call->gtInlineCandidateInfo = inlineInfo; + inlineInfo = call->SetSingleInlineCandidate(compiler, inlineInfo); // If there was a ret expr for this call, we need to create a new one // and append it just after the call. @@ -826,7 +892,7 @@ class IndirectCallTransformer //------------------------------------------------------------------------ // CreateElse: create else block. This executes the unaltered indirect call. // - virtual void CreateElse() + virtual void CreateElse() override { elseBlock = CreateAndInsertBasicBlock(BBJ_NONE, thenBlock); elseBlock->bbFlags |= currBlock->bbFlags & BBF_SPLIT_GAINED; @@ -844,16 +910,23 @@ class IndirectCallTransformer newStmt->SetRootNode(assign); } + UINT32 elseLikelihood = 100; + for (UINT8 i = 0; i < origCall->GetGDVCandidatesCount(); i++) + { + elseLikelihood -= origCall->GetInlineCandidateInfo(i)->likelihood; + } + elseBlock->inheritWeightPercentage(currBlock, elseLikelihood > 100 /*overflow*/ ? 0 : elseLikelihood); + // For stub calls, restore the stub address. For everything else, // null out the candidate info field. if (call->IsVirtualStub()) { - JITDUMP("Restoring stub addr %p from candidate info\n", call->gtInlineCandidateInfo->stubAddr); - call->gtStubCallStubAddr = call->gtInlineCandidateInfo->stubAddr; + JITDUMP("Restoring stub addr %p from candidate info\n", call->GetInlineCandidateInfo()->stubAddr); + call->gtStubCallStubAddr = call->GetInlineCandidateInfo()->stubAddr; } else { - call->gtInlineCandidateInfo = nullptr; + call->ClearInlineInfo(); } compiler->fgInsertStmtAtEnd(elseBlock, newStmt); @@ -863,6 +936,14 @@ class IndirectCallTransformer stmt->SetRootNode(compiler->gtNewNothingNode()); } + //------------------------------------------------------------------------ + // SetWeights: set weights for new blocks. + // + virtual void SetWeights() override + { + remainderBlock->inheritWeight(currBlock); + } + // For chained gdv, we modify the expansion as follows: // // We verify the check block has two BBJ_NONE/ALWAYS predecessors, one of @@ -963,8 +1044,6 @@ class IndirectCallTransformer const unsigned maxStatementDup = JitConfig.JitGuardedDevirtualizationChainStatements(); unsigned chainStatementDup = 0; unsigned chainNodeDup = 0; - unsigned chainLikelihood = 0; - GenTreeCall* chainedCall = nullptr; // Helper class to check a statement for uncloneable nodes and count // the total number of nodes @@ -1024,10 +1103,10 @@ class IndirectCallTransformer GenTreeCall* const call = root->AsCall(); if (call->IsGuardedDevirtualizationCandidate() && - (call->gtGuardedDevirtualizationCandidateInfo->likelihood >= gdvChainLikelihood)) + (call->GetInlineCandidateInfo()->likelihood >= gdvChainLikelihood)) { JITDUMP("GDV call at [%06u] has likelihood %u >= %u; chaining (%u stmts, %u nodes to dup).\n", - compiler->dspTreeID(call), call->gtGuardedDevirtualizationCandidateInfo->likelihood, + compiler->dspTreeID(call), call->GetInlineCandidateInfo()->likelihood, gdvChainLikelihood, chainStatementDup, chainNodeDup); call->gtCallMoreFlags |= GTF_CALL_M_GUARDED_DEVIRT_CHAIN; @@ -1157,7 +1236,7 @@ class IndirectCallTransformer //------------------------------------------------------------------------ // CreateCheck: create check blocks, that checks dictionary size and does null test. // - virtual void CreateCheck() override + virtual void CreateCheck(UINT8 checkIdx) override { GenTreeCall::UseIterator argsIter = origCall->Args().begin(); GenTree* nullCheck = argsIter.GetUse()->GetNode(); @@ -1184,7 +1263,7 @@ class IndirectCallTransformer // CreateThen: create then block, that is executed if the checks succeed. // This simply returns the handle. // - virtual void CreateThen() override + virtual void CreateThen(UINT8 checkIdx) override { thenBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, checkBlock2); diff --git a/src/coreclr/jit/inline.cpp b/src/coreclr/jit/inline.cpp index 4e86755491b8f0..ed0d2de5f0b0b3 100644 --- a/src/coreclr/jit/inline.cpp +++ b/src/coreclr/jit/inline.cpp @@ -713,8 +713,13 @@ void InlineResult::Report() // a failing observation yet, do so now. if (IsFailure() && (m_Call != nullptr)) { - // compiler should have revoked candidacy on the call by now - assert((m_Call->gtFlags & GTF_CALL_INLINE_CANDIDATE) == 0); + if (m_Call->GetGDVCandidatesCount() < 2) + { + // Compiler should have revoked candidacy on the call by now + // However, in GDV case we might see cases where some of the + // candidates are inlineable and some are not. + assert((m_Call->gtFlags & GTF_CALL_INLINE_CANDIDATE) == 0); + } if (m_Call->gtInlineObservation == InlineObservation::CALLEE_UNUSED_INITIAL) { diff --git a/src/coreclr/jit/jitconfigvalues.h b/src/coreclr/jit/jitconfigvalues.h index 2501fc0d2693af..0efea82200cc4c 100644 --- a/src/coreclr/jit/jitconfigvalues.h +++ b/src/coreclr/jit/jitconfigvalues.h @@ -498,6 +498,9 @@ CONFIG_STRING(JitGuardedDevirtualizationRange, W("JitGuardedDevirtualizationRang CONFIG_INTEGER(JitRandomGuardedDevirtualization, W("JitRandomGuardedDevirtualization"), 0) #endif // DEBUG +// Max number of type checks for a virtual call +CONFIG_INTEGER(JitGuardedDevirtualizationCheckCount, W("JitGuardedDevirtualizationCheckCount"), 3) + // Enable insertion of patchpoints into Tier0 methods with loops. CONFIG_INTEGER(TC_OnStackReplacement, W("TC_OnStackReplacement"), 0) // Initial patchpoint counter value used by jitted code diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 5e6c05a338f6ea..6bb82148e1d5f7 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -64,16 +64,16 @@ GenTree* Compiler::fgMorphIntoHelperCall(GenTree* tree, int helper, GenTreeCall: GenTreeCall* call = tree->AsCall(); - call->gtCallType = CT_HELPER; - call->gtCallMethHnd = eeFindHelper(helper); - call->gtCallThisArg = nullptr; - call->gtCallArgs = args; - call->gtCallLateArgs = nullptr; - call->fgArgInfo = nullptr; - call->gtRetClsHnd = nullptr; - call->gtCallMoreFlags = GTF_CALL_M_EMPTY; - call->gtInlineCandidateInfo = nullptr; - call->gtControlExpr = nullptr; + call->gtCallType = CT_HELPER; + call->gtCallMethHnd = eeFindHelper(helper); + call->gtCallThisArg = nullptr; + call->gtCallArgs = args; + call->gtCallLateArgs = nullptr; + call->fgArgInfo = nullptr; + call->gtRetClsHnd = nullptr; + call->gtCallMoreFlags = GTF_CALL_M_EMPTY; + call->gtControlExpr = nullptr; + call->ClearInlineInfo(); #ifdef UNIX_X86_ABI call->gtFlags |= GTF_CALL_POP_ARGS; #endif // UNIX_X86_ABI