diff --git a/src/coreclr/jit/fgbasic.cpp b/src/coreclr/jit/fgbasic.cpp index 58d698443b3dd8..1f3def222fff24 100644 --- a/src/coreclr/jit/fgbasic.cpp +++ b/src/coreclr/jit/fgbasic.cpp @@ -887,6 +887,8 @@ void Compiler::fgFindJumpTargets(const BYTE* codeAddr, IL_OFFSET codeSize, Fixed compInlineResult->NoteDouble(InlineObservation::CALLSITE_PROFILE_FREQUENCY, 1.0); // Observe force inline state and code size. compInlineResult->NoteBool(InlineObservation::CALLEE_IS_FORCE_INLINE, isForceInline); + compInlineResult->NoteBool(InlineObservation::CALLEE_IS_INTRINSIC_TYPE, + (info.compClassAttr & CORINFO_FLG_INTRINSIC_TYPE) != 0); compInlineResult->NoteInt(InlineObservation::CALLEE_IL_CODE_SIZE, codeSize); // Determine if call site is within a try. diff --git a/src/coreclr/jit/inline.cpp b/src/coreclr/jit/inline.cpp index 45f97e4a211989..3a6cc6b0b4d33a 100644 --- a/src/coreclr/jit/inline.cpp +++ b/src/coreclr/jit/inline.cpp @@ -840,6 +840,7 @@ InlineStrategy::InlineStrategy(Compiler* compiler) , m_MaxInlineSize(DEFAULT_MAX_INLINE_SIZE) , m_MaxInlineDepth(DEFAULT_MAX_INLINE_DEPTH) , m_MaxForceInlineDepth(DEFAULT_MAX_FORCE_INLINE_DEPTH) + , m_OverBudgetIntrinsicInlineCount(0) , m_InitialTimeBudget(0) , m_InitialTimeEstimate(0) , m_CurrentTimeBudget(0) diff --git a/src/coreclr/jit/inline.def b/src/coreclr/jit/inline.def index 05f19bcbd6458a..aaa7ce52dc82d1 100644 --- a/src/coreclr/jit/inline.def +++ b/src/coreclr/jit/inline.def @@ -99,6 +99,7 @@ INLINE_OBSERVATION(IS_FORCE_INLINE, bool, "aggressive inline attribu INLINE_OBSERVATION(IS_INSTANCE_CTOR, bool, "instance constructor", INFORMATION, CALLEE) INLINE_OBSERVATION(IS_PROFITABLE_INLINE, bool, "profitable inline", INFORMATION, CALLEE) INLINE_OBSERVATION(IS_SIZE_DECREASING_INLINE, bool, "size decreasing inline", INFORMATION, CALLEE) +INLINE_OBSERVATION(IS_INTRINSIC_TYPE, bool, "callee class marked as Intrinsic", INFORMATION, CALLEE) INLINE_OBSERVATION(LOG_REPLAY_ACCEPT, bool, "accepted by log replay", INFORMATION, CALLEE) INLINE_OBSERVATION(LOOKS_LIKE_WRAPPER, bool, "thin wrapper around a call", INFORMATION, CALLEE) INLINE_OBSERVATION(MAXSTACK, int, "maxstack", INFORMATION, CALLEE) diff --git a/src/coreclr/jit/inline.h b/src/coreclr/jit/inline.h index c320f72fb5d7a1..83d74587789366 100644 --- a/src/coreclr/jit/inline.h +++ b/src/coreclr/jit/inline.h @@ -986,6 +986,24 @@ class InlineStrategy return m_MaxForceInlineDepth; } + // Maximum number of over-budget [Intrinsic]-type inlines allowed per root method. + enum + { + MAX_OVER_BUDGET_INTRINSIC_INLINES = 50 + }; + + // Number of over-budget inlines admitted because the callee was on an [Intrinsic] type. + unsigned GetOverBudgetIntrinsicInlineCount() const + { + return m_OverBudgetIntrinsicInlineCount; + } + + // Note an over-budget inline that was admitted due to the callee's [Intrinsic] type. + void NoteOverBudgetIntrinsicInline() + { + m_OverBudgetIntrinsicInlineCount++; + } + // Number of successful inlines into the root unsigned GetInlineCount() const { @@ -1139,6 +1157,7 @@ class InlineStrategy unsigned m_MaxInlineSize; unsigned m_MaxInlineDepth; unsigned m_MaxForceInlineDepth; + unsigned m_OverBudgetIntrinsicInlineCount; int m_InitialTimeBudget; int m_InitialTimeEstimate; int m_CurrentTimeBudget; diff --git a/src/coreclr/jit/inlinepolicy.cpp b/src/coreclr/jit/inlinepolicy.cpp index dd1e6cc23f8297..620a42612168e1 100644 --- a/src/coreclr/jit/inlinepolicy.cpp +++ b/src/coreclr/jit/inlinepolicy.cpp @@ -288,6 +288,10 @@ void DefaultPolicy::NoteBool(InlineObservation obs, bool value) m_IsForceInlineKnown = true; break; + case InlineObservation::CALLEE_IS_INTRINSIC_TYPE: + m_IsIntrinsicType = value; + break; + case InlineObservation::CALLEE_IS_INSTANCE_CTOR: m_IsInstanceCtor = value; break; @@ -506,6 +510,18 @@ bool DefaultPolicy::BudgetCheck() const allowOverBudget = true; } + if (!allowOverBudget && m_IsIntrinsicType && + (strategy->GetOverBudgetIntrinsicInlineCount() < InlineStrategy::MAX_OVER_BUDGET_INTRINSIC_INLINES)) + { + // Callees from [Intrinsic]-marked types (e.g. Span, Vector, hardware intrinsic + // ISA classes) need to be inlined for codegen quality even when we're out of budget. + // Cap the number of such admissions per root method to keep JIT throughput bounded. + JITDUMP("Allowing over-budget for intrinsic types (count: %u)\n", + strategy->GetOverBudgetIntrinsicInlineCount()); + strategy->NoteOverBudgetIntrinsicInline(); + allowOverBudget = true; + } + if (!allowOverBudget && m_IsNoReturnKnown && m_IsNoReturn) { // We're not going to inline no-return calls anyway @@ -1027,6 +1043,7 @@ void DefaultPolicy::OnDumpXml(FILE* file, unsigned indent) const XATTR_B(m_IsNoReturn) XATTR_B(m_IsNoReturnKnown) XATTR_B(m_InsideThrowBlock) + XATTR_B(m_IsIntrinsicType) } #endif @@ -1855,14 +1872,25 @@ double ExtendedDefaultPolicy::DetermineMultiplier() const double profileTrustCoef = (double)JitConfig.JitExtDefaultPolicyProfTrust() / 10.0; const double profileScale = (double)JitConfig.JitExtDefaultPolicyProfScale() / 10.0; + double profileBoost; if (m_RootCompiler->fgHaveTrustedProfileWeights()) { - multiplier *= (1.0 - profileTrustCoef) + min(m_ProfileFrequency, 1.0) * profileScale; + profileBoost = (1.0 - profileTrustCoef) + min(m_ProfileFrequency, 1.0) * profileScale; } else { - multiplier *= min(m_ProfileFrequency, 1.0) * profileScale; + profileBoost = min(m_ProfileFrequency, 1.0) * profileScale; } + + if ((profileBoost < 1.0) && m_IsIntrinsicType) + { + // Don't apply the profile-frequency-based penalty for callees from [Intrinsic]-marked types + // (e.g. Span, Vector) - JIT relies on inlining these for codegen quality regardless + // of how cold the call site is. + profileBoost = 1.0; + } + multiplier *= profileBoost; + JITDUMP("\nCallsite has profile data: %g. Multiplier limited to %g.", m_ProfileFrequency, multiplier); } diff --git a/src/coreclr/jit/inlinepolicy.h b/src/coreclr/jit/inlinepolicy.h index 4277a61fa78151..5516ba4ad7a74c 100644 --- a/src/coreclr/jit/inlinepolicy.h +++ b/src/coreclr/jit/inlinepolicy.h @@ -113,6 +113,7 @@ class DefaultPolicy : public LegalPolicy , m_ConstArgFeedsIsKnownConst(false) , m_ArgFeedsIsKnownConst(false) , m_InsideThrowBlock(false) + , m_IsIntrinsicType(false) { // empty } @@ -189,6 +190,7 @@ class DefaultPolicy : public LegalPolicy bool m_ConstArgFeedsIsKnownConst : 1; bool m_ArgFeedsIsKnownConst : 1; bool m_InsideThrowBlock : 1; + bool m_IsIntrinsicType : 1; }; // ExtendedDefaultPolicy is a slightly more aggressive variant of