JIT: relax inliner heuristics for callees on [Intrinsic] types

src/coreclr/jit/fgbasic.cpp

@@ -887,6 +887,8 @@ void Compiler::fgFindJumpTargets(const BYTE* codeAddr, IL_OFFSET codeSize, Fixed
         compInlineResult->NoteDouble(InlineObservation::CALLSITE_PROFILE_FREQUENCY, 1.0);
         // Observe force inline state and code size.
         compInlineResult->NoteBool(InlineObservation::CALLEE_IS_FORCE_INLINE, isForceInline);
+        compInlineResult->NoteBool(InlineObservation::CALLEE_IS_INTRINSIC_TYPE,
+                                   (info.compClassAttr & CORINFO_FLG_INTRINSIC_TYPE) != 0);
         compInlineResult->NoteInt(InlineObservation::CALLEE_IL_CODE_SIZE, codeSize);
 
         // Determine if call site is within a try.

src/coreclr/jit/inline.cpp

@@ -840,6 +840,7 @@ InlineStrategy::InlineStrategy(Compiler* compiler)
     , m_MaxInlineSize(DEFAULT_MAX_INLINE_SIZE)
     , m_MaxInlineDepth(DEFAULT_MAX_INLINE_DEPTH)
     , m_MaxForceInlineDepth(DEFAULT_MAX_FORCE_INLINE_DEPTH)
+    , m_OverBudgetIntrinsicInlineCount(0)
     , m_InitialTimeBudget(0)
     , m_InitialTimeEstimate(0)
     , m_CurrentTimeBudget(0)

src/coreclr/jit/inline.def

@@ -99,6 +99,7 @@ INLINE_OBSERVATION(IS_FORCE_INLINE,           bool,   "aggressive inline attribu
 INLINE_OBSERVATION(IS_INSTANCE_CTOR,          bool,   "instance constructor",                 INFORMATION, CALLEE)
 INLINE_OBSERVATION(IS_PROFITABLE_INLINE,      bool,   "profitable inline",                    INFORMATION, CALLEE)
 INLINE_OBSERVATION(IS_SIZE_DECREASING_INLINE, bool,   "size decreasing inline",               INFORMATION, CALLEE)
+INLINE_OBSERVATION(IS_INTRINSIC_TYPE,         bool,   "callee class marked as Intrinsic",     INFORMATION, CALLEE)
 INLINE_OBSERVATION(LOG_REPLAY_ACCEPT,         bool,   "accepted by log replay",               INFORMATION, CALLEE)
 INLINE_OBSERVATION(LOOKS_LIKE_WRAPPER,        bool,   "thin wrapper around a call",           INFORMATION, CALLEE)
 INLINE_OBSERVATION(MAXSTACK,                  int,    "maxstack",                             INFORMATION, CALLEE)

src/coreclr/jit/inline.h

@@ -986,6 +986,24 @@ class InlineStrategy
         return m_MaxForceInlineDepth;
     }
 
+    // Maximum number of over-budget [Intrinsic]-type inlines allowed per root method.
+    enum
+    {
+        MAX_OVER_BUDGET_INTRINSIC_INLINES = 50
+    };
+
+    // Number of over-budget inlines admitted because the callee was on an [Intrinsic] type.
+    unsigned GetOverBudgetIntrinsicInlineCount() const
+    {
+        return m_OverBudgetIntrinsicInlineCount;
+    }
+
+    // Note an over-budget inline that was admitted due to the callee's [Intrinsic] type.
+    void NoteOverBudgetIntrinsicInline()
+    {
+        m_OverBudgetIntrinsicInlineCount++;
+    }
+
     // Number of successful inlines into the root
     unsigned GetInlineCount() const
     {
@@ -1139,6 +1157,7 @@ class InlineStrategy
     unsigned          m_MaxInlineSize;
     unsigned          m_MaxInlineDepth;
     unsigned          m_MaxForceInlineDepth;
+    unsigned          m_OverBudgetIntrinsicInlineCount;
     int               m_InitialTimeBudget;
     int               m_InitialTimeEstimate;
     int               m_CurrentTimeBudget;

src/coreclr/jit/inlinepolicy.cpp

@@ -288,6 +288,10 @@ void DefaultPolicy::NoteBool(InlineObservation obs, bool value)
                 m_IsForceInlineKnown = true;
                 break;
 
+            case InlineObservation::CALLEE_IS_INTRINSIC_TYPE:
+                m_IsIntrinsicType = value;
+                break;
+
             case InlineObservation::CALLEE_IS_INSTANCE_CTOR:
                 m_IsInstanceCtor = value;
                 break;
@@ -506,6 +510,18 @@ bool DefaultPolicy::BudgetCheck() const
             allowOverBudget = true;
         }
 
+        if (!allowOverBudget && m_IsIntrinsicType &&
+            (strategy->GetOverBudgetIntrinsicInlineCount() < InlineStrategy::MAX_OVER_BUDGET_INTRINSIC_INLINES))
+        {
+            // Callees from [Intrinsic]-marked types (e.g. Span<T>, Vector<T>, hardware intrinsic
+            // ISA classes) need to be inlined for codegen quality even when we're out of budget.
+            // Cap the number of such admissions per root method to keep JIT throughput bounded.
+            JITDUMP("Allowing over-budget for intrinsic types (count: %u)\n",
+                    strategy->GetOverBudgetIntrinsicInlineCount());
+            strategy->NoteOverBudgetIntrinsicInline();
+            allowOverBudget = true;
+        }
+
         if (!allowOverBudget && m_IsNoReturnKnown && m_IsNoReturn)
         {
             // We're not going to inline no-return calls anyway
@@ -1027,6 +1043,7 @@ void DefaultPolicy::OnDumpXml(FILE* file, unsigned indent) const
     XATTR_B(m_IsNoReturn)
     XATTR_B(m_IsNoReturnKnown)
     XATTR_B(m_InsideThrowBlock)
+    XATTR_B(m_IsIntrinsicType)
 }
 #endif
 
@@ -1855,14 +1872,25 @@ double ExtendedDefaultPolicy::DetermineMultiplier()
         const double profileTrustCoef = (double)JitConfig.JitExtDefaultPolicyProfTrust() / 10.0;
         const double profileScale     = (double)JitConfig.JitExtDefaultPolicyProfScale() / 10.0;
 
+        double profileBoost;
         if (m_RootCompiler->fgHaveTrustedProfileWeights())
         {
-            multiplier *= (1.0 - profileTrustCoef) + min(m_ProfileFrequency, 1.0) * profileScale;
+            profileBoost = (1.0 - profileTrustCoef) + min(m_ProfileFrequency, 1.0) * profileScale;
         }
         else
         {
-            multiplier *= min(m_ProfileFrequency, 1.0) * profileScale;
+            profileBoost = min(m_ProfileFrequency, 1.0) * profileScale;
         }
+
+        if ((profileBoost < 1.0) && m_IsIntrinsicType)
+        {
+            // Don't apply the profile-frequency-based penalty for callees from [Intrinsic]-marked types
+            // (e.g. Span<T>, Vector<T>) - JIT relies on inlining these for codegen quality regardless
+            // of how cold the call site is.
+            profileBoost = 1.0;
+        }
+        multiplier *= profileBoost;
+
         JITDUMP("\nCallsite has profile data: %g.  Multiplier limited to %g.", m_ProfileFrequency, multiplier);
     }
 

src/coreclr/jit/inlinepolicy.h

@@ -113,6 +113,7 @@ class DefaultPolicy : public LegalPolicy
         , m_ConstArgFeedsIsKnownConst(false)
         , m_ArgFeedsIsKnownConst(false)
         , m_InsideThrowBlock(false)
+        , m_IsIntrinsicType(false)
     {
         // empty
     }
@@ -189,6 +190,7 @@ class DefaultPolicy : public LegalPolicy
     bool                    m_ConstArgFeedsIsKnownConst  : 1;
     bool                    m_ArgFeedsIsKnownConst       : 1;
     bool                    m_InsideThrowBlock           : 1;
+    bool                    m_IsIntrinsicType            : 1;
 };
 
 // ExtendedDefaultPolicy is a slightly more aggressive variant of