From 0fb9719dae0f40060e41e815dee1df887533c88d Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 20 Apr 2026 13:51:57 -0700 Subject: [PATCH 1/6] Fix IJW OverflowException with 17+ by-ref parameters (#127166) StubSigBuilder::EnsureEnoughQuickBytes only doubled the buffer size once, so when the required size exceeded 2x the current buffer, the CQuickBytes inline buffer (512 bytes) overflowed into the adjacent m_nItems field. This was triggered by PR #106000 which changed ConvertToInternalSignature to preserve custom modifiers (bSkipCustomModifier=FALSE). Each preserved modifier adds 10 bytes (ELEMENT_TYPE_CMOD_INTERNAL + required byte + 8-byte TypeHandle pointer) to the signature. For 18+ parameters with custom modifiers, the total signature exceeded 512 bytes, causing a buffer overflow that corrupted m_nItems and led to an OverflowException. The fix loops the doubling until the buffer is large enough. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/coreclr/vm/stubgen.cpp | 7 +- .../IjwNativeCallingManagedDll.cpp | 72 +++++++++++++++++++ .../NativeCallingManaged.cs | 16 +++++ 3 files changed, 94 insertions(+), 1 deletion(-) diff --git a/src/coreclr/vm/stubgen.cpp b/src/coreclr/vm/stubgen.cpp index 06c99d1cb0cbd6..731e807ba69857 100644 --- a/src/coreclr/vm/stubgen.cpp +++ b/src/coreclr/vm/stubgen.cpp @@ -1956,7 +1956,12 @@ void StubSigBuilder::EnsureEnoughQuickBytes(size_t cbToAppend) SIZE_T cbBuffer = m_qbSigBuffer.Size(); if ((m_cbSig + cbToAppend) >= cbBuffer) { - m_qbSigBuffer.ReSizeThrows(2 * cbBuffer); + SIZE_T cbNew = 2 * cbBuffer; + while ((m_cbSig + cbToAppend) >= cbNew) + { + cbNew *= 2; + } + m_qbSigBuffer.ReSizeThrows(cbNew); m_pbSigCursor = ((BYTE*)m_qbSigBuffer.Ptr()) + m_cbSig; } } diff --git a/src/tests/Interop/IJW/IjwNativeCallingManagedDll/IjwNativeCallingManagedDll.cpp b/src/tests/Interop/IJW/IjwNativeCallingManagedDll/IjwNativeCallingManagedDll.cpp index e581c7ae185e11..8cc34329fe5fa1 100644 --- a/src/tests/Interop/IJW/IjwNativeCallingManagedDll/IjwNativeCallingManagedDll.cpp +++ b/src/tests/Interop/IJW/IjwNativeCallingManagedDll/IjwNativeCallingManagedDll.cpp @@ -16,6 +16,50 @@ extern "C" DLL_EXPORT int __cdecl NativeEntryPoint() return NativeFunction(); } +struct IntWrapper +{ + int Value; +}; + +#pragma managed +int64_t ManagedSum18ByRef(const int& a0, + const IntWrapper& a1, + const IntWrapper& a2, + const IntWrapper& a3, + const IntWrapper& a4, + const IntWrapper& a5, + const IntWrapper& a6, + const IntWrapper& a7, + const IntWrapper& a8, + const IntWrapper& a9, + const IntWrapper& a10, + const IntWrapper& a11, + const IntWrapper& a12, + const IntWrapper& a13, + const IntWrapper& a14, + const IntWrapper& a15, + const IntWrapper& a16, + const int& a17); + +#pragma unmanaged +int64_t NativeSum18ByRef() +{ + int a0 = 0; + IntWrapper w[16]; + for (int i = 0; i < 16; ++i) + { + w[i].Value = i + 1; + } + int a17 = 17; + return ManagedSum18ByRef(a0, w[0], w[1], w[2], w[3], w[4], w[5], w[6], w[7], + w[8], w[9], w[10], w[11], w[12], w[13], w[14], w[15], a17); +} + +extern "C" DLL_EXPORT int64_t __cdecl NativeSum18ByRefEntryPoint() +{ + return NativeSum18ByRef(); +} + #pragma managed // Needed to provide a regression case for https://github.com/dotnet/runtime/issues/110365 @@ -39,6 +83,11 @@ public ref class TestClass return NativeFunction(); } + int64_t ManagedEntryPointSum18ByRef() + { + return NativeSum18ByRef(); + } + static void ChangeReturnedValue(int i) { s_valueToReturnStorage.valueToReturn = i; @@ -58,3 +107,26 @@ int ManagedCallee() { return TestClass::GetReturnValue(); } + +int64_t ManagedSum18ByRef(const int& a0, + const IntWrapper& a1, + const IntWrapper& a2, + const IntWrapper& a3, + const IntWrapper& a4, + const IntWrapper& a5, + const IntWrapper& a6, + const IntWrapper& a7, + const IntWrapper& a8, + const IntWrapper& a9, + const IntWrapper& a10, + const IntWrapper& a11, + const IntWrapper& a12, + const IntWrapper& a13, + const IntWrapper& a14, + const IntWrapper& a15, + const IntWrapper& a16, + const int& a17) +{ + return (int64_t)a0 + a1.Value + a2.Value + a3.Value + a4.Value + a5.Value + a6.Value + a7.Value + a8.Value + a9.Value + + a10.Value + a11.Value + a12.Value + a13.Value + a14.Value + a15.Value + a16.Value + a17; +} diff --git a/src/tests/Interop/IJW/NativeCallingManaged/NativeCallingManaged.cs b/src/tests/Interop/IJW/NativeCallingManaged/NativeCallingManaged.cs index 1b21d075150c2b..d81d7b159f6f29 100644 --- a/src/tests/Interop/IJW/NativeCallingManaged/NativeCallingManaged.cs +++ b/src/tests/Interop/IJW/NativeCallingManaged/NativeCallingManaged.cs @@ -36,6 +36,22 @@ public static int TestEntryPoint() } TestFramework.EndTestCase(); + // Regression test for https://github.com/dotnet/runtime/issues/127166: + // Native code calling a managed function with 17+ by-ref parameters + // hit an OverflowException because StubSigBuilder::EnsureEnoughQuickBytes + // only doubled the buffer size once, which was insufficient when the + // internal signature (with preserved custom modifiers) exceeded 512 bytes. + TestFramework.BeginTestCase("Call managed method with 18 by-ref parameters from native"); + MethodInfo sum18Method = testType.GetMethod("ManagedEntryPointSum18ByRef"); + long sum = (long)sum18Method.Invoke(testInstance, null); + const long expectedSum = 153; // 0 + (1+2+...+16) + 17 + if (sum != expectedSum) + { + TestFramework.LogError("IJW", "Incorrect sum returned: " + sum + " (expected " + expectedSum + ")"); + success = false; + } + TestFramework.EndTestCase(); + return success ? 100 : 99; } } From 3fa12874cf037091f629d69e77595fa9629dbd8d Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 20 Apr 2026 14:09:10 -0700 Subject: [PATCH 2/6] Address review: overflow-safe growth loop, remove unused export - Use S_SIZE_T for checked arithmetic in EnsureEnoughQuickBytes to prevent infinite loop or undersized allocation on SIZE_T overflow. - Remove unnecessary NativeSum18ByRefEntryPoint DLL export. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/coreclr/vm/stubgen.cpp | 10 ++++++---- .../IjwNativeCallingManagedDll.cpp | 5 ----- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/coreclr/vm/stubgen.cpp b/src/coreclr/vm/stubgen.cpp index 731e807ba69857..b59d1329d0ec98 100644 --- a/src/coreclr/vm/stubgen.cpp +++ b/src/coreclr/vm/stubgen.cpp @@ -1956,12 +1956,14 @@ void StubSigBuilder::EnsureEnoughQuickBytes(size_t cbToAppend) SIZE_T cbBuffer = m_qbSigBuffer.Size(); if ((m_cbSig + cbToAppend) >= cbBuffer) { - SIZE_T cbNew = 2 * cbBuffer; - while ((m_cbSig + cbToAppend) >= cbNew) + S_SIZE_T cbNew = S_SIZE_T(cbBuffer) * S_SIZE_T(2); + while (!cbNew.IsOverflow() && (m_cbSig + cbToAppend) >= cbNew.Value()) { - cbNew *= 2; + cbNew *= S_SIZE_T(2); } - m_qbSigBuffer.ReSizeThrows(cbNew); + if (cbNew.IsOverflow()) + COMPlusThrowHR(COR_E_OVERFLOW); + m_qbSigBuffer.ReSizeThrows(cbNew.Value()); m_pbSigCursor = ((BYTE*)m_qbSigBuffer.Ptr()) + m_cbSig; } } diff --git a/src/tests/Interop/IJW/IjwNativeCallingManagedDll/IjwNativeCallingManagedDll.cpp b/src/tests/Interop/IJW/IjwNativeCallingManagedDll/IjwNativeCallingManagedDll.cpp index 8cc34329fe5fa1..a4c1257575af51 100644 --- a/src/tests/Interop/IJW/IjwNativeCallingManagedDll/IjwNativeCallingManagedDll.cpp +++ b/src/tests/Interop/IJW/IjwNativeCallingManagedDll/IjwNativeCallingManagedDll.cpp @@ -55,11 +55,6 @@ int64_t NativeSum18ByRef() w[8], w[9], w[10], w[11], w[12], w[13], w[14], w[15], a17); } -extern "C" DLL_EXPORT int64_t __cdecl NativeSum18ByRefEntryPoint() -{ - return NativeSum18ByRef(); -} - #pragma managed // Needed to provide a regression case for https://github.com/dotnet/runtime/issues/110365 From 065a47a09cc78c93b57c35d802cc937cb881c77d Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 20 Apr 2026 17:05:11 -0700 Subject: [PATCH 3/6] Update src/coreclr/vm/stubgen.cpp Co-authored-by: Jan Kotas --- src/coreclr/vm/stubgen.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/coreclr/vm/stubgen.cpp b/src/coreclr/vm/stubgen.cpp index b59d1329d0ec98..3ee25fdaac32dc 100644 --- a/src/coreclr/vm/stubgen.cpp +++ b/src/coreclr/vm/stubgen.cpp @@ -1956,12 +1956,9 @@ void StubSigBuilder::EnsureEnoughQuickBytes(size_t cbToAppend) SIZE_T cbBuffer = m_qbSigBuffer.Size(); if ((m_cbSig + cbToAppend) >= cbBuffer) { - S_SIZE_T cbNew = S_SIZE_T(cbBuffer) * S_SIZE_T(2); - while (!cbNew.IsOverflow() && (m_cbSig + cbToAppend) >= cbNew.Value()) - { - cbNew *= S_SIZE_T(2); - } - if (cbNew.IsOverflow()) + SIZE_T cbNew = max(m_cbSig + cbToAppend, 2 * cbBuffer); + // Detect integer overflow + if ((cbNew - m_cbSig) < cbToAppend) COMPlusThrowHR(COR_E_OVERFLOW); m_qbSigBuffer.ReSizeThrows(cbNew.Value()); m_pbSigCursor = ((BYTE*)m_qbSigBuffer.Ptr()) + m_cbSig; From 3ba11ce1781053cf8cca2a26872eae4b9cb91f7a Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 20 Apr 2026 17:05:24 -0700 Subject: [PATCH 4/6] Apply suggestion from @jkotas Co-authored-by: Jan Kotas --- src/coreclr/vm/stubgen.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/vm/stubgen.cpp b/src/coreclr/vm/stubgen.cpp index 3ee25fdaac32dc..c8accb70710a71 100644 --- a/src/coreclr/vm/stubgen.cpp +++ b/src/coreclr/vm/stubgen.cpp @@ -1954,7 +1954,7 @@ void StubSigBuilder::EnsureEnoughQuickBytes(size_t cbToAppend) STANDARD_VM_CONTRACT; SIZE_T cbBuffer = m_qbSigBuffer.Size(); - if ((m_cbSig + cbToAppend) >= cbBuffer) + if ((cbBuffer - m_cbSig) < cbToAppend) { SIZE_T cbNew = max(m_cbSig + cbToAppend, 2 * cbBuffer); // Detect integer overflow From 06bdeaeba780f5d6cd865efc766e4f8f422068da Mon Sep 17 00:00:00 2001 From: Jan Kotas Date: Mon, 20 Apr 2026 17:07:46 -0700 Subject: [PATCH 5/6] Apply suggestion from @jkotas --- src/coreclr/vm/stubgen.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/vm/stubgen.cpp b/src/coreclr/vm/stubgen.cpp index c8accb70710a71..69cea47ae68380 100644 --- a/src/coreclr/vm/stubgen.cpp +++ b/src/coreclr/vm/stubgen.cpp @@ -1960,7 +1960,7 @@ void StubSigBuilder::EnsureEnoughQuickBytes(size_t cbToAppend) // Detect integer overflow if ((cbNew - m_cbSig) < cbToAppend) COMPlusThrowHR(COR_E_OVERFLOW); - m_qbSigBuffer.ReSizeThrows(cbNew.Value()); + m_qbSigBuffer.ReSizeThrows(cbNew); m_pbSigCursor = ((BYTE*)m_qbSigBuffer.Ptr()) + m_cbSig; } } From 46b2f084eeabb4d9eac12ee9b2558e03d1f19cdd Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 22 Apr 2026 11:22:01 -0700 Subject: [PATCH 6/6] Apply suggestion from @jkoritzinsky --- src/coreclr/vm/stubgen.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/vm/stubgen.cpp b/src/coreclr/vm/stubgen.cpp index 69cea47ae68380..08bc275e23cf6e 100644 --- a/src/coreclr/vm/stubgen.cpp +++ b/src/coreclr/vm/stubgen.cpp @@ -1956,7 +1956,7 @@ void StubSigBuilder::EnsureEnoughQuickBytes(size_t cbToAppend) SIZE_T cbBuffer = m_qbSigBuffer.Size(); if ((cbBuffer - m_cbSig) < cbToAppend) { - SIZE_T cbNew = max(m_cbSig + cbToAppend, 2 * cbBuffer); + SIZE_T cbNew = max((SIZE_T)(m_cbSig + cbToAppend), 2 * cbBuffer); // Detect integer overflow if ((cbNew - m_cbSig) < cbToAppend) COMPlusThrowHR(COR_E_OVERFLOW);