Rework built-in COM->CLR stubs to build on the UnmanagedCallersOnly model

docs/design/coreclr/botr/guide-for-porting.md

@@ -368,8 +368,7 @@ Here is an annotated list of the stubs implemented for Unix on Arm64.
         FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP can be implemented as
         performance needs require.
 
-    7.  `ComCallPreStub`/ `COMToCLRDispatchHelper` /`GenericComCallStub` - not
-        necessary for non-Windows platforms at this time
+    7.  `ComCallPreStub` - not necessary for non-Windows platforms at this time
 
     8.  `TheUMEntryPrestub`/ `UMThunkStub` - used to enter the runtime from
         non-managed code through entrypoints generated from the

src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs

@@ -637,6 +637,19 @@ internal static void ClearNative(IntPtr pUnk)
                 Marshal.Release(pUnk);
             }
         }
+
+        internal static object GetObjectForComCallableWrapperIUnknown(IntPtr unk)
+        {
+            object? retObject = null;
+            GetObjectForComCallableWrapperIUnknown(unk, ObjectHandleOnStack.Create(ref retObject));
+            return retObject!;
+        }
+
+        [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "InterfaceMarshaler_GetObjectForComCallableWrapperIUnknown")]
+        private static partial void GetObjectForComCallableWrapperIUnknown(IntPtr unk, ObjectHandleOnStack retObject);
+
+        [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "InterfaceMarshaler_ValidateComVisibilityForIUnknown")]
+        internal static partial void ValidateComVisibilityForIUnknown(IntPtr unk);
     }  // class InterfaceMarshaler
 #endif // FEATURE_COMINTEROP
 

src/coreclr/debug/daccess/dacdbiimplstackwalk.cpp

@@ -603,42 +603,6 @@ HRESULT STDMETHODCALLTYPE DacDbiInterfaceImpl::EnumerateInternalFrames(VMPTR_Thr
                 frameData.vmCurrentAppDomainToken.SetHostPtr(pAppDomain);
 
                 MethodDesc * pMD = pFrame->GetFunction();
-    #if defined(FEATURE_COMINTEROP)
-                if (frameData.stubFrame.frameType == STUBFRAME_U2M)
-                {
-                    _ASSERTE(pMD == NULL);
-
-                    // U2M transition frame generally don't store the target MD because we know what the target
-                    // is by looking at the callee stack frame.  However, for reverse COM interop, we can try
-                    // to get the MD for the interface.
-                    //
-                    // Note that some reverse COM interop cases don't have an intermediate interface MD, so
-                    // pMD may still be NULL.
-                    //
-                    // Even if there is an MD on the ComMethodFrame, it could be in a different appdomain than
-                    // the ComMethodFrame itself.  The only known scenario is a cross-appdomain reverse COM
-                    // interop call.  We need to check for this case.  The end result is that GetFunction() and
-                    // GetFunctionToken() on ICDInternalFrame will return NULL.
-
-                    // Minidumps without full memory don't guarantee to capture the CCW since we can do without
-                    // it.  In this case, pMD will remain NULL.
-                    EX_TRY_ALLOW_DATATARGET_MISSING_MEMORY
-                    {
-                        if (pFrame->GetFrameIdentifier() == FrameIdentifier::ComMethodFrame)
-                        {
-                            ComMethodFrame * pCOMFrame = dac_cast<PTR_ComMethodFrame>(pFrame);
-                            PTR_VOID pUnkStackSlot     = pCOMFrame->GetPointerToArguments();
-                            PTR_IUnknown pUnk          = dac_cast<PTR_IUnknown>(*dac_cast<PTR_TADDR>(pUnkStackSlot));
-                            ComCallWrapper * pCCW      = ComCallWrapper::GetWrapperFromIP(pUnk);
-
-                            ComCallMethodDesc * pCMD = NULL;
-                            pCMD = dac_cast<PTR_ComCallMethodDesc>(pCOMFrame->ComMethodFrame::GetDatum());
-                            pMD  = pCMD->GetInterfaceMethodDesc();
-                        }
-                    }
-                    EX_END_CATCH_ALLOW_DATATARGET_MISSING_MEMORY
-                }
-    #endif // FEATURE_COMINTEROP
 
                 Module *     pModule = (pMD ? pMD->GetModule() : NULL);
                 DomainAssembly * pDomainAssembly = (pModule ? pModule->GetDomainAssembly() : NULL);
@@ -902,7 +866,7 @@ void DacDbiInterfaceImpl::InitFrameData(StackFrameIterator *   pIter,
         pFrameData->v.fVarArgs = pMD->IsVarArg();
 
         // Check if this is a NoMetadata method or if the method should be hidden.
-        // These methods should not be visible in the debugger both for convenience and 
+        // These methods should not be visible in the debugger both for convenience and
         // because they don't have backing metadata. For more information see comments in
         // MethodDesc::IsNoMetadata and MethodDesc::IsDiagnosticsHidden.
         pFrameData->v.fNoMetadata = pMD->IsNoMetadata() || pMD->IsDiagnosticsHidden();

src/coreclr/debug/ee/frameinfo.cpp

@@ -801,28 +801,8 @@ void FrameInfo::InitForM2UInternalFrame(CrawlFrame * pCF)
 void FrameInfo::InitForU2MInternalFrame(CrawlFrame * pCF)
 {
     _ASSERTE(pCF != NULL);
-    MethodDesc * pMDHint = NULL;
-
-#ifdef FEATURE_COMINTEROP
-    Frame * pFrame = pCF->GetFrame();
-    _ASSERTE(pFrame != NULL);
-
-
-    // For regular U2M PInvoke cases, we don't care about MD b/c it's just going to
-    // be the next frame.
-    // If we're a COM2CLR call, perhaps we can get the MD for the interface.
-    if (pFrame->GetFrameIdentifier() == FrameIdentifier::ComMethodFrame)
-    {
-        ComMethodFrame* pCOMFrame = dac_cast<PTR_ComMethodFrame> (pFrame);
-        ComCallMethodDesc* pCMD = reinterpret_cast<ComCallMethodDesc *> (pCOMFrame->ComMethodFrame::GetDatum());
-        pMDHint = pCMD->GetInterfaceMethodDesc();
-
-        // Some COM-interop cases don't have an intermediate interface method desc, so
-        // pMDHint may be null.
-    }
-#endif
 
-    InitFromStubHelper(pCF, pMDHint, STUBFRAME_U2M);
+    InitFromStubHelper(pCF, NULL, STUBFRAME_U2M);
     InitForScratchFrameInfo();
 }
 

src/coreclr/inc/CrstTypes.def

@@ -463,7 +463,7 @@ Crst UnresolvedClassLock
 End
 
 Crst WrapperTemplate
-    AcquiredBefore ExecutableAllocatorLock
+    AcquiredBefore LoaderHeap
 End
 
 Crst UMEntryThunkCache
@@ -536,4 +536,4 @@ End
 
 Crst CallStubCache
     AcquiredBefore LoaderHeap
-End
+End

src/coreclr/inc/clrtypes.h

@@ -215,9 +215,6 @@ static_assert(sizeof(CLR_BOOL) == 1);
 #define CLR_BOOL_MAX    BOOL_MAX
 #define CLR_BOOL_MIN    BOOL_MIN
 
-#define CLR_NAN_32 0xFFC00000
-#define CLR_NAN_64 0xFFF8000000000000LL
-
 // ================================================================================
 // Simple utility functions
 // ================================================================================

src/coreclr/inc/crsttypes_generated.h

@@ -235,7 +235,7 @@ int g_rgCrstLevelMap[] =
     2,          // CrstUnwindInfoTablePendingLock
     3,          // CrstUnwindInfoTablePublishLock
     3,          // CrstVSDIndirectionCellLock
-    2,          // CrstWrapperTemplate
+    3,          // CrstWrapperTemplate
 };
 
 // An array mapping CrstType to a stringized name.

src/coreclr/vm/CMakeLists.txt

@@ -661,7 +661,6 @@ elseif(CLR_CMAKE_TARGET_WIN32)
             ${ARCH_SOURCES_DIR}/CachedInterfaceDispatchCoreCLR.asm
             ${ARCH_SOURCES_DIR}/CallDescrWorkerAMD64.asm
             ${ARCH_SOURCES_DIR}/ComCallPreStub.asm
-            ${ARCH_SOURCES_DIR}/GenericComCallStubs.asm
             ${ARCH_SOURCES_DIR}/GenericCLRToCOMCallStubs.asm
             ${ARCH_SOURCES_DIR}/getstate.asm
             ${ARCH_SOURCES_DIR}/JitHelpers_Fast.asm

src/coreclr/vm/FrameTypes.h

@@ -19,9 +19,7 @@ FRAME_TYPE_NAME(SoftwareExceptionFrame)
 FRAME_TYPE_NAME(FuncEvalFrame)
 #endif // DEBUGGING_SUPPORTED
 #ifdef FEATURE_COMINTEROP
-FRAME_TYPE_NAME(ComMethodFrame)
 FRAME_TYPE_NAME(CLRToCOMMethodFrame)
-FRAME_TYPE_NAME(ComPrestubMethodFrame)
 #endif // FEATURE_COMINTEROP
 FRAME_TYPE_NAME(PInvokeCalliFrame)
 #ifdef FEATURE_HIJACK

src/coreclr/vm/amd64/AsmHelpers.asm

@@ -106,41 +106,6 @@ NESTED_ENTRY JIT_RareDisableHelper, _TEXT
 
 NESTED_END JIT_RareDisableHelper, _TEXT
 
-
-; extern "C" void setFPReturn(int fpSize, INT64 retVal);
-LEAF_ENTRY setFPReturn, _TEXT
-        cmp     ecx, 4
-        je      setFPReturn4
-        cmp     ecx, 8
-        jne     setFPReturnNot8
-        mov     [rsp+10h], rdx
-        movsd   xmm0, real8 ptr [rsp+10h]
-setFPReturnNot8:
-        REPRET
-
-setFPReturn4:
-        mov     [rsp+10h], rdx
-        movss   xmm0, real4 ptr [rsp+10h]
-        ret
-LEAF_END setFPReturn, _TEXT
-
-
-; extern "C" void getFPReturn(int fpSize, INT64 *retval);
-LEAF_ENTRY getFPReturn, _TEXT
-        cmp     ecx, 4
-        je      getFPReturn4
-        cmp     ecx, 8
-        jne     getFPReturnNot8
-        movsd   real8 ptr [rdx], xmm0
-getFPReturnNot8:
-        REPRET
-
-getFPReturn4:
-        movss   real4 ptr [rdx], xmm0
-        ret
-LEAF_END getFPReturn, _TEXT
-
-
 ; A JITted method's return address was hijacked to return to us here.
 ; VOID OnHijackTripThread()
 NESTED_ENTRY OnHijackTripThread, _TEXT

src/coreclr/vm/amd64/ComCallPreStub.asm

@@ -6,126 +6,75 @@ ifdef FEATURE_COMINTEROP
 include AsmMacros.inc
 include asmconstants.inc
 
-; extern "C" const BYTE* ComPreStubWorker(ComPrestubMethodFrame *pPFrame, UINT64 *pErrorResult)
+; extern "C" const BYTE* ComPreStubWorker(UMEntryThunkData* pEntryThunkInfo);
 extern ComPreStubWorker:proc
-extern JIT_FailFast:proc
-
-
-; extern "C" VOID ComCallPreStub()
-NESTED_ENTRY ComCallPreStub, _TEXT
-
-;
-; Stack layout:
-;
-; (stack parameters)
-; ...
-; r9
-; r8
-; rdx
-; rcx
-; ComPrestubMethodFrame::m_ReturnAddress
-; ComPrestubMethodFrame::m_pFuncDesc
-; Frame::m_Next
-; FrameIdentifier::ComPrestubMethodFrame        <-- rsp + ComCallPreStub_ComPrestubMethodFrame_OFFSET
-; HRESULT                                       <-- rsp + ComCallPreStub_ERRORRETVAL_OFFSET
-; (optional padding to qword align xmm save area)
-; xmm3
-; xmm2
-; xmm1
-; xmm0                                          <-- rsp + ComCallPreStub_XMM_SAVE_OFFSET
-; callee's r9
-; callee's r8
-; callee's rdx
-; callee's rcx
-
-ComCallPreStub_STACK_FRAME_SIZE = 0
-
-; ComPrestubMethodFrame MUST be the highest part of the stack frame,
-; immediately below the return address, so that
-; ComPrestubMethodFrame::m_ReturnAddress and m_FuncDesc are in the right place.
-ComCallPreStub_STACK_FRAME_SIZE = ComCallPreStub_STACK_FRAME_SIZE + SIZEOF__ComPrestubMethodFrame - 8
-ComCallPreStub_ComPrestubMethodFrame_NEGOFFSET = ComCallPreStub_STACK_FRAME_SIZE
-
-; CalleeSavedRegisters MUST be immediately below ComPrestubMethodFrame
-ComCallPreStub_STACK_FRAME_SIZE = ComCallPreStub_STACK_FRAME_SIZE + 8*8
-ComCallPreStub_CalleeSavedRegisters_NEGOFFSET = ComCallPreStub_STACK_FRAME_SIZE
-
-; UINT64 (out param to ComPreStubWorker)
-ComCallPreStub_STACK_FRAME_SIZE = ComCallPreStub_STACK_FRAME_SIZE + 8
-ComCallPreStub_ERRORRETVAL_NEGOFFSET = ComCallPreStub_STACK_FRAME_SIZE
+extern UMEntryPrestubUnwindFrameChainHandler:proc
 
-; Ensure that the offset of the XMM save area will be 16-byte aligned.
-if ((ComCallPreStub_STACK_FRAME_SIZE + SIZEOF__Frame + 8) mod 16) ne 0
-ComCallPreStub_STACK_FRAME_SIZE = ComCallPreStub_STACK_FRAME_SIZE + 8
-endif
+NESTED_ENTRY ComCallPreStub, _TEXT, UMEntryPrestubUnwindFrameChainHandler
 
-; FP parameters (xmm0-xmm3)
-ComCallPreStub_STACK_FRAME_SIZE = ComCallPreStub_STACK_FRAME_SIZE + 40h
-ComCallPreStub_XMM_SAVE_NEGOFFSET = ComCallPreStub_STACK_FRAME_SIZE
+ComCallPreStub_STACK_FRAME_SIZE = SIZEOF_MAX_OUTGOING_ARGUMENT_HOMES
 
-; Callee scratch area
-ComCallPreStub_STACK_FRAME_SIZE = ComCallPreStub_STACK_FRAME_SIZE + 20h
+; XMM save area
+ComCallPreStub_XMM_SAVE_OFFSET = ComCallPreStub_STACK_FRAME_SIZE
+ComCallPreStub_STACK_FRAME_SIZE = ComCallPreStub_STACK_FRAME_SIZE + SIZEOF_MAX_FP_ARG_SPILL
 
-; Now we have the full size of the stack frame.  The offsets have been computed relative to the
-; top, so negate them to make them relative to the post-prologue rsp.
-ComCallPreStub_ComPrestubMethodFrame_OFFSET = ComCallPreStub_STACK_FRAME_SIZE - ComCallPreStub_ComPrestubMethodFrame_NEGOFFSET
-ComCallPreStub_ERRORRETVAL_OFFSET           = ComCallPreStub_STACK_FRAME_SIZE - ComCallPreStub_ERRORRETVAL_NEGOFFSET
-ComCallPreStub_XMM_SAVE_OFFSET              = ComCallPreStub_STACK_FRAME_SIZE - ComCallPreStub_XMM_SAVE_NEGOFFSET
+ComCallPreStub_NONVOL_REG_SAVE_OFFSET = ComCallPreStub_STACK_FRAME_SIZE
+ComCallPreStub_STACK_FRAME_SIZE = ComCallPreStub_STACK_FRAME_SIZE + 8 ; for one non-volatile register to preserve the secret argument
 
-        .allocstack     8                       ; ComPrestubMethodFrame::m_pFuncDesc, pushed by prepad
-
-        alloc_stack     SIZEOF__ComPrestubMethodFrame - 2*8
+; Ensure that the new rsp will be 16-byte aligned.  Note that the caller has
+; already pushed the return address.
+if ((ComCallPreStub_STACK_FRAME_SIZE + 8) MOD 16) ne 0
+ComCallPreStub_STACK_FRAME_SIZE = ComCallPreStub_STACK_FRAME_SIZE + 8
+endif
 
-        ;
-        ; Save ComPrestubMethodFrame* to pass to ComPreStubWorker
-        ;
-        mov             r10, rsp
+        alloc_stack     ComCallPreStub_STACK_FRAME_SIZE
 
-        ;
-        ; Allocate callee scratch area and save FP parameters
-        ;
-        alloc_stack     ComCallPreStub_ComPrestubMethodFrame_OFFSET
+        save_reg_postrsp    rcx, ComCallPreStub_STACK_FRAME_SIZE + 8h
+        save_reg_postrsp    rdx, ComCallPreStub_STACK_FRAME_SIZE + 10h
+        save_reg_postrsp    r8,  ComCallPreStub_STACK_FRAME_SIZE + 18h
+        save_reg_postrsp    r9,  ComCallPreStub_STACK_FRAME_SIZE + 20h
 
-        ;
-        ; Save argument registers
-        ;
-        SAVE_ARGUMENT_REGISTERS ComCallPreStub_STACK_FRAME_SIZE + 8h
+        save_xmm128_postrsp xmm0, ComCallPreStub_XMM_SAVE_OFFSET
+        save_xmm128_postrsp xmm1, ComCallPreStub_XMM_SAVE_OFFSET + 10h
+        save_xmm128_postrsp xmm2, ComCallPreStub_XMM_SAVE_OFFSET + 20h
+        save_xmm128_postrsp xmm3, ComCallPreStub_XMM_SAVE_OFFSET + 30h
 
-        ;
-        ; spill the fp args
-        ;
-        SAVE_FLOAT_ARGUMENT_REGISTERS ComCallPreStub_XMM_SAVE_OFFSET
+        save_reg_postrsp    r12, ComCallPreStub_NONVOL_REG_SAVE_OFFSET
 
         END_PROLOGUE
 
         ;
-        ; Resolve target.
+        ; Do prestub-specific stuff
         ;
-        mov             rcx, r10
-        lea             rdx, [rsp + ComCallPreStub_ERRORRETVAL_OFFSET]
+        mov             rcx, METHODDESC_REGISTER
+        mov             r12, METHODDESC_REGISTER
         call            ComPreStubWorker
-        test            rax, rax
-        jz              ExitError
 
-        ;
-        ; Restore FP parameters
-        ;
-        RESTORE_FLOAT_ARGUMENT_REGISTERS ComCallPreStub_XMM_SAVE_OFFSET
+        ; Restore the secret argument
+        mov             METHODDESC_REGISTER, r12
 
         ;
-        ; Restore integer parameters
+        ; we're going to tail call to the exec stub that we just setup
+        ; or return the specified error
         ;
-        RESTORE_ARGUMENT_REGISTERS ComCallPreStub_STACK_FRAME_SIZE + 8h
 
-        add             rsp, ComCallPreStub_ComPrestubMethodFrame_OFFSET + SIZEOF__ComPrestubMethodFrame - 8
+        mov             rcx, [rsp + ComCallPreStub_STACK_FRAME_SIZE + 8h]
+        mov             rdx, [rsp + ComCallPreStub_STACK_FRAME_SIZE + 10h]
+        mov             r8,  [rsp + ComCallPreStub_STACK_FRAME_SIZE + 18h]
+        mov             r9,  [rsp + ComCallPreStub_STACK_FRAME_SIZE + 20h]
 
-        TAILJMP_RAX
+        movdqa          xmm0, xmmword ptr [rsp + ComCallPreStub_XMM_SAVE_OFFSET]
+        movdqa          xmm1, xmmword ptr [rsp + ComCallPreStub_XMM_SAVE_OFFSET + 10h]
+        movdqa          xmm2, xmmword ptr [rsp + ComCallPreStub_XMM_SAVE_OFFSET + 20h]
+        movdqa          xmm3, xmmword ptr [rsp + ComCallPreStub_XMM_SAVE_OFFSET + 30h]
 
-ExitError:
-        mov             rax, [rsp + ComCallPreStub_ERRORRETVAL_OFFSET]
-        add             rsp, ComCallPreStub_ComPrestubMethodFrame_OFFSET + SIZEOF__ComPrestubMethodFrame - 8
+        mov             r12, [rsp + ComCallPreStub_NONVOL_REG_SAVE_OFFSET]
 
-        ret
+        ;
+        ; epilogue
+        ;
+        add             rsp, ComCallPreStub_STACK_FRAME_SIZE
+        TAILJMP_RAX
 
 NESTED_END ComCallPreStub, _TEXT