From 24de52af74973e75e7d9fe9c83210aee8b076201 Mon Sep 17 00:00:00 2001 From: vsadov Date: Thu, 27 Feb 2020 16:11:03 -0800 Subject: [PATCH 01/11] Managed `LdelemaRef` and `StelemRef` --- .../Runtime/CompilerServices/CastHelpers.cs | 86 +++++++++++++++++++ .../RuntimeHelpers.CoreCLR.cs | 12 +++ src/coreclr/src/inc/jithelpers.h | 8 +- src/coreclr/src/vm/ecall.cpp | 7 ++ src/coreclr/src/vm/ecalllist.h | 1 + src/coreclr/src/vm/metasig.h | 2 + src/coreclr/src/vm/mscorlib.h | 2 + 7 files changed, 112 insertions(+), 6 deletions(-) diff --git a/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs b/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs index b0905c93f506cd..49bf5a36ab3470 100644 --- a/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs +++ b/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs @@ -21,6 +21,7 @@ namespace System.Runtime.CompilerServices internal static unsafe class CastHelpers { private static int[]? s_table; + private static readonly MethodTable* pObjMt = RuntimeHelpers.GetMethodTable(new object()); [DebuggerDisplay("Source = {_source}; Target = {_targetAndResult & ~1}; Result = {_targetAndResult & 1}; VersionNum = {_version & ((1 << 29) - 1)}; Distance = {_version >> 29};")] [StructLayout(LayoutKind.Sequential)] @@ -168,6 +169,9 @@ private static CastResult TryGet(nuint source, nuint target) [MethodImpl(MethodImplOptions.InternalCall)] private static extern ref byte Unbox_Helper(void* toTypeHnd, object obj); + [MethodImpl(MethodImplOptions.InternalCall)] + private static extern void WriteBarrier(ref object? dst, object obj); + // IsInstanceOf test used for unusual cases (naked type parameters, variant generic types) // Unlike the IsInstanceOfInterface and IsInstanceOfClass functions, // this test must deal with all kinds of type tests @@ -491,5 +495,87 @@ private static ref byte Unbox(void* toTypeHnd, object obj) return ref Unbox_Helper(toTypeHnd, obj); } + + internal struct ArrayElement + { + public object? Value; + } + + private static ref object? ThrowArrayMismatchException() + { + throw new ArrayTypeMismatchException(); + } + + private static ref object? LdelemaRef(Array array, int index, void* type) + { + ArrayElement[] arr = Unsafe.As(array); + + // this will throw appropriate exceptions if array is null or access is out of range. + nuint elementType = RuntimeHelpers.GetMethodTable(arr)->ElementType; + ref object? element = ref arr[index].Value; + + if (elementType != (nuint)type) + goto throwArrayMismatch; + + return ref element; + + throwArrayMismatch: + return ref ThrowArrayMismatchException(); + } + + private static void StelemRef(Array array, int index, object? obj) + { + ArrayElement[] arr = Unsafe.As(array); + + // this will throw appropriate exceptions if array is null or access is out of range. + nuint elementType = RuntimeHelpers.GetMethodTable(arr)->ElementType; + ref object? element = ref arr[index].Value; + + if (obj == null) + goto assigningNull; + + if (elementType != (nuint)RuntimeHelpers.GetMethodTable(obj)) + goto notExactMatch; + + doWrite: + WriteBarrier(ref element, obj); + return; + + assigningNull: + element = null; + return; + + notExactMatch: + if (elementType == (nuint)pObjMt) + goto doWrite; + + StelemRef_Helper(ref element, elementType, obj); + } + + private static void StelemRef_Helper(ref object? element, nuint elementType, object obj) + { + CastResult result = TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)elementType); + if (result == CastResult.CanCast) + { + WriteBarrier(ref element, obj); + return; + } + + StelemRef_Helper_Slow(ref element, elementType, obj); + } + + private static void StelemRef_Helper_Slow(ref object? element, nuint elementType, object obj) + { + Debug.Assert(obj != null); + + obj = IsInstanceOfAny_NoCacheLookup((void*)elementType, obj); + if (obj != null) + { + WriteBarrier(ref element, obj); + return; + } + + throw new ArrayTypeMismatchException(); + } } } diff --git a/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs b/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs index 9936ff6b5dc51a..eae8c8e697fed1 100644 --- a/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs +++ b/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs @@ -304,6 +304,8 @@ internal unsafe struct MethodTable public ushort InterfaceCount; [FieldOffset(ParentMethodTableOffset)] public MethodTable* ParentMethodTable; + [FieldOffset(ElementTypeOffset)] + public nuint ElementType; [FieldOffset(InterfaceMapOffset)] public MethodTable** InterfaceMap; @@ -322,6 +324,16 @@ internal unsafe struct MethodTable #endif ; +#if TARGET_64BIT + private const int ElementTypeOffset = 0x30 +#else + private const int ElementTypeOffset = 0x20 +#endif +#if DEBUG + + sizeof(nuint) // adjust for debug_m_szClassName +#endif + ; + #if TARGET_64BIT private const int InterfaceMapOffset = 0x38 #else diff --git a/src/coreclr/src/inc/jithelpers.h b/src/coreclr/src/inc/jithelpers.h index b1ce9b13c8c031..8087f4f89799b5 100644 --- a/src/coreclr/src/inc/jithelpers.h +++ b/src/coreclr/src/inc/jithelpers.h @@ -107,12 +107,8 @@ JITHELPER(CORINFO_HELP_UNBOX_NULLABLE, JIT_Unbox_Nullable, CORINFO_HELP_SIG_4_STACK) JITHELPER(CORINFO_HELP_GETREFANY, JIT_GetRefAny, CORINFO_HELP_SIG_8_STACK) -#if defined(TARGET_ARM) - DYNAMICJITHELPER(CORINFO_HELP_ARRADDR_ST, JIT_Stelem_Ref, CORINFO_HELP_SIG_4_STACK) -#else - JITHELPER(CORINFO_HELP_ARRADDR_ST, JIT_Stelem_Ref, CORINFO_HELP_SIG_4_STACK) -#endif // TARGET_ARM - JITHELPER(CORINFO_HELP_LDELEMA_REF, JIT_Ldelema_Ref, CORINFO_HELP_SIG_4_STACK) + DYNAMICJITHELPER(CORINFO_HELP_ARRADDR_ST, NULL, CORINFO_HELP_SIG_4_STACK) + DYNAMICJITHELPER(CORINFO_HELP_LDELEMA_REF, NULL, CORINFO_HELP_SIG_4_STACK) // Exceptions JITHELPER(CORINFO_HELP_THROW, IL_Throw, CORINFO_HELP_SIG_REG_ONLY) diff --git a/src/coreclr/src/vm/ecall.cpp b/src/coreclr/src/vm/ecall.cpp index 692b058e4d378d..bae203456f91fe 100644 --- a/src/coreclr/src/vm/ecall.cpp +++ b/src/coreclr/src/vm/ecall.cpp @@ -195,6 +195,13 @@ void ECall::PopulateManagedCastHelpers() pDest = pMD->GetMultiCallableAddrOfCode(); SetJitHelperFunction(CORINFO_HELP_UNBOX, pDest); + pMD = MscorlibBinder::GetMethod((BinderMethodID)(METHOD__CASTHELPERS__STELEMREF)); + pDest = pMD->GetMultiCallableAddrOfCode(); + SetJitHelperFunction(CORINFO_HELP_ARRADDR_ST, pDest); + + pMD = MscorlibBinder::GetMethod((BinderMethodID)(METHOD__CASTHELPERS__LDELEMAREF)); + pDest = pMD->GetMultiCallableAddrOfCode(); + SetJitHelperFunction(CORINFO_HELP_LDELEMA_REF, pDest); #endif //CROSSGEN_COMPILE } diff --git a/src/coreclr/src/vm/ecalllist.h b/src/coreclr/src/vm/ecalllist.h index abcd3905d18f9b..73fd9a5faeca3a 100644 --- a/src/coreclr/src/vm/ecalllist.h +++ b/src/coreclr/src/vm/ecalllist.h @@ -718,6 +718,7 @@ FCFuncStart(gCastHelpers) FCFuncElement("IsInstanceOfAny_NoCacheLookup", ::IsInstanceOfAny_NoCacheLookup) FCFuncElement("ChkCastAny_NoCacheLookup", ::ChkCastAny_NoCacheLookup) FCFuncElement("Unbox_Helper", ::Unbox_Helper) + FCFuncElement("WriteBarrier", ::JIT_WriteBarrier) FCFuncEnd() FCFuncStart(gArrayFuncs) diff --git a/src/coreclr/src/vm/metasig.h b/src/coreclr/src/vm/metasig.h index f6309e352407c8..8699224e0f1175 100644 --- a/src/coreclr/src/vm/metasig.h +++ b/src/coreclr/src/vm/metasig.h @@ -611,6 +611,8 @@ DEFINE_METASIG_T(IM(Int_RetReadOnlyRefT, i, Q(INATTRIBUTE) r(G(0)))) DEFINE_METASIG(GM(RetT, IMAGE_CEE_CS_CALLCONV_DEFAULT, 1, _, M(0))) DEFINE_METASIG_T(SM(Array_Int_Array_Int_Int_RetVoid, C(ARRAY) i C(ARRAY) i i, v)) +DEFINE_METASIG_T(SM(Array_Int_Obj_RetVoid, C(ARRAY) i j, v)) +DEFINE_METASIG_T(SM(Array_Int_PtrVoid_RetRefObj, C(ARRAY) i P(v), r(j))) // Undefine macros in case we include the file again in the compilation unit diff --git a/src/coreclr/src/vm/mscorlib.h b/src/coreclr/src/vm/mscorlib.h index dfcf9bc704a18b..3ce6149ded554a 100644 --- a/src/coreclr/src/vm/mscorlib.h +++ b/src/coreclr/src/vm/mscorlib.h @@ -1462,6 +1462,8 @@ DEFINE_METHOD(CASTHELPERS, CHKCASTINTERFACE, ChkCastInterface, SM_Ptr DEFINE_METHOD(CASTHELPERS, CHKCASTCLASS, ChkCastClass, SM_PtrVoid_Obj_RetObj) DEFINE_METHOD(CASTHELPERS, CHKCASTCLASSSPECIAL, ChkCastClassSpecial, SM_PtrVoid_Obj_RetObj) DEFINE_METHOD(CASTHELPERS, UNBOX, Unbox, SM_PtrVoid_Obj_RetRefByte) +DEFINE_METHOD(CASTHELPERS, STELEMREF, StelemRef, SM_Array_Int_Obj_RetVoid) +DEFINE_METHOD(CASTHELPERS, LDELEMAREF, LdelemaRef, SM_Array_Int_PtrVoid_RetRefObj) DEFINE_CLASS_U(CompilerServices, LAHashDependentHashTracker, LAHashDependentHashTrackerObject) DEFINE_FIELD_U(_dependentHandle, LAHashDependentHashTrackerObject,_dependentHandle) From 944e213cabd019cc4c098ff2e10921b300b29ee0 Mon Sep 17 00:00:00 2001 From: vsadov Date: Sat, 29 Feb 2020 15:12:44 -0800 Subject: [PATCH 02/11] support ARM64 and OSX --- src/coreclr/src/vm/amd64/jithelpers_fast.S | 9 +++++++++ src/coreclr/src/vm/arm64/asmhelpers.S | 15 ++++++++++++++- src/coreclr/src/vm/arm64/asmhelpers.asm | 16 +++++++++++++++- src/coreclr/src/vm/ecalllist.h | 2 +- src/coreclr/src/vm/jitinterface.h | 10 ++++++++++ 5 files changed, 49 insertions(+), 3 deletions(-) diff --git a/src/coreclr/src/vm/amd64/jithelpers_fast.S b/src/coreclr/src/vm/amd64/jithelpers_fast.S index d5c9eaec923775..1a46ebe376b553 100644 --- a/src/coreclr/src/vm/amd64/jithelpers_fast.S +++ b/src/coreclr/src/vm/amd64/jithelpers_fast.S @@ -399,6 +399,15 @@ LEAF_END_MARKED JIT_ByRefWriteBarrier, _TEXT // helpers use this global variable to jump to it. This variable is set in InitThreadManager. .global _JIT_WriteBarrier_Loc .zerofill __DATA,__common,_JIT_WriteBarrier_Loc,8,3 + +// ------------------------------------------------------------------ +// __declspec(naked) void F_CALL_CONV JIT_WriteBarrier_Callable(Object **dst, Object* val) +.balign 16 +LEAF_ENTRY JIT_WriteBarrier_Callable, _TEXT + // JIT_WriteBarrier(Object** dst, Object* src) + jmp [rip + C_FUNC(JIT_WriteBarrier_Loc)] +LEAF_END JIT_Stelem_DoWrite, _TEXT + #endif // FEATURE_WRITEBARRIER_COPY // TODO: put definition for this in asmconstants.h diff --git a/src/coreclr/src/vm/arm64/asmhelpers.S b/src/coreclr/src/vm/arm64/asmhelpers.S index 21b55ea431f7ec..5a46ad31f9acd0 100644 --- a/src/coreclr/src/vm/arm64/asmhelpers.S +++ b/src/coreclr/src/vm/arm64/asmhelpers.S @@ -1369,7 +1369,7 @@ NESTED_END JIT_Stelem_Ref_NotExactMatch, _TEXT // __declspec(naked) void F_CALL_CONV JIT_Stelem_DoWrite(PtrArray* array, unsigned idx, Object* val) LEAF_ENTRY JIT_Stelem_DoWrite, _TEXT - // Setup args for JIT_WriteBarrier. x14 = &array->m_array[idx] x15 = val + // Setup args for JIT_WriteBarrier. x14 = &array->m_array[idx] ; x15 = val add x14, x0, #PtrArray__m_Array // x14 = &array->m_array add x14, x14, x1, LSL #3 mov x15, x2 // x15 = val @@ -1379,6 +1379,19 @@ LEAF_ENTRY JIT_Stelem_DoWrite, _TEXT b C_FUNC(JIT_WriteBarrier) LEAF_END JIT_Stelem_DoWrite, _TEXT +// ------------------------------------------------------------------ +// __declspec(naked) void F_CALL_CONV JIT_WriteBarrier_Callable(Object **dst, Object* val) +LEAF_ENTRY JIT_WriteBarrier_Callable, _TEXT + + // Setup args for JIT_WriteBarrier. x14 = dst ; x15 = val + mov x14, x0 // x14 = dst + mov x15, x1 // x15 = val + + // Branch to the write barrier (which is already correctly overwritten with + // single or multi-proc code based on the current CPU + b C_FUNC(JIT_WriteBarrier) +LEAF_END JIT_Stelem_DoWrite, _TEXT + #ifdef PROFILING_SUPPORTED // ------------------------------------------------------------------ diff --git a/src/coreclr/src/vm/arm64/asmhelpers.asm b/src/coreclr/src/vm/arm64/asmhelpers.asm index bf730eb9254f3c..b7305f5718dbcb 100644 --- a/src/coreclr/src/vm/arm64/asmhelpers.asm +++ b/src/coreclr/src/vm/arm64/asmhelpers.asm @@ -1607,7 +1607,7 @@ DoWrite ; __declspec(naked) void F_CALL_CONV JIT_Stelem_DoWrite(PtrArray* array, unsigned idx, Object* val) LEAF_ENTRY JIT_Stelem_DoWrite - ; Setup args for JIT_WriteBarrier. x14 = &array->m_array[idx] x15 = val + ; Setup args for JIT_WriteBarrier. x14 = &array->m_array[idx] ; x15 = val add x14, x0, #PtrArray__m_Array ; x14 = &array->m_array add x14, x14, x1, LSL #3 mov x15, x2 ; x15 = val @@ -1617,6 +1617,20 @@ DoWrite b JIT_WriteBarrier LEAF_END +; ------------------------------------------------------------------ +; __declspec(naked) void F_CALL_CONV JIT_WriteBarrier_Callable(Object **dst, Object* val) + LEAF_ENTRY JIT_WriteBarrier_Callable + + ; Setup args for JIT_WriteBarrier. x14 = dst ; x15 = val + mov x14, x0 ; x14 = dst + mov x15, x1 ; x15 = val + + ; Branch to the write barrier (which is already correctly overwritten with + ; single or multi-proc code based on the current CPU + b JIT_WriteBarrier + + LEAF_END + #ifdef PROFILING_SUPPORTED ; ------------------------------------------------------------------ diff --git a/src/coreclr/src/vm/ecalllist.h b/src/coreclr/src/vm/ecalllist.h index 73fd9a5faeca3a..613588a9722e1d 100644 --- a/src/coreclr/src/vm/ecalllist.h +++ b/src/coreclr/src/vm/ecalllist.h @@ -718,7 +718,7 @@ FCFuncStart(gCastHelpers) FCFuncElement("IsInstanceOfAny_NoCacheLookup", ::IsInstanceOfAny_NoCacheLookup) FCFuncElement("ChkCastAny_NoCacheLookup", ::ChkCastAny_NoCacheLookup) FCFuncElement("Unbox_Helper", ::Unbox_Helper) - FCFuncElement("WriteBarrier", ::JIT_WriteBarrier) + FCFuncElement("WriteBarrier", ::WriteBarrier_Helper) FCFuncEnd() FCFuncStart(gArrayFuncs) diff --git a/src/coreclr/src/vm/jitinterface.h b/src/coreclr/src/vm/jitinterface.h index c1e0bb8e49d48f..30b7ad8e787e1e 100644 --- a/src/coreclr/src/vm/jitinterface.h +++ b/src/coreclr/src/vm/jitinterface.h @@ -248,6 +248,16 @@ extern "C" FCDECL2(Object*, ChkCastAny_NoCacheLookup, CORINFO_CLASS_HANDLE type, extern "C" FCDECL2(Object*, IsInstanceOfAny_NoCacheLookup, CORINFO_CLASS_HANDLE type, Object* obj); extern "C" FCDECL2(LPVOID, Unbox_Helper, CORINFO_CLASS_HANDLE type, Object* obj); +#if defined(TARGET_ARM64) || defined(FEATURE_WRITEBARRIER_COPY) +// ARM64 JIT_WriteBarrier uses speciall ABI and thus is not callable directly +// Copied write barriers must be called at a different location +extern "C" FCDECL2(VOID, JIT_WriteBarrier_Callable, Object **dst, Object *ref); +#define WriteBarrier_Helper JIT_WriteBarrier_Callable +#else +// in other cases the regular JIT helper is callable. +#define WriteBarrier_Helper JIT_WriteBarrier +#endif + extern "C" FCDECL1(void, JIT_InternalThrow, unsigned exceptNum); extern "C" FCDECL1(void*, JIT_InternalThrowFromHelper, unsigned exceptNum); From 53b3d8e55a61cb722a432bb42077e51c4ee5c221 Mon Sep 17 00:00:00 2001 From: vsadov Date: Sat, 29 Feb 2020 19:49:49 -0800 Subject: [PATCH 03/11] removed old JIT_Stelem_Ref implementation --- src/coreclr/src/vm/amd64/JitHelpers_Fast.asm | 123 ----------------- src/coreclr/src/vm/amd64/cgencpu.h | 1 - src/coreclr/src/vm/amd64/jithelpers_fast.S | 138 +------------------ src/coreclr/src/vm/arm/asmhelpers.S | 97 ------------- src/coreclr/src/vm/arm/asmhelpers.asm | 101 -------------- src/coreclr/src/vm/arm/cgencpu.h | 3 - src/coreclr/src/vm/arm64/asmhelpers.S | 96 ------------- src/coreclr/src/vm/arm64/asmhelpers.asm | 100 -------------- src/coreclr/src/vm/arm64/cgencpu.h | 2 - src/coreclr/src/vm/i386/cgencpu.h | 1 - src/coreclr/src/vm/i386/jitinterfacex86.cpp | 98 ------------- src/coreclr/src/vm/i386/stublinkerx86.cpp | 1 - src/coreclr/src/vm/jithelpers.cpp | 60 -------- src/coreclr/src/vm/jitinterface.h | 6 - src/coreclr/src/vm/threads.cpp | 2 +- 15 files changed, 2 insertions(+), 827 deletions(-) diff --git a/src/coreclr/src/vm/amd64/JitHelpers_Fast.asm b/src/coreclr/src/vm/amd64/JitHelpers_Fast.asm index 54c0019a944c63..662c2d4509f829 100644 --- a/src/coreclr/src/vm/amd64/JitHelpers_Fast.asm +++ b/src/coreclr/src/vm/amd64/JitHelpers_Fast.asm @@ -390,129 +390,6 @@ endif LEAF_END_MARKED JIT_ByRefWriteBarrier, _TEXT -g_pObjectClass equ ?g_pObjectClass@@3PEAVMethodTable@@EA - -EXTERN g_pObjectClass:qword -extern ArrayStoreCheck:proc -extern ObjIsInstanceOfCached:proc - -; TODO: put definition for this in asmconstants.h -CanCast equ 1 - -;__declspec(naked) void F_CALL_CONV JIT_Stelem_Ref(PtrArray* array, unsigned idx, Object* val) -LEAF_ENTRY JIT_Stelem_Ref, _TEXT - ; check for null PtrArray* - test rcx, rcx - je ThrowNullReferenceException - - ; we only want the lower 32-bits of edx, it might be dirty - or edx, edx - - ; check that index is in bounds - cmp edx, dword ptr [rcx + OFFSETOF__PtrArray__m_NumComponents] ; 8h -> array size offset - jae ThrowIndexOutOfRangeException - - ; r10 = Array MT - mov r10, [rcx] - - ; if we're assigning a null object* then we don't need a write barrier - test r8, r8 - jz AssigningNull - - mov r9, [r10 + OFFSETOF__MethodTable__m_ElementType] ; 10h -> typehandle offset - - ; check for exact match - cmp r9, [r8] - jne NotExactMatch - - DoWrite: - lea rcx, [rcx + 8*rdx + OFFSETOF__PtrArray__m_Array] - mov rdx, r8 - - ; JIT_WriteBarrier(Object** dst, Object* src) - jmp JIT_WriteBarrier - - AssigningNull: - ; write barrier is not needed for assignment of NULL references - mov [rcx + 8*rdx + OFFSETOF__PtrArray__m_Array], r8 - ret - - NotExactMatch: - cmp r9, [g_pObjectClass] - je DoWrite - - jmp JIT_Stelem_Ref__ObjIsInstanceOfCached_Helper - - ThrowNullReferenceException: - mov rcx, CORINFO_NullReferenceException_ASM - jmp JIT_InternalThrow - - ThrowIndexOutOfRangeException: - mov rcx, CORINFO_IndexOutOfRangeException_ASM - jmp JIT_InternalThrow -LEAF_END JIT_Stelem_Ref, _TEXT - -NESTED_ENTRY JIT_Stelem_Ref__ObjIsInstanceOfCached_Helper, _TEXT - alloc_stack MIN_SIZE - save_reg_postrsp rcx, MIN_SIZE + 8h - save_reg_postrsp rdx, MIN_SIZE + 10h - save_reg_postrsp r8, MIN_SIZE + 18h - END_PROLOGUE - - ; need to get TypeHandle before setting rcx to be the Obj* because that trashes the PtrArray* - mov rdx, r9 - mov rcx, r8 - - ; TypeHandle::CastResult ObjIsInstanceOfCached(Object *pElement, TypeHandle toTypeHnd) - call ObjIsInstanceOfCached - - mov rcx, [rsp + MIN_SIZE + 8h] - mov rdx, [rsp + MIN_SIZE + 10h] - mov r8, [rsp + MIN_SIZE + 18h] - - cmp eax, CanCast - jne NeedCheck - - lea rcx, [rcx + 8*rdx + OFFSETOF__PtrArray__m_Array] - mov rdx, r8 - add rsp, MIN_SIZE - - ; JIT_WriteBarrier(Object** dst, Object* src) - jmp JIT_WriteBarrier - - NeedCheck: - add rsp, MIN_SIZE - jmp JIT_Stelem_Ref__ArrayStoreCheck_Helper -NESTED_END JIT_Stelem_Ref__ObjIsInstanceOfCached_Helper, _TEXT - -; Need to save r8 to provide a stack address for the Object* -NESTED_ENTRY JIT_Stelem_Ref__ArrayStoreCheck_Helper, _TEXT - alloc_stack MIN_SIZE - save_reg_postrsp rcx, MIN_SIZE + 8h - save_reg_postrsp rdx, MIN_SIZE + 10h - save_reg_postrsp r8, MIN_SIZE + 18h - END_PROLOGUE - - lea rcx, [rsp + MIN_SIZE + 18h] - lea rdx, [rsp + MIN_SIZE + 8h] - - ; HCIMPL2(FC_INNER_RET, ArrayStoreCheck, Object** pElement, PtrArray** pArray) - call ArrayStoreCheck - - mov rcx, [rsp + MIN_SIZE + 8h] - mov rdx, [rsp + MIN_SIZE + 10h] - mov r8, [rsp + MIN_SIZE + 18h] - - lea rcx, [rcx + 8*rdx + OFFSETOF__PtrArray__m_Array] - mov rdx, r8 - add rsp, MIN_SIZE - - ; JIT_WriteBarrier(Object** dst, Object* src) - jmp JIT_WriteBarrier - -NESTED_END JIT_Stelem_Ref__ArrayStoreCheck_Helper, _TEXT - - extern JIT_FailFast:proc extern s_gsCookie:qword diff --git a/src/coreclr/src/vm/amd64/cgencpu.h b/src/coreclr/src/vm/amd64/cgencpu.h index 59f0f7a5a11f6a..2b537bf2fc8d5d 100644 --- a/src/coreclr/src/vm/amd64/cgencpu.h +++ b/src/coreclr/src/vm/amd64/cgencpu.h @@ -528,7 +528,6 @@ inline BOOL ClrFlushInstructionCache(LPCVOID pCodeAddr, size_t sizeOfCode) #define JIT_GetSharedGCStaticBaseNoCtor JIT_GetSharedGCStaticBaseNoCtor_SingleAppDomain #define JIT_GetSharedNonGCStaticBaseNoCtor JIT_GetSharedNonGCStaticBaseNoCtor_SingleAppDomain -#define JIT_Stelem_Ref JIT_Stelem_Ref //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Call counting diff --git a/src/coreclr/src/vm/amd64/jithelpers_fast.S b/src/coreclr/src/vm/amd64/jithelpers_fast.S index 1a46ebe376b553..a51079931414ff 100644 --- a/src/coreclr/src/vm/amd64/jithelpers_fast.S +++ b/src/coreclr/src/vm/amd64/jithelpers_fast.S @@ -395,7 +395,7 @@ LEAF_ENTRY JIT_ByRefWriteBarrier, _TEXT LEAF_END_MARKED JIT_ByRefWriteBarrier, _TEXT #ifdef FEATURE_WRITEBARRIER_COPY - // When JIT_WriteBarrier is copied into an allocated page, the JIT_Stelem_Ref and its + // When JIT_WriteBarrier is copied into an allocated page, // helpers use this global variable to jump to it. This variable is set in InitThreadManager. .global _JIT_WriteBarrier_Loc .zerofill __DATA,__common,_JIT_WriteBarrier_Loc,8,3 @@ -410,142 +410,6 @@ LEAF_END JIT_Stelem_DoWrite, _TEXT #endif // FEATURE_WRITEBARRIER_COPY -// TODO: put definition for this in asmconstants.h -#define CanCast 1 - -//__declspec(naked) void F_CALL_CONV JIT_Stelem_Ref(PtrArray* array, unsigned idx, Object* val) -.balign 16 -LEAF_ENTRY JIT_Stelem_Ref, _TEXT - // check for null PtrArray* - test rdi, rdi - je LOCAL_LABEL(ThrowNullReferenceException) - - // we only want the lower 32-bits of rsi, it might be dirty - or esi, esi - - // check that index is in bounds - cmp esi, dword ptr [rdi + OFFSETOF__PtrArray__m_NumComponents] // 8h -> array size offset - jae LOCAL_LABEL(ThrowIndexOutOfRangeException) - - // r10 = Array MT - mov r10, [rdi] - - // if we're assigning a null object* then we don't need a write barrier - test rdx, rdx - jz LOCAL_LABEL(AssigningNull) - - mov rcx, [r10 + OFFSETOF__MethodTable__m_ElementType] // 10h -> typehandle offset - - // check for exact match - cmp rcx, [rdx] - jne LOCAL_LABEL(NotExactMatch) - - LOCAL_LABEL(DoWrite): - lea rdi, [rdi + 8*rsi] - add rdi, OFFSETOF__PtrArray__m_Array - mov rsi, rdx - - // JIT_WriteBarrier(Object** dst, Object* src) -#ifdef FEATURE_WRITEBARRIER_COPY - jmp [rip + C_FUNC(JIT_WriteBarrier_Loc)] -#else - jmp C_FUNC(JIT_WriteBarrier) -#endif - - LOCAL_LABEL(AssigningNull): - // write barrier is not needed for assignment of NULL references - mov [rdi + 8*rsi + OFFSETOF__PtrArray__m_Array], rdx - ret - - LOCAL_LABEL(NotExactMatch): - PREPARE_EXTERNAL_VAR g_pObjectClass, r11 - cmp rcx, [r11] - je LOCAL_LABEL(DoWrite) - - jmp C_FUNC(JIT_Stelem_Ref__ObjIsInstanceOfCached_Helper) - - LOCAL_LABEL(ThrowNullReferenceException): - mov rdi, CORINFO_NullReferenceException_ASM - jmp C_FUNC(JIT_InternalThrow) - - LOCAL_LABEL(ThrowIndexOutOfRangeException): - mov rdi, CORINFO_IndexOutOfRangeException_ASM - jmp C_FUNC(JIT_InternalThrow) -LEAF_END JIT_Stelem_Ref, _TEXT - -LEAF_ENTRY JIT_Stelem_Ref__ObjIsInstanceOfCached_Helper, _TEXT - push_nonvol_reg rbp - mov rbp, rsp - set_cfa_register rbp, 16 - - sub rsp, 0x20 - mov [rbp - 0x08], rdi - mov [rbp - 0x10], rsi - mov [rbp - 0x18], rdx - - // need to get TypeHandle before setting rcx to be the Obj* because that trashes the PtrArray* - mov rsi, rcx - mov rdi, rdx - - // TypeHandle::CastResult ObjIsInstanceOfCached(Object *pElement, TypeHandle toTypeHnd) - call C_FUNC(ObjIsInstanceOfCached) - - mov rdi, [rbp - 0x08] - mov rsi, [rbp - 0x10] - mov rdx, [rbp - 0x18] - - RESET_FRAME_WITH_RBP - - cmp eax, CanCast - jne LOCAL_LABEL(NeedCheck) - - lea rdi, [rdi + 8*rsi] - add rdi, OFFSETOF__PtrArray__m_Array - mov rsi, rdx - - // JIT_WriteBarrier(Object** dst, Object* src) -#ifdef FEATURE_WRITEBARRIER_COPY - jmp [rip + C_FUNC(JIT_WriteBarrier_Loc)] -#else - jmp C_FUNC(JIT_WriteBarrier) -#endif - - LOCAL_LABEL(NeedCheck): - jmp C_FUNC(JIT_Stelem_Ref__ArrayStoreCheck_Helper) -LEAF_END JIT_Stelem_Ref__ObjIsInstanceOfCached_Helper, _TEXT - -// Need to save reg to provide a stack address for the Object* -LEAF_ENTRY JIT_Stelem_Ref__ArrayStoreCheck_Helper, _TEXT - push_nonvol_reg rbp - mov rbp, rsp - set_cfa_register rbp, 16 - - sub rsp, 0x20 - mov [rbp - 0x10], rdi - mov [rbp - 0x18], rsi - mov [rbp - 0x20], rdx - - mov rdi, rsp - lea rsi, [rbp - 0x10] - // HCIMPL2(FC_INNER_RET, ArrayStoreCheck, Object** pElement, PtrArray** pArray) - call C_FUNC(ArrayStoreCheck) - mov rdi, [rbp - 0x10] - mov rsi, [rbp - 0x18] - mov rdx, [rbp - 0x20] - - lea rdi, [rdi + 8*rsi] - add rdi, OFFSETOF__PtrArray__m_Array - mov rsi, rdx - - RESET_FRAME_WITH_RBP - - // JIT_WriteBarrier(Object** dst, Object* src) -#ifdef FEATURE_WRITEBARRIER_COPY - jmp [rip + C_FUNC(JIT_WriteBarrier_Loc)] -#else - jmp C_FUNC(JIT_WriteBarrier) -#endif -LEAF_END JIT_Stelem_Ref__ArrayStoreCheck_Helper, _TEXT // The following helper will access ("probe") a word on each page of the stack // starting with the page right beneath rsp down to the one pointed to by r11. diff --git a/src/coreclr/src/vm/arm/asmhelpers.S b/src/coreclr/src/vm/arm/asmhelpers.S index 23a24992352468..3d7c96edc2754d 100644 --- a/src/coreclr/src/vm/arm/asmhelpers.S +++ b/src/coreclr/src/vm/arm/asmhelpers.S @@ -1057,103 +1057,6 @@ LOCAL_LABEL(CallCppHelper3): #endif -// ------------------------------------------------------------------ -// __declspec(naked) void F_CALL_CONV JIT_Stelem_Ref(PtrArray* array, unsigned idx, Object* val) - LEAF_ENTRY JIT_Stelem_Ref, _TEXT - - // We retain arguments as they were passed and use r0 == array// r1 == idx// r2 == val - - // check for null array - cbz r0, LOCAL_LABEL(ThrowNullReferenceException) - - // idx bounds check - ldr r3,[r0,#ArrayBase__m_NumComponents] - cmp r3,r1 - bls LOCAL_LABEL(ThrowIndexOutOfRangeException) - - // fast path to null assignment (doesn't need any write-barriers) - cbz r2, LOCAL_LABEL(AssigningNull) - - // Verify the array-type and val-type matches before writing - ldr r12, [r0] // r12 = array MT - ldr r3, [r2] // r3 = val->GetMethodTable() - ldr r12, [r12, #MethodTable__m_ElementType] // array->GetArrayElementTypeHandle() - cmp r3, r12 - beq C_FUNC(JIT_Stelem_DoWrite) - - // Types didnt match but allow writing into an array of objects - ldr r3, =g_pObjectClass - ldr r3, [r3] // r3 = *g_pObjectClass - cmp r3, r12 // array type matches with Object* - beq C_FUNC(JIT_Stelem_DoWrite) - - // array type and val type do not exactly match. Raise frame and do detailed match - b C_FUNC(JIT_Stelem_Ref_NotExactMatch) - -LOCAL_LABEL(AssigningNull): - // Assigning null doesn't need write barrier - adds r0, r0, r1, LSL #2 // r0 = r0 + (r1 x 4) = array->m_array[idx] - str r2, [r0, #PtrArray__m_Array] // array->m_array[idx] = val - bx lr - -LOCAL_LABEL(ThrowNullReferenceException): - // Tail call JIT_InternalThrow(NullReferenceException) - ldr r0, =CORINFO_NullReferenceException_ASM - b C_FUNC(JIT_InternalThrow) - -LOCAL_LABEL(ThrowIndexOutOfRangeException): - // Tail call JIT_InternalThrow(NullReferenceException) - ldr r0, =CORINFO_IndexOutOfRangeException_ASM - b C_FUNC(JIT_InternalThrow) - - LEAF_END JIT_Stelem_Ref, _TEXT - -// ------------------------------------------------------------------ -// __declspec(naked) void F_CALL_CONV JIT_Stelem_Ref_NotExactMatch(PtrArray* array, -// unsigned idx, Object* val) -// r12 = array->GetArrayElementTypeHandle() -// - NESTED_ENTRY JIT_Stelem_Ref_NotExactMatch, _TEXT, NoHandler - PROLOG_PUSH {lr} - PROLOG_PUSH {r0-r2} - - CHECK_STACK_ALIGNMENT - - // allow in case val can be casted to array element type - // call ObjIsInstanceOfCached(val, array->GetArrayElementTypeHandle()) - mov r1, r12 // array->GetArrayElementTypeHandle() - mov r0, r2 - bl C_FUNC(ObjIsInstanceOfCached) - cmp r0, TypeHandle_CanCast - beq LOCAL_LABEL(DoWrite) // ObjIsInstance returned TypeHandle::CanCast - - // check via raising frame -LOCAL_LABEL(NeedFrame): - mov r1, sp // r1 = &array - adds r0, sp, #8 // r0 = &val - bl C_FUNC(ArrayStoreCheck) // ArrayStoreCheck(&val, &array) - -LOCAL_LABEL(DoWrite): - EPILOG_POP {r0-r2} - EPILOG_POP {lr} - b C_FUNC(JIT_Stelem_DoWrite) - - NESTED_END JIT_Stelem_Ref_NotExactMatch, _TEXT - -// ------------------------------------------------------------------ -// __declspec(naked) void F_CALL_CONV JIT_Stelem_DoWrite(PtrArray* array, unsigned idx, Object* val) - LEAF_ENTRY JIT_Stelem_DoWrite, _TEXT - - // Setup args for JIT_WriteBarrier. r0 = &array->m_array[idx]// r1 = val - adds r0, #PtrArray__m_Array // r0 = &array->m_array - adds r0, r0, r1, LSL #2 - mov r1, r2 // r1 = val - - // Branch to the write barrier (which is already correctly overwritten with - // single or multi-proc code based on the current CPU - b C_FUNC(JIT_WriteBarrier) - - LEAF_END JIT_Stelem_DoWrite, _TEXT #define __wbScratch r3 #define pShadow r7 diff --git a/src/coreclr/src/vm/arm/asmhelpers.asm b/src/coreclr/src/vm/arm/asmhelpers.asm index 9d16e474d674ed..1479c1d4f58a22 100644 --- a/src/coreclr/src/vm/arm/asmhelpers.asm +++ b/src/coreclr/src/vm/arm/asmhelpers.asm @@ -24,10 +24,7 @@ IMPORT PreStubWorker IMPORT PreStubGetMethodDescForCompactEntryPoint IMPORT NDirectImportWorker - IMPORT ObjIsInstanceOfCached - IMPORT ArrayStoreCheck IMPORT VSD_ResolveWorker - IMPORT $g_pObjectClass #ifdef WRITE_BARRIER_CHECK SETALIAS g_GCShadow, ?g_GCShadow@@3PAEA @@ -1461,104 +1458,6 @@ CallCppHelper3 LEAF_END -; ------------------------------------------------------------------ -; __declspec(naked) void F_CALL_CONV JIT_Stelem_Ref(PtrArray* array, unsigned idx, Object* val) - LEAF_ENTRY JIT_Stelem_Ref - - ; We retain arguments as they were passed and use r0 == array; r1 == idx; r2 == val - - ; check for null array - cbz r0, ThrowNullReferenceException - - ; idx bounds check - ldr r3,[r0,#ArrayBase__m_NumComponents] - cmp r3,r1 - bls ThrowIndexOutOfRangeException - - ; fast path to null assignment (doesn't need any write-barriers) - cbz r2, AssigningNull - - ; Verify the array-type and val-type matches before writing - ldr r12, [r0] ; r12 = array MT - ldr r3, [r2] ; r3 = val->GetMethodTable() - ldr r12, [r12, #MethodTable__m_ElementType] ; array->GetArrayElementTypeHandle() - cmp r3, r12 - beq JIT_Stelem_DoWrite - - ; Types didnt match but allow writing into an array of objects - ldr r3, =$g_pObjectClass - ldr r3, [r3] ; r3 = *g_pObjectClass - cmp r3, r12 ; array type matches with Object* - beq JIT_Stelem_DoWrite - - ; array type and val type do not exactly match. Raise frame and do detailed match - b JIT_Stelem_Ref_NotExactMatch - -AssigningNull - ; Assigning null doesn't need write barrier - adds r0, r1, LSL #2 ; r0 = r0 + (r1 x 4) = array->m_array[idx] - str r2, [r0, #PtrArray__m_Array] ; array->m_array[idx] = val - bx lr - -ThrowNullReferenceException - ; Tail call JIT_InternalThrow(NullReferenceException) - ldr r0, =CORINFO_NullReferenceException_ASM - b JIT_InternalThrow - -ThrowIndexOutOfRangeException - ; Tail call JIT_InternalThrow(NullReferenceException) - ldr r0, =CORINFO_IndexOutOfRangeException_ASM - b JIT_InternalThrow - - LEAF_END - -; ------------------------------------------------------------------ -; __declspec(naked) void F_CALL_CONV JIT_Stelem_Ref_NotExactMatch(PtrArray* array, -; unsigned idx, Object* val) -; r12 = array->GetArrayElementTypeHandle() -; - NESTED_ENTRY JIT_Stelem_Ref_NotExactMatch - PROLOG_PUSH {lr} - PROLOG_PUSH {r0-r2} - - CHECK_STACK_ALIGNMENT - - ; allow in case val can be casted to array element type - ; call ObjIsInstanceOfCached(val, array->GetArrayElementTypeHandle()) - mov r1, r12 ; array->GetArrayElementTypeHandle() - mov r0, r2 - bl ObjIsInstanceOfCached - cmp r0, TypeHandle_CanCast - beq DoWrite ; ObjIsInstance returned TypeHandle::CanCast - - ; check via raising frame -NeedFrame - mov r1, sp ; r1 = &array - adds r0, sp, #8 ; r0 = &val - bl ArrayStoreCheck ; ArrayStoreCheck(&val, &array) - -DoWrite - EPILOG_POP {r0-r2} - EPILOG_POP {lr} - EPILOG_BRANCH JIT_Stelem_DoWrite - - NESTED_END - -; ------------------------------------------------------------------ -; __declspec(naked) void F_CALL_CONV JIT_Stelem_DoWrite(PtrArray* array, unsigned idx, Object* val) - LEAF_ENTRY JIT_Stelem_DoWrite - - ; Setup args for JIT_WriteBarrier. r0 = &array->m_array[idx]; r1 = val - adds r0, #PtrArray__m_Array ; r0 = &array->m_array - adds r0, r1, LSL #2 - mov r1, r2 ; r1 = val - - ; Branch to the write barrier (which is already correctly overwritten with - ; single or multi-proc code based on the current CPU - b JIT_WriteBarrier - - LEAF_END - ; ------------------------------------------------------------------ ; GC write barrier support. ; diff --git a/src/coreclr/src/vm/arm/cgencpu.h b/src/coreclr/src/vm/arm/cgencpu.h index 7b6da30fce6dfb..edd80606ffd3c2 100644 --- a/src/coreclr/src/vm/arm/cgencpu.h +++ b/src/coreclr/src/vm/arm/cgencpu.h @@ -1059,9 +1059,6 @@ inline BOOL ClrFlushInstructionCache(LPCVOID pCodeAddr, size_t sizeOfCode) #define JIT_GetSharedGCStaticBaseNoCtor JIT_GetSharedGCStaticBaseNoCtor_SingleAppDomain #define JIT_GetSharedNonGCStaticBaseNoCtor JIT_GetSharedNonGCStaticBaseNoCtor_SingleAppDomain -#ifndef TARGET_UNIX -#define JIT_Stelem_Ref JIT_Stelem_Ref -#endif //------------------------------------------------------------------------ // diff --git a/src/coreclr/src/vm/arm64/asmhelpers.S b/src/coreclr/src/vm/arm64/asmhelpers.S index 5a46ad31f9acd0..ce8b71b837a92c 100644 --- a/src/coreclr/src/vm/arm64/asmhelpers.S +++ b/src/coreclr/src/vm/arm64/asmhelpers.S @@ -1282,102 +1282,6 @@ LEAF_ENTRY JIT_GetSharedGCStaticBaseNoCtor_SingleAppDomain, _TEXT ret lr LEAF_END JIT_GetSharedGCStaticBaseNoCtor_SingleAppDomain, _TEXT -// ------------------------------------------------------------------ -// __declspec(naked) void F_CALL_CONV JIT_Stelem_Ref(PtrArray* array, unsigned idx, Object* val) -LEAF_ENTRY JIT_Stelem_Ref, _TEXT - // We retain arguments as they were passed and use x0 == array x1 == idx x2 == val - - // check for null array - cbz x0, LOCAL_LABEL(ThrowNullReferenceException) - - // idx bounds check - ldr x3,[x0,#ArrayBase__m_NumComponents] - cmp x3, x1 - bls LOCAL_LABEL(ThrowIndexOutOfRangeException) - - // fast path to null assignment (doesn't need any write-barriers) - cbz x2, LOCAL_LABEL(AssigningNull) - - // Verify the array-type and val-type matches before writing - ldr x12, [x0] // x12 = array MT - ldr x3, [x2] // x3 = val->GetMethodTable() - ldr x12, [x12, #MethodTable__m_ElementType] // array->GetArrayElementTypeHandle() - cmp x3, x12 - beq C_FUNC(JIT_Stelem_DoWrite) - - // Types didnt match but allow writing into an array of objects - PREPARE_EXTERNAL_VAR g_pObjectClass, x3 - ldr x3, [x3] // x3 = *g_pObjectClass - cmp x3, x12 // array type matches with Object* - beq C_FUNC(JIT_Stelem_DoWrite) - - // array type and val type do not exactly match. Raise frame and do detailed match - b C_FUNC(JIT_Stelem_Ref_NotExactMatch) - -LOCAL_LABEL(AssigningNull): - // Assigning null doesn't need write barrier - add x0, x0, x1, LSL #3 // x0 = x0 + (x1 x 8) = array->m_array[idx] - str x2, [x0, #PtrArray__m_Array] // array->m_array[idx] = val - ret - -LOCAL_LABEL(ThrowNullReferenceException): - // Tail call JIT_InternalThrow(NullReferenceException) - ldr x0, =CORINFO_NullReferenceException_ASM - b C_FUNC(JIT_InternalThrow) - -LOCAL_LABEL(ThrowIndexOutOfRangeException): - // Tail call JIT_InternalThrow(NullReferenceException) - ldr x0, =CORINFO_IndexOutOfRangeException_ASM - b C_FUNC(JIT_InternalThrow) - -LEAF_END JIT_Stelem_Ref, _TEXT - -// ------------------------------------------------------------------ -// __declspec(naked) void F_CALL_CONV JIT_Stelem_Ref_NotExactMatch(PtrArray* array, -// unsigned idx, Object* val) -// x12 = array->GetArrayElementTypeHandle() -// -NESTED_ENTRY JIT_Stelem_Ref_NotExactMatch, _TEXT, NoHandler - PROLOG_SAVE_REG_PAIR_INDEXED fp, lr, -48 - // Spill callee saved registers - PROLOG_SAVE_REG_PAIR x0, x1, 16 - PROLOG_SAVE_REG x2, 32 - - // allow in case val can be casted to array element type - // call ObjIsInstanceOfCached(val, array->GetArrayElementTypeHandle()) - mov x1, x12 // array->GetArrayElementTypeHandle() - mov x0, x2 - bl C_FUNC(ObjIsInstanceOfCached) - cmp x0, TypeHandle_CanCast - beq LOCAL_LABEL(DoWrite) // ObjIsInstance returned TypeHandle::CanCast - - // check via raising frame -LOCAL_LABEL(NeedFrame): - add x1, sp, #16 // x1 = &array - add x0, sp, #32 // x0 = &val - - bl C_FUNC(ArrayStoreCheck) // ArrayStoreCheck(&val, &array) - -LOCAL_LABEL(DoWrite): - EPILOG_RESTORE_REG_PAIR x0, x1, 16 - EPILOG_RESTORE_REG x2, 32 - EPILOG_RESTORE_REG_PAIR_INDEXED fp, lr, 48 - b C_FUNC(JIT_Stelem_DoWrite) -NESTED_END JIT_Stelem_Ref_NotExactMatch, _TEXT - -// ------------------------------------------------------------------ -// __declspec(naked) void F_CALL_CONV JIT_Stelem_DoWrite(PtrArray* array, unsigned idx, Object* val) -LEAF_ENTRY JIT_Stelem_DoWrite, _TEXT - - // Setup args for JIT_WriteBarrier. x14 = &array->m_array[idx] ; x15 = val - add x14, x0, #PtrArray__m_Array // x14 = &array->m_array - add x14, x14, x1, LSL #3 - mov x15, x2 // x15 = val - - // Branch to the write barrier (which is already correctly overwritten with - // single or multi-proc code based on the current CPU - b C_FUNC(JIT_WriteBarrier) -LEAF_END JIT_Stelem_DoWrite, _TEXT // ------------------------------------------------------------------ // __declspec(naked) void F_CALL_CONV JIT_WriteBarrier_Callable(Object **dst, Object* val) diff --git a/src/coreclr/src/vm/arm64/asmhelpers.asm b/src/coreclr/src/vm/arm64/asmhelpers.asm index b7305f5718dbcb..11e7d0fef0643b 100644 --- a/src/coreclr/src/vm/arm64/asmhelpers.asm +++ b/src/coreclr/src/vm/arm64/asmhelpers.asm @@ -36,11 +36,6 @@ IMPORT DynamicHelperWorker #endif - IMPORT ObjIsInstanceOfCached - IMPORT ArrayStoreCheck - SETALIAS g_pObjectClass, ?g_pObjectClass@@3PEAVMethodTable@@EA - IMPORT $g_pObjectClass - #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP IMPORT g_sw_ww_table #endif @@ -1521,101 +1516,6 @@ CallHelper2 ret lr LEAF_END -; ------------------------------------------------------------------ -;__declspec(naked) void F_CALL_CONV JIT_Stelem_Ref(PtrArray* array, unsigned idx, Object* val) - LEAF_ENTRY JIT_Stelem_Ref - ; We retain arguments as they were passed and use x0 == array x1 == idx x2 == val - - ; check for null array - cbz x0, ThrowNullReferenceException - - ; idx bounds check - ldr x3,[x0,#ArrayBase__m_NumComponents] - cmp x3, x1 - bls ThrowIndexOutOfRangeException - - ; fast path to null assignment (doesn't need any write-barriers) - cbz x2, AssigningNull - - ; Verify the array-type and val-type matches before writing - ldr x12, [x0] ; x12 = array MT - ldr x3, [x2] ; x3 = val->GetMethodTable() - ldr x12, [x12, #MethodTable__m_ElementType] ; array->GetArrayElementTypeHandle() - cmp x3, x12 - beq JIT_Stelem_DoWrite - - ; Types didnt match but allow writing into an array of objects - ldr x3, =$g_pObjectClass - ldr x3, [x3] ; x3 = *g_pObjectClass - cmp x3, x12 ; array type matches with Object* - beq JIT_Stelem_DoWrite - - ; array type and val type do not exactly match. Raise frame and do detailed match - b JIT_Stelem_Ref_NotExactMatch - -AssigningNull - ; Assigning null doesn't need write barrier - add x0, x0, x1, LSL #3 ; x0 = x0 + (x1 x 8) = array->m_array[idx] - str x2, [x0, #PtrArray__m_Array] ; array->m_array[idx] = val - ret - -ThrowNullReferenceException - ; Tail call JIT_InternalThrow(NullReferenceException) - ldr x0, =CORINFO_NullReferenceException_ASM - b JIT_InternalThrow - -ThrowIndexOutOfRangeException - ; Tail call JIT_InternalThrow(NullReferenceException) - ldr x0, =CORINFO_IndexOutOfRangeException_ASM - b JIT_InternalThrow - - LEAF_END - -; ------------------------------------------------------------------ -; __declspec(naked) void F_CALL_CONV JIT_Stelem_Ref_NotExactMatch(PtrArray* array, -; unsigned idx, Object* val) -; x12 = array->GetArrayElementTypeHandle() -; - NESTED_ENTRY JIT_Stelem_Ref_NotExactMatch - PROLOG_SAVE_REG_PAIR fp, lr, #-48! - stp x0, x1, [sp, #16] - str x2, [sp, #32] - - ; allow in case val can be casted to array element type - ; call ObjIsInstanceOfCached(val, array->GetArrayElementTypeHandle()) - mov x1, x12 ; array->GetArrayElementTypeHandle() - mov x0, x2 - bl ObjIsInstanceOfCached - cmp x0, TypeHandle_CanCast - beq DoWrite ; ObjIsInstance returned TypeHandle::CanCast - - ; check via raising frame -NeedFrame - add x1, sp, #16 ; x1 = &array - add x0, sp, #32 ; x0 = &val - - bl ArrayStoreCheck ; ArrayStoreCheck(&val, &array) - -DoWrite - ldp x0, x1, [sp, #16] - ldr x2, [sp, #32] - EPILOG_RESTORE_REG_PAIR fp, lr, #48! - EPILOG_BRANCH JIT_Stelem_DoWrite - NESTED_END - -; ------------------------------------------------------------------ -; __declspec(naked) void F_CALL_CONV JIT_Stelem_DoWrite(PtrArray* array, unsigned idx, Object* val) - LEAF_ENTRY JIT_Stelem_DoWrite - - ; Setup args for JIT_WriteBarrier. x14 = &array->m_array[idx] ; x15 = val - add x14, x0, #PtrArray__m_Array ; x14 = &array->m_array - add x14, x14, x1, LSL #3 - mov x15, x2 ; x15 = val - - ; Branch to the write barrier (which is already correctly overwritten with - ; single or multi-proc code based on the current CPU - b JIT_WriteBarrier - LEAF_END ; ------------------------------------------------------------------ ; __declspec(naked) void F_CALL_CONV JIT_WriteBarrier_Callable(Object **dst, Object* val) diff --git a/src/coreclr/src/vm/arm64/cgencpu.h b/src/coreclr/src/vm/arm64/cgencpu.h index 7f9bef9da9c41f..999a40df1e84fd 100644 --- a/src/coreclr/src/vm/arm64/cgencpu.h +++ b/src/coreclr/src/vm/arm64/cgencpu.h @@ -100,8 +100,6 @@ static_assert(((STACK_ELEM_SIZE & (STACK_ELEM_SIZE-1)) == 0), "STACK_ELEM_SIZE m #define JIT_GetSharedGCStaticBaseNoCtor JIT_GetSharedGCStaticBaseNoCtor_SingleAppDomain #define JIT_GetSharedNonGCStaticBaseNoCtor JIT_GetSharedNonGCStaticBaseNoCtor_SingleAppDomain -#define JIT_Stelem_Ref JIT_Stelem_Ref - //********************************************************************** // Frames //********************************************************************** diff --git a/src/coreclr/src/vm/i386/cgencpu.h b/src/coreclr/src/vm/i386/cgencpu.h index b9c924fcd4d9a5..3ab93a7eca42a7 100644 --- a/src/coreclr/src/vm/i386/cgencpu.h +++ b/src/coreclr/src/vm/i386/cgencpu.h @@ -547,7 +547,6 @@ inline BOOL ClrFlushInstructionCache(LPCVOID pCodeAddr, size_t sizeOfCode) #ifndef TARGET_UNIX #define JIT_NewCrossContext JIT_NewCrossContext -#define JIT_Stelem_Ref JIT_Stelem_Ref #endif // TARGET_UNIX //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/coreclr/src/vm/i386/jitinterfacex86.cpp b/src/coreclr/src/vm/i386/jitinterfacex86.cpp index 5a585cbd9cd100..d26e91b784b9e1 100644 --- a/src/coreclr/src/vm/i386/jitinterfacex86.cpp +++ b/src/coreclr/src/vm/i386/jitinterfacex86.cpp @@ -96,104 +96,6 @@ extern "C" void STDCALL WriteBarrierAssert(BYTE* ptr, Object* obj) #endif // _DEBUG #ifndef TARGET_UNIX -/****************************************************************************/ -/* assigns 'val to 'array[idx], after doing all the proper checks */ - -/* note that we can do almost as well in portable code, but this - squezes the last little bit of perf out */ - -__declspec(naked) void F_CALL_CONV JIT_Stelem_Ref(PtrArray* array, unsigned idx, Object* val) -{ - STATIC_CONTRACT_THROWS; - STATIC_CONTRACT_GC_TRIGGERS; - - enum { CanCast = TypeHandle::CanCast, - }; - - __asm { - mov EAX, [ESP+4] // EAX = val - - test ECX, ECX - je ThrowNullReferenceException - - cmp EDX, [ECX+4]; // test if in bounds - jae ThrowIndexOutOfRangeException - - test EAX, EAX - jz Assigning0 - - push EDX - mov EDX, [ECX] - mov EDX, [EDX]MethodTable.m_ElementTypeHnd - - cmp EDX, [EAX] // do we have an exact match - jne NotExactMatch - -DoWrite2: - pop EDX - lea EDX, [ECX + 4*EDX + 8] - call JIT_WriteBarrierEAX - ret 4 - -Assigning0: - // write barrier is not necessary for assignment of NULL references - mov [ECX + 4*EDX + 8], EAX - ret 4 - -DoWrite: - mov EAX, [ESP+4] // EAX = val - lea EDX, [ECX + 4*EDX + 8] - call JIT_WriteBarrierEAX - ret 4 - -NotExactMatch: - cmp EDX, [g_pObjectClass] // are we assigning to Array of objects - je DoWrite2 - - // push EDX // caller-save ECX and EDX - push ECX - - push EDX // element type handle - push EAX // object - - call ObjIsInstanceOfCached - - pop ECX // caller-restore ECX and EDX - pop EDX - - cmp EAX, CanCast - je DoWrite - - // Call the helper that knows how to erect a frame - push EDX - push ECX - - lea ECX, [ESP+8+4] // ECX = address of object being stored - lea EDX, [ESP] // EDX = address of array - - call ArrayStoreCheck - - pop ECX // these might have been updated! - pop EDX - - cmp EAX, EAX // set zero flag - jnz Epilog // This jump never happens, it keeps the epilog walker happy - - jmp DoWrite - -ThrowNullReferenceException: - mov ECX, CORINFO_NullReferenceException - jmp Throw - -ThrowIndexOutOfRangeException: - mov ECX, CORINFO_IndexOutOfRangeException - -Throw: - call JIT_InternalThrowFromHelper -Epilog: - ret 4 - } -} HCIMPL1_V(INT32, JIT_Dbl2IntOvf, double val) { diff --git a/src/coreclr/src/vm/i386/stublinkerx86.cpp b/src/coreclr/src/vm/i386/stublinkerx86.cpp index fe997620753558..3e9180cff240b3 100644 --- a/src/coreclr/src/vm/i386/stublinkerx86.cpp +++ b/src/coreclr/src/vm/i386/stublinkerx86.cpp @@ -4398,7 +4398,6 @@ VOID StubLinkerCPU::EmitArrayOpStub(const ArrayOpScript* pArrayOpScript) // Try to call the fast helper first ( ObjIsInstanceOfCached ). // If that fails we will fall back to calling the slow helper ( ArrayStoreCheck ) that erects a frame. - // See also JitInterfaceX86::JIT_Stelem_Ref #ifdef TARGET_AMD64 // RCX contains pointer to object to check (Object*) diff --git a/src/coreclr/src/vm/jithelpers.cpp b/src/coreclr/src/vm/jithelpers.cpp index f483aa00a128db..e7c0c7fd6ab141 100644 --- a/src/coreclr/src/vm/jithelpers.cpp +++ b/src/coreclr/src/vm/jithelpers.cpp @@ -2857,66 +2857,6 @@ HCIMPL2(LPVOID, ArrayStoreCheck, Object** pElement, PtrArray** pArray) } HCIMPLEND -/****************************************************************************/ -/* assigns 'val to 'array[idx], after doing all the proper checks */ - -HCIMPL3(void, JIT_Stelem_Ref_Portable, PtrArray* array, unsigned idx, Object *val) -{ - FCALL_CONTRACT; - - if (!array) - { - FCThrowVoid(kNullReferenceException); - } - if (idx >= array->GetNumComponents()) - { - FCThrowVoid(kIndexOutOfRangeException); - } - - if (val) - { - MethodTable *valMT = val->GetMethodTable(); - TypeHandle arrayElemTH = array->GetArrayElementTypeHandle(); - - if (arrayElemTH != TypeHandle(valMT) && arrayElemTH != TypeHandle(g_pObjectClass)) - { - TypeHandle::CastResult result = ObjIsInstanceOfCached(val, arrayElemTH); - if (result != TypeHandle::CanCast) - { - // FCALL_CONTRACT increase ForbidGC count. Normally, HELPER_METHOD_FRAME macros decrease the count. - // But to avoid perf hit, we manually decrease the count here before calling another HCCALL. - ENDFORBIDGC(); - - if (HCCALL2(ArrayStoreCheck,(Object**)&val, (PtrArray**)&array) != NULL) - { - // This return is never executed. It helps epilog walker to find its way out. - return; - } - } - } - -#ifdef TARGET_ARM64 - SetObjectReference((OBJECTREF*)&array->m_Array[idx], ObjectToOBJECTREF(val)); -#else - // The performance gain of the optimized JIT_Stelem_Ref in - // jitinterfacex86.cpp is mainly due to calling JIT_WriteBarrier - // By calling write barrier directly here, - // we can avoid translating in-line assembly from MSVC to gcc - // while keeping most of the performance gain. - HCCALL2(JIT_WriteBarrier, (Object **)&array->m_Array[idx], val); -#endif - - } - else - { - // no need to go through write-barrier for NULL - ClearObjectReference(&array->m_Array[idx]); - } -} -HCIMPLEND - - - //======================================================================== // // VALUETYPE/BYREF HELPERS diff --git a/src/coreclr/src/vm/jitinterface.h b/src/coreclr/src/vm/jitinterface.h index 30b7ad8e787e1e..79be98cde02daa 100644 --- a/src/coreclr/src/vm/jitinterface.h +++ b/src/coreclr/src/vm/jitinterface.h @@ -209,12 +209,6 @@ extern FCDECL2(Object*, JIT_NewArr1VC_MP_FastPortable, CORINFO_CLASS_HANDLE arra extern FCDECL2(Object*, JIT_NewArr1OBJ_MP_FastPortable, CORINFO_CLASS_HANDLE arrayMT, INT_PTR size); extern FCDECL2(Object*, JIT_NewArr1, CORINFO_CLASS_HANDLE arrayMT, INT_PTR size); -#ifndef JIT_Stelem_Ref -#define JIT_Stelem_Ref JIT_Stelem_Ref_Portable -#endif -EXTERN_C FCDECL3(void, JIT_Stelem_Ref, PtrArray* array, unsigned idx, Object* val); -EXTERN_C FCDECL3(void, JIT_Stelem_Ref_Portable, PtrArray* array, unsigned idx, Object* val); - EXTERN_C FCDECL_MONHELPER(JITutil_MonEnterWorker, Object* obj); EXTERN_C FCDECL2(void, JITutil_MonReliableEnter, Object* obj, BYTE* pbLockTaken); EXTERN_C FCDECL3(void, JITutil_MonTryEnter, Object* obj, INT32 timeOut, BYTE* pbLockTaken); diff --git a/src/coreclr/src/vm/threads.cpp b/src/coreclr/src/vm/threads.cpp index 63565b1b0a631e..13f1a379d779c5 100644 --- a/src/coreclr/src/vm/threads.cpp +++ b/src/coreclr/src/vm/threads.cpp @@ -1085,7 +1085,7 @@ void InitThreadManager() memcpy(s_barrierCopy, (BYTE*)JIT_PatchedCodeStart, (BYTE*)JIT_PatchedCodeLast - (BYTE*)JIT_PatchedCodeStart); - // Store the JIT_WriteBarrier copy location to a global variable so that the JIT_Stelem_Ref and its helpers + // Store the JIT_WriteBarrier copy location to a global variable so that helpers // can jump to it. JIT_WriteBarrier_Loc = GetWriteBarrierCodeLocation((void*)JIT_WriteBarrier); From ecdbaa979abbb704f798e912d4ad920912abd546 Mon Sep 17 00:00:00 2001 From: vsadov Date: Sat, 29 Feb 2020 21:54:46 -0800 Subject: [PATCH 04/11] removed JIT_Ldelema_Ref --- src/coreclr/src/vm/jithelpers.cpp | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/src/coreclr/src/vm/jithelpers.cpp b/src/coreclr/src/vm/jithelpers.cpp index e7c0c7fd6ab141..bb59a8015ab102 100644 --- a/src/coreclr/src/vm/jithelpers.cpp +++ b/src/coreclr/src/vm/jithelpers.cpp @@ -2806,32 +2806,6 @@ HCIMPL3(Object*, JIT_NewMDArrNonVarArg, CORINFO_CLASS_HANDLE classHnd, unsigned } HCIMPLEND -/*************************************************************/ -/* returns '&array[idx], after doing all the proper checks */ - -#include -HCIMPL3(void*, JIT_Ldelema_Ref, PtrArray* array, unsigned idx, CORINFO_CLASS_HANDLE type) -{ - FCALL_CONTRACT; - - RuntimeExceptionKind except; - // This has been carefully arranged to ensure that in the common - // case the branches are predicted properly (fall through). - // and that we dont spill registers unnecessarily etc. - if (array != 0) - if (idx < array->GetNumComponents()) - if (array->GetArrayElementTypeHandle() == TypeHandle(type)) - return(&array->m_Array[idx]); - else - except = kArrayTypeMismatchException; - else - except = kIndexOutOfRangeException; - else - except = kNullReferenceException; - - FCThrow(except); -} -HCIMPLEND #include //=========================================================================== From 94997d5915a92a9a9c542ba3640637fdcae54f51 Mon Sep 17 00:00:00 2001 From: vsadov Date: Sun, 1 Mar 2020 15:36:58 -0800 Subject: [PATCH 05/11] Attributes for stack/debug hiding. --- .../Runtime/CompilerServices/CastHelpers.cs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs b/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs index 49bf5a36ab3470..4bf43016dcabe0 100644 --- a/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs +++ b/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs @@ -501,11 +501,17 @@ internal struct ArrayElement public object? Value; } + [DebuggerHidden] + [StackTraceHidden] + [DebuggerStepThrough] private static ref object? ThrowArrayMismatchException() { throw new ArrayTypeMismatchException(); } + [DebuggerHidden] + [StackTraceHidden] + [DebuggerStepThrough] private static ref object? LdelemaRef(Array array, int index, void* type) { ArrayElement[] arr = Unsafe.As(array); @@ -523,6 +529,9 @@ internal struct ArrayElement return ref ThrowArrayMismatchException(); } + [DebuggerHidden] + [StackTraceHidden] + [DebuggerStepThrough] private static void StelemRef(Array array, int index, object? obj) { ArrayElement[] arr = Unsafe.As(array); @@ -552,6 +561,9 @@ private static void StelemRef(Array array, int index, object? obj) StelemRef_Helper(ref element, elementType, obj); } + [DebuggerHidden] + [StackTraceHidden] + [DebuggerStepThrough] private static void StelemRef_Helper(ref object? element, nuint elementType, object obj) { CastResult result = TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)elementType); @@ -564,6 +576,9 @@ private static void StelemRef_Helper(ref object? element, nuint elementType, obj StelemRef_Helper_Slow(ref element, elementType, obj); } + [DebuggerHidden] + [StackTraceHidden] + [DebuggerStepThrough] private static void StelemRef_Helper_Slow(ref object? element, nuint elementType, object obj) { Debug.Assert(obj != null); From d970d8ad8a4c14eab767cadffbc47a432e983f23 Mon Sep 17 00:00:00 2001 From: vsadov Date: Sun, 1 Mar 2020 20:39:20 -0800 Subject: [PATCH 06/11] PR feedback --- .../Runtime/CompilerServices/CastHelpers.cs | 32 +++++++++---------- .../RuntimeHelpers.CoreCLR.cs | 2 +- src/coreclr/src/vm/amd64/jithelpers_fast.S | 2 +- src/coreclr/src/vm/arm64/asmhelpers.S | 2 +- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs b/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs index 4bf43016dcabe0..e52a823ae7b2a2 100644 --- a/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs +++ b/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs @@ -21,7 +21,7 @@ namespace System.Runtime.CompilerServices internal static unsafe class CastHelpers { private static int[]? s_table; - private static readonly MethodTable* pObjMt = RuntimeHelpers.GetMethodTable(new object()); + private static readonly MethodTable* pObjMt = (MethodTable*)RuntimeTypeHandle.GetValueInternal(typeof(object).TypeHandle); [DebuggerDisplay("Source = {_source}; Target = {_targetAndResult & ~1}; Result = {_targetAndResult & 1}; VersionNum = {_version & ((1 << 29) - 1)}; Distance = {_version >> 29};")] [StructLayout(LayoutKind.Sequential)] @@ -253,7 +253,7 @@ private static CastResult TryGet(nuint source, nuint target) return obj; slowPath: - return IsInstanceHelper(toTypeHnd, obj); + return IsInstance_Helper(toTypeHnd, obj); } [DebuggerHidden] @@ -308,14 +308,14 @@ private static CastResult TryGet(nuint source, nuint target) return obj; slowPath: - return IsInstanceHelper(toTypeHnd, obj); + return IsInstance_Helper(toTypeHnd, obj); } [DebuggerHidden] [StackTraceHidden] [DebuggerStepThrough] [MethodImpl(MethodImplOptions.NoInlining)] - private static object? IsInstanceHelper(void* toTypeHnd, object obj) + private static object? IsInstance_Helper(void* toTypeHnd, object obj) { CastResult result = TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)toTypeHnd); if (result == CastResult.CanCast) @@ -368,7 +368,7 @@ private static CastResult TryGet(nuint source, nuint target) [StackTraceHidden] [DebuggerStepThrough] [MethodImpl(MethodImplOptions.NoInlining)] - private static object? ChkCastHelper(void* toTypeHnd, object obj) + private static object? ChkCast_Helper(void* toTypeHnd, object obj) { CastResult result = TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)toTypeHnd); if (result == CastResult.CanCast) @@ -420,7 +420,7 @@ private static CastResult TryGet(nuint source, nuint target) return obj; slowPath: - return ChkCastHelper(toTypeHnd, obj); + return ChkCast_Helper(toTypeHnd, obj); } [DebuggerHidden] @@ -481,7 +481,7 @@ private static CastResult TryGet(nuint source, nuint target) return obj; slowPath: - return ChkCastHelper(toTypeHnd, obj); + return ChkCast_Helper(toTypeHnd, obj); } [DebuggerHidden] @@ -517,10 +517,10 @@ internal struct ArrayElement ArrayElement[] arr = Unsafe.As(array); // this will throw appropriate exceptions if array is null or access is out of range. - nuint elementType = RuntimeHelpers.GetMethodTable(arr)->ElementType; + void* elementType = RuntimeHelpers.GetMethodTable(arr)->ElementType; ref object? element = ref arr[index].Value; - if (elementType != (nuint)type) + if (elementType != type) goto throwArrayMismatch; return ref element; @@ -537,13 +537,13 @@ private static void StelemRef(Array array, int index, object? obj) ArrayElement[] arr = Unsafe.As(array); // this will throw appropriate exceptions if array is null or access is out of range. - nuint elementType = RuntimeHelpers.GetMethodTable(arr)->ElementType; + void* elementType = RuntimeHelpers.GetMethodTable(arr)->ElementType; ref object? element = ref arr[index].Value; if (obj == null) goto assigningNull; - if (elementType != (nuint)RuntimeHelpers.GetMethodTable(obj)) + if (elementType != RuntimeHelpers.GetMethodTable(obj)) goto notExactMatch; doWrite: @@ -555,7 +555,7 @@ private static void StelemRef(Array array, int index, object? obj) return; notExactMatch: - if (elementType == (nuint)pObjMt) + if (elementType == pObjMt) goto doWrite; StelemRef_Helper(ref element, elementType, obj); @@ -564,7 +564,7 @@ private static void StelemRef(Array array, int index, object? obj) [DebuggerHidden] [StackTraceHidden] [DebuggerStepThrough] - private static void StelemRef_Helper(ref object? element, nuint elementType, object obj) + private static void StelemRef_Helper(ref object? element, void* elementType, object obj) { CastResult result = TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)elementType); if (result == CastResult.CanCast) @@ -573,17 +573,17 @@ private static void StelemRef_Helper(ref object? element, nuint elementType, obj return; } - StelemRef_Helper_Slow(ref element, elementType, obj); + StelemRef_Helper_NoCacheLookup(ref element, elementType, obj); } [DebuggerHidden] [StackTraceHidden] [DebuggerStepThrough] - private static void StelemRef_Helper_Slow(ref object? element, nuint elementType, object obj) + private static void StelemRef_Helper_NoCacheLookup(ref object? element, void* elementType, object obj) { Debug.Assert(obj != null); - obj = IsInstanceOfAny_NoCacheLookup((void*)elementType, obj); + obj = IsInstanceOfAny_NoCacheLookup(elementType, obj); if (obj != null) { WriteBarrier(ref element, obj); diff --git a/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs b/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs index eae8c8e697fed1..117e7e26dc6ea2 100644 --- a/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs +++ b/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs @@ -305,7 +305,7 @@ internal unsafe struct MethodTable [FieldOffset(ParentMethodTableOffset)] public MethodTable* ParentMethodTable; [FieldOffset(ElementTypeOffset)] - public nuint ElementType; + public void* ElementType; [FieldOffset(InterfaceMapOffset)] public MethodTable** InterfaceMap; diff --git a/src/coreclr/src/vm/amd64/jithelpers_fast.S b/src/coreclr/src/vm/amd64/jithelpers_fast.S index a51079931414ff..e55d60711810ba 100644 --- a/src/coreclr/src/vm/amd64/jithelpers_fast.S +++ b/src/coreclr/src/vm/amd64/jithelpers_fast.S @@ -406,7 +406,7 @@ LEAF_END_MARKED JIT_ByRefWriteBarrier, _TEXT LEAF_ENTRY JIT_WriteBarrier_Callable, _TEXT // JIT_WriteBarrier(Object** dst, Object* src) jmp [rip + C_FUNC(JIT_WriteBarrier_Loc)] -LEAF_END JIT_Stelem_DoWrite, _TEXT +LEAF_END JIT_WriteBarrier_Callable, _TEXT #endif // FEATURE_WRITEBARRIER_COPY diff --git a/src/coreclr/src/vm/arm64/asmhelpers.S b/src/coreclr/src/vm/arm64/asmhelpers.S index ce8b71b837a92c..4706319b30a622 100644 --- a/src/coreclr/src/vm/arm64/asmhelpers.S +++ b/src/coreclr/src/vm/arm64/asmhelpers.S @@ -1294,7 +1294,7 @@ LEAF_ENTRY JIT_WriteBarrier_Callable, _TEXT // Branch to the write barrier (which is already correctly overwritten with // single or multi-proc code based on the current CPU b C_FUNC(JIT_WriteBarrier) -LEAF_END JIT_Stelem_DoWrite, _TEXT +LEAF_END JIT_WriteBarrier_Callable, _TEXT #ifdef PROFILING_SUPPORTED From 35e548fb25296fb73b868b8d9ef332e7b6d21fc1 Mon Sep 17 00:00:00 2001 From: vsadov Date: Sun, 1 Mar 2020 22:20:42 -0800 Subject: [PATCH 07/11] Avoid needing static initializer. --- .../src/System/Runtime/CompilerServices/CastHelpers.cs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs b/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs index e52a823ae7b2a2..829500f7320a2c 100644 --- a/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs +++ b/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs @@ -21,7 +21,6 @@ namespace System.Runtime.CompilerServices internal static unsafe class CastHelpers { private static int[]? s_table; - private static readonly MethodTable* pObjMt = (MethodTable*)RuntimeTypeHandle.GetValueInternal(typeof(object).TypeHandle); [DebuggerDisplay("Source = {_source}; Target = {_targetAndResult & ~1}; Result = {_targetAndResult & 1}; VersionNum = {_version & ((1 << 29) - 1)}; Distance = {_version >> 29};")] [StructLayout(LayoutKind.Sequential)] @@ -534,11 +533,9 @@ internal struct ArrayElement [DebuggerStepThrough] private static void StelemRef(Array array, int index, object? obj) { - ArrayElement[] arr = Unsafe.As(array); - // this will throw appropriate exceptions if array is null or access is out of range. - void* elementType = RuntimeHelpers.GetMethodTable(arr)->ElementType; - ref object? element = ref arr[index].Value; + void* elementType = RuntimeHelpers.GetMethodTable(array)->ElementType; + ref object? element = ref Unsafe.As(array)[index].Value; if (obj == null) goto assigningNull; @@ -555,7 +552,7 @@ private static void StelemRef(Array array, int index, object? obj) return; notExactMatch: - if (elementType == pObjMt) + if (array.GetType() == typeof(object[])) goto doWrite; StelemRef_Helper(ref element, elementType, obj); From 590031e13af0169a0ee83bd4c099a0889452f81d Mon Sep 17 00:00:00 2001 From: vsadov Date: Mon, 2 Mar 2020 22:37:07 -0800 Subject: [PATCH 08/11] couple small tweaks --- .../Runtime/CompilerServices/CastHelpers.cs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs b/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs index 829500f7320a2c..857e24877ed737 100644 --- a/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs +++ b/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs @@ -513,19 +513,14 @@ internal struct ArrayElement [DebuggerStepThrough] private static ref object? LdelemaRef(Array array, int index, void* type) { - ArrayElement[] arr = Unsafe.As(array); - // this will throw appropriate exceptions if array is null or access is out of range. - void* elementType = RuntimeHelpers.GetMethodTable(arr)->ElementType; - ref object? element = ref arr[index].Value; - - if (elementType != type) - goto throwArrayMismatch; + ref object? element = ref Unsafe.As(array)[index].Value; + void* elementType = RuntimeHelpers.GetMethodTable(array)->ElementType; - return ref element; + if (elementType == type) + return ref element; - throwArrayMismatch: - return ref ThrowArrayMismatchException(); + return ref ThrowArrayMismatchException(); } [DebuggerHidden] @@ -534,8 +529,8 @@ internal struct ArrayElement private static void StelemRef(Array array, int index, object? obj) { // this will throw appropriate exceptions if array is null or access is out of range. - void* elementType = RuntimeHelpers.GetMethodTable(array)->ElementType; ref object? element = ref Unsafe.As(array)[index].Value; + void* elementType = RuntimeHelpers.GetMethodTable(array)->ElementType; if (obj == null) goto assigningNull; From a8bf8de147890b732c9d65cb3061ef03daf62034 Mon Sep 17 00:00:00 2001 From: vsadov Date: Tue, 3 Mar 2020 18:44:30 -0800 Subject: [PATCH 09/11] Avoid PreStub indirection in array element JIT helpers. --- .../Runtime/CompilerServices/CastHelpers.cs | 2 ++ src/coreclr/src/vm/appdomain.cpp | 14 +++++++------- src/coreclr/src/vm/ecall.cpp | 16 ++++++++++++++-- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs b/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs index 857e24877ed737..660b96e7a00040 100644 --- a/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs +++ b/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs @@ -511,6 +511,7 @@ internal struct ArrayElement [DebuggerHidden] [StackTraceHidden] [DebuggerStepThrough] + [MethodImpl(MethodImplOptions.AggressiveOptimization)] private static ref object? LdelemaRef(Array array, int index, void* type) { // this will throw appropriate exceptions if array is null or access is out of range. @@ -526,6 +527,7 @@ internal struct ArrayElement [DebuggerHidden] [StackTraceHidden] [DebuggerStepThrough] + [MethodImpl(MethodImplOptions.AggressiveOptimization)] private static void StelemRef(Array array, int index, object? obj) { // this will throw appropriate exceptions if array is null or access is out of range. diff --git a/src/coreclr/src/vm/appdomain.cpp b/src/coreclr/src/vm/appdomain.cpp index 241e1b4bd61673..34c1e3358c9499 100644 --- a/src/coreclr/src/vm/appdomain.cpp +++ b/src/coreclr/src/vm/appdomain.cpp @@ -1986,13 +1986,6 @@ void SystemDomain::LoadBaseSystemClasses() g_TypedReferenceMT = MscorlibBinder::GetClass(CLASS__TYPED_REFERENCE); - // further loading of nonprimitive types may need casting support. - // initialize cast cache here. -#ifndef CROSSGEN_COMPILE - CastCache::Initialize(); - ECall::PopulateManagedCastHelpers(); -#endif // CROSSGEN_COMPILE - // unfortunately, the following cannot be delay loaded since the jit // uses it to compute method attributes within a function that cannot // handle Complus exception and the following call goes through a path @@ -2006,6 +1999,13 @@ void SystemDomain::LoadBaseSystemClasses() CrossLoaderAllocatorHashSetup::EnsureTypesLoaded(); #endif + // further loading of nonprimitive types may need casting support. + // initialize cast cache here. +#ifndef CROSSGEN_COMPILE + CastCache::Initialize(); + ECall::PopulateManagedCastHelpers(); +#endif // CROSSGEN_COMPILE + // used by IsImplicitInterfaceOfSZArray MscorlibBinder::GetClass(CLASS__IENUMERABLEGENERIC); MscorlibBinder::GetClass(CLASS__ICOLLECTIONGENERIC); diff --git a/src/coreclr/src/vm/ecall.cpp b/src/coreclr/src/vm/ecall.cpp index bae203456f91fe..86497a89c1cd23 100644 --- a/src/coreclr/src/vm/ecall.cpp +++ b/src/coreclr/src/vm/ecall.cpp @@ -195,12 +195,24 @@ void ECall::PopulateManagedCastHelpers() pDest = pMD->GetMultiCallableAddrOfCode(); SetJitHelperFunction(CORINFO_HELP_UNBOX, pDest); + // Array element accessors are more perf sensitive than other managed helpers and indirection + // costs introduced by PreStub could be noticeable (7% to 30% depending on platform). + // Other helpers are either more complex, less common, or have their trivial case inlined by the JIT, + // so indirection is not as big concern. + // We JIT-compile the following helpers eagerly here to avoid indirection costs. + pMD = MscorlibBinder::GetMethod((BinderMethodID)(METHOD__CASTHELPERS__STELEMREF)); - pDest = pMD->GetMultiCallableAddrOfCode(); + pMD->DoPrestub(NULL); + // This helper is marked AggressiveOptimization and its native code is in its final form. + // Get the code directly to avoid PreStub indirection. + pDest = pMD->GetNativeCode(); SetJitHelperFunction(CORINFO_HELP_ARRADDR_ST, pDest); pMD = MscorlibBinder::GetMethod((BinderMethodID)(METHOD__CASTHELPERS__LDELEMAREF)); - pDest = pMD->GetMultiCallableAddrOfCode(); + pMD->DoPrestub(NULL); + // This helper is marked AggressiveOptimization and its native code is in its final form. + // Get the code directly to avoid PreStub indirection. + pDest = pMD->GetNativeCode(); SetJitHelperFunction(CORINFO_HELP_LDELEMA_REF, pDest); #endif //CROSSGEN_COMPILE } From f412beba9614d5a25c841dcdf0ec813bad92b2a4 Mon Sep 17 00:00:00 2001 From: vsadov Date: Tue, 3 Mar 2020 22:12:51 -0800 Subject: [PATCH 10/11] Initialize `gcCoverLock` a bit earlier --- src/coreclr/src/vm/ceemain.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/coreclr/src/vm/ceemain.cpp b/src/coreclr/src/vm/ceemain.cpp index cad8bc2154b139..3f500009422d4c 100644 --- a/src/coreclr/src/vm/ceemain.cpp +++ b/src/coreclr/src/vm/ceemain.cpp @@ -964,6 +964,10 @@ void EEStartupHelper(COINITIEE fFlags) SystemDomain::System()->PublishAppDomainAndInformDebugger(SystemDomain::System()->DefaultDomain()); #endif +#ifdef HAVE_GCCOVER + MethodDesc::Init(); +#endif + #endif // CROSSGEN_COMPILE SystemDomain::System()->Init(); @@ -1044,10 +1048,6 @@ void EEStartupHelper(COINITIEE fFlags) #endif // _DEBUG -#ifdef HAVE_GCCOVER - MethodDesc::Init(); -#endif - #endif // !CROSSGEN_COMPILE ErrExit: ; From 00a91b89753231f9df19f143a68df1d4f4913de1 Mon Sep 17 00:00:00 2001 From: vsadov Date: Wed, 4 Mar 2020 11:14:54 -0800 Subject: [PATCH 11/11] Added comment about revisiting after #5857 is fixed --- src/coreclr/src/vm/ecall.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/coreclr/src/vm/ecall.cpp b/src/coreclr/src/vm/ecall.cpp index 86497a89c1cd23..7231cd3ff8f15a 100644 --- a/src/coreclr/src/vm/ecall.cpp +++ b/src/coreclr/src/vm/ecall.cpp @@ -201,6 +201,9 @@ void ECall::PopulateManagedCastHelpers() // so indirection is not as big concern. // We JIT-compile the following helpers eagerly here to avoid indirection costs. + //TODO: revise if this specialcasing is still needed when crossgen supports tailcall optimizations + // see: https://github.com/dotnet/runtime/issues/5857 + pMD = MscorlibBinder::GetMethod((BinderMethodID)(METHOD__CASTHELPERS__STELEMREF)); pMD->DoPrestub(NULL); // This helper is marked AggressiveOptimization and its native code is in its final form.