From 2bbc0dce126b38922682fe7f5f46d805029d438f Mon Sep 17 00:00:00 2001 From: dotnet-bot Date: Fri, 7 Apr 2017 09:15:14 -0700 Subject: [PATCH 1/7] ProjectX: Hook up class lib functions with type manager The class lib functions are already hooked up to code manager. However, there are cases where we look for class lib functions from EEtype, such as GetClasslibException on an eeype. This change attaches the class lib function array to every type manager and sets up a runtime helper to get to the class lib function GetRuntimeException from an EEtype. [tfs-changeset: 1653592] --- .../CompilerHelpers/StartupCodeHelpers.cs | 8 ++-- src/Native/Bootstrap/main.cpp | 8 ++-- src/Native/Runtime/EHHelpers.cpp | 29 +++++++++++- src/Native/Runtime/RuntimeInstance.cpp | 4 +- src/Native/Runtime/TypeManager.cpp | 19 ++++++-- src/Native/Runtime/TypeManager.h | 7 ++- .../src/System/Runtime/EEType.Runtime.cs | 2 +- .../src/System/Runtime/ExceptionHandling.cs | 44 +++++++++++++++++-- .../src/System/Runtime/InternalCalls.cs | 9 +++- .../src/System/Runtime/RuntimeImports.cs | 2 +- .../src/System/Runtime/RuntimeImports.cs | 2 +- 11 files changed, 107 insertions(+), 27 deletions(-) diff --git a/src/Common/src/Internal/Runtime/CompilerHelpers/StartupCodeHelpers.cs b/src/Common/src/Internal/Runtime/CompilerHelpers/StartupCodeHelpers.cs index 4102beb5a5e..a91aef630f2 100644 --- a/src/Common/src/Internal/Runtime/CompilerHelpers/StartupCodeHelpers.cs +++ b/src/Common/src/Internal/Runtime/CompilerHelpers/StartupCodeHelpers.cs @@ -25,10 +25,10 @@ public static TypeManagerHandle[] Modules } [NativeCallable(EntryPoint = "InitializeModules", CallingConvention = CallingConvention.Cdecl)] - internal static void InitializeModules(IntPtr osModule, IntPtr moduleHeaders, int count) + internal static unsafe void InitializeModules(IntPtr osModule, IntPtr moduleHeaders, int count, IntPtr* pClasslibFunctions, int nClasslibFunctions) { RuntimeImports.RhpRegisterOsModule(osModule); - TypeManagerHandle[] modules = CreateTypeManagers(osModule, moduleHeaders, count); + TypeManagerHandle[] modules = CreateTypeManagers(osModule, moduleHeaders, count, pClasslibFunctions, nClasslibFunctions); for (int i = 0; i < modules.Length; i++) { @@ -48,7 +48,7 @@ internal static void InitializeModules(IntPtr osModule, IntPtr moduleHeaders, in } } - private static unsafe TypeManagerHandle[] CreateTypeManagers(IntPtr osModule, IntPtr moduleHeaders, int count) + private static unsafe TypeManagerHandle[] CreateTypeManagers(IntPtr osModule, IntPtr moduleHeaders, int count, IntPtr* pClasslibFunctions, int nClasslibFunctions) { // Count the number of modules so we can allocate an array to hold the TypeManager objects. // At this stage of startup, complex collection classes will not work. @@ -67,7 +67,7 @@ private static unsafe TypeManagerHandle[] CreateTypeManagers(IntPtr osModule, In for (int i = 0; i < count; i++) { if (((IntPtr*)moduleHeaders)[i] != IntPtr.Zero) - modules[moduleIndex++] = RuntimeImports.RhpCreateTypeManager(osModule, ((IntPtr*)moduleHeaders)[i]); + modules[moduleIndex++] = RuntimeImports.RhpCreateTypeManager(osModule, ((IntPtr*)moduleHeaders)[i], pClasslibFunctions, nClasslibFunctions); } return modules; diff --git a/src/Native/Bootstrap/main.cpp b/src/Native/Bootstrap/main.cpp index a4d77ab9338..232cd8c6d0d 100644 --- a/src/Native/Bootstrap/main.cpp +++ b/src/Native/Bootstrap/main.cpp @@ -211,7 +211,7 @@ extern "C" void RhpUniversalTransition_DebugStepTailCall() } extern "C" void CCWAddRef() { - throw "CCWAddRef"; + throw "CCWAddRef"; } void* RtRHeaderWrapper(); @@ -266,7 +266,7 @@ static const pfn c_classlibFunctions[] = { #endif // !CPPCODEGEN -extern "C" void InitializeModules(void* osModule, void ** modules, int count); +extern "C" void InitializeModules(void* osModule, void ** modules, int count, void ** pClasslibFunctions, int nClasslibFunctions); #if defined(_WIN32) extern "C" int __managed__Main(int argc, wchar_t* argv[]); @@ -308,9 +308,9 @@ int main(int argc, char* argv[]) #endif #ifndef CPPCODEGEN - InitializeModules(osModule, __modules_a, (int)((__modules_z - __modules_a))); + InitializeModules(osModule, __modules_a, (int)((__modules_z - __modules_a)), (void **)&c_classlibFunctions, _countof(c_classlibFunctions)); #else // !CPPCODEGEN - InitializeModules(nullptr, (void**)RtRHeaderWrapper(), 2); + InitializeModules(nullptr, (void**)RtRHeaderWrapper(), 2, nullptr, 0); #endif // !CPPCODEGEN int retval; diff --git a/src/Native/Runtime/EHHelpers.cpp b/src/Native/Runtime/EHHelpers.cpp index 3e0887c648c..64b5f27ce0c 100644 --- a/src/Native/Runtime/EHHelpers.cpp +++ b/src/Native/Runtime/EHHelpers.cpp @@ -25,6 +25,8 @@ #include "threadstore.h" #include "threadstore.inl" #include "stressLog.h" +#include "rhbinder.h" +#include "eetype.h" // Find the code manager containing the given address, which might be a return address from a managed function. The // address may be to another managed function, or it may be to an unmanaged function. The address may also refer to @@ -38,7 +40,6 @@ static ICodeManager * FindCodeManagerForClasslibFunction(void * address) if (pCodeManager != NULL) return pCodeManager; - // @TODO: CORERT: Do we need to make this work for CoreRT? // Less common, we will look for the address in any of the sections of the module. This is slower, but is // necessary for EEType pointers and jump stubs. Module * pModule = pRI->FindModuleByAddress(address); @@ -67,7 +68,7 @@ COOP_PINVOKE_HELPER(Boolean, RhpEHEnumNext, (EHEnum* pEHEnum, EHClause* pEHClaus // Unmanaged helper to locate one of two classlib-provided functions that the runtime needs to // implement throwing of exceptions out of Rtm, and fail-fast. This may return NULL if the classlib // found via the provided address does not have the necessary exports. -COOP_PINVOKE_HELPER(void *, RhpGetClasslibFunction, (void * address, ClasslibFunctionId functionId)) +COOP_PINVOKE_HELPER(void *, RhpGetClasslibFunctionFromCodeAddress, (void * address, ClasslibFunctionId functionId)) { // Find the code manager for the given address, which is an address into some managed module. It could // be code, or it could be an EEType. No matter what, it's an address into a managed module in some non-Rtm @@ -83,6 +84,30 @@ COOP_PINVOKE_HELPER(void *, RhpGetClasslibFunction, (void * address, ClasslibFun return pCodeManager->GetClasslibFunction(functionId); } +// Unmanaged helper to locate one of two classlib-provided functions that the runtime needs to +// implement throwing of exceptions out of Rtm, and fail-fast. This may return NULL if the classlib +// found via the provided address does not have the necessary exports. +COOP_PINVOKE_HELPER(void *, RhpGetClasslibFunctionFromEEtype, (EEType * pEEtype, ClasslibFunctionId functionId)) +{ + if (pEEtype->HasTypeManager()) + { + return pEEtype->GetTypeManagerPtr()->AsTypeManager()->GetClasslibFunction(functionId); + } + else + { + RuntimeInstance * pRI = GetRuntimeInstance(); + Module * pModule = pRI->FindModuleByAddress(pEEtype); + if (pModule != NULL) + { + return pModule->GetClasslibFunction(functionId); + } + else + { + return NULL; + } + } +} + COOP_PINVOKE_HELPER(void, RhpValidateExInfoStack, ()) { Thread * pThisThread = ThreadStore::GetCurrentThread(); diff --git a/src/Native/Runtime/RuntimeInstance.cpp b/src/Native/Runtime/RuntimeInstance.cpp index bb0fff7fd18..665b776366c 100644 --- a/src/Native/Runtime/RuntimeInstance.cpp +++ b/src/Native/Runtime/RuntimeInstance.cpp @@ -429,9 +429,9 @@ bool RuntimeInstance::RegisterTypeManager(TypeManager * pTypeManager) return true; } -COOP_PINVOKE_HELPER(TypeManagerHandle, RhpCreateTypeManager, (HANDLE osModule, void* pModuleHeader)) +COOP_PINVOKE_HELPER(TypeManagerHandle, RhpCreateTypeManager, (HANDLE osModule, void* pModuleHeader, PTR_PTR_VOID pClasslibFunctions, UInt32 nClasslibFunctions)) { - TypeManager * typeManager = TypeManager::Create(osModule, pModuleHeader); + TypeManager * typeManager = TypeManager::Create(osModule, pModuleHeader, pClasslibFunctions, nClasslibFunctions); GetRuntimeInstance()->RegisterTypeManager(typeManager); return TypeManagerHandle::Create(typeManager); } diff --git a/src/Native/Runtime/TypeManager.cpp b/src/Native/Runtime/TypeManager.cpp index b09de7fba0a..2b0da3f140e 100644 --- a/src/Native/Runtime/TypeManager.cpp +++ b/src/Native/Runtime/TypeManager.cpp @@ -24,7 +24,7 @@ #include "TypeManager.h" /* static */ -TypeManager * TypeManager::Create(HANDLE osModule, void * pModuleHeader) +TypeManager * TypeManager::Create(HANDLE osModule, void * pModuleHeader, void** pClasslibFunctions, UInt32 nClasslibFunctions) { ReadyToRunHeader * pReadyToRunHeader = (ReadyToRunHeader *)pModuleHeader; @@ -38,11 +38,12 @@ TypeManager * TypeManager::Create(HANDLE osModule, void * pModuleHeader) if (pReadyToRunHeader->MajorVersion != ReadyToRunHeaderConstants::CurrentMajorVersion) return nullptr; - return new (nothrow) TypeManager(osModule, pReadyToRunHeader); + return new (nothrow) TypeManager(osModule, pReadyToRunHeader, pClasslibFunctions, nClasslibFunctions); } -TypeManager::TypeManager(HANDLE osModule, ReadyToRunHeader * pHeader) - : m_osModule(osModule), m_pHeader(pHeader), m_pDispatchMapTable(nullptr) +TypeManager::TypeManager(HANDLE osModule, ReadyToRunHeader * pHeader, void** pClasslibFunctions, UInt32 nClasslibFunctions) + : m_osModule(osModule), m_pHeader(pHeader), m_pDispatchMapTable(nullptr), + m_pClasslibFunctions(pClasslibFunctions), m_nClasslibFunctions(nClasslibFunctions) { int length; m_pStaticsGCDataSection = (UInt8*)GetModuleSection(ReadyToRunSectionType::GCStaticRegion, &length); @@ -73,6 +74,16 @@ void * TypeManager::GetModuleSection(ReadyToRunSectionType sectionId, int * leng return nullptr; } +void * TypeManager::GetClasslibFunction(ClasslibFunctionId functionId) +{ + uint32_t id = (uint32_t)functionId; + + if (id >= m_nClasslibFunctions) + return nullptr; + + return m_pClasslibFunctions[id]; +} + DispatchMap** TypeManager::GetDispatchMapLookupTable() { if (m_pDispatchMapTable == nullptr) diff --git a/src/Native/Runtime/TypeManager.h b/src/Native/Runtime/TypeManager.h index df095a33695..1a5baa216a0 100644 --- a/src/Native/Runtime/TypeManager.h +++ b/src/Native/Runtime/TypeManager.h @@ -18,15 +18,18 @@ class TypeManager UInt8* m_pStaticsGCDataSection; UInt8* m_pThreadStaticsDataSection; UInt32* m_pTlsIndex; // Pointer to TLS index if this module uses thread statics + void** m_pClasslibFunctions; + UInt32 m_nClasslibFunctions; - TypeManager(HANDLE osModule, ReadyToRunHeader * pHeader); + TypeManager(HANDLE osModule, ReadyToRunHeader * pHeader, void** pClasslibFunctions, UInt32 nClasslibFunctions); public: - static TypeManager * Create(HANDLE osModule, void * pModuleHeader); + static TypeManager * Create(HANDLE osModule, void * pModuleHeader, void** pClasslibFunctions, UInt32 nClasslibFunctions); void * GetModuleSection(ReadyToRunSectionType sectionId, int * length); DispatchMap ** GetDispatchMapLookupTable(); void EnumStaticGCRefs(void * pfnCallback, void * pvCallbackData); HANDLE GetOsModuleHandle(); + void* GetClasslibFunction(ClasslibFunctionId functionId); private: diff --git a/src/Runtime.Base/src/System/Runtime/EEType.Runtime.cs b/src/Runtime.Base/src/System/Runtime/EEType.Runtime.cs index 4463ed6b85e..b517cb77f78 100644 --- a/src/Runtime.Base/src/System/Runtime/EEType.Runtime.cs +++ b/src/Runtime.Base/src/System/Runtime/EEType.Runtime.cs @@ -50,7 +50,7 @@ internal Exception GetClasslibException(ExceptionIDs id) return RelatedParameterType->GetClasslibException(id); } - return EH.GetClasslibException(id, GetAssociatedModuleAddress()); + return EH.GetClasslibExceptionFromEEType(id, GetAssociatedModuleAddress()); #endif } diff --git a/src/Runtime.Base/src/System/Runtime/ExceptionHandling.cs b/src/Runtime.Base/src/System/Runtime/ExceptionHandling.cs index 48eb0c950ab..c4f7b07c3c6 100644 --- a/src/Runtime.Base/src/System/Runtime/ExceptionHandling.cs +++ b/src/Runtime.Base/src/System/Runtime/ExceptionHandling.cs @@ -162,7 +162,7 @@ internal static void FailFastViaClasslib(RhFailFastReason reason, object unhandl { // Find the classlib function that will fail fast. This is a RuntimeExport function from the // classlib module, and is therefore managed-callable. - IntPtr pFailFastFunction = (IntPtr)InternalCalls.RhpGetClasslibFunction(classlibAddress, + IntPtr pFailFastFunction = (IntPtr)InternalCalls.RhpGetClasslibFunctionFromCodeAddress(classlibAddress, ClassLibFunctionId.FailFast); if (pFailFastFunction == IntPtr.Zero) @@ -214,7 +214,7 @@ internal static unsafe void UnhandledExceptionFailFastViaClasslib( RhFailFastReason reason, object unhandledException, IntPtr classlibAddress, ref ExInfo exInfo) { IntPtr pFailFastFunction = - (IntPtr)InternalCalls.RhpGetClasslibFunction(classlibAddress, ClassLibFunctionId.FailFast); + (IntPtr)InternalCalls.RhpGetClasslibFunctionFromCodeAddress(classlibAddress, ClassLibFunctionId.FailFast); if (pFailFastFunction == IntPtr.Zero) { @@ -253,7 +253,7 @@ private enum RhEHFrameType private static void AppendExceptionStackFrameViaClasslib(object exception, IntPtr IP, ref bool isFirstRethrowFrame, ref bool isFirstFrame) { - IntPtr pAppendStackFrame = (IntPtr)InternalCalls.RhpGetClasslibFunction(IP, + IntPtr pAppendStackFrame = (IntPtr)InternalCalls.RhpGetClasslibFunctionFromCodeAddress(IP, ClassLibFunctionId.AppendExceptionStackFrame); if (pAppendStackFrame != IntPtr.Zero) @@ -284,7 +284,7 @@ internal static Exception GetClasslibException(ExceptionIDs id, IntPtr address) // Find the classlib function that will give us the exception object we want to throw. This // is a RuntimeExport function from the classlib module, and is therefore managed-callable. IntPtr pGetRuntimeExceptionFunction = - (IntPtr)InternalCalls.RhpGetClasslibFunction(address, ClassLibFunctionId.GetRuntimeException); + (IntPtr)InternalCalls.RhpGetClasslibFunctionFromCodeAddress(address, ClassLibFunctionId.GetRuntimeException); // Return the exception object we get from the classlib. Exception e = null; @@ -309,6 +309,42 @@ internal static Exception GetClasslibException(ExceptionIDs id, IntPtr address) return e; } + // Given an ExceptionID and an EEtype address, get an exception object of a type that the module containing + // the given address will understand. This finds the classlib-defined GetRuntimeException function and asks + // it for the exception object. + internal static Exception GetClasslibExceptionFromEEType(ExceptionIDs id, IntPtr pEEType) + { + // Find the classlib function that will give us the exception object we want to throw. This + // is a RuntimeExport function from the classlib module, and is therefore managed-callable. + IntPtr pGetRuntimeExceptionFunction = IntPtr.Zero; + if (pEEType != IntPtr.Zero) + { + pGetRuntimeExceptionFunction = (IntPtr)InternalCalls.RhpGetClasslibFunctionFromEEtype(pEEType, ClassLibFunctionId.GetRuntimeException); + } + + // Return the exception object we get from the classlib. + Exception e = null; + try + { + e = CalliIntrinsics.Call(pGetRuntimeExceptionFunction, id); + } + catch + { + // disallow all exceptions leaking out of callbacks + } + + // If the helper fails to yield an object, then we fail-fast. + if (e == null) + { + FailFastViaClasslib( + RhFailFastReason.ClassLibDidNotTranslateExceptionID, + null, + pEEType); + } + + return e; + } + // RhExceptionHandling_ functions are used to throw exceptions out of our asm helpers. We tail-call from // the asm helpers to these functions, which performs the throw. The tail-call is important: it ensures that // the stack is crawlable from within these functions. diff --git a/src/Runtime.Base/src/System/Runtime/InternalCalls.cs b/src/Runtime.Base/src/System/Runtime/InternalCalls.cs index 3d420ef7a49..26bc3336ef3 100644 --- a/src/Runtime.Base/src/System/Runtime/InternalCalls.cs +++ b/src/Runtime.Base/src/System/Runtime/InternalCalls.cs @@ -224,10 +224,15 @@ internal static Int32 RhEndNoGCRegion() [ManuallyManaged(GcPollPolicy.Never)] internal extern static unsafe IntPtr RhpUpdateDispatchCellCache(IntPtr pCell, IntPtr pTargetCode, EEType* pInstanceType, ref DispatchCellInfo newCellInfo); - [RuntimeImport(Redhawk.BaseName, "RhpGetClasslibFunction")] + [RuntimeImport(Redhawk.BaseName, "RhpGetClasslibFunctionFromCodeAddress")] [MethodImpl(MethodImplOptions.InternalCall)] [ManuallyManaged(GcPollPolicy.Never)] - internal extern static unsafe void* RhpGetClasslibFunction(IntPtr address, EH.ClassLibFunctionId id); + internal extern static unsafe void* RhpGetClasslibFunctionFromCodeAddress(IntPtr address, EH.ClassLibFunctionId id); + + [RuntimeImport(Redhawk.BaseName, "RhpGetClasslibFunctionFromEEtype")] + [MethodImpl(MethodImplOptions.InternalCall)] + [ManuallyManaged(GcPollPolicy.Never)] + internal extern static unsafe void* RhpGetClasslibFunctionFromEEtype(IntPtr pEEType, EH.ClassLibFunctionId id); // // StackFrameIterator diff --git a/src/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs b/src/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs index 52c62b5981d..1df7edb1692 100644 --- a/src/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs +++ b/src/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs @@ -468,7 +468,7 @@ internal static unsafe bool RhFindBlob(TypeManagerHandle typeManagerHandle, uint [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "RhpCreateTypeManager")] - internal static extern unsafe TypeManagerHandle RhpCreateTypeManager(IntPtr osModule, IntPtr moduleHeader); + internal static extern unsafe TypeManagerHandle RhpCreateTypeManager(IntPtr osModule, IntPtr moduleHeader, IntPtr* pClasslibFunctions, int nClasslibFunctions); [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "RhpRegisterOsModule")] diff --git a/src/Test.CoreLib/src/System/Runtime/RuntimeImports.cs b/src/Test.CoreLib/src/System/Runtime/RuntimeImports.cs index 1c8c1e35f39..ecdda3bdc07 100644 --- a/src/Test.CoreLib/src/System/Runtime/RuntimeImports.cs +++ b/src/Test.CoreLib/src/System/Runtime/RuntimeImports.cs @@ -56,7 +56,7 @@ internal static IntPtr RhGetModuleSection(TypeManagerHandle module, ReadyToRunSe [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "RhpCreateTypeManager")] - internal static extern unsafe TypeManagerHandle RhpCreateTypeManager(IntPtr osModule, IntPtr moduleHeader); + internal static extern unsafe TypeManagerHandle RhpCreateTypeManager(IntPtr osModule, IntPtr moduleHeader, IntPtr* pClasslibFunctions, int nClasslibFunctions); [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "RhpRegisterOsModule")] From 2c030fb7b54c3a7175b47805cae2afc37b37a800 Mon Sep 17 00:00:00 2001 From: Brian Robbins Date: Fri, 7 Apr 2017 10:25:37 -0700 Subject: [PATCH 2/7] Fix for uapaot build in corefx from breaks due to EventSource (CS # 1653496). [tfs-changeset: 1653598] --- .../shared/System/Diagnostics/Tracing/EventSourceException.cs | 4 ++-- .../shared/System/Diagnostics/Tracing/StubEnvironment.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/System.Private.CoreLib/shared/System/Diagnostics/Tracing/EventSourceException.cs b/src/System.Private.CoreLib/shared/System/Diagnostics/Tracing/EventSourceException.cs index 89d3ee5554d..88b8da93cdd 100644 --- a/src/System.Private.CoreLib/shared/System/Diagnostics/Tracing/EventSourceException.cs +++ b/src/System.Private.CoreLib/shared/System/Diagnostics/Tracing/EventSourceException.cs @@ -18,7 +18,7 @@ namespace System.Diagnostics.Tracing /// /// Exception that is thrown when an error occurs during EventSource operation. /// -#if (!ES_BUILD_PCL && !ES_BUILD_PN) +#if !ES_BUILD_PCL [Serializable] #endif public class EventSourceException : Exception @@ -40,7 +40,7 @@ public EventSourceException(string message) : base(message) { } /// public EventSourceException(string message, Exception innerException) : base(message, innerException) { } -#if (!ES_BUILD_PCL && !ES_BUILD_PN) +#if !ES_BUILD_PCL /// /// Initializes a new instance of the EventSourceException class with serialized data. /// diff --git a/src/System.Private.CoreLib/shared/System/Diagnostics/Tracing/StubEnvironment.cs b/src/System.Private.CoreLib/shared/System/Diagnostics/Tracing/StubEnvironment.cs index add2812038e..b365841d5f6 100644 --- a/src/System.Private.CoreLib/shared/System/Diagnostics/Tracing/StubEnvironment.cs +++ b/src/System.Private.CoreLib/shared/System/Diagnostics/Tracing/StubEnvironment.cs @@ -370,7 +370,7 @@ public PermissionSetAttribute(System.Security.SecurityAction action) { } #if ES_BUILD_PN namespace System { - public static class AppDomain + internal static class AppDomain { public static int GetCurrentThreadId() { From adb2909feea5492c50db49d525350dd1ff01f043 Mon Sep 17 00:00:00 2001 From: Atsushi Kanamori Date: Fri, 7 Apr 2017 11:29:11 -0700 Subject: [PATCH 3/7] NotImplementedException annotation. (#3254) Added github comments to NotImplementedExceptions (that I know about) and removed a few false-positive NotImplementationException calls. --- src/System.Private.CoreLib/src/System/IO/Stream.cs | 4 ++-- .../src/System/Reflection/Assembly.CoreRT.cs | 2 +- .../src/System/Reflection/AssemblyName.cs | 4 ++-- .../src/System/Reflection/MethodBase.CoreRT.cs | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/System.Private.CoreLib/src/System/IO/Stream.cs b/src/System.Private.CoreLib/src/System/IO/Stream.cs index fefe19b7424..3356990047d 100644 --- a/src/System.Private.CoreLib/src/System/IO/Stream.cs +++ b/src/System.Private.CoreLib/src/System/IO/Stream.cs @@ -660,7 +660,7 @@ public override int ReadByte() public override IAsyncResult BeginRead(byte[] buffer, int offset, int count, AsyncCallback callback, Object state) { - throw new NotImplementedException(); + throw new NotImplementedException(); // TODO: https://github.com/dotnet/corert/issues/3251 //bool overridesBeginRead = _stream.HasOverriddenBeginEndRead(); //lock (_stream) @@ -712,7 +712,7 @@ public override void WriteByte(byte b) public override IAsyncResult BeginWrite(byte[] buffer, int offset, int count, AsyncCallback callback, Object state) { - throw new NotImplementedException(); + throw new NotImplementedException(); // TODO: https://github.com/dotnet/corert/issues/3251 //bool overridesBeginWrite = _stream.HasOverriddenBeginEndWrite(); //lock (_stream) diff --git a/src/System.Private.CoreLib/src/System/Reflection/Assembly.CoreRT.cs b/src/System.Private.CoreLib/src/System/Reflection/Assembly.CoreRT.cs index c909c87a81a..e7e59817e5e 100644 --- a/src/System.Private.CoreLib/src/System/Reflection/Assembly.CoreRT.cs +++ b/src/System.Private.CoreLib/src/System/Reflection/Assembly.CoreRT.cs @@ -14,7 +14,7 @@ public abstract partial class Assembly : ICustomAttributeProvider, ISerializable public static Assembly GetEntryAssembly() => Internal.Runtime.CompilerHelpers.StartupCodeHelpers.GetEntryAssembly(); [System.Runtime.CompilerServices.Intrinsic] - public static Assembly GetExecutingAssembly() { throw new NotImplementedException(); } + public static Assembly GetExecutingAssembly() { throw NotImplemented.ByDesign; } //Implemented by toolchain. public static Assembly GetCallingAssembly() { throw new PlatformNotSupportedException(); } diff --git a/src/System.Private.CoreLib/src/System/Reflection/AssemblyName.cs b/src/System.Private.CoreLib/src/System/Reflection/AssemblyName.cs index ed880072ee2..3cdaef295f5 100644 --- a/src/System.Private.CoreLib/src/System/Reflection/AssemblyName.cs +++ b/src/System.Private.CoreLib/src/System/Reflection/AssemblyName.cs @@ -224,8 +224,8 @@ public void OnDeserialization(object sender) _siInfo = null; } - public static AssemblyName GetAssemblyName(string assemblyFile) { throw new NotImplementedException(); } - public static bool ReferenceMatchesDefinition(AssemblyName reference, AssemblyName definition) { throw new NotImplementedException(); } + public static AssemblyName GetAssemblyName(string assemblyFile) { throw new NotImplementedException(); } // TODO: https://github.com/dotnet/corert/issues/3253 + public static bool ReferenceMatchesDefinition(AssemblyName reference, AssemblyName definition) { throw new NotImplementedException(); } // TODO: https://github.com/dotnet/corert/issues/1861 internal static string EscapeCodeBase(string codebase) { throw new PlatformNotSupportedException(); } diff --git a/src/System.Private.CoreLib/src/System/Reflection/MethodBase.CoreRT.cs b/src/System.Private.CoreLib/src/System/Reflection/MethodBase.CoreRT.cs index 8d53973ce4c..3064d5b0f2b 100644 --- a/src/System.Private.CoreLib/src/System/Reflection/MethodBase.CoreRT.cs +++ b/src/System.Private.CoreLib/src/System/Reflection/MethodBase.CoreRT.cs @@ -12,7 +12,7 @@ public abstract partial class MethodBase : MemberInfo public static MethodBase GetMethodFromHandle(RuntimeMethodHandle handle, RuntimeTypeHandle declaringType) => ReflectionAugments.ReflectionCoreCallbacks.GetMethodFromHandle(handle, declaringType); [System.Runtime.CompilerServices.Intrinsic] - public static MethodBase GetCurrentMethod() { throw new NotImplementedException(); } + public static MethodBase GetCurrentMethod() { throw NotImplemented.ByDesign; } //Implemented by toolchain. // This is not an api but needs to be declared public so that System.Private.Reflection.Core can access (and override it) public virtual ParameterInfo[] GetParametersNoCopy() => GetParameters(); From 0d330f37e084703af8b6de8b9c84a3027e5bf8ba Mon Sep 17 00:00:00 2001 From: Daniel Harvey Date: Fri, 7 Apr 2017 12:35:04 -0700 Subject: [PATCH 4/7] Port MemoryFailPoint from CoreCLR [tfs-changeset: 1653620] --- .../Windows/kernel32/Interop.Memory.cs | 81 +++ src/Native/Runtime/GCHelpers.cpp | 8 + .../src/Resources/Strings.resx | 9 + .../src/System.Private.CoreLib.csproj | 4 + .../src/System/Runtime/MemoryFailPoint.cs | 502 ++++++++++++++++++ .../src/System/Runtime/RuntimeImports.cs | 12 +- 6 files changed, 612 insertions(+), 4 deletions(-) create mode 100644 src/Common/src/Interop/Windows/kernel32/Interop.Memory.cs create mode 100644 src/System.Private.CoreLib/src/System/Runtime/MemoryFailPoint.cs diff --git a/src/Common/src/Interop/Windows/kernel32/Interop.Memory.cs b/src/Common/src/Interop/Windows/kernel32/Interop.Memory.cs new file mode 100644 index 00000000000..96e6ef643fd --- /dev/null +++ b/src/Common/src/Interop/Windows/kernel32/Interop.Memory.cs @@ -0,0 +1,81 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Runtime.InteropServices; + +internal partial class Interop +{ + internal partial class Kernel32 + { + [DllImport(Libraries.Kernel32)] + unsafe internal static extern void* VirtualAlloc(void* address, UIntPtr numBytes, int commitOrReserve, int pageProtectionMode); + + [DllImport(Libraries.Kernel32)] + unsafe internal static extern bool VirtualFree(void* address, UIntPtr numBytes, int pageFreeMode); + + unsafe internal static bool GlobalMemoryStatusEx(ref MEMORYSTATUSEX buffer) + { + buffer.length = sizeof(MEMORYSTATUSEX); + return GlobalMemoryStatusExNative(ref buffer); + } + + [DllImport(Libraries.Kernel32, SetLastError = true, EntryPoint = "GlobalMemoryStatusEx")] + private static extern bool GlobalMemoryStatusExNative(ref MEMORYSTATUSEX buffer); + + [DllImport(Libraries.Kernel32, SetLastError = true)] + unsafe internal static extern UIntPtr VirtualQuery(void* address, ref MEMORY_BASIC_INFORMATION buffer, UIntPtr sizeOfBuffer); + + [DllImport(Libraries.Kernel32, SetLastError = true)] + unsafe internal static extern UIntPtr GetSystemInfo(ref SYSTEM_INFO info); + + internal const int MEM_COMMIT = 0x1000; + internal const int MEM_RESERVE = 0x2000; + internal const int MEM_RELEASE = 0x8000; + internal const int MEM_FREE = 0x10000; + internal const int PAGE_READWRITE = 0x04; + + [StructLayout(LayoutKind.Sequential)] + internal struct SYSTEM_INFO + { + internal int dwOemId; + internal int dwPageSize; + internal UIntPtr lpMinimumApplicationAddress; + internal UIntPtr lpMaximumApplicationAddress; + internal UIntPtr dwActiveProcessorMask; + internal int dwNumberOfProcessors; + internal int dwProcessorType; + internal int dwAllocationGranularity; + internal short wProcessorLevel; + internal short wProcessorRevision; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct MEMORYSTATUSEX + { + // The length field must be set to the size of this data structure. + internal int length; + internal int memoryLoad; + internal ulong totalPhys; + internal ulong availPhys; + internal ulong totalPageFile; + internal ulong availPageFile; + internal ulong totalVirtual; + internal ulong availVirtual; + internal ulong availExtendedVirtual; + } + + [StructLayout(LayoutKind.Sequential)] + internal unsafe struct MEMORY_BASIC_INFORMATION + { + internal void* BaseAddress; + internal void* AllocationBase; + internal uint AllocationProtect; + internal UIntPtr RegionSize; + internal uint State; + internal uint Protect; + internal uint Type; + } + } +} diff --git a/src/Native/Runtime/GCHelpers.cpp b/src/Native/Runtime/GCHelpers.cpp index a09ac5fc9ae..405ff7c8b5f 100644 --- a/src/Native/Runtime/GCHelpers.cpp +++ b/src/Native/Runtime/GCHelpers.cpp @@ -197,3 +197,11 @@ COOP_PINVOKE_HELPER(Int32, RhWaitForFullGCComplete, (Int32 millisecondsTimeout)) int timeout = millisecondsTimeout == -1 ? INFINITE : millisecondsTimeout; return GCHeapUtilities::GetGCHeap()->WaitForFullGCComplete(millisecondsTimeout); } + +COOP_PINVOKE_HELPER(Int64, RhGetGCSegmentSize, ()) +{ + size_t first = GCHeapUtilities::GetGCHeap()->GetValidSegmentSize(Boolean_true); + size_t second = GCHeapUtilities::GetGCHeap()->GetValidSegmentSize(Boolean_false); + + return (first > second) ? first : second; +} diff --git a/src/System.Private.CoreLib/src/Resources/Strings.resx b/src/System.Private.CoreLib/src/Resources/Strings.resx index cc832925e0f..9e72e0ea592 100644 --- a/src/System.Private.CoreLib/src/Resources/Strings.resx +++ b/src/System.Private.CoreLib/src/Resources/Strings.resx @@ -1065,6 +1065,15 @@ String was not recognized as a valid TimeSpan. + + Insufficient available memory to meet the expected demands of an operation at this time. Please try again later. + + + Insufficient memory to meet the expected demands of an operation, and this system is likely to never satisfy this request. If this is a 32 bit system, consider booting in 3 GB mode. + + + Insufficient available memory to meet the expected demands of an operation at this time, possibly due to virtual address space fragmentation. Please try again later. + Null object cannot be converted to a value type. diff --git a/src/System.Private.CoreLib/src/System.Private.CoreLib.csproj b/src/System.Private.CoreLib/src/System.Private.CoreLib.csproj index 2df17edb78a..fd561edfd2f 100644 --- a/src/System.Private.CoreLib/src/System.Private.CoreLib.csproj +++ b/src/System.Private.CoreLib/src/System.Private.CoreLib.csproj @@ -607,11 +607,15 @@ + Interop\Windows\ntdll\Interop.ZeroMemory.cs + + Interop\Windows\kernel32\Interop.Memory.cs + Interop\Windows\kernel32\Interop.MultiByteToWideChar.cs diff --git a/src/System.Private.CoreLib/src/System/Runtime/MemoryFailPoint.cs b/src/System.Private.CoreLib/src/System/Runtime/MemoryFailPoint.cs new file mode 100644 index 00000000000..1712a0465fd --- /dev/null +++ b/src/System.Private.CoreLib/src/System/Runtime/MemoryFailPoint.cs @@ -0,0 +1,502 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +/*============================================================ +** +** +** +** Provides a way for an app to not start an operation unless +** there's a reasonable chance there's enough memory +** available for the operation to succeed. +** +** +===========================================================*/ + +using System; +using System.IO; +using Microsoft.Win32; +using System.Runtime.InteropServices; +using System.Threading; +using System.Runtime.CompilerServices; +using System.Runtime.ConstrainedExecution; +using System.Runtime.Versioning; +using System.Diagnostics; +using System.Diagnostics.Contracts; + +/* + This class allows an application to fail before starting certain + activities. The idea is to fail early instead of failing in the middle + of some long-running operation to increase the survivability of the + application and ensure you don't have to write tricky code to handle an + OOM anywhere in your app's code (which implies state corruption, meaning you + should unload the appdomain, if you have a transacted environment to ensure + rollback of individual transactions). This is an incomplete tool to attempt + hoisting all your OOM failures from anywhere in your worker methods to one + particular point where it is easier to handle an OOM failure, and you can + optionally choose to not start a workitem if it will likely fail. This does + not help the performance of your code directly (other than helping to avoid + AD unloads). The point is to avoid starting work if it is likely to fail. + The Enterprise Services team has used these memory gates effectively in the + unmanaged world for a decade. + + In Whidbey, we will simply check to see if there is enough memory available + in the OS's page file & attempt to ensure there might be enough space free + within the process's address space (checking for address space fragmentation + as well). We will not commit or reserve any memory. To avoid race conditions with + other threads using MemoryFailPoints, we'll also keep track of a + process-wide amount of memory "reserved" via all currently-active + MemoryFailPoints. This has two problems: + 1) This can account for memory twice. If a thread creates a + MemoryFailPoint for 100 MB then allocates 99 MB, we'll see 99 MB + less free memory and 100 MB less reserved memory. Yet, subtracting + off the 100 MB is necessary because the thread may not have started + allocating memory yet. Disposing of this class immediately after + front-loaded allocations have completed is a great idea. + 2) This is still vulnerable to race conditions with other threads that don't use + MemoryFailPoints. + So this class is far from perfect. But it may be good enough to + meaningfully reduce the frequency of OutOfMemoryExceptions in managed apps. + + In Orcas or later, we might allocate some memory from the OS and add it + to a allocation context for this thread. Obviously, at that point we need + some way of conveying when we release this block of memory. So, we + implemented IDisposable on this type in Whidbey and expect all users to call + this from within a using block to provide lexical scope for their memory + usage. The call to Dispose (implicit with the using block) will give us an + opportunity to release this memory, perhaps. We anticipate this will give + us the possibility of a more effective design in a future version. + + In Orcas, we may also need to differentiate between allocations that would + go into the normal managed heap vs. the large object heap, or we should + consider checking for enough free space in both locations (with any + appropriate adjustments to ensure the memory is contiguous). +*/ + +namespace System.Runtime +{ + public sealed class MemoryFailPoint : CriticalFinalizerObject, IDisposable + { + // Find the top section of user mode memory. Avoid the last 64K. + // Windows reserves that block for the kernel, apparently, and doesn't + // let us ask about that memory. But since we ask for memory in 1 MB + // chunks, we don't have to special case this. Also, we need to + // deal with 32 bit machines in 3 GB mode. + // Using Win32's GetSystemInfo should handle all this for us. + private static readonly ulong s_topOfMemory; + + // Walking the address space is somewhat expensive, taking around half + // a millisecond. Doing that per transaction limits us to a max of + // ~2000 transactions/second. Instead, let's do this address space + // walk once every 10 seconds, or when we will likely fail. This + // amortization scheme can reduce the cost of a memory gate by about + // a factor of 100. + private static long s_hiddenLastKnownFreeAddressSpace = 0; + private static long s_hiddenLastTimeCheckingAddressSpace = 0; + private const int CheckThreshold = 10 * 1000; // 10 seconds + + private static long LastKnownFreeAddressSpace + { + get { return Volatile.Read(ref s_hiddenLastKnownFreeAddressSpace); } + set { Volatile.Write(ref s_hiddenLastKnownFreeAddressSpace, value); } + } + + private static long AddToLastKnownFreeAddressSpace(long addend) + { + return Interlocked.Add(ref s_hiddenLastKnownFreeAddressSpace, addend); + } + + private static long LastTimeCheckingAddressSpace + { + get { return Volatile.Read(ref s_hiddenLastTimeCheckingAddressSpace); } + set { Volatile.Write(ref s_hiddenLastTimeCheckingAddressSpace, value); } + } + + // When allocating memory segment by segment, we've hit some cases + // where there are only 22 MB of memory available on the machine, + // we need 1 16 MB segment, and the OS does not succeed in giving us + // that memory. Reasons for this could include: + // 1) The GC does allocate memory when doing a collection. + // 2) Another process on the machine could grab that memory. + // 3) Some other part of the runtime might grab this memory. + // If we build in a little padding, we can help protect + // ourselves against some of these cases, and we want to err on the + // conservative side with this class. + private const int LowMemoryFudgeFactor = 16 << 20; + + // Round requested size to a 16MB multiple to have a better granularity + // when checking for available memory. + private const int MemoryCheckGranularity = 16; + + // Note: This may become dynamically tunable in the future. + // Also note that we can have different segment sizes for the normal vs. + // large object heap. We currently use the max of the two. + private static readonly ulong s_GCSegmentSize = RuntimeImports.RhGetGCSegmentSize(); + + // For multi-threaded workers, we want to ensure that if two workers + // use a MemoryFailPoint at the same time, and they both succeed, that + // they don't trample over each other's memory. Keep a process-wide + // count of "reserved" memory, and decrement this in Dispose and + // in the critical finalizer. See + // SharedStatics.MemoryFailPointReservedMemory + + private ulong _reservedMemory; // The size of this request (from user) + private bool _mustSubtractReservation; // Did we add data to SharedStatics? + + static MemoryFailPoint() + { + Interop.Kernel32.SYSTEM_INFO info = new Interop.Kernel32.SYSTEM_INFO(); + Interop.Kernel32.GetSystemInfo(ref info); + s_topOfMemory = (ulong)info.lpMaximumApplicationAddress; + } + + // We can remove this link demand in a future version - we will + // have scenarios for this in partial trust in the future, but + // we're doing this just to restrict this in case the code below + // is somehow incorrect. + public MemoryFailPoint(int sizeInMegabytes) + { + if (sizeInMegabytes <= 0) + throw new ArgumentOutOfRangeException(nameof(sizeInMegabytes), SR.ArgumentOutOfRange_NeedNonNegNum); + + ulong size = ((ulong)sizeInMegabytes) << 20; + _reservedMemory = size; + + // Check to see that we both have enough memory on the system + // and that we have enough room within the user section of the + // process's address space. Also, we need to use the GC segment + // size, not the amount of memory the user wants to allocate. + // Consider correcting this to reflect free memory within the GC + // heap, and to check both the normal & large object heaps. + ulong segmentSize = (ulong)(Math.Ceiling((double)size / s_GCSegmentSize) * s_GCSegmentSize); + if (segmentSize >= s_topOfMemory) + throw new InsufficientMemoryException(SR.InsufficientMemory_MemFailPoint_TooBig); + + ulong requestedSizeRounded = (ulong)(Math.Ceiling((double)sizeInMegabytes / MemoryCheckGranularity) * MemoryCheckGranularity); + //re-convert into bytes + requestedSizeRounded <<= 20; + + ulong availPageFile = 0; // available VM (physical + page file) + ulong totalAddressSpaceFree = 0; // non-contiguous free address space + + // Check for available memory, with 2 attempts at getting more + // memory. + // Stage 0: If we don't have enough, trigger a GC. + // Stage 1: If we don't have enough, try growing the swap file. + // Stage 2: Update memory state, then fail or leave loop. + // + // (In the future, we could consider adding another stage after + // Stage 0 to run finalizers. However, before doing that make sure + // that we could abort this constructor when we call + // GC.WaitForPendingFinalizers, noting that this method uses a CER + // so it can't be aborted, and we have a critical finalizer. It + // would probably work, but do some thinking first.) + for (int stage = 0; stage < 3; stage++) + { + CheckForAvailableMemory(out availPageFile, out totalAddressSpaceFree); + + // If we have enough room, then skip some stages. + // Note that multiple threads can still lead to a race condition for our free chunk + // of address space, which can't be easily solved. + ulong reserved = MemoryFailPointReservedMemory; + ulong segPlusReserved = segmentSize + reserved; + bool overflow = segPlusReserved < segmentSize || segPlusReserved < reserved; + bool needPageFile = availPageFile < (requestedSizeRounded + reserved + LowMemoryFudgeFactor) || overflow; + bool needAddressSpace = totalAddressSpaceFree < segPlusReserved || overflow; + + // Ensure our cached amount of free address space is not stale. + long now = Environment.TickCount; // Handle wraparound. + if ((now > LastTimeCheckingAddressSpace + CheckThreshold || now < LastTimeCheckingAddressSpace) || + LastKnownFreeAddressSpace < (long)segmentSize) + { + CheckForFreeAddressSpace(segmentSize, false); + } + bool needContiguousVASpace = (ulong)LastKnownFreeAddressSpace < segmentSize; + + if (!needPageFile && !needAddressSpace && !needContiguousVASpace) + break; + + switch (stage) + { + case 0: + // The GC will release empty segments to the OS. This will + // relieve us from having to guess whether there's + // enough memory in either GC heap, and whether + // internal fragmentation will prevent those + // allocations from succeeding. + GC.Collect(); + continue; + + case 1: + // Do this step if and only if the page file is too small. + if (!needPageFile) + continue; + + // Attempt to grow the OS's page file. Note that we ignore + // any allocation routines from the host intentionally. + RuntimeHelpers.PrepareConstrainedRegions(); + try + { + } + finally + { + // This shouldn't overflow due to the if clauses above. + UIntPtr numBytes = new UIntPtr(segmentSize); + unsafe + { + void* pMemory = Interop.Kernel32.VirtualAlloc(null, numBytes, Interop.Kernel32.MEM_COMMIT, Interop.Kernel32.PAGE_READWRITE); + if (pMemory != null) + { + bool r = Interop.Kernel32.VirtualFree(pMemory, UIntPtr.Zero, Interop.Kernel32.MEM_RELEASE); + if (!r) + throw Win32Marshal.GetExceptionForWin32Error(Marshal.GetLastWin32Error()); + } + } + } + continue; + + case 2: + // The call to CheckForAvailableMemory above updated our + // state. + if (needPageFile || needAddressSpace) + { + InsufficientMemoryException e = new InsufficientMemoryException(SR.InsufficientMemory_MemFailPoint); +#if DEBUG + e.Data["MemFailPointState"] = new MemoryFailPointState(sizeInMegabytes, segmentSize, + needPageFile, needAddressSpace, needContiguousVASpace, + availPageFile >> 20, totalAddressSpaceFree >> 20, + LastKnownFreeAddressSpace >> 20, reserved); +#endif + throw e; + } + + if (needContiguousVASpace) + { + InsufficientMemoryException e = new InsufficientMemoryException(SR.InsufficientMemory_MemFailPoint_VAFrag); +#if DEBUG + e.Data["MemFailPointState"] = new MemoryFailPointState(sizeInMegabytes, segmentSize, + needPageFile, needAddressSpace, needContiguousVASpace, + availPageFile >> 20, totalAddressSpaceFree >> 20, + LastKnownFreeAddressSpace >> 20, reserved); +#endif + throw e; + } + + break; + + default: + Debug.Assert(false, "Fell through switch statement!"); + break; + } + } + + // Success - we have enough room the last time we checked. + // Now update our shared state in a somewhat atomic fashion + // and handle a simple race condition with other MemoryFailPoint instances. + AddToLastKnownFreeAddressSpace(-((long)size)); + if (LastKnownFreeAddressSpace < 0) + CheckForFreeAddressSpace(segmentSize, true); + + RuntimeHelpers.PrepareConstrainedRegions(); + try + { + } + finally + { + AddMemoryFailPointReservation((long)size); + _mustSubtractReservation = true; + } + } + + private static void CheckForAvailableMemory(out ulong availPageFile, out ulong totalAddressSpaceFree) + { + bool r; + Interop.Kernel32.MEMORYSTATUSEX memory = new Interop.Kernel32.MEMORYSTATUSEX(); + r = Interop.Kernel32.GlobalMemoryStatusEx(ref memory); + if (!r) + throw Win32Marshal.GetExceptionForWin32Error(Marshal.GetLastWin32Error()); + availPageFile = memory.availPageFile; + totalAddressSpaceFree = memory.availVirtual; + } + + // Based on the shouldThrow parameter, this will throw an exception, or + // returns whether there is enough space. In all cases, we update + // our last known free address space, hopefully avoiding needing to + // probe again. + private static unsafe bool CheckForFreeAddressSpace(ulong size, bool shouldThrow) + { + // Start walking the address space at 0. VirtualAlloc may wrap + // around the address space. We don't need to find the exact + // pages that VirtualAlloc would return - we just need to + // know whether VirtualAlloc could succeed. + ulong freeSpaceAfterGCHeap = MemFreeAfterAddress(null, size); + + // We may set these without taking a lock - I don't believe + // this will hurt, as long as we never increment this number in + // the Dispose method. If we do an extra bit of checking every + // once in a while, but we avoid taking a lock, we may win. + LastKnownFreeAddressSpace = (long)freeSpaceAfterGCHeap; + LastTimeCheckingAddressSpace = Environment.TickCount; + + if (freeSpaceAfterGCHeap < size && shouldThrow) + throw new InsufficientMemoryException(SR.InsufficientMemory_MemFailPoint_VAFrag); + return freeSpaceAfterGCHeap >= size; + } + + // Returns the amount of consecutive free memory available in a block + // of pages. If we didn't have enough address space, we still return + // a positive value < size, to help potentially avoid the overhead of + // this check if we use a MemoryFailPoint with a smaller size next. + private static unsafe ulong MemFreeAfterAddress(void* address, ulong size) + { + if (size >= s_topOfMemory) + return 0; + + ulong largestFreeRegion = 0; + Interop.Kernel32.MEMORY_BASIC_INFORMATION memInfo = new Interop.Kernel32.MEMORY_BASIC_INFORMATION(); + UIntPtr sizeOfMemInfo = (UIntPtr)sizeof(Interop.Kernel32.MEMORY_BASIC_INFORMATION); + + while (((ulong)address) + size < s_topOfMemory) + { + UIntPtr r = Interop.Kernel32.VirtualQuery(address, ref memInfo, sizeOfMemInfo); + if (r == UIntPtr.Zero) + throw Win32Marshal.GetExceptionForWin32Error(Marshal.GetLastWin32Error()); + + ulong regionSize = memInfo.RegionSize.ToUInt64(); + if (memInfo.State == Interop.Kernel32.MEM_FREE) + { + if (regionSize >= size) + return regionSize; + else + largestFreeRegion = Math.Max(largestFreeRegion, regionSize); + } + address = (void*)((ulong)address + regionSize); + } + return largestFreeRegion; + } + + [MethodImpl(MethodImplOptions.InternalCall)] + private static extern void GetMemorySettings(out ulong maxGCSegmentSize, out ulong topOfMemory); + + ~MemoryFailPoint() + { + Dispose(false); + } + + // Applications must call Dispose, which conceptually "releases" the + // memory that was "reserved" by the MemoryFailPoint. This affects a + // global count of reserved memory in this version (helping to throttle + // future MemoryFailPoints) in this version. We may in the + // future create an allocation context and release it in the Dispose + // method. While the finalizer will eventually free this block of + // memory, apps will help their performance greatly by calling Dispose. + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + private void Dispose(bool disposing) + { + // This is just bookkeeping to ensure multiple threads can really + // get enough memory, and this does not actually reserve memory + // within the GC heap. + if (_mustSubtractReservation) + { + RuntimeHelpers.PrepareConstrainedRegions(); + try + { + } + finally + { + AddMemoryFailPointReservation(-((long)_reservedMemory)); + _mustSubtractReservation = false; + } + } + + /* + // Prototype performance + // Let's pretend that we returned at least some free memory to + // the GC heap. We don't know this is true - the objects could + // have a longer lifetime, and the memory could be elsewhere in the + // GC heap. Additionally, we subtracted off the segment size, not + // this size. That's ok - we don't mind if this slowly degrades + // and requires us to refresh the value a little bit sooner. + // But releasing the memory here should help us avoid probing for + // free address space excessively with large workItem sizes. + Interlocked.Add(ref LastKnownFreeAddressSpace, _reservedMemory); + */ + } + + // This is the total amount of memory currently "reserved" via + // all MemoryFailPoints allocated within the process. + // Stored as a long because we need to use Interlocked.Add. + private static long s_memFailPointReservedMemory; + + internal static long AddMemoryFailPointReservation(long size) + { + // Size can legitimately be negative - see Dispose. + return Interlocked.Add(ref s_memFailPointReservedMemory, (long)size); + } + + internal static ulong MemoryFailPointReservedMemory + { + get + { + Debug.Assert(Volatile.Read(ref s_memFailPointReservedMemory) >= 0, "Process-wide MemoryFailPoint reserved memory was negative!"); + return (ulong)Volatile.Read(ref s_memFailPointReservedMemory); + } + } + +#if DEBUG + [Serializable] + internal sealed class MemoryFailPointState + { + private ulong _segmentSize; + private int _allocationSizeInMB; + private bool _needPageFile; + private bool _needAddressSpace; + private bool _needContiguousVASpace; + private ulong _availPageFile; + private ulong _totalFreeAddressSpace; + private long _lastKnownFreeAddressSpace; + private ulong _reservedMem; + private String _stackTrace; // Where did we fail, for additional debugging. + + internal MemoryFailPointState(int allocationSizeInMB, ulong segmentSize, bool needPageFile, bool needAddressSpace, bool needContiguousVASpace, ulong availPageFile, ulong totalFreeAddressSpace, long lastKnownFreeAddressSpace, ulong reservedMem) + { + _allocationSizeInMB = allocationSizeInMB; + _segmentSize = segmentSize; + _needPageFile = needPageFile; + _needAddressSpace = needAddressSpace; + _needContiguousVASpace = needContiguousVASpace; + _availPageFile = availPageFile; + _totalFreeAddressSpace = totalFreeAddressSpace; + _lastKnownFreeAddressSpace = lastKnownFreeAddressSpace; + _reservedMem = reservedMem; + try + { + _stackTrace = Environment.StackTrace; + } + catch (System.Security.SecurityException) + { + _stackTrace = "no permission"; + } + catch (OutOfMemoryException) + { + _stackTrace = "out of memory"; + } + } + + public override String ToString() + { + return String.Format(System.Globalization.CultureInfo.InvariantCulture, "MemoryFailPoint detected insufficient memory to guarantee an operation could complete. Checked for {0} MB, for allocation size of {1} MB. Need page file? {2} Need Address Space? {3} Need Contiguous address space? {4} Avail page file: {5} MB Total free VA space: {6} MB Contiguous free address space (found): {7} MB Space reserved by process's MemoryFailPoints: {8} MB", + _segmentSize >> 20, _allocationSizeInMB, _needPageFile, + _needAddressSpace, _needContiguousVASpace, + _availPageFile >> 20, _totalFreeAddressSpace >> 20, + _lastKnownFreeAddressSpace >> 20, _reservedMem); + } + } +#endif + } +} \ No newline at end of file diff --git a/src/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs b/src/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs index 1df7edb1692..3f7b1c86891 100644 --- a/src/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs +++ b/src/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs @@ -163,6 +163,10 @@ internal static void RhWaitForPendingFinalizers(bool allowReentrantWait) [RuntimeImport(RuntimeLibrary, "RhpShutdown")] internal static extern void RhpShutdown(); + [MethodImpl(MethodImplOptions.InternalCall)] + [RuntimeImport(RuntimeLibrary, "RhGetGCSegmentSize")] + internal static extern ulong RhGetGCSegmentSize(); + // // calls for GCHandle. // These methods are needed to implement GCHandle class like functionality (optional) @@ -640,10 +644,10 @@ internal static uint RhGetLoadedModules(TypeManagerHandle[] resultArray) // simply let it be pulled off the stack. internal struct ConservativelyReportedRegionDesc { - private IntPtr ptr1; - private IntPtr ptr2; - private IntPtr ptr3; - private IntPtr ptr4; + private IntPtr _ptr1; + private IntPtr _ptr2; + private IntPtr _ptr3; + private IntPtr _ptr4; } [MethodImplAttribute(MethodImplOptions.InternalCall)] From 53b764ea69fc3ec96448752e7fc4fe9ff39752bb Mon Sep 17 00:00:00 2001 From: Tijoy Tom Kalathiparambil Date: Fri, 7 Apr 2017 13:38:28 -0700 Subject: [PATCH 5/7] This's vitek's change. Make EventSource.EventMetadata and EventDescriptor public so that analysis roots them. These types were public in the separate System.Diagnostics.Tracing assembly, now the move to CoreLib marked them as private. While they are technically private (it's a contract used between the library and the ILC toolchain), we need them to be rooted and exported from shared library for the system to work. For now I'm simply marking them as public again. A cleaner solution might be to use .rd.xml to root them and modify shared library definition to force export them. [tfs-changeset: 1653635] --- .../Diagnostics/Tracing/EventDescriptor.cs | 16 +++++++++++++++- .../System/Diagnostics/Tracing/EventSource.cs | 16 +++++++++++++++- .../Diagnostics/Tracing/EventSource_CoreRT.cs | 17 +++++++++++++++-- 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/src/System.Private.CoreLib/shared/System/Diagnostics/Tracing/EventDescriptor.cs b/src/System.Private.CoreLib/shared/System/Diagnostics/Tracing/EventDescriptor.cs index a1dcf1c5d9a..8fb471a99f7 100644 --- a/src/System.Private.CoreLib/shared/System/Diagnostics/Tracing/EventDescriptor.cs +++ b/src/System.Private.CoreLib/shared/System/Diagnostics/Tracing/EventDescriptor.cs @@ -25,7 +25,21 @@ namespace System.Diagnostics.Tracing #if !CORECLR && !ES_BUILD_PN [System.Security.Permissions.HostProtection(MayLeakOnAbort = true)] #endif // !CORECLR && !ES_BUILD_PN - internal struct EventDescriptor + + /* + EventDescriptor was public in the separate System.Diagnostics.Tracing assembly(pre NS2.0), + now the move to CoreLib marked them as private. + While they are technically private (it's a contract used between the library and the ILC toolchain), + we need them to be rooted and exported from shared library for the system to work. + For now I'm simply marking them as public again.A cleaner solution might be to use.rd.xml to + root them and modify shared library definition to force export them. + */ +#if ES_BUILD_PN + public +#else + internal +#endif + struct EventDescriptor { # region private [FieldOffset(0)] diff --git a/src/System.Private.CoreLib/shared/System/Diagnostics/Tracing/EventSource.cs b/src/System.Private.CoreLib/shared/System/Diagnostics/Tracing/EventSource.cs index 6a1726501d4..9197ed3b726 100644 --- a/src/System.Private.CoreLib/shared/System/Diagnostics/Tracing/EventSource.cs +++ b/src/System.Private.CoreLib/shared/System/Diagnostics/Tracing/EventSource.cs @@ -2505,7 +2505,21 @@ protected override void OnControllerCommand(ControllerCommand command, IDictiona /// descriptor as well as some stuff we added specifically for EventSource. see the /// code:m_eventData for where we use this. /// - internal partial struct EventMetadata + + /* + EventMetadata was public in the separate System.Diagnostics.Tracing assembly(pre NS2.0), + now the move to CoreLib marked them as private. + While they are technically private (it's a contract used between the library and the ILC toolchain), + we need them to be rooted and exported from shared library for the system to work. + For now I'm simply marking them as public again.A cleaner solution might be to use.rd.xml to + root them and modify shared library definition to force export them. + */ +#if ES_BUILD_PN + public +#else + internal +#endif + partial struct EventMetadata { public EventDescriptor Descriptor; public EventTags Tags; diff --git a/src/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventSource_CoreRT.cs b/src/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventSource_CoreRT.cs index 97c4b8eed93..9e778549c6d 100644 --- a/src/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventSource_CoreRT.cs +++ b/src/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventSource_CoreRT.cs @@ -149,9 +149,22 @@ private Type GetDataType(EventMetadata eventData, int parameterId) return dataType; } - private static readonly bool m_EventSourcePreventRecursion = true; + private static readonly bool m_EventSourcePreventRecursion = true; - internal partial struct EventMetadata + /* + EventMetadata was public in the separate System.Diagnostics.Tracing assembly(pre NS2.0), + now the move to CoreLib marked them as private. + While they are technically private (it's a contract used between the library and the ILC toolchain), + we need them to be rooted and exported from shared library for the system to work. + For now I'm simply marking them as public again.A cleaner solution might be to use.rd.xml to + root them and modify shared library definition to force export them. + */ +#if ES_BUILD_PN + public +#else + internal +#endif + partial struct EventMetadata { public EventMetadata(EventDescriptor descriptor, EventTags tags, From 875a5776e51d45bb9910dd7c3dce51f3c1ba457b Mon Sep 17 00:00:00 2001 From: dotnet-bot Date: Fri, 7 Apr 2017 13:46:12 -0700 Subject: [PATCH 6/7] Hook jitinterface into GetRuntimeLookupSignatureName --Implement GetRuntimeLookupSignatureNameForClass/GetRuntimeLookupSignatureNameForMethod in bridge code --GetRuntimeLookupSignatureNameForMethod is used for all usg lookups, regardless of whether method is generic/nongeneric. In the case of a nongeneric method, I do lookup using TypeDesc instead of MethodDesc. --Call these from jitinterface instead of the stub code. --Made method NativeLayoutSignature public so its callable from the bridge --Made class NativeLayoutSignatureNode public as method NativeLayoutSignature is now public --Merged method signature and type signature hashtables into a single hashtable GlobalNameLookUpSignatureMap ----This lets us avoid creating multiple SSRs for identical types ----This is a possible perf hit since we're indexing on char* now instead of handles - there are a couple options in the future to mitigate this [tfs-changeset: 1653636] --- .../Compiler/DependencyAnalysis/NativeLayoutSignatureNode.cs | 2 +- .../src/Compiler/DependencyAnalysis/NodeFactory.NativeLayout.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ILCompiler.Compiler/src/Compiler/DependencyAnalysis/NativeLayoutSignatureNode.cs b/src/ILCompiler.Compiler/src/Compiler/DependencyAnalysis/NativeLayoutSignatureNode.cs index d59dad5dcde..4877d7e67d2 100644 --- a/src/ILCompiler.Compiler/src/Compiler/DependencyAnalysis/NativeLayoutSignatureNode.cs +++ b/src/ILCompiler.Compiler/src/Compiler/DependencyAnalysis/NativeLayoutSignatureNode.cs @@ -15,7 +15,7 @@ namespace ILCompiler.DependencyAnalysis /// to the TypeManager that contains the native layout info blob of interest, and the second item /// is an offset into that native layout info blob /// - class NativeLayoutSignatureNode : ObjectNode, ISymbolNode + public class NativeLayoutSignatureNode : ObjectNode, ISymbolNode { private TypeSystemEntity _identity; private Utf8String _identityPrefix; diff --git a/src/ILCompiler.Compiler/src/Compiler/DependencyAnalysis/NodeFactory.NativeLayout.cs b/src/ILCompiler.Compiler/src/Compiler/DependencyAnalysis/NodeFactory.NativeLayout.cs index 4b87bf21e25..5ba89fc2b01 100644 --- a/src/ILCompiler.Compiler/src/Compiler/DependencyAnalysis/NodeFactory.NativeLayout.cs +++ b/src/ILCompiler.Compiler/src/Compiler/DependencyAnalysis/NodeFactory.NativeLayout.cs @@ -386,7 +386,7 @@ public bool Equals(NativeLayoutSignatureKey other) } private NodeCache _nativeLayoutSignatureNodes; - internal NativeLayoutSignatureNode NativeLayoutSignature(NativeLayoutSavedVertexNode signature, Utf8String identityPrefix, TypeSystemEntity identity) + public NativeLayoutSignatureNode NativeLayoutSignature(NativeLayoutSavedVertexNode signature, Utf8String identityPrefix, TypeSystemEntity identity) { return _nativeLayoutSignatureNodes.GetOrAdd(new NativeLayoutSignatureKey(signature, identityPrefix, identity)); } From 3d64c6ca2f65ea3bd93612bfdf35c69b82baab25 Mon Sep 17 00:00:00 2001 From: Atsushi Kanamori Date: Fri, 7 Apr 2017 14:22:15 -0700 Subject: [PATCH 7/7] Incorporate CR feedback on Marvin hash (#3263) - String.GetHashCode() now uses Marvin unconditionally. - Make the Marvin routines more general purpose and easier to use. - Expanded the testing to odd-sized byte arrays (i.e. not strings), and fixed the bug found. Original data obtained by exporting SymCryptMarvin32() from CoreClr.dll and P/Invoking to it from test app. --- .../src/System/Marvin.cs | 32 ++++++++----- .../src/System/String.Comparison.cs | 47 +------------------ 2 files changed, 22 insertions(+), 57 deletions(-) diff --git a/src/System.Private.CoreLib/src/System/Marvin.cs b/src/System.Private.CoreLib/src/System/Marvin.cs index b4ac62be0d8..91a7c1021bc 100644 --- a/src/System.Private.CoreLib/src/System/Marvin.cs +++ b/src/System.Private.CoreLib/src/System/Marvin.cs @@ -9,14 +9,28 @@ namespace System { internal static class Marvin { - public static uint ComputeStringHash(ref byte data, uint count, ulong seed) + /// + /// Convenience method to compute a Marvin hash and collapse it into a 32-bit hash. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int ComputeHash32(ref byte data, int count, ulong seed) + { + long hash64 = ComputeHash(ref data, count, seed); + return ((int)(hash64 >> 32)) ^ (int)hash64; + } + + /// + /// Computes a 64-hash using the Marvin algorithm. + /// + public static long ComputeHash(ref byte data, int count, ulong seed) { + uint ucount = (uint)count; uint p0 = (uint)seed; uint p1 = (uint)(seed >> 32); int byteOffset = 0; // declared as signed int so we don't have to cast everywhere (it's passed to Unsafe.Add() and used for nothing else.) - while (count >= 8) + while (ucount >= 8) { p0 += Unsafe.As(ref Unsafe.Add(ref data, byteOffset)); Block(ref p0, ref p1); @@ -25,10 +39,10 @@ public static uint ComputeStringHash(ref byte data, uint count, ulong seed) Block(ref p0, ref p1); byteOffset += 8; - count -= 8; + ucount -= 8; } - switch (count) + switch (ucount) { case 4: p0 += Unsafe.As(ref Unsafe.Add(ref data, byteOffset)); @@ -66,22 +80,18 @@ public static uint ComputeStringHash(ref byte data, uint count, ulong seed) goto case 3; case 3: - p0 += 0x80000000u | Unsafe.As(ref Unsafe.Add(ref data, byteOffset)) | Unsafe.Add(ref data, byteOffset + 2); + p0 += 0x80000000u | (((uint)(Unsafe.Add(ref data, byteOffset + 2))) << 16)| (uint)(Unsafe.As(ref Unsafe.Add(ref data, byteOffset))); break; default: Debug.Fail("Should not get here."); - throw new InvalidOperationException(); + break; } Block(ref p0, ref p1); Block(ref p0, ref p1); - // At this point, p0 and p1 contains the 8-byte Marvin hash result. If we need a general purpose Marvin implementation in the future, - // this could be refactored to stop here. For now, String.GetHashCode() is the only user of this function and he wants an 4-byte hash code - // so this last step is specific to String.GetHashCode(). - - return p0 ^ p1; + return (((long)p1) << 32) | p0; } [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/src/System.Private.CoreLib/src/System/String.Comparison.cs b/src/System.Private.CoreLib/src/System/String.Comparison.cs index 8280142cf18..09f28181cbc 100644 --- a/src/System.Private.CoreLib/src/System/String.Comparison.cs +++ b/src/System.Private.CoreLib/src/System/String.Comparison.cs @@ -997,52 +997,7 @@ public static bool Equals(String a, String b, StringComparison comparisonType) // they will return the same hash code. public override int GetHashCode() { -#if FEATURE_RANDOMIZED_STRING_HASHING - return (int)Marvin.ComputeStringHash(ref Unsafe.As(ref _firstChar), (uint)(_stringLength * 2), Marvin.DefaultSeed); -#else - unsafe - { - fixed (char* src = &_firstChar) - { -#if BIT64 - int hash1 = 5381; -#else - int hash1 = (5381 << 16) + 5381; -#endif - int hash2 = hash1; - -#if BIT64 - int c; - char* s = src; - while ((c = s[0]) != 0) - { - hash1 = ((hash1 << 5) + hash1) ^ c; - c = s[1]; - if (c == 0) - break; - hash2 = ((hash2 << 5) + hash2) ^ c; - s += 2; - } -#else - // 32bit machines. - int* pint = (int*)src; - int len = this.Length; - while (len > 0) - { - hash1 = ((hash1 << 5) + hash1 + (hash1 >> 27)) ^ pint[0]; - if (len <= 2) - { - break; - } - hash2 = ((hash2 << 5) + hash2 + (hash2 >> 27)) ^ pint[1]; - pint += 2; - len -= 4; - } -#endif - return hash1 + (hash2 * 1566083941); - } - } -#endif // FEATURE_RANDOMIZED_STRING_HASHING + return Marvin.ComputeHash32(ref Unsafe.As(ref _firstChar), _stringLength * 2, Marvin.DefaultSeed); } // Determines whether a specified string is a prefix of the current instance