From 9a8a047a35c253255f74f2dfcb86a750981b974a Mon Sep 17 00:00:00 2001 From: Woongsuk Cho Date: Wed, 9 Nov 2022 07:10:06 +0900 Subject: [PATCH 1/7] Return NULL for mmap fail case on Unix If mmap failed, "MAP_FAILED" is returned not "NULL". The windows implememtation of GetRWMapping returns "NULL" for fail case, and the caller function is also checking "NULL". So, change Unix implementation to return "NULL" for fail case. --- src/coreclr/minipal/Unix/doublemapping.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/coreclr/minipal/Unix/doublemapping.cpp b/src/coreclr/minipal/Unix/doublemapping.cpp index 57ce0c09f283df..27e6418be430f6 100644 --- a/src/coreclr/minipal/Unix/doublemapping.cpp +++ b/src/coreclr/minipal/Unix/doublemapping.cpp @@ -208,7 +208,12 @@ void* VMToOSInterface::GetRWMapping(void *mapperHandle, void* pStart, size_t off { #ifndef TARGET_OSX int fd = (int)(size_t)mapperHandle; - return mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, offset); + void* result = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, offset); + if (result == MAP_FAILED) + { + result = NULL; + } + return result; #else // TARGET_OSX #ifdef TARGET_AMD64 vm_address_t startRW; From 8eb572415cea7ac252e262a719470583107d1c6e Mon Sep 17 00:00:00 2001 From: Woongsuk Cho Date: Wed, 9 Nov 2022 07:30:16 +0900 Subject: [PATCH 2/7] call memset when mmap succeeds --- src/coreclr/minipal/Unix/doublemapping.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/coreclr/minipal/Unix/doublemapping.cpp b/src/coreclr/minipal/Unix/doublemapping.cpp index 27e6418be430f6..9e07d05a8b4229 100644 --- a/src/coreclr/minipal/Unix/doublemapping.cpp +++ b/src/coreclr/minipal/Unix/doublemapping.cpp @@ -198,8 +198,10 @@ bool VMToOSInterface::ReleaseDoubleMappedMemory(void *mapperHandle, void* pStart { #ifndef TARGET_OSX int fd = (int)(size_t)mapperHandle; - mmap(pStart, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fd, offset); - memset(pStart, 0, size); + if (mmap(pStart, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fd, offset) != MAP_FAILED) + { + memset(pStart, 0, size); + } #endif // TARGET_OSX return munmap(pStart, size) != -1; } From 3b7535206115b1de9baac2bba3d967b2da16287a Mon Sep 17 00:00:00 2001 From: Woongsuk Cho Date: Wed, 9 Nov 2022 08:05:55 +0900 Subject: [PATCH 3/7] check MAP_FAILED instead of NULL --- src/coreclr/nativeaot/Runtime/unix/PalRedhawkUnix.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/nativeaot/Runtime/unix/PalRedhawkUnix.cpp b/src/coreclr/nativeaot/Runtime/unix/PalRedhawkUnix.cpp index e54ce9a015141f..e72c83a15b1dad 100644 --- a/src/coreclr/nativeaot/Runtime/unix/PalRedhawkUnix.cpp +++ b/src/coreclr/nativeaot/Runtime/unix/PalRedhawkUnix.cpp @@ -743,7 +743,7 @@ REDHAWK_PALEXPORT _Ret_maybenull_ _Post_writable_byte_size_(size) void* REDHAWK_ void * pRetVal = mmap(pAddress, alignedSize, unixProtect, flags, -1, 0); - if (pRetVal != NULL) + if (pRetVal != MAP_FAILED) { void * pAlignedRetVal = (void *)(((size_t)pRetVal + (Alignment - 1)) & ~(Alignment - 1)); size_t startPadding = (size_t)pAlignedRetVal - (size_t)pRetVal; From c50ee0e04751e675fdd5c9968e092621b24d29b8 Mon Sep 17 00:00:00 2001 From: Woongsuk Cho Date: Wed, 9 Nov 2022 19:02:53 +0900 Subject: [PATCH 4/7] return false for failing mmap --- src/coreclr/minipal/Unix/doublemapping.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/coreclr/minipal/Unix/doublemapping.cpp b/src/coreclr/minipal/Unix/doublemapping.cpp index 9e07d05a8b4229..e38c84b752e96b 100644 --- a/src/coreclr/minipal/Unix/doublemapping.cpp +++ b/src/coreclr/minipal/Unix/doublemapping.cpp @@ -200,8 +200,9 @@ bool VMToOSInterface::ReleaseDoubleMappedMemory(void *mapperHandle, void* pStart int fd = (int)(size_t)mapperHandle; if (mmap(pStart, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fd, offset) != MAP_FAILED) { - memset(pStart, 0, size); + return false; } + memset(pStart, 0, size); #endif // TARGET_OSX return munmap(pStart, size) != -1; } From d9d1f0a145b4a120aae3207468d55c9cc196baf6 Mon Sep 17 00:00:00 2001 From: Woongsuk Cho Date: Wed, 9 Nov 2022 20:01:54 +0900 Subject: [PATCH 5/7] Call g_fatalErrorHandler if releasing failed --- src/coreclr/utilcode/executableallocator.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/coreclr/utilcode/executableallocator.cpp b/src/coreclr/utilcode/executableallocator.cpp index 7872954adc485c..aac65064aecbce 100644 --- a/src/coreclr/utilcode/executableallocator.cpp +++ b/src/coreclr/utilcode/executableallocator.cpp @@ -453,7 +453,10 @@ void ExecutableAllocator::Release(void* pRX) if (pBlock != NULL) { - VMToOSInterface::ReleaseDoubleMappedMemory(m_doubleMemoryMapperHandle, pRX, pBlock->offset, pBlock->size); + if (VMToOSInterface::ReleaseDoubleMappedMemory(m_doubleMemoryMapperHandle, pRX, pBlock->offset, pBlock->size) == false) + { + g_fatalErrorHandler(COR_E_EXECUTIONENGINE, W("Releasing the double mapped memory failed")); + } // Put the released block into the free block list pBlock->baseRX = NULL; pBlock->next = m_pFirstFreeBlockRX; From 936acd61d43c25f65cc9be73dfa5771ad3ab9f1f Mon Sep 17 00:00:00 2001 From: "ws77.cho" Date: Thu, 10 Nov 2022 06:09:38 +0900 Subject: [PATCH 6/7] Update src/coreclr/utilcode/executableallocator.cpp Co-authored-by: Jan Vorlicek --- src/coreclr/utilcode/executableallocator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/utilcode/executableallocator.cpp b/src/coreclr/utilcode/executableallocator.cpp index aac65064aecbce..078800995d1334 100644 --- a/src/coreclr/utilcode/executableallocator.cpp +++ b/src/coreclr/utilcode/executableallocator.cpp @@ -453,7 +453,7 @@ void ExecutableAllocator::Release(void* pRX) if (pBlock != NULL) { - if (VMToOSInterface::ReleaseDoubleMappedMemory(m_doubleMemoryMapperHandle, pRX, pBlock->offset, pBlock->size) == false) + if (!VMToOSInterface::ReleaseDoubleMappedMemory(m_doubleMemoryMapperHandle, pRX, pBlock->offset, pBlock->size)) { g_fatalErrorHandler(COR_E_EXECUTIONENGINE, W("Releasing the double mapped memory failed")); } From 3e1850c1945d7af7e0e003ae38e6da12ddd8af0a Mon Sep 17 00:00:00 2001 From: Woongsuk Cho Date: Thu, 10 Nov 2022 06:16:30 +0900 Subject: [PATCH 7/7] Fix wrong condition check --- src/coreclr/minipal/Unix/doublemapping.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/minipal/Unix/doublemapping.cpp b/src/coreclr/minipal/Unix/doublemapping.cpp index e38c84b752e96b..cb65e5e284e2b3 100644 --- a/src/coreclr/minipal/Unix/doublemapping.cpp +++ b/src/coreclr/minipal/Unix/doublemapping.cpp @@ -198,7 +198,7 @@ bool VMToOSInterface::ReleaseDoubleMappedMemory(void *mapperHandle, void* pStart { #ifndef TARGET_OSX int fd = (int)(size_t)mapperHandle; - if (mmap(pStart, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fd, offset) != MAP_FAILED) + if (mmap(pStart, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fd, offset) == MAP_FAILED) { return false; }