JIT: Disallow return buffer aliasing for unmanaged calls

src/coreclr/jit/importer.cpp

@@ -1219,7 +1219,7 @@ GenTree* Compiler::impAssignStructPtr(GenTree*             destAddr,
 
     var_types asgType = src->TypeGet();
 
-    if (src->gtOper == GT_CALL)
+    if (src->IsCall())
     {
         GenTreeCall* srcCall = src->AsCall();
         if (srcCall->TreatAsShouldHaveRetBufArg(this))
@@ -10995,6 +10995,17 @@ GenTree* Compiler::impFixupCallStructReturn(GenTreeCall* call, CORINFO_CLASS_HAN
     {
         assert(returnType == TYP_UNKNOWN);
         call->gtCallMoreFlags |= GTF_CALL_M_RETBUFFARG;
+
+        if (call->IsUnmanaged())
+        {
+            // Native ABIs do not allow retbufs to alias anything.
+            // This is allowed by the managed ABI and impAssignStructPtr will
+            // never introduce copies due to this.
+            unsigned tmpNum = lvaGrabTemp(true DEBUGARG("Retbuf for unmanaged call"));
+            impAssignTempGen(tmpNum, call, retClsHnd, (unsigned)CHECK_SPILL_ALL);
+            return gtNewLclvNode(tmpNum, lvaGetDesc(tmpNum)->TypeGet());
+        }
+
         return call;
     }
 

src/tests/JIT/Directed/aliasing_retbuf/CMakeLists.txt

@@ -0,0 +1,7 @@
+project(AliasingRetBufNative)
+
+# This test always needs to be optimized to hit the problem.
+set(CMAKE_BUILD_TYPE Release)
+
+add_library(AliasingRetBufNative SHARED aliasing_retbuf_native.cpp)
+

src/tests/JIT/Directed/aliasing_retbuf/aliasing_retbuf.cs

@@ -0,0 +1,130 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+public unsafe class AliasingRetBuf
+{
+    public static int Main()
+    {
+        int failures = 0;
+
+        Foo f = new Foo { A = 1, B = 2, C = 3 };
+        CallPtrPInvoke(&f);
+        if (f.A != 2 || f.B != 3 || f.C != 1)
+        {
+            Console.WriteLine("FAIL: After CallPtrPInvoke: {0}", f);
+            failures |= 1;
+        }
+
+        f = new Foo { A = 1, B = 2, C = 3 };
+        CallRefPInvoke(ref f);
+        if (f.A != 2 || f.B != 3 || f.C != 1)
+        {
+            Console.WriteLine("FAIL: After CallRefPInvoke: {0}", f);
+            failures |= 2;
+        }
+
+        f = new Foo { A = 1, B = 2, C = 3 };
+        CallStructFieldPInvoke(ref f);
+        if (f.A != 2 || f.B != 3 || f.C != 1)
+        {
+            Console.WriteLine("FAIL: After CallStructFieldPInvoke: {0}", f);
+            failures |= 4;
+        }
+
+        IntPtr lib = NativeLibrary.Load(nameof(AliasingRetBufNative), typeof(AliasingRetBufNative).Assembly, null);
+        IntPtr export = NativeLibrary.GetExport(lib, "TransposeRetBuf");
+
+        f = new Foo { A = 3, B = 2, C = 1 };
+        CallPtrFPtr(&f, (delegate* unmanaged[Cdecl, SuppressGCTransition]<Foo*, Foo>)export);
+        if (f.A != 2 || f.B != 1 || f.C != 3)
+        {
+            Console.WriteLine("FAIL: After CallPtrFPtr: {0}", f);
+            failures |= 8;
+        }
+
+        f = new Foo { A = 3, B = 2, C = 1 };
+        CallStructFieldFPtr(ref f, (delegate* unmanaged[Cdecl, SuppressGCTransition]<Foo*, Foo>)export);
+        if (f.A != 2 || f.B != 1 || f.C != 3)
+        {
+            Console.WriteLine("FAIL: After CallStructFieldFPtr: {0}", f);
+            failures |= 16;
+        }
+
+        f = new Foo { A = 3, B = 2, C = 1 };
+        CallRefFPtr(ref f, (delegate* unmanaged[Cdecl, SuppressGCTransition]<ref Foo, Foo>)export);
+        if (f.A != 2 || f.B != 1 || f.C != 3)
+        {
+            Console.WriteLine("FAIL: After CallRefFPtr: {0}", f);
+            failures |= 32;
+        }
+
+        if (failures == 0)
+        {
+            Console.WriteLine("PASS");
+        }
+
+        return 100 + failures;
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)]
+    private static void CallPtrPInvoke(Foo* fi)
+    {
+        *fi = AliasingRetBufNative.TransposeRetBufPtr(fi);
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)]
+    private static void CallRefPInvoke(ref Foo fi)
+    {
+        fi = AliasingRetBufNative.TransposeRetBufRef(ref fi);
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)]
+    private static void CallStructFieldPInvoke(ref Foo fi)
+    {
+        Fooer fooer = new() { F = fi };
+        fooer.F = AliasingRetBufNative.TransposeRetBufPtr(&fooer.F);
+        fi = fooer.F;
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)]
+    private static void CallPtrFPtr(Foo* fi, delegate* unmanaged[Cdecl, SuppressGCTransition]<Foo*, Foo> fptr)
+    {
+        *fi = fptr(fi);
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)]
+    private static void CallRefFPtr(ref Foo fi, delegate* unmanaged[Cdecl, SuppressGCTransition]<ref Foo, Foo> fptr)
+    {
+        fi = fptr(ref fi);
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)]
+    private static void CallStructFieldFPtr(ref Foo fi, delegate* unmanaged[Cdecl, SuppressGCTransition]<Foo*, Foo> fptr)
+    {
+        Fooer fooer = new() { F = fi };
+        fooer.F = fi;
+        fooer.F = fptr(&fooer.F);
+        fi = fooer.F;
+    }
+
+    private record struct Foo(nint A, nint B, nint C);
+    private struct Fooer
+    {
+        public Foo F;
+    }
+
+    static class AliasingRetBufNative
+    {
+        [DllImport(nameof(AliasingRetBufNative), EntryPoint = "TransposeRetBuf", CallingConvention = CallingConvention.Cdecl)]
+        [SuppressGCTransition]
+        public static unsafe extern Foo TransposeRetBufPtr(Foo* fi);
+
+        [DllImport(nameof(AliasingRetBufNative), EntryPoint = "TransposeRetBuf", CallingConvention = CallingConvention.Cdecl)]
+        [SuppressGCTransition]
+        public static unsafe extern Foo TransposeRetBufRef(ref Foo fi);
+    }
+}

src/tests/JIT/Directed/aliasing_retbuf/aliasing_retbuf.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <Optimize>True</Optimize>
+    <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="$(MSBuildProjectName).cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <CMakeProjectReference Include="CMakeLists.txt" />
+  </ItemGroup>
+</Project>

src/tests/JIT/Directed/aliasing_retbuf/aliasing_retbuf_native.cpp

@@ -0,0 +1,30 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+#define CALLCONV
+
+#if defined(_MSC_VER)
+#define EXPORT_API extern "C" __declspec(dllexport)
+#ifdef _WIN32
+#undef CALLCONV
+#define CALLCONV __cdecl
+#endif
+#else
+#define EXPORT_API extern "C" __attribute__((visibility("default")))
+#endif
+
+struct Foo
+{
+    void* A;
+    void* B;
+    void* C;
+};
+
+EXPORT_API Foo CALLCONV TransposeRetBuf(Foo* fi)
+{
+    Foo f;
+    f.A = fi->B;
+    f.B = fi->C;
+    f.C = fi->A;
+    return f;
+}

src/tests/issues.targets

@@ -3395,6 +3395,9 @@
         <ExcludeList Include="$(XUnitTestBinBase)/JIT/Directed/callconv/CdeclMemberFunction/CdeclMemberFunctionTest/*">
             <Issue>https://github.com/dotnet/runtime/issues/41519</Issue>
         </ExcludeList>
+        <ExcludeList Include="$(XUnitTestBinBase)/JIT/Directed/aliasing_retbuf/aliasing_retbuf/*">
+            <Issue>https://github.com/dotnet/runtime/issues/41519</Issue>
+        </ExcludeList>
         <ExcludeList Include = "$(XunitTestBinBase)/Loader/classloader/TypeInitialization/CircularCctors/CircularCctorFourThreadsBFI/**">
             <Issue>https://github.com/dotnet/runtime/issues/41472</Issue>
         </ExcludeList>