[NativeAOT] Print OOM message before Abort() on Linux

src/coreclr/nativeaot/System.Private.CoreLib/src/System/RuntimeExceptionHelpers.cs

@@ -201,16 +201,23 @@ internal static void SerializeCrashInfo(RhFailFastReason reason, string? message
             int previousState = Interlocked.CompareExchange(ref s_crashInfoPresent, -1, 0);
             if (previousState == 0)
             {
-                CrashInfo crashInfo = new();
+                try
+                {
+                    CrashInfo crashInfo = new();
 
-                crashInfo.Open(reason, Thread.CurrentOSThreadId, message ?? GetStringForFailFastReason(reason));
-                if (exception != null)
+                    crashInfo.Open(reason, Thread.CurrentOSThreadId, message ?? GetStringForFailFastReason(reason));
+                    if (exception != null)
+                    {
+                        crashInfo.WriteException(exception);
+                    }
+                    crashInfo.Close();
+                    s_triageBufferAddress = crashInfo.TriageBufferAddress;
+                    s_triageBufferSize = crashInfo.TriageBufferSize;
+                }
+                catch
                 {
-                    crashInfo.WriteException(exception);
+                    // If crash info serialization fails (for example, due to OOM), proceed without it.
                 }
-                crashInfo.Close();
-                s_triageBufferAddress = crashInfo.TriageBufferAddress;
-                s_triageBufferSize = crashInfo.TriageBufferSize;
 
                 s_crashInfoPresent = 1;
             }
@@ -235,8 +242,19 @@ internal static unsafe void FailFast(string? message = null, Exception? exceptio
             ulong previousThreadId = Interlocked.CompareExchange(ref s_crashingThreadId, currentThreadId, 0);
             if (previousThreadId == 0)
             {
-                bool minimalFailFast = (exception == PreallocatedOutOfMemoryException.Instance);
-                if (!minimalFailFast)
+                bool minimalFailFast = exception == PreallocatedOutOfMemoryException.Instance;
+                if (minimalFailFast)
+                {
+                    // Minimal OOM fail-fast path: avoid heap allocations as much as possible, but still
+                    // report that OOM is the reason for the crash.
+                    try
+                    {
+                        Internal.Console.Error.Write("Process terminated. System.OutOfMemoryException");
+                        Internal.Console.Error.WriteLine();
+                    }
+                    catch { }
+                }
+                else
                 {
                     Internal.Console.Error.Write(((exception == null) || (reason is RhFailFastReason.EnvironmentFailFast or RhFailFastReason.AssertionFailure)) ?
                         "Process terminated. " : "Unhandled exception. ");
@@ -266,8 +284,21 @@ internal static unsafe void FailFast(string? message = null, Exception? exceptio
 
                     if ((exception != null) && (reason is not RhFailFastReason.AssertionFailure))
                     {
-                        Internal.Console.Error.Write(exception.ToString());
-                        Internal.Console.Error.WriteLine();
+                        try
+                        {
+                            Internal.Console.Error.Write(exception.ToString());
+                            Internal.Console.Error.WriteLine();
+                        }
+                        catch
+                        {
+                            // If ToString() fails (for example, due to OOM), fall back to printing just the type name.
+                            try
+                            {
+                                Internal.Console.Error.Write(exception.GetType().FullName);
+                                Internal.Console.Error.WriteLine();
+                            }
+                            catch { }
+                        }
                     }
 
 #if TARGET_WINDOWS

src/tests/nativeaot/SmokeTests/OomHandling/OomHandling.cs

@@ -0,0 +1,118 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+// This test verifies that an out-of-memory condition in a NativeAOT process
+// produces a diagnostic message on stderr before the process terminates.
+//
+// The test spawns itself as a subprocess with a small GC heap limit set via
+// DOTNET_GCHeapHardLimit so that the subprocess reliably runs out of memory.
+// The outer process then validates that the subprocess wrote the expected
+// OOM message to its standard error stream.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Threading.Tasks;
+
+class OomHandlingTest
+{
+    const int Pass = 100;
+    const int Fail = -1;
+    const int TimeoutMilliseconds = 30 * 1000;
+
+    const string AllocateSmallArg = "--allocate-small";
+    const string AllocateLargeArg = "--allocate-large";
+    // Both the minimal OOM fail-fast path ("Process terminated. System.OutOfMemoryException")
+    // and the standard unhandled-exception path ("Unhandled exception. System.OutOfMemoryException...")
+    // contain this token. The test validates that some OOM diagnostic is printed rather than
+    // just "Aborted" with no context.
+    const string ExpectedToken = "OutOfMemoryException";
+
+    static int Main(string[] args)
+    {
+        if (args.Length > 0 && args[0] == AllocateSmallArg)
+        {
+            // Subprocess mode: allocate until OOM is triggered.
+            // Phase 1: fill quickly with large blocks to use most of the heap.
+            // Phase 2: exhaust remaining scraps with small allocations so that
+            // virtually no memory is left when OOM is finally thrown.
+            var list = new List<object>();
+            try { while (true) list.Add(new byte[16 * 1024]); } catch (OutOfMemoryException) { }
+            while (true) list.Add(new object());
+        }
+
+        if (args.Length > 0 && args[0] == AllocateLargeArg)
+        {
+            // Subprocess mode: allocate 128 KB chunks until OOM is triggered.
+            // This leaves some free memory when OOM fires, exercising the code
+            // path where GetRuntimeException may allocate a new OutOfMemoryException.
+            var list = new List<byte[]>();
+            while (true) list.Add(new byte[128 * 1024]);
+        }
+
+        // Controller mode: launch subprocesses with a GC heap limit and verify their output.
+        string? processPath = Environment.ProcessPath;
+        if (processPath == null)
+        {
+            Console.WriteLine("ProcessPath is null, skipping test.");
+            return Pass;
+        }
+
+        int result = RunSubprocess(processPath, AllocateSmallArg, "small allocations");
+        if (result != Pass)
+            return result;
+
+        result = RunSubprocess(processPath, AllocateLargeArg, "large allocations");
+        return result;
+    }
+
+    static int RunSubprocess(string processPath, string allocateArg, string description)
+    {
+        Console.WriteLine($"Testing OOM with {description}...");
+
+        var psi = new ProcessStartInfo(processPath, allocateArg)
+        {
+            RedirectStandardError = true,
+            UseShellExecute = false,
+        };
+        // 0x2000000 = 32 MB GC heap limit: small enough to exhaust quickly but large enough for startup.
+        psi.Environment["DOTNET_GCHeapHardLimit"] = "2000000";
+
+        using Process? p = Process.Start(psi);
+        if (p is null)
+        {
+            Console.WriteLine("Failed to start subprocess.");
+            return Fail;
+        }
+
+        // Read stderr asynchronously so that WaitForExit can enforce the timeout.
+        // A synchronous ReadToEnd() would block until the child exits, defeating the timeout.
+        Task<string> stderrTask = p.StandardError.ReadToEndAsync();
+        if (!p.WaitForExit(TimeoutMilliseconds))
+        {
+            p.Kill(true);
+            p.WaitForExit();
+            _ = stderrTask.GetAwaiter().GetResult();
+            Console.WriteLine($"Subprocess timed out after {TimeoutMilliseconds / 1000} seconds.");
+            return Fail;
+        }
+        string stderr = stderrTask.GetAwaiter().GetResult();
+
+        Console.WriteLine($"Subprocess exit code: {p.ExitCode}");
+        Console.WriteLine($"Subprocess stderr: {stderr}");
+
+        if (p.ExitCode == 0 || p.ExitCode == Pass)
+        {
+            Console.WriteLine("Expected a non-success exit code from the OOM subprocess.");
+            return Fail;
+        }
+
+        if (!stderr.Contains(ExpectedToken))
+        {
+            Console.WriteLine($"Expected stderr to contain: {ExpectedToken}");
+            return Fail;
+        }
+
+        return Pass;
+    }
+}

src/tests/nativeaot/SmokeTests/OomHandling/OomHandling.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <CLRTestPriority>0</CLRTestPriority>
+    <!-- This test spawns a subprocess; not supported on mobile or browser platforms -->
+    <CLRTestTargetUnsupported Condition="'$(TargetsAppleMobile)' == 'true' or '$(TargetsAndroid)' == 'true' or '$(TargetsBrowser)' == 'true'">true</CLRTestTargetUnsupported>
+    <RequiresProcessIsolation>true</RequiresProcessIsolation>
+    <ReferenceXUnitWrapperGenerator>false</ReferenceXUnitWrapperGenerator>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="OomHandling.cs" />
+  </ItemGroup>
+</Project>