Fix race in BackgroundService exception aggregation during Host shutdown

src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs

@@ -30,6 +30,7 @@ internal sealed class Host : IHost, IAsyncDisposable
         private IEnumerable<IHostedLifecycleService>? _hostedLifecycleServices;
         private bool _hostStarting;
         private bool _hostStopped;
+        private List<Task>? _backgroundServiceTasks;
         private List<Exception>? _backgroundServiceExceptions;
 
         public Host(IServiceProvider services,
@@ -126,7 +127,12 @@ await ForeachService(_hostedServices, cancellationToken, concurrent, abortOnFirs
 
                         if (service is BackgroundService backgroundService)
                         {
-                            _ = TryExecuteBackgroundServiceAsync(backgroundService);
+                            Task monitorTask = TryExecuteBackgroundServiceAsync(backgroundService);
+                            List<Task> bgTasks = LazyInitializer.EnsureInitialized(ref _backgroundServiceTasks);
+                            lock (bgTasks)
+                            {
+                                bgTasks.Add(monitorTask);
+                            }
                         }
                     }).ConfigureAwait(false);
 
@@ -289,6 +295,23 @@ await ForeachService(reversedLifetimeServices, cancellationToken, concurrent, ab
 
                 _hostStopped = true;
 
+                // Ensure all background service monitoring tasks have finished processing
+                // exceptions before we read them. Without this, there's a race: when a
+                // BackgroundService's ExecuteTask faults, both BackgroundService.StopAsync
+                // (which Host awaits) and TryExecuteBackgroundServiceAsync (fire-and-forget)
+                // have continuations scheduled. If StopAsync's continuation runs first, the
+                // Host may read _backgroundServiceExceptions before the monitoring task has
+                // added its exception.
+                if (_backgroundServiceTasks is not null)
+                {
+                    Task bgMonitoringTasks = Task.WhenAll(_backgroundServiceTasks);
+                    var tcs = new TaskCompletionSource<object?>(TaskCreationOptions.RunContinuationsAsynchronously);
+                    using (cancellationToken.Register(s => ((TaskCompletionSource<object?>)s!).TrySetCanceled(), tcs))
+                    {
+                        await Task.WhenAny(bgMonitoringTasks, tcs.Task).ConfigureAwait(false);
+                    }
+                }
+
                 // If background services faulted and caused the host to stop, rethrow the exceptions
                 // so they propagate and cause a non-zero exit code.
                 List<Exception>? backgroundServiceExceptions = Volatile.Read(ref _backgroundServiceExceptions);

src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionTests.cs

@@ -180,6 +180,38 @@ public async Task BackgroundServiceExceptionAndStopException_ThrowsAggregateExce
                 Assert.IsType<InvalidOperationException>(ex));
         }
 
+        /// <summary>
+        /// Regression test for a race where the fire-and-forget TryExecuteBackgroundServiceAsync
+        /// has not yet recorded its exception by the time Host.StopAsync reads the exception list.
+        /// DelayedMonitorFaultService overrides ExecuteTask so that the monitoring task sees a
+        /// separately-controlled task that faults 200ms after StopAsync returns,
+        /// reproducing the window in which the exception would be lost without the fix.
+        /// </summary>
+        [Fact]
+        public async Task BackgroundService_DelayedMonitoringException_ThrowsAggregateException()
+        {
+            var builder = new HostBuilder()
+                .ConfigureServices(services =>
+                {
+                    services.Configure<HostOptions>(options =>
+                    {
+                        options.BackgroundServiceExceptionBehavior = BackgroundServiceExceptionBehavior.StopHost;
+                    });
+                    services.AddHostedService<SynchronousFailureService>();
+                    services.AddHostedService<DelayedMonitorFaultService>();
+                });
+
+            var aggregateException = await Assert.ThrowsAsync<AggregateException>(async () =>
+            {
+                await builder.Build().RunAsync();
+            });
+
+            Assert.Equal(2, aggregateException.InnerExceptions.Count);
+
+            Assert.All(aggregateException.InnerExceptions, ex =>
+                Assert.IsType<InvalidOperationException>(ex));
+        }
+
         /// <summary>
         /// Tests that when a BackgroundService throws an exception with Ignore behavior,
         /// the host does not throw and continues to run until stopped.
@@ -294,6 +326,43 @@ private class StopFailureService : IHostedService
             public Task StopAsync(CancellationToken cancellationToken) => throw new InvalidOperationException("Stop failure");
         }
 
+        /// <summary>
+        /// A BackgroundService that overrides <see cref="ExecuteTask"/> to return a separately
+        /// controlled task. The internal _executeTask (used by BackgroundService.StopAsync) completes
+        /// normally on cancellation, but the overridden ExecuteTask (monitored by
+        /// TryExecuteBackgroundServiceAsync) faults 200ms after StopAsync, deterministically
+        /// reproducing the race window.
+        /// </summary>
+        private class DelayedMonitorFaultService : BackgroundService
+        {
+            private readonly TaskCompletionSource<object?> _monitorTcs = new(TaskCreationOptions.RunContinuationsAsynchronously);
+
+            public override Task? ExecuteTask => _monitorTcs.Task;
+
+            protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+            {
+                try
+                {
+                    await Task.Delay(Timeout.Infinite, stoppingToken);
+                }
+                catch (OperationCanceledException)
+                {
+                }
+            }
+
+            public override async Task StopAsync(CancellationToken cancellationToken)
+            {
+                await base.StopAsync(cancellationToken);
+                _ = Task.Run(async () =>
+                {
+                    // This is testing that ExecuteTask delays stopping of the host, so it can't be triggered by a deterministic signal.
+                    // It shouldn't cause any flakiness: incorrect ordering could cause the test to succeed when it should fail, but it shouldn't cause the test to fail when it should succeed.
+                    await Task.Delay(200);
+                    _monitorTcs.TrySetException(new InvalidOperationException("Delayed monitor failure"));
+                });
+            }
+        }
+
         private class SuccessfulService : BackgroundService
         {
             private readonly IHostApplicationLifetime _lifetime;