From d3d79f659fd96309540d452ed661998fd9c5a693 Mon Sep 17 00:00:00 2001 From: Dan Moseley Date: Sun, 15 Mar 2026 16:10:16 -0600 Subject: [PATCH 1/3] Fix race in BackgroundService exception aggregation during Host shutdown TryExecuteBackgroundServiceAsync tasks were fire-and-forget, creating a race where Host.StopAsync could read _backgroundServiceExceptions before the monitoring tasks had added their exceptions. When multiple BackgroundServices fault, this caused some exceptions to be silently lost. The fix stores the monitoring tasks and awaits them (with shutdown timeout) in StopAsync before reading the exception list. Fix #125589 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/Internal/Host.cs | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs b/src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs index 99159d0563d44e..c196d1f1c4d487 100644 --- a/src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs +++ b/src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs @@ -30,6 +30,7 @@ internal sealed class Host : IHost, IAsyncDisposable private IEnumerable? _hostedLifecycleServices; private bool _hostStarting; private bool _hostStopped; + private List? _backgroundServiceTasks; private List? _backgroundServiceExceptions; public Host(IServiceProvider services, @@ -126,7 +127,7 @@ await ForeachService(_hostedServices, cancellationToken, concurrent, abortOnFirs if (service is BackgroundService backgroundService) { - _ = TryExecuteBackgroundServiceAsync(backgroundService); + (_backgroundServiceTasks ??= new()).Add(TryExecuteBackgroundServiceAsync(backgroundService)); } }).ConfigureAwait(false); @@ -289,6 +290,23 @@ await ForeachService(reversedLifetimeServices, cancellationToken, concurrent, ab _hostStopped = true; + // Ensure all background service monitoring tasks have finished processing + // exceptions before we read them. Without this, there's a race: when a + // BackgroundService's ExecuteTask faults, both BackgroundService.StopAsync + // (which Host awaits) and TryExecuteBackgroundServiceAsync (fire-and-forget) + // have continuations scheduled. If StopAsync's continuation runs first, the + // Host may read _backgroundServiceExceptions before the monitoring task has + // added its exception. + if (_backgroundServiceTasks is not null) + { + Task bgMonitoringTasks = Task.WhenAll(_backgroundServiceTasks); + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using (cancellationToken.Register(s => ((TaskCompletionSource)s!).TrySetCanceled(), tcs)) + { + await Task.WhenAny(bgMonitoringTasks, tcs.Task).ConfigureAwait(false); + } + } + // If background services faulted and caused the host to stop, rethrow the exceptions // so they propagate and cause a non-zero exit code. List? backgroundServiceExceptions = Volatile.Read(ref _backgroundServiceExceptions); From 81d9d2ead0bbd1acea675e4e110b706bca430650 Mon Sep 17 00:00:00 2001 From: Dan Moseley Date: Sun, 15 Mar 2026 16:22:12 -0600 Subject: [PATCH 2/3] Make _backgroundServiceTasks thread-safe for concurrent start Use LazyInitializer.EnsureInitialized + lock, matching the existing pattern used for _backgroundServiceExceptions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Microsoft.Extensions.Hosting/src/Internal/Host.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs b/src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs index c196d1f1c4d487..efeb09405ff2ba 100644 --- a/src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs +++ b/src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs @@ -127,7 +127,12 @@ await ForeachService(_hostedServices, cancellationToken, concurrent, abortOnFirs if (service is BackgroundService backgroundService) { - (_backgroundServiceTasks ??= new()).Add(TryExecuteBackgroundServiceAsync(backgroundService)); + Task monitorTask = TryExecuteBackgroundServiceAsync(backgroundService); + List bgTasks = LazyInitializer.EnsureInitialized(ref _backgroundServiceTasks); + lock (bgTasks) + { + bgTasks.Add(monitorTask); + } } }).ConfigureAwait(false); From ca42dd05c677e2d414f83a11ca02b2f7477a367d Mon Sep 17 00:00:00 2001 From: Petr Onderka Date: Mon, 16 Mar 2026 11:58:44 +0100 Subject: [PATCH 3/3] Added test verifying late exceptions are captured --- .../BackgroundServiceExceptionTests.cs | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionTests.cs b/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionTests.cs index 05e30776023592..99abcf8e860cab 100644 --- a/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionTests.cs +++ b/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionTests.cs @@ -180,6 +180,38 @@ public async Task BackgroundServiceExceptionAndStopException_ThrowsAggregateExce Assert.IsType(ex)); } + /// + /// Regression test for a race where the fire-and-forget TryExecuteBackgroundServiceAsync + /// has not yet recorded its exception by the time Host.StopAsync reads the exception list. + /// DelayedMonitorFaultService overrides ExecuteTask so that the monitoring task sees a + /// separately-controlled task that faults 200ms after StopAsync returns, + /// reproducing the window in which the exception would be lost without the fix. + /// + [Fact] + public async Task BackgroundService_DelayedMonitoringException_ThrowsAggregateException() + { + var builder = new HostBuilder() + .ConfigureServices(services => + { + services.Configure(options => + { + options.BackgroundServiceExceptionBehavior = BackgroundServiceExceptionBehavior.StopHost; + }); + services.AddHostedService(); + services.AddHostedService(); + }); + + var aggregateException = await Assert.ThrowsAsync(async () => + { + await builder.Build().RunAsync(); + }); + + Assert.Equal(2, aggregateException.InnerExceptions.Count); + + Assert.All(aggregateException.InnerExceptions, ex => + Assert.IsType(ex)); + } + /// /// Tests that when a BackgroundService throws an exception with Ignore behavior, /// the host does not throw and continues to run until stopped. @@ -294,6 +326,43 @@ private class StopFailureService : IHostedService public Task StopAsync(CancellationToken cancellationToken) => throw new InvalidOperationException("Stop failure"); } + /// + /// A BackgroundService that overrides to return a separately + /// controlled task. The internal _executeTask (used by BackgroundService.StopAsync) completes + /// normally on cancellation, but the overridden ExecuteTask (monitored by + /// TryExecuteBackgroundServiceAsync) faults 200ms after StopAsync, deterministically + /// reproducing the race window. + /// + private class DelayedMonitorFaultService : BackgroundService + { + private readonly TaskCompletionSource _monitorTcs = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public override Task? ExecuteTask => _monitorTcs.Task; + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + try + { + await Task.Delay(Timeout.Infinite, stoppingToken); + } + catch (OperationCanceledException) + { + } + } + + public override async Task StopAsync(CancellationToken cancellationToken) + { + await base.StopAsync(cancellationToken); + _ = Task.Run(async () => + { + // This is testing that ExecuteTask delays stopping of the host, so it can't be triggered by a deterministic signal. + // It shouldn't cause any flakiness: incorrect ordering could cause the test to succeed when it should fail, but it shouldn't cause the test to fail when it should succeed. + await Task.Delay(200); + _monitorTcs.TrySetException(new InvalidOperationException("Delayed monitor failure")); + }); + } + } + private class SuccessfulService : BackgroundService { private readonly IHostApplicationLifetime _lifetime;