diff --git a/src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs b/src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs index 99159d0563d44e..efeb09405ff2ba 100644 --- a/src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs +++ b/src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs @@ -30,6 +30,7 @@ internal sealed class Host : IHost, IAsyncDisposable private IEnumerable? _hostedLifecycleServices; private bool _hostStarting; private bool _hostStopped; + private List? _backgroundServiceTasks; private List? _backgroundServiceExceptions; public Host(IServiceProvider services, @@ -126,7 +127,12 @@ await ForeachService(_hostedServices, cancellationToken, concurrent, abortOnFirs if (service is BackgroundService backgroundService) { - _ = TryExecuteBackgroundServiceAsync(backgroundService); + Task monitorTask = TryExecuteBackgroundServiceAsync(backgroundService); + List bgTasks = LazyInitializer.EnsureInitialized(ref _backgroundServiceTasks); + lock (bgTasks) + { + bgTasks.Add(monitorTask); + } } }).ConfigureAwait(false); @@ -289,6 +295,23 @@ await ForeachService(reversedLifetimeServices, cancellationToken, concurrent, ab _hostStopped = true; + // Ensure all background service monitoring tasks have finished processing + // exceptions before we read them. Without this, there's a race: when a + // BackgroundService's ExecuteTask faults, both BackgroundService.StopAsync + // (which Host awaits) and TryExecuteBackgroundServiceAsync (fire-and-forget) + // have continuations scheduled. If StopAsync's continuation runs first, the + // Host may read _backgroundServiceExceptions before the monitoring task has + // added its exception. + if (_backgroundServiceTasks is not null) + { + Task bgMonitoringTasks = Task.WhenAll(_backgroundServiceTasks); + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using (cancellationToken.Register(s => ((TaskCompletionSource)s!).TrySetCanceled(), tcs)) + { + await Task.WhenAny(bgMonitoringTasks, tcs.Task).ConfigureAwait(false); + } + } + // If background services faulted and caused the host to stop, rethrow the exceptions // so they propagate and cause a non-zero exit code. List? backgroundServiceExceptions = Volatile.Read(ref _backgroundServiceExceptions); diff --git a/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionTests.cs b/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionTests.cs index 05e30776023592..99abcf8e860cab 100644 --- a/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionTests.cs +++ b/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/BackgroundServiceExceptionTests.cs @@ -180,6 +180,38 @@ public async Task BackgroundServiceExceptionAndStopException_ThrowsAggregateExce Assert.IsType(ex)); } + /// + /// Regression test for a race where the fire-and-forget TryExecuteBackgroundServiceAsync + /// has not yet recorded its exception by the time Host.StopAsync reads the exception list. + /// DelayedMonitorFaultService overrides ExecuteTask so that the monitoring task sees a + /// separately-controlled task that faults 200ms after StopAsync returns, + /// reproducing the window in which the exception would be lost without the fix. + /// + [Fact] + public async Task BackgroundService_DelayedMonitoringException_ThrowsAggregateException() + { + var builder = new HostBuilder() + .ConfigureServices(services => + { + services.Configure(options => + { + options.BackgroundServiceExceptionBehavior = BackgroundServiceExceptionBehavior.StopHost; + }); + services.AddHostedService(); + services.AddHostedService(); + }); + + var aggregateException = await Assert.ThrowsAsync(async () => + { + await builder.Build().RunAsync(); + }); + + Assert.Equal(2, aggregateException.InnerExceptions.Count); + + Assert.All(aggregateException.InnerExceptions, ex => + Assert.IsType(ex)); + } + /// /// Tests that when a BackgroundService throws an exception with Ignore behavior, /// the host does not throw and continues to run until stopped. @@ -294,6 +326,43 @@ private class StopFailureService : IHostedService public Task StopAsync(CancellationToken cancellationToken) => throw new InvalidOperationException("Stop failure"); } + /// + /// A BackgroundService that overrides to return a separately + /// controlled task. The internal _executeTask (used by BackgroundService.StopAsync) completes + /// normally on cancellation, but the overridden ExecuteTask (monitored by + /// TryExecuteBackgroundServiceAsync) faults 200ms after StopAsync, deterministically + /// reproducing the race window. + /// + private class DelayedMonitorFaultService : BackgroundService + { + private readonly TaskCompletionSource _monitorTcs = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public override Task? ExecuteTask => _monitorTcs.Task; + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + try + { + await Task.Delay(Timeout.Infinite, stoppingToken); + } + catch (OperationCanceledException) + { + } + } + + public override async Task StopAsync(CancellationToken cancellationToken) + { + await base.StopAsync(cancellationToken); + _ = Task.Run(async () => + { + // This is testing that ExecuteTask delays stopping of the host, so it can't be triggered by a deterministic signal. + // It shouldn't cause any flakiness: incorrect ordering could cause the test to succeed when it should fail, but it shouldn't cause the test to fail when it should succeed. + await Task.Delay(200); + _monitorTcs.TrySetException(new InvalidOperationException("Delayed monitor failure")); + }); + } + } + private class SuccessfulService : BackgroundService { private readonly IHostApplicationLifetime _lifetime;