From 08e56ceb0c9374f44fb7091a9622c46fb4287ba6 Mon Sep 17 00:00:00 2001 From: Szymon Pobiega Date: Sun, 24 Mar 2024 22:31:48 +0100 Subject: [PATCH 1/4] Add exception logging to DomainEvents --- .../DomainEvents.cs | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/src/ServiceControl.DomainEvents/DomainEvents.cs b/src/ServiceControl.DomainEvents/DomainEvents.cs index 6df5c980c1..5714b464d1 100644 --- a/src/ServiceControl.DomainEvents/DomainEvents.cs +++ b/src/ServiceControl.DomainEvents/DomainEvents.cs @@ -3,9 +3,12 @@ using System; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; + using NServiceBus.Logging; public class DomainEvents : IDomainEvents { + static readonly ILog Log = LogManager.GetLogger(); + readonly IServiceProvider serviceProvider; public DomainEvents(IServiceProvider serviceProvider) => this.serviceProvider = serviceProvider; @@ -14,15 +17,31 @@ public async Task Raise(T domainEvent) where T : IDomainEvent var handlers = serviceProvider.GetServices>(); foreach (var handler in handlers) { - await handler.Handle(domainEvent) - .ConfigureAwait(false); + try + { + await handler.Handle(domainEvent) + .ConfigureAwait(false); + } + catch (Exception e) + { + Log.Error($"Unexpected error publishing domain event {typeof(T)}", e); + throw; + } } var ieventHandlers = serviceProvider.GetServices>(); foreach (var handler in ieventHandlers) { - await handler.Handle(domainEvent) + try + { + await handler.Handle(domainEvent) .ConfigureAwait(false); + } + catch (Exception e) + { + Log.Error($"Unexpected error publishing domain event {typeof(T)}", e); + throw; + } } } } From a91bdc7fdae60f2bbb77ee3e961a0730642d8ed0 Mon Sep 17 00:00:00 2001 From: Szymon Pobiega Date: Sun, 24 Mar 2024 22:32:04 +0100 Subject: [PATCH 2/4] Add NServiceBus reference --- .../ServiceControl.DomainEvents.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ServiceControl.DomainEvents/ServiceControl.DomainEvents.csproj b/src/ServiceControl.DomainEvents/ServiceControl.DomainEvents.csproj index a4da52cc6b..8b8037bc48 100644 --- a/src/ServiceControl.DomainEvents/ServiceControl.DomainEvents.csproj +++ b/src/ServiceControl.DomainEvents/ServiceControl.DomainEvents.csproj @@ -6,6 +6,7 @@ + From 8fc526e1f914ef7f236a1b0910038fe9e18416e8 Mon Sep 17 00:00:00 2001 From: Szymon Pobiega Date: Sun, 24 Mar 2024 22:32:19 +0100 Subject: [PATCH 3/4] Add debug logging to EndpointInstanceMonitor --- src/ServiceControl/Monitoring/EndpointInstanceMonitor.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/ServiceControl/Monitoring/EndpointInstanceMonitor.cs b/src/ServiceControl/Monitoring/EndpointInstanceMonitor.cs index 2941c43b9b..e055fe46ce 100644 --- a/src/ServiceControl/Monitoring/EndpointInstanceMonitor.cs +++ b/src/ServiceControl/Monitoring/EndpointInstanceMonitor.cs @@ -5,6 +5,8 @@ namespace ServiceControl.Monitoring using Contracts.HeartbeatMonitoring; using EndpointControl.Contracts; using Infrastructure.DomainEvents; + using NLog.Fluent; + using NServiceBus.Logging; using ServiceControl.Operations; using ServiceControl.Persistence; @@ -38,6 +40,7 @@ public async Task UpdateStatus(HeartbeatStatus newStatus, DateTime? latestTimest if (newStatus != status) { await RaiseStateChangeEvents(newStatus, latestTimestamp); + Log.DebugFormat("Endpoint {0} status updated from {1} to {2}", Id.LogicalName, status, newStatus); } lastSeen = latestTimestamp; @@ -132,6 +135,8 @@ public KnownEndpointsView GetKnownView() }; } + static readonly ILog Log = LogManager.GetLogger(); + IDomainEvents domainEvents; DateTime? lastSeen; HeartbeatStatus status; From 93729c635f8e8d280f89eedcf71a56453976e0e2 Mon Sep 17 00:00:00 2001 From: Szymon Pobiega Date: Sun, 24 Mar 2024 22:45:35 +0100 Subject: [PATCH 4/4] Add configurable bulk insert timeout --- .../DatabaseConfiguration.cs | 6 +++- .../RavenPersistenceConfiguration.cs | 34 +++++++++++++++++-- .../UnitOfWork/RavenAuditUnitOfWorkFactory.cs | 2 +- .../SharedEmbeddedServer.cs | 2 +- 4 files changed, 39 insertions(+), 5 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/DatabaseConfiguration.cs b/src/ServiceControl.Audit.Persistence.RavenDB/DatabaseConfiguration.cs index 7bb08d6fa8..f1cb948d5e 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/DatabaseConfiguration.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/DatabaseConfiguration.cs @@ -11,7 +11,8 @@ public DatabaseConfiguration(string name, TimeSpan auditRetentionPeriod, int maxBodySizeToStore, int minimumStorageLeftRequiredForIngestion, - ServerConfiguration serverConfiguration) + ServerConfiguration serverConfiguration, + TimeSpan bulkInsertCommitTimeout) { Name = name; ExpirationProcessTimerInSeconds = expirationProcessTimerInSeconds; @@ -19,6 +20,7 @@ public DatabaseConfiguration(string name, AuditRetentionPeriod = auditRetentionPeriod; MaxBodySizeToStore = maxBodySizeToStore; ServerConfiguration = serverConfiguration; + BulkInsertCommitTimeout = bulkInsertCommitTimeout; MinimumStorageLeftRequiredForIngestion = minimumStorageLeftRequiredForIngestion; } @@ -37,5 +39,7 @@ public DatabaseConfiguration(string name, public int MaxBodySizeToStore { get; } public int MinimumStorageLeftRequiredForIngestion { get; internal set; } //Setting for ATT only + + public TimeSpan BulkInsertCommitTimeout { get; } } } diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistenceConfiguration.cs b/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistenceConfiguration.cs index 0f227d5906..6ff26714ff 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistenceConfiguration.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistenceConfiguration.cs @@ -16,6 +16,7 @@ public class RavenPersistenceConfiguration : IPersistenceConfiguration public const string LogPathKey = "LogPath"; public const string RavenDbLogLevelKey = "RavenDBLogLevel"; public const string MinimumStorageLeftRequiredForIngestionKey = "MinimumStorageLeftRequiredForIngestion"; + public const string BulkInsertCommitTimeoutInSecondsKey = "BulkInsertCommitTimeoutInSeconds"; public IEnumerable ConfigurationKeys => new[]{ DatabaseNameKey, @@ -25,7 +26,8 @@ public class RavenPersistenceConfiguration : IPersistenceConfiguration ExpirationProcessTimerInSecondsKey, LogPathKey, RavenDbLogLevelKey, - MinimumStorageLeftRequiredForIngestionKey + MinimumStorageLeftRequiredForIngestionKey, + BulkInsertCommitTimeoutInSecondsKey }; public string Name => "RavenDB"; @@ -101,6 +103,8 @@ internal static DatabaseConfiguration GetDatabaseConfiguration(PersistenceSettin var expirationProcessTimerInSeconds = GetExpirationProcessTimerInSeconds(settings); + var bulkInsertTimeout = TimeSpan.FromSeconds(GetBulkInsertCommitTimeout(settings)); + return new DatabaseConfiguration( databaseName, expirationProcessTimerInSeconds, @@ -108,7 +112,8 @@ internal static DatabaseConfiguration GetDatabaseConfiguration(PersistenceSettin settings.AuditRetentionPeriod, settings.MaxBodySizeToStore, minimumStorageLeftRequiredForIngestion, - serverConfiguration); + serverConfiguration, + bulkInsertTimeout); } static int GetExpirationProcessTimerInSeconds(PersistenceSettings settings) @@ -135,6 +140,30 @@ static int GetExpirationProcessTimerInSeconds(PersistenceSettings settings) return expirationProcessTimerInSeconds; } + static int GetBulkInsertCommitTimeout(PersistenceSettings settings) + { + var bulkInsertCommitTimeoutInSeconds = BulkInsertCommitTimeoutInSecondsDefault; + + if (settings.PersisterSpecificSettings.TryGetValue(BulkInsertCommitTimeoutInSecondsKey, out var bulkInsertCommitTimeoutString)) + { + bulkInsertCommitTimeoutInSeconds = int.Parse(bulkInsertCommitTimeoutString); + } + + if (bulkInsertCommitTimeoutInSeconds < 0) + { + Logger.Error($"BulkInsertCommitTimeout cannot be negative. Defaulting to {BulkInsertCommitTimeoutInSecondsDefault}"); + return BulkInsertCommitTimeoutInSecondsDefault; + } + + if (bulkInsertCommitTimeoutInSeconds > TimeSpan.FromHours(1).TotalSeconds) + { + Logger.Error($"BulkInsertCommitTimeout cannot be larger than {TimeSpan.FromHours(1).TotalSeconds}. Defaulting to {BulkInsertCommitTimeoutInSecondsDefault}"); + return BulkInsertCommitTimeoutInSecondsDefault; + } + + return bulkInsertCommitTimeoutInSeconds; + } + static string GetLogPath(PersistenceSettings settings) { if (!settings.PersisterSpecificSettings.TryGetValue(LogPathKey, out var logPath)) @@ -151,5 +180,6 @@ static string GetLogPath(PersistenceSettings settings) static readonly ILog Logger = LogManager.GetLogger(typeof(RavenPersistenceConfiguration)); const int ExpirationProcessTimerInSecondsDefault = 600; + const int BulkInsertCommitTimeoutInSecondsDefault = 60; } } \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/UnitOfWork/RavenAuditUnitOfWorkFactory.cs b/src/ServiceControl.Audit.Persistence.RavenDB/UnitOfWork/RavenAuditUnitOfWorkFactory.cs index 751fdba882..b404ac935d 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/UnitOfWork/RavenAuditUnitOfWorkFactory.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/UnitOfWork/RavenAuditUnitOfWorkFactory.cs @@ -20,7 +20,7 @@ public RavenAuditIngestionUnitOfWorkFactory(IRavenDocumentStoreProvider document public IAuditIngestionUnitOfWork StartNew(int batchSize) { - var timedCancellationSource = new CancellationTokenSource(TimeSpan.FromMinutes(1)); + var timedCancellationSource = new CancellationTokenSource(databaseConfiguration.BulkInsertCommitTimeout); var bulkInsert = documentStoreProvider.GetDocumentStore() .BulkInsert(new BulkInsertOptions { SkipOverwriteIfUnchanged = true, }, timedCancellationSource.Token); diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/SharedEmbeddedServer.cs b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/SharedEmbeddedServer.cs index 97a87dbae9..8463147951 100644 --- a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/SharedEmbeddedServer.cs +++ b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/SharedEmbeddedServer.cs @@ -33,7 +33,7 @@ public static async Task GetInstance(CancellationToken cancell var logsMode = "Operations"; var serverUrl = $"http://localhost:{PortUtility.FindAvailablePort(33334)}"; - embeddedDatabase = EmbeddedDatabase.Start(new DatabaseConfiguration("audit", 60, true, TimeSpan.FromMinutes(5), 120000, 5, new ServerConfiguration(dbPath, serverUrl, logPath, logsMode))); + embeddedDatabase = EmbeddedDatabase.Start(new DatabaseConfiguration("audit", 60, true, TimeSpan.FromMinutes(5), 120000, 5, new ServerConfiguration(dbPath, serverUrl, logPath, logsMode), TimeSpan.FromSeconds(60))); //make sure that the database is up using var documentStore = await embeddedDatabase.Connect(cancellationToken);