diff --git a/src/Persisters.Primary.Includes.props b/src/Persisters.Primary.Includes.props index 8b2eb0241d..ddc16fb034 100644 --- a/src/Persisters.Primary.Includes.props +++ b/src/Persisters.Primary.Includes.props @@ -1,5 +1,6 @@ + \ No newline at end of file diff --git a/src/ServiceControl.Audit/Persistence/PersistenceHostBuilderExtensions.cs b/src/ServiceControl.Audit/Persistence/PersistenceHostBuilderExtensions.cs index ca546ea612..a9e468fcac 100644 --- a/src/ServiceControl.Audit/Persistence/PersistenceHostBuilderExtensions.cs +++ b/src/ServiceControl.Audit/Persistence/PersistenceHostBuilderExtensions.cs @@ -15,7 +15,8 @@ public static IHostBuilder SetupPersistence(this IHostBuilder hostBuilder, { var lifecycle = persistence.Configure(serviceCollection); - serviceCollection.AddHostedService(_ => new PersistenceLifecycleHostedService(lifecycle)); + serviceCollection.AddSingleton(new PersistenceLifecycleHostedService(lifecycle)); + serviceCollection.AddHostedService(sp => sp.GetRequiredService()); }); return hostBuilder; diff --git a/src/ServiceControl.Persistence.RavenDb/CustomChecks/CheckRavenDBIndexErrors.cs b/src/ServiceControl.Persistence.RavenDb/CustomChecks/CheckRavenDBIndexErrors.cs index 27f038687d..c076725f57 100644 --- a/src/ServiceControl.Persistence.RavenDb/CustomChecks/CheckRavenDBIndexErrors.cs +++ b/src/ServiceControl.Persistence.RavenDb/CustomChecks/CheckRavenDBIndexErrors.cs @@ -41,8 +41,8 @@ public override Task PerformCheck() return CheckResult.Failed(message); } - static ILog Logger = LogManager.GetLogger(); + static readonly ILog Logger = LogManager.GetLogger(); - IDocumentStore store; + readonly IDocumentStore store; } } diff --git a/src/ServiceControl.Persistence.RavenDb/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb/ErrorMessagesDataStore.cs index 2de540abd0..261413ed30 100644 --- a/src/ServiceControl.Persistence.RavenDb/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb/ErrorMessagesDataStore.cs @@ -136,28 +136,6 @@ SortInfo sortInfo } } - public async Task>> GetAllMessagesForEndpoint( - string searchTerms, - string receivingEndpointName, - PagingInfo pagingInfo, - SortInfo sortInfo - ) - { - using (var session = documentStore.OpenAsyncSession()) - { - var results = await session.Query() - .Statistics(out var stats) - .Search(x => x.Query, searchTerms) - .Where(m => m.ReceivingEndpointName == receivingEndpointName) - .Sort(sortInfo) - .Paging(pagingInfo) - .TransformWith() - .ToListAsync(); - - return new QueryResult>(results, stats.ToQueryStatsInfo()); - } - } - public async Task FailedMessageFetch(string failedMessageId) { using (var session = documentStore.OpenAsyncSession()) diff --git a/src/ServiceControl.Persistence.RavenDb/EventLogDataStore.cs b/src/ServiceControl.Persistence.RavenDb/EventLogDataStore.cs index 6fbafb64f6..6299ffda67 100644 --- a/src/ServiceControl.Persistence.RavenDb/EventLogDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb/EventLogDataStore.cs @@ -29,7 +29,9 @@ public async Task Add(EventLogItem logItem) { using (var session = documentStore.OpenAsyncSession()) { - var results = await session.Query().Statistics(out var stats) + var results = await session + .Query() + .Statistics(out var stats) .OrderByDescending(p => p.RaisedAt) .Paging(pagingInfo) .ToListAsync(); diff --git a/src/ServiceControl.Persistence.RavenDb/RavenDbCustomCheckDataStore.cs b/src/ServiceControl.Persistence.RavenDb/RavenDbCustomCheckDataStore.cs index 483e3b3f01..7cb7c02612 100644 --- a/src/ServiceControl.Persistence.RavenDb/RavenDbCustomCheckDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb/RavenDbCustomCheckDataStore.cs @@ -34,7 +34,7 @@ public async Task UpdateCustomCheckStatus(CustomCheckDetail de { customCheck = new CustomCheck { - Id = id + Id = MakeId(id) }; } @@ -54,6 +54,11 @@ public async Task UpdateCustomCheckStatus(CustomCheckDetail de return status; } + static string MakeId(Guid id) + { + return $"CustomChecks/{id}"; + } + public async Task>> GetStats(PagingInfo paging, string status = null) { using (var session = store.OpenAsyncSession()) @@ -73,7 +78,7 @@ public async Task>> GetStats(PagingInfo paging, s public async Task DeleteCustomCheck(Guid id) { - await store.AsyncDatabaseCommands.DeleteAsync(store.Conventions.DefaultFindFullDocumentKeyFromNonStringIdentifier(id, typeof(CustomCheck), false), null); + await store.AsyncDatabaseCommands.DeleteAsync(MakeId(id), null); } public async Task GetNumberOfFailedChecks() diff --git a/src/ServiceControl.Persistence.RavenDb/RavenDbPersistence.cs b/src/ServiceControl.Persistence.RavenDb/RavenDbPersistence.cs index bab32ac5bd..c80cb622a5 100644 --- a/src/ServiceControl.Persistence.RavenDb/RavenDbPersistence.cs +++ b/src/ServiceControl.Persistence.RavenDb/RavenDbPersistence.cs @@ -78,10 +78,7 @@ public void Configure(IServiceCollection serviceCollection) serviceCollection.AddSingleton(); } - public IPersistenceLifecycle CreateLifecycle() - { - return new RavenDbPersistenceLifecycle(ravenStartup, documentStore); - } + public void ConfigureLifecycle(IServiceCollection serviceCollection) => serviceCollection.AddSingleton(new RavenDbPersistenceLifecycle(ravenStartup, documentStore)); public IPersistenceInstaller CreateInstaller() { diff --git a/src/ServiceControl.Persistence.RavenDb5/.editorconfig b/src/ServiceControl.Persistence.RavenDb5/.editorconfig new file mode 100644 index 0000000000..ff993b49bb --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/.editorconfig @@ -0,0 +1,4 @@ +[*.cs] + +# Justification: ServiceControl app has no synchronization context +dotnet_diagnostic.CA2007.severity = none diff --git a/src/ServiceControl.Persistence.RavenDb5/Chunker.cs b/src/ServiceControl.Persistence.RavenDb5/Chunker.cs new file mode 100644 index 0000000000..bae4cb0da8 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Chunker.cs @@ -0,0 +1,40 @@ +namespace ServiceControl.Infrastructure.RavenDB +{ + using System; + using System.Threading; + + static class Chunker + { + public static int ExecuteInChunks(int total, Func action, T1 t1, T2 t2, CancellationToken cancellationToken = default) + { + if (total == 0) + { + return 0; + } + + if (total < CHUNK_SIZE) + { + return action(t1, t2, 0, total - 1); + } + + int start = 0, end = CHUNK_SIZE - 1; + var chunkCount = 0; + do + { + chunkCount += action(t1, t2, start, end); + + start = end + 1; + end += CHUNK_SIZE; + if (end >= total) + { + end = total - 1; + } + } + while (start < total && !cancellationToken.IsCancellationRequested); + + return chunkCount; + } + + const int CHUNK_SIZE = 500; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckFreeDiskSpace.cs b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckFreeDiskSpace.cs new file mode 100644 index 0000000000..14fb9ae8d9 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckFreeDiskSpace.cs @@ -0,0 +1,72 @@ +namespace ServiceControl.Operations +{ + using System; + using System.IO; + using System.Threading.Tasks; + using NServiceBus.CustomChecks; + using NServiceBus.Logging; + using Persistence.RavenDb; + + class CheckFreeDiskSpace : CustomCheck + { + public CheckFreeDiskSpace(RavenDBPersisterSettings settings) : base("ServiceControl database", "Storage space", TimeSpan.FromMinutes(5)) + { + dataPath = settings.DatabasePath; + percentageThreshold = settings.DataSpaceRemainingThreshold; + + Logger.Debug($"Check ServiceControl data drive space remaining custom check starting. Threshold {percentageThreshold:P0}"); + } + + public override Task PerformCheck() + { + var dataPathRoot = Path.GetPathRoot(dataPath); + + if (dataPathRoot == null) + { + throw new Exception($"Unable to find the root of the data path {dataPath}"); + } + + var dataDriveInfo = new DriveInfo(dataPathRoot); + var availableFreeSpace = (decimal)dataDriveInfo.AvailableFreeSpace; + var totalSpace = (decimal)dataDriveInfo.TotalSize; + + var percentRemaining = (decimal)dataDriveInfo.AvailableFreeSpace / dataDriveInfo.TotalSize; + + if (Logger.IsDebugEnabled) + { + Logger.Debug($"Free space: {availableFreeSpace:N0}B | Total: {totalSpace:N0}B | Percent remaining {percentRemaining:P1}"); + } + + return percentRemaining > percentageThreshold + ? CheckResult.Pass + : CheckResult.Failed($"{percentRemaining:P0} disk space remaining on data drive '{dataDriveInfo.VolumeLabel} ({dataDriveInfo.RootDirectory})' on '{Environment.MachineName}'."); + } + + public static void Validate(RavenDBPersisterSettings settings) + { + var threshold = settings.DataSpaceRemainingThreshold; + + string message; + + if (threshold < 0) + { + message = $"{RavenDbPersistenceConfiguration.DataSpaceRemainingThresholdKey} is invalid, minimum value is 0."; + Logger.Fatal(message); + throw new Exception(message); + } + + if (threshold > 100) + { + message = $"{RavenDbPersistenceConfiguration.DataSpaceRemainingThresholdKey} is invalid, maximum value is 100."; + Logger.Fatal(message); + throw new Exception(message); + } + } + + readonly string dataPath; + readonly decimal percentageThreshold; + + public const int DataSpaceRemainingThresholdDefault = 20; + static readonly ILog Logger = LogManager.GetLogger(typeof(CheckFreeDiskSpace)); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckMinimumStorageRequiredForIngestion.cs b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckMinimumStorageRequiredForIngestion.cs new file mode 100644 index 0000000000..a696a1d45d --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckMinimumStorageRequiredForIngestion.cs @@ -0,0 +1,90 @@ +namespace ServiceControl.Operations +{ + using System; + using System.IO; + using System.Threading.Tasks; + using NServiceBus.CustomChecks; + using NServiceBus.Logging; + using Persistence.RavenDb; + using ServiceControl.Persistence; + + class CheckMinimumStorageRequiredForIngestion : CustomCheck + { + public CheckMinimumStorageRequiredForIngestion( + MinimumRequiredStorageState stateHolder, + RavenDBPersisterSettings settings) + : base("Message Ingestion Process", "ServiceControl Health", TimeSpan.FromSeconds(5)) + { + this.stateHolder = stateHolder; + this.settings = settings; + + dataPathRoot = Path.GetPathRoot(settings.DatabasePath); + } + + public override Task PerformCheck() + { + percentageThreshold = settings.MinimumStorageLeftRequiredForIngestion / 100m; + + if (dataPathRoot == null) + { + stateHolder.CanIngestMore = true; + return SuccessResult; + } + + Logger.Debug($"Check ServiceControl data drive space starting. Threshold {percentageThreshold:P0}"); + + var dataDriveInfo = new DriveInfo(dataPathRoot); + var availableFreeSpace = (decimal)dataDriveInfo.AvailableFreeSpace; + var totalSpace = (decimal)dataDriveInfo.TotalSize; + + var percentRemaining = (decimal)dataDriveInfo.AvailableFreeSpace / dataDriveInfo.TotalSize; + + if (Logger.IsDebugEnabled) + { + Logger.Debug($"Free space: {availableFreeSpace:N0}B | Total: {totalSpace:N0}B | Percent remaining {percentRemaining:P1}"); + } + + if (percentRemaining > percentageThreshold) + { + stateHolder.CanIngestMore = true; + return SuccessResult; + } + + var message = $"Error message ingestion stopped! {percentRemaining:P0} disk space remaining on data drive '{dataDriveInfo.VolumeLabel} ({dataDriveInfo.RootDirectory})' on '{Environment.MachineName}'. This is less than {percentageThreshold}% - the minimal required space configured. The threshold can be set using the {RavenBootstrapper.MinimumStorageLeftRequiredForIngestionKey} configuration setting."; + Logger.Warn(message); + stateHolder.CanIngestMore = false; + return CheckResult.Failed(message); + } + + public static void Validate(RavenDBPersisterSettings settings) + { + int threshold = settings.MinimumStorageLeftRequiredForIngestion; + + string message; + if (threshold < 0) + { + message = $"{RavenBootstrapper.MinimumStorageLeftRequiredForIngestionKey} is invalid, minimum value is 0."; + Logger.Fatal(message); + throw new Exception(message); + } + + if (threshold > 100) + { + message = $"{RavenBootstrapper.MinimumStorageLeftRequiredForIngestionKey} is invalid, maximum value is 100."; + Logger.Fatal(message); + throw new Exception(message); + } + } + + public const int MinimumStorageLeftRequiredForIngestionDefault = 5; + + readonly MinimumRequiredStorageState stateHolder; + readonly RavenDBPersisterSettings settings; + readonly string dataPathRoot; + + decimal percentageThreshold; + + static readonly Task SuccessResult = Task.FromResult(CheckResult.Pass); + static readonly ILog Logger = LogManager.GetLogger(typeof(CheckMinimumStorageRequiredForIngestion)); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexErrors.cs b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexErrors.cs new file mode 100644 index 0000000000..83156e61d8 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexErrors.cs @@ -0,0 +1,50 @@ +namespace ServiceControl +{ + using System; + using System.Text; + using System.Threading.Tasks; + using NServiceBus.CustomChecks; + using NServiceBus.Logging; + using Raven.Client.Documents; + using Raven.Client.Documents.Operations.Indexes; + + class CheckRavenDBIndexErrors : CustomCheck + { + public CheckRavenDBIndexErrors(IDocumentStore store) + : base("Error Database Index Errors", "ServiceControl Health", TimeSpan.FromMinutes(5)) + { + this.store = store; + } + + public override Task PerformCheck() + { + var indexErrors = store.Maintenance.Send(new GetIndexErrorsOperation()); + + if (indexErrors.Length == 0) + { + return CheckResult.Pass; + } + + var text = new StringBuilder(); + text.AppendLine("Detected RavenDB index errors, please start maintenance mode and resolve the following issues:"); + + foreach (var indexError in indexErrors) + { + foreach (var indexingError in indexError.Errors) + { + text.AppendLine($"- Index [{indexError.Name}] error: {indexError.Name} (Action: {indexingError.Action}, Doc: {indexingError.Document}, At: {indexingError.Timestamp})"); + } + } + + text.AppendLine().AppendLine("See: https://docs.particular.net/search?q=servicecontrol+troubleshooting"); + + var message = text.ToString(); + Logger.Error(message); + return CheckResult.Failed(message); + } + + static readonly ILog Logger = LogManager.GetLogger(); + + readonly IDocumentStore store; + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexLag.cs b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexLag.cs new file mode 100644 index 0000000000..a6bc6acc4b --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexLag.cs @@ -0,0 +1,91 @@ +namespace ServiceControl +{ + using System; + using System.Linq; + using System.Text; + using System.Threading.Tasks; + using NServiceBus.CustomChecks; + using NServiceBus.Logging; + using Raven.Client.Documents; + using Raven.Client.Documents.Operations; + using CustomCheck = NServiceBus.CustomChecks.CustomCheck; + + class CheckRavenDBIndexLag : CustomCheck + { + public CheckRavenDBIndexLag(IDocumentStore store) + : base("Error Database Index Lag", "ServiceControl Health", TimeSpan.FromMinutes(5)) + { + this.store = store; + } + + public override async Task PerformCheck() + { + var statistics = await store.Maintenance.SendAsync(new GetStatisticsOperation()); + var indexes = statistics.Indexes.OrderBy(x => x.Name).ToArray(); + + CreateDiagnosticsLogEntry(statistics, indexes); + + var indexCountWithTooMuchLag = CheckAndReportIndexesWithTooMuchIndexLag(indexes); + + if (indexCountWithTooMuchLag > 0) + { + return CheckResult.Failed($"At least one index significantly stale. Please run maintenance mode if this custom check persists to ensure index(es) can recover. See log file for more details. Visit https://docs.particular.net/search?q=servicecontrol+troubleshooting for more information."); + } + + return CheckResult.Pass; + } + + static int CheckAndReportIndexesWithTooMuchIndexLag(IndexInformation[] indexes) + { + int indexCountWithTooMuchLag = 0; + + foreach (var indexStats in indexes) + { + if (indexStats.LastIndexingTime.HasValue) + { + var indexLag = DateTime.UtcNow - indexStats.LastIndexingTime.Value; // TODO: Ensure audit ravendb5 persistence uses the same index lag behavior based on time + + if (indexLag > IndexLagThresholdError) + { + indexCountWithTooMuchLag++; + Log.Error($"Index [{indexStats.Name}] IndexingLag {indexLag} is above error threshold ({IndexLagThresholdError}). Launch in maintenance mode to let indexes catch up."); + } + else if (indexLag > IndexLagThresholdWarning) + { + indexCountWithTooMuchLag++; + Log.Warn($"Index [{indexStats.Name}] IndexingLag {indexLag} is above warning threshold ({IndexLagThresholdWarning}). Launch in maintenance mode to let indexes catch up."); + } + } + } + + return indexCountWithTooMuchLag; + } + + static void CreateDiagnosticsLogEntry(DatabaseStatistics statistics, IndexInformation[] indexes) + { + if (!Log.IsDebugEnabled) + { + return; + } + + var report = new StringBuilder(); + report.AppendLine("Internal RavenDB index health report:"); + report.AppendLine($"- DB Size: {statistics.SizeOnDisk.HumaneSize}"); + report.AppendLine($"- LastIndexingTime {statistics.LastIndexingTime:u}"); + + foreach (var indexStats in indexes) + { + report.AppendLine($"- Index [{indexStats.Name,-44}] State: {indexStats.State}, Stale: {indexStats.IsStale,-5}, Priority: {indexStats.Priority,-6}, LastIndexingTime: {indexStats.LastIndexingTime:u}"); + } + Log.Debug(report.ToString()); + } + + // TODO: RavenDB 3.5 had IndexLag thresholds that were number of document writes, and I converted to times. Revisit these numbers before shipping + // For IndexLag as document writes, 10k was a warning, 100k was an error. These TimeSpans assume same # of writes / 250 writes/sec + static readonly TimeSpan IndexLagThresholdWarning = TimeSpan.FromSeconds(40); // Assuming 10_000 writes at 250 writes/sec + static readonly TimeSpan IndexLagThresholdError = TimeSpan.FromSeconds(400); // Assuming 100_000 writes at 250 writes/sec + static readonly ILog Log = LogManager.GetLogger(); + + readonly IDocumentStore store; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/DatabaseSetup.cs b/src/ServiceControl.Persistence.RavenDb5/DatabaseSetup.cs new file mode 100644 index 0000000000..0b53d5782c --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/DatabaseSetup.cs @@ -0,0 +1,88 @@ +namespace ServiceControl.Persistence.RavenDb5 +{ + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using Raven.Client.Documents; + using Raven.Client.Documents.Indexes; + using Raven.Client.Documents.Operations; + using Raven.Client.Documents.Operations.Expiration; + using Raven.Client.Exceptions; + using Raven.Client.Exceptions.Database; + using Raven.Client.ServerWide; + using Raven.Client.ServerWide.Operations; + using ServiceControl.MessageFailures.Api; + using ServiceControl.Operations; + using ServiceControl.Persistence; + using ServiceControl.Recoverability; + + class DatabaseSetup + { + public DatabaseSetup(RavenDBPersisterSettings settings) + { + this.settings = settings; + } + + public async Task Execute(IDocumentStore documentStore, CancellationToken cancellationToken) + { + try + { + await documentStore.Maintenance.ForDatabase(settings.DatabaseName).SendAsync(new GetStatisticsOperation(), cancellationToken); + } + catch (DatabaseDoesNotExistException) + { + try + { + await documentStore.Maintenance.Server + .SendAsync(new CreateDatabaseOperation(new DatabaseRecord(settings.DatabaseName)), cancellationToken); + } + catch (ConcurrencyException) + { + // The database was already created before calling CreateDatabaseOperation + } + } + + var indexList = new List { + new ArchivedGroupsViewIndex(), + new CustomChecksIndex(), + new FailedErrorImportIndex(), + new FailedMessageFacetsIndex(), + new FailedMessageRetries_ByBatch(), + new FailedMessageViewIndex(), + new FailureGroupsViewIndex(), + new GroupCommentIndex(), + new KnownEndpointIndex(), + new MessagesViewIndex(), + new QueueAddressIndex(), + new RetryBatches_ByStatusAndSession(), + new RetryBatches_ByStatus_ReduceInitialBatchSize() + + }; + + //TODO: Handle full text search + //if (settings.EnableFullTextSearch) + //{ + // indexList.Add(new MessagesViewIndexWithFullTextSearch()); + // await documentStore.Maintenance.SendAsync(new DeleteIndexOperation("MessagesViewIndex"), cancellationToken); + //} + //else + //{ + // indexList.Add(new MessagesViewIndex()); + // await documentStore.Maintenance + // .SendAsync(new DeleteIndexOperation("MessagesViewIndexWithFullTextSearch"), cancellationToken); + //} + + await IndexCreation.CreateIndexesAsync(indexList, documentStore, null, null, cancellationToken); + + var expirationConfig = new ExpirationConfiguration + { + Disabled = false, + DeleteFrequencyInSec = settings.ExpirationProcessTimerInSeconds + }; + + await documentStore.Maintenance.SendAsync(new ConfigureExpirationOperation(expirationConfig), cancellationToken); + } + + readonly RavenDBPersisterSettings settings; + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/FailedMessageIdGenerator.cs b/src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/FailedMessageIdGenerator.cs new file mode 100644 index 0000000000..4935378ec9 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/FailedMessageIdGenerator.cs @@ -0,0 +1,11 @@ +static class FailedMessageIdGenerator +{ + public const string CollectionName = "FailedMessages"; + + public static string MakeDocumentId(string messageUniqueId) + { + return $"{CollectionName}/{messageUniqueId}"; + } + + public static string GetMessageIdFromDocumentId(string failedMessageDocumentId) => failedMessageDocumentId.Substring(CollectionName.Length + 1); +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/KnownEndpointIdGenerator.cs b/src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/KnownEndpointIdGenerator.cs new file mode 100644 index 0000000000..ea368580af --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/KnownEndpointIdGenerator.cs @@ -0,0 +1,7 @@ +using System; + +static class KnownEndpointIdGenerator +{ + const string CollectionName = "KnownEndpoint"; + public static string MakeDocumentId(Guid endpointId) => $"{CollectionName}/{endpointId}"; +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/MessageBodyIdGenerator.cs b/src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/MessageBodyIdGenerator.cs new file mode 100644 index 0000000000..9891ffdc14 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/MessageBodyIdGenerator.cs @@ -0,0 +1,9 @@ +static class MessageBodyIdGenerator +{ + const string CollectionName = "messagebodies"; + + public static string MakeDocumentId(string messageUniqueId) + { + return $"{CollectionName}/{messageUniqueId}"; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Editing/EditFailedMessageManager.cs b/src/ServiceControl.Persistence.RavenDb5/Editing/EditFailedMessageManager.cs new file mode 100644 index 0000000000..1f87798ea0 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Editing/EditFailedMessageManager.cs @@ -0,0 +1,53 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System; + using System.Threading.Tasks; + using ServiceControl.MessageFailures; + using ServiceControl.Persistence.Recoverability.Editing; + using Raven.Client.Documents.Session; + + class EditFailedMessageManager : AbstractSessionManager, IEditFailedMessagesManager + { + readonly IAsyncDocumentSession session; + FailedMessage failedMessage; + + public EditFailedMessageManager(IAsyncDocumentSession session) + : base(session) + { + this.session = session; + } + + public async Task GetFailedMessage(string failedMessageId) + { + failedMessage = await session.LoadAsync(FailedMessageIdGenerator.MakeDocumentId(failedMessageId)); + return failedMessage; + } + + public async Task GetCurrentEditingMessageId(string failedMessageId) + { + var edit = await session.LoadAsync(FailedMessageEdit.MakeDocumentId(failedMessageId)); + return edit?.EditId; + } + + public Task SetCurrentEditingMessageId(string editingMessageId) + { + if (failedMessage == null) + { + throw new InvalidOperationException("No failed message loaded"); + } + return session.StoreAsync(new FailedMessageEdit + { + Id = FailedMessageEdit.MakeDocumentId(failedMessage.UniqueMessageId), + FailedMessageId = failedMessage.Id, + EditId = editingMessageId + }); + } + + public Task SetFailedMessageAsResolved() + { + // Instance is tracked by the document session + failedMessage.Status = FailedMessageStatus.Resolved; + return Task.CompletedTask; + } + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/Editing/FailedMessageEdit.cs b/src/ServiceControl.Persistence.RavenDb5/Editing/FailedMessageEdit.cs new file mode 100644 index 0000000000..2bc0e7a1e0 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Editing/FailedMessageEdit.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Persistence.Recoverability.Editing +{ + class FailedMessageEdit + { + public string Id { get; set; } + public string FailedMessageId { get; set; } + public string EditId { get; set; } + + public static string MakeDocumentId(string failedMessageId) + { + return $"{CollectionName}/{failedMessageId}"; + } + + const string CollectionName = "FailedMessageEdit"; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Editing/NotificationsManager.cs b/src/ServiceControl.Persistence.RavenDb5/Editing/NotificationsManager.cs new file mode 100644 index 0000000000..8857860ad9 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Editing/NotificationsManager.cs @@ -0,0 +1,38 @@ +namespace ServiceControl.Persistence.RavenDb.Editing +{ + using System; + using System.Threading.Tasks; + using Notifications; + using Raven.Client.Documents.Session; + + class NotificationsManager : AbstractSessionManager, INotificationsManager + { + static readonly TimeSpan CacheTimeoutDefault = TimeSpan.FromMinutes(5); // Raven requires this to be at least 1 second + + public NotificationsManager(IAsyncDocumentSession session) : base(session) + { + } + + public async Task LoadSettings(TimeSpan? cacheTimeout = null) + { + + using (Session.Advanced.DocumentStore.AggressivelyCacheFor(cacheTimeout ?? CacheTimeoutDefault)) + { + var settings = await Session + .LoadAsync(NotificationsSettings.SingleDocumentId); + + if (settings == null) + { + settings = new NotificationsSettings + { + Id = NotificationsSettings.SingleDocumentId + }; + + await Session.StoreAsync(settings); + } + + return settings; + } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/EmbeddedDatabase.cs b/src/ServiceControl.Persistence.RavenDb5/EmbeddedDatabase.cs new file mode 100644 index 0000000000..536d5247fc --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/EmbeddedDatabase.cs @@ -0,0 +1,235 @@ +namespace ServiceControl.Persistence.RavenDb5 +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Globalization; + using System.IO; + using System.Threading; + using System.Threading.Tasks; + using ByteSizeLib; + using NServiceBus.Logging; + using Raven.Client.Documents; + using Raven.Client.Documents.Conventions; + using Raven.Embedded; + + public class EmbeddedDatabase : IDisposable + { + EmbeddedDatabase(RavenDBPersisterSettings configuration) + { + this.configuration = configuration; + ServerUrl = configuration.ServerUrl; + } + + public string ServerUrl { get; private set; } + + static (string LicenseFileName, string ServerDirectory) GetRavenLicenseFileNameAndServerDirectory() + { + var licenseFileName = "RavenLicense.json"; + var localRavenLicense = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, licenseFileName); + if (File.Exists(localRavenLicense)) + { + return (localRavenLicense, null); + } + + //TODO: refactor this to extract the folder name to a constant + localRavenLicense = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Persisters", "RavenDB5", licenseFileName); + if (!File.Exists(localRavenLicense)) + { + throw new Exception($"RavenDB license not found. Make sure the RavenDB license file, '{licenseFileName}', " + + $"is stored in the '{AppDomain.CurrentDomain.BaseDirectory}' folder or in the 'Persisters/RavenDB5' subfolder."); + } + + // By default RavenDB 5 searches its binaries in the RavenDBServer right below the BaseDirectory. + // If we're loading from Persisters/RavenDB5 we also have to signal RavenDB where are binaries + var serverDirectory = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Persisters", "RavenDB5", "RavenDBServer"); + + return (localRavenLicense, serverDirectory); + } + + internal static EmbeddedDatabase Start(RavenDBPersisterSettings settings) + { + var licenseFileNameAndServerDirectory = GetRavenLicenseFileNameAndServerDirectory(); + + var nugetPackagesPath = Path.Combine(settings.DatabasePath, "Packages", "NuGet"); + + logger.InfoFormat("Loading RavenDB license from {0}", licenseFileNameAndServerDirectory.LicenseFileName); + var serverOptions = new ServerOptions + { + CommandLineArgs = new List + { + $"--License.Path=\"{licenseFileNameAndServerDirectory.LicenseFileName}\"", + $"--Logs.Mode={settings.LogsMode}", + // HINT: If this is not set, then Raven will pick a default location relative to the server binaries + // See https://github.com/ravendb/ravendb/issues/15694 + $"--Indexing.NuGetPackagesPath=\"{nugetPackagesPath}\"" + }, + AcceptEula = true, + DataDirectory = settings.DatabasePath, + ServerUrl = settings.ServerUrl, + LogsPath = settings.LogPath + }; + + if (!string.IsNullOrWhiteSpace(licenseFileNameAndServerDirectory.ServerDirectory)) + { + serverOptions.ServerDirectory = licenseFileNameAndServerDirectory.ServerDirectory; + } + + var embeddedDatabase = new EmbeddedDatabase(settings); + + embeddedDatabase.Start(serverOptions); + + RecordStartup(settings); + + return embeddedDatabase; + } + + void Start(ServerOptions serverOptions) + { + EmbeddedServer.Instance.ServerProcessExited += (sender, args) => + { + if (sender is Process process && process.HasExited && process.ExitCode != 0) + { + logger.Warn($"RavenDB server process exited unexpectedly with exitCode: {process.ExitCode}. Process will be restarted."); + + restartRequired = true; + } + }; + + EmbeddedServer.Instance.StartServer(serverOptions); + + var _ = Task.Run(async () => + { + while (!shutdownTokenSource.IsCancellationRequested) + { + try + { + await Task.Delay(delayBetweenRestarts, shutdownTokenSource.Token); + + if (restartRequired) + { + logger.Info("Restarting RavenDB server process"); + + await EmbeddedServer.Instance.RestartServerAsync(); + restartRequired = false; + + logger.Info("RavenDB server process restarted successfully."); + } + } + catch (OperationCanceledException) + { + //no-op + } + catch (Exception e) + { + logger.Fatal($"RavenDB server restart failed. Restart will be retried in {delayBetweenRestarts}.", e); + } + } + }); + } + + public async Task Connect(CancellationToken cancellationToken) + { + var dbOptions = new DatabaseOptions(configuration.DatabaseName) + { + Conventions = new DocumentConventions + { + SaveEnumsAsIntegers = true + } + }; + + //TODO: copied from Audit. In Audit FindClrType so I guess this is not needed. Confirm and remove + //if (configuration.FindClrType != null) + //{ + // dbOptions.Conventions.FindClrType += configuration.FindClrType; + //} + + var store = await EmbeddedServer.Instance.GetDocumentStoreAsync(dbOptions, cancellationToken); + + var databaseSetup = new DatabaseSetup(configuration); + await databaseSetup.Execute(store, cancellationToken); + + return store; + } + + public void Dispose() + { + shutdownTokenSource.Cancel(); + EmbeddedServer.Instance?.Dispose(); + } + + static void RecordStartup(RavenDBPersisterSettings settings) + { + var dataSize = DataSize(settings); + var folderSize = FolderSize(settings); + + var startupMessage = $@" +------------------------------------------------------------- +Database Size: {ByteSize.FromBytes(dataSize).ToString("#.##", CultureInfo.InvariantCulture)} +Database Folder Size: {ByteSize.FromBytes(folderSize).ToString("#.##", CultureInfo.InvariantCulture)} +-------------------------------------------------------------"; + + logger.Info(startupMessage); + } + + static long DataSize(RavenDBPersisterSettings settings) + { + var datafilePath = Path.Combine(settings.DatabasePath, "data"); + + try + { + var info = new FileInfo(datafilePath); + if (!info.Exists) + { + return -1; + } + return info.Length; + } + catch + { + return -1; + } + } + + static long FolderSize(RavenDBPersisterSettings settings) + { + try + { + var dir = new DirectoryInfo(settings.DatabasePath); + var dirSize = DirSize(dir); + return dirSize; + } + catch + { + return -1; + } + } + + static long DirSize(DirectoryInfo d) + { + long size = 0; + if (d.Exists) + { + FileInfo[] fis = d.GetFiles(); + foreach (FileInfo fi in fis) + { + size += fi.Length; + } + + DirectoryInfo[] dis = d.GetDirectories(); + foreach (DirectoryInfo di in dis) + { + size += DirSize(di); + } + } + + return size; + } + CancellationTokenSource shutdownTokenSource = new CancellationTokenSource(); + bool restartRequired; + readonly RavenDBPersisterSettings configuration; + + static TimeSpan delayBetweenRestarts = TimeSpan.FromSeconds(60); + static readonly ILog logger = LogManager.GetLogger(); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/EndpointDetailsParser.cs b/src/ServiceControl.Persistence.RavenDb5/EndpointDetailsParser.cs new file mode 100644 index 0000000000..1cda01af30 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/EndpointDetailsParser.cs @@ -0,0 +1,120 @@ +namespace ServiceControl +{ + using System; + using System.Collections.Generic; + using Infrastructure; + using NServiceBus; + using ServiceControl.Operations; + + class EndpointDetailsParser + { + public static EndpointDetails SendingEndpoint(IReadOnlyDictionary headers) + { + var endpointDetails = new EndpointDetails(); + + DictionaryExtensions.CheckIfKeyExists(Headers.OriginatingEndpoint, headers, s => endpointDetails.Name = s); + DictionaryExtensions.CheckIfKeyExists("NServiceBus.OriginatingMachine", headers, s => endpointDetails.Host = s); + DictionaryExtensions.CheckIfKeyExists(Headers.OriginatingHostId, headers, s => endpointDetails.HostId = Guid.Parse(s)); + + if (!string.IsNullOrEmpty(endpointDetails.Name) && !string.IsNullOrEmpty(endpointDetails.Host)) + { + return endpointDetails; + } + + string address = null; + DictionaryExtensions.CheckIfKeyExists(Headers.OriginatingAddress, headers, s => address = s); + + if (address != null) + { + var queueAndMachinename = ExtractQueueAndMachineName(address); + endpointDetails.Name = queueAndMachinename.Queue; + endpointDetails.Host = queueAndMachinename.Machine; + return endpointDetails; + } + + return null; + } + + public static EndpointDetails ReceivingEndpoint(IReadOnlyDictionary headers) + { + var endpoint = new EndpointDetails(); + + if (headers.TryGetValue(Headers.HostId, out var hostIdHeader)) + { + endpoint.HostId = Guid.Parse(hostIdHeader); + } + + if (headers.TryGetValue(Headers.HostDisplayName, out var hostDisplayNameHeader)) + { + endpoint.Host = hostDisplayNameHeader; + } + else + { + DictionaryExtensions.CheckIfKeyExists("NServiceBus.ProcessingMachine", headers, s => endpoint.Host = s); + } + + DictionaryExtensions.CheckIfKeyExists(Headers.ProcessingEndpoint, headers, s => endpoint.Name = s); + + if (!string.IsNullOrEmpty(endpoint.Name) && !string.IsNullOrEmpty(endpoint.Host)) + { + return endpoint; + } + + string address = null; + //use the failed q to determine the receiving endpoint + DictionaryExtensions.CheckIfKeyExists("NServiceBus.FailedQ", headers, s => address = s); + + // If we have a failed queue, then construct an endpoint from the failed queue information + if (address != null) + { + var queueAndMachinename = ExtractQueueAndMachineName(address); + + if (string.IsNullOrEmpty(endpoint.Name)) + { + endpoint.Name = queueAndMachinename.Queue; + } + + if (string.IsNullOrEmpty(endpoint.Host)) + { + endpoint.Host = queueAndMachinename.Machine; + } + + // If we've been now able to get the endpoint details, return the new info. + if (!string.IsNullOrEmpty(endpoint.Name) && !string.IsNullOrEmpty(endpoint.Host)) + { + return endpoint; + } + } + + return null; + } + + static QueueAndMachine ExtractQueueAndMachineName(string address) + { + var atIndex = address?.IndexOf("@", StringComparison.InvariantCulture); + + if (atIndex.HasValue && atIndex.Value > -1) + { + var queue = address.Substring(0, atIndex.Value); + var machine = address.Substring(atIndex.Value + 1); + return new QueueAndMachine + { + Queue = queue, + Machine = machine + }; + } + + return new QueueAndMachine + { + Queue = address, + Machine = null + }; + } + + struct QueueAndMachine + { + public string Queue; + public string Machine; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs new file mode 100644 index 0000000000..19db4f2f6b --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -0,0 +1,748 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading.Tasks; + using Editing; + using NServiceBus.Logging; + using Raven.Client.Documents; + using Raven.Client.Documents.Commands; + using Raven.Client.Documents.Linq; + using Raven.Client.Documents.Operations; + using Raven.Client.Documents.Queries; + using Raven.Client.Documents.Queries.Facets; + using Raven.Client.Documents.Session; + using ServiceControl.CompositeViews.Messages; + using ServiceControl.EventLog; + using ServiceControl.MessageFailures; + using ServiceControl.MessageFailures.Api; + using ServiceControl.Operations; + using ServiceControl.Persistence.Infrastructure; + using ServiceControl.Recoverability; + + class ErrorMessagesDataStore : IErrorMessageDataStore + { + readonly IDocumentStore documentStore; + + public ErrorMessagesDataStore(IDocumentStore documentStore) + { + this.documentStore = documentStore; + + } + + public async Task>> GetAllMessages( + PagingInfo pagingInfo, + SortInfo sortInfo, + bool includeSystemMessages + ) + { + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session.Query() + .IncludeSystemMessagesWhere(includeSystemMessages) + .Statistics(out var stats) + .Sort(sortInfo) + .Paging(pagingInfo) + //.TransformWith() + .TransformToMessagesView() + .ToListAsync(); + + return new QueryResult>(results, stats.ToQueryStatsInfo()); + } + } + + public async Task>> GetAllMessagesForEndpoint( + string endpointName, + PagingInfo pagingInfo, + SortInfo sortInfo, + bool includeSystemMessages + ) + { + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session.Query() + .IncludeSystemMessagesWhere(includeSystemMessages) + .Where(m => m.ReceivingEndpointName == endpointName) + .Statistics(out var stats) + .Sort(sortInfo) + .Paging(pagingInfo) + //.TransformWith() + .TransformToMessagesView() + .ToListAsync(); + + return new QueryResult>(results, stats.ToQueryStatsInfo()); + } + } + + public async Task>> SearchEndpointMessages( + string endpointName, + string searchKeyword, + PagingInfo pagingInfo, + SortInfo sortInfo + ) + { + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session.Query() + .Statistics(out var stats) + .Search(x => x.Query, searchKeyword) + .Where(m => m.ReceivingEndpointName == endpointName) + .Sort(sortInfo) + .Paging(pagingInfo) + //.TransformWith() + .TransformToMessagesView() + .ToListAsync(); + + return new QueryResult>(results, stats.ToQueryStatsInfo()); + } + } + + public async Task>> GetAllMessagesByConversation( + string conversationId, + PagingInfo pagingInfo, + SortInfo sortInfo, + bool includeSystemMessages + ) + { + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session.Query() + .Statistics(out var stats) + .Where(m => m.ConversationId == conversationId) + .Sort(sortInfo) + .Paging(pagingInfo) + //.TransformWith() + .TransformToMessagesView() + .ToListAsync(); + + return new QueryResult>(results, stats.ToQueryStatsInfo()); + } + } + + public async Task>> GetAllMessagesForSearch( + string searchTerms, + PagingInfo pagingInfo, + SortInfo sortInfo + ) + { + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session.Query() + .Statistics(out var stats) + .Search(x => x.Query, searchTerms) + .Sort(sortInfo) + .Paging(pagingInfo) + //.TransformWith() + .TransformToMessagesView() + .ToListAsync(); + + return new QueryResult>(results, stats.ToQueryStatsInfo()); + } + } + + public async Task FailedMessageFetch(string failedMessageId) + { + using (var session = documentStore.OpenAsyncSession()) + { + return await session.LoadAsync(failedMessageId); + } + } + + public async Task FailedMessageMarkAsArchived(string failedMessageId) + { + using (var session = documentStore.OpenAsyncSession()) + { + var failedMessage = await session.LoadAsync(failedMessageId); + + if (failedMessage.Status != FailedMessageStatus.Archived) + { + failedMessage.Status = FailedMessageStatus.Archived; + } + + await session.SaveChangesAsync(); + } + } + + public async Task FailedMessagesFetch(Guid[] ids) + { + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session.LoadAsync(ids.Select(g => g.ToString())); + return results.Values.Where(x => x != null).ToArray(); + } + } + + public async Task StoreFailedErrorImport(FailedErrorImport failure) + { + using (var session = documentStore.OpenAsyncSession()) + { + await session.StoreAsync(failure); + + await session.SaveChangesAsync(); + } + } + + public Task CreateEditFailedMessageManager() + { + var session = documentStore.OpenAsyncSession(); + var manager = new EditFailedMessageManager(session); + return Task.FromResult((IEditFailedMessagesManager)manager); + } + + public async Task> GetFailureGroupView(string groupId, string status, string modified) + { + using (var session = documentStore.OpenAsyncSession()) + { + var document = await session.Advanced + .AsyncDocumentQuery() + .Statistics(out var stats) + .WhereEquals(group => group.Id, groupId) + .FilterByStatusWhere(status) + .FilterByLastModifiedRange(modified) + .FirstOrDefaultAsync(); + + return new QueryResult(document, stats.ToQueryStatsInfo()); + } + } + + public async Task> GetFailureGroupsByClassifier(string classifier) + { + using (var session = documentStore.OpenAsyncSession()) + { + var groups = session + .Query() + .Where(v => v.Type == classifier); + + var results = await groups + .OrderByDescending(x => x.Last) + .Take(200) // only show 200 groups + .ToListAsync(); + + return results; + } + } + + public async Task>> ErrorGet( + string status, + string modified, + string queueAddress, + PagingInfo pagingInfo, + SortInfo sortInfo + ) + { + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session.Advanced + .AsyncDocumentQuery() + .Statistics(out var stats) + .FilterByStatusWhere(status) + .FilterByLastModifiedRange(modified) + .FilterByQueueAddress(queueAddress) + .Sort(sortInfo) + .Paging(pagingInfo) + // TODO: Fix SetResultTransformer + //.SetResultTransformer(new FailedMessageViewTransformer().TransformerName) + .SelectFields() + .ToListAsync(); + + return new QueryResult>(results, stats.ToQueryStatsInfo()); + } + } + + public async Task ErrorsHead( + string status, + string modified, + string queueAddress + ) + { + using (var session = documentStore.OpenAsyncSession()) + { + var stats = await session.Advanced + .AsyncDocumentQuery() + .FilterByStatusWhere(status) + .FilterByLastModifiedRange(modified) + .FilterByQueueAddress(queueAddress) + .GetQueryResultAsync(); + + return stats.ToQueryStatsInfo(); + } + } + + public async Task>> ErrorsByEndpointName( + string status, + string endpointName, + string modified, + PagingInfo pagingInfo, + SortInfo sortInfo + ) + { + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session.Advanced + .AsyncDocumentQuery() + .Statistics(out var stats) + .FilterByStatusWhere(status) + .AndAlso() + .WhereEquals("ReceivingEndpointName", endpointName) + .FilterByLastModifiedRange(modified) + .Sort(sortInfo) + .Paging(pagingInfo) + // TODO: Fix SetResultTransformer + //.SetResultTransformer(new FailedMessageViewTransformer().TransformerName) + .SelectFields() + .ToListAsync(); + + return new QueryResult>(results, stats.ToQueryStatsInfo()); + } + } + + public async Task> ErrorsSummary() + { + using (var session = documentStore.OpenAsyncSession()) + { + var facetResults = await session.Query() + .AggregateBy(new List + { + new Facet + { + FieldName = "Name", + DisplayFieldName = "Endpoints" + }, + new Facet + { + FieldName = "Host", + DisplayFieldName = "Hosts" + }, + new Facet + { + FieldName = "MessageType", + DisplayFieldName = "Message types" + } + }).ExecuteAsync(); + + var results = facetResults + .ToDictionary( + x => x.Key, + x => (object)x.Value + ); + + return results; + } + } + + public async Task ErrorBy(Guid failedMessageId) + { + using (var session = documentStore.OpenAsyncSession()) + { + var message = await session.LoadAsync(failedMessageId.ToString()); + return message; + } + } + + public async Task ErrorBy(string failedMessageId) + { + using (var session = documentStore.OpenAsyncSession()) + { + var message = await session.LoadAsync(FailedMessageIdGenerator.MakeDocumentId(failedMessageId)); + return message; + } + } + + public Task CreateNotificationsManager() + { + var session = documentStore.OpenAsyncSession(); + var manager = new NotificationsManager(session); + + return Task.FromResult(manager); + } + + public async Task ErrorLastBy(Guid failedMessageId) + { + using (var session = documentStore.OpenAsyncSession()) + { + var message = await session.LoadAsync(failedMessageId.ToString()); + if (message == null) + { + return null; + } + var result = Map(message, session); + return result; + } + } + + static FailedMessageView Map(FailedMessage message, IAsyncDocumentSession session) + { + var processingAttempt = message.ProcessingAttempts.Last(); + + var metadata = processingAttempt.MessageMetadata; + var failureDetails = processingAttempt.FailureDetails; + var wasEdited = message.ProcessingAttempts.Last().Headers.ContainsKey("ServiceControl.EditOf"); + + var failedMsgView = new FailedMessageView + { + Id = message.UniqueMessageId, + MessageType = metadata.GetAsStringOrNull("MessageType"), + IsSystemMessage = metadata.GetOrDefault("IsSystemMessage"), + TimeSent = metadata.GetAsNullableDateTime("TimeSent"), + MessageId = metadata.GetAsStringOrNull("MessageId"), + Exception = failureDetails.Exception, + QueueAddress = failureDetails.AddressOfFailingEndpoint, + NumberOfProcessingAttempts = message.ProcessingAttempts.Count, + Status = message.Status, + TimeOfFailure = failureDetails.TimeOfFailure, + LastModified = session.Advanced.GetLastModifiedFor(message).Value, + Edited = wasEdited, + EditOf = wasEdited ? message.ProcessingAttempts.Last().Headers["ServiceControl.EditOf"] : "" + }; + + try + { + failedMsgView.SendingEndpoint = metadata.GetOrDefault("SendingEndpoint"); + } + catch (Exception ex) + { + Logger.Warn($"Unable to parse SendingEndpoint from metadata for messageId {message.UniqueMessageId}", ex); + failedMsgView.SendingEndpoint = EndpointDetailsParser.SendingEndpoint(processingAttempt.Headers); + } + + try + { + failedMsgView.ReceivingEndpoint = metadata.GetOrDefault("ReceivingEndpoint"); + } + catch (Exception ex) + { + Logger.Warn($"Unable to parse ReceivingEndpoint from metadata for messageId {message.UniqueMessageId}", ex); + failedMsgView.ReceivingEndpoint = EndpointDetailsParser.ReceivingEndpoint(processingAttempt.Headers); + } + + return failedMsgView; + } + + + public async Task EditComment(string groupId, string comment) + { + using (var session = documentStore.OpenAsyncSession()) + { + var groupComment = + await session.LoadAsync(GroupComment.MakeId(groupId)) + ?? new GroupComment { Id = GroupComment.MakeId(groupId) }; + + groupComment.Comment = comment; + + await session.StoreAsync(groupComment); + await session.SaveChangesAsync(); + } + } + + public async Task DeleteComment(string groupId) + { + using (var session = documentStore.OpenAsyncSession()) + { + session.Delete(GroupComment.MakeId(groupId)); + await session.SaveChangesAsync(); + } + } + + public async Task>> GetGroupErrors( + string groupId, + string status, + string modified, + SortInfo sortInfo, + PagingInfo pagingInfo + ) + { + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session.Advanced + .AsyncDocumentQuery() + .Statistics(out var stats) + .WhereEquals(view => view.FailureGroupId, groupId) + .FilterByStatusWhere(status) + .FilterByLastModifiedRange(modified) + .Sort(sortInfo) + .Paging(pagingInfo) + // TODO: Fix SetResultTransformer + //.SetResultTransformer(FailedMessageViewTransformer.Name) + .SelectFields() + .ToListAsync(); + + return results.ToQueryResult(stats); + } + } + + public async Task GetGroupErrorsCount(string groupId, string status, string modified) + { + using (var session = documentStore.OpenAsyncSession()) + { + var queryResult = await session.Advanced + .AsyncDocumentQuery() + .WhereEquals(view => view.FailureGroupId, groupId) + .FilterByStatusWhere(status) + .FilterByLastModifiedRange(modified) + .GetQueryResultAsync(); + + return queryResult.ToQueryStatsInfo(); + } + } + + public async Task>> GetGroup(string groupId, string status, string modified) + { + using (var session = documentStore.OpenAsyncSession()) + { + var queryResult = await session.Advanced + .AsyncDocumentQuery() + .Statistics(out var stats) + .WhereEquals(group => group.Id, groupId) + .FilterByStatusWhere(status) + .FilterByLastModifiedRange(modified) + .ToListAsync(); + + return queryResult.ToQueryResult(stats); + } + } + + public async Task MarkMessageAsResolved(string failedMessageId) + { + using (var session = documentStore.OpenAsyncSession()) + { + session.Advanced.UseOptimisticConcurrency = true; + + var failedMessage = await session.LoadAsync(failedMessageId); + + if (failedMessage == null) + { + return false; + } + + failedMessage.Status = FailedMessageStatus.Resolved; + + await session.SaveChangesAsync(); + + return true; + } + } + + public async Task ProcessPendingRetries(DateTime periodFrom, DateTime periodTo, string queueAddress, Func processCallback) + { + using (var session = documentStore.OpenAsyncSession()) + { + var prequery = session.Advanced + .AsyncDocumentQuery() + .WhereEquals("Status", (int)FailedMessageStatus.RetryIssued) + .AndAlso() + .WhereBetween("LastModified", periodFrom.Ticks, periodTo.Ticks); + + if (!string.IsNullOrWhiteSpace(queueAddress)) + { + prequery = prequery.AndAlso() + .WhereEquals(options => options.QueueAddress, queueAddress); + } + + var query = prequery + // TODO: Fix SetResultTransformer + //.SetResultTransformer(new FailedMessageViewTransformer().TransformerName) + .SelectFields(); + + await using (var ie = await session.Advanced.StreamAsync(query)) + { + while (await ie.MoveNextAsync()) + { + await processCallback(ie.Current.Document.Id); + } + } + } + } + + class DocumentPatchResult + { + public string Document { get; set; } + } + + public async Task<(string[] ids, int count)> UnArchiveMessagesByRange(DateTime from, DateTime to, DateTime cutOff) + { + // TODO: Make sure this new implementation actually works, not going to delete the old implementation (commented below) until then + var patch = new PatchByQueryOperation(new IndexQuery + { + Query = $@"from index '{new FailedMessageViewIndex().IndexName} as msg + where msg.LastModified >= args.From and msg.LastModified <= args.To + where msg.Status == args.Archived + update + {{ + msg.Status = args.Unresolved + }}", + QueryParameters = + { + { "From", from }, + { "To", to }, + { "Unresolved", (int)FailedMessageStatus.Unresolved }, + { "Archived", (int)FailedMessageStatus.Archived }, + } + }, new QueryOperationOptions + { + AllowStale = true, + RetrieveDetails = true + }); + + var operation = await documentStore.Operations.SendAsync(patch); + + var result = await operation.WaitForCompletionAsync(); + + var ids = result.Details.OfType() + .Select(d => d.Id) + .ToArray(); + + // TODO: Are we *really* returning an array AND the length of the same array? + return (ids, ids.Length); + + // var options = new BulkOperationOptions + // { + // AllowStale = true + // }; + + // var result = await documentStore.AsyncDatabaseCommands.UpdateByIndexAsync( + // new FailedMessageViewIndex().IndexName, + // new IndexQuery + // { + // Query = string.Format(CultureInfo.InvariantCulture, "LastModified:[{0} TO {1}] AND Status:{2}", from.Ticks, to.Ticks, (int)FailedMessageStatus.Archived), + // Cutoff = cutOff + // }, new ScriptedPatchRequest + // { + // Script = @" + //if(this.Status === archivedStatus) { + // this.Status = unresolvedStatus; + //} + //", + // Values = + // { + // {"archivedStatus", (int)FailedMessageStatus.Archived}, + // {"unresolvedStatus", (int)FailedMessageStatus.Unresolved} + // } + // }, options); + + // var patchedDocumentIds = (await result.WaitForCompletionAsync()) + // .JsonDeserialization(); + + // return ( + // patchedDocumentIds.Select(x => FailedMessageIdGenerator.GetMessageIdFromDocumentId(x.Document)).ToArray(), + // patchedDocumentIds.Length + // ); + } + + public async Task<(string[] ids, int count)> UnArchiveMessages(IEnumerable failedMessageIds) + { + Dictionary failedMessages; + + using (var session = documentStore.OpenAsyncSession()) + { + session.Advanced.UseOptimisticConcurrency = true; + + var documentIds = failedMessageIds.Select(FailedMessageIdGenerator.MakeDocumentId); + + failedMessages = await session.LoadAsync(documentIds); + + foreach (var failedMessage in failedMessages.Values) + { + if (failedMessage.Status == FailedMessageStatus.Archived) + { + failedMessage.Status = FailedMessageStatus.Unresolved; + } + } + + await session.SaveChangesAsync(); + } + + return ( + failedMessages.Values.Select(x => x.UniqueMessageId).ToArray(), // TODO: (ramon) I don't think we can use Keys here as UniqueMessageId is something different than failedMessageId right? + failedMessages.Count + ); + } + + public async Task RevertRetry(string messageUniqueId) + { + using (var session = documentStore.OpenAsyncSession()) + { + var failedMessage = await session + .LoadAsync(FailedMessageIdGenerator.MakeDocumentId(messageUniqueId)); + if (failedMessage != null) + { + failedMessage.Status = FailedMessageStatus.Unresolved; + } + + var failedMessageRetry = await session + .LoadAsync(FailedMessageRetry.MakeDocumentId(messageUniqueId)); + if (failedMessageRetry != null) + { + session.Delete(failedMessageRetry); + } + + await session.SaveChangesAsync(); + } + } + + public async Task RemoveFailedMessageRetryDocument(string uniqueMessageId) + { + using var session = documentStore.OpenAsyncSession(); + await session.Advanced.RequestExecutor.ExecuteAsync(new DeleteDocumentCommand(FailedMessageRetry.MakeDocumentId(uniqueMessageId), null), session.Advanced.Context); + } + + // TODO: Once using .NET, consider using IAsyncEnumerable here as this is an unbounded query + public async Task GetRetryPendingMessages(DateTime from, DateTime to, string queueAddress) + { + var ids = new List(); + + using (var session = documentStore.OpenAsyncSession()) + { + var query = session.Advanced + .AsyncDocumentQuery() + .WhereEquals("Status", (int)FailedMessageStatus.RetryIssued) + .AndAlso() + .WhereBetween(options => options.LastModified, from.Ticks, to.Ticks) + .AndAlso() + .WhereEquals(o => o.QueueAddress, queueAddress) + // TODO: Fix SetResultTransformer + //.SetResultTransformer(FailedMessageViewTransformer.Name) + .SelectFields(new[] { "Id" }); + + await using (var ie = await session.Advanced.StreamAsync(query)) + { + while (await ie.MoveNextAsync()) + { + ids.Add(ie.Current.Document.Id); + } + } + } + + return ids.ToArray(); + } + + public Task FetchFromFailedMessage(string uniqueMessageId) + { + throw new NotSupportedException("Body not stored embedded"); + } + + public async Task StoreEventLogItem(EventLogItem logItem) + { + using (var session = documentStore.OpenAsyncSession()) + { + await session.StoreAsync(logItem); + await session.SaveChangesAsync(); + } + } + + public async Task StoreFailedMessagesForTestsOnly(params FailedMessage[] failedMessages) + { + using (var session = documentStore.OpenAsyncSession()) + { + foreach (var message in failedMessages) + { + await session.StoreAsync(message); + } + + await session.SaveChangesAsync(); + } + } + + static readonly ILog Logger = LogManager.GetLogger(); + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/EventLogDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/EventLogDataStore.cs new file mode 100644 index 0000000000..597f52958b --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/EventLogDataStore.cs @@ -0,0 +1,43 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System.Collections.Generic; + using System.Threading.Tasks; + using EventLog; + using Persistence.Infrastructure; + using Raven.Client.Documents; + using Raven.Client.Documents.Session; + + class EventLogDataStore : IEventLogDataStore + { + readonly IDocumentStore documentStore; + + public EventLogDataStore(IDocumentStore documentStore) + { + this.documentStore = documentStore; + } + + public async Task Add(EventLogItem logItem) + { + using (var session = documentStore.OpenAsyncSession()) + { + await session.StoreAsync(logItem); + await session.SaveChangesAsync(); + } + } + + public async Task<(IList, int, string)> GetEventLogItems(PagingInfo pagingInfo) + { + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session + .Query() + .Statistics(out var stats) + .OrderByDescending(p => p.RaisedAt, OrderingType.Double) + .Paging(pagingInfo) + .ToListAsync(); + + return (results, stats.TotalResults, stats.ResultEtag.ToString()); + } + } + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/Extensions.cs b/src/ServiceControl.Persistence.RavenDb5/Extensions.cs new file mode 100644 index 0000000000..55f2936ace --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Extensions.cs @@ -0,0 +1,32 @@ +namespace ServiceControl.Infrastructure.RavenDB +{ + using System; + using System.Threading; + using Newtonsoft.Json.Linq; + using Raven.Client.Documents.Conventions; + using Raven.Client.Documents.Queries; + + static class Extensions + { + //public static void Query(this DocumentDatabase db, string index, IndexQuery query, Action onItem, TState state, CancellationToken cancellationToken = default) + //{ + // var results = db.Queries.Query(index, query, cancellationToken); + // foreach (var doc in results.Results) + // { + // onItem(doc, state); + // } + //} + + // TODO: This polyfill of RavenDB 3.5 is a guess based loosely on https://github.com/ravendb/ravendb/blob/v3.5/Raven.Client.Lightweight/Document/DocumentConvention.cs#L151 + public static string DefaultFindFullDocumentKeyFromNonStringIdentifier(this DocumentConventions conventions, T id, Type collectionType, bool allowNull) + { + if (allowNull && id.Equals(default(T))) + { + return null; + } + + var collectionName = conventions.FindCollectionName(collectionType); + return $"{collectionName}{conventions.IdentityPartsSeparator}{id}"; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Extensions/DictionaryExtensions.cs b/src/ServiceControl.Persistence.RavenDb5/Extensions/DictionaryExtensions.cs new file mode 100644 index 0000000000..7aabb6ea83 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Extensions/DictionaryExtensions.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Infrastructure +{ + using System; + using System.Collections.Generic; + + static class DictionaryExtensions + { + public static void CheckIfKeyExists(string key, IReadOnlyDictionary headers, Action actionToInvokeWhenKeyIsFound) + { + if (headers.TryGetValue(key, out var value)) + { + actionToInvokeWhenKeyIsFound(value); + } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Extensions/QueryResultConvert.cs b/src/ServiceControl.Persistence.RavenDb5/Extensions/QueryResultConvert.cs new file mode 100644 index 0000000000..ba7100e879 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Extensions/QueryResultConvert.cs @@ -0,0 +1,15 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System.Collections.Generic; + using Persistence.Infrastructure; + using Raven.Client.Documents.Session; + + static class QueryResultConvert + { + public static QueryResult> ToQueryResult(this IList result, QueryStatistics stats) + where T : class + { + return new QueryResult>(result, stats.ToQueryStatsInfo()); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/ExternalIntegrationRequestsDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ExternalIntegrationRequestsDataStore.cs new file mode 100644 index 0000000000..0ea74e7346 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/ExternalIntegrationRequestsDataStore.cs @@ -0,0 +1,217 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Reactive.Linq; + using System.Threading; + using System.Threading.Tasks; + using ExternalIntegrations; + using Microsoft.Extensions.Hosting; + using NServiceBus; + using NServiceBus.Logging; + using ServiceBus.Management.Infrastructure.Extensions; + using Raven.Client.Documents; + using Raven.Client.Documents.Changes; + + class ExternalIntegrationRequestsDataStore + : IExternalIntegrationRequestsDataStore + , IHostedService + , IAsyncDisposable + { + public ExternalIntegrationRequestsDataStore(RavenDBPersisterSettings settings, IDocumentStore documentStore, CriticalError criticalError) + { + this.settings = settings; + this.documentStore = documentStore; + + circuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker( + "EventDispatcher", + TimeSpan.FromMinutes(5), // TODO: Shouldn't be magic value but coming from settings + ex => criticalError.Raise("Repeated failures when dispatching external integration events.", ex), + TimeSpan.FromSeconds(20) // TODO: Shouldn't be magic value but coming from settings + ); + } + + const string KeyPrefix = "ExternalIntegrationDispatchRequests"; + + public async Task StoreDispatchRequest(IEnumerable dispatchRequests) + { + using (var session = documentStore.OpenAsyncSession()) + { + foreach (var dispatchRequest in dispatchRequests) + { + if (dispatchRequest.Id != null) + { + throw new ArgumentException("Items cannot have their Id property set"); + } + + dispatchRequest.Id = KeyPrefix + "/" + Guid.NewGuid(); // TODO: Key is generated to persistence + await session.StoreAsync(dispatchRequest); + } + + await session.SaveChangesAsync(); + } + } + + public void Subscribe(Func callback) + { + if (this.callback != null) + { + throw new InvalidOperationException("Subscription already exists."); + } + + this.callback = callback ?? throw new ArgumentNullException(nameof(callback)); + + StartDispatcher(); + } + + void StartDispatcher() + { + task = StartDispatcherTask(tokenSource.Token); + } + + async Task StartDispatcherTask(CancellationToken cancellationToken) + { + try + { + await DispatchEvents(cancellationToken); + do + { + try + { + await signal.WaitHandle.WaitOneAsync(cancellationToken); + signal.Reset(); + } + catch (OperationCanceledException) + { + break; + } + + await DispatchEvents(cancellationToken); + } + while (!cancellationToken.IsCancellationRequested); + } + catch (OperationCanceledException) + { + // ignore + } + catch (Exception ex) + { + Logger.Error("An exception occurred when dispatching external integration events", ex); + await circuitBreaker.Failure(ex); + + if (!tokenSource.IsCancellationRequested) + { + StartDispatcher(); + } + } + } + + async Task DispatchEvents(CancellationToken cancellationToken) + { + bool more; + + do + { + more = await TryDispatchEventBatch(); + + circuitBreaker.Success(); + + if (more && !cancellationToken.IsCancellationRequested) + { + //if there is more events to dispatch we sleep for a bit and then we go again + await Task.Delay(1000, CancellationToken.None); + } + } + while (!cancellationToken.IsCancellationRequested && more); + } + + async Task TryDispatchEventBatch() + { + using (var session = documentStore.OpenAsyncSession()) + { + var awaitingDispatching = await session + .Query() + .Statistics(out var stats) + .Take(settings.ExternalIntegrationsDispatchingBatchSize) + .ToListAsync(); + + if (awaitingDispatching.Count == 0) + { + //TODO: this should ensure we query again if the result is potentially stale + // if ☝️ is not true we will need to use/parse the ChangeVector when document is written and compare to ResultEtag + return stats.IsStale; + } + + var allContexts = awaitingDispatching.Select(r => r.DispatchContext).ToArray(); + if (Logger.IsDebugEnabled) + { + Logger.Debug($"Dispatching {allContexts.Length} events."); + } + + await callback(allContexts); + + foreach (var dispatchedEvent in awaitingDispatching) + { + session.Delete(dispatchedEvent); + } + + await session.SaveChangesAsync(); + } + + return true; + } + + public Task StartAsync(CancellationToken cancellationToken) + { + subscription = documentStore + .Changes() + .ForDocumentsStartingWith(KeyPrefix) + .Where(c => c.Type == DocumentChangeTypes.Put) + .Subscribe(d => + { + signal.Set(); + }); + + return Task.CompletedTask; + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + await DisposeAsync(); + } + + public async ValueTask DisposeAsync() + { + if (isDisposed) + { + return; + } + + isDisposed = true; + subscription?.Dispose(); + tokenSource?.Cancel(); + + if (task != null) + { + await task; + } + + tokenSource?.Dispose(); + circuitBreaker?.Dispose(); + } + + readonly RavenDBPersisterSettings settings; + readonly IDocumentStore documentStore; + readonly CancellationTokenSource tokenSource = new CancellationTokenSource(); + readonly RepeatedFailuresOverTimeCircuitBreaker circuitBreaker; + + IDisposable subscription; + Task task; + ManualResetEventSlim signal = new ManualResetEventSlim(); + Func callback; + bool isDisposed; + + static ILog Logger = LogManager.GetLogger(typeof(ExternalIntegrationRequestsDataStore)); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/FailedErrorImportDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/FailedErrorImportDataStore.cs new file mode 100644 index 0000000000..1a4de2512c --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/FailedErrorImportDataStore.cs @@ -0,0 +1,74 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using NServiceBus.Logging; + using Raven.Client.Documents; + using Raven.Client.Documents.Commands; + using ServiceControl.Operations; + + class FailedErrorImportDataStore : IFailedErrorImportDataStore + { + readonly IDocumentStore store; + + static readonly ILog Logger = LogManager.GetLogger(typeof(FailedErrorImportDataStore)); + + public FailedErrorImportDataStore(IDocumentStore store) + { + this.store = store; + } + + public async Task ProcessFailedErrorImports(Func processMessage, CancellationToken cancellationToken) + { + var succeeded = 0; + var failed = 0; + using (var session = store.OpenAsyncSession()) + { + var query = session.Query(); + await using var stream = await session.Advanced.StreamAsync(query, cancellationToken); + while (!cancellationToken.IsCancellationRequested && await stream.MoveNextAsync()) + { + var transportMessage = stream.Current.Document.Message; + try + { + await processMessage(transportMessage); + + await session.Advanced.RequestExecutor.ExecuteAsync(new DeleteDocumentCommand(stream.Current.Id, null), session.Advanced.Context, token: cancellationToken); + + succeeded++; + + if (Logger.IsDebugEnabled) + { + Logger.Debug($"Successfully re-imported failed error message {transportMessage.Id}."); + } + } + catch (OperationCanceledException) + { + // no-op + } + catch (Exception e) + { + Logger.Error($"Error while attempting to re-import failed error message {transportMessage.Id}.", e); + failed++; + } + } + } + + Logger.Info($"Done re-importing failed errors. Successfully re-imported {succeeded} messages. Failed re-importing {failed} messages."); + + if (failed > 0) + { + Logger.Warn($"{failed} messages could not be re-imported. This could indicate a problem with the data. Contact Particular support if you need help with recovering the messages."); + } + } + + public async Task QueryContainsFailedImports() + { + using var session = store.OpenAsyncSession(); + var query = session.Query(); + await using var ie = await session.Advanced.StreamAsync(query); + return await ie.MoveNextAsync(); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/FailedMessageViewIndexNotifications.cs b/src/ServiceControl.Persistence.RavenDb5/FailedMessageViewIndexNotifications.cs new file mode 100644 index 0000000000..92a5945fdc --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/FailedMessageViewIndexNotifications.cs @@ -0,0 +1,111 @@ +namespace ServiceControl.MessageFailures +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using Api; + using Microsoft.Extensions.Hosting; + using NServiceBus.Logging; + using Persistence; + using Raven.Client; + using Raven.Client.Documents; + + class FailedMessageViewIndexNotifications + : IFailedMessageViewIndexNotifications + , IDisposable + , IHostedService + { + public FailedMessageViewIndexNotifications(IDocumentStore store) + { + this.store = store; + } + + void OnNext() + { + try + { + UpdatedCount().GetAwaiter().GetResult(); + } + catch (Exception ex) + { + logging.WarnFormat("Failed to emit MessageFailuresUpdated - {0}", ex); + } + } + + async Task UpdatedCount() + { + using (var session = store.OpenAsyncSession()) + { + var failedUnresolvedMessageCount = await session + .Query() + .CountAsync(p => p.Status == FailedMessageStatus.Unresolved); + + var failedArchivedMessageCount = await session + .Query() + .CountAsync(p => p.Status == FailedMessageStatus.Archived); + + if (lastUnresolvedCount == failedUnresolvedMessageCount && lastArchivedCount == failedArchivedMessageCount) + { + return; + } + + lastUnresolvedCount = failedUnresolvedMessageCount; + lastArchivedCount = failedArchivedMessageCount; + + if (subscriber != null) + { + await subscriber(new FailedMessageTotals + { + ArchivedTotal = failedArchivedMessageCount, + UnresolvedTotal = failedUnresolvedMessageCount + }); + } + } + } + + public IDisposable Subscribe(Func callback) + { + if (callback is null) + { + throw new ArgumentNullException(nameof(callback)); + } + + if (!(subscriber is null)) + { + throw new InvalidOperationException("Already a subscriber, only a single subscriber supported"); + } + + subscriber = callback; + return this; + } + + public void Dispose() + { + subscriber = null; + subscription?.Dispose(); + } + + public Task StartAsync(CancellationToken cancellationToken) + { + subscription = store + .Changes() + .ForIndex(new FailedMessageViewIndex().IndexName) + .Subscribe(d => OnNext()); + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) + { + Dispose(); + return Task.CompletedTask; + } + + readonly IDocumentStore store; + readonly ILog logging = LogManager.GetLogger(typeof(FailedMessageViewIndexNotifications)); + + Func subscriber; + IDisposable subscription; + int lastUnresolvedCount; + int lastArchivedCount; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs b/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs new file mode 100644 index 0000000000..ba986ed298 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs @@ -0,0 +1,148 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.Hosting; + using Newtonsoft.Json.Linq; + using NServiceBus.Logging; + using Raven.Client.Documents; + using Raven.Client.Documents.Operations; + using Raven.Client.Exceptions; + using ServiceControl.MessageFailures; + using ServiceControl.MessageFailures.Api; + using ServiceControl.Persistence.Infrastructure; + using ServiceControl.Recoverability; + + class FailedMessageReclassifier : IReclassifyFailedMessages + { + readonly IDocumentStore store; + readonly IEnumerable classifiers; + + public FailedMessageReclassifier(IDocumentStore store, IHostApplicationLifetime applicationLifetime, IEnumerable classifiers) + { + this.store = store; + this.classifiers = classifiers; + + applicationLifetime?.ApplicationStopping.Register(() => { abort = true; }); + } + + public async Task ReclassifyFailedMessages(bool force) + { + logger.Info("Reclassification of failures started."); + + var failedMessagesReclassified = 0; + var currentBatch = new List>(); + + using (var session = store.OpenAsyncSession()) + { + ReclassifyErrorSettings settings = null; + + if (!force) + { + settings = await session.LoadAsync(ReclassifyErrorSettings.IdentifierCase); + + if (settings != null && settings.ReclassificationDone) + { + logger.Info("Skipping reclassification of failures as classification has already been done."); + return 0; + } + } + + var query = session.Query() + .Where(f => f.Status == FailedMessageStatus.Unresolved); + + var totalMessagesReclassified = 0; + + await using (var stream = await session.Advanced.StreamAsync(query.OfType())) + { + while (!abort && await stream.MoveNextAsync()) + { + currentBatch.Add(Tuple.Create(stream.Current.Document.Id, new ClassifiableMessageDetails(stream.Current.Document))); + + if (currentBatch.Count == BatchSize) + { + failedMessagesReclassified += ReclassifyBatch(store, currentBatch, classifiers); + currentBatch.Clear(); + + totalMessagesReclassified += BatchSize; + logger.Info($"Reclassification of batch of {BatchSize} failed messages completed. Total messages reclassified: {totalMessagesReclassified}"); + } + } + } + + if (currentBatch.Any()) + { + ReclassifyBatch(store, currentBatch, classifiers); + } + + logger.Info($"Reclassification of failures ended. Reclassified {failedMessagesReclassified} messages"); + + settings ??= new ReclassifyErrorSettings(); + + settings.ReclassificationDone = true; + await session.StoreAsync(settings); + await session.SaveChangesAsync(); + + return failedMessagesReclassified; + } + } + + int ReclassifyBatch(IDocumentStore store, IEnumerable> docs, IEnumerable classifiers) + { + var failedMessagesReclassified = 0; + + Parallel.ForEach(docs, doc => + { + var failureGroups = GetClassificationGroups(doc.Item2, classifiers).Select(JObject.FromObject); + + try + { + store.Operations.Send(new PatchOperation(doc.Item1, null, new PatchRequest + { + Script = @"this.FailureGroups = args.Value", + Values = + { + { "Value", new JArray(failureGroups) } + } + })); + + Interlocked.Increment(ref failedMessagesReclassified); + } + catch (ConcurrencyException) + { + // Ignore concurrency exceptions + } + }); + + return failedMessagesReclassified; + } + + IEnumerable GetClassificationGroups(ClassifiableMessageDetails details, IEnumerable classifiers) + { + foreach (var classifier in classifiers) + { + var classification = classifier.ClassifyFailure(details); + if (classification == null) + { + continue; + } + + var id = DeterministicGuid.MakeId(classifier.Name, classification).ToString(); + + yield return new FailedMessage.FailureGroup + { + Id = id, + Title = classification, + Type = classifier.Name + }; + } + } + + readonly ILog logger = LogManager.GetLogger(); + const int BatchSize = 1000; + bool abort; + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/ArchivedGroupsViewIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/ArchivedGroupsViewIndex.cs new file mode 100644 index 0000000000..5d373b119b --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/ArchivedGroupsViewIndex.cs @@ -0,0 +1,44 @@ +namespace ServiceControl.Recoverability +{ + using System.Linq; + using MessageFailures; + using Raven.Client.Documents.Indexes; + + class ArchivedGroupsViewIndex : AbstractIndexCreationTask + { + public ArchivedGroupsViewIndex() + { + Map = docs => from doc in docs + where doc.Status == FailedMessageStatus.Archived + let failureTimes = doc.ProcessingAttempts.Select(x => x.FailureDetails.TimeOfFailure) + from failureGroup in doc.FailureGroups + select new FailureGroupView + { + Id = failureGroup.Id, + Title = failureGroup.Title, + Count = 1, + First = failureTimes.Min(), + Last = failureTimes.Max(), + Type = failureGroup.Type + }; + + Reduce = results => from result in results + group result by new + { + result.Id, + result.Title, + result.Type + } + into g + select new FailureGroupView + { + Id = g.Key.Id, + Title = g.Key.Title, + Count = g.Sum(x => x.Count), + First = g.Min(x => x.First), + Last = g.Max(x => x.Last), + Type = g.Key.Type + }; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/CustomChecksIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/CustomChecksIndex.cs new file mode 100644 index 0000000000..7dd1833657 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/CustomChecksIndex.cs @@ -0,0 +1,21 @@ +namespace ServiceControl.Persistence +{ + using System.Linq; + using ServiceControl.Contracts.CustomChecks; + using Raven.Client.Documents.Indexes; + + class CustomChecksIndex : AbstractIndexCreationTask + { + public CustomChecksIndex() + { + Map = docs => from cc in docs + select new CustomCheck + { + Status = cc.Status, + ReportedAt = cc.ReportedAt, + Category = cc.Category, + CustomCheckId = cc.CustomCheckId + }; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedErrorImportIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedErrorImportIndex.cs new file mode 100644 index 0000000000..653e2f0f96 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedErrorImportIndex.cs @@ -0,0 +1,18 @@ +namespace ServiceControl.Operations +{ + using System.Linq; + using Raven.Client.Documents.Indexes; + + class FailedErrorImportIndex : AbstractIndexCreationTask + { + public FailedErrorImportIndex() + { + Map = docs => from cc in docs + select new FailedErrorImport + { + Id = cc.Id, + Message = cc.Message + }; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageFacetsIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageFacetsIndex.cs new file mode 100644 index 0000000000..5335c9f8d9 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageFacetsIndex.cs @@ -0,0 +1,26 @@ +namespace ServiceControl.MessageFailures.Api +{ + using System.Linq; + using ServiceControl.Operations; + using Raven.Client.Documents.Indexes; + + class FailedMessageFacetsIndex : AbstractIndexCreationTask + { + public FailedMessageFacetsIndex() + { + Map = failures => from failure in failures + where failure.Status == FailedMessageStatus.Unresolved + let t = (EndpointDetails)failure.ProcessingAttempts.Last().MessageMetadata["ReceivingEndpoint"] + select new + { + t.Name, + t.Host, + MessageType = failure.ProcessingAttempts.Last().MessageMetadata["MessageType"] + }; + + Index("Name", FieldIndexing.Exact); //to avoid lower casing + Index("Host", FieldIndexing.Exact); //to avoid lower casing + Index("MessageType", FieldIndexing.Exact); //to avoid lower casing + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageRetries_ByBatch.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageRetries_ByBatch.cs new file mode 100644 index 0000000000..3a782c6a4e --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageRetries_ByBatch.cs @@ -0,0 +1,18 @@ +namespace ServiceControl.Persistence +{ + using System.Linq; + using Raven.Client.Documents.Indexes; + using ServiceControl.Recoverability; + + class FailedMessageRetries_ByBatch : AbstractIndexCreationTask + { + public FailedMessageRetries_ByBatch() + { + Map = docs => from doc in docs + select new + { + doc.RetryBatchId + }; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageViewIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageViewIndex.cs new file mode 100644 index 0000000000..b390b2c5cb --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageViewIndex.cs @@ -0,0 +1,39 @@ +namespace ServiceControl.MessageFailures.Api +{ + using System; + using System.Linq; + using ServiceControl.Operations; + using Raven.Client.Documents.Indexes; + + class FailedMessageViewIndex : AbstractIndexCreationTask + { + public FailedMessageViewIndex() + { + Map = messages => from message in messages + let processingAttemptsLast = message.ProcessingAttempts.Last() + select new + { + MessageId = processingAttemptsLast.MessageMetadata["MessageId"], + MessageType = processingAttemptsLast.MessageMetadata["MessageType"], + message.Status, + TimeSent = (DateTime)processingAttemptsLast.MessageMetadata["TimeSent"], + ReceivingEndpointName = ((EndpointDetails)processingAttemptsLast.MessageMetadata["ReceivingEndpoint"]).Name, + QueueAddress = processingAttemptsLast.FailureDetails.AddressOfFailingEndpoint, + processingAttemptsLast.FailureDetails.TimeOfFailure, + LastModified = MetadataFor(message).Value("@last-modified").Ticks + }; + } + + public class SortAndFilterOptions : IHaveStatus + { + public string MessageId { get; set; } + public DateTime TimeSent { get; set; } + public string MessageType { get; set; } + public string ReceivingEndpointName { get; set; } + public string QueueAddress { get; set; } + public DateTime TimeOfFailure { get; set; } + public long LastModified { get; set; } + public FailedMessageStatus Status { get; set; } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessages_ByGroup.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessages_ByGroup.cs new file mode 100644 index 0000000000..7fbe6207f4 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessages_ByGroup.cs @@ -0,0 +1,31 @@ +namespace ServiceControl.Recoverability +{ + using System; + using System.Linq; + using MessageFailures; + using Raven.Client.Documents.Indexes; + + class FailedMessages_ByGroup : AbstractIndexCreationTask + { + public FailedMessages_ByGroup() + { + Map = docs => from doc in docs + let processingAttemptsLast = doc.ProcessingAttempts.Last() + from failureGroup in doc.FailureGroups + select new FailureGroupMessageView + { + Id = doc.Id, + MessageId = doc.UniqueMessageId, + FailureGroupId = failureGroup.Id, + FailureGroupName = failureGroup.Title, + Status = doc.Status, + MessageType = (string)processingAttemptsLast.MessageMetadata["MessageType"], + TimeSent = (DateTime)processingAttemptsLast.MessageMetadata["TimeSent"], + TimeOfFailure = processingAttemptsLast.FailureDetails.TimeOfFailure, + LastModified = MetadataFor(doc).Value("@last-modified").Ticks + }; + + StoreAllFields(FieldStorage.Yes); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailureGroupsViewIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailureGroupsViewIndex.cs new file mode 100644 index 0000000000..104bff6816 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailureGroupsViewIndex.cs @@ -0,0 +1,44 @@ +namespace ServiceControl.Recoverability +{ + using System.Linq; + using MessageFailures; + using Raven.Client.Documents.Indexes; + + class FailureGroupsViewIndex : AbstractIndexCreationTask + { + public FailureGroupsViewIndex() + { + Map = docs => from doc in docs + where doc.Status == FailedMessageStatus.Unresolved + let failureTimes = doc.ProcessingAttempts.Select(x => x.FailureDetails.TimeOfFailure) + from failureGroup in doc.FailureGroups + select new FailureGroupView + { + Id = failureGroup.Id, + Title = failureGroup.Title, + Count = 1, + First = failureTimes.Min(), + Last = failureTimes.Max(), + Type = failureGroup.Type + }; + + Reduce = results => from result in results + group result by new + { + result.Id, + result.Title, + result.Type + } + into g + select new FailureGroupView + { + Id = g.Key.Id, + Title = g.Key.Title, + Count = g.Sum(x => x.Count), + First = g.Min(x => x.First), + Last = g.Max(x => x.Last), + Type = g.Key.Type + }; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/GroupCommentIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/GroupCommentIndex.cs new file mode 100644 index 0000000000..3bfb3ea353 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/GroupCommentIndex.cs @@ -0,0 +1,14 @@ +namespace ServiceControl.Recoverability +{ + using System.Linq; + using MessageFailures; + using Raven.Client.Documents.Indexes; + + class GroupCommentIndex : AbstractIndexCreationTask + { + public GroupCommentIndex() + { + Map = docs => docs.Select(gc => new GroupComment { Id = gc.Id, Comment = gc.Comment }); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/KnownEndpointIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/KnownEndpointIndex.cs new file mode 100644 index 0000000000..1b25b507c2 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/KnownEndpointIndex.cs @@ -0,0 +1,21 @@ +namespace ServiceControl.Persistence +{ + using System.Linq; + using Raven.Client.Documents.Indexes; + + class KnownEndpointIndex : AbstractIndexCreationTask + { + public KnownEndpointIndex() + { + Map = messages => from message in messages + select new + { + EndpointDetails_Name = message.EndpointDetails.Name, + EndpointDetails_Host = message.EndpointDetails.Host, + message.HostDisplayName, + message.Monitored, + message.HasTemporaryId + }; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/MessagesViewIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/MessagesViewIndex.cs new file mode 100644 index 0000000000..73f5401e2a --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/MessagesViewIndex.cs @@ -0,0 +1,60 @@ +namespace ServiceControl.Persistence +{ + using System; + using System.Linq; + using Lucene.Net.Analysis.Standard; + using Raven.Client.Documents.Indexes; + using ServiceControl.MessageFailures; + using ServiceControl.Operations; + + // TODO: Consider renaming to FailedMessageIndex + public class MessagesViewIndex : AbstractIndexCreationTask + { + public MessagesViewIndex() + { + Map = messages => from message in messages + let last = message.ProcessingAttempts.Last() + select new SortAndFilterOptions + { + MessageId = last.MessageId, + MessageType = (string)last.MessageMetadata["MessageType"], + IsSystemMessage = (bool)last.MessageMetadata["IsSystemMessage"], + Status = message.Status == FailedMessageStatus.Archived + ? MessageStatus.ArchivedFailure + : message.Status == FailedMessageStatus.Resolved + ? MessageStatus.ResolvedSuccessfully + : message.ProcessingAttempts.Count == 1 + ? MessageStatus.Failed + : MessageStatus.RepeatedFailure, + TimeSent = (DateTime)last.MessageMetadata["TimeSent"], + ProcessedAt = last.AttemptedAt, + ReceivingEndpointName = ((EndpointDetails)last.MessageMetadata["ReceivingEndpoint"]).Name, + CriticalTime = null, + ProcessingTime = null, + DeliveryTime = null, + Query = last.MessageMetadata.Select(_ => _.Value.ToString()).Union(new[] { string.Join(" ", last.Headers.Select(x => x.Value)) }).ToArray(), + ConversationId = (string)last.MessageMetadata["ConversationId"] + }; + + Index(x => x.Query, FieldIndexing.Search); + + Analyze(x => x.Query, typeof(StandardAnalyzer).AssemblyQualifiedName); + } + + public class SortAndFilterOptions + { + public string MessageId { get; set; } + public string MessageType { get; set; } + public bool IsSystemMessage { get; set; } + public MessageStatus Status { get; set; } + public DateTime ProcessedAt { get; set; } + public string ReceivingEndpointName { get; set; } + public TimeSpan? CriticalTime { get; set; } + public TimeSpan? ProcessingTime { get; set; } + public TimeSpan? DeliveryTime { get; set; } + public string ConversationId { get; set; } + public string[] Query { get; set; } + public DateTime TimeSent { get; set; } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/QueueAddressIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/QueueAddressIndex.cs new file mode 100644 index 0000000000..b106c9bdcb --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/QueueAddressIndex.cs @@ -0,0 +1,28 @@ +namespace ServiceControl.MessageFailures.Api +{ + using System.Linq; + using Raven.Client.Documents.Indexes; + + class QueueAddressIndex : AbstractIndexCreationTask + { + public QueueAddressIndex() + { + Map = messages => from message in messages + let processingAttemptsLast = message.ProcessingAttempts.Last() + select new + { + PhysicalAddress = processingAttemptsLast.FailureDetails.AddressOfFailingEndpoint, + FailedMessageCount = 1 + }; + + Reduce = results => from result in results + group result by result.PhysicalAddress + into g + select new + { + PhysicalAddress = g.Key, + FailedMessageCount = g.Sum(m => m.FailedMessageCount) + }; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/RetryBatches_ByStatusAndSession.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/RetryBatches_ByStatusAndSession.cs new file mode 100644 index 0000000000..f63a0d43aa --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/RetryBatches_ByStatusAndSession.cs @@ -0,0 +1,18 @@ +namespace ServiceControl.Persistence +{ + using System.Linq; + using Raven.Client.Documents.Indexes; + + class RetryBatches_ByStatusAndSession : AbstractIndexCreationTask + { + public RetryBatches_ByStatusAndSession() + { + Map = docs => from doc in docs + select new + { + doc.RetrySessionId, + doc.Status + }; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/RetryBatches_ByStatus_ReduceInitialBatchSize.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/RetryBatches_ByStatus_ReduceInitialBatchSize.cs new file mode 100644 index 0000000000..e8dd133c0d --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/RetryBatches_ByStatus_ReduceInitialBatchSize.cs @@ -0,0 +1,45 @@ +namespace ServiceControl.Persistence +{ + using System.Linq; + using Raven.Client.Documents.Indexes; + + class RetryBatches_ByStatus_ReduceInitialBatchSize : AbstractIndexCreationTask + { + public RetryBatches_ByStatus_ReduceInitialBatchSize() + { + Map = docs => from doc in docs + select new + { + doc.RequestId, + doc.RetryType, + HasStagingBatches = doc.Status == RetryBatchStatus.Staging, + HasForwardingBatches = doc.Status == RetryBatchStatus.Forwarding, + doc.InitialBatchSize, + doc.Originator, + doc.Classifier, + doc.StartTime, + doc.Last + }; + + Reduce = results => from result in results + group result by new + { + result.RequestId, + result.RetryType + } + into g + select new + { + g.Key.RequestId, + g.Key.RetryType, + g.First().Originator, + HasStagingBatches = g.Any(x => x.HasStagingBatches), + HasForwardingBatches = g.Any(x => x.HasForwardingBatches), + InitialBatchSize = g.Sum(x => x.InitialBatchSize), + g.First().StartTime, + Last = g.Max(x => x.Last), + g.First().Classifier + }; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/MigratedTypeAwareBinder.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/MigratedTypeAwareBinder.cs new file mode 100644 index 0000000000..e1fd124395 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/MigratedTypeAwareBinder.cs @@ -0,0 +1,45 @@ +namespace ServiceControl.Infrastructure.RavenDB +{ + using System; + using System.Linq; + using Newtonsoft.Json.Serialization; + using ServiceControl.Contracts.CustomChecks; + using ServiceControl.Contracts.Operations; + using ServiceControl.MessageAuditing; + using ServiceControl.MessageFailures; + using ServiceControl.Operations; + using ServiceControl.Persistence; + using ServiceControl.Recoverability; + using static ServiceControl.MessageFailures.FailedMessage; + + class MigratedTypeAwareBinder : DefaultSerializationBinder + { + public override Type BindToType(string assemblyName, string typeName) + { + var className = GetClassName(typeName); + return className switch + { + nameof(CustomCheck) => typeof(CustomCheck), + nameof(CustomCheckDetail) => typeof(CustomCheckDetail), + nameof(EndpointDetails) => typeof(EndpointDetails), + nameof(ExceptionDetails) => typeof(ExceptionDetails), + nameof(FailedMessage) => typeof(FailedMessage), + nameof(FailureDetails) => typeof(FailureDetails), + nameof(FailureGroup) => typeof(FailureGroup), + nameof(GroupComment) => typeof(GroupComment), + nameof(KnownEndpoint) => typeof(KnownEndpoint), + nameof(ProcessedMessage) => typeof(ProcessedMessage), + nameof(ProcessingAttempt) => typeof(ProcessingAttempt), + nameof(FailedMessageRetry) => typeof(FailedMessageRetry), + nameof(RetryBatch) => typeof(RetryBatch), + nameof(RetryBatchNowForwarding) => typeof(RetryBatchNowForwarding), + _ => base.BindToType(assemblyName, typeName), + }; + } + + string GetClassName(string typeName) + { + return typeName.Split('.').Last(); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/IDataMigration.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/IDataMigration.cs new file mode 100644 index 0000000000..25fe4a0511 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/IDataMigration.cs @@ -0,0 +1,10 @@ +namespace ServiceControl.Infrastructure.RavenDB +{ + using System.Threading.Tasks; + using Raven.Client.Documents; + + interface IDataMigration + { + Task Migrate(IDocumentStore store); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration.cs new file mode 100644 index 0000000000..0b3f0253d4 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration.cs @@ -0,0 +1,37 @@ +namespace ServiceControl.Infrastructure.RavenDB +{ + using System.Linq; + using System.Threading.Tasks; + using Raven.Client.Documents; + using Raven.Client.Documents.Commands; + using ServiceControl.Persistence; + + // TODO: I don't know if we can delete this because no prior Raven5 database will exist, or if it's an ongoing need to purge these things on every startup + class PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration : IDataMigration + { + public Task Migrate(IDocumentStore store) + { + using (var session = store.OpenSession()) + { + var endpoints = session.Query().ToList(); + + foreach (var knownEndpoints in endpoints.GroupBy(e => e.EndpointDetails.Host + e.EndpointDetails.Name)) + { + var fixedIdsCount = knownEndpoints.Count(e => !e.HasTemporaryId); + + //If we have knowEndpoints with non temp ids, we should delete all temp ids ones. + if (fixedIdsCount > 0) + { + foreach (var key in knownEndpoints.Where(e => e.HasTemporaryId)) + { + var documentId = store.Conventions.DefaultFindFullDocumentKeyFromNonStringIdentifier(key.Id, typeof(KnownEndpoint), false); + session.Advanced.RequestExecutor.Execute(new DeleteDocumentCommand(documentId, null), session.Advanced.Context); + } + } + } + } + + return Task.CompletedTask; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/MessageTypeConverter.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/MessageTypeConverter.cs new file mode 100644 index 0000000000..9cc3ea9d0b --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/MessageTypeConverter.cs @@ -0,0 +1,24 @@ +namespace ServiceControl.Infrastructure.RavenDB.Subscriptions +{ + using System; + using Newtonsoft.Json; + using NServiceBus.Unicast.Subscriptions; + + class MessageTypeConverter : JsonConverter + { + public override bool CanConvert(Type objectType) + { + return objectType == typeof(MessageType); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + return new MessageType(serializer.Deserialize(reader)); + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + serializer.Serialize(writer, value.ToString()); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/RavenDbSubscriptionStorage.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/RavenDbSubscriptionStorage.cs new file mode 100644 index 0000000000..b56b86263c --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/RavenDbSubscriptionStorage.cs @@ -0,0 +1,248 @@ +namespace ServiceControl.Infrastructure.RavenDB.Subscriptions +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Security.Cryptography; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using NServiceBus; + using NServiceBus.Extensibility; + using NServiceBus.Logging; + using NServiceBus.Settings; + using NServiceBus.Unicast.Subscriptions; + using NServiceBus.Unicast.Subscriptions.MessageDrivenSubscriptions; + using Raven.Client; + using Raven.Client.Documents; + using Raven.Client.Documents.Commands; + using Raven.Client.Documents.Session; + using ServiceControl.Persistence; + using ServiceControl.Persistence.RavenDb.Infrastructure; + + class RavenDbSubscriptionStorage : IServiceControlSubscriptionStorage + { + public RavenDbSubscriptionStorage(IDocumentStore store, ReadOnlySettings settings) : + this(store, settings.EndpointName(), settings.LocalAddress(), settings.GetAvailableTypes().Implementing().Select(e => new MessageType(e)).ToArray()) + { + } + + public RavenDbSubscriptionStorage(IDocumentStore store, string endpointName, string localAddress, MessageType[] locallyHandledEventTypes) + { + this.store = store; + localClient = new SubscriptionClient + { + Endpoint = endpointName, + TransportAddress = localAddress + }; + + this.locallyHandledEventTypes = locallyHandledEventTypes; + + + SetSubscriptions(new Subscriptions()).GetAwaiter().GetResult(); + } + + public async Task Initialize() + { + using (var session = store.OpenAsyncSession()) + { + var primeSubscriptions = await LoadSubscriptions(session) ?? await MigrateSubscriptions(session, localClient); + + await SetSubscriptions(primeSubscriptions); + } + } + + public async Task Subscribe(Subscriber subscriber, MessageType messageType, ContextBag context) + { + if (subscriber.Endpoint == localClient.Endpoint) + { + return; + } + + try + { + await subscriptionsLock.WaitAsync(); + + if (AddOrUpdateSubscription(messageType, subscriber)) + { + await SaveSubscriptions(); + } + } + finally + { + subscriptionsLock.Release(); + } + } + + public async Task Unsubscribe(Subscriber subscriber, MessageType messageType, ContextBag context) + { + try + { + await subscriptionsLock.WaitAsync(); + + var needsSave = false; + if (subscriptions.All.TryGetValue(FormatId(messageType), out var subscription)) + { + var client = CreateSubscriptionClient(subscriber); + if (subscription.Subscribers.Remove(client)) + { + needsSave = true; + } + } + + if (needsSave) + { + await SaveSubscriptions(); + } + } + finally + { + subscriptionsLock.Release(); + } + } + + public Task> GetSubscriberAddressesForMessage(IEnumerable messageTypes, ContextBag context) + { + return Task.FromResult(messageTypes.SelectMany(x => subscriptionsLookup[x]).Distinct()); + } + + bool AddOrUpdateSubscription(MessageType messageType, Subscriber subscriber) + { + var key = FormatId(messageType); + + var subscriptionClient = CreateSubscriptionClient(subscriber); + + if (subscriptions.All.TryGetValue(key, out var subscription)) + { + if (subscription.Subscribers.Contains(subscriptionClient)) + { + return false; + } + + subscription.Subscribers.Add(subscriptionClient); + return true; + } + + // New Subscription + subscription = new Subscription + { + Id = key, + Subscribers = new List + { + subscriptionClient + }, + MessageType = messageType + }; + subscriptions.All.Add(key, subscription); + return true; + } + + static SubscriptionClient CreateSubscriptionClient(Subscriber subscriber) + { + //When the subscriber is running V6 and UseLegacyMessageDrivenSubscriptionMode is enabled at the subscriber the 'subcriber.Endpoint' value is null + var endpoint = subscriber.Endpoint ?? subscriber.TransportAddress.Split('@').First(); + var subscriptionClient = new SubscriptionClient + { + TransportAddress = subscriber.TransportAddress, + Endpoint = endpoint + }; + return subscriptionClient; + } + + async Task SaveSubscriptions() + { + using (var session = store.OpenAsyncSession()) + { + await session.StoreAsync(subscriptions, Subscriptions.SingleDocumentId); + UpdateLookup(); + await session.SaveChangesAsync(); + } + } + + void UpdateLookup() + { + subscriptionsLookup = (from subscription in subscriptions.All.Values + from client in subscription.Subscribers + select new + { + subscription.MessageType, + Subscriber = new Subscriber(client.TransportAddress, client.Endpoint) + }).Union(from eventType in locallyHandledEventTypes + select new + { + MessageType = eventType, + Subscriber = new Subscriber(localClient.TransportAddress, localClient.Endpoint) + } + ).ToLookup(x => x.MessageType, x => x.Subscriber); + } + + string FormatId(MessageType messageType) + { + // use MD5 hash to get a 16-byte hash of the string + var inputBytes = Encoding.Default.GetBytes($"{messageType.TypeName}/{messageType.Version.Major}"); + using (var provider = new MD5CryptoServiceProvider()) + { + var hashBytes = provider.ComputeHash(inputBytes); + + // generate a guid from the hash: + var id = new Guid(hashBytes); + return $"Subscriptions/{id}"; + } + } + + async Task SetSubscriptions(Subscriptions newSubscriptions) + { + try + { + await subscriptionsLock.WaitAsync(); + + subscriptions = newSubscriptions; + UpdateLookup(); + } + finally + { + subscriptionsLock.Release(); + } + } + + static Task LoadSubscriptions(IAsyncDocumentSession session) + => session.LoadAsync(Subscriptions.SingleDocumentId); + + static async Task MigrateSubscriptions(IAsyncDocumentSession session, SubscriptionClient localClient) + { + logger.Info("Migrating subscriptions to new format"); + + var subscriptions = new Subscriptions(); + + var stream = await session.Advanced.StreamAsync("Subscriptions"); + + while (await stream.MoveNextAsync()) + { + var existingSubscription = stream.Current.Document; + existingSubscription.Subscribers.Remove(localClient); + subscriptions.All.Add(existingSubscription.Id.Replace("Subscriptions/", string.Empty), existingSubscription); + await session.Advanced.RequestExecutor.ExecuteAsync(new DeleteDocumentCommand(stream.Current.Id, null), session.Advanced.Context); + } + + await session.StoreAsync(subscriptions, Subscriptions.SingleDocumentId); + await session.SaveChangesAsync(); + return subscriptions; + } + + IDocumentStore store; + SubscriptionClient localClient; + Subscriptions subscriptions; + ILookup subscriptionsLookup; + MessageType[] locallyHandledEventTypes; + + SemaphoreSlim subscriptionsLock = new SemaphoreSlim(1); + + static ILog logger = LogManager.GetLogger(); + } + + class Subscriptions + { + public IDictionary All { get; set; } = new Dictionary(); + public const string SingleDocumentId = "Subscriptions/All"; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/Subscription.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/Subscription.cs new file mode 100644 index 0000000000..51a1d8e541 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/Subscription.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Infrastructure.RavenDB.Subscriptions +{ + using System.Collections.Generic; + using Newtonsoft.Json; + using NServiceBus.Unicast.Subscriptions; + + class Subscription + { + public string Id { get; set; } + + [JsonConverter(typeof(MessageTypeConverter))] + public MessageType MessageType { get; set; } + + public List Subscribers { get; set; } = new List(); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/SubscriptionClient.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/SubscriptionClient.cs new file mode 100644 index 0000000000..9a845df083 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/SubscriptionClient.cs @@ -0,0 +1,31 @@ +namespace ServiceControl.Infrastructure.RavenDB.Subscriptions +{ + using System; + + class SubscriptionClient + { + public string TransportAddress { get; set; } + + public string Endpoint { get; set; } + + public override bool Equals(object obj) + { + if (obj is null) + { + return false; + } + + if (ReferenceEquals(this, obj)) + { + return true; + } + + return obj is SubscriptionClient client && Equals(client); + } + + bool Equals(SubscriptionClient obj) => string.Equals(TransportAddress, obj.TransportAddress, + StringComparison.InvariantCultureIgnoreCase); + + public override int GetHashCode() => TransportAddress.ToLowerInvariant().GetHashCode(); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/TypeExtensions.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/TypeExtensions.cs new file mode 100644 index 0000000000..20c25c6d90 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/TypeExtensions.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Persistence.RavenDb.Infrastructure +{ + using System; + using System.Collections.Generic; + using System.Linq; + + static class TypeExtensions + { + public static IEnumerable Implementing(this IEnumerable types) + { + return from type in types + where typeof(T).IsAssignableFrom(type) && !type.IsAbstract && !type.IsInterface + select type; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/InternalsVisibleTo.cs b/src/ServiceControl.Persistence.RavenDb5/InternalsVisibleTo.cs new file mode 100644 index 0000000000..cbd15879aa --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/InternalsVisibleTo.cs @@ -0,0 +1,7 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("ServiceControl.UnitTests")] +[assembly: InternalsVisibleTo("ServiceControl.PersistenceTests")] +[assembly: InternalsVisibleTo("ServiceControl.Persistence.Tests.RavenDb")] +[assembly: InternalsVisibleTo("ServiceControl.AcceptanceTests.RavenDB")] +[assembly: InternalsVisibleTo("ServiceControl.MultiInstance.AcceptanceTests")] diff --git a/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs new file mode 100644 index 0000000000..cc9ac0f7db --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs @@ -0,0 +1,47 @@ +namespace ServiceControl.Persistence.RavenDb.MessageRedirects +{ + using System; + using System.Threading.Tasks; + using Raven.Client.Documents; + using Raven.Client.Documents.Queries; + using ServiceControl.Persistence.MessageRedirects; + + class MessageRedirectsDataStore : IMessageRedirectsDataStore + { + readonly IDocumentStore store; + + public MessageRedirectsDataStore(IDocumentStore store) + { + this.store = store; + } + + public async Task GetOrCreate() + { + using (var session = store.OpenAsyncSession()) + { + var redirects = await session.LoadAsync(DefaultId); + + if (redirects != null) + { + redirects.ETag = session.Advanced.GetChangeVectorFor(redirects); + redirects.LastModified = RavenQuery.LastModified(redirects); + + return redirects; + } + + return new MessageRedirectsCollection(); + } + } + + public async Task Save(MessageRedirectsCollection redirects) + { + using (var session = store.OpenAsyncSession()) + { + await session.StoreAsync(redirects, redirects.ETag, DefaultId); + await session.SaveChangesAsync(); + } + } + + const string DefaultId = "messageredirects"; + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/MetadataExtensions.cs b/src/ServiceControl.Persistence.RavenDb5/MetadataExtensions.cs new file mode 100644 index 0000000000..7ac89dd9a4 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/MetadataExtensions.cs @@ -0,0 +1,40 @@ +namespace ServiceControl +{ + using System; + using System.Collections.Generic; + + static class MetadataExtensions + { + public static T GetOrDefault(this IDictionary metadata, string key) + { + if (metadata.TryGetValue(key, out var foundValue)) + { + return (T)foundValue; + } + + return default; + } + + public static string GetAsStringOrNull(this IDictionary metadata, string key) + { + if (metadata.TryGetValue(key, out var foundValue)) + { + return foundValue?.ToString(); + } + + return null; + } + + public static DateTime? GetAsNullableDateTime(this IDictionary metadata, string key) + { + var datetimeAsString = metadata.GetAsStringOrNull(key); + + if (datetimeAsString != null) + { + return DateTime.Parse(datetimeAsString); + } + + return null; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/QueueAddressStore.cs b/src/ServiceControl.Persistence.RavenDb5/QueueAddressStore.cs new file mode 100644 index 0000000000..764ad9cdcc --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/QueueAddressStore.cs @@ -0,0 +1,48 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System.Collections.Generic; + using System.Linq; + using System.Threading.Tasks; + using Raven.Client.Documents; + using ServiceControl.MessageFailures; + using ServiceControl.MessageFailures.Api; + using ServiceControl.Persistence.Infrastructure; + + class QueueAddressStore : IQueueAddressStore + { + readonly IDocumentStore documentStore; + + public QueueAddressStore(IDocumentStore documentStore) + { + this.documentStore = documentStore; + } + + public async Task>> GetAddresses(PagingInfo pagingInfo) + { + using var session = documentStore.OpenAsyncSession(); + var addresses = await session + .Query() + .Statistics(out var stats) + .Paging(pagingInfo) + .ToListAsync(); + + var result = new QueryResult>(addresses, stats.ToQueryStatsInfo()); + return result; + } + + public async Task>> GetAddressesBySearchTerm(string search, PagingInfo pagingInfo) + { + using var session = documentStore.OpenAsyncSession(); + var failedMessageQueues = await session + .Query() + .Statistics(out var stats) + .Paging(pagingInfo) + .Where(q => q.PhysicalAddress.StartsWith(search)) + .OrderBy(q => q.PhysicalAddress) + .ToListAsync(); + + var result = new QueryResult>(failedMessageQueues, stats.ToQueryStatsInfo()); + return result; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs b/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs new file mode 100644 index 0000000000..5a65756a95 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs @@ -0,0 +1,60 @@ +namespace ServiceControl.Operations.BodyStorage.RavenAttachments +{ + using System.IO; + using System.Threading.Tasks; + using Raven.Client.Documents; + using Raven.Client.Documents.Operations.Attachments; + + // TODO: For Raven5, look at how the Audit instance is implementing this, as Attachments won't exist + // and there will be no need for a fallback method on a new persistence + // Ramon: Don't understand the comment, audit RavenDB 5 is using attachments.... + class RavenAttachmentsBodyStorage : IBodyStorage + { + const string AttachmentName = "body"; + readonly IDocumentStore documentStore; + + public RavenAttachmentsBodyStorage(IDocumentStore documentStore) + { + this.documentStore = documentStore; + } + + public async Task Store(string messageId, string contentType, int bodySize, Stream bodyStream) + { + // var id = MessageBodyIdGenerator.MakeDocumentId(messageId); // TODO: Not needed? Not used by audit + + using var session = documentStore.OpenAsyncSession(); + + // Following is possible to but not documented in the Raven docs. + //session.Advanced.Attachments.Store(messageId,"body",bodyStream,contentType); + // https://ravendb.net/docs/article-page/5.4/csharp/client-api/operations/attachments/get-attachment + _ = await documentStore.Operations.SendAsync( + new PutAttachmentOperation(messageId, + AttachmentName, + bodyStream, + contentType)); + } + + public async Task TryFetch(string messageId) + { + //var messageId = MessageBodyIdGenerator.MakeDocumentId(bodyId); // TODO: Not needed? Not used by audit + + using var session = documentStore.OpenAsyncSession(); + + var result = await session.Advanced.Attachments.GetAsync(messageId, AttachmentName); + + if (result == null) + { + return null; + } + + return new MessageBodyStreamResult + { + HasResult = true, + Stream = (MemoryStream)result.Stream, + ContentType = result.Details.ContentType, + BodySize = (int)result.Details.Size, + Etag = result.Details.ChangeVector + }; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs b/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs new file mode 100644 index 0000000000..20cdd35b8a --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs @@ -0,0 +1,17 @@ +namespace ServiceControl.Persistence.RavenDb +{ + static class RavenBootstrapper + { + public const string DatabasePathKey = "DbPath"; + public const string HostNameKey = "HostName"; + public const string DatabaseMaintenancePortKey = "DatabaseMaintenancePort"; + public const string ExposeRavenDBKey = "ExposeRavenDB"; + public const string ExpirationProcessTimerInSecondsKey = "ExpirationProcessTimerInSeconds"; + public const string RunInMemoryKey = "RavenDB35/RunInMemory"; + public const string ConnectionStringKey = "RavenDB5/ConnectionString"; + public const string MinimumStorageLeftRequiredForIngestionKey = "MinimumStorageLeftRequiredForIngestion"; + public const string DatabaseNameKey = "RavenDB5/DatabaseName"; + public const string LogsPathKey = "LogPath"; + public const string LogsModeKey = "LogMode"; + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs new file mode 100644 index 0000000000..a23954884b --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs @@ -0,0 +1,32 @@ +using System; +using ServiceControl.Operations; +using ServiceControl.Persistence; + +class RavenDBPersisterSettings : PersistenceSettings +{ + public string HostName { get; set; } = "localhost"; + public int DatabaseMaintenancePort { get; set; } = DatabaseMaintenancePortDefault; + public string DatabaseMaintenanceUrl => $"http://{HostName}:{DatabaseMaintenancePort}"; + public bool ExposeRavenDB { get; set; } + public int ExpirationProcessTimerInSeconds { get; set; } = ExpirationProcessTimerInSecondsDefault; + public bool RunInMemory { get; set; } + public int MinimumStorageLeftRequiredForIngestion { get; set; } = CheckMinimumStorageRequiredForIngestion.MinimumStorageLeftRequiredForIngestionDefault; + public int DataSpaceRemainingThreshold { get; set; } = CheckFreeDiskSpace.DataSpaceRemainingThresholdDefault; + public TimeSpan ErrorRetentionPeriod { get; set; } + public TimeSpan EventsRetentionPeriod { get; set; } + public TimeSpan? AuditRetentionPeriod { get; set; } + public int ExternalIntegrationsDispatchingBatchSize { get; set; } = 100; + + //TODO: these are newly added settings, we should remove any duplication + public string ServerUrl { get; set; } + public string ConnectionString { get; set; } + public bool UseEmbeddedServer => string.IsNullOrWhiteSpace(ConnectionString); + public string LogPath { get; set; } + public string LogsMode { get; set; } = LogsModeDefault; + public string DatabaseName { get; set; } = DatabaseNameDefault; + + public const string DatabaseNameDefault = "audit"; + public const int DatabaseMaintenancePortDefault = 33334; + public const int ExpirationProcessTimerInSecondsDefault = 600; + public const string LogsModeDefault = "Information"; +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs new file mode 100644 index 0000000000..d94e267f04 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs @@ -0,0 +1,109 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System; + using System.Collections.Generic; + using System.Threading.Tasks; + using Raven.Client.Documents; + using Raven.Client.Documents.Commands; + using Raven.Client.Documents.Linq; + using Raven.Client.Documents.Session; + using ServiceControl.Contracts.CustomChecks; + using ServiceControl.Persistence; + using ServiceControl.Persistence.Infrastructure; + + class RavenDbCustomCheckDataStore : ICustomChecksDataStore + { + public RavenDbCustomCheckDataStore(IDocumentStore store) + { + this.store = store; + } + + public async Task UpdateCustomCheckStatus(CustomCheckDetail detail) + { + var status = CheckStateChange.Unchanged; + var id = MakeId(detail.GetDeterministicId()); + + using (var session = store.OpenAsyncSession()) + { + var customCheck = await session.LoadAsync(id); + + if (customCheck == null || + (customCheck.Status == Status.Fail && !detail.HasFailed) || + (customCheck.Status == Status.Pass && detail.HasFailed)) + { + customCheck ??= new CustomCheck { Id = id }; + + status = CheckStateChange.Changed; + } + + customCheck.CustomCheckId = detail.CustomCheckId; + customCheck.Category = detail.Category; + customCheck.Status = detail.HasFailed ? Status.Fail : Status.Pass; + customCheck.ReportedAt = detail.ReportedAt; + customCheck.FailureReason = detail.FailureReason; + customCheck.OriginatingEndpoint = detail.OriginatingEndpoint; + await session.StoreAsync(customCheck); + await session.SaveChangesAsync(); + } + + return status; + } + + static string MakeId(Guid id) + { + return $"CustomChecks/{id}"; + } + + public async Task>> GetStats(PagingInfo paging, string status = null) + { + using var session = store.OpenAsyncSession(); + var query = + session.Query().Statistics(out var stats); + + query = AddStatusFilter(query, status); + + var results = await query + .Paging(paging) + .ToListAsync(); + + return new QueryResult>(results, new QueryStatsInfo($"{stats.ResultEtag}", stats.TotalResults, stats.IsStale)); + } + + public async Task DeleteCustomCheck(Guid id) + { + var documentId = MakeId(id); + using var session = store.OpenAsyncSession(new SessionOptions { NoTracking = true, NoCaching = true }); + await session.Advanced.RequestExecutor.ExecuteAsync(new DeleteDocumentCommand(documentId, null), session.Advanced.Context); + } + + public async Task GetNumberOfFailedChecks() + { + using var session = store.OpenAsyncSession(); + var failedCustomCheckCount = await session.Query().CountAsync(p => p.Status == Status.Fail); + + return failedCustomCheckCount; + } + + static IRavenQueryable AddStatusFilter(IRavenQueryable query, string status) + { + if (status == null) + { + return query; + } + + if (status == "fail") + { + query = query.Where(c => c.Status == Status.Fail); + } + + if (status == "pass") + { + query = query.Where(c => c.Status == Status.Pass); + } + + return query; + } + + readonly IDocumentStore store; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbEmbeddedPersistenceLifecycle.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbEmbeddedPersistenceLifecycle.cs new file mode 100644 index 0000000000..7df1fa9489 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbEmbeddedPersistenceLifecycle.cs @@ -0,0 +1,46 @@ +namespace ServiceControl.Persistence.RavenDb5 +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using Raven.Client.Documents; + using ServiceControl.Persistence; + + class RavenDbEmbeddedPersistenceLifecycle : IPersistenceLifecycle + { + public RavenDbEmbeddedPersistenceLifecycle(RavenDBPersisterSettings databaseConfiguration) + { + this.databaseConfiguration = databaseConfiguration; + } + + public IDocumentStore GetDocumentStore() + { + if (documentStore == null) + { + throw new InvalidOperationException("Document store is not available until the persistence have been started"); + } + + return documentStore; + } + + public async Task Start(CancellationToken cancellationToken) + { + database = EmbeddedDatabase.Start(databaseConfiguration); + + documentStore = await database.Connect(cancellationToken); + } + + public Task Stop(CancellationToken cancellationToken) + { + documentStore?.Dispose(); + database?.Dispose(); + + return Task.CompletedTask; + } + + IDocumentStore documentStore; + EmbeddedDatabase database; + + readonly RavenDBPersisterSettings databaseConfiguration; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbExternalPersistenceLifecycle.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbExternalPersistenceLifecycle.cs new file mode 100644 index 0000000000..910afddeb1 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbExternalPersistenceLifecycle.cs @@ -0,0 +1,64 @@ +namespace ServiceControl.Persistence.RavenDb5 +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using Raven.Client.Documents; + using Raven.Client.Documents.Conventions; + using ServiceControl.Persistence; + + class RavenDbExternalPersistenceLifecycle : IPersistenceLifecycle + { + public RavenDbExternalPersistenceLifecycle(RavenDBPersisterSettings settings) + { + this.settings = settings; + } + + public IDocumentStore GetDocumentStore() + { + if (documentStore == null) + { + throw new InvalidOperationException("Document store is not available until the persistence have been started"); + } + + return documentStore; + } + + public async Task Start(CancellationToken cancellationToken) + { + var store = new DocumentStore + { + Database = settings.DatabaseName, + Urls = new[] { settings.ConnectionString }, + Conventions = new DocumentConventions + { + SaveEnumsAsIntegers = true + } + }; + + //TODO: copied from Audit, not sure if needed (never assigned). Check and remove + //if (settings.FindClrType != null) + //{ + // store.Conventions.FindClrType += settings.FindClrType; + //} + + store.Initialize(); + + documentStore = store; + + var databaseSetup = new DatabaseSetup(settings); + await databaseSetup.Execute(store, cancellationToken).ConfigureAwait(false); + } + + public Task Stop(CancellationToken cancellationToken) + { + documentStore?.Dispose(); + + return Task.CompletedTask; + } + + IDocumentStore documentStore; + + readonly RavenDBPersisterSettings settings; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbInstaller.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbInstaller.cs new file mode 100644 index 0000000000..296df56084 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbInstaller.cs @@ -0,0 +1,22 @@ +namespace ServiceControl.Persistence.RavenDb5 +{ + using System.Threading; + using System.Threading.Tasks; + using ServiceControl.Persistence; + + class RavenDbInstaller : IPersistenceInstaller + { + public RavenDbInstaller(IPersistenceLifecycle lifecycle) + { + this.lifecycle = lifecycle; + } + + public async Task Install(CancellationToken cancellationToken) + { + await lifecycle.Start(cancellationToken); + await lifecycle.Stop(cancellationToken); + } + + readonly IPersistenceLifecycle lifecycle; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbMonitoringDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbMonitoringDataStore.cs new file mode 100644 index 0000000000..3d6613bda2 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbMonitoringDataStore.cs @@ -0,0 +1,119 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System; + using System.Collections.Generic; + using System.Threading.Tasks; + using ServiceControl.Operations; + using ServiceControl.Persistence; + using Raven.Client.Documents; + + class RavenDbMonitoringDataStore : IMonitoringDataStore + { + public RavenDbMonitoringDataStore(IDocumentStore store) + { + this.store = store; + } + + public async Task CreateIfNotExists(EndpointDetails endpoint) + { + var id = endpoint.GetDeterministicId(); + + using var session = store.OpenAsyncSession(); + + var knownEndpoint = await session.LoadAsync(id.ToString()); + + if (knownEndpoint != null) + { + return; + } + + knownEndpoint = new KnownEndpoint + { + Id = id, + EndpointDetails = endpoint, + HostDisplayName = endpoint.Host, + Monitored = false + }; + + await session.StoreAsync(knownEndpoint); + + await session.SaveChangesAsync(); + } + + public async Task CreateOrUpdate(EndpointDetails endpoint, IEndpointInstanceMonitoring endpointInstanceMonitoring) + { + var id = endpoint.GetDeterministicId(); + + using var session = store.OpenAsyncSession(); + + var knownEndpoint = await session.LoadAsync(id.ToString()); + + if (knownEndpoint == null) + { + knownEndpoint = new KnownEndpoint + { + Id = id, + EndpointDetails = endpoint, + HostDisplayName = endpoint.Host, + Monitored = true + }; + + await session.StoreAsync(knownEndpoint); + } + else + { + knownEndpoint.Monitored = endpointInstanceMonitoring.IsMonitored(id); + } + + await session.SaveChangesAsync(); + } + + public async Task UpdateEndpointMonitoring(EndpointDetails endpoint, bool isMonitored) + { + var id = endpoint.GetDeterministicId(); + + using var session = store.OpenAsyncSession(); + + var knownEndpoint = await session.LoadAsync(id.ToString()); + + if (knownEndpoint != null) + { + knownEndpoint.Monitored = isMonitored; + + await session.SaveChangesAsync(); + } + } + + public async Task WarmupMonitoringFromPersistence(IEndpointInstanceMonitoring endpointInstanceMonitoring) + { + using var session = store.OpenAsyncSession(); + await using var endpointsEnumerator = await session.Advanced.StreamAsync(session.Query()); + + while (await endpointsEnumerator.MoveNextAsync()) + { + var endpoint = endpointsEnumerator.Current.Document; + + endpointInstanceMonitoring.DetectEndpointFromPersistentStore(endpoint.EndpointDetails, endpoint.Monitored); + } + } + + public async Task Delete(Guid endpointId) + { + using var session = store.OpenAsyncSession(); + session.Delete(KnownEndpointIdGenerator.MakeDocumentId(endpointId)); + await session.SaveChangesAsync(); + } + + public async Task> GetAllKnownEndpoints() + { + using var session = store.OpenAsyncSession(); + + var knownEndpoints = await session.Query() + .ToListAsync(); + + return knownEndpoints.ToArray(); + } + + readonly IDocumentStore store; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs new file mode 100644 index 0000000000..58bf61e6ee --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs @@ -0,0 +1,106 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using MessageRedirects; + using Microsoft.Extensions.DependencyInjection; + using Persistence.Recoverability; + using RavenDb5; + using Recoverability; + using ServiceControl.CustomChecks; + using ServiceControl.Infrastructure.RavenDB.Subscriptions; + using ServiceControl.MessageFailures; + using ServiceControl.Operations; + using ServiceControl.Operations.BodyStorage; + using ServiceControl.Operations.BodyStorage.RavenAttachments; + using ServiceControl.Persistence.MessageRedirects; + using ServiceControl.Persistence.UnitOfWork; + using ServiceControl.Recoverability; + + class RavenDbPersistence : IPersistence + { + public RavenDbPersistence(RavenDBPersisterSettings settings) + { + this.settings = settings; + } + + public void Configure(IServiceCollection serviceCollection) + { + if (settings.MaintenanceMode) + { + return; + } + + serviceCollection.AddSingleton(settings); + + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddUnitOfWorkFactory(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(p => p.GetRequiredService()); + serviceCollection.AddHostedService(p => p.GetRequiredService()); + + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(p => p.GetRequiredService()); + serviceCollection.AddHostedService(p => p.GetRequiredService()); + + serviceCollection.AddCustomCheck(); + serviceCollection.AddCustomCheck(); + serviceCollection.AddCustomCheck(); + serviceCollection.AddCustomCheck(); + + serviceCollection.AddSingleton(); + + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + } + + public void ConfigureLifecycle(IServiceCollection serviceCollection) + { + if (settings.UseEmbeddedServer) + { + var embedded = new RavenDbEmbeddedPersistenceLifecycle(settings); + + serviceCollection.AddSingleton(embedded); + serviceCollection.AddSingleton(_ => embedded.GetDocumentStore()); + + return; + } + + var external = new RavenDbExternalPersistenceLifecycle(settings); + + serviceCollection.AddSingleton(external); + serviceCollection.AddSingleton(_ => external.GetDocumentStore()); + } + + public IPersistenceInstaller CreateInstaller() + { + var serviceCollection = new ServiceCollection(); + + serviceCollection.AddSingleton(settings); + ConfigureLifecycle(serviceCollection); + + var lifecycle = serviceCollection.BuildServiceProvider().GetRequiredService(); + + return new RavenDbInstaller(lifecycle); + } + + readonly RavenDBPersisterSettings settings; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs new file mode 100644 index 0000000000..c7a0f2d8fe --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs @@ -0,0 +1,81 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System; + using ServiceControl.Operations; + + class RavenDbPersistenceConfiguration : IPersistenceConfiguration + { + public const string DataSpaceRemainingThresholdKey = "DataSpaceRemainingThreshold"; + const string AuditRetentionPeriodKey = "AuditRetentionPeriod"; + const string ErrorRetentionPeriodKey = "ErrorRetentionPeriod"; + const string EventsRetentionPeriodKey = "EventsRetentionPeriod"; + const string ExternalIntegrationsDispatchingBatchSizeKey = "ExternalIntegrationsDispatchingBatchSize"; + const string MaintenanceModeKey = "MaintenanceMode"; + + public PersistenceSettings CreateSettings(Func tryReadSetting) + { + T GetRequiredSetting(string key) + { + var (exists, value) = tryReadSetting(key, typeof(T)); + + if (exists) + { + return (T)value; + } + + throw new Exception($"Setting {key} of type {typeof(T)} is required"); + } + + T GetSetting(string key, T defaultValue) + { + var (exists, value) = tryReadSetting(key, typeof(T)); + + if (exists) + { + return (T)value; + } + else + { + return defaultValue; + } + } + + var settings = new RavenDBPersisterSettings + { + ConnectionString = GetSetting(RavenBootstrapper.ConnectionStringKey, default), + DatabaseName = GetSetting(RavenBootstrapper.DatabaseNameKey, RavenDBPersisterSettings.DatabaseNameDefault), + DatabasePath = GetSetting(RavenBootstrapper.DatabasePathKey, default), + HostName = GetSetting(RavenBootstrapper.HostNameKey, "localhost"), + DatabaseMaintenancePort = GetSetting(RavenBootstrapper.DatabaseMaintenancePortKey, RavenDBPersisterSettings.DatabaseMaintenancePortDefault), + ExposeRavenDB = GetSetting(RavenBootstrapper.ExposeRavenDBKey, false), + ExpirationProcessTimerInSeconds = GetSetting(RavenBootstrapper.ExpirationProcessTimerInSecondsKey, 600), + RunInMemory = GetSetting(RavenBootstrapper.RunInMemoryKey, false), + MinimumStorageLeftRequiredForIngestion = GetSetting(RavenBootstrapper.MinimumStorageLeftRequiredForIngestionKey, CheckMinimumStorageRequiredForIngestion.MinimumStorageLeftRequiredForIngestionDefault), + DataSpaceRemainingThreshold = GetSetting(DataSpaceRemainingThresholdKey, CheckFreeDiskSpace.DataSpaceRemainingThresholdDefault), + ErrorRetentionPeriod = GetRequiredSetting(ErrorRetentionPeriodKey), + EventsRetentionPeriod = GetSetting(EventsRetentionPeriodKey, TimeSpan.FromDays(14)), + AuditRetentionPeriod = GetSetting(AuditRetentionPeriodKey, TimeSpan.Zero), + ExternalIntegrationsDispatchingBatchSize = GetSetting(ExternalIntegrationsDispatchingBatchSizeKey, 100), + MaintenanceMode = GetSetting(MaintenanceModeKey, false), + LogPath = GetRequiredSetting(RavenBootstrapper.LogsPathKey), + LogsMode = GetSetting(RavenBootstrapper.LogsModeKey, RavenDBPersisterSettings.LogsModeDefault) + }; + + CheckFreeDiskSpace.Validate(settings); + CheckMinimumStorageRequiredForIngestion.Validate(settings); + return settings; + } + + public IPersistence Create(PersistenceSettings settings) + { + var specificSettings = (RavenDBPersisterSettings)settings; + + //var documentStore = new EmbeddableDocumentStore(); + //RavenBootstrapper.Configure(documentStore, specificSettings); + + //var ravenStartup = new RavenStartup(); + + return new RavenDbPersistence(specificSettings); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenLicense.json b/src/ServiceControl.Persistence.RavenDb5/RavenLicense.json new file mode 100644 index 0000000000..9afe8a04d1 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenLicense.json @@ -0,0 +1,16 @@ +{ +"Id": "64c6a174-3f3a-4e7d-ac5d-b3eedd801460", +"Name": "ParticularNservicebus (Israel)", +"Keys": [ +"kWznr3iTRWkzuYjFJH8IvS1LY", +"t5GJId7H/h5i4YjW2Z7YPgUoR", +"igl/wrjx4y6pmXcChQRnTu1TK", +"Q1scjkLMB2aaH9VZ5G2s4E0gz", +"08GaHnSOHpRVz6SgjFGcAqnEb", +"c0ZhNNOGsBcrOVx+KOq7+ggHs", +"MtKI8e8mCRcgiaTOkPURpAAyU", +"GKEkFCisMLQ4vMAcREjM0NRYX", +"GhufAh8AnwIgAJ8CIwBBMkMMR", +"AY4OTw9Pp8CISBiP1g=" +] +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs b/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs new file mode 100644 index 0000000000..4e9988311e --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs @@ -0,0 +1,406 @@ +namespace ServiceControl.Persistence +{ + using System; + using System.Collections.Generic; + using System.Globalization; + using System.Linq; + using System.Linq.Expressions; + using System.Net.Http; + using Raven.Client.Documents.Linq; + using Raven.Client.Documents.Session; + using ServiceControl.MessageFailures; + using ServiceControl.Persistence.Infrastructure; + + static class RavenQueryExtensions + { + public static IRavenQueryable IncludeSystemMessagesWhere( + this IRavenQueryable source, bool includeSystemMessages) + { + if (!includeSystemMessages) + { + return source.Where(m => !m.IsSystemMessage); + } + + return source; + } + + public static IQueryable Paging(this IOrderedQueryable source, PagingInfo pagingInfo) + => source + .Skip(pagingInfo.Offset) + .Take(pagingInfo.PageSize); + + public static IRavenQueryable Sort(this IRavenQueryable source, SortInfo sortInfo) + where TSource : MessagesViewIndex.SortAndFilterOptions + { + Expression> keySelector; + switch (sortInfo.Sort) + { + case "id": + case "message_id": + keySelector = m => m.MessageId; + break; + + case "message_type": + keySelector = m => m.MessageType; + break; + + case "critical_time": + keySelector = m => m.CriticalTime; + break; + + case "delivery_time": + keySelector = m => m.DeliveryTime; + break; + + case "processing_time": + keySelector = m => m.ProcessingTime; + break; + + case "processed_at": + keySelector = m => m.ProcessedAt; + break; + + case "status": + keySelector = m => m.Status; + break; + + default: + keySelector = m => m.TimeSent; + break; + } + + if (sortInfo.Direction == "asc") + { + return source.OrderBy(keySelector); + } + + return source.OrderByDescending(keySelector); + } + + public static IAsyncDocumentQuery Paging(this IAsyncDocumentQuery source, PagingInfo pagingInfo) + { + var maxResultsPerPage = pagingInfo.PageSize; + if (maxResultsPerPage < 1) + { + maxResultsPerPage = 50; + } + + var page = pagingInfo.Page; + if (page < 1) + { + page = 1; + } + + var skipResults = (page - 1) * maxResultsPerPage; + + return source.Skip(skipResults) + .Take(maxResultsPerPage); + } + + public static IAsyncDocumentQuery Sort(this IAsyncDocumentQuery source, SortInfo sortInfo) + { + var descending = true; + + var direction = sortInfo.Direction; + if (direction == "asc") + { + descending = false; + } + + string keySelector; + + var sort = sortInfo.Sort; + if (!AsyncDocumentQuerySortOptions.Contains(sort)) + { + sort = "time_sent"; + } + + switch (sort) + { + case "id": + case "message_id": + keySelector = "MessageId"; + break; + + case "message_type": + keySelector = "MessageType"; + break; + + case "status": + keySelector = "Status"; + break; + + case "modified": + keySelector = "LastModified"; + break; + + case "time_of_failure": + keySelector = "TimeOfFailure"; + break; + + default: + keySelector = "TimeSent"; + break; + } + + return source.AddOrder(keySelector, descending); + } + + + public static IAsyncDocumentQuery FilterByStatusWhere(this IAsyncDocumentQuery source, string status) + { + if (status == null) + { + return source; + } + + var filters = status.Replace(" ", string.Empty).Split(','); + var excludes = new List(); + var includes = new List(); + + foreach (var filter in filters) + { + FailedMessageStatus failedMessageStatus; + + if (filter.StartsWith("-")) + { + if (Enum.TryParse(filter.Substring(1), true, out failedMessageStatus)) + { + excludes.Add((int)failedMessageStatus); + } + + continue; + } + + if (Enum.TryParse(filter, true, out failedMessageStatus)) + { + includes.Add((int)failedMessageStatus); + } + } + + if (includes.Any()) + { + source.WhereIn("Status", includes.Cast()); + } + + foreach (var exclude in excludes) + { + source.WhereNotEquals("Status", exclude); + } + + return source; + } + + + public static IAsyncDocumentQuery FilterByLastModifiedRange(this IAsyncDocumentQuery source, string modified) + { + if (modified == null) + { + return source; + } + + var filters = modified.Split(SplitChars, StringSplitOptions.None); + if (filters.Length != 2) + { + throw new Exception("Invalid modified date range, dates need to be in ISO8601 format and it needs to be a range eg. 2016-03-11T00:27:15.474Z...2016-03-16T03:27:15.474Z"); + } + + try + { + var from = DateTime.Parse(filters[0], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); + var to = DateTime.Parse(filters[1], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); + + source.AndAlso(); + source.WhereBetween("LastModified", from.Ticks, to.Ticks); + } + catch (Exception) + { + throw new Exception("Invalid modified date range, dates need to be in ISO8601 format and it needs to be a range eg. 2016-03-11T00:27:15.474Z...2016-03-16T03:27:15.474Z"); + } + + return source; + } + + public static IAsyncDocumentQuery FilterByQueueAddress(this IAsyncDocumentQuery source, string queueAddress) + { + if (string.IsNullOrWhiteSpace(queueAddress)) + { + return source; + } + + source.AndAlso(); + source.WhereEquals("QueueAddress", queueAddress.ToLowerInvariant()); + + return source; + } + + public static IRavenQueryable IncludeSystemMessagesWhere( + this IRavenQueryable source, HttpRequestMessage request) + { + var includeSystemMessages = request.GetQueryStringValue("include_system_messages", false); + return !includeSystemMessages ? source.Where(m => !m.IsSystemMessage) : source; + } + + public static IOrderedQueryable Paging(this IOrderedQueryable source, HttpRequestMessage request) + { + var maxResultsPerPage = request.GetQueryStringValue("per_page", 50); + if (maxResultsPerPage < 1) + { + maxResultsPerPage = 50; + } + + var page = request.GetQueryStringValue("page", 1); + + if (page < 1) + { + page = 1; + } + + var skipResults = (page - 1) * maxResultsPerPage; + + return (IOrderedQueryable)source.Skip(skipResults) + .Take(maxResultsPerPage); + } + + public static IRavenQueryable Paging(this IRavenQueryable source, HttpRequestMessage request) + { + var maxResultsPerPage = request.GetQueryStringValue("per_page", 50); + if (maxResultsPerPage < 1) + { + maxResultsPerPage = 50; + } + + var page = request.GetQueryStringValue("page", 1); + + if (page < 1) + { + page = 1; + } + + var skipResults = (page - 1) * maxResultsPerPage; + + return source.Skip(skipResults) + .Take(maxResultsPerPage); + } + + public static IRavenQueryable Sort(this IRavenQueryable source, HttpRequestMessage request, + Expression> defaultKeySelector = null, string defaultSortDirection = "desc") + where TSource : MessagesViewIndex.SortAndFilterOptions + { + var direction = request.GetQueryStringValue("direction", defaultSortDirection); + if (direction != "asc" && direction != "desc") + { + direction = defaultSortDirection; + } + + Expression> keySelector; + var sort = request.GetQueryStringValue("sort", "time_sent"); + if (!RavenQueryableSortOptions.Contains(sort)) + { + sort = "time_sent"; + } + + switch (sort) + { + case "id": + case "message_id": + keySelector = m => m.MessageId; + break; + + case "message_type": + keySelector = m => m.MessageType; + break; + + case "critical_time": + keySelector = m => m.CriticalTime; + break; + + case "delivery_time": + keySelector = m => m.DeliveryTime; + break; + + case "processing_time": + keySelector = m => m.ProcessingTime; + break; + + case "processed_at": + keySelector = m => m.ProcessedAt; + break; + + case "status": + keySelector = m => m.Status; + break; + + default: + if (defaultKeySelector == null) + { + keySelector = m => m.TimeSent; + } + else + { + keySelector = defaultKeySelector; + } + + break; + } + + if (direction == "asc") + { + return source.OrderBy(keySelector); + } + + return source.OrderByDescending(keySelector); + } + + public static T GetQueryStringValue(this HttpRequestMessage request, string key, T defaultValue = default) + { + Dictionary queryStringDictionary; + if (!request.Properties.TryGetValue("QueryStringAsDictionary", out var dictionaryAsObject)) + { + queryStringDictionary = request.GetQueryNameValuePairs().ToDictionary(x => x.Key, x => x.Value, StringComparer.OrdinalIgnoreCase); + request.Properties["QueryStringAsDictionary"] = queryStringDictionary; + } + else + { + queryStringDictionary = (Dictionary)dictionaryAsObject; + } + + queryStringDictionary.TryGetValue(key, out var value); + if (string.IsNullOrEmpty(value)) + { + return defaultValue; + } + + return (T)Convert.ChangeType(value, typeof(T)); + } + + static HashSet AsyncDocumentQuerySortOptions = new HashSet + { + "id", + "message_id", + "message_type", + "time_sent", + "status", + "modified", + "time_of_failure" + }; + + static HashSet RavenQueryableSortOptions = new HashSet + { + "processed_at", + "id", + "message_type", + "time_sent", + "critical_time", + "delivery_time", + "processing_time", + "status", + "message_id" + }; + + static string[] SplitChars = + { + "..." + }; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenQueryStatisticsExtensions.cs b/src/ServiceControl.Persistence.RavenDb5/RavenQueryStatisticsExtensions.cs new file mode 100644 index 0000000000..fa5746ccfa --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenQueryStatisticsExtensions.cs @@ -0,0 +1,18 @@ +namespace ServiceControl.Persistence +{ + using Raven.Client.Documents.Session; + using ServiceControl.Persistence.Infrastructure; + + static class RavenQueryStatisticsExtensions + { + public static QueryStatsInfo ToQueryStatsInfo(this QueryStatistics stats) + { + return new QueryStatsInfo($"{stats.ResultEtag}", stats.TotalResults, stats.IsStale); + } + + public static QueryStatsInfo ToQueryStatsInfo(this Raven.Client.Documents.Queries.QueryResult queryResult) + { + return new QueryStatsInfo(queryResult.ResultEtag.ToString(), queryResult.TotalResults, queryResult.IsStale); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenStartup.cs b/src/ServiceControl.Persistence.RavenDb5/RavenStartup.cs new file mode 100644 index 0000000000..3a535665ec --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenStartup.cs @@ -0,0 +1,20 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System.Threading.Tasks; + using NServiceBus.Logging; + using Raven.Client; + using Raven.Client.Documents; + using Raven.Client.Documents.Indexes; + + class RavenStartup + { + public async Task CreateIndexesAsync(IDocumentStore documentStore) + { + Logger.Info("Index creation started"); + await IndexCreation.CreateIndexesAsync(typeof(RavenBootstrapper).Assembly, documentStore); + Logger.Info("Index creation complete"); + } + + static readonly ILog Logger = LogManager.GetLogger(); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveBatch.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveBatch.cs new file mode 100644 index 0000000000..7dfc624019 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveBatch.cs @@ -0,0 +1,15 @@ +namespace ServiceControl.Recoverability +{ + using System.Collections.Generic; + + class ArchiveBatch //raven + { + public string Id { get; set; } + public List DocumentIds { get; set; } = new List(); + + public static string MakeId(string requestId, ArchiveType archiveType, int batchNumber) + { + return $"ArchiveOperations/{(int)archiveType}/{requestId}/{batchNumber}"; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveDocumentManager.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveDocumentManager.cs new file mode 100644 index 0000000000..22cd9ba04b --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveDocumentManager.cs @@ -0,0 +1,157 @@ +namespace ServiceControl.Recoverability +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading.Tasks; + using MessageFailures; + using NServiceBus.Logging; + using Raven.Client.Documents; + using Raven.Client.Documents.Commands.Batches; + using Raven.Client.Documents.Operations; + using Raven.Client.Documents.Session; + + class ArchiveDocumentManager + { + public Task LoadArchiveOperation(IAsyncDocumentSession session, string groupId, ArchiveType archiveType) + { + return session.LoadAsync(ArchiveOperation.MakeId(groupId, archiveType)); + } + + public async Task CreateArchiveOperation(IAsyncDocumentSession session, string groupId, ArchiveType archiveType, int numberOfMessages, string groupName, int batchSize) + { + var operation = new ArchiveOperation + { + Id = ArchiveOperation.MakeId(groupId, archiveType), + RequestId = groupId, + ArchiveType = archiveType, + TotalNumberOfMessages = numberOfMessages, + NumberOfMessagesArchived = 0, + Started = DateTime.Now, + GroupName = groupName, + NumberOfBatches = (int)Math.Ceiling(numberOfMessages / (float)batchSize), + CurrentBatch = 0 + }; + + await session.StoreAsync(operation); + + var documentCount = 0; + var indexQuery = session.Query(new FailedMessages_ByGroup().IndexName); + + var docQuery = indexQuery + .Where(failure => failure.FailureGroupId == groupId) + .Where(failure => failure.Status == FailedMessageStatus.Unresolved) + .Select(document => document.Id); + + var docs = await StreamResults(session, docQuery); + + var batches = docs + .GroupBy(d => documentCount++ / batchSize); + + foreach (var batch in batches) + { + var archiveBatch = new ArchiveBatch + { + Id = ArchiveBatch.MakeId(groupId, archiveType, batch.Key), + DocumentIds = batch.ToList() + }; + + await session.StoreAsync(archiveBatch); + } + + return operation; + } + + async Task> StreamResults(IAsyncDocumentSession session, IQueryable query) + { + var results = new List(); + await using (var enumerator = await session.Advanced.StreamAsync(query)) + { + while (await enumerator.MoveNextAsync()) + { + results.Add(enumerator.Current.Document); + } + } + + return results; + } + + public Task GetArchiveBatch(IAsyncDocumentSession session, string archiveOperationId, int batchNumber) + { + return session.LoadAsync($"{archiveOperationId}/{batchNumber}"); + } + + public async Task GetGroupDetails(IAsyncDocumentSession session, string groupId) + { + var group = await session.Query() + .FirstOrDefaultAsync(x => x.Id == groupId); + + return new GroupDetails + { + NumberOfMessagesInGroup = group?.Count ?? 0, + GroupName = group?.Title ?? "Undefined" + }; + } + + public void ArchiveMessageGroupBatch(IAsyncDocumentSession session, ArchiveBatch batch) + { + var patchCommands = batch?.DocumentIds.Select(documentId => new PatchCommandData(documentId, null, patchRequest)); + + if (patchCommands != null) + { + session.Advanced.Defer(patchCommands.ToArray()); + session.Advanced.Defer(new DeleteCommandData(batch.Id, null)); + } + } + + public async Task WaitForIndexUpdateOfArchiveOperation(IDocumentStore store, string requestId, TimeSpan timeToWait) + { + using (var session = store.OpenAsyncSession()) + { + var indexQuery = session.Query(new FailedMessages_ByGroup().IndexName) + .Customize(x => x.WaitForNonStaleResults(timeToWait)); + + var docQuery = indexQuery + .Where(failure => failure.FailureGroupId == requestId) + .Select(document => document.Id); + + try + { + await docQuery.AnyAsync(); + + return true; + } + catch + { + return false; + } + } + } + + public Task UpdateArchiveOperation(IAsyncDocumentSession session, ArchiveOperation archiveOperation) + { + return session.StoreAsync(archiveOperation); + } + + public async Task RemoveArchiveOperation(IDocumentStore store, ArchiveOperation archiveOperation) + { + using (var session = store.OpenAsyncSession()) + { + session.Advanced.Defer(new DeleteCommandData(archiveOperation.Id, null)); + await session.SaveChangesAsync(); + + logger.Info($"Removing ArchiveOperation {archiveOperation.Id} completed"); + } + } + + static PatchRequest patchRequest = new PatchRequest { Script = @$"this.Status = {(int)FailedMessageStatus.Archived}" }; + + public class GroupDetails + { + public string GroupName { get; set; } + public int NumberOfMessagesInGroup { get; set; } + } + + static ILog logger = LogManager.GetLogger(); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveOperation.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveOperation.cs new file mode 100644 index 0000000000..8fedc64233 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveOperation.cs @@ -0,0 +1,21 @@ +namespace ServiceControl.Recoverability +{ + using System; + + class ArchiveOperation // raven + { + public string Id { get; set; } + public string RequestId { get; set; } + public string GroupName { get; set; } + public ArchiveType ArchiveType { get; set; } + public int TotalNumberOfMessages { get; set; } + public int NumberOfMessagesArchived { get; set; } + public DateTime Started { get; set; } + public int NumberOfBatches { get; set; } + public int CurrentBatch { get; set; } + public static string MakeId(string requestId, ArchiveType archiveType) + { + return $"ArchiveOperations/{(int)archiveType}/{requestId}"; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveOperationExtensions.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveOperationExtensions.cs new file mode 100644 index 0000000000..8ec9667654 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveOperationExtensions.cs @@ -0,0 +1,39 @@ +namespace ServiceControl.Persistence.RavenDb.Recoverability +{ + using ServiceControl.Recoverability; + + static class ArchiveOperationExtensions + { + public static ArchiveOperation ToArchiveOperation(this InMemoryArchive a) + { + return new ArchiveOperation + { + ArchiveType = a.ArchiveType, + GroupName = a.GroupName, + Id = ArchiveOperation.MakeId(a.RequestId, a.ArchiveType), + NumberOfMessagesArchived = a.NumberOfMessagesArchived, + RequestId = a.RequestId, + Started = a.Started, + TotalNumberOfMessages = a.TotalNumberOfMessages, + NumberOfBatches = a.NumberOfBatches, + CurrentBatch = a.CurrentBatch + }; + } + + public static UnarchiveOperation ToUnarchiveOperation(this InMemoryUnarchive u) + { + return new UnarchiveOperation + { + ArchiveType = u.ArchiveType, + GroupName = u.GroupName, + Id = UnarchiveOperation.MakeId(u.RequestId, u.ArchiveType), + NumberOfMessagesUnarchived = u.NumberOfMessagesUnarchived, + RequestId = u.RequestId, + Started = u.Started, + TotalNumberOfMessages = u.TotalNumberOfMessages, + NumberOfBatches = u.NumberOfBatches, + CurrentBatch = u.CurrentBatch + }; + } + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchivingManager.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchivingManager.cs new file mode 100644 index 0000000000..2526acdbfb --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchivingManager.cs @@ -0,0 +1,112 @@ +namespace ServiceControl.Recoverability +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading.Tasks; + using Infrastructure.DomainEvents; + + class ArchivingManager + { + public ArchivingManager(IDomainEvents domainEvents, OperationsManager operationsManager) + { + this.domainEvents = domainEvents; + this.operationsManager = operationsManager; + } + + public bool IsArchiveInProgressFor(string requestId) + { + return operationsManager.ArchiveOperations.Keys.Any(key => key.EndsWith($"/{requestId}")); + } + + internal IEnumerable GetArchivalOperations() + { + return operationsManager.ArchiveOperations.Values; + } + + public bool IsOperationInProgressFor(string requestId, ArchiveType archiveType) + { + return operationsManager.IsOperationInProgressFor(requestId, archiveType); + } + + InMemoryArchive GetOrCreate(ArchiveType archiveType, string requestId) + { + if (!operationsManager.ArchiveOperations.TryGetValue(InMemoryArchive.MakeId(requestId, archiveType), out var summary)) + { + summary = new InMemoryArchive(requestId, archiveType, domainEvents); + operationsManager.ArchiveOperations[InMemoryArchive.MakeId(requestId, archiveType)] = summary; + } + + return summary; + } + + public Task StartArchiving(ArchiveOperation archiveOperation) + { + var summary = GetOrCreate(archiveOperation.ArchiveType, archiveOperation.RequestId); + + summary.TotalNumberOfMessages = archiveOperation.TotalNumberOfMessages; + summary.NumberOfMessagesArchived = archiveOperation.NumberOfMessagesArchived; + summary.Started = archiveOperation.Started; + summary.GroupName = archiveOperation.GroupName; + summary.NumberOfBatches = archiveOperation.NumberOfBatches; + summary.CurrentBatch = archiveOperation.CurrentBatch; + + return summary.Start(); + } + + public Task StartArchiving(string requestId, ArchiveType archiveType) + { + var summary = GetOrCreate(archiveType, requestId); + + summary.TotalNumberOfMessages = 0; + summary.NumberOfMessagesArchived = 0; + summary.Started = DateTime.Now; + summary.GroupName = "Undefined"; + summary.NumberOfBatches = 0; + summary.CurrentBatch = 0; + + return summary.Start(); + } + + public InMemoryArchive GetStatusForArchiveOperation(string requestId, ArchiveType archiveType) + { + operationsManager.ArchiveOperations.TryGetValue(InMemoryArchive.MakeId(requestId, archiveType), out var summary); + + return summary; + } + + public Task BatchArchived(string requestId, ArchiveType archiveType, int numberOfMessagesArchivedInBatch) + { + var summary = GetOrCreate(archiveType, requestId); + + return summary.BatchArchived(numberOfMessagesArchivedInBatch); + } + + public Task ArchiveOperationFinalizing(string requestId, ArchiveType archiveType) + { + var summary = GetOrCreate(archiveType, requestId); + return summary.FinalizeArchive(); + } + + public Task ArchiveOperationCompleted(string requestId, ArchiveType archiveType) + { + var summary = GetOrCreate(archiveType, requestId); + return summary.Complete(); + } + + public void DismissArchiveOperation(string requestId, ArchiveType archiveType) + { + RemoveArchiveOperation(requestId, archiveType); + } + + void RemoveArchiveOperation(string requestId, ArchiveType archiveType) + { + operationsManager.ArchiveOperations.Remove(InMemoryArchive.MakeId(requestId, archiveType)); + } + + IDomainEvents domainEvents; + + OperationsManager operationsManager; + + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/MessageArchiver.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/MessageArchiver.cs new file mode 100644 index 0000000000..bdeba84603 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/MessageArchiver.cs @@ -0,0 +1,243 @@ +namespace ServiceControl.Persistence.RavenDb.Recoverability +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading.Tasks; + using NServiceBus.Logging; + using Raven.Client; + using Raven.Client.Documents; + using ServiceControl.Infrastructure.DomainEvents; + using ServiceControl.Persistence.Recoverability; + using ServiceControl.Recoverability; + + class MessageArchiver : IArchiveMessages + { + public MessageArchiver(IDocumentStore store, OperationsManager operationsManager, IDomainEvents domainEvents) + { + this.store = store; + this.domainEvents = domainEvents; + this.operationsManager = operationsManager; + + archiveDocumentManager = new ArchiveDocumentManager(); + archivingManager = new ArchivingManager(domainEvents, operationsManager); + + unarchiveDocumentManager = new UnarchiveDocumentManager(); + unarchivingManager = new UnarchivingManager(domainEvents, operationsManager); + } + + public async Task ArchiveAllInGroup(string groupId) + { + logger.Info($"Archiving of {groupId} started"); + ArchiveOperation archiveOperation; + + using (var session = store.OpenAsyncSession()) + { + session.Advanced.UseOptimisticConcurrency = true; // Ensure 2 messages don't split the same operation into batches at once + + archiveOperation = await archiveDocumentManager.LoadArchiveOperation(session, groupId, ArchiveType.FailureGroup); + + if (archiveOperation == null) + { + var groupDetails = await archiveDocumentManager.GetGroupDetails(session, groupId); + if (groupDetails.NumberOfMessagesInGroup == 0) + { + logger.Warn($"No messages to archive in group {groupId}"); + return; + } + + logger.Info($"Splitting group {groupId} into batches"); + archiveOperation = await archiveDocumentManager.CreateArchiveOperation(session, groupId, ArchiveType.FailureGroup, groupDetails.NumberOfMessagesInGroup, groupDetails.GroupName, batchSize); + await session.SaveChangesAsync(); + + logger.Info($"Group {groupId} has been split into {archiveOperation.NumberOfBatches} batches"); + } + } + + await archivingManager.StartArchiving(archiveOperation); + + while (archiveOperation.CurrentBatch < archiveOperation.NumberOfBatches) + { + using (var batchSession = store.OpenAsyncSession()) + { + var nextBatch = await archiveDocumentManager.GetArchiveBatch(batchSession, archiveOperation.Id, archiveOperation.CurrentBatch); + if (nextBatch == null) + { + // We're only here in the case where Raven indexes are stale + logger.Warn($"Attempting to archive a batch ({archiveOperation.Id}/{archiveOperation.CurrentBatch}) which appears to already have been archived."); + } + else + { + logger.Info($"Archiving {nextBatch.DocumentIds.Count} messages from group {groupId} starting"); + } + + archiveDocumentManager.ArchiveMessageGroupBatch(batchSession, nextBatch); + + await archivingManager.BatchArchived(archiveOperation.RequestId, archiveOperation.ArchiveType, nextBatch?.DocumentIds.Count ?? 0); + + archiveOperation = archivingManager.GetStatusForArchiveOperation(archiveOperation.RequestId, archiveOperation.ArchiveType).ToArchiveOperation(); + + await archiveDocumentManager.UpdateArchiveOperation(batchSession, archiveOperation); + + await batchSession.SaveChangesAsync(); + + if (nextBatch != null) + { + await domainEvents.Raise(new FailedMessageGroupBatchArchived + { + // Remove `FailedMessages/` prefix and publish pure GUIDs without Raven collection name + FailedMessagesIds = nextBatch.DocumentIds.Select(id => id.Replace("FailedMessages/", "")).ToArray() + }); + } + + if (nextBatch != null) + { + logger.Info($"Archiving of {nextBatch.DocumentIds.Count} messages from group {groupId} completed"); + } + } + } + + logger.Info($"Archiving of group {groupId} is complete. Waiting for index updates."); + await archivingManager.ArchiveOperationFinalizing(archiveOperation.RequestId, archiveOperation.ArchiveType); + if (!await archiveDocumentManager.WaitForIndexUpdateOfArchiveOperation(store, archiveOperation.RequestId, TimeSpan.FromMinutes(5)) + ) + { + logger.Warn($"Archiving group {groupId} completed but index not updated."); + } + + await archivingManager.ArchiveOperationCompleted(archiveOperation.RequestId, archiveOperation.ArchiveType); + await archiveDocumentManager.RemoveArchiveOperation(store, archiveOperation); + + await domainEvents.Raise(new FailedMessageGroupArchived + { + GroupId = groupId, + GroupName = archiveOperation.GroupName, + MessagesCount = archiveOperation.TotalNumberOfMessages, + }); + + logger.Info($"Archiving of group {groupId} completed"); + } + + public async Task UnarchiveAllInGroup(string groupId) + { + logger.Info($"Unarchiving of {groupId} started"); + UnarchiveOperation unarchiveOperation; + + using (var session = store.OpenAsyncSession()) + { + session.Advanced.UseOptimisticConcurrency = true; // Ensure 2 messages don't split the same operation into batches at once + + unarchiveOperation = await unarchiveDocumentManager.LoadUnarchiveOperation(session, groupId, ArchiveType.FailureGroup); + + if (unarchiveOperation == null) + { + var groupDetails = await unarchiveDocumentManager.GetGroupDetails(session, groupId); + if (groupDetails.NumberOfMessagesInGroup == 0) + { + logger.Warn($"No messages to unarchive in group {groupId}"); + + return; + } + + logger.Info($"Splitting group {groupId} into batches"); + unarchiveOperation = await unarchiveDocumentManager.CreateUnarchiveOperation(session, groupId, ArchiveType.FailureGroup, groupDetails.NumberOfMessagesInGroup, groupDetails.GroupName, batchSize); + await session.SaveChangesAsync(); + + logger.Info($"Group {groupId} has been split into {unarchiveOperation.NumberOfBatches} batches"); + } + } + + await unarchivingManager.StartUnarchiving(unarchiveOperation); + + while (unarchiveOperation.CurrentBatch < unarchiveOperation.NumberOfBatches) + { + using (var batchSession = store.OpenAsyncSession()) + { + var nextBatch = await unarchiveDocumentManager.GetUnarchiveBatch(batchSession, unarchiveOperation.Id, unarchiveOperation.CurrentBatch); + if (nextBatch == null) + { + // We're only here in the case where Raven indexes are stale + logger.Warn($"Attempting to unarchive a batch ({unarchiveOperation.Id}/{unarchiveOperation.CurrentBatch}) which appears to already have been archived."); + } + else + { + logger.Info($"Unarchiving {nextBatch.DocumentIds.Count} messages from group {groupId} starting"); + } + + unarchiveDocumentManager.UnarchiveMessageGroupBatch(batchSession, nextBatch); + + await unarchivingManager.BatchUnarchived(unarchiveOperation.RequestId, unarchiveOperation.ArchiveType, nextBatch?.DocumentIds.Count ?? 0); + + unarchiveOperation = unarchivingManager.GetStatusForUnarchiveOperation(unarchiveOperation.RequestId, unarchiveOperation.ArchiveType).ToUnarchiveOperation(); + + await unarchiveDocumentManager.UpdateUnarchiveOperation(batchSession, unarchiveOperation); + + await batchSession.SaveChangesAsync(); + + if (nextBatch != null) + { + await domainEvents.Raise(new FailedMessageGroupBatchUnarchived + { + // Remove `FailedMessages/` prefix and publish pure GUIDs without Raven collection name + FailedMessagesIds = nextBatch.DocumentIds.Select(id => id.Replace("FailedMessages/", "")).ToArray() + }); + } + + if (nextBatch != null) + { + logger.Info($"Unarchiving of {nextBatch.DocumentIds.Count} messages from group {groupId} completed"); + } + } + } + + logger.Info($"Unarchiving of group {groupId} is complete. Waiting for index updates."); + await unarchivingManager.UnarchiveOperationFinalizing(unarchiveOperation.RequestId, unarchiveOperation.ArchiveType); + if (!await unarchiveDocumentManager.WaitForIndexUpdateOfUnarchiveOperation(store, unarchiveOperation.RequestId, TimeSpan.FromMinutes(5)) + ) + { + logger.Warn($"Unarchiving group {groupId} completed but index not updated."); + } + + logger.Info($"Unarchiving of group {groupId} completed"); + await unarchivingManager.UnarchiveOperationCompleted(unarchiveOperation.RequestId, unarchiveOperation.ArchiveType); + await unarchiveDocumentManager.RemoveUnarchiveOperation(store, unarchiveOperation); + + await domainEvents.Raise(new FailedMessageGroupUnarchived + { + GroupId = groupId, + GroupName = unarchiveOperation.GroupName, + MessagesCount = unarchiveOperation.TotalNumberOfMessages, + }); + } + + public bool IsOperationInProgressFor(string groupId, ArchiveType archiveType) + { + return operationsManager.IsOperationInProgressFor(groupId, archiveType); + } + + public bool IsArchiveInProgressFor(string groupId) + => archivingManager.IsArchiveInProgressFor(groupId); + + public void DismissArchiveOperation(string groupId, ArchiveType archiveType) + => archivingManager.DismissArchiveOperation(groupId, archiveType); + + public Task StartArchiving(string groupId, ArchiveType archiveType) + => archivingManager.StartArchiving(groupId, archiveType); + + public Task StartUnarchiving(string groupId, ArchiveType archiveType) + => unarchivingManager.StartUnarchiving(groupId, archiveType); + + public IEnumerable GetArchivalOperations() + => archivingManager.GetArchivalOperations(); + + readonly IDocumentStore store; + readonly OperationsManager operationsManager; + readonly IDomainEvents domainEvents; + readonly ArchiveDocumentManager archiveDocumentManager; + readonly ArchivingManager archivingManager; + readonly UnarchiveDocumentManager unarchiveDocumentManager; + readonly UnarchivingManager unarchivingManager; + static readonly ILog logger = LogManager.GetLogger(); + const int batchSize = 1000; + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveBatch.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveBatch.cs new file mode 100644 index 0000000000..d8ee487d19 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveBatch.cs @@ -0,0 +1,15 @@ +namespace ServiceControl.Recoverability +{ + using System.Collections.Generic; + + class UnarchiveBatch //raven + { + public string Id { get; set; } + public List DocumentIds { get; set; } = new List(); + + public static string MakeId(string requestId, ArchiveType archiveType, int batchNumber) + { + return $"UnarchiveOperations/{(int)archiveType}/{requestId}/{batchNumber}"; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveDocumentManager.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveDocumentManager.cs new file mode 100644 index 0000000000..3287cc5289 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveDocumentManager.cs @@ -0,0 +1,150 @@ +namespace ServiceControl.Recoverability +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading.Tasks; + using MessageFailures; + using Raven.Client.Documents; + using Raven.Client.Documents.Commands.Batches; + using Raven.Client.Documents.Operations; + using Raven.Client.Documents.Session; + + class UnarchiveDocumentManager + { + public Task LoadUnarchiveOperation(IAsyncDocumentSession session, string groupId, ArchiveType archiveType) + { + return session.LoadAsync(UnarchiveOperation.MakeId(groupId, archiveType)); + } + + public async Task CreateUnarchiveOperation(IAsyncDocumentSession session, string groupId, ArchiveType archiveType, int numberOfMessages, string groupName, int batchSize) + { + var operation = new UnarchiveOperation + { + Id = UnarchiveOperation.MakeId(groupId, archiveType), + RequestId = groupId, + ArchiveType = archiveType, + TotalNumberOfMessages = numberOfMessages, + NumberOfMessagesUnarchived = 0, + Started = DateTime.Now, + GroupName = groupName, + NumberOfBatches = (int)Math.Ceiling(numberOfMessages / (float)batchSize), + CurrentBatch = 0 + }; + + await session.StoreAsync(operation); + + var documentCount = 0; + var indexQuery = session.Query(new FailedMessages_ByGroup().IndexName); + + var docQuery = indexQuery + .Where(failure => failure.FailureGroupId == groupId) + .Where(failure => failure.Status == FailedMessageStatus.Archived) + .Select(document => document.Id); + + var docs = await StreamResults(session, docQuery); + + var batches = docs + .GroupBy(d => documentCount++ / batchSize); + + foreach (var batch in batches) + { + var unarchiveBatch = new UnarchiveBatch + { + Id = UnarchiveBatch.MakeId(groupId, archiveType, batch.Key), + DocumentIds = batch.ToList() + }; + + await session.StoreAsync(unarchiveBatch); + } + + return operation; + } + + async Task> StreamResults(IAsyncDocumentSession session, IQueryable query) + { + var results = new List(); + await using (var enumerator = await session.Advanced.StreamAsync(query)) + { + while (await enumerator.MoveNextAsync()) + { + results.Add(enumerator.Current.Document); + } + } + + return results; + } + + public Task GetUnarchiveBatch(IAsyncDocumentSession session, string unUnarchiveOperationId, int batchNumber) + { + return session.LoadAsync($"{unUnarchiveOperationId}/{batchNumber}"); + } + + public async Task GetGroupDetails(IAsyncDocumentSession session, string groupId) + { + var group = await session.Query() + .FirstOrDefaultAsync(x => x.Id == groupId); + + return new GroupDetails + { + NumberOfMessagesInGroup = group?.Count ?? 0, + GroupName = group?.Title ?? "Undefined" + }; + } + + public void UnarchiveMessageGroupBatch(IAsyncDocumentSession session, UnarchiveBatch batch) + { + var patchCommands = batch?.DocumentIds.Select(documentId => new PatchCommandData(documentId, null, patchRequest)); + + if (patchCommands != null) + { + session.Advanced.Defer(patchCommands.ToArray()); + session.Advanced.Defer(new DeleteCommandData(batch.Id, null)); + } + } + + public async Task WaitForIndexUpdateOfUnarchiveOperation(IDocumentStore store, string requestId, TimeSpan timeToWait) + { + using (var session = store.OpenAsyncSession()) + { + var indexQuery = session.Query(new FailedMessages_ByGroup().IndexName) + .Customize(x => x.WaitForNonStaleResults(timeToWait)); + + var docQuery = indexQuery + .Where(failure => failure.FailureGroupId == requestId) + .Select(document => document.Id); + + try + { + await docQuery.AnyAsync(); + + return true; + } + catch + { + return false; + } + } + } + + public async Task UpdateUnarchiveOperation(IAsyncDocumentSession session, UnarchiveOperation unarchiveOperation) + { + await session.StoreAsync(unarchiveOperation); + } + + public async Task RemoveUnarchiveOperation(IDocumentStore store, UnarchiveOperation unarchiveOperation) + { + using var session = store.OpenAsyncSession(); + session.Advanced.Defer(new DeleteCommandData(unarchiveOperation.Id, null)); + await session.SaveChangesAsync(); + } + + static PatchRequest patchRequest = new PatchRequest { Script = $@"this.Status = {(int)FailedMessageStatus.Unresolved}" }; + + public class GroupDetails + { + public string GroupName { get; set; } + public int NumberOfMessagesInGroup { get; set; } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveOperation.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveOperation.cs new file mode 100644 index 0000000000..4227ce05f5 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveOperation.cs @@ -0,0 +1,21 @@ +namespace ServiceControl.Recoverability +{ + using System; + + class UnarchiveOperation // raven + { + public string Id { get; set; } + public string RequestId { get; set; } + public string GroupName { get; set; } + public ArchiveType ArchiveType { get; set; } + public int TotalNumberOfMessages { get; set; } + public int NumberOfMessagesUnarchived { get; set; } + public DateTime Started { get; set; } + public int NumberOfBatches { get; set; } + public int CurrentBatch { get; set; } + public static string MakeId(string requestId, ArchiveType archiveType) + { + return $"UnarchiveOperations/{(int)archiveType}/{requestId}"; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchivingManager.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchivingManager.cs new file mode 100644 index 0000000000..0644b29dc1 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchivingManager.cs @@ -0,0 +1,110 @@ +namespace ServiceControl.Recoverability +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading.Tasks; + using Infrastructure.DomainEvents; + + class UnarchivingManager + { + public UnarchivingManager(IDomainEvents domainEvents, OperationsManager operationsManager) + { + this.domainEvents = domainEvents; + this.operationsManager = operationsManager; + } + + public bool IsUnarchiveInProgressFor(string requestId) + { + return operationsManager.UnarchiveOperations.Keys.Any(key => key.EndsWith($"/{requestId}")); + } + + internal IEnumerable GetUnarchivalOperations() + { + return operationsManager.UnarchiveOperations.Values; + } + + public bool IsOperationInProgressFor(string requestId, ArchiveType archiveType) + { + return operationsManager.IsOperationInProgressFor(requestId, archiveType); + } + + InMemoryUnarchive GetOrCreate(ArchiveType archiveType, string requestId) + { + if (!operationsManager.UnarchiveOperations.TryGetValue(InMemoryUnarchive.MakeId(requestId, archiveType), out var summary)) + { + summary = new InMemoryUnarchive(requestId, archiveType, domainEvents); + operationsManager.UnarchiveOperations[InMemoryUnarchive.MakeId(requestId, archiveType)] = summary; + } + + return summary; + } + + public Task StartUnarchiving(UnarchiveOperation archiveOperation) + { + var summary = GetOrCreate(archiveOperation.ArchiveType, archiveOperation.RequestId); + + summary.TotalNumberOfMessages = archiveOperation.TotalNumberOfMessages; + summary.NumberOfMessagesUnarchived = archiveOperation.NumberOfMessagesUnarchived; + summary.Started = archiveOperation.Started; + summary.GroupName = archiveOperation.GroupName; + summary.NumberOfBatches = archiveOperation.NumberOfBatches; + summary.CurrentBatch = archiveOperation.CurrentBatch; + + return summary.Start(); + } + + public Task StartUnarchiving(string requestId, ArchiveType archiveType) + { + var summary = GetOrCreate(archiveType, requestId); + + summary.TotalNumberOfMessages = 0; + summary.NumberOfMessagesUnarchived = 0; + summary.Started = DateTime.Now; + summary.GroupName = "Undefined"; + summary.NumberOfBatches = 0; + summary.CurrentBatch = 0; + + return summary.Start(); + } + + public InMemoryUnarchive GetStatusForUnarchiveOperation(string requestId, ArchiveType archiveType) + { + operationsManager.UnarchiveOperations.TryGetValue(InMemoryUnarchive.MakeId(requestId, archiveType), out var summary); + + return summary; + } + + public Task BatchUnarchived(string requestId, ArchiveType archiveType, int numberOfMessagesArchivedInBatch) + { + var summary = GetOrCreate(archiveType, requestId); + + return summary.BatchUnarchived(numberOfMessagesArchivedInBatch); + } + + public Task UnarchiveOperationFinalizing(string requestId, ArchiveType archiveType) + { + var summary = GetOrCreate(archiveType, requestId); + return summary.FinalizeUnarchive(); + } + + public Task UnarchiveOperationCompleted(string requestId, ArchiveType archiveType) + { + var summary = GetOrCreate(archiveType, requestId); + return summary.Complete(); + } + + public void DismissArchiveOperation(string requestId, ArchiveType archiveType) + { + RemoveUnarchiveOperation(requestId, archiveType); + } + + void RemoveUnarchiveOperation(string requestId, ArchiveType archiveType) + { + operationsManager.UnarchiveOperations.Remove(InMemoryUnarchive.MakeId(requestId, archiveType)); + } + + IDomainEvents domainEvents; + OperationsManager operationsManager; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/GroupsDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/GroupsDataStore.cs new file mode 100644 index 0000000000..49577f38ad --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/GroupsDataStore.cs @@ -0,0 +1,60 @@ +namespace ServiceControl.Persistence.RavenDb.Recoverability +{ + using System.Collections.Generic; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Raven.Client.Documents; + using Raven.Client.Documents.Linq; + using ServiceControl.MessageFailures; + using ServiceControl.Recoverability; + + class GroupsDataStore : IGroupsDataStore + { + readonly IDocumentStore store; + + public GroupsDataStore(IDocumentStore store) + { + this.store = store; + } + + public async Task> GetFailureGroupsByClassifier(string classifier, string classifierFilter) + { + using (var session = store.OpenAsyncSession()) + { + var query = Queryable.Where(session.Query(), v => v.Type == classifier); + + if (!string.IsNullOrWhiteSpace(classifierFilter)) + { + query = query.Where(v => v.Title == classifierFilter); + } + + var groups = await query.OrderByDescending(x => x.Last) + .Take(200) + .ToListAsync(); + + var commentIds = groups.Select(x => GroupComment.MakeId(x.Id)).ToArray(); + var comments = await session.Query().Where(x => x.Id.In(commentIds)) + .ToListAsync(CancellationToken.None); + + foreach (var group in groups) + { + group.Comment = comments.FirstOrDefault(x => x.Id == GroupComment.MakeId(group.Id))?.Comment; + } + + return groups; + } + } + + public async Task GetCurrentForwardingBatch() + { + using (var session = store.OpenAsyncSession()) + { + var nowForwarding = await session.Include(r => r.RetryBatchId) + .LoadAsync(RetryBatchNowForwarding.Id); + + return nowForwarding == null ? null : await session.LoadAsync(nowForwarding.RetryBatchId); + } + } + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/RetryHistoryDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/RetryHistoryDataStore.cs new file mode 100644 index 0000000000..21fc002aa1 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/RetryHistoryDataStore.cs @@ -0,0 +1,80 @@ +namespace ServiceControl.Persistence.RavenDb.Recoverability +{ + using System; + using System.Threading.Tasks; + using ServiceControl.Recoverability; + using Raven.Client.Documents; + + class RetryHistoryDataStore : IRetryHistoryDataStore + { + public RetryHistoryDataStore(IDocumentStore documentStore) + { + this.documentStore = documentStore; + } + + public async Task GetRetryHistory() + { + using var session = documentStore.OpenAsyncSession(); + var id = RetryHistory.MakeId(); + var retryHistory = await session.LoadAsync(id); + + retryHistory ??= RetryHistory.CreateNew(); + + return retryHistory; + } + + public async Task RecordRetryOperationCompleted(string requestId, RetryType retryType, DateTime startTime, DateTime completionTime, + string originator, string classifier, bool messageFailed, int numberOfMessagesProcessed, DateTime lastProcessed, int retryHistoryDepth) + { + using var session = documentStore.OpenAsyncSession(); + var retryHistory = await session.LoadAsync(RetryHistory.MakeId()) ?? RetryHistory.CreateNew(); + + retryHistory.AddToUnacknowledged(new UnacknowledgedRetryOperation + { + RequestId = requestId, + RetryType = retryType, + StartTime = startTime, + CompletionTime = completionTime, + Originator = originator, + Classifier = classifier, + Failed = messageFailed, + NumberOfMessagesProcessed = numberOfMessagesProcessed, + Last = lastProcessed + }); + + retryHistory.AddToHistory(new HistoricRetryOperation + { + RequestId = requestId, + RetryType = retryType, + StartTime = startTime, + CompletionTime = completionTime, + Originator = originator, + Failed = messageFailed, + NumberOfMessagesProcessed = numberOfMessagesProcessed + }, retryHistoryDepth); + + await session.StoreAsync(retryHistory); + await session.SaveChangesAsync(); + } + + public async Task AcknowledgeRetryGroup(string groupId) + { + using var session = documentStore.OpenAsyncSession(); + var retryHistory = await session.LoadAsync(RetryHistory.MakeId()); + if (retryHistory != null) + { + if (retryHistory.Acknowledge(groupId, RetryType.FailureGroup)) + { + await session.StoreAsync(retryHistory); + await session.SaveChangesAsync(); + + return true; + } + } + + return false; + } + + readonly IDocumentStore documentStore; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RepeatedFailuresOverTimeCircuitBreaker.cs b/src/ServiceControl.Persistence.RavenDb5/RepeatedFailuresOverTimeCircuitBreaker.cs new file mode 100644 index 0000000000..09cb5b0700 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RepeatedFailuresOverTimeCircuitBreaker.cs @@ -0,0 +1,74 @@ +namespace NServiceBus +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using Logging; + + class RepeatedFailuresOverTimeCircuitBreaker : IDisposable + { + public RepeatedFailuresOverTimeCircuitBreaker(string name, TimeSpan timeToWaitBeforeTriggering, Action triggerAction, TimeSpan delayAfterFailure) + { + this.delayAfterFailure = delayAfterFailure; + this.name = name; + this.triggerAction = triggerAction; + this.timeToWaitBeforeTriggering = timeToWaitBeforeTriggering; + + timer = new Timer(CircuitBreakerTriggered); + } + + public void Dispose() + { + //Injected + } + + public void Success() + { + var oldValue = Interlocked.Exchange(ref failureCount, 0); + + if (oldValue == 0) + { + return; + } + + timer.Change(System.Threading.Timeout.Infinite, System.Threading.Timeout.Infinite); + Logger.InfoFormat("The circuit breaker for {0} is now disarmed", name); + } + + public Task Failure(Exception exception) + { + lastException = exception; + var newValue = Interlocked.Increment(ref failureCount); + + if (newValue == 1) + { + timer.Change(timeToWaitBeforeTriggering, NoPeriodicTriggering); + Logger.WarnFormat("The circuit breaker for {0} is now in the armed state", name); + } + + return Task.Delay(delayAfterFailure); + } + + void CircuitBreakerTriggered(object state) + { + if (Interlocked.Read(ref failureCount) > 0) + { + Logger.WarnFormat("The circuit breaker for {0} will now be triggered", name); + triggerAction(lastException); + } + } + + readonly TimeSpan delayAfterFailure; + + long failureCount; + Exception lastException; + + string name; + Timer timer; + TimeSpan timeToWaitBeforeTriggering; + Action triggerAction; + + static TimeSpan NoPeriodicTriggering = TimeSpan.FromMilliseconds(-1); + static ILog Logger = LogManager.GetLogger(); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RetryBatchesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RetryBatchesDataStore.cs new file mode 100644 index 0000000000..d20d0b84da --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RetryBatchesDataStore.cs @@ -0,0 +1,95 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System; + using System.Collections.Generic; + using System.Threading.Tasks; + using MessageFailures; + using NServiceBus.Logging; + using Raven.Client.Documents; + using Raven.Client.Documents.Commands; + using Raven.Client.Documents.Commands.Batches; + using Raven.Client.Documents.Operations; + using Raven.Client.Exceptions; + using ServiceControl.Recoverability; + + class RetryBatchesDataStore : IRetryBatchesDataStore + { + readonly IDocumentStore documentStore; + + static readonly ILog Log = LogManager.GetLogger(typeof(RetryBatchesDataStore)); + + public RetryBatchesDataStore(IDocumentStore documentStore) + { + this.documentStore = documentStore; + } + + public Task CreateRetryBatchesManager() + { + var session = documentStore.OpenAsyncSession(); + var manager = new RetryBatchesManager(session); + + return Task.FromResult(manager); + } + + public async Task RecordFailedStagingAttempt(IReadOnlyCollection messages, + IReadOnlyDictionary failedMessageRetriesById, Exception e, + int maxStagingAttempts, string stagingId) + { + var commands = new ICommandData[messages.Count]; + var commandIndex = 0; + foreach (var failedMessage in messages) + { + var failedMessageRetry = failedMessageRetriesById[failedMessage.Id]; + + Log.Warn($"Attempt {1} of {maxStagingAttempts} to stage a retry message {failedMessage.UniqueMessageId} failed", e); + + commands[commandIndex] = new PatchCommandData(failedMessageRetry.Id, null, new PatchRequest + { + Script = @"this.StageAttempts = args.Value", + Values = + { + {"Value", 1 } + } + }); + + commandIndex++; + } + + + try + { + using var session = documentStore.OpenAsyncSession(); + + var batch = new SingleNodeBatchCommand(documentStore.Conventions, session.Advanced.Context, commands); + await session.Advanced.RequestExecutor.ExecuteAsync(batch, session.Advanced.Context); + } + catch (ConcurrencyException) + { + Log.DebugFormat( + "Ignoring concurrency exception while incrementing staging attempt count for {0}", + stagingId); + } + } + + public async Task IncrementAttemptCounter(FailedMessageRetry message) + { + try + { + await documentStore.Operations.SendAsync(new PatchOperation(message.Id, null, new PatchRequest + { + Script = @"this.StageAttempts += 1" + })); + } + catch (ConcurrencyException) + { + Log.DebugFormat("Ignoring concurrency exception while incrementing staging attempt count for {0}", message.FailedMessageId); + } + } + + public async Task DeleteFailedMessageRetry(string uniqueMessageId) + { + using var session = documentStore.OpenAsyncSession(); + await session.Advanced.RequestExecutor.ExecuteAsync(new DeleteDocumentCommand(FailedMessageRetry.MakeDocumentId(uniqueMessageId), null), session.Advanced.Context); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RetryBatchesManager.cs b/src/ServiceControl.Persistence.RavenDb5/RetryBatchesManager.cs new file mode 100644 index 0000000000..43da58b7c2 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RetryBatchesManager.cs @@ -0,0 +1,68 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System.Collections.Generic; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using MessageFailures; + using Persistence.MessageRedirects; + using Raven.Client.Documents; + using Raven.Client.Documents.Session; + using ServiceControl.Recoverability; + + class RetryBatchesManager : AbstractSessionManager, IRetryBatchesManager + { + public RetryBatchesManager(IAsyncDocumentSession session) : base(session) + { + } + + public void Delete(RetryBatch retryBatch) => Session.Delete(retryBatch); + + public void Delete(RetryBatchNowForwarding forwardingBatch) => Session.Delete(forwardingBatch); + + public async Task GetFailedMessageRetries(IList stagingBatchFailureRetries) + { + var result = await Session.LoadAsync(stagingBatchFailureRetries); + return result.Values.ToArray(); + } + + public void Evict(FailedMessageRetry failedMessageRetry) => Session.Advanced.Evict(failedMessageRetry); + + public async Task GetFailedMessages(Dictionary.KeyCollection keys) + { + var result = await Session.LoadAsync(keys); + return result.Values.ToArray(); + } + + public async Task GetRetryBatchNowForwarding() => + await Session.Include(r => r.RetryBatchId) + .LoadAsync(RetryBatchNowForwarding.Id); + + public async Task GetRetryBatch(string retryBatchId, CancellationToken cancellationToken) => + await Session.LoadAsync(retryBatchId, cancellationToken); + + public async Task GetStagingBatch() + { + return await Session.Query() + .Include(b => b.FailureRetries) + .FirstOrDefaultAsync(b => b.Status == RetryBatchStatus.Staging); + } + + public async Task Store(RetryBatchNowForwarding retryBatchNowForwarding) => + await Session.StoreAsync(retryBatchNowForwarding, RetryBatchNowForwarding.Id); + + public async Task GetOrCreateMessageRedirectsCollection() + { + var redirects = await Session.LoadAsync(MessageRedirectsCollection.DefaultId); + + if (redirects != null) + { + redirects.ETag = Session.Advanced.GetChangeVectorFor(redirects); + redirects.LastModified = Session.Advanced.GetLastModifiedFor(redirects)!.Value; + return redirects; + } + + return new MessageRedirectsCollection(); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs new file mode 100644 index 0000000000..d387f658e6 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs @@ -0,0 +1,264 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading.Tasks; + using MessageFailures; + using NServiceBus.Logging; + using Persistence.Infrastructure; + using Raven.Client.Documents; + using Raven.Client.Documents.Commands.Batches; + using Raven.Client.Documents.Linq; + using Raven.Client.Documents.Operations; + using Raven.Client.Exceptions; + using ServiceControl.MessageFailures.Api; + using ServiceControl.Recoverability; + + class RetryDocumentDataStore : IRetryDocumentDataStore + { + readonly IDocumentStore store; + + public RetryDocumentDataStore(IDocumentStore store) + { + this.store = store; + } + + public async Task StageRetryByUniqueMessageIds(string batchDocumentId, string requestId, RetryType retryType, string[] messageIds, + DateTime startTime, + DateTime? last = null, string originator = null, string batchName = null, string classifier = null) + { + var commands = new ICommandData[messageIds.Length]; + + for (var i = 0; i < messageIds.Length; i++) + { + commands[i] = CreateFailedMessageRetryDocument(batchDocumentId, messageIds[i]); + } + + using (var session = store.OpenAsyncSession()) + { + var batch = new SingleNodeBatchCommand(store.Conventions, session.Advanced.Context, commands); + await session.Advanced.RequestExecutor.ExecuteAsync(batch, session.Advanced.Context); + } + } + + public async Task MoveBatchToStaging(string batchDocumentId) + { + try + { + await store.Operations.SendAsync(new PatchOperation(batchDocumentId, null, new PatchRequest + { + Script = @"this.Status = args.Status", + Values = + { + {"Status", (int)RetryBatchStatus.Staging } + } + })); + } + catch (ConcurrencyException) + { + log.DebugFormat("Ignoring concurrency exception while moving batch to staging {0}", batchDocumentId); + } + } + + public async Task CreateBatchDocument(string retrySessionId, string requestId, RetryType retryType, string[] failedMessageRetryIds, + string originator, + DateTime startTime, DateTime? last = null, string batchName = null, string classifier = null) + { + var batchDocumentId = RetryBatch.MakeDocumentId(Guid.NewGuid().ToString()); + using (var session = store.OpenAsyncSession()) + { + await session.StoreAsync(new RetryBatch + { + Id = batchDocumentId, + Context = batchName, + RequestId = requestId, + RetryType = retryType, + Originator = originator, + Classifier = classifier, + StartTime = startTime, + Last = last, + InitialBatchSize = failedMessageRetryIds.Length, + RetrySessionId = retrySessionId, + FailureRetries = failedMessageRetryIds, + Status = RetryBatchStatus.MarkingDocuments + }); + await session.SaveChangesAsync(); + } + + return batchDocumentId; + } + + public async Task>> QueryOrphanedBatches(string retrySessionId, DateTime cutoff) + { + using (var session = store.OpenAsyncSession()) + { + var orphanedBatches = await session + .Query() + + // TODO: Cutoff no longer exists but guidance isn't clear how to handle this: + // https://ravendb.net/docs/article-page/5.4/Csharp/indexes/stale-indexes + // https://ravendb.net/docs/article-page/5.4/csharp/client-api/session/querying/how-to-customize-query#waitfornonstaleresults + + //.Customize(c => c.BeforeQueryExecuted(index => index.Cutoff = cutoff)) + .Customize(c => c.WaitForNonStaleResults()) // (ramon) I think this is valid as at start orphaned batches should be retrieved based on non-stale results I would assume? + + .Where(b => b.Status == RetryBatchStatus.MarkingDocuments && b.RetrySessionId != retrySessionId) + .Statistics(out var stats) + .ToListAsync(); + + return orphanedBatches.ToQueryResult(stats); + } + } + + public async Task> QueryAvailableBatches() + { + using (var session = store.OpenAsyncSession()) + { + var results = await session.Query() + .Where(b => b.HasStagingBatches || b.HasForwardingBatches) + .ToListAsync(); + return results; + } + } + + static ICommandData CreateFailedMessageRetryDocument(string batchDocumentId, string messageId) + { + return new PatchCommandData(FailedMessageRetry.MakeDocumentId(messageId), null, patch: null, patchIfMissing: new PatchRequest + { + Script = @"this.FailedMessageId = args.MessageId + this.RetryBatchId = args.BatchDocumentId", + Values = + { + { "MessageId", FailedMessageIdGenerator.MakeDocumentId(messageId) }, + { "BatchDocumentId", batchDocumentId } + } + }); + } + + static ILog log = LogManager.GetLogger(typeof(RetryDocumentDataStore)); + + // TODO: Verify Stream queries in this file, which were the result of joining overly-complex IndexBasedBulkRetryRequest + // which was in this file, as well as the FailedMessages_UniqueMessageIdAndTimeOfFailures transformer, since transformers + // are not supported in RavenDB 5. I don't know what all the other properties of IndexBasedBulkRetryRequest were ever for, + // since they weren't used in this class. I also don't know what the other comments that were in each streaming query method + // were for either. + + public async Task GetBatchesForAll(DateTime cutoff, Func callback) + { + // StartRetryForIndex("All", RetryType.All, DateTime.UtcNow, originator: "all messages"); + //public void StartRetryForIndex(string requestId, RetryType retryType, DateTime startTime, Expression> filter = null, string originator = null, string classifier = null) + //StartRetryForIndex(endpoint, RetryType.AllForEndpoint, DateTime.UtcNow, m => m.ReceivingEndpointName == endpoint, $"all messages for endpoint {endpoint}"); + + using (var session = store.OpenAsyncSession()) + { + var query = session.Query() + .Where(d => d.Status == FailedMessageStatus.Unresolved) + .Select(m => new + { + UniqueMessageId = m.MessageId, + LatestTimeOfFailure = m.TimeOfFailure + }); + + await using (var stream = await session.Advanced.StreamAsync(query)) + { + while (await stream.MoveNextAsync()) + { + var current = stream.Current.Document; + await callback(current.UniqueMessageId, current.LatestTimeOfFailure); + } + } + } + } + + public async Task GetBatchesForEndpoint(DateTime cutoff, string endpoint, Func callback) + { + //ForIndex + //StartRetryForIndex(endpoint, RetryType.AllForEndpoint, DateTime.UtcNow, m => m.ReceivingEndpointName == endpoint, $"all messages for endpoint {endpoint}"); + + using (var session = store.OpenAsyncSession()) + { + var query = session.Query() + .Where(d => d.Status == FailedMessageStatus.Unresolved) + .Where(m => m.ReceivingEndpointName == endpoint) + .Select(m => new + { + UniqueMessageId = m.MessageId, + LatestTimeOfFailure = m.TimeOfFailure + }); + + await using (var stream = await session.Advanced.StreamAsync(query)) + { + while (await stream.MoveNextAsync()) + { + var current = stream.Current.Document; + await callback(current.UniqueMessageId, current.LatestTimeOfFailure); + } + } + } + } + + public async Task GetBatchesForFailedQueueAddress(DateTime cutoff, string failedQueueAddress, FailedMessageStatus status, Func callback) + { + //ForIndex + //StartRetryForIndex(failedQueueAddress, RetryType.ByQueueAddress, DateTime.UtcNow, m => m.QueueAddress == failedQueueAddress && m.Status == status, ); + + using (var session = store.OpenAsyncSession()) + { + var query = session.Query() + .Where(d => d.Status == FailedMessageStatus.Unresolved) + .Where(m => m.QueueAddress == failedQueueAddress && m.Status == status) + .Select(m => new + { + UniqueMessageId = m.MessageId, + LatestTimeOfFailure = m.TimeOfFailure + }); + + await using (var stream = await session.Advanced.StreamAsync(query)) + { + while (await stream.MoveNextAsync()) + { + var current = stream.Current.Document; + await callback(current.UniqueMessageId, current.LatestTimeOfFailure); + } + } + } + } + + public async Task GetBatchesForFailureGroup(string groupId, string groupTitle, string groupType, DateTime cutoff, Func callback) + { + //retries.StartRetryForIndex(message.GroupId, RetryType.FailureGroup, started, x => x.FailureGroupId == message.GroupId, originator, group?.Type); + + using (var session = store.OpenAsyncSession()) + { + var query = session.Query() + .Where(d => d.Status == FailedMessageStatus.Unresolved) + .Where(m => m.FailureGroupId == groupId) + .Select(m => new + { + UniqueMessageId = m.MessageId, + LatestTimeOfFailure = m.TimeOfFailure + }); + + await using (var stream = await session.Advanced.StreamAsync(query)) + { + while (await stream.MoveNextAsync()) + { + var current = stream.Current.Document; + await callback(current.UniqueMessageId, current.LatestTimeOfFailure); + } + } + } + } + + public async Task QueryFailureGroupViewOnGroupId(string groupId) + { + using (var session = store.OpenAsyncSession()) + { + var group = await session.Query() + .FirstOrDefaultAsync(x => x.Id == groupId); + return group; + } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/SagaAudit/NoImplementationSagaAuditDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/SagaAudit/NoImplementationSagaAuditDataStore.cs new file mode 100644 index 0000000000..0ee0bf02e7 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/SagaAudit/NoImplementationSagaAuditDataStore.cs @@ -0,0 +1,14 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System; + using System.Threading.Tasks; + using ServiceControl.Persistence.Infrastructure; + using ServiceControl.SagaAudit; + + class NoImplementationSagaAuditDataStore : ISagaAuditDataStore + { + public Task StoreSnapshot(SagaSnapshot sagaSnapshot) => throw new NotImplementedException(); + + public Task> GetSagaById(Guid sagaId) => Task.FromResult(QueryResult.Empty()); + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/ServiceControl.Persistence.RavenDb5.csproj b/src/ServiceControl.Persistence.RavenDb5/ServiceControl.Persistence.RavenDb5.csproj new file mode 100644 index 0000000000..85fe63bef3 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/ServiceControl.Persistence.RavenDb5.csproj @@ -0,0 +1,29 @@ + + + + net472 + 8.0 + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Transactions/RavenTransactionalDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/Transactions/RavenTransactionalDataStore.cs new file mode 100644 index 0000000000..f67b84c65c --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Transactions/RavenTransactionalDataStore.cs @@ -0,0 +1,15 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System.Threading.Tasks; + using Raven.Client; + using Raven.Client.Documents.Session; + + abstract class AbstractSessionManager : IDataSessionManager + { + protected IAsyncDocumentSession Session { get; } + + protected AbstractSessionManager(IAsyncDocumentSession session) => Session = session; + public Task SaveChanges() => Session.SaveChangesAsync(); + public void Dispose() => Session.Dispose(); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs b/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs new file mode 100644 index 0000000000..77c425d968 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs @@ -0,0 +1,45 @@ +namespace ServiceControl.MessageFailures.Api +{ + using System; + using System.Linq; + using ServiceControl.CompositeViews.Messages; + + class FailedMessageViewTransformer : FakeAbstractTransformerCreationTask // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers + { + public FailedMessageViewTransformer() + { + TransformResults = failures => from failure in failures + let rec = failure.ProcessingAttempts.Last() + let edited = rec.Headers["ServiceControl.EditOf"] != null + select new + { + Id = failure.UniqueMessageId, + MessageType = rec.MessageMetadata["MessageType"], + IsSystemMessage = (bool)rec.MessageMetadata["IsSystemMessage"], + SendingEndpoint = rec.MessageMetadata["SendingEndpoint"], + ReceivingEndpoint = rec.MessageMetadata["ReceivingEndpoint"], + TimeSent = (DateTime?)rec.MessageMetadata["TimeSent"], + MessageId = rec.MessageMetadata["MessageId"], + rec.FailureDetails.Exception, + QueueAddress = rec.FailureDetails.AddressOfFailingEndpoint, + NumberOfProcessingAttempts = failure.ProcessingAttempts.Count, + failure.Status, + rec.FailureDetails.TimeOfFailure, + //LastModified = MetadataFor(failure)["@last-modified"].Value(), + Edited = edited, + EditOf = edited ? rec.Headers["ServiceControl.EditOf"] : "" + }; + } + + public static string Name + { + get + { + transformerName ??= new FailedMessageViewTransformer().TransformerName; + return transformerName; + } + } + + static string transformerName; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs new file mode 100644 index 0000000000..4f4f58fc94 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs @@ -0,0 +1,93 @@ +namespace ServiceControl.CompositeViews.Messages +{ + using System; + using System.Collections; + using System.Collections.Generic; + using System.Linq; + using System.Linq.Expressions; + using MessageFailures; + using Raven.Client.Documents.Linq; + using ServiceControl.Persistence; + + class FakeAbstractTransformerCreationTask + { + public Expression, IEnumerable>> TransformResults; + public string TransformerName; + } + + class MessagesViewTransformer : FakeAbstractTransformerCreationTask // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers + { + public MessagesViewTransformer() + { + TransformResults = messages => from message in messages + let metadata = + message.ProcessingAttempts != null + ? message.ProcessingAttempts.Last().MessageMetadata + : message.MessageMetadata + let headers = + message.ProcessingAttempts != null ? message.ProcessingAttempts.Last().Headers : message.Headers + let processedAt = + message.ProcessingAttempts != null + ? message.ProcessingAttempts.Last().AttemptedAt + : message.ProcessedAt + let status = + message.ProcessingAttempts == null + ? !(bool)message.MessageMetadata["IsRetried"] + ? MessageStatus.Successful + : MessageStatus.ResolvedSuccessfully + : message.Status == FailedMessageStatus.Resolved + ? MessageStatus.ResolvedSuccessfully + : message.Status == FailedMessageStatus.RetryIssued + ? MessageStatus.RetryIssued + : message.Status == FailedMessageStatus.Archived + ? MessageStatus.ArchivedFailure + : message.ProcessingAttempts.Count == 1 + ? MessageStatus.Failed + : MessageStatus.RepeatedFailure + select new + { + Id = message.UniqueMessageId, + MessageId = metadata["MessageId"], + MessageType = metadata["MessageType"], + SendingEndpoint = metadata["SendingEndpoint"], + ReceivingEndpoint = metadata["ReceivingEndpoint"], + TimeSent = (DateTime?)metadata["TimeSent"], + ProcessedAt = processedAt, + CriticalTime = (TimeSpan)metadata["CriticalTime"], + ProcessingTime = (TimeSpan)metadata["ProcessingTime"], + DeliveryTime = (TimeSpan)metadata["DeliveryTime"], + IsSystemMessage = (bool)metadata["IsSystemMessage"], + ConversationId = metadata["ConversationId"], + //the reason the we need to use a KeyValuePair is that raven seems to interpret the values and convert them + // to real types. In this case it was the NServiceBus.Temporary.DelayDeliveryWith header to was converted to a timespan + Headers = headers.Select(header => new KeyValuePair(header.Key, header.Value)), + Status = status, + MessageIntent = metadata["MessageIntent"], + BodyUrl = metadata["BodyUrl"], + BodySize = (int)metadata["ContentLength"], + InvokedSagas = metadata["InvokedSagas"], + OriginatesFromSaga = metadata["OriginatesFromSaga"] + }; + } + + public class Input + { + public string Id { get; set; } + public string UniqueMessageId { get; set; } + public DateTime ProcessedAt { get; set; } + public Dictionary Headers { get; set; } + public Dictionary MessageMetadata { get; set; } + public List ProcessingAttempts { get; set; } + public FailedMessageStatus Status { get; set; } + } + } + + public static class MessageViewTransformerExtension + { + public static IQueryable TransformToMessagesView(this IQueryable source) + { + return source.Select(m => new MessagesView()); + } + } +} + diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs new file mode 100644 index 0000000000..f1f67516c3 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs @@ -0,0 +1,34 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System.Collections.Concurrent; + using System.Threading.Tasks; + using ServiceControl.Persistence.UnitOfWork; + using Raven.Client.Documents; + using Raven.Client.Documents.Commands.Batches; + + class RavenDbIngestionUnitOfWork : IngestionUnitOfWorkBase + { + readonly IDocumentStore store; + readonly ConcurrentBag commands; + + public RavenDbIngestionUnitOfWork(IDocumentStore store) + { + this.store = store; + commands = new ConcurrentBag(); + Monitoring = new RavenDbMonitoringIngestionUnitOfWork(this); + Recoverability = new RavenDbRecoverabilityIngestionUnitOfWork(this); + } + + internal void AddCommand(ICommandData command) => commands.Add(command); + + public override async Task Complete() + { + using (var session = store.OpenAsyncSession()) + { + // not really interested in the batch results since a batch is atomic + session.Advanced.Defer(commands.ToArray()); + await session.SaveChangesAsync(); + } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWorkFactory.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWorkFactory.cs new file mode 100644 index 0000000000..60d64a8e2c --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWorkFactory.cs @@ -0,0 +1,27 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System.Threading.Tasks; + using ServiceControl.Persistence.UnitOfWork; + using Raven.Client; + using Raven.Client.Documents; + + class RavenDbIngestionUnitOfWorkFactory : IIngestionUnitOfWorkFactory + { + readonly IDocumentStore store; + readonly MinimumRequiredStorageState customCheckState; + + public RavenDbIngestionUnitOfWorkFactory(IDocumentStore store, MinimumRequiredStorageState customCheckState) + { + this.store = store; + this.customCheckState = customCheckState; + } + + public ValueTask StartNew() + => new ValueTask(new RavenDbIngestionUnitOfWork(store)); + + public bool CanIngestMore() + { + return customCheckState.CanIngestMore; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs new file mode 100644 index 0000000000..e3a3114b93 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs @@ -0,0 +1,47 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System.Threading.Tasks; + using Newtonsoft.Json.Linq; + using ServiceControl.Persistence.UnitOfWork; + using Raven.Client.Documents.Commands.Batches; + using Raven.Client.Documents.Operations; + + class RavenDbMonitoringIngestionUnitOfWork : IMonitoringIngestionUnitOfWork + { + RavenDbIngestionUnitOfWork parentUnitOfWork; + + public RavenDbMonitoringIngestionUnitOfWork(RavenDbIngestionUnitOfWork parentUnitOfWork) + { + this.parentUnitOfWork = parentUnitOfWork; + } + + public Task RecordKnownEndpoint(KnownEndpoint knownEndpoint) + { + parentUnitOfWork.AddCommand(CreateKnownEndpointsPutCommand(knownEndpoint)); + return Task.CompletedTask; + } + + static PatchCommandData CreateKnownEndpointsPutCommand(KnownEndpoint endpoint) + { + var document = JObject.FromObject(endpoint); + document["@metadata"] = KnownEndpointMetadata; + + return new PatchCommandData(endpoint.Id.ToString(), null, new PatchRequest + { + //TODO: check if this works + Script = $"put('{KnownEndpoint.CollectionName}/', {document}" + }); + } + + static RavenDbMonitoringIngestionUnitOfWork() + { + KnownEndpointMetadata = JObject.Parse($@" + {{ + ""@collection"": ""{KnownEndpoint.CollectionName}"", + ""Raven-Clr-Type"": ""{typeof(KnownEndpoint).AssemblyQualifiedName}"" + }}"); + } + + static readonly JObject KnownEndpointMetadata; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs new file mode 100644 index 0000000000..0b02f6ff58 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs @@ -0,0 +1,129 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System.Collections.Generic; + using System.Threading.Tasks; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + using ServiceControl.MessageFailures; + using ServiceControl.Persistence.UnitOfWork; + using ServiceControl.Recoverability; + using Raven.Client.Documents.Commands.Batches; + using Raven.Client.Documents.Operations; + + class RavenDbRecoverabilityIngestionUnitOfWork : IRecoverabilityIngestionUnitOfWork + { + RavenDbIngestionUnitOfWork parentUnitOfWork; + + public RavenDbRecoverabilityIngestionUnitOfWork(RavenDbIngestionUnitOfWork parentUnitOfWork) + { + this.parentUnitOfWork = parentUnitOfWork; + } + + public Task RecordFailedProcessingAttempt( + string uniqueMessageId, + FailedMessage.ProcessingAttempt processingAttempt, + List groups) + { + parentUnitOfWork.AddCommand( + CreateFailedMessagesPatchCommand(uniqueMessageId, processingAttempt, groups) + ); + return Task.CompletedTask; + } + + public Task RecordSuccessfulRetry(string retriedMessageUniqueId) + { + var failedMessageDocumentId = FailedMessageIdGenerator.MakeDocumentId(retriedMessageUniqueId); + var failedMessageRetryDocumentId = FailedMessageRetry.MakeDocumentId(retriedMessageUniqueId); + + parentUnitOfWork.AddCommand(new PatchCommandData(failedMessageDocumentId, null, new PatchRequest + { + Script = $@"this.nameof(FailedMessage.Status) = {(int)FailedMessageStatus.Resolved};" + })); + + parentUnitOfWork.AddCommand(new DeleteCommandData(failedMessageRetryDocumentId, null)); + return Task.CompletedTask; + } + + ICommandData CreateFailedMessagesPatchCommand(string uniqueMessageId, FailedMessage.ProcessingAttempt processingAttempt, + List groups) + { + var documentId = FailedMessageIdGenerator.MakeDocumentId(uniqueMessageId); + + var serializedGroups = JToken.FromObject(groups); + var serializedAttempt = JToken.FromObject(processingAttempt, Serializer); + + //HINT: RavenDB 3.5 is using Lodash v4.13.1 to provide javascript utility functions + // https://ravendb.net/docs/article-page/3.5/csharp/client-api/commands/patches/how-to-use-javascript-to-patch-your-documents#methods-objects-and-variables + return new PatchCommandData(documentId, null, new PatchRequest + { + Script = $@"this.{nameof(FailedMessage.Status)} = status; + this.{nameof(FailedMessage.FailureGroups)} = failureGroups; + + var newAttempts = this.{nameof(FailedMessage.ProcessingAttempts)}; + + //De-duplicate attempts by AttemptedAt value + + var duplicateIndex = _.findIndex(this.{nameof(FailedMessage.ProcessingAttempts)}, function(a){{ + return a.{nameof(FailedMessage.ProcessingAttempt.AttemptedAt)} === attempt.{nameof(FailedMessage.ProcessingAttempt.AttemptedAt)}; + }}); + + if(duplicateIndex === -1){{ + newAttempts = _.union(newAttempts, [attempt]); + }} + + //Trim to the latest MaxProcessingAttempts + + newAttempts = _.sortBy(newAttempts, function(a) {{ + return a.{nameof(FailedMessage.ProcessingAttempt.AttemptedAt)}; + }}); + + if(newAttempts.length > {MaxProcessingAttempts}) + {{ + newAttempts = _.slice(newAttempts, newAttempts.length - {MaxProcessingAttempts}, newAttempts.length); + }} + + this.{nameof(FailedMessage.ProcessingAttempts)} = newAttempts; + ", + Values = new Dictionary + { + {"status", (int)FailedMessageStatus.Unresolved}, + {"failureGroups", serializedGroups}, + {"attempt", serializedAttempt} + }, + }, + patchIfMissing: new PatchRequest + { + Script = $@"this.{nameof(FailedMessage.Status)} = status; + this.{nameof(FailedMessage.FailureGroups)} = failureGroups; + this.{nameof(FailedMessage.ProcessingAttempts)} = [attempt]; + this.{nameof(FailedMessage.UniqueMessageId)} = uniqueMessageId; + this.@metadata = {FailedMessageMetadata} + ", + Values = new Dictionary + { + {"status", (int)FailedMessageStatus.Unresolved}, + {"failureGroups", serializedGroups}, + {"attempt", serializedAttempt}, + {"uniqueMessageId", uniqueMessageId} + } + }); + } + + static RavenDbRecoverabilityIngestionUnitOfWork() + { + //TODO: check if this actually works + Serializer = JsonSerializer.CreateDefault(); + Serializer.TypeNameHandling = TypeNameHandling.Auto; + + FailedMessageMetadata = JObject.Parse($@" + {{ + ""@collection"": ""{FailedMessageIdGenerator.CollectionName}"", + ""Raven-Clr-Type"": ""{typeof(FailedMessage).AssemblyQualifiedName}"" + }}"); + } + + static int MaxProcessingAttempts = 10; + static readonly JObject FailedMessageMetadata; + static readonly JsonSerializer Serializer; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/WaitHandleExtensions.cs b/src/ServiceControl.Persistence.RavenDb5/WaitHandleExtensions.cs new file mode 100644 index 0000000000..5535209353 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/WaitHandleExtensions.cs @@ -0,0 +1,44 @@ +namespace ServiceBus.Management.Infrastructure.Extensions +{ + using System; + using System.Threading; + using System.Threading.Tasks; + + static class WaitHandleExtensions + { + public static async Task WaitOneAsync(this WaitHandle handle, int millisecondsTimeout, CancellationToken cancellationToken = default) + { + RegisteredWaitHandle registeredHandle = null; + var tokenRegistration = default(CancellationTokenRegistration); + try + { + var tcs = new TaskCompletionSource(); + registeredHandle = ThreadPool.RegisterWaitForSingleObject( + handle, + (state, timedOut) => ((TaskCompletionSource)state).TrySetResult(!timedOut), + tcs, + millisecondsTimeout, + true); + tokenRegistration = cancellationToken.Register( + state => ((TaskCompletionSource)state).TrySetCanceled(), + tcs); + return await tcs.Task; + } + finally + { + registeredHandle?.Unregister(null); + tokenRegistration.Dispose(); + } + } + + public static Task WaitOneAsync(this WaitHandle handle, TimeSpan timeout, CancellationToken cancellationToken = default) + { + return handle.WaitOneAsync((int)timeout.TotalMilliseconds, cancellationToken); + } + + public static Task WaitOneAsync(this WaitHandle handle, CancellationToken cancellationToken = default) + { + return handle.WaitOneAsync(Timeout.Infinite, cancellationToken); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/persistence.manifest b/src/ServiceControl.Persistence.RavenDb5/persistence.manifest new file mode 100644 index 0000000000..4dace6b01e --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/persistence.manifest @@ -0,0 +1,24 @@ +{ + "Version": "1.0.0", + "Name": "RavenDB5", + "DisplayName": "RavenDB 5", + "Description": "RavenDB 5 ServiceControl persister", + "TypeName": "ServiceControl.Persistence.RavenDb.RavenDbPersistenceConfiguration, ServiceControl.Persistence.RavenDb5", + "Settings": [ + { + "Name": "ServiceControl/DBPath", + "Mandatory": true + }, + { + "Name": "ServiceControl/HostName", + "Mandatory": true + }, + { + "Name": "ServiceControl/DatabaseMaintenancePort", + "Mandatory": true + } + ], + "SettingsWithPathsToCleanup": [ + "ServiceControl/DBPath" + ] +} diff --git a/src/ServiceControl.Persistence.Tests/CustomChecksDataStoreTests.cs b/src/ServiceControl.Persistence.Tests/CustomChecksDataStoreTests.cs index d35058f7e7..589dfa591d 100644 --- a/src/ServiceControl.Persistence.Tests/CustomChecksDataStoreTests.cs +++ b/src/ServiceControl.Persistence.Tests/CustomChecksDataStoreTests.cs @@ -117,7 +117,7 @@ public async Task Should_delete_custom_checks() await CompleteDatabaseOperation(); var storedChecks = await CustomChecks.GetStats(new PagingInfo()); - var check = storedChecks.Results.Where(c => c.Id == checkId).ToList(); + var check = storedChecks.Results.Where(c => c.Id == checkId.ToString()).ToList(); Assert.AreEqual(0, check.Count); } diff --git a/src/ServiceControl.Persistence/CustomCheck.cs b/src/ServiceControl.Persistence/CustomCheck.cs index e0ee96eed3..584598a68f 100644 --- a/src/ServiceControl.Persistence/CustomCheck.cs +++ b/src/ServiceControl.Persistence/CustomCheck.cs @@ -6,7 +6,7 @@ public class CustomCheck { - public Guid Id { get; set; } + public string Id { get; set; } public string CustomCheckId { get; set; } public string Category { get; set; } public Status Status { get; set; } diff --git a/src/ServiceControl.Persistence/IErrorMessageDatastore.cs b/src/ServiceControl.Persistence/IErrorMessageDatastore.cs index 910783554e..865d19e208 100644 --- a/src/ServiceControl.Persistence/IErrorMessageDatastore.cs +++ b/src/ServiceControl.Persistence/IErrorMessageDatastore.cs @@ -17,7 +17,6 @@ public interface IErrorMessageDataStore Task>> GetAllMessagesForEndpoint(string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages); Task>> GetAllMessagesByConversation(string conversationId, PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages); Task>> GetAllMessagesForSearch(string searchTerms, PagingInfo pagingInfo, SortInfo sortInfo); - Task>> GetAllMessagesForEndpoint(string searchTerms, string receivingEndpointName, PagingInfo pagingInfo, SortInfo sortInfo); Task FailedMessageFetch(string failedMessageId); Task FailedMessageMarkAsArchived(string failedMessageId); Task FailedMessagesFetch(Guid[] ids); diff --git a/src/ServiceControl.Persistence/IPersistence.cs b/src/ServiceControl.Persistence/IPersistence.cs index 2a4bef4cdd..b974f1b261 100644 --- a/src/ServiceControl.Persistence/IPersistence.cs +++ b/src/ServiceControl.Persistence/IPersistence.cs @@ -6,6 +6,6 @@ public interface IPersistence { void Configure(IServiceCollection serviceCollection); IPersistenceInstaller CreateInstaller(); - IPersistenceLifecycle CreateLifecycle(); + void ConfigureLifecycle(IServiceCollection serviceCollection); } } diff --git a/src/ServiceControl.sln b/src/ServiceControl.sln index dfae66114e..b4779e2aa7 100644 --- a/src/ServiceControl.sln +++ b/src/ServiceControl.sln @@ -181,6 +181,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ServiceControl.Persistence. EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ServiceControl.AcceptanceTests.RavenDB", "ServiceControl.AcceptanceTests.RavenDB\ServiceControl.AcceptanceTests.RavenDB.csproj", "{33D5D084-FABB-4F65-A046-1C6626DF9AFB}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ServiceControl.Persistence.RavenDb5", "ServiceControl.Persistence.RavenDb5\ServiceControl.Persistence.RavenDb5.csproj", "{84627DC0-7B43-480D-80CF-A4DDA514A4EE}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1015,6 +1017,18 @@ Global {33D5D084-FABB-4F65-A046-1C6626DF9AFB}.Release|x64.Build.0 = Release|Any CPU {33D5D084-FABB-4F65-A046-1C6626DF9AFB}.Release|x86.ActiveCfg = Release|Any CPU {33D5D084-FABB-4F65-A046-1C6626DF9AFB}.Release|x86.Build.0 = Release|Any CPU + {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Debug|x64.ActiveCfg = Debug|Any CPU + {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Debug|x64.Build.0 = Debug|Any CPU + {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Debug|x86.ActiveCfg = Debug|Any CPU + {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Debug|x86.Build.0 = Debug|Any CPU + {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Release|Any CPU.Build.0 = Release|Any CPU + {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Release|x64.ActiveCfg = Release|Any CPU + {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Release|x64.Build.0 = Release|Any CPU + {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Release|x86.ActiveCfg = Release|Any CPU + {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1097,6 +1111,7 @@ Global {B93B8B7E-BEAF-4645-AA5C-2A1CADEC3D4B} = {9AF9D3C7-E859-451B-BA4D-B954D289213A} {F2BD40E4-A077-429A-8A22-1C80AFC240E8} = {350F72AB-142D-4AAD-9EF1-1A83DC991D87} {33D5D084-FABB-4F65-A046-1C6626DF9AFB} = {350F72AB-142D-4AAD-9EF1-1A83DC991D87} + {84627DC0-7B43-480D-80CF-A4DDA514A4EE} = {9B52418E-BF18-4D25-BE17-4B56D3FB1154} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {3B9E5B72-F580-465A-A22C-2D2148AF4EB4} diff --git a/src/ServiceControl/App.config b/src/ServiceControl/App.config index 0f471c0fbc..3305284eca 100644 --- a/src/ServiceControl/App.config +++ b/src/ServiceControl/App.config @@ -23,8 +23,8 @@ These settings are only here so that we can debug ServiceControl while developin --> - - + + diff --git a/src/ServiceControl/MessageFailures/Handlers/UnArchiveMessagesByRangeHandler.cs b/src/ServiceControl/MessageFailures/Handlers/UnArchiveMessagesByRangeHandler.cs index 7c98893fb7..b581e53037 100644 --- a/src/ServiceControl/MessageFailures/Handlers/UnArchiveMessagesByRangeHandler.cs +++ b/src/ServiceControl/MessageFailures/Handlers/UnArchiveMessagesByRangeHandler.cs @@ -1,7 +1,5 @@ namespace ServiceControl.MessageFailures.Handlers { - using System.Collections.Generic; - using System.Linq; using System.Threading.Tasks; using Contracts.MessageFailures; using Infrastructure.DomainEvents; diff --git a/src/ServiceControl/Persistence/PersistenceFactory.cs b/src/ServiceControl/Persistence/PersistenceFactory.cs index 9345e97b56..80dfdb4bb2 100644 --- a/src/ServiceControl/Persistence/PersistenceFactory.cs +++ b/src/ServiceControl/Persistence/PersistenceFactory.cs @@ -3,7 +3,6 @@ namespace ServiceControl.Persistence using System; using System.IO; using System.Linq; - using System.Web.Hosting; using ServiceBus.Management.Infrastructure.Settings; static class PersistenceFactory diff --git a/src/ServiceControl/Persistence/PersistenceHostBuilderExtensions.cs b/src/ServiceControl/Persistence/PersistenceHostBuilderExtensions.cs index ef34ba16d2..02b1300e81 100644 --- a/src/ServiceControl/Persistence/PersistenceHostBuilderExtensions.cs +++ b/src/ServiceControl/Persistence/PersistenceHostBuilderExtensions.cs @@ -20,9 +20,10 @@ public static IHostBuilder SetupPersistence(this IHostBuilder hostBuilder, Setti public static void CreatePersisterLifecyle(IServiceCollection serviceCollection, IPersistence persistence) { - var lifecycle = persistence.CreateLifecycle(); + persistence.ConfigureLifecycle(serviceCollection); + // lifecycle needs to be started before any other hosted service - serviceCollection.AddHostedService(_ => new PersistenceLifecycleHostedService(lifecycle)); + serviceCollection.AddHostedService(); persistence.Configure(serviceCollection); } }