From ff374312c3c898d1da2c287080b3c261f77a9c44 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 6 Sep 2023 13:44:50 +0200 Subject: [PATCH 01/78] Cloned the RavenDB 3.5 persister project to RavenDB5, updated the license and manifest files and an incomplete project change --- .../.editorconfig | 4 + ...ssages_UniqueMessageIdAndTimeOfFailures.cs | 27 + .../Chunker.cs | 40 + .../CustomChecks/CheckFreeDiskSpace.cs | 72 ++ ...CheckMinimumStorageRequiredForIngestion.cs | 90 +++ .../CustomChecks/CheckRavenDBIndexErrors.cs | 48 ++ .../CustomChecks/CheckRavenDBIndexLag.cs | 89 +++ .../FailedAuditImportCustomCheck.cs | 43 + .../FailedMessageIdGenerator.cs | 11 + .../MessageBodyIdGenerator.cs | 9 + .../Editing/EditFailedMessageManager.cs | 54 ++ .../Editing/FailedMessageEdit.cs | 16 + .../Editing/NotificationsManager.cs | 38 + .../EndpointDetailsParser.cs | 120 +++ .../ErrorMessagesDataStore.cs | 738 ++++++++++++++++++ .../EventLogDataStore.cs | 41 + .../Expiration/AuditMessageCleaner.cs | 166 ++++ .../Expiration/ErrorMessageCleaner.cs | 174 +++++ .../Expiration/EventLogItemsCleaner.cs | 112 +++ .../Expiration/ExpiredDocumentsCleaner.cs | 49 ++ .../ExpiredDocumentsCleanerBundle.cs | 125 +++ .../Expiration/ExpiryErrorMessageIndex.cs | 23 + .../Expiration/ExpiryEventLogItemsIndex.cs | 21 + .../Expiration/ExpiryProcessedMessageIndex.cs | 20 + .../Extensions.cs | 20 + .../Extensions/DictionaryExtensions.cs | 16 + .../Extensions/QueryResultConvert.cs | 15 + .../ExternalIntegrationRequestsDataStore.cs | 219 ++++++ .../FailedErrorImportDataStore.cs | 79 ++ .../FailedMessageViewIndexNotifications.cs | 110 +++ .../FailedMessageReclassifier.cs | 154 ++++ .../Indexes/ArchivedGroupsViewIndex.cs | 46 ++ .../Indexes/CustomChecksIndex.cs | 23 + .../Indexes/ExpirySagaAuditIndex.cs | 26 + .../Indexes/FailedErrorImportIndex.cs | 20 + .../Indexes/FailedMessageFacetsIndex.cs | 29 + .../Indexes/FailedMessageRetries_ByBatch.cs | 20 + .../Indexes/FailedMessageViewIndex.cs | 41 + .../Indexes/FailedMessages_ByGroup.cs | 34 + .../Indexes/FailureGroupsViewIndex.cs | 46 ++ .../Indexes/GroupCommentIndex.cs | 16 + .../Indexes/ImportFailedAudits.cs | 20 + .../Indexes/KnownEndpointIndex.cs | 23 + .../Indexes/MessagesViewIndex.cs | 80 ++ .../Indexes/QueueAddressIndex.cs | 30 + .../RetryBatches_ByStatusAndSession.cs | 20 + ...Batches_ByStatus_ReduceInitialBatchSize.cs | 47 ++ .../Indexes/SagaDetailsIndex.cs | 57 ++ .../Indexes/SagaHistoryCleaner.cs | 112 +++ .../Infrastructure/MigratedTypeAwareBinder.cs | 68 ++ .../Migrations/IDataMigration.cs | 10 + ...mporaryIdsThatAreDuplicateDataMigration.cs | 32 + .../Subscriptions/LegacyAddress.cs | 98 +++ .../Subscriptions/MessageTypeConverter.cs | 24 + .../RavenDbSubscriptionStorage.cs | 247 ++++++ .../Subscriptions/Subscription.cs | 45 ++ .../Subscriptions/SubscriptionClient.cs | 31 + .../SubscriptionsLegacyAddressConverter.cs | 50 ++ .../Infrastructure/TypeExtensions.cs | 16 + .../InternalsVisibleTo.cs | 7 + .../MessageRedirectsDataStore.cs | 46 ++ .../MetadataExtensions.cs | 40 + .../QueueAddressStore.cs | 53 ++ .../RavenAttachmentsBodyStorage.cs | 112 +++ .../RavenBootstrapper.cs | 232 ++++++ .../RavenDBPersisterSettings.cs | 24 + .../RavenDbCustomCheckDataStore.cs | 113 +++ .../RavenDbInstaller.cs | 40 + .../RavenDbMonitoringDataStore.cs | 131 ++++ .../RavenDbPersistence.cs | 95 +++ .../RavenDbPersistenceConfiguration.cs | 80 ++ .../RavenDbPersistenceLifecycle.cs | 62 ++ .../RavenLicense.json | 16 + .../RavenQueryExtensions.cs | 421 ++++++++++ .../RavenQueryStatisticsExtensions.cs | 17 + .../RavenStartup.cs | 19 + .../Recoverability/Archiving/ArchiveBatch.cs | 15 + .../Archiving/ArchiveDocumentManager.cs | 164 ++++ .../Archiving/ArchiveOperation.cs | 21 + .../Archiving/ArchiveOperationExtensions.cs | 39 + .../Archiving/ArchivingManager.cs | 112 +++ .../Archiving/MessageArchiver.cs | 242 ++++++ .../Archiving/UnarchiveBatch.cs | 15 + .../Archiving/UnarchiveDocumentManager.cs | 159 ++++ .../Archiving/UnarchiveOperation.cs | 21 + .../Archiving/UnarchivingManager.cs | 110 +++ .../Recoverability/GroupsDataStore.cs | 60 ++ .../Recoverability/RetryHistoryDataStore.cs | 87 +++ .../RepeatedFailuresOverTimeCircuitBreaker.cs | 74 ++ .../RetryBatchesDataStore.cs | 96 +++ .../RetryBatchesManager.cs | 58 ++ .../RetryDocumentDataStore.cs | 303 +++++++ .../SagaAudit/AuditRetentionCustomCheck.cs | 50 ++ .../SagaAudit/SagaAuditDataStore.cs | 45 ++ .../SagaAudit/SagaListIndex.cs | 45 ++ ...ServiceControl.Persistence.RavenDb5.csproj | 28 + .../RavenTransactionalDataStore.cs | 14 + .../FailedMessageViewTransformer.cs | 49 ++ .../Transformers/MessagesBodyTransformer.cs | 59 ++ .../Transformers/MessagesViewTransformer.cs | 76 ++ .../UnitOfWork/RavenDbIngestionUnitOfWork.cs | 28 + .../RavenDbIngestionUnitOfWorkFactory.cs | 26 + .../RavenDbMonitoringIngestionUnitOfWork.cs | 42 + ...avenDbRecoverabilityIngestionUnitOfWork.cs | 137 ++++ .../WaitHandleExtensions.cs | 44 ++ .../persistence.manifest | 24 + src/ServiceControl.sln | 15 + 107 files changed, 7820 insertions(+) create mode 100644 src/ServiceControl.Persistence.RavenDb5/.editorconfig create mode 100644 src/ServiceControl.Persistence.RavenDb5/BulkRetry/FailedMessages_UniqueMessageIdAndTimeOfFailures.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Chunker.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckFreeDiskSpace.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckMinimumStorageRequiredForIngestion.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexErrors.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexLag.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/CustomChecks/FailedAuditImportCustomCheck.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/FailedMessageIdGenerator.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/MessageBodyIdGenerator.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Editing/EditFailedMessageManager.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Editing/FailedMessageEdit.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Editing/NotificationsManager.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/EndpointDetailsParser.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/EventLogDataStore.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Expiration/AuditMessageCleaner.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Expiration/ErrorMessageCleaner.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Expiration/EventLogItemsCleaner.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiredDocumentsCleaner.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiredDocumentsCleanerBundle.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryErrorMessageIndex.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryEventLogItemsIndex.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryProcessedMessageIndex.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Extensions.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Extensions/DictionaryExtensions.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Extensions/QueryResultConvert.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/ExternalIntegrationRequestsDataStore.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/FailedErrorImportDataStore.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/FailedMessageViewIndexNotifications.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Indexes/ArchivedGroupsViewIndex.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Indexes/CustomChecksIndex.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Indexes/ExpirySagaAuditIndex.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Indexes/FailedErrorImportIndex.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageFacetsIndex.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageRetries_ByBatch.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageViewIndex.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessages_ByGroup.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Indexes/FailureGroupsViewIndex.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Indexes/GroupCommentIndex.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Indexes/ImportFailedAudits.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Indexes/KnownEndpointIndex.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Indexes/MessagesViewIndex.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Indexes/QueueAddressIndex.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Indexes/RetryBatches_ByStatusAndSession.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Indexes/RetryBatches_ByStatus_ReduceInitialBatchSize.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Indexes/SagaDetailsIndex.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Indexes/SagaHistoryCleaner.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Infrastructure/MigratedTypeAwareBinder.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/IDataMigration.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/LegacyAddress.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/MessageTypeConverter.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/RavenDbSubscriptionStorage.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/Subscription.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/SubscriptionClient.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/SubscriptionsLegacyAddressConverter.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Infrastructure/TypeExtensions.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/InternalsVisibleTo.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/MetadataExtensions.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/QueueAddressStore.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/RavenDbInstaller.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/RavenDbMonitoringDataStore.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceLifecycle.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/RavenLicense.json create mode 100644 src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/RavenQueryStatisticsExtensions.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/RavenStartup.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveBatch.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveDocumentManager.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveOperation.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveOperationExtensions.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchivingManager.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/MessageArchiver.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveBatch.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveDocumentManager.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveOperation.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchivingManager.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Recoverability/GroupsDataStore.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Recoverability/RetryHistoryDataStore.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/RepeatedFailuresOverTimeCircuitBreaker.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/RetryBatchesDataStore.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/RetryBatchesManager.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/SagaAudit/AuditRetentionCustomCheck.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaAuditDataStore.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaListIndex.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/ServiceControl.Persistence.RavenDb5.csproj create mode 100644 src/ServiceControl.Persistence.RavenDb5/Transactions/RavenTransactionalDataStore.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesBodyTransformer.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWorkFactory.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/WaitHandleExtensions.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/persistence.manifest diff --git a/src/ServiceControl.Persistence.RavenDb5/.editorconfig b/src/ServiceControl.Persistence.RavenDb5/.editorconfig new file mode 100644 index 0000000000..ff993b49bb --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/.editorconfig @@ -0,0 +1,4 @@ +[*.cs] + +# Justification: ServiceControl app has no synchronization context +dotnet_diagnostic.CA2007.severity = none diff --git a/src/ServiceControl.Persistence.RavenDb5/BulkRetry/FailedMessages_UniqueMessageIdAndTimeOfFailures.cs b/src/ServiceControl.Persistence.RavenDb5/BulkRetry/FailedMessages_UniqueMessageIdAndTimeOfFailures.cs new file mode 100644 index 0000000000..ba15cb1158 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/BulkRetry/FailedMessages_UniqueMessageIdAndTimeOfFailures.cs @@ -0,0 +1,27 @@ +namespace ServiceControl.Recoverability +{ + using System; + using System.Linq; + using MessageFailures; + using Raven.Client.Indexes; + + class FailedMessages_UniqueMessageIdAndTimeOfFailures : AbstractTransformerCreationTask + { + public FailedMessages_UniqueMessageIdAndTimeOfFailures() + { + TransformResults = failedMessages => from failedMessage in failedMessages + select new + { + failedMessage.UniqueMessageId, + LatestTimeOfFailure = failedMessage.ProcessingAttempts.Max(x => x.FailureDetails.TimeOfFailure) + }; + } + + public struct Result + { + public string UniqueMessageId { get; set; } + + public DateTime LatestTimeOfFailure { get; set; } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Chunker.cs b/src/ServiceControl.Persistence.RavenDb5/Chunker.cs new file mode 100644 index 0000000000..bae4cb0da8 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Chunker.cs @@ -0,0 +1,40 @@ +namespace ServiceControl.Infrastructure.RavenDB +{ + using System; + using System.Threading; + + static class Chunker + { + public static int ExecuteInChunks(int total, Func action, T1 t1, T2 t2, CancellationToken cancellationToken = default) + { + if (total == 0) + { + return 0; + } + + if (total < CHUNK_SIZE) + { + return action(t1, t2, 0, total - 1); + } + + int start = 0, end = CHUNK_SIZE - 1; + var chunkCount = 0; + do + { + chunkCount += action(t1, t2, start, end); + + start = end + 1; + end += CHUNK_SIZE; + if (end >= total) + { + end = total - 1; + } + } + while (start < total && !cancellationToken.IsCancellationRequested); + + return chunkCount; + } + + const int CHUNK_SIZE = 500; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckFreeDiskSpace.cs b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckFreeDiskSpace.cs new file mode 100644 index 0000000000..14fb9ae8d9 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckFreeDiskSpace.cs @@ -0,0 +1,72 @@ +namespace ServiceControl.Operations +{ + using System; + using System.IO; + using System.Threading.Tasks; + using NServiceBus.CustomChecks; + using NServiceBus.Logging; + using Persistence.RavenDb; + + class CheckFreeDiskSpace : CustomCheck + { + public CheckFreeDiskSpace(RavenDBPersisterSettings settings) : base("ServiceControl database", "Storage space", TimeSpan.FromMinutes(5)) + { + dataPath = settings.DatabasePath; + percentageThreshold = settings.DataSpaceRemainingThreshold; + + Logger.Debug($"Check ServiceControl data drive space remaining custom check starting. Threshold {percentageThreshold:P0}"); + } + + public override Task PerformCheck() + { + var dataPathRoot = Path.GetPathRoot(dataPath); + + if (dataPathRoot == null) + { + throw new Exception($"Unable to find the root of the data path {dataPath}"); + } + + var dataDriveInfo = new DriveInfo(dataPathRoot); + var availableFreeSpace = (decimal)dataDriveInfo.AvailableFreeSpace; + var totalSpace = (decimal)dataDriveInfo.TotalSize; + + var percentRemaining = (decimal)dataDriveInfo.AvailableFreeSpace / dataDriveInfo.TotalSize; + + if (Logger.IsDebugEnabled) + { + Logger.Debug($"Free space: {availableFreeSpace:N0}B | Total: {totalSpace:N0}B | Percent remaining {percentRemaining:P1}"); + } + + return percentRemaining > percentageThreshold + ? CheckResult.Pass + : CheckResult.Failed($"{percentRemaining:P0} disk space remaining on data drive '{dataDriveInfo.VolumeLabel} ({dataDriveInfo.RootDirectory})' on '{Environment.MachineName}'."); + } + + public static void Validate(RavenDBPersisterSettings settings) + { + var threshold = settings.DataSpaceRemainingThreshold; + + string message; + + if (threshold < 0) + { + message = $"{RavenDbPersistenceConfiguration.DataSpaceRemainingThresholdKey} is invalid, minimum value is 0."; + Logger.Fatal(message); + throw new Exception(message); + } + + if (threshold > 100) + { + message = $"{RavenDbPersistenceConfiguration.DataSpaceRemainingThresholdKey} is invalid, maximum value is 100."; + Logger.Fatal(message); + throw new Exception(message); + } + } + + readonly string dataPath; + readonly decimal percentageThreshold; + + public const int DataSpaceRemainingThresholdDefault = 20; + static readonly ILog Logger = LogManager.GetLogger(typeof(CheckFreeDiskSpace)); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckMinimumStorageRequiredForIngestion.cs b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckMinimumStorageRequiredForIngestion.cs new file mode 100644 index 0000000000..a696a1d45d --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckMinimumStorageRequiredForIngestion.cs @@ -0,0 +1,90 @@ +namespace ServiceControl.Operations +{ + using System; + using System.IO; + using System.Threading.Tasks; + using NServiceBus.CustomChecks; + using NServiceBus.Logging; + using Persistence.RavenDb; + using ServiceControl.Persistence; + + class CheckMinimumStorageRequiredForIngestion : CustomCheck + { + public CheckMinimumStorageRequiredForIngestion( + MinimumRequiredStorageState stateHolder, + RavenDBPersisterSettings settings) + : base("Message Ingestion Process", "ServiceControl Health", TimeSpan.FromSeconds(5)) + { + this.stateHolder = stateHolder; + this.settings = settings; + + dataPathRoot = Path.GetPathRoot(settings.DatabasePath); + } + + public override Task PerformCheck() + { + percentageThreshold = settings.MinimumStorageLeftRequiredForIngestion / 100m; + + if (dataPathRoot == null) + { + stateHolder.CanIngestMore = true; + return SuccessResult; + } + + Logger.Debug($"Check ServiceControl data drive space starting. Threshold {percentageThreshold:P0}"); + + var dataDriveInfo = new DriveInfo(dataPathRoot); + var availableFreeSpace = (decimal)dataDriveInfo.AvailableFreeSpace; + var totalSpace = (decimal)dataDriveInfo.TotalSize; + + var percentRemaining = (decimal)dataDriveInfo.AvailableFreeSpace / dataDriveInfo.TotalSize; + + if (Logger.IsDebugEnabled) + { + Logger.Debug($"Free space: {availableFreeSpace:N0}B | Total: {totalSpace:N0}B | Percent remaining {percentRemaining:P1}"); + } + + if (percentRemaining > percentageThreshold) + { + stateHolder.CanIngestMore = true; + return SuccessResult; + } + + var message = $"Error message ingestion stopped! {percentRemaining:P0} disk space remaining on data drive '{dataDriveInfo.VolumeLabel} ({dataDriveInfo.RootDirectory})' on '{Environment.MachineName}'. This is less than {percentageThreshold}% - the minimal required space configured. The threshold can be set using the {RavenBootstrapper.MinimumStorageLeftRequiredForIngestionKey} configuration setting."; + Logger.Warn(message); + stateHolder.CanIngestMore = false; + return CheckResult.Failed(message); + } + + public static void Validate(RavenDBPersisterSettings settings) + { + int threshold = settings.MinimumStorageLeftRequiredForIngestion; + + string message; + if (threshold < 0) + { + message = $"{RavenBootstrapper.MinimumStorageLeftRequiredForIngestionKey} is invalid, minimum value is 0."; + Logger.Fatal(message); + throw new Exception(message); + } + + if (threshold > 100) + { + message = $"{RavenBootstrapper.MinimumStorageLeftRequiredForIngestionKey} is invalid, maximum value is 100."; + Logger.Fatal(message); + throw new Exception(message); + } + } + + public const int MinimumStorageLeftRequiredForIngestionDefault = 5; + + readonly MinimumRequiredStorageState stateHolder; + readonly RavenDBPersisterSettings settings; + readonly string dataPathRoot; + + decimal percentageThreshold; + + static readonly Task SuccessResult = Task.FromResult(CheckResult.Pass); + static readonly ILog Logger = LogManager.GetLogger(typeof(CheckMinimumStorageRequiredForIngestion)); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexErrors.cs b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexErrors.cs new file mode 100644 index 0000000000..27f038687d --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexErrors.cs @@ -0,0 +1,48 @@ +namespace ServiceControl +{ + using System; + using System.Linq; + using System.Text; + using System.Threading.Tasks; + using NServiceBus.CustomChecks; + using NServiceBus.Logging; + using Raven.Client; + + class CheckRavenDBIndexErrors : CustomCheck + { + public CheckRavenDBIndexErrors(IDocumentStore store) + : base("Error Database Index Errors", "ServiceControl Health", TimeSpan.FromMinutes(5)) + { + this.store = store; + } + + public override Task PerformCheck() + { + var statistics = store.DatabaseCommands.GetStatistics(); + var indexes = statistics.Indexes.OrderBy(x => x.Name).ToArray(); + + if (statistics.Errors.Length == 0) + { + return CheckResult.Pass; + } + + var text = new StringBuilder(); + text.AppendLine("Detected RavenDB index errors, please start maintenance mode and resolve the following issues:"); + + foreach (var indexError in statistics.Errors) + { + text.AppendLine($"- Index [{indexError.IndexName}] error: {indexError.Error} (Action: {indexError.Action}, Doc: {indexError.Document}, At: {indexError.Timestamp})"); + } + + text.AppendLine().AppendLine("See: https://docs.particular.net/search?q=servicecontrol+troubleshooting"); + + var message = text.ToString(); + Logger.Error(message); + return CheckResult.Failed(message); + } + + static ILog Logger = LogManager.GetLogger(); + + IDocumentStore store; + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexLag.cs b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexLag.cs new file mode 100644 index 0000000000..f138e0efa0 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexLag.cs @@ -0,0 +1,89 @@ +namespace ServiceControl +{ + using System; + using System.Linq; + using System.Text; + using System.Threading.Tasks; + using NServiceBus.CustomChecks; + using NServiceBus.Logging; + using Raven.Client; + using Raven.Abstractions.Data; + using CustomCheck = NServiceBus.CustomChecks.CustomCheck; + + class CheckRavenDBIndexLag : CustomCheck + { + public CheckRavenDBIndexLag(IDocumentStore store) + : base("Error Database Index Lag", "ServiceControl Health", TimeSpan.FromMinutes(5)) + { + this.store = store; + } + + public override Task PerformCheck() + { + var statistics = store.DatabaseCommands.GetStatistics(); + var indexes = statistics.Indexes.OrderBy(x => x.Name).ToArray(); + + CreateDiagnosticsLogEntry(statistics, indexes); + + var indexCountWithTooMuchLag = CheckAndReportIndexesWithTooMuchIndexLag(indexes); + + if (indexCountWithTooMuchLag > 0) + { + return CheckResult.Failed($"At least one index significantly stale. Please run maintenance mode if this custom check persists to ensure index(es) can recover. See log file for more details. Visit https://docs.particular.net/search?q=servicecontrol+troubleshooting for more information."); + } + + return CheckResult.Pass; + } + + static int CheckAndReportIndexesWithTooMuchIndexLag(IndexStats[] indexes) + { + int indexCountWithTooMuchLag = 0; + + foreach (var indexStats in indexes) + { + // IndexingLag is the number of documents that the index is behind, it is not a time unit. + var indexLag = indexStats.IndexingLag.GetValueOrDefault(); + indexLag = Math.Abs(indexLag); + + if (indexLag > IndexLagThresholdError) + { + indexCountWithTooMuchLag++; + Log.Error($"Index [{indexStats.Name}] IndexingLag {indexLag:n0} is above error threshold ({IndexLagThresholdError:n0}). Launch in maintenance mode to let indexes catch up."); + } + else if (indexLag > IndexLagThresholdWarning) + { + indexCountWithTooMuchLag++; + Log.Warn($"Index [{indexStats.Name}] IndexingLag {indexLag:n0} is above warning threshold ({IndexLagThresholdWarning:n0}). Launch in maintenance mode to let indexes catch up."); + } + } + + return indexCountWithTooMuchLag; + } + + static void CreateDiagnosticsLogEntry(DatabaseStatistics statistics, IndexStats[] indexes) + { + if (!Log.IsDebugEnabled) + { + return; + } + + var report = new StringBuilder(); + report.AppendLine("Internal RavenDB index health report:"); + + foreach (var indexStats in indexes) + { + // IndexingLag is the number of documents that the index is behind, it is not a time unit. + var indexLag = indexStats.IndexingLag.GetValueOrDefault(); + indexLag = Math.Abs(indexLag); + report.AppendLine($"- Index [{indexStats.Name,-44}] Stale: {statistics.StaleIndexes.Contains(indexStats.Name),-5}, Lag: {indexLag,9:n0}, Valid: {indexStats.IsInvalidIndex,-5}, LastIndexed: {indexStats.LastIndexedTimestamp:u}, LastIndexing: {indexStats.LastIndexingTime:u}"); + } + Log.Debug(report.ToString()); + } + + const int IndexLagThresholdWarning = 10000; + const int IndexLagThresholdError = 100000; + static readonly ILog Log = LogManager.GetLogger(); + + readonly IDocumentStore store; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/FailedAuditImportCustomCheck.cs b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/FailedAuditImportCustomCheck.cs new file mode 100644 index 0000000000..bada097bf3 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/FailedAuditImportCustomCheck.cs @@ -0,0 +1,43 @@ +namespace ServiceControl.Operations +{ + using System; + using System.Threading.Tasks; + using NServiceBus.CustomChecks; + using NServiceBus.Logging; + using Raven.Client; + + class FailedAuditImportCustomCheck : CustomCheck + { + public FailedAuditImportCustomCheck(IDocumentStore store) + : base("Audit Message Ingestion", "ServiceControl Health", TimeSpan.FromHours(1)) + { + this.store = store; + } + + public override async Task PerformCheck() + { + using (var session = store.OpenAsyncSession()) + { + var query = session.Query(); + using (var ie = await session.Advanced.StreamAsync(query) + ) + { + if (await ie.MoveNextAsync()) + { + Logger.Warn(message); + return CheckResult.Failed(message); + } + } + } + + return CheckResult.Pass; + } + + readonly IDocumentStore store; + + const string message = @"One or more audit messages have failed to import properly into ServiceControl and have been stored in the ServiceControl database. +The import of these messages could have failed for a number of reasons and ServiceControl is not able to automatically reimport them. For guidance on how to resolve this see https://docs.particular.net/servicecontrol/import-failed-messages"; + + static readonly ILog Logger = LogManager.GetLogger(typeof(FailedAuditImportCustomCheck)); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/FailedMessageIdGenerator.cs b/src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/FailedMessageIdGenerator.cs new file mode 100644 index 0000000000..4935378ec9 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/FailedMessageIdGenerator.cs @@ -0,0 +1,11 @@ +static class FailedMessageIdGenerator +{ + public const string CollectionName = "FailedMessages"; + + public static string MakeDocumentId(string messageUniqueId) + { + return $"{CollectionName}/{messageUniqueId}"; + } + + public static string GetMessageIdFromDocumentId(string failedMessageDocumentId) => failedMessageDocumentId.Substring(CollectionName.Length + 1); +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/MessageBodyIdGenerator.cs b/src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/MessageBodyIdGenerator.cs new file mode 100644 index 0000000000..9891ffdc14 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/MessageBodyIdGenerator.cs @@ -0,0 +1,9 @@ +static class MessageBodyIdGenerator +{ + const string CollectionName = "messagebodies"; + + public static string MakeDocumentId(string messageUniqueId) + { + return $"{CollectionName}/{messageUniqueId}"; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Editing/EditFailedMessageManager.cs b/src/ServiceControl.Persistence.RavenDb5/Editing/EditFailedMessageManager.cs new file mode 100644 index 0000000000..3664c36fb8 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Editing/EditFailedMessageManager.cs @@ -0,0 +1,54 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System; + using System.Threading.Tasks; + using Raven.Abstractions.Data; + using Raven.Client; + using ServiceControl.MessageFailures; + using ServiceControl.Persistence.Recoverability.Editing; + + class EditFailedMessageManager : AbstractSessionManager, IEditFailedMessagesManager + { + readonly IAsyncDocumentSession session; + FailedMessage failedMessage; + + public EditFailedMessageManager(IAsyncDocumentSession session) + : base(session) + { + this.session = session; + } + + public async Task GetFailedMessage(string failedMessageId) + { + failedMessage = await session.LoadAsync(FailedMessageIdGenerator.MakeDocumentId(failedMessageId)); + return failedMessage; + } + + public async Task GetCurrentEditingMessageId(string failedMessageId) + { + var edit = await session.LoadAsync(FailedMessageEdit.MakeDocumentId(failedMessageId)); + return edit?.EditId; + } + + public Task SetCurrentEditingMessageId(string editingMessageId) + { + if (failedMessage == null) + { + throw new InvalidOperationException("No failed message loaded"); + } + return session.StoreAsync(new FailedMessageEdit + { + Id = FailedMessageEdit.MakeDocumentId(failedMessage.UniqueMessageId), + FailedMessageId = failedMessage.Id, + EditId = editingMessageId + }, Etag.Empty); + } + + public Task SetFailedMessageAsResolved() + { + // Instance is tracked by the document session + failedMessage.Status = FailedMessageStatus.Resolved; + return Task.CompletedTask; + } + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/Editing/FailedMessageEdit.cs b/src/ServiceControl.Persistence.RavenDb5/Editing/FailedMessageEdit.cs new file mode 100644 index 0000000000..2bc0e7a1e0 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Editing/FailedMessageEdit.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Persistence.Recoverability.Editing +{ + class FailedMessageEdit + { + public string Id { get; set; } + public string FailedMessageId { get; set; } + public string EditId { get; set; } + + public static string MakeDocumentId(string failedMessageId) + { + return $"{CollectionName}/{failedMessageId}"; + } + + const string CollectionName = "FailedMessageEdit"; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Editing/NotificationsManager.cs b/src/ServiceControl.Persistence.RavenDb5/Editing/NotificationsManager.cs new file mode 100644 index 0000000000..dd4d8fc4b3 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Editing/NotificationsManager.cs @@ -0,0 +1,38 @@ +namespace ServiceControl.Persistence.RavenDb.Editing +{ + using System; + using System.Threading.Tasks; + using Notifications; + using Raven.Client; + + class NotificationsManager : AbstractSessionManager, INotificationsManager + { + static readonly TimeSpan CacheTimeoutDefault = TimeSpan.FromMinutes(5); // Raven requires this to be at least 1 second + + public NotificationsManager(IAsyncDocumentSession session) : base(session) + { + } + + public async Task LoadSettings(TimeSpan? cacheTimeout = null) + { + + using (Session.Advanced.DocumentStore.AggressivelyCacheFor(cacheTimeout ?? CacheTimeoutDefault)) + { + var settings = await Session + .LoadAsync(NotificationsSettings.SingleDocumentId); + + if (settings == null) + { + settings = new NotificationsSettings + { + Id = NotificationsSettings.SingleDocumentId + }; + + await Session.StoreAsync(settings); + } + + return settings; + } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/EndpointDetailsParser.cs b/src/ServiceControl.Persistence.RavenDb5/EndpointDetailsParser.cs new file mode 100644 index 0000000000..1cda01af30 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/EndpointDetailsParser.cs @@ -0,0 +1,120 @@ +namespace ServiceControl +{ + using System; + using System.Collections.Generic; + using Infrastructure; + using NServiceBus; + using ServiceControl.Operations; + + class EndpointDetailsParser + { + public static EndpointDetails SendingEndpoint(IReadOnlyDictionary headers) + { + var endpointDetails = new EndpointDetails(); + + DictionaryExtensions.CheckIfKeyExists(Headers.OriginatingEndpoint, headers, s => endpointDetails.Name = s); + DictionaryExtensions.CheckIfKeyExists("NServiceBus.OriginatingMachine", headers, s => endpointDetails.Host = s); + DictionaryExtensions.CheckIfKeyExists(Headers.OriginatingHostId, headers, s => endpointDetails.HostId = Guid.Parse(s)); + + if (!string.IsNullOrEmpty(endpointDetails.Name) && !string.IsNullOrEmpty(endpointDetails.Host)) + { + return endpointDetails; + } + + string address = null; + DictionaryExtensions.CheckIfKeyExists(Headers.OriginatingAddress, headers, s => address = s); + + if (address != null) + { + var queueAndMachinename = ExtractQueueAndMachineName(address); + endpointDetails.Name = queueAndMachinename.Queue; + endpointDetails.Host = queueAndMachinename.Machine; + return endpointDetails; + } + + return null; + } + + public static EndpointDetails ReceivingEndpoint(IReadOnlyDictionary headers) + { + var endpoint = new EndpointDetails(); + + if (headers.TryGetValue(Headers.HostId, out var hostIdHeader)) + { + endpoint.HostId = Guid.Parse(hostIdHeader); + } + + if (headers.TryGetValue(Headers.HostDisplayName, out var hostDisplayNameHeader)) + { + endpoint.Host = hostDisplayNameHeader; + } + else + { + DictionaryExtensions.CheckIfKeyExists("NServiceBus.ProcessingMachine", headers, s => endpoint.Host = s); + } + + DictionaryExtensions.CheckIfKeyExists(Headers.ProcessingEndpoint, headers, s => endpoint.Name = s); + + if (!string.IsNullOrEmpty(endpoint.Name) && !string.IsNullOrEmpty(endpoint.Host)) + { + return endpoint; + } + + string address = null; + //use the failed q to determine the receiving endpoint + DictionaryExtensions.CheckIfKeyExists("NServiceBus.FailedQ", headers, s => address = s); + + // If we have a failed queue, then construct an endpoint from the failed queue information + if (address != null) + { + var queueAndMachinename = ExtractQueueAndMachineName(address); + + if (string.IsNullOrEmpty(endpoint.Name)) + { + endpoint.Name = queueAndMachinename.Queue; + } + + if (string.IsNullOrEmpty(endpoint.Host)) + { + endpoint.Host = queueAndMachinename.Machine; + } + + // If we've been now able to get the endpoint details, return the new info. + if (!string.IsNullOrEmpty(endpoint.Name) && !string.IsNullOrEmpty(endpoint.Host)) + { + return endpoint; + } + } + + return null; + } + + static QueueAndMachine ExtractQueueAndMachineName(string address) + { + var atIndex = address?.IndexOf("@", StringComparison.InvariantCulture); + + if (atIndex.HasValue && atIndex.Value > -1) + { + var queue = address.Substring(0, atIndex.Value); + var machine = address.Substring(atIndex.Value + 1); + return new QueueAndMachine + { + Queue = queue, + Machine = machine + }; + } + + return new QueueAndMachine + { + Queue = address, + Machine = null + }; + } + + struct QueueAndMachine + { + public string Queue; + public string Machine; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs new file mode 100644 index 0000000000..2de540abd0 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -0,0 +1,738 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System; + using System.Collections.Generic; + using System.Globalization; + using System.Linq; + using System.Text; + using System.Threading.Tasks; + using CompositeViews.Messages; + using Editing; + using NServiceBus.Logging; + using Raven.Abstractions.Data; + using Raven.Abstractions.Extensions; + using Raven.Client; + using Raven.Client.Linq; + using Raven.Json.Linq; + using ServiceControl.EventLog; + using ServiceControl.MessageFailures; + using ServiceControl.MessageFailures.Api; + using ServiceControl.Operations; + using ServiceControl.Persistence.Infrastructure; + using ServiceControl.Recoverability; + + class ErrorMessagesDataStore : IErrorMessageDataStore + { + readonly IDocumentStore documentStore; + + public ErrorMessagesDataStore(IDocumentStore documentStore) + { + this.documentStore = documentStore; + + } + + public async Task>> GetAllMessages( + PagingInfo pagingInfo, + SortInfo sortInfo, + bool includeSystemMessages + ) + { + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session.Query() + .IncludeSystemMessagesWhere(includeSystemMessages) + .Statistics(out var stats) + .Sort(sortInfo) + .Paging(pagingInfo) + .TransformWith() + .ToListAsync(); + + return new QueryResult>(results, stats.ToQueryStatsInfo()); + } + } + + public async Task>> GetAllMessagesForEndpoint( + string endpointName, + PagingInfo pagingInfo, + SortInfo sortInfo, + bool includeSystemMessages + ) + { + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session.Query() + .IncludeSystemMessagesWhere(includeSystemMessages) + .Where(m => m.ReceivingEndpointName == endpointName) + .Statistics(out var stats) + .Sort(sortInfo) + .Paging(pagingInfo) + .TransformWith() + .ToListAsync(); + + return new QueryResult>(results, stats.ToQueryStatsInfo()); + } + } + + public async Task>> SearchEndpointMessages( + string endpointName, + string searchKeyword, + PagingInfo pagingInfo, + SortInfo sortInfo + ) + { + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session.Query() + .Statistics(out var stats) + .Search(x => x.Query, searchKeyword) + .Where(m => m.ReceivingEndpointName == endpointName) + .Sort(sortInfo) + .Paging(pagingInfo) + .TransformWith() + .ToListAsync(); + + return new QueryResult>(results, stats.ToQueryStatsInfo()); + } + } + + public async Task>> GetAllMessagesByConversation( + string conversationId, + PagingInfo pagingInfo, + SortInfo sortInfo, + bool includeSystemMessages + ) + { + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session.Query() + .Statistics(out var stats) + .Where(m => m.ConversationId == conversationId) + .Sort(sortInfo) + .Paging(pagingInfo) + .TransformWith() + .ToListAsync(); + + return new QueryResult>(results, stats.ToQueryStatsInfo()); + } + } + + public async Task>> GetAllMessagesForSearch( + string searchTerms, + PagingInfo pagingInfo, + SortInfo sortInfo + ) + { + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session.Query() + .Statistics(out var stats) + .Search(x => x.Query, searchTerms) + .Sort(sortInfo) + .Paging(pagingInfo) + .TransformWith() + .ToListAsync(); + + return new QueryResult>(results, stats.ToQueryStatsInfo()); + } + } + + public async Task>> GetAllMessagesForEndpoint( + string searchTerms, + string receivingEndpointName, + PagingInfo pagingInfo, + SortInfo sortInfo + ) + { + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session.Query() + .Statistics(out var stats) + .Search(x => x.Query, searchTerms) + .Where(m => m.ReceivingEndpointName == receivingEndpointName) + .Sort(sortInfo) + .Paging(pagingInfo) + .TransformWith() + .ToListAsync(); + + return new QueryResult>(results, stats.ToQueryStatsInfo()); + } + } + + public async Task FailedMessageFetch(string failedMessageId) + { + using (var session = documentStore.OpenAsyncSession()) + { + return await session.LoadAsync(new Guid(failedMessageId)); + } + } + + public async Task FailedMessageMarkAsArchived(string failedMessageId) + { + using (var session = documentStore.OpenAsyncSession()) + { + var failedMessage = await session.LoadAsync(new Guid(failedMessageId)); + + if (failedMessage.Status != FailedMessageStatus.Archived) + { + failedMessage.Status = FailedMessageStatus.Archived; + } + + await session.SaveChangesAsync(); + } + } + + public async Task FailedMessagesFetch(Guid[] ids) + { + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session.LoadAsync(ids.Cast()); + return results.Where(x => x != null).ToArray(); + } + } + + public async Task StoreFailedErrorImport(FailedErrorImport failure) + { + using (var session = documentStore.OpenAsyncSession()) + { + await session.StoreAsync(failure); + + await session.SaveChangesAsync(); + } + } + + public Task CreateEditFailedMessageManager() + { + var session = documentStore.OpenAsyncSession(); + var manager = new EditFailedMessageManager(session); + return Task.FromResult((IEditFailedMessagesManager)manager); + } + + public async Task> GetFailureGroupView(string groupId, string status, string modified) + { + using (var session = documentStore.OpenAsyncSession()) + { + var document = await session.Advanced + .AsyncDocumentQuery() + .Statistics(out var stats) + .WhereEquals(group => group.Id, groupId) + .FilterByStatusWhere(status) + .FilterByLastModifiedRange(modified) + .FirstOrDefaultAsync(); + + return new QueryResult(document, stats.ToQueryStatsInfo()); + } + } + + public async Task> GetFailureGroupsByClassifier(string classifier) + { + using (var session = documentStore.OpenAsyncSession()) + { + var groups = session + .Query() + .Where(v => v.Type == classifier); + + var results = await groups + .OrderByDescending(x => x.Last) + .Take(200) // only show 200 groups + .ToListAsync(); + + return results; + } + } + + public async Task>> ErrorGet( + string status, + string modified, + string queueAddress, + PagingInfo pagingInfo, + SortInfo sortInfo + ) + { + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session.Advanced + .AsyncDocumentQuery() + .Statistics(out var stats) + .FilterByStatusWhere(status) + .FilterByLastModifiedRange(modified) + .FilterByQueueAddress(queueAddress) + .Sort(sortInfo) + .Paging(pagingInfo) + .SetResultTransformer(new FailedMessageViewTransformer().TransformerName) + .SelectFields() + .ToListAsync(); + + return new QueryResult>(results, stats.ToQueryStatsInfo()); + } + } + + public async Task ErrorsHead( + string status, + string modified, + string queueAddress + ) + { + using (var session = documentStore.OpenAsyncSession()) + { + var stats = await session.Advanced + .AsyncDocumentQuery() + .FilterByStatusWhere(status) + .FilterByLastModifiedRange(modified) + .FilterByQueueAddress(queueAddress) + .QueryResultAsync(); + + return stats.ToQueryStatsInfo(); + } + } + + public async Task>> ErrorsByEndpointName( + string status, + string endpointName, + string modified, + PagingInfo pagingInfo, + SortInfo sortInfo + ) + { + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session.Advanced + .AsyncDocumentQuery() + .Statistics(out var stats) + .FilterByStatusWhere(status) + .AndAlso() + .WhereEquals("ReceivingEndpointName", endpointName) + .FilterByLastModifiedRange(modified) + .Sort(sortInfo) + .Paging(pagingInfo) + .SetResultTransformer(new FailedMessageViewTransformer().TransformerName) + .SelectFields() + .ToListAsync(); + + return new QueryResult>(results, stats.ToQueryStatsInfo()); + } + } + + public async Task> ErrorsSummary() + { + using (var session = documentStore.OpenAsyncSession()) + { + var facetResults = await session.Query() + .ToFacetsAsync(new List + { + new Facet + { + Name = "Name", + DisplayName = "Endpoints" + }, + new Facet + { + Name = "Host", + DisplayName = "Hosts" + }, + new Facet + { + Name = "MessageType", + DisplayName = "Message types" + } + }); + + var results = facetResults + .Results + .ToDictionary( + x => x.Key, + x => (object)x.Value + ); + + return results; + } + } + + public async Task ErrorBy(Guid failedMessageId) + { + using (var session = documentStore.OpenAsyncSession()) + { + var message = await session.LoadAsync(failedMessageId); + return message; + } + } + + public async Task ErrorBy(string failedMessageId) + { + using (var session = documentStore.OpenAsyncSession()) + { + var message = await session.LoadAsync(FailedMessageIdGenerator.MakeDocumentId(failedMessageId)); + return message; + } + } + + public Task CreateNotificationsManager() + { + var session = documentStore.OpenAsyncSession(); + var manager = new NotificationsManager(session); + + return Task.FromResult(manager); + } + + public async Task ErrorLastBy(Guid failedMessageId) + { + using (var session = documentStore.OpenAsyncSession()) + { + var message = await session.LoadAsync(failedMessageId); + if (message == null) + { + return null; + } + var result = Map(message, session); + return result; + } + } + + static FailedMessageView Map(FailedMessage message, IAsyncDocumentSession session) + { + var processingAttempt = message.ProcessingAttempts.Last(); + + var metadata = processingAttempt.MessageMetadata; + var failureDetails = processingAttempt.FailureDetails; + var wasEdited = message.ProcessingAttempts.Last().Headers.ContainsKey("ServiceControl.EditOf"); + + var failedMsgView = new FailedMessageView + { + Id = message.UniqueMessageId, + MessageType = metadata.GetAsStringOrNull("MessageType"), + IsSystemMessage = metadata.GetOrDefault("IsSystemMessage"), + TimeSent = metadata.GetAsNullableDatetime("TimeSent"), + MessageId = metadata.GetAsStringOrNull("MessageId"), + Exception = failureDetails.Exception, + QueueAddress = failureDetails.AddressOfFailingEndpoint, + NumberOfProcessingAttempts = message.ProcessingAttempts.Count, + Status = message.Status, + TimeOfFailure = failureDetails.TimeOfFailure, + LastModified = session.Advanced.GetMetadataFor(message)["Last-Modified"].Value(), + Edited = wasEdited, + EditOf = wasEdited ? message.ProcessingAttempts.Last().Headers["ServiceControl.EditOf"] : "" + }; + + try + { + failedMsgView.SendingEndpoint = metadata.GetOrDefault("SendingEndpoint"); + } + catch (Exception ex) + { + Logger.Warn($"Unable to parse SendingEndpoint from metadata for messageId {message.UniqueMessageId}", ex); + failedMsgView.SendingEndpoint = EndpointDetailsParser.SendingEndpoint(processingAttempt.Headers); + } + + try + { + failedMsgView.ReceivingEndpoint = metadata.GetOrDefault("ReceivingEndpoint"); + } + catch (Exception ex) + { + Logger.Warn($"Unable to parse ReceivingEndpoint from metadata for messageId {message.UniqueMessageId}", ex); + failedMsgView.ReceivingEndpoint = EndpointDetailsParser.ReceivingEndpoint(processingAttempt.Headers); + } + + return failedMsgView; + } + + + public async Task EditComment(string groupId, string comment) + { + using (var session = documentStore.OpenAsyncSession()) + { + var groupComment = + await session.LoadAsync(GroupComment.MakeId(groupId)) + ?? new GroupComment { Id = GroupComment.MakeId(groupId) }; + + groupComment.Comment = comment; + + await session.StoreAsync(groupComment); + await session.SaveChangesAsync(); + } + } + + public async Task DeleteComment(string groupId) + { + using (var session = documentStore.OpenAsyncSession()) + { + session.Delete(GroupComment.MakeId(groupId)); + await session.SaveChangesAsync(); + } + } + + public async Task>> GetGroupErrors( + string groupId, + string status, + string modified, + SortInfo sortInfo, + PagingInfo pagingInfo + ) + { + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session.Advanced + .AsyncDocumentQuery() + .Statistics(out var stats) + .WhereEquals(view => view.FailureGroupId, groupId) + .FilterByStatusWhere(status) + .FilterByLastModifiedRange(modified) + .Sort(sortInfo) + .Paging(pagingInfo) + .SetResultTransformer(FailedMessageViewTransformer.Name) + .SelectFields() + .ToListAsync(); + + return results.ToQueryResult(stats); + } + } + + public async Task GetGroupErrorsCount(string groupId, string status, string modified) + { + using (var session = documentStore.OpenAsyncSession()) + { + var queryResult = await session.Advanced + .AsyncDocumentQuery() + .WhereEquals(view => view.FailureGroupId, groupId) + .FilterByStatusWhere(status) + .FilterByLastModifiedRange(modified) + .QueryResultAsync(); + + return queryResult.ToQueryStatsInfo(); + } + } + + public async Task>> GetGroup(string groupId, string status, string modified) + { + using (var session = documentStore.OpenAsyncSession()) + { + var queryResult = await session.Advanced + .AsyncDocumentQuery() + .Statistics(out var stats) + .WhereEquals(group => group.Id, groupId) + .FilterByStatusWhere(status) + .FilterByLastModifiedRange(modified) + .ToListAsync(); + + return queryResult.ToQueryResult(stats); + } + } + + public async Task MarkMessageAsResolved(string failedMessageId) + { + using (var session = documentStore.OpenAsyncSession()) + { + session.Advanced.UseOptimisticConcurrency = true; + + var failedMessage = await session.LoadAsync(new Guid(failedMessageId)); + + if (failedMessage == null) + { + return false; + } + + failedMessage.Status = FailedMessageStatus.Resolved; + + await session.SaveChangesAsync(); + + return true; + } + } + + public async Task ProcessPendingRetries(DateTime periodFrom, DateTime periodTo, string queueAddress, Func processCallback) + { + using (var session = documentStore.OpenAsyncSession()) + { + var prequery = session.Advanced + .AsyncDocumentQuery() + .WhereEquals("Status", (int)FailedMessageStatus.RetryIssued) + .AndAlso() + .WhereBetweenOrEqual("LastModified", periodFrom.Ticks, periodTo.Ticks); + + if (!string.IsNullOrWhiteSpace(queueAddress)) + { + prequery = prequery.AndAlso() + .WhereEquals(options => options.QueueAddress, queueAddress); + } + + var query = prequery + .SetResultTransformer(new FailedMessageViewTransformer().TransformerName) + .SelectFields(); + + using (var ie = await session.Advanced.StreamAsync(query)) + { + while (await ie.MoveNextAsync()) + { + await processCallback(ie.Current.Document.Id); + } + } + } + } + + class DocumentPatchResult + { + public string Document { get; set; } + } + + public async Task<(string[] ids, int count)> UnArchiveMessagesByRange(DateTime from, DateTime to, DateTime cutOff) + { + var options = new BulkOperationOptions + { + AllowStale = true + }; + + var result = await documentStore.AsyncDatabaseCommands.UpdateByIndexAsync( + new FailedMessageViewIndex().IndexName, + new IndexQuery + { + Query = string.Format(CultureInfo.InvariantCulture, "LastModified:[{0} TO {1}] AND Status:{2}", from.Ticks, to.Ticks, (int)FailedMessageStatus.Archived), + Cutoff = cutOff + }, new ScriptedPatchRequest + { + Script = @" +if(this.Status === archivedStatus) { + this.Status = unresolvedStatus; +} +", + Values = + { + {"archivedStatus", (int)FailedMessageStatus.Archived}, + {"unresolvedStatus", (int)FailedMessageStatus.Unresolved} + } + }, options); + + var patchedDocumentIds = (await result.WaitForCompletionAsync()) + .JsonDeserialization(); + + return ( + patchedDocumentIds.Select(x => FailedMessageIdGenerator.GetMessageIdFromDocumentId(x.Document)).ToArray(), + patchedDocumentIds.Length + ); + } + + public async Task<(string[] ids, int count)> UnArchiveMessages(IEnumerable failedMessageIds) + { + FailedMessage[] failedMessages; + + using (var session = documentStore.OpenAsyncSession()) + { + session.Advanced.UseOptimisticConcurrency = true; + + var documentIds = failedMessageIds.Select(FailedMessageIdGenerator.MakeDocumentId); + + failedMessages = await session.LoadAsync(documentIds); + + foreach (var failedMessage in failedMessages) + { + if (failedMessage.Status == FailedMessageStatus.Archived) + { + failedMessage.Status = FailedMessageStatus.Unresolved; + } + } + + await session.SaveChangesAsync(); + } + + return ( + failedMessages.Select(x => x.UniqueMessageId).ToArray(), + failedMessages.Length + ); + } + + public async Task RevertRetry(string messageUniqueId) + { + using (var session = documentStore.OpenAsyncSession()) + { + var failedMessage = await session + .LoadAsync(FailedMessageIdGenerator.MakeDocumentId(messageUniqueId)); + if (failedMessage != null) + { + failedMessage.Status = FailedMessageStatus.Unresolved; + } + + var failedMessageRetry = await session + .LoadAsync(FailedMessageRetry.MakeDocumentId(messageUniqueId)); + if (failedMessageRetry != null) + { + session.Delete(failedMessageRetry); + } + + await session.SaveChangesAsync(); + } + } + + public Task RemoveFailedMessageRetryDocument(string uniqueMessageId) + { + return documentStore.AsyncDatabaseCommands.DeleteAsync(FailedMessageRetry.MakeDocumentId(uniqueMessageId), null); + } + + // TODO: Once using .NET, consider using IAsyncEnumerable here as this is an unbounded query + public async Task GetRetryPendingMessages(DateTime from, DateTime to, string queueAddress) + { + var ids = new List(); + + using (var session = documentStore.OpenAsyncSession()) + { + var query = session.Advanced + .AsyncDocumentQuery() + .WhereEquals("Status", (int)FailedMessageStatus.RetryIssued) + .AndAlso() + .WhereBetweenOrEqual(options => options.LastModified, from.Ticks, to.Ticks) + .AndAlso() + .WhereEquals(o => o.QueueAddress, queueAddress) + .SetResultTransformer(FailedMessageViewTransformer.Name) + .SelectFields(new[] { "Id" }); + + using (var ie = await session.Advanced.StreamAsync(query)) + { + while (await ie.MoveNextAsync()) + { + ids.Add(ie.Current.Document.Id); + } + } + } + + return ids.ToArray(); + } + + public async Task FetchFromFailedMessage(string uniqueMessageId) + { + string documentId = FailedMessageIdGenerator.MakeDocumentId(uniqueMessageId); + var results = await documentStore.AsyncDatabaseCommands.GetAsync(new[] { documentId }, null, + transformer: MessagesBodyTransformer.Name); + + string resultBody = ((results.Results?.SingleOrDefault()?["$values"] as RavenJArray)?.SingleOrDefault() as RavenJObject) + ?.ToObject()?.Body; + + if (resultBody != null) + { + return Encoding.UTF8.GetBytes(resultBody); + } + + return null; + } + + public async Task StoreEventLogItem(EventLogItem logItem) + { + using (var session = documentStore.OpenAsyncSession()) + { + await session.StoreAsync(logItem); + await session.SaveChangesAsync(); + } + } + + public async Task StoreFailedMessagesForTestsOnly(params FailedMessage[] failedMessages) + { + using (var session = documentStore.OpenAsyncSession()) + { + foreach (var message in failedMessages) + { + await session.StoreAsync(message); + } + + await session.SaveChangesAsync(); + } + } + + static readonly ILog Logger = LogManager.GetLogger(); + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/EventLogDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/EventLogDataStore.cs new file mode 100644 index 0000000000..6fbafb64f6 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/EventLogDataStore.cs @@ -0,0 +1,41 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System.Collections.Generic; + using System.Threading.Tasks; + using EventLog; + using Persistence.Infrastructure; + using Raven.Client; + using Raven.Client.Linq; + + class EventLogDataStore : IEventLogDataStore + { + readonly IDocumentStore documentStore; + + public EventLogDataStore(IDocumentStore documentStore) + { + this.documentStore = documentStore; + } + + public async Task Add(EventLogItem logItem) + { + using (var session = documentStore.OpenAsyncSession()) + { + await session.StoreAsync(logItem); + await session.SaveChangesAsync(); + } + } + + public async Task<(IList, int, string)> GetEventLogItems(PagingInfo pagingInfo) + { + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session.Query().Statistics(out var stats) + .OrderByDescending(p => p.RaisedAt) + .Paging(pagingInfo) + .ToListAsync(); + + return (results, stats.TotalResults, stats.ResultEtag.ToString()); + } + } + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/AuditMessageCleaner.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/AuditMessageCleaner.cs new file mode 100644 index 0000000000..74a9d3b3af --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Expiration/AuditMessageCleaner.cs @@ -0,0 +1,166 @@ +namespace ServiceControl.Infrastructure.RavenDB.Expiration +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Linq; + using System.Threading; + using NServiceBus.Logging; + using Raven.Abstractions; + using Raven.Abstractions.Commands; + using Raven.Abstractions.Data; + using Raven.Abstractions.Exceptions; + using Raven.Database; + using Raven.Json.Linq; + + static class AuditMessageCleaner + { + public static void Clean(int deletionBatchSize, DocumentDatabase database, DateTime expiryThreshold, CancellationToken cancellationToken = default) + { + var stopwatch = Stopwatch.StartNew(); + var items = new List(deletionBatchSize); + var attachments = new List(deletionBatchSize); + var itemsAndAttachements = Tuple.Create(items, attachments); + var indexName = new ExpiryProcessedMessageIndex().IndexName; + + try + { + var query = new IndexQuery + { + Start = 0, + PageSize = deletionBatchSize, + Cutoff = SystemTime.UtcNow, + DisableCaching = true, + Query = $"ProcessedAt:[* TO {expiryThreshold.Ticks}]", + FieldsToFetch = new[] + { + "__document_id", + "MessageMetadata.MessageId", + "MessageMetadata.BodyNotStored" + }, + SortedFields = new[] + { + new SortedField("ProcessedAt") + { + Field = "ProcessedAt", + Descending = false + } + } + }; + + database.Query(indexName, query, (doc, state) => + { + var id = doc.Value("__document_id"); + if (string.IsNullOrEmpty(id)) + { + return; + } + + state.Item1.Add(new DeleteCommandData + { + Key = id + }); + + if (TryGetBodyId(doc, out var bodyId)) + { + state.Item2.Add(bodyId); + } + }, + itemsAndAttachements, cancellationToken); + } + catch (IndexDisabledException ex) + { + logger.Error($"Unable to cleanup audit messages. The index ${indexName} was disabled.", ex); + return; + } + catch (OperationCanceledException) + { + logger.Info("Cleanup operation cancelled"); + return; + } + + if (cancellationToken.IsCancellationRequested) + { + return; + } + + var deletedAuditDocuments = Chunker.ExecuteInChunks(items.Count, (itemsForBatch, db, s, e) => + { + if (logger.IsDebugEnabled) + { + logger.Debug($"Batching deletion of {s}-{e} audit documents."); + } + + var results = db.Batch(itemsForBatch.GetRange(s, e - s + 1), CancellationToken.None); + if (logger.IsDebugEnabled) + { + logger.Debug($"Batching deletion of {s}-{e} audit documents completed."); + } + + return results.Count(x => x.Deleted == true); + }, items, database, cancellationToken); + + var deletedAttachments = Chunker.ExecuteInChunks(attachments.Count, (att, db, s, e) => + { + var deleted = 0; + if (logger.IsDebugEnabled) + { + logger.Debug($"Batching deletion of {s}-{e} attachment audit documents."); + } + + db.TransactionalStorage.Batch(accessor => + { + for (var idx = s; idx <= e; idx++) + { + //We want to continue using attachments for now +#pragma warning disable 618 + accessor.Attachments.DeleteAttachment(att[idx], null); +#pragma warning restore 618 + deleted++; + } + }); + if (logger.IsDebugEnabled) + { + logger.Debug($"Batching deletion of {s}-{e} attachment audit documents completed."); + } + + return deleted; + }, attachments, database, cancellationToken); + + if (deletedAttachments + deletedAuditDocuments == 0) + { + if (logger.IsDebugEnabled) + { + logger.Debug("No expired audit documents found"); + } + } + else + { + if (logger.IsDebugEnabled) + { + logger.Debug($"Deleted {deletedAuditDocuments} expired audit documents and {deletedAttachments} message body attachments. Batch execution took {stopwatch.ElapsedMilliseconds} ms"); + } + } + } + + static bool TryGetBodyId(RavenJObject doc, out string bodyId) + { + bodyId = null; + if (doc.Value("MessageMetadata.BodyNotStored")) + { + return false; + } + + var messageId = doc.Value("MessageMetadata.MessageId"); + if (messageId == null) + { + return false; + } + + bodyId = $"messagebodies/{messageId}"; + return true; + } + + static ILog logger = LogManager.GetLogger(typeof(AuditMessageCleaner)); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/ErrorMessageCleaner.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/ErrorMessageCleaner.cs new file mode 100644 index 0000000000..8e91475081 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Expiration/ErrorMessageCleaner.cs @@ -0,0 +1,174 @@ +namespace ServiceControl.Infrastructure.RavenDB.Expiration +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Linq; + using System.Threading; + using NServiceBus.Logging; + using Raven.Abstractions; + using Raven.Abstractions.Commands; + using Raven.Abstractions.Data; + using Raven.Abstractions.Exceptions; + using Raven.Database; + using ServiceControl.MessageFailures; + using ServiceControl.Recoverability; + + static class ErrorMessageCleaner + { + public static void Clean(int deletionBatchSize, DocumentDatabase database, DateTime expiryThreshold, CancellationToken cancellationToken = default) + { + var stopwatch = Stopwatch.StartNew(); + var items = new List(deletionBatchSize); + var attachments = new List(deletionBatchSize); + var failedRetryItems = new List(deletionBatchSize); + var indexName = new ExpiryErrorMessageIndex().IndexName; + + var itemsAndAttachements = new + { + items, + attachments, + failedRetryItems + }; + + try + { + var query = new IndexQuery + { + Start = 0, + PageSize = deletionBatchSize, + Cutoff = SystemTime.UtcNow, + DisableCaching = true, + Query = $"Status:[2 TO 4] AND LastModified:[* TO {expiryThreshold.Ticks}]", + FieldsToFetch = new[] + { + "__document_id", + "ProcessingAttempts[0].MessageId" + }, + SortedFields = new[] + { + new SortedField("LastModified") + { + Field = "LastModified", + Descending = false + } + } + }; + + database.Query(indexName, query, (doc, state) => + { + var id = doc.Value("__document_id"); + if (string.IsNullOrEmpty(id)) + { + return; + } + + var failedMessageRetryId = FailedMessageRetry.MakeDocumentId(FailedMessageIdGenerator.GetMessageIdFromDocumentId(id)); + state.failedRetryItems.Add(new DeleteCommandData + { + Key = failedMessageRetryId + }); + + state.items.Add(new DeleteCommandData + { + Key = id + }); + var bodyid = doc.Value("ProcessingAttempts[0].MessageId"); + state.attachments.Add(bodyid); + }, + itemsAndAttachements, cancellationToken); + } + catch (IndexDisabledException ex) + { + logger.Error($"Unable to cleanup error messages. The index ${indexName} was disabled.", ex); + return; + } + catch (OperationCanceledException) + { + logger.Info("Cleanup operation cancelled"); + return; + } + + if (cancellationToken.IsCancellationRequested) + { + return; + } + + var deletedFailedMessageRetry = Chunker.ExecuteInChunks(failedRetryItems.Count, (itemsForBatch, db, s, e) => + { + if (logger.IsDebugEnabled) + { + logger.Debug($"Batching deletion of {s}-{e} FailedMessageRetry documents."); + } + + var results = db.Batch(itemsForBatch.GetRange(s, e - s + 1), CancellationToken.None); + if (logger.IsDebugEnabled) + { + logger.Debug($"Batching deletion of {s}-{e} FailedMessageRetry documents completed."); + } + + return results.Count(x => x.Deleted == true); + }, failedRetryItems, database, cancellationToken); + + var deletedAttachments = Chunker.ExecuteInChunks(attachments.Count, (atts, db, s, e) => + { + var deleted = 0; + if (logger.IsDebugEnabled) + { + logger.Debug($"Batching deletion of {s}-{e} attachment error documents."); + } + + db.TransactionalStorage.Batch(accessor => + { + for (var idx = s; idx <= e; idx++) + { + //We want to continue using attachments for now +#pragma warning disable 618 + accessor.Attachments.DeleteAttachment("messagebodies/" + attachments[idx], null); +#pragma warning restore 618 + deleted++; + } + }); + if (logger.IsDebugEnabled) + { + logger.Debug($"Batching deletion of {s}-{e} attachment error documents completed."); + } + + return deleted; + }, attachments, database, cancellationToken); + + var deletedFailedMessage = Chunker.ExecuteInChunks(items.Count, (itemsForBatch, db, s, e) => + { + if (logger.IsDebugEnabled) + { + logger.Debug($"Batching deletion of {s}-{e} error documents."); + } + + var results = db.Batch(itemsForBatch.GetRange(s, e - s + 1), CancellationToken.None); + if (logger.IsDebugEnabled) + { + logger.Debug($"Batching deletion of {s}-{e} error documents completed."); + } + + return results.Count(x => x.Deleted == true); + }, items, database, cancellationToken); + + if (deletedFailedMessage + deletedAttachments + deletedFailedMessageRetry == 0) + { + if (logger.IsDebugEnabled) + { + logger.Debug("No expired error documents found"); + } + } + else + { + if (logger.IsDebugEnabled) + { + logger.Debug($"Deleted {deletedFailedMessage} expired error documents and {deletedAttachments} message body attachments. Batch execution took {stopwatch.ElapsedMilliseconds} ms"); + } + } + } + + static ILog logger = LogManager.GetLogger(typeof(ErrorMessageCleaner)); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/EventLogItemsCleaner.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/EventLogItemsCleaner.cs new file mode 100644 index 0000000000..3e037c35cb --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Expiration/EventLogItemsCleaner.cs @@ -0,0 +1,112 @@ +namespace ServiceControl.Infrastructure.RavenDB.Expiration +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Linq; + using System.Threading; + using NServiceBus.Logging; + using Raven.Abstractions; + using Raven.Abstractions.Commands; + using Raven.Abstractions.Data; + using Raven.Abstractions.Exceptions; + using Raven.Database; + + static class EventLogItemsCleaner + { + public static void Clean(int deletionBatchSize, DocumentDatabase database, DateTime expiryThreshold, CancellationToken cancellationToken = default) + { + var stopwatch = Stopwatch.StartNew(); + var items = new List(deletionBatchSize); + var indexName = new ExpiryEventLogItemsIndex().IndexName; + + try + { + var query = new IndexQuery + { + Start = 0, + DisableCaching = true, + Cutoff = SystemTime.UtcNow, + PageSize = deletionBatchSize, + Query = $"LastModified:[* TO {expiryThreshold.Ticks}]", + FieldsToFetch = new[] + { + "__document_id" + }, + SortedFields = new[] + { + new SortedField("LastModified") + { + Field = "LastModified", + Descending = false + } + } + }; + + database.Query(indexName, query, (doc, commands) => + { + var id = doc.Value("__document_id"); + if (string.IsNullOrEmpty(id)) + { + return; + } + + commands.Add(new DeleteCommandData + { + Key = id + }); + }, + items, cancellationToken); + } + catch (IndexDisabledException ex) + { + logger.Error($"Unable to cleanup event log items. The index ${indexName} was disabled.", ex); + return; + } + catch (OperationCanceledException) + { + logger.Info("Cleanup operation cancelled"); + return; + } + + if (cancellationToken.IsCancellationRequested) + { + return; + } + + var deletionCount = Chunker.ExecuteInChunks(items.Count, (itemsForBatch, db, s, e) => + { + if (logger.IsDebugEnabled) + { + logger.Debug($"Batching deletion of {s}-{e} event log documents."); + } + + var results = db.Batch(itemsForBatch.GetRange(s, e - s + 1), CancellationToken.None); + + if (logger.IsDebugEnabled) + { + logger.Debug($"Batching deletion of {s}-{e} event log documents completed."); + } + + return results.Count(x => x.Deleted == true); + }, items, database, cancellationToken); + + if (deletionCount == 0) + { + if (logger.IsDebugEnabled) + { + logger.Debug("No expired event log documents found"); + } + } + else + { + if (logger.IsDebugEnabled) + { + logger.Debug($"Deleted {deletionCount} expired event log documents. Batch execution took {stopwatch.ElapsedMilliseconds} ms"); + } + } + } + + static ILog logger = LogManager.GetLogger(typeof(EventLogItemsCleaner)); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiredDocumentsCleaner.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiredDocumentsCleaner.cs new file mode 100644 index 0000000000..ec610a1bc1 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiredDocumentsCleaner.cs @@ -0,0 +1,49 @@ +namespace ServiceControl.Infrastructure.RavenDB.Expiration +{ + using System.Globalization; + using System.Threading; + using System.Threading.Tasks; + using BackgroundTasks; + using NServiceBus.Logging; + using Raven.Abstractions; + using Raven.Database; + using SagaAudit; + + class ExpiredDocumentsCleaner + { + public static Task RunCleanup(int deletionBatchSize, DocumentDatabase database, RavenDBPersisterSettings settings, CancellationToken cancellationToken = default) + { + var threshold = SystemTime.UtcNow.Add(settings.ErrorRetentionPeriod); + + if (logger.IsDebugEnabled) + { + logger.Debug($"Trying to find expired FailedMessage documents to delete (with threshold {threshold.ToString(Default.DateTimeFormatsToWrite, CultureInfo.InvariantCulture)})"); + } + ErrorMessageCleaner.Clean(deletionBatchSize, database, threshold, cancellationToken); + + threshold = SystemTime.UtcNow.Add(settings.EventsRetentionPeriod); + + if (logger.IsDebugEnabled) + { + logger.Debug($"Trying to find expired EventLogItem documents to delete (with threshold {threshold.ToString(Default.DateTimeFormatsToWrite, CultureInfo.InvariantCulture)})"); + } + EventLogItemsCleaner.Clean(deletionBatchSize, database, threshold, cancellationToken); + + if (settings.AuditRetentionPeriod.HasValue) + { + threshold = SystemTime.UtcNow.Add(settings.AuditRetentionPeriod.Value); + + if (logger.IsDebugEnabled) + { + logger.Debug($"Trying to find expired ProcessedMessage and SagaHistory documents to delete (with threshold {threshold.ToString(Default.DateTimeFormatsToWrite, CultureInfo.InvariantCulture)})"); + } + AuditMessageCleaner.Clean(deletionBatchSize, database, threshold, cancellationToken); + SagaHistoryCleaner.Clean(deletionBatchSize, database, threshold, cancellationToken); + } + + return Task.FromResult(TimerJobExecutionResult.ScheduleNextExecution); + } + + static ILog logger = LogManager.GetLogger(typeof(ExpiredDocumentsCleaner)); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiredDocumentsCleanerBundle.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiredDocumentsCleanerBundle.cs new file mode 100644 index 0000000000..bcaf23c33b --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiredDocumentsCleanerBundle.cs @@ -0,0 +1,125 @@ +namespace ServiceControl.Infrastructure.RavenDB.Expiration +{ + using System; + using System.ComponentModel.Composition; + using System.Threading.Tasks; + using BackgroundTasks; + using NServiceBus.Logging; + using Persistence.RavenDb; + using Raven.Database; + using Raven.Database.Plugins; + + [InheritedExport(typeof(IStartupTask))] + [ExportMetadata("Bundle", "customDocumentExpiration")] + class ExpiredDocumentsCleanerBundle : IStartupTask, IDisposable + { + // TODO: Ensure that the timers are started when the persister starts! + + public void Dispose() + { + lock (this) + { + if (timer == null) + { + return; + } + + var stopTask = timer.Stop(); + var delayTask = Task.Delay(TimeSpan.FromSeconds(30)); + var composite = Task.WhenAny(stopTask, delayTask); + + var finishedTask = composite.GetAwaiter().GetResult(); + if (finishedTask == delayTask) + { + Logger.Error("Cleanup process did not finish on time. Forcing shutdown."); + } + else + { + Logger.Info("Expired documents cleanup process stopped."); + } + + timer = null; + } + } + + public void Execute(DocumentDatabase database) + { + var persistenceSettings = RavenBootstrapper.Settings; + + if (ExpirationProcessTimerInSeconds == 0) + { + return; + } + + var due = TimeSpan.FromSeconds(ExpirationProcessTimerInSeconds); + var deletionBatchSize = ExpirationProcessBatchSize; + + Logger.Info($"Running deletion of expired documents every {ExpirationProcessTimerInSeconds} seconds"); + Logger.Info($"Deletion batch size set to {deletionBatchSize}"); + Logger.Info($"Retention period for errors is {persistenceSettings.ErrorRetentionPeriod}"); + + var auditRetention = persistenceSettings.AuditRetentionPeriod; + + if (auditRetention.HasValue) + { + Logger.InfoFormat("Retention period for audits and saga history is {0}", persistenceSettings.AuditRetentionPeriod); + } + + timer = new TimerJob( + token => ExpiredDocumentsCleaner.RunCleanup(deletionBatchSize, database, persistenceSettings, token), due, due, e => { Logger.Error("Error when trying to find expired documents", e); }); + } + + int ExpirationProcessTimerInSeconds + { + get + { + var expirationProcessTimerInSeconds = settings.ExpirationProcessTimerInSeconds; + + if (expirationProcessTimerInSeconds < 0) + { + Logger.Error($"ExpirationProcessTimerInSeconds cannot be negative. Defaulting to {ExpirationProcessTimerInSecondsDefault}"); + return ExpirationProcessTimerInSecondsDefault; + } + + if (expirationProcessTimerInSeconds > TimeSpan.FromHours(3).TotalSeconds) + { + Logger.Error($"ExpirationProcessTimerInSeconds cannot be larger than {TimeSpan.FromHours(3).TotalSeconds}. Defaulting to {ExpirationProcessTimerInSecondsDefault}"); + return ExpirationProcessTimerInSecondsDefault; + } + + return expirationProcessTimerInSeconds; + } + } + + public int ExpirationProcessBatchSize + { + get + { + var expirationProcessBatchSize = settings.ExpirationProcessBatchSize; + + if (expirationProcessBatchSize < 1) + { + Logger.Error($"ExpirationProcessBatchSize cannot be less than 1. Defaulting to {ExpirationProcessBatchSizeDefault}"); + return ExpirationProcessBatchSizeDefault; + } + + if (expirationProcessBatchSize < ExpirationProcessBatchSizeMinimum) + { + Logger.Error($"ExpirationProcessBatchSize cannot be less than {ExpirationProcessBatchSizeMinimum}. Defaulting to {ExpirationProcessBatchSizeDefault}"); + return ExpirationProcessBatchSizeDefault; + } + + return expirationProcessBatchSize; + } + } + + public const int ExpirationProcessTimerInSecondsDefault = 600; + public const int ExpirationProcessBatchSizeDefault = 65512; + const int ExpirationProcessBatchSizeMinimum = 10240; + + readonly RavenDBPersisterSettings settings = RavenBootstrapper.Settings; + + static readonly ILog Logger = LogManager.GetLogger(typeof(ExpiredDocumentsCleanerBundle)); + TimerJob timer; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryErrorMessageIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryErrorMessageIndex.cs new file mode 100644 index 0000000000..39f18df6c8 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryErrorMessageIndex.cs @@ -0,0 +1,23 @@ +namespace ServiceControl.Infrastructure.RavenDB.Expiration +{ + using System; + using System.Linq; + using MessageFailures; + using Raven.Client.Indexes; + + class ExpiryErrorMessageIndex : AbstractIndexCreationTask + { + public ExpiryErrorMessageIndex() + { + Map = messages => from message in messages + where message.Status != FailedMessageStatus.Unresolved + select new + { + message.Status, + LastModified = MetadataFor(message).Value("Last-Modified").Ticks + }; + + DisableInMemoryIndexing = true; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryEventLogItemsIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryEventLogItemsIndex.cs new file mode 100644 index 0000000000..13ebc15cfe --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryEventLogItemsIndex.cs @@ -0,0 +1,21 @@ +namespace ServiceControl.Infrastructure.RavenDB.Expiration +{ + using System; + using System.Linq; + using EventLog; + using Raven.Client.Indexes; + + class ExpiryEventLogItemsIndex : AbstractIndexCreationTask + { + public ExpiryEventLogItemsIndex() + { + Map = messages => from message in messages + select new + { + LastModified = MetadataFor(message).Value("Last-Modified").Ticks + }; + + DisableInMemoryIndexing = true; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryProcessedMessageIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryProcessedMessageIndex.cs new file mode 100644 index 0000000000..3d6e24725b --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryProcessedMessageIndex.cs @@ -0,0 +1,20 @@ +namespace ServiceControl.Infrastructure.RavenDB.Expiration +{ + using System.Linq; + using MessageAuditing; + using Raven.Client.Indexes; + + class ExpiryProcessedMessageIndex : AbstractIndexCreationTask + { + public ExpiryProcessedMessageIndex() + { + Map = messages => from message in messages + select new + { + ProcessedAt = message.ProcessedAt.Ticks + }; + + DisableInMemoryIndexing = true; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Extensions.cs b/src/ServiceControl.Persistence.RavenDb5/Extensions.cs new file mode 100644 index 0000000000..5f2184db8f --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Extensions.cs @@ -0,0 +1,20 @@ +namespace ServiceControl.Infrastructure.RavenDB +{ + using System; + using System.Threading; + using Raven.Abstractions.Data; + using Raven.Database; + using Raven.Json.Linq; + + static class Extensions + { + public static void Query(this DocumentDatabase db, string index, IndexQuery query, Action onItem, TState state, CancellationToken cancellationToken = default) + { + var results = db.Queries.Query(index, query, cancellationToken); + foreach (var doc in results.Results) + { + onItem(doc, state); + } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Extensions/DictionaryExtensions.cs b/src/ServiceControl.Persistence.RavenDb5/Extensions/DictionaryExtensions.cs new file mode 100644 index 0000000000..7aabb6ea83 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Extensions/DictionaryExtensions.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Infrastructure +{ + using System; + using System.Collections.Generic; + + static class DictionaryExtensions + { + public static void CheckIfKeyExists(string key, IReadOnlyDictionary headers, Action actionToInvokeWhenKeyIsFound) + { + if (headers.TryGetValue(key, out var value)) + { + actionToInvokeWhenKeyIsFound(value); + } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Extensions/QueryResultConvert.cs b/src/ServiceControl.Persistence.RavenDb5/Extensions/QueryResultConvert.cs new file mode 100644 index 0000000000..9d47302837 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Extensions/QueryResultConvert.cs @@ -0,0 +1,15 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System.Collections.Generic; + using Persistence.Infrastructure; + using Raven.Client; + + static class QueryResultConvert + { + public static QueryResult> ToQueryResult(this IList result, RavenQueryStatistics stats) + where T : class + { + return new QueryResult>(result, stats.ToQueryStatsInfo()); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/ExternalIntegrationRequestsDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ExternalIntegrationRequestsDataStore.cs new file mode 100644 index 0000000000..ebb0db036d --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/ExternalIntegrationRequestsDataStore.cs @@ -0,0 +1,219 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Reactive.Linq; + using System.Threading; + using System.Threading.Tasks; + using ExternalIntegrations; + using Microsoft.Extensions.Hosting; + using NServiceBus; + using NServiceBus.Logging; + using Raven.Abstractions.Data; + using Raven.Client; + using Raven.Client.Linq; + using ServiceBus.Management.Infrastructure.Extensions; + + class ExternalIntegrationRequestsDataStore + : IExternalIntegrationRequestsDataStore + , IHostedService + , IAsyncDisposable + { + public ExternalIntegrationRequestsDataStore(RavenDBPersisterSettings settings, IDocumentStore documentStore, CriticalError criticalError) + { + this.settings = settings; + this.documentStore = documentStore; + + circuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker( + "EventDispatcher", + TimeSpan.FromMinutes(5), // TODO: Shouldn't be magic value but coming from settings + ex => criticalError.Raise("Repeated failures when dispatching external integration events.", ex), + TimeSpan.FromSeconds(20) // TODO: Shouldn't be magic value but coming from settings + ); + } + + const string KeyPrefix = "ExternalIntegrationDispatchRequests"; + + public async Task StoreDispatchRequest(IEnumerable dispatchRequests) + { + using (var session = documentStore.OpenAsyncSession()) + { + foreach (var dispatchRequest in dispatchRequests) + { + if (dispatchRequest.Id != null) + { + throw new ArgumentException("Items cannot have their Id property set"); + } + + dispatchRequest.Id = KeyPrefix + "/" + Guid.NewGuid(); // TODO: Key is generated to persistence + await session.StoreAsync(dispatchRequest); + } + + await session.SaveChangesAsync(); + } + } + + public void Subscribe(Func callback) + { + if (this.callback != null) + { + throw new InvalidOperationException("Subscription already exists."); + } + + this.callback = callback ?? throw new ArgumentNullException(nameof(callback)); + + StartDispatcher(); + } + + void StartDispatcher() + { + task = StartDispatcherTask(tokenSource.Token); + } + + async Task StartDispatcherTask(CancellationToken cancellationToken) + { + try + { + await DispatchEvents(cancellationToken); + do + { + try + { + await signal.WaitHandle.WaitOneAsync(cancellationToken); + signal.Reset(); + } + catch (OperationCanceledException) + { + break; + } + + await DispatchEvents(cancellationToken); + } + while (!cancellationToken.IsCancellationRequested); + } + catch (OperationCanceledException) + { + // ignore + } + catch (Exception ex) + { + Logger.Error("An exception occurred when dispatching external integration events", ex); + await circuitBreaker.Failure(ex); + + if (!tokenSource.IsCancellationRequested) + { + StartDispatcher(); + } + } + } + + async Task DispatchEvents(CancellationToken cancellationToken) + { + bool more; + + do + { + more = await TryDispatchEventBatch(); + + circuitBreaker.Success(); + + if (more && !cancellationToken.IsCancellationRequested) + { + //if there is more events to dispatch we sleep for a bit and then we go again + await Task.Delay(1000, CancellationToken.None); + } + } + while (!cancellationToken.IsCancellationRequested && more); + } + + async Task TryDispatchEventBatch() + { + using (var session = documentStore.OpenAsyncSession()) + { + var awaitingDispatching = await session + .Query() + .Statistics(out var stats) + .Take(settings.ExternalIntegrationsDispatchingBatchSize) + .ToListAsync(); + + if (awaitingDispatching.Count == 0) + { + // If the index hasn't caught up, try again + return stats.IndexEtag.CompareTo(latestEtag) < 0; + } + + var allContexts = awaitingDispatching.Select(r => r.DispatchContext).ToArray(); + if (Logger.IsDebugEnabled) + { + Logger.Debug($"Dispatching {allContexts.Length} events."); + } + + await callback(allContexts); + + foreach (var dispatchedEvent in awaitingDispatching) + { + session.Delete(dispatchedEvent); + } + + await session.SaveChangesAsync(); + } + + return true; + } + + public Task StartAsync(CancellationToken cancellationToken) + { + subscription = documentStore + .Changes() + .ForDocumentsStartingWith(KeyPrefix) + .Where(c => c.Type == DocumentChangeTypes.Put) + .Subscribe(d => + { + latestEtag = Etag.Max(d.Etag, latestEtag); + signal.Set(); + }); + + return Task.CompletedTask; + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + await DisposeAsync(); + } + + public async ValueTask DisposeAsync() + { + if (isDisposed) + { + return; + } + + isDisposed = true; + subscription?.Dispose(); + tokenSource?.Cancel(); + + if (task != null) + { + await task; + } + + tokenSource?.Dispose(); + circuitBreaker?.Dispose(); + } + + readonly RavenDBPersisterSettings settings; + readonly IDocumentStore documentStore; + readonly CancellationTokenSource tokenSource = new CancellationTokenSource(); + readonly RepeatedFailuresOverTimeCircuitBreaker circuitBreaker; + + IDisposable subscription; + Etag latestEtag = Etag.Empty; + Task task; + ManualResetEventSlim signal = new ManualResetEventSlim(); + Func callback; + bool isDisposed; + + static ILog Logger = LogManager.GetLogger(typeof(ExternalIntegrationRequestsDataStore)); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/FailedErrorImportDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/FailedErrorImportDataStore.cs new file mode 100644 index 0000000000..3d96697bd1 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/FailedErrorImportDataStore.cs @@ -0,0 +1,79 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using NServiceBus.Logging; + using Raven.Client; + using ServiceControl.Operations; + + class FailedErrorImportDataStore : IFailedErrorImportDataStore + { + readonly IDocumentStore store; + + static readonly ILog Logger = LogManager.GetLogger(typeof(FailedErrorImportDataStore)); + + public FailedErrorImportDataStore(IDocumentStore store) + { + this.store = store; + } + + public async Task ProcessFailedErrorImports(Func processMessage, CancellationToken cancellationToken) + { + var succeeded = 0; + var failed = 0; + using (var session = store.OpenAsyncSession()) + { + var query = session.Query(); + using (var stream = await session.Advanced.StreamAsync(query, cancellationToken) + ) + { + while (!cancellationToken.IsCancellationRequested && await stream.MoveNextAsync()) + { + var transportMessage = stream.Current.Document.Message; + try + { + await processMessage(transportMessage); + + await store.AsyncDatabaseCommands.DeleteAsync(stream.Current.Key, null, cancellationToken); + succeeded++; + + if (Logger.IsDebugEnabled) + { + Logger.Debug($"Successfully re-imported failed error message {transportMessage.Id}."); + } + } + catch (OperationCanceledException) + { + // no-op + } + catch (Exception e) + { + Logger.Error($"Error while attempting to re-import failed error message {transportMessage.Id}.", e); + failed++; + } + } + } + } + + Logger.Info($"Done re-importing failed errors. Successfully re-imported {succeeded} messages. Failed re-importing {failed} messages."); + + if (failed > 0) + { + Logger.Warn($"{failed} messages could not be re-imported. This could indicate a problem with the data. Contact Particular support if you need help with recovering the messages."); + } + } + + public async Task QueryContainsFailedImports() + { + using (var session = store.OpenAsyncSession()) + { + var query = session.Query(); + using (var ie = await session.Advanced.StreamAsync(query)) + { + return await ie.MoveNextAsync(); + } + } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/FailedMessageViewIndexNotifications.cs b/src/ServiceControl.Persistence.RavenDb5/FailedMessageViewIndexNotifications.cs new file mode 100644 index 0000000000..1ab4388de0 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/FailedMessageViewIndexNotifications.cs @@ -0,0 +1,110 @@ +namespace ServiceControl.MessageFailures +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using Api; + using Microsoft.Extensions.Hosting; + using NServiceBus.Logging; + using Persistence; + using Raven.Client; + + class FailedMessageViewIndexNotifications + : IFailedMessageViewIndexNotifications + , IDisposable + , IHostedService + { + public FailedMessageViewIndexNotifications(IDocumentStore store) + { + this.store = store; + } + + void OnNext() + { + try + { + UpdatedCount().GetAwaiter().GetResult(); + } + catch (Exception ex) + { + logging.WarnFormat("Failed to emit MessageFailuresUpdated - {0}", ex); + } + } + + async Task UpdatedCount() + { + using (var session = store.OpenAsyncSession()) + { + var failedUnresolvedMessageCount = await session + .Query() + .CountAsync(p => p.Status == FailedMessageStatus.Unresolved); + + var failedArchivedMessageCount = await session + .Query() + .CountAsync(p => p.Status == FailedMessageStatus.Archived); + + if (lastUnresolvedCount == failedUnresolvedMessageCount && lastArchivedCount == failedArchivedMessageCount) + { + return; + } + + lastUnresolvedCount = failedUnresolvedMessageCount; + lastArchivedCount = failedArchivedMessageCount; + + if (subscriber != null) + { + await subscriber(new FailedMessageTotals + { + ArchivedTotal = failedArchivedMessageCount, + UnresolvedTotal = failedUnresolvedMessageCount + }); + } + } + } + + public IDisposable Subscribe(Func callback) + { + if (callback is null) + { + throw new ArgumentNullException(nameof(callback)); + } + + if (!(subscriber is null)) + { + throw new InvalidOperationException("Already a subscriber, only a single subscriber supported"); + } + + subscriber = callback; + return this; + } + + public void Dispose() + { + subscriber = null; + subscription?.Dispose(); + } + + public Task StartAsync(CancellationToken cancellationToken) + { + subscription = store + .Changes() + .ForIndex(new FailedMessageViewIndex().IndexName) + .Subscribe(d => OnNext()); + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) + { + Dispose(); + return Task.CompletedTask; + } + + readonly IDocumentStore store; + readonly ILog logging = LogManager.GetLogger(typeof(FailedMessageViewIndexNotifications)); + + Func subscriber; + IDisposable subscription; + int lastUnresolvedCount; + int lastArchivedCount; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs b/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs new file mode 100644 index 0000000000..3da00aee79 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs @@ -0,0 +1,154 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.Hosting; + using NServiceBus.Logging; + using Raven.Abstractions.Data; + using Raven.Abstractions.Exceptions; + using Raven.Client; + using Raven.Json.Linq; + using ServiceControl.MessageFailures; + using ServiceControl.MessageFailures.Api; + using ServiceControl.Persistence.Infrastructure; + using ServiceControl.Recoverability; + + class FailedMessageReclassifier : IReclassifyFailedMessages + { + readonly IDocumentStore store; + readonly IEnumerable classifiers; + + public FailedMessageReclassifier(IDocumentStore store, IHostApplicationLifetime applicationLifetime, IEnumerable classifiers) + { + this.store = store; + this.classifiers = classifiers; + + applicationLifetime?.ApplicationStopping.Register(() => { abort = true; }); + } + + public async Task ReclassifyFailedMessages(bool force) + { + logger.Info("Reclassification of failures started."); + + var failedMessagesReclassified = 0; + var currentBatch = new List>(); + + using (var session = store.OpenAsyncSession()) + { + ReclassifyErrorSettings settings = null; + + if (!force) + { + settings = await session.LoadAsync(ReclassifyErrorSettings.IdentifierCase); + + if (settings != null && settings.ReclassificationDone) + { + logger.Info("Skipping reclassification of failures as classification has already been done."); + return 0; + } + } + + var query = session.Query() + .Where(f => f.Status == FailedMessageStatus.Unresolved); + + var totalMessagesReclassified = 0; + + using (var stream = await session.Advanced.StreamAsync(query.As()) + ) + { + while (!abort && await stream.MoveNextAsync()) + { + currentBatch.Add(Tuple.Create(stream.Current.Document.Id, new ClassifiableMessageDetails(stream.Current.Document))); + + if (currentBatch.Count == BatchSize) + { + failedMessagesReclassified += ReclassifyBatch(store, currentBatch, classifiers); + currentBatch.Clear(); + + totalMessagesReclassified += BatchSize; + logger.Info($"Reclassification of batch of {BatchSize} failed messages completed. Total messages reclassified: {totalMessagesReclassified}"); + } + } + } + + if (currentBatch.Any()) + { + ReclassifyBatch(store, currentBatch, classifiers); + } + + logger.Info($"Reclassification of failures ended. Reclassified {failedMessagesReclassified} messages"); + + if (settings == null) + { + settings = new ReclassifyErrorSettings(); + } + + settings.ReclassificationDone = true; + await session.StoreAsync(settings); + await session.SaveChangesAsync(); + + return failedMessagesReclassified; + } + } + + int ReclassifyBatch(IDocumentStore store, IEnumerable> docs, IEnumerable classifiers) + { + var failedMessagesReclassified = 0; + + Parallel.ForEach(docs, doc => + { + var failureGroups = GetClassificationGroups(doc.Item2, classifiers).Select(RavenJObject.FromObject); + + try + { + store.DatabaseCommands.Patch(doc.Item1, + new[] + { + new PatchRequest + { + Type = PatchCommandType.Set, + Name = "FailureGroups", + Value = new RavenJArray(failureGroups) + } + }); + + Interlocked.Increment(ref failedMessagesReclassified); + } + catch (ConcurrencyException) + { + // Ignore concurrency exceptions + } + }); + + return failedMessagesReclassified; + } + + IEnumerable GetClassificationGroups(ClassifiableMessageDetails details, IEnumerable classifiers) + { + foreach (var classifier in classifiers) + { + var classification = classifier.ClassifyFailure(details); + if (classification == null) + { + continue; + } + + var id = DeterministicGuid.MakeId(classifier.Name, classification).ToString(); + + yield return new FailedMessage.FailureGroup + { + Id = id, + Title = classification, + Type = classifier.Name + }; + } + } + + readonly ILog logger = LogManager.GetLogger(); + const int BatchSize = 1000; + bool abort; + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/ArchivedGroupsViewIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/ArchivedGroupsViewIndex.cs new file mode 100644 index 0000000000..6de54efd63 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/ArchivedGroupsViewIndex.cs @@ -0,0 +1,46 @@ +namespace ServiceControl.Recoverability +{ + using System.Linq; + using MessageFailures; + using Raven.Client.Indexes; + + class ArchivedGroupsViewIndex : AbstractIndexCreationTask + { + public ArchivedGroupsViewIndex() + { + Map = docs => from doc in docs + where doc.Status == FailedMessageStatus.Archived + let failureTimes = doc.ProcessingAttempts.Select(x => x.FailureDetails.TimeOfFailure) + from failureGroup in doc.FailureGroups + select new FailureGroupView + { + Id = failureGroup.Id, + Title = failureGroup.Title, + Count = 1, + First = failureTimes.Min(), + Last = failureTimes.Max(), + Type = failureGroup.Type + }; + + Reduce = results => from result in results + group result by new + { + result.Id, + result.Title, + result.Type + } + into g + select new FailureGroupView + { + Id = g.Key.Id, + Title = g.Key.Title, + Count = g.Sum(x => x.Count), + First = g.Min(x => x.First), + Last = g.Max(x => x.Last), + Type = g.Key.Type + }; + + DisableInMemoryIndexing = true; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/CustomChecksIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/CustomChecksIndex.cs new file mode 100644 index 0000000000..83a96221c4 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/CustomChecksIndex.cs @@ -0,0 +1,23 @@ +namespace ServiceControl.Persistence +{ + using System.Linq; + using Raven.Client.Indexes; + using ServiceControl.Contracts.CustomChecks; + + class CustomChecksIndex : AbstractIndexCreationTask + { + public CustomChecksIndex() + { + Map = docs => from cc in docs + select new CustomCheck + { + Status = cc.Status, + ReportedAt = cc.ReportedAt, + Category = cc.Category, + CustomCheckId = cc.CustomCheckId + }; + + DisableInMemoryIndexing = true; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/ExpirySagaAuditIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/ExpirySagaAuditIndex.cs new file mode 100644 index 0000000000..fb6e83706c --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/ExpirySagaAuditIndex.cs @@ -0,0 +1,26 @@ +namespace ServiceControl.SagaAudit +{ + using System; + using System.Linq; + using Raven.Client.Indexes; + + class ExpirySagaAuditIndex : AbstractMultiMapIndexCreationTask + { + public ExpirySagaAuditIndex() + { + AddMap(messages => from message in messages + select new + { + LastModified = MetadataFor(message).Value("Last-Modified").Ticks + }); + + AddMap(sagaHistories => from sagaHistory in sagaHistories + select new + { + LastModified = MetadataFor(sagaHistory).Value("Last-Modified").Ticks + }); + + DisableInMemoryIndexing = true; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedErrorImportIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedErrorImportIndex.cs new file mode 100644 index 0000000000..0816d9ca4e --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedErrorImportIndex.cs @@ -0,0 +1,20 @@ +namespace ServiceControl.Operations +{ + using System.Linq; + using Raven.Client.Indexes; + + class FailedErrorImportIndex : AbstractIndexCreationTask + { + public FailedErrorImportIndex() + { + Map = docs => from cc in docs + select new FailedErrorImport + { + Id = cc.Id, + Message = cc.Message + }; + + DisableInMemoryIndexing = true; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageFacetsIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageFacetsIndex.cs new file mode 100644 index 0000000000..6ecba0affc --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageFacetsIndex.cs @@ -0,0 +1,29 @@ +namespace ServiceControl.MessageFailures.Api +{ + using System.Linq; + using Raven.Abstractions.Indexing; + using Raven.Client.Indexes; + using ServiceControl.Operations; + + class FailedMessageFacetsIndex : AbstractIndexCreationTask + { + public FailedMessageFacetsIndex() + { + Map = failures => from failure in failures + where failure.Status == FailedMessageStatus.Unresolved + let t = (EndpointDetails)failure.ProcessingAttempts.Last().MessageMetadata["ReceivingEndpoint"] + select new + { + t.Name, + t.Host, + MessageType = failure.ProcessingAttempts.Last().MessageMetadata["MessageType"] + }; + + Index("Name", FieldIndexing.NotAnalyzed); //to avoid lower casing + Index("Host", FieldIndexing.NotAnalyzed); //to avoid lower casing + Index("MessageType", FieldIndexing.NotAnalyzed); //to avoid lower casing + + DisableInMemoryIndexing = true; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageRetries_ByBatch.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageRetries_ByBatch.cs new file mode 100644 index 0000000000..2ff6a5d4ba --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageRetries_ByBatch.cs @@ -0,0 +1,20 @@ +namespace ServiceControl.Persistence +{ + using System.Linq; + using Raven.Client.Indexes; + using ServiceControl.Recoverability; + + class FailedMessageRetries_ByBatch : AbstractIndexCreationTask + { + public FailedMessageRetries_ByBatch() + { + Map = docs => from doc in docs + select new + { + doc.RetryBatchId + }; + + DisableInMemoryIndexing = true; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageViewIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageViewIndex.cs new file mode 100644 index 0000000000..11e3514eec --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageViewIndex.cs @@ -0,0 +1,41 @@ +namespace ServiceControl.MessageFailures.Api +{ + using System; + using System.Linq; + using Raven.Client.Indexes; + using ServiceControl.Operations; + + class FailedMessageViewIndex : AbstractIndexCreationTask + { + public FailedMessageViewIndex() + { + Map = messages => from message in messages + let processingAttemptsLast = message.ProcessingAttempts.Last() + select new + { + MessageId = processingAttemptsLast.MessageMetadata["MessageId"], + MessageType = processingAttemptsLast.MessageMetadata["MessageType"], + message.Status, + TimeSent = (DateTime)processingAttemptsLast.MessageMetadata["TimeSent"], + ReceivingEndpointName = ((EndpointDetails)processingAttemptsLast.MessageMetadata["ReceivingEndpoint"]).Name, + QueueAddress = processingAttemptsLast.FailureDetails.AddressOfFailingEndpoint, + processingAttemptsLast.FailureDetails.TimeOfFailure, + LastModified = MetadataFor(message).Value("Last-Modified").Ticks + }; + + DisableInMemoryIndexing = true; + } + + public class SortAndFilterOptions : IHaveStatus + { + public string MessageId { get; set; } + public DateTime TimeSent { get; set; } + public string MessageType { get; set; } + public string ReceivingEndpointName { get; set; } + public string QueueAddress { get; set; } + public DateTime TimeOfFailure { get; set; } + public long LastModified { get; set; } + public FailedMessageStatus Status { get; set; } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessages_ByGroup.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessages_ByGroup.cs new file mode 100644 index 0000000000..d7b96da4d2 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessages_ByGroup.cs @@ -0,0 +1,34 @@ +namespace ServiceControl.Recoverability +{ + using System; + using System.Linq; + using MessageFailures; + using Raven.Abstractions.Indexing; + using Raven.Client.Indexes; + + class FailedMessages_ByGroup : AbstractIndexCreationTask + { + public FailedMessages_ByGroup() + { + Map = docs => from doc in docs + let processingAttemptsLast = doc.ProcessingAttempts.Last() + from failureGroup in doc.FailureGroups + select new FailureGroupMessageView + { + Id = doc.Id, + MessageId = doc.UniqueMessageId, + FailureGroupId = failureGroup.Id, + FailureGroupName = failureGroup.Title, + Status = doc.Status, + MessageType = (string)processingAttemptsLast.MessageMetadata["MessageType"], + TimeSent = (DateTime)processingAttemptsLast.MessageMetadata["TimeSent"], + TimeOfFailure = processingAttemptsLast.FailureDetails.TimeOfFailure, + LastModified = MetadataFor(doc).Value("Last-Modified").Ticks + }; + + StoreAllFields(FieldStorage.Yes); + + DisableInMemoryIndexing = true; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailureGroupsViewIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailureGroupsViewIndex.cs new file mode 100644 index 0000000000..47f2402d7c --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailureGroupsViewIndex.cs @@ -0,0 +1,46 @@ +namespace ServiceControl.Recoverability +{ + using System.Linq; + using MessageFailures; + using Raven.Client.Indexes; + + class FailureGroupsViewIndex : AbstractIndexCreationTask + { + public FailureGroupsViewIndex() + { + Map = docs => from doc in docs + where doc.Status == FailedMessageStatus.Unresolved + let failureTimes = doc.ProcessingAttempts.Select(x => x.FailureDetails.TimeOfFailure) + from failureGroup in doc.FailureGroups + select new FailureGroupView + { + Id = failureGroup.Id, + Title = failureGroup.Title, + Count = 1, + First = failureTimes.Min(), + Last = failureTimes.Max(), + Type = failureGroup.Type + }; + + Reduce = results => from result in results + group result by new + { + result.Id, + result.Title, + result.Type + } + into g + select new FailureGroupView + { + Id = g.Key.Id, + Title = g.Key.Title, + Count = g.Sum(x => x.Count), + First = g.Min(x => x.First), + Last = g.Max(x => x.Last), + Type = g.Key.Type + }; + + DisableInMemoryIndexing = true; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/GroupCommentIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/GroupCommentIndex.cs new file mode 100644 index 0000000000..e5b86901c6 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/GroupCommentIndex.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Recoverability +{ + using System.Linq; + using MessageFailures; + using Raven.Client.Indexes; + + class GroupCommentIndex : AbstractIndexCreationTask + { + public GroupCommentIndex() + { + Map = docs => docs.Select(gc => new GroupComment { Id = gc.Id, Comment = gc.Comment }); + + DisableInMemoryIndexing = true; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/ImportFailedAudits.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/ImportFailedAudits.cs new file mode 100644 index 0000000000..10b8970b81 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/ImportFailedAudits.cs @@ -0,0 +1,20 @@ +namespace ServiceControl.Operations +{ + using System.Linq; + using Raven.Client.Indexes; + + class FailedAuditImportIndex : AbstractIndexCreationTask + { + public FailedAuditImportIndex() + { + Map = docs => from cc in docs + select new FailedAuditImport + { + Id = cc.Id, + Message = cc.Message + }; + + DisableInMemoryIndexing = true; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/KnownEndpointIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/KnownEndpointIndex.cs new file mode 100644 index 0000000000..297724d957 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/KnownEndpointIndex.cs @@ -0,0 +1,23 @@ +namespace ServiceControl.Persistence +{ + using System.Linq; + using Raven.Client.Indexes; + + class KnownEndpointIndex : AbstractIndexCreationTask + { + public KnownEndpointIndex() + { + Map = messages => from message in messages + select new + { + EndpointDetails_Name = message.EndpointDetails.Name, + EndpointDetails_Host = message.EndpointDetails.Host, + message.HostDisplayName, + message.Monitored, + message.HasTemporaryId + }; + + DisableInMemoryIndexing = true; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/MessagesViewIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/MessagesViewIndex.cs new file mode 100644 index 0000000000..3b3acd9476 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/MessagesViewIndex.cs @@ -0,0 +1,80 @@ +namespace ServiceControl.Persistence +{ + using System; + using System.Linq; + using Lucene.Net.Analysis.Standard; + using Raven.Abstractions.Indexing; + using Raven.Client.Indexes; + using ServiceControl.MessageAuditing; + using ServiceControl.MessageFailures; + using ServiceControl.Operations; + + class MessagesViewIndex : AbstractMultiMapIndexCreationTask + { + public MessagesViewIndex() + { + AddMap(messages => from message in messages + select new SortAndFilterOptions + { + MessageId = (string)message.MessageMetadata["MessageId"], + MessageType = (string)message.MessageMetadata["MessageType"], + IsSystemMessage = (bool)message.MessageMetadata["IsSystemMessage"], + Status = (bool)message.MessageMetadata["IsRetried"] ? MessageStatus.ResolvedSuccessfully : MessageStatus.Successful, + TimeSent = (DateTime)message.MessageMetadata["TimeSent"], + ProcessedAt = message.ProcessedAt, + ReceivingEndpointName = ((EndpointDetails)message.MessageMetadata["ReceivingEndpoint"]).Name, + CriticalTime = (TimeSpan?)message.MessageMetadata["CriticalTime"], + ProcessingTime = (TimeSpan?)message.MessageMetadata["ProcessingTime"], + DeliveryTime = (TimeSpan?)message.MessageMetadata["DeliveryTime"], + Query = message.MessageMetadata.Select(_ => _.Value.ToString()).Union(new[] { string.Join(" ", message.Headers.Select(x => x.Value)) }).ToArray(), + ConversationId = (string)message.MessageMetadata["ConversationId"] + }); + + AddMap(messages => from message in messages + let last = message.ProcessingAttempts.Last() + select new SortAndFilterOptions + { + MessageId = last.MessageId, + MessageType = (string)last.MessageMetadata["MessageType"], + IsSystemMessage = (bool)last.MessageMetadata["IsSystemMessage"], + Status = message.Status == FailedMessageStatus.Archived + ? MessageStatus.ArchivedFailure + : message.Status == FailedMessageStatus.Resolved + ? MessageStatus.ResolvedSuccessfully + : message.ProcessingAttempts.Count == 1 + ? MessageStatus.Failed + : MessageStatus.RepeatedFailure, + TimeSent = (DateTime)last.MessageMetadata["TimeSent"], + ProcessedAt = last.AttemptedAt, + ReceivingEndpointName = ((EndpointDetails)last.MessageMetadata["ReceivingEndpoint"]).Name, + CriticalTime = null, + ProcessingTime = null, + DeliveryTime = null, + Query = last.MessageMetadata.Select(_ => _.Value.ToString()).Union(new[] { string.Join(" ", last.Headers.Select(x => x.Value)) }).ToArray(), + ConversationId = (string)last.MessageMetadata["ConversationId"] + }); + + Index(x => x.Query, FieldIndexing.Analyzed); + + Analyze(x => x.Query, typeof(StandardAnalyzer).AssemblyQualifiedName); + + DisableInMemoryIndexing = true; + } + + public class SortAndFilterOptions + { + public string MessageId { get; set; } + public string MessageType { get; set; } + public bool IsSystemMessage { get; set; } + public MessageStatus Status { get; set; } + public DateTime ProcessedAt { get; set; } + public string ReceivingEndpointName { get; set; } + public TimeSpan? CriticalTime { get; set; } + public TimeSpan? ProcessingTime { get; set; } + public TimeSpan? DeliveryTime { get; set; } + public string ConversationId { get; set; } + public string[] Query { get; set; } + public DateTime TimeSent { get; set; } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/QueueAddressIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/QueueAddressIndex.cs new file mode 100644 index 0000000000..5a31449809 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/QueueAddressIndex.cs @@ -0,0 +1,30 @@ +namespace ServiceControl.MessageFailures.Api +{ + using System.Linq; + using Raven.Client.Indexes; + + class QueueAddressIndex : AbstractIndexCreationTask + { + public QueueAddressIndex() + { + Map = messages => from message in messages + let processingAttemptsLast = message.ProcessingAttempts.Last() + select new + { + PhysicalAddress = processingAttemptsLast.FailureDetails.AddressOfFailingEndpoint, + FailedMessageCount = 1 + }; + + Reduce = results => from result in results + group result by result.PhysicalAddress + into g + select new + { + PhysicalAddress = g.Key, + FailedMessageCount = g.Sum(m => m.FailedMessageCount) + }; + + DisableInMemoryIndexing = true; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/RetryBatches_ByStatusAndSession.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/RetryBatches_ByStatusAndSession.cs new file mode 100644 index 0000000000..70ad6da482 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/RetryBatches_ByStatusAndSession.cs @@ -0,0 +1,20 @@ +namespace ServiceControl.Persistence +{ + using System.Linq; + using Raven.Client.Indexes; + + class RetryBatches_ByStatusAndSession : AbstractIndexCreationTask + { + public RetryBatches_ByStatusAndSession() + { + Map = docs => from doc in docs + select new + { + doc.RetrySessionId, + doc.Status + }; + + DisableInMemoryIndexing = true; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/RetryBatches_ByStatus_ReduceInitialBatchSize.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/RetryBatches_ByStatus_ReduceInitialBatchSize.cs new file mode 100644 index 0000000000..c20490177b --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/RetryBatches_ByStatus_ReduceInitialBatchSize.cs @@ -0,0 +1,47 @@ +namespace ServiceControl.Persistence +{ + using System.Linq; + using Raven.Client.Indexes; + + class RetryBatches_ByStatus_ReduceInitialBatchSize : AbstractIndexCreationTask + { + public RetryBatches_ByStatus_ReduceInitialBatchSize() + { + Map = docs => from doc in docs + select new + { + doc.RequestId, + doc.RetryType, + HasStagingBatches = doc.Status == RetryBatchStatus.Staging, + HasForwardingBatches = doc.Status == RetryBatchStatus.Forwarding, + doc.InitialBatchSize, + doc.Originator, + doc.Classifier, + doc.StartTime, + doc.Last + }; + + Reduce = results => from result in results + group result by new + { + result.RequestId, + result.RetryType + } + into g + select new + { + g.Key.RequestId, + g.Key.RetryType, + g.First().Originator, + HasStagingBatches = g.Any(x => x.HasStagingBatches), + HasForwardingBatches = g.Any(x => x.HasForwardingBatches), + InitialBatchSize = g.Sum(x => x.InitialBatchSize), + g.First().StartTime, + Last = g.Max(x => x.Last), + g.First().Classifier + }; + + DisableInMemoryIndexing = true; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/SagaDetailsIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/SagaDetailsIndex.cs new file mode 100644 index 0000000000..36ecd37a53 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/SagaDetailsIndex.cs @@ -0,0 +1,57 @@ +namespace ServiceControl.SagaAudit +{ + using System.Linq; + using Raven.Client.Indexes; + + class SagaDetailsIndex : AbstractMultiMapIndexCreationTask + { + public SagaDetailsIndex() + { + AddMap(docs => from doc in docs + select new + { + doc.SagaId, + Id = doc.SagaId, + doc.SagaType, + Changes = new[] + { + new SagaStateChange + { + Endpoint = doc.Endpoint, + FinishTime = doc.FinishTime, + InitiatingMessage = doc.InitiatingMessage, + OutgoingMessages = doc.OutgoingMessages, + StartTime = doc.StartTime, + StateAfterChange = doc.StateAfterChange, + Status = doc.Status + } + } + }); + + //Legacy so we still scan old sagahistories + AddMap(docs => from doc in docs + select new + { + doc.SagaId, + Id = doc.SagaId, + doc.SagaType, + doc.Changes + } + ); + + Reduce = results => from result in results + group result by result.SagaId + into g + let first = g.First() + select new SagaHistory + { + Id = first.SagaId, + SagaId = first.SagaId, + SagaType = first.SagaType, + Changes = g.SelectMany(x => x.Changes) + .OrderByDescending(x => x.FinishTime) + .ToList() + }; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/SagaHistoryCleaner.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/SagaHistoryCleaner.cs new file mode 100644 index 0000000000..e0b6782807 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/SagaHistoryCleaner.cs @@ -0,0 +1,112 @@ +namespace ServiceControl.SagaAudit +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Linq; + using System.Threading; + using NServiceBus.Logging; + using Raven.Abstractions; + using Raven.Abstractions.Commands; + using Raven.Abstractions.Data; + using Raven.Abstractions.Exceptions; + using Raven.Database; + using ServiceControl.Infrastructure.RavenDB; + + static class SagaHistoryCleaner + { + public static void Clean(int deletionBatchSize, DocumentDatabase database, DateTime expiryThreshold, CancellationToken cancellationToken = default) + { + var stopwatch = Stopwatch.StartNew(); + var items = new List(deletionBatchSize); + var indexName = new ExpirySagaAuditIndex().IndexName; + + try + { + var query = new IndexQuery + { + Start = 0, + DisableCaching = true, + Cutoff = SystemTime.UtcNow, + PageSize = deletionBatchSize, + Query = $"LastModified:[* TO {expiryThreshold.Ticks}]", + FieldsToFetch = new[] + { + "__document_id" + }, + SortedFields = new[] + { + new SortedField("LastModified") + { + Field = "LastModified", + Descending = false + } + } + }; + + database.Query(indexName, query, (doc, commands) => + { + var id = doc.Value("__document_id"); + if (string.IsNullOrEmpty(id)) + { + return; + } + + commands.Add(new DeleteCommandData + { + Key = id + }); + }, + items, cancellationToken); + } + catch (IndexDisabledException ex) + { + logger.Error($"Unable to cleanup saga history. The index ${indexName} was disabled.", ex); + return; + } + catch (OperationCanceledException) + { + logger.Info("Cleanup operation cancelled"); + return; + } + + if (cancellationToken.IsCancellationRequested) + { + return; + } + + var deletionCount = Chunker.ExecuteInChunks(items.Count, (itemsForBatch, db, s, e) => + { + if (logger.IsDebugEnabled) + { + logger.Debug($"Batching deletion of {s}-{e} saga history documents."); + } + + var results = db.Batch(itemsForBatch.GetRange(s, e - s + 1), CancellationToken.None); + if (logger.IsDebugEnabled) + { + logger.Debug($"Batching deletion of {s}-{e} saga history documents completed."); + } + + return results.Count(x => x.Deleted == true); + }, items, database, cancellationToken); + + if (deletionCount == 0) + { + if (logger.IsDebugEnabled) + { + logger.Debug("No expired saga history documents found"); + } + } + else + { + if (logger.IsDebugEnabled) + { + logger.Debug($"Deleted {deletionCount} expired saga history documents. Batch execution took {stopwatch.ElapsedMilliseconds} ms"); + } + } + } + + static ILog logger = LogManager.GetLogger(typeof(SagaHistoryCleaner)); + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/MigratedTypeAwareBinder.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/MigratedTypeAwareBinder.cs new file mode 100644 index 0000000000..0dcaa04fd0 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/MigratedTypeAwareBinder.cs @@ -0,0 +1,68 @@ +namespace ServiceControl.Infrastructure.RavenDB +{ + using System; + using System.Linq; + using Raven.Imports.Newtonsoft.Json.Serialization; + using SagaAudit; + using ServiceControl.Contracts.CustomChecks; + using ServiceControl.Contracts.Operations; + using ServiceControl.MessageAuditing; + using ServiceControl.MessageFailures; + using ServiceControl.Operations; + using ServiceControl.Persistence; + using ServiceControl.Recoverability; + using static ServiceControl.MessageFailures.FailedMessage; + + class MigratedTypeAwareBinder : DefaultSerializationBinder + { + public override Type BindToType(string assemblyName, string typeName) + { + if (typeName == "ServiceControl.SagaAudit.SagaInfo" && assemblyName == "ServiceControl") + { + return typeof(SagaInfo); + } + + var className = GetClassName(typeName); + switch (className) + { + case nameof(CustomCheck): + return typeof(CustomCheck); + case nameof(CustomCheckDetail): + return typeof(CustomCheckDetail); + case nameof(EndpointDetails): + return typeof(EndpointDetails); + case nameof(ExceptionDetails): + return typeof(ExceptionDetails); + case nameof(FailedMessage): + return typeof(FailedMessage); + case nameof(FailureDetails): + return typeof(FailureDetails); + case nameof(FailureGroup): + return typeof(FailureGroup); + case nameof(GroupComment): + return typeof(GroupComment); + case nameof(KnownEndpoint): + return typeof(KnownEndpoint); + case nameof(ProcessedMessage): + return typeof(ProcessedMessage); + case nameof(ProcessingAttempt): + return typeof(ProcessingAttempt); + case nameof(FailedMessageRetry): + return typeof(FailedMessageRetry); + case nameof(RetryBatch): + return typeof(RetryBatch); + case nameof(RetryBatchNowForwarding): + return typeof(RetryBatchNowForwarding); + case nameof(SagaInfo): + return typeof(SagaInfo); + default: + return base.BindToType(assemblyName, typeName); + } + } + + string GetClassName(string typeName) + { + return typeName.Split('.').Last(); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/IDataMigration.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/IDataMigration.cs new file mode 100644 index 0000000000..006e2e7c8f --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/IDataMigration.cs @@ -0,0 +1,10 @@ +namespace ServiceControl.Infrastructure.RavenDB +{ + using System.Threading.Tasks; + using Raven.Client; + + interface IDataMigration + { + Task Migrate(IDocumentStore store); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration.cs new file mode 100644 index 0000000000..3980de5820 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration.cs @@ -0,0 +1,32 @@ +namespace ServiceControl.Infrastructure.RavenDB +{ + using System.Linq; + using System.Threading.Tasks; + using Raven.Abstractions.Extensions; + using Raven.Client; + using ServiceControl.Persistence; + + class PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration : IDataMigration + { + public Task Migrate(IDocumentStore store) + { + using (var session = store.OpenSession()) + { + var endpoints = session.Query().ToList(); + + foreach (var knownEndpoints in endpoints.GroupBy(e => e.EndpointDetails.Host + e.EndpointDetails.Name)) + { + var fixedIdsCount = knownEndpoints.Count(e => !e.HasTemporaryId); + + //If we have knowEndpoints with non temp ids, we should delete all temp ids ones. + if (fixedIdsCount > 0) + { + knownEndpoints.Where(e => e.HasTemporaryId).ForEach(k => { store.DatabaseCommands.Delete(store.Conventions.DefaultFindFullDocumentKeyFromNonStringIdentifier(k.Id, typeof(KnownEndpoint), false), null); }); + } + } + } + + return Task.CompletedTask; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/LegacyAddress.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/LegacyAddress.cs new file mode 100644 index 0000000000..81e3f1677b --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/LegacyAddress.cs @@ -0,0 +1,98 @@ +namespace ServiceControl.Infrastructure.RavenDB.Subscriptions +{ + using System; + using System.Collections.Generic; + using Raven.Imports.Newtonsoft.Json.Linq; + using Raven.Json.Linq; + + class LegacyAddress + { + public string Queue { get; set; } + public string Machine { get; set; } + public static List ParseMultipleToSubscriptionClient(List addresses) => addresses.ConvertAll(ParseToSubscriptionClient); + public static List ConvertMultipleToLegacyAddress(List subscriptions) => subscriptions.ConvertAll(ConvertToLegacyAddress); + + public static SubscriptionClient ParseToSubscriptionClient(LegacyAddress address) + { + var queue = address.Queue; + var machine = address.Machine; + + // Previously known as IgnoreMachineName (for brokers) + if (string.IsNullOrEmpty(machine)) + { + return new SubscriptionClient + { + TransportAddress = queue, + Endpoint = null + }; + } + + return new SubscriptionClient + { + TransportAddress = queue + "@" + machine, + Endpoint = null + }; + } + + public static LegacyAddress ConvertToLegacyAddress(SubscriptionClient client) + { + var atIndex = client.TransportAddress?.IndexOf("@", StringComparison.InvariantCulture); + + if (atIndex.HasValue && atIndex.Value > -1) + { + return new LegacyAddress + { + Queue = client.TransportAddress.Substring(0, atIndex.Value), + Machine = client.TransportAddress.Substring(atIndex.Value + 1) + }; + } + + return new LegacyAddress + { + Queue = client.TransportAddress, + Machine = null + }; + } + + public static string ParseToString(Func tokenSelector) + { + var token = tokenSelector(); + + // When we have the new timeout data we just return the value + if (token.Type == JTokenType.String) + { + return token.Value(); + } + + var queue = token.Value("Queue"); + var machine = token.Value("Machine"); + + // Previously known as IgnoreMachineName (for brokers) + if (string.IsNullOrEmpty(machine)) + { + return queue; + } + + return queue + "@" + machine; + } + + public override bool Equals(object obj) + { + if (obj is null) + { + return false; + } + + if (ReferenceEquals(this, obj)) + { + return true; + } + + return obj is LegacyAddress address && Equals(address); + } + + bool Equals(LegacyAddress obj) => string.Equals(Queue, obj.Queue) && string.Equals(Machine, obj.Machine); + + public override int GetHashCode() => Queue.GetHashCode() ^ Machine.GetHashCode(); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/MessageTypeConverter.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/MessageTypeConverter.cs new file mode 100644 index 0000000000..2805858866 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/MessageTypeConverter.cs @@ -0,0 +1,24 @@ +namespace ServiceControl.Infrastructure.RavenDB.Subscriptions +{ + using System; + using NServiceBus.Unicast.Subscriptions; + using Raven.Imports.Newtonsoft.Json; + + class MessageTypeConverter : JsonConverter + { + public override bool CanConvert(Type objectType) + { + return objectType == typeof(MessageType); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + return new MessageType(serializer.Deserialize(reader)); + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + serializer.Serialize(writer, value.ToString()); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/RavenDbSubscriptionStorage.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/RavenDbSubscriptionStorage.cs new file mode 100644 index 0000000000..a64bd817c3 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/RavenDbSubscriptionStorage.cs @@ -0,0 +1,247 @@ +namespace ServiceControl.Infrastructure.RavenDB.Subscriptions +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Security.Cryptography; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using NServiceBus; + using NServiceBus.Extensibility; + using NServiceBus.Logging; + using NServiceBus.Settings; + using NServiceBus.Unicast.Subscriptions; + using NServiceBus.Unicast.Subscriptions.MessageDrivenSubscriptions; + using Raven.Client; + using ServiceControl.Persistence; + using ServiceControl.Persistence.RavenDb.Infrastructure; + + class RavenDbSubscriptionStorage : IServiceControlSubscriptionStorage + { + public RavenDbSubscriptionStorage(IDocumentStore store, ReadOnlySettings settings) : + this(store, settings.EndpointName(), settings.LocalAddress(), settings.GetAvailableTypes().Implementing().Select(e => new MessageType(e)).ToArray()) + { + } + + public RavenDbSubscriptionStorage(IDocumentStore store, string endpointName, string localAddress, MessageType[] locallyHandledEventTypes) + { + this.store = store; + localClient = new SubscriptionClient + { + Endpoint = endpointName, + TransportAddress = localAddress + }; + + this.locallyHandledEventTypes = locallyHandledEventTypes; + + + SetSubscriptions(new Subscriptions()).GetAwaiter().GetResult(); + } + + public async Task Initialize() + { + using (var session = store.OpenAsyncSession()) + { + var primeSubscriptions = await LoadSubscriptions(session) ?? await MigrateSubscriptions(session, localClient); + + await SetSubscriptions(primeSubscriptions); + } + } + + public async Task Subscribe(Subscriber subscriber, MessageType messageType, ContextBag context) + { + if (subscriber.Endpoint == localClient.Endpoint) + { + return; + } + + try + { + await subscriptionsLock.WaitAsync(); + + if (AddOrUpdateSubscription(messageType, subscriber)) + { + await SaveSubscriptions(); + } + } + finally + { + subscriptionsLock.Release(); + } + } + + public async Task Unsubscribe(Subscriber subscriber, MessageType messageType, ContextBag context) + { + try + { + await subscriptionsLock.WaitAsync(); + + var needsSave = false; + if (subscriptions.All.TryGetValue(FormatId(messageType), out var subscription)) + { + var client = CreateSubscriptionClient(subscriber); + if (subscription.Subscribers.Remove(client)) + { + needsSave = true; + } + } + + if (needsSave) + { + await SaveSubscriptions(); + } + } + finally + { + subscriptionsLock.Release(); + } + } + + public Task> GetSubscriberAddressesForMessage(IEnumerable messageTypes, ContextBag context) + { + return Task.FromResult(messageTypes.SelectMany(x => subscriptionsLookup[x]).Distinct()); + } + + bool AddOrUpdateSubscription(MessageType messageType, Subscriber subscriber) + { + var key = FormatId(messageType); + + var subscriptionClient = CreateSubscriptionClient(subscriber); + + if (subscriptions.All.TryGetValue(key, out var subscription)) + { + if (subscription.Subscribers.Contains(subscriptionClient)) + { + return false; + } + + subscription.Subscribers.Add(subscriptionClient); + return true; + } + + // New Subscription + subscription = new Subscription + { + Id = key, + Subscribers = new List + { + subscriptionClient + }, + MessageType = messageType + }; + subscriptions.All.Add(key, subscription); + return true; + } + + static SubscriptionClient CreateSubscriptionClient(Subscriber subscriber) + { + //When the subscriber is running V6 and UseLegacyMessageDrivenSubscriptionMode is enabled at the subscriber the 'subcriber.Endpoint' value is null + var endpoint = subscriber.Endpoint ?? subscriber.TransportAddress.Split('@').First(); + var subscriptionClient = new SubscriptionClient + { + TransportAddress = subscriber.TransportAddress, + Endpoint = endpoint + }; + return subscriptionClient; + } + + async Task SaveSubscriptions() + { + using (var session = store.OpenAsyncSession()) + { + await session.StoreAsync(subscriptions, Subscriptions.SingleDocumentId); + UpdateLookup(); + await session.SaveChangesAsync(); + } + } + + void UpdateLookup() + { + subscriptionsLookup = (from subscription in subscriptions.All.Values + from client in subscription.Subscribers + select new + { + subscription.MessageType, + Subscriber = new Subscriber(client.TransportAddress, client.Endpoint) + }).Union(from eventType in locallyHandledEventTypes + select new + { + MessageType = eventType, + Subscriber = new Subscriber(localClient.TransportAddress, localClient.Endpoint) + } + ).ToLookup(x => x.MessageType, x => x.Subscriber); + } + + string FormatId(MessageType messageType) + { + // use MD5 hash to get a 16-byte hash of the string + var inputBytes = Encoding.Default.GetBytes($"{messageType.TypeName}/{messageType.Version.Major}"); + using (var provider = new MD5CryptoServiceProvider()) + { + var hashBytes = provider.ComputeHash(inputBytes); + + // generate a guid from the hash: + var id = new Guid(hashBytes); + return $"Subscriptions/{id}"; + } + } + + async Task SetSubscriptions(Subscriptions newSubscriptions) + { + try + { + await subscriptionsLock.WaitAsync(); + + subscriptions = newSubscriptions; + UpdateLookup(); + } + finally + { + subscriptionsLock.Release(); + } + } + + static Task LoadSubscriptions(IAsyncDocumentSession session) + => session.LoadAsync(Subscriptions.SingleDocumentId); + + static async Task MigrateSubscriptions(IAsyncDocumentSession session, SubscriptionClient localClient) + { + logger.Info("Migrating subscriptions to new format"); + + var subscriptions = new Subscriptions(); + + using (var stream = await session.Advanced.StreamAsync("Subscriptions") + ) + { + while (await stream.MoveNextAsync()) + { + var existingSubscription = stream.Current.Document; + existingSubscription.Subscribers.Remove(localClient); + subscriptions.All.Add(existingSubscription.Id.Replace("Subscriptions/", string.Empty), existingSubscription); + await session.Advanced.DocumentStore.AsyncDatabaseCommands.DeleteAsync(stream.Current.Key, null); + } + } + + await session.StoreAsync(subscriptions, Subscriptions.SingleDocumentId); + await session.SaveChangesAsync(); + return subscriptions; + } + + IDocumentStore store; + SubscriptionClient localClient; + Subscriptions subscriptions; + ILookup subscriptionsLookup; + MessageType[] locallyHandledEventTypes; + + SemaphoreSlim subscriptionsLock = new SemaphoreSlim(1); + + static ILog logger = LogManager.GetLogger(); + } + + class Subscriptions + { + public IDictionary All { get; set; } = new Dictionary(); + public const string SingleDocumentId = "Subscriptions/All"; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/Subscription.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/Subscription.cs new file mode 100644 index 0000000000..542d7e6bf0 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/Subscription.cs @@ -0,0 +1,45 @@ +namespace ServiceControl.Infrastructure.RavenDB.Subscriptions +{ + using System.Collections.Generic; + using NServiceBus.Unicast.Subscriptions; + using Raven.Imports.Newtonsoft.Json; + + class Subscription + { + public string Id { get; set; } + + [JsonConverter(typeof(MessageTypeConverter))] + public MessageType MessageType { get; set; } + + public List Subscribers + { + get + { + if (subscribers == null) + { + subscribers = new List(); + } + + return subscribers; + } + set => subscribers = value; + } + + [JsonProperty("Clients")] + public List LegacySubscriptions + { + get + { + if (legacySubscriptions == null) + { + legacySubscriptions = new List(); + } + + return legacySubscriptions; + } + } + + List subscribers; + List legacySubscriptions; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/SubscriptionClient.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/SubscriptionClient.cs new file mode 100644 index 0000000000..9a845df083 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/SubscriptionClient.cs @@ -0,0 +1,31 @@ +namespace ServiceControl.Infrastructure.RavenDB.Subscriptions +{ + using System; + + class SubscriptionClient + { + public string TransportAddress { get; set; } + + public string Endpoint { get; set; } + + public override bool Equals(object obj) + { + if (obj is null) + { + return false; + } + + if (ReferenceEquals(this, obj)) + { + return true; + } + + return obj is SubscriptionClient client && Equals(client); + } + + bool Equals(SubscriptionClient obj) => string.Equals(TransportAddress, obj.TransportAddress, + StringComparison.InvariantCultureIgnoreCase); + + public override int GetHashCode() => TransportAddress.ToLowerInvariant().GetHashCode(); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/SubscriptionsLegacyAddressConverter.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/SubscriptionsLegacyAddressConverter.cs new file mode 100644 index 0000000000..8b6f7cb5c9 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/SubscriptionsLegacyAddressConverter.cs @@ -0,0 +1,50 @@ +namespace ServiceControl.Infrastructure.RavenDB.Subscriptions +{ + using System.Linq; + using Raven.Client.Listeners; + using Raven.Json.Linq; + + class SubscriptionsLegacyAddressConverter : IDocumentConversionListener + { + public void BeforeConversionToDocument(string key, object entity, RavenJObject metadata) + { + if (!(entity is Subscription subscription)) + { + return; + } + + var converted = LegacyAddress.ConvertMultipleToLegacyAddress(subscription.Subscribers); + subscription.LegacySubscriptions.Clear(); + subscription.LegacySubscriptions.AddRange(converted); + } + + public void AfterConversionToDocument(string key, object entity, RavenJObject document, RavenJObject metadata) + { + } + + public void BeforeConversionToEntity(string key, RavenJObject document, RavenJObject metadata) + { + } + + public void AfterConversionToEntity(string key, RavenJObject document, RavenJObject metadata, object entity) + { + if (!(entity is Subscription subscription)) + { + return; + } + + var clients = document["Clients"]; + + if (clients != null) + { + var converted = LegacyAddress.ParseMultipleToSubscriptionClient(subscription.LegacySubscriptions); + + var legacySubscriptions = converted.Except(subscription.Subscribers).ToArray(); + foreach (var legacySubscription in legacySubscriptions) + { + subscription.Subscribers.Add(legacySubscription); + } + } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/TypeExtensions.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/TypeExtensions.cs new file mode 100644 index 0000000000..20c25c6d90 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/TypeExtensions.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Persistence.RavenDb.Infrastructure +{ + using System; + using System.Collections.Generic; + using System.Linq; + + static class TypeExtensions + { + public static IEnumerable Implementing(this IEnumerable types) + { + return from type in types + where typeof(T).IsAssignableFrom(type) && !type.IsAbstract && !type.IsInterface + select type; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/InternalsVisibleTo.cs b/src/ServiceControl.Persistence.RavenDb5/InternalsVisibleTo.cs new file mode 100644 index 0000000000..cbd15879aa --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/InternalsVisibleTo.cs @@ -0,0 +1,7 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("ServiceControl.UnitTests")] +[assembly: InternalsVisibleTo("ServiceControl.PersistenceTests")] +[assembly: InternalsVisibleTo("ServiceControl.Persistence.Tests.RavenDb")] +[assembly: InternalsVisibleTo("ServiceControl.AcceptanceTests.RavenDB")] +[assembly: InternalsVisibleTo("ServiceControl.MultiInstance.AcceptanceTests")] diff --git a/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs new file mode 100644 index 0000000000..842342cb8d --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs @@ -0,0 +1,46 @@ +namespace ServiceControl.Persistence.RavenDb.MessageRedirects +{ + using System; + using System.Threading.Tasks; + using Raven.Client; + using ServiceControl.Persistence.MessageRedirects; + + class MessageRedirectsDataStore : IMessageRedirectsDataStore + { + readonly IDocumentStore store; + + public MessageRedirectsDataStore(IDocumentStore store) + { + this.store = store; + } + + public async Task GetOrCreate() + { + using (var session = store.OpenAsyncSession()) + { + var redirects = await session.LoadAsync(DefaultId); + + if (redirects != null) + { + redirects.ETag = session.Advanced.GetEtagFor(redirects); + redirects.LastModified = session.Advanced.GetMetadataFor(redirects).Value("Last-Modified"); + + return redirects; + } + + return new MessageRedirectsCollection(); + } + } + + public async Task Save(MessageRedirectsCollection redirects) + { + using (var session = store.OpenAsyncSession()) + { + await session.StoreAsync(redirects, redirects.ETag, DefaultId); + await session.SaveChangesAsync(); + } + } + + const string DefaultId = "messageredirects"; + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/MetadataExtensions.cs b/src/ServiceControl.Persistence.RavenDb5/MetadataExtensions.cs new file mode 100644 index 0000000000..ddfb3ce644 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/MetadataExtensions.cs @@ -0,0 +1,40 @@ +namespace ServiceControl +{ + using System; + using System.Collections.Generic; + + static class MetadataExtensions + { + public static T GetOrDefault(this IDictionary metadata, string key) + { + if (metadata.TryGetValue(key, out var foundValue)) + { + return (T)foundValue; + } + + return default; + } + + public static string GetAsStringOrNull(this IDictionary metadata, string key) + { + if (metadata.TryGetValue(key, out var foundValue)) + { + return foundValue?.ToString(); + } + + return null; + } + + public static DateTime? GetAsNullableDatetime(this IDictionary metadata, string key) + { + var datetimeAsString = metadata.GetAsStringOrNull(key); + + if (datetimeAsString != null) + { + return DateTime.Parse(datetimeAsString); + } + + return null; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/QueueAddressStore.cs b/src/ServiceControl.Persistence.RavenDb5/QueueAddressStore.cs new file mode 100644 index 0000000000..c951bb94d7 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/QueueAddressStore.cs @@ -0,0 +1,53 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System.Collections.Generic; + using System.Linq; + using System.Threading.Tasks; + using Raven.Client; + using Raven.Client.Linq; + using ServiceControl.MessageFailures; + using ServiceControl.MessageFailures.Api; + using ServiceControl.Persistence.Infrastructure; + + class QueueAddressStore : IQueueAddressStore + { + readonly IDocumentStore documentStore; + + public QueueAddressStore(IDocumentStore documentStore) + { + this.documentStore = documentStore; + } + + public async Task>> GetAddresses(PagingInfo pagingInfo) + { + using (var session = documentStore.OpenAsyncSession()) + { + var addresses = await session + .Query() + .Statistics(out var stats) + .Paging(pagingInfo) + .ToListAsync(); + + var result = new QueryResult>(addresses, stats.ToQueryStatsInfo()); + return result; + } + } + + public async Task>> GetAddressesBySearchTerm(string search, PagingInfo pagingInfo) + { + using (var session = documentStore.OpenAsyncSession()) + { + var failedMessageQueues = await session + .Query() + .Statistics(out var stats) + .Where(q => q.PhysicalAddress.StartsWith(search)) + .OrderBy(q => q.PhysicalAddress) + .Paging(pagingInfo) + .ToListAsync(); + + var result = new QueryResult>(failedMessageQueues, stats.ToQueryStatsInfo()); + return result; + } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs b/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs new file mode 100644 index 0000000000..8fdc936d17 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs @@ -0,0 +1,112 @@ +namespace ServiceControl.Operations.BodyStorage.RavenAttachments +{ + using System; + using System.IO; + using System.Threading.Tasks; + using Raven.Client; + using Raven.Json.Linq; + using ServiceControl.CompositeViews.Messages; + using ServiceControl.Persistence; + + // For Raven5, look at how the Audit instance is implementing this, as Attachments won't exist + // and there will be no need for a fallback method on a new persistence + class RavenAttachmentsBodyStorage : IBodyStorage + { + readonly IDocumentStore documentStore; + + public RavenAttachmentsBodyStorage(IDocumentStore documentStore) + { + this.documentStore = documentStore; + } + + public Task Store(string bodyId, string contentType, int bodySize, Stream bodyStream) + { + var id = MessageBodyIdGenerator.MakeDocumentId(bodyId); + //We want to continue using attachments for now +#pragma warning disable 618 + return documentStore.AsyncDatabaseCommands.PutAttachmentAsync(id, null, bodyStream, + new RavenJObject +#pragma warning restore 618 + { + {"ContentType", contentType}, + {"ContentLength", bodySize} + }); + } + + // The RavenDB5 implementation, like the Audit instance, will not use Attachments + public async Task TryFetch(string bodyId) + { + bodyId = bodyId?.Replace("/", @"\"); + + // First try loading from index + + using (var session = documentStore.OpenAsyncSession()) + { + var message = await session.Query() + .Statistics(out var stats) + .TransformWith() + .FirstOrDefaultAsync(f => f.MessageId == bodyId); + + if (message != null) + { + if (message.Body == null && message.BodyNotStored) + { + return new MessageBodyStreamResult { HasResult = false }; + } + + if (message.Body != null) + { + var stream = new MemoryStream(); + var writer = new StreamWriter(stream); + // ReSharper disable once MethodHasAsyncOverload, operations are MemoryStream so do not require async overloads + writer.Write(message.Body); + // ReSharper disable once MethodHasAsyncOverload, operations are MemoryStream so do not require async overloads + writer.Flush(); + stream.Position = 0; + + return new MessageBodyStreamResult + { + HasResult = true, + Stream = stream, + ContentType = message.ContentType, + BodySize = message.BodySize, + Etag = stats.IndexEtag + }; + } + else if (message.BodySize == 0) + { + return new MessageBodyStreamResult + { + HasResult = true, + Stream = new MemoryStream(Array.Empty()), + ContentType = message.ContentType, + BodySize = message.BodySize, + Etag = stats.IndexEtag + }; + } + } + } + + var id = MessageBodyIdGenerator.MakeDocumentId(bodyId); + + //We want to continue using attachments for now +#pragma warning disable 618 + var attachment = await documentStore.AsyncDatabaseCommands.GetAttachmentAsync(id); +#pragma warning restore 618 + + if (attachment != null) + { + return new MessageBodyStreamResult + { + HasResult = true, + Stream = (MemoryStream)attachment.Data(), + ContentType = attachment.Metadata["ContentType"].Value(), + BodySize = attachment.Metadata["ContentLength"].Value(), + Etag = attachment.Etag + }; + } + + return null; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs b/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs new file mode 100644 index 0000000000..16863103e5 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs @@ -0,0 +1,232 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System; + using System.ComponentModel.Composition.Hosting; + using System.Globalization; + using System.IO; + using System.Runtime.Serialization; + using ByteSizeLib; + using NServiceBus.Logging; + using Raven.Abstractions.Data; + using Raven.Client.Document; + using Raven.Client.Embedded; + using ServiceControl.Infrastructure.RavenDB; + using ServiceControl.Infrastructure.RavenDB.Subscriptions; + + static class RavenBootstrapper + { + public const string DatabasePathKey = "DbPath"; + public const string HostNameKey = "HostName"; + public const string DatabaseMaintenancePortKey = "DatabaseMaintenancePort"; + public const string ExposeRavenDBKey = "ExposeRavenDB"; + public const string ExpirationProcessTimerInSecondsKey = "ExpirationProcessTimerInSeconds"; + public const string ExpirationProcessBatchSizeKey = "ExpirationProcessBatchSize"; + public const string RunCleanupBundleKey = "RavenDB35/RunCleanupBundle"; + public const string RunInMemoryKey = "RavenDB35/RunInMemory"; + public const string MinimumStorageLeftRequiredForIngestionKey = "MinimumStorageLeftRequiredForIngestion"; + + + public static RavenDBPersisterSettings Settings { get; private set; } + + public static void Configure(EmbeddableDocumentStore documentStore, RavenDBPersisterSettings settings) + { + Settings = settings; + + var runInMemory = settings.RunInMemory; + + if (runInMemory) + { + documentStore.RunInMemory = true; + } + else + { + var dbPath = settings.DatabasePath; + + if (string.IsNullOrEmpty(dbPath)) + { + throw new InvalidOperationException($"{DatabasePathKey} is mandatory"); + } + + Directory.CreateDirectory(dbPath); + + documentStore.DataDirectory = dbPath; + documentStore.Configuration.CompiledIndexCacheDirectory = dbPath; + documentStore.Listeners.RegisterListener(new SubscriptionsLegacyAddressConverter()); + } + + var exposeRavenDB = settings.ExposeRavenDB; + + documentStore.UseEmbeddedHttpServer = settings.MaintenanceMode || exposeRavenDB; + documentStore.EnlistInDistributedTransactions = false; + + var localRavenLicense = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "RavenLicense.xml"); + if (File.Exists(localRavenLicense)) + { + Logger.InfoFormat("Loading RavenDB license found from {0}", localRavenLicense); + documentStore.Configuration.Settings["Raven/License"] = ReadAllTextWithoutLocking(localRavenLicense); + } + else + { + Logger.InfoFormat("Loading Embedded RavenDB license"); + documentStore.Configuration.Settings["Raven/License"] = ReadLicense(); + } + + //This is affects only remote access to the database in maintenance mode and enables access without authentication + documentStore.Configuration.Settings["Raven/AnonymousAccess"] = "Admin"; + documentStore.Configuration.Settings["Raven/Licensing/AllowAdminAnonymousAccessForCommercialUse"] = "true"; + + var runCleanupBundle = settings.RunCleanupBundle; + + if (runCleanupBundle) + { + documentStore.Configuration.Settings.Add("Raven/ActiveBundles", "CustomDocumentExpiration"); + } + + documentStore.Configuration.DisableClusterDiscovery = true; + documentStore.Configuration.ResetIndexOnUncleanShutdown = true; + + if (settings.DatabaseMaintenancePort == 0) + { + throw new Exception($"{DatabaseMaintenancePortKey} is mandatory."); + } + + documentStore.Configuration.Port = settings.DatabaseMaintenancePort; + + if (string.IsNullOrEmpty(settings.HostName)) + { + throw new Exception($"{HostNameKey} is mandatory."); + } + + var hostName = settings.HostName; + + documentStore.Configuration.HostName = hostName == "*" || hostName == "+" + ? "localhost" + : hostName; + documentStore.Conventions.SaveEnumsAsIntegers = true; + documentStore.Conventions.CustomizeJsonSerializer = serializer => serializer.Binder = MigratedTypeAwareBinder; + documentStore.Configuration.Catalog.Catalogs.Add(new AssemblyCatalog(typeof(RavenBootstrapper).Assembly)); + documentStore.Conventions.FindClrType = (id, doc, metadata) => + { + var clrtype = metadata.Value(Constants.RavenClrType); + + // The CLR type cannot be assumed to be always there + if (clrtype == null) + { + return null; + } + + if (clrtype.EndsWith(".Subscription, NServiceBus.Core")) + { + clrtype = ReflectionUtil.GetFullNameWithoutVersionInformation(typeof(Subscription)); + } + else if (clrtype.EndsWith(".Subscription, NServiceBus.RavenDB")) + { + clrtype = ReflectionUtil.GetFullNameWithoutVersionInformation(typeof(Subscription)); + } + + return clrtype; + }; + + if (settings.MaintenanceMode) + { + Logger.InfoFormat($"RavenDB is now accepting requests on {settings.DatabaseMaintenanceUrl}"); + } + + if (settings.RunInMemory == false) + { + RecordStartup(); + } + } + + static void RecordStartup() + { + var dataSize = DataSize(); + var folderSize = FolderSize(); + + var startupMessage = $@" +------------------------------------------------------------- +Database Size: {ByteSize.FromBytes(dataSize).ToString("#.##", CultureInfo.InvariantCulture)} +Database Folder Size: {ByteSize.FromBytes(folderSize).ToString("#.##", CultureInfo.InvariantCulture)} +-------------------------------------------------------------"; + + Logger.Info(startupMessage); + } + + public static string ReadLicense() + { + using (var resourceStream = typeof(RavenBootstrapper).Assembly.GetManifestResourceStream("ServiceControl.Persistence.RavenDb.RavenLicense.xml")) + using (var reader = new StreamReader(resourceStream)) + { + return reader.ReadToEnd(); + } + } + + static string ReadAllTextWithoutLocking(string path) + { + using (var fileStream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) + using (var textReader = new StreamReader(fileStream)) + { + return textReader.ReadToEnd(); + } + } + + static long DataSize() + { + var datafilePath = Path.Combine(Settings.DatabasePath, "data"); + + try + { + var info = new FileInfo(datafilePath); + if (!info.Exists) + { + return -1; + } + return info.Length; + } + catch + { + return -1; + } + } + + static long FolderSize() + { + try + { + var dir = new DirectoryInfo(Settings.DatabasePath); + var dirSize = DirSize(dir); + return dirSize; + } + catch + { + return -1; + } + } + + static long DirSize(DirectoryInfo d) + { + long size = 0; + if (d.Exists) + { + FileInfo[] fis = d.GetFiles(); + foreach (FileInfo fi in fis) + { + size += fi.Length; + } + + DirectoryInfo[] dis = d.GetDirectories(); + foreach (DirectoryInfo di in dis) + { + size += DirSize(di); + } + } + + return size; + } + + static readonly SerializationBinder MigratedTypeAwareBinder = new MigratedTypeAwareBinder(); + + static readonly ILog Logger = LogManager.GetLogger(typeof(RavenBootstrapper)); + + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs new file mode 100644 index 0000000000..a15e1e4e3f --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs @@ -0,0 +1,24 @@ +using System; +using ServiceControl.Infrastructure.RavenDB.Expiration; +using ServiceControl.Operations; +using ServiceControl.Persistence; + +class RavenDBPersisterSettings : PersistenceSettings +{ + public string HostName { get; set; } = "localhost"; + public int DatabaseMaintenancePort { get; set; } = DatabaseMaintenancePortDefault; + public string DatabaseMaintenanceUrl => $"http://{HostName}:{DatabaseMaintenancePort}"; + public bool ExposeRavenDB { get; set; } + public int ExpirationProcessTimerInSeconds { get; set; } = ExpiredDocumentsCleanerBundle.ExpirationProcessTimerInSecondsDefault; + public int ExpirationProcessBatchSize { get; set; } = ExpiredDocumentsCleanerBundle.ExpirationProcessBatchSizeDefault; + public bool RunCleanupBundle { get; set; } + public bool RunInMemory { get; set; } + public int MinimumStorageLeftRequiredForIngestion { get; set; } = CheckMinimumStorageRequiredForIngestion.MinimumStorageLeftRequiredForIngestionDefault; + public int DataSpaceRemainingThreshold { get; set; } = CheckFreeDiskSpace.DataSpaceRemainingThresholdDefault; + public TimeSpan ErrorRetentionPeriod { get; set; } + public TimeSpan EventsRetentionPeriod { get; set; } + public TimeSpan? AuditRetentionPeriod { get; set; } + public int ExternalIntegrationsDispatchingBatchSize { get; set; } = 100; + + public const int DatabaseMaintenancePortDefault = 33334; +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs new file mode 100644 index 0000000000..483e3b3f01 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs @@ -0,0 +1,113 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading.Tasks; + using Raven.Client; + using Raven.Client.Linq; + using ServiceControl.Contracts.CustomChecks; + using ServiceControl.Persistence; + using ServiceControl.Persistence.Infrastructure; + + class RavenDbCustomCheckDataStore : ICustomChecksDataStore + { + public RavenDbCustomCheckDataStore(IDocumentStore store) + { + this.store = store; + } + + public async Task UpdateCustomCheckStatus(CustomCheckDetail detail) + { + var status = CheckStateChange.Unchanged; + var id = detail.GetDeterministicId(); + + using (var session = store.OpenAsyncSession()) + { + var customCheck = await session.LoadAsync(id); + + if (customCheck == null || + (customCheck.Status == Status.Fail && !detail.HasFailed) || + (customCheck.Status == Status.Pass && detail.HasFailed)) + { + if (customCheck == null) + { + customCheck = new CustomCheck + { + Id = id + }; + } + + status = CheckStateChange.Changed; + } + + customCheck.CustomCheckId = detail.CustomCheckId; + customCheck.Category = detail.Category; + customCheck.Status = detail.HasFailed ? Status.Fail : Status.Pass; + customCheck.ReportedAt = detail.ReportedAt; + customCheck.FailureReason = detail.FailureReason; + customCheck.OriginatingEndpoint = detail.OriginatingEndpoint; + await session.StoreAsync(customCheck); + await session.SaveChangesAsync(); + } + + return status; + } + + public async Task>> GetStats(PagingInfo paging, string status = null) + { + using (var session = store.OpenAsyncSession()) + { + var query = + session.Query().Statistics(out var stats); + + query = AddStatusFilter(query, status); + + var results = await query + .Paging(paging) + .ToListAsync(); + + return new QueryResult>(results, new QueryStatsInfo(stats.IndexEtag, stats.TotalResults, stats.IsStale)); + } + } + + public async Task DeleteCustomCheck(Guid id) + { + await store.AsyncDatabaseCommands.DeleteAsync(store.Conventions.DefaultFindFullDocumentKeyFromNonStringIdentifier(id, typeof(CustomCheck), false), null); + } + + public async Task GetNumberOfFailedChecks() + { + using (var session = store.OpenAsyncSession()) + { + var failedCustomCheckCount = await session.Query().CountAsync(p => p.Status == Status.Fail); + + return failedCustomCheckCount; + } + } + + static IRavenQueryable AddStatusFilter(IRavenQueryable query, string status) + { + if (status == null) + { + return query; + } + + if (status == "fail") + { + query = query.Where(c => c.Status == Status.Fail); + } + + if (status == "pass") + { + query = query.Where(c => c.Status == Status.Pass); + } + + return query; + } + + public Task Setup() => Task.CompletedTask; + + IDocumentStore store; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbInstaller.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbInstaller.cs new file mode 100644 index 0000000000..c940a0448d --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbInstaller.cs @@ -0,0 +1,40 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System.Threading; + using System.Threading.Tasks; + using NServiceBus.Logging; + using Raven.Client.Embedded; + using ServiceControl.Infrastructure.RavenDB; + + class RavenDbInstaller : IPersistenceInstaller + { + public RavenDbInstaller(EmbeddableDocumentStore documentStore, RavenStartup ravenStartup) + { + this.documentStore = documentStore; + this.ravenStartup = ravenStartup; + } + + public async Task Install(CancellationToken cancellationToken = default) + { + Logger.Info("Database initialization starting"); + documentStore.Initialize(); + Logger.Info("Database initialization complete"); + + Logger.Info("Index creation started"); + await ravenStartup.CreateIndexesAsync(documentStore); + Logger.Info("Index creation complete"); + + Logger.Info("Data migrations starting"); + var purgeTempIdKnownEndpoints = new PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration(); + await purgeTempIdKnownEndpoints.Migrate(documentStore); + Logger.Info("Data migrations complete"); + + } + + readonly EmbeddableDocumentStore documentStore; + readonly RavenStartup ravenStartup; + + static readonly ILog Logger = LogManager.GetLogger(typeof(RavenDbPersistenceLifecycle)); + + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbMonitoringDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbMonitoringDataStore.cs new file mode 100644 index 0000000000..f04a89718b --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbMonitoringDataStore.cs @@ -0,0 +1,131 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading.Tasks; + using Raven.Client; + using ServiceControl.Operations; + using ServiceControl.Persistence; + + class RavenDbMonitoringDataStore : IMonitoringDataStore + { + public RavenDbMonitoringDataStore(IDocumentStore store) + { + this.store = store; + } + + public async Task CreateIfNotExists(EndpointDetails endpoint) + { + var id = endpoint.GetDeterministicId(); + + using (var session = store.OpenAsyncSession()) + { + var knownEndpoint = await session.LoadAsync(id); + + if (knownEndpoint != null) + { + return; + } + + knownEndpoint = new KnownEndpoint + { + Id = id, + EndpointDetails = endpoint, + HostDisplayName = endpoint.Host, + Monitored = false + }; + + await session.StoreAsync(knownEndpoint); + + await session.SaveChangesAsync(); + } + } + + public async Task CreateOrUpdate(EndpointDetails endpoint, IEndpointInstanceMonitoring endpointInstanceMonitoring) + { + var id = endpoint.GetDeterministicId(); + + using (var session = store.OpenAsyncSession()) + { + var knownEndpoint = await session.LoadAsync(id); + + if (knownEndpoint == null) + { + knownEndpoint = new KnownEndpoint + { + Id = id, + EndpointDetails = endpoint, + HostDisplayName = endpoint.Host, + Monitored = true + }; + + await session.StoreAsync(knownEndpoint); + } + else + { + knownEndpoint.Monitored = endpointInstanceMonitoring.IsMonitored(id); + } + + await session.SaveChangesAsync(); + } + } + + public async Task UpdateEndpointMonitoring(EndpointDetails endpoint, bool isMonitored) + { + var id = endpoint.GetDeterministicId(); + + using (var session = store.OpenAsyncSession()) + { + var knownEndpoint = await session.LoadAsync(id); + + if (knownEndpoint != null) + { + knownEndpoint.Monitored = isMonitored; + + await session.SaveChangesAsync(); + } + } + } + + public async Task WarmupMonitoringFromPersistence(IEndpointInstanceMonitoring endpointInstanceMonitoring) + { + using (var session = store.OpenAsyncSession()) + { + using (var endpointsEnumerator = await session.Advanced.StreamAsync(session.Query()) + ) + { + while (await endpointsEnumerator.MoveNextAsync()) + { + var endpoint = endpointsEnumerator.Current.Document; + + endpointInstanceMonitoring.DetectEndpointFromPersistentStore(endpoint.EndpointDetails, endpoint.Monitored); + } + } + } + } + + public async Task Delete(Guid endpointId) + { + using (var session = store.OpenAsyncSession()) + { + session.Delete(endpointId); + await session.SaveChangesAsync(); + } + } + public async Task> GetAllKnownEndpoints() + { + using (var session = store.OpenAsyncSession()) + { + var knownEndpoints = await session.Query() + .ToListAsync(); + + return knownEndpoints.ToArray(); + } + } + + public Task Setup() => Task.CompletedTask; + + IDocumentStore store; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs new file mode 100644 index 0000000000..bab32ac5bd --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs @@ -0,0 +1,95 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using MessageRedirects; + using Microsoft.Extensions.DependencyInjection; + using Persistence.Recoverability; + using Raven.Client; + using Raven.Client.Embedded; + using Recoverability; + using ServiceControl.CustomChecks; + using ServiceControl.Infrastructure.RavenDB.Subscriptions; + using ServiceControl.MessageFailures; + using ServiceControl.Operations; + using ServiceControl.Operations.BodyStorage; + using ServiceControl.Operations.BodyStorage.RavenAttachments; + using ServiceControl.Persistence.MessageRedirects; + using ServiceControl.Persistence.RavenDb.SagaAudit; + using ServiceControl.Persistence.UnitOfWork; + using ServiceControl.Recoverability; + + class RavenDbPersistence : IPersistence + { + public RavenDbPersistence(RavenDBPersisterSettings settings, EmbeddableDocumentStore documentStore, RavenStartup ravenStartup) + { + this.settings = settings; + this.documentStore = documentStore; + this.ravenStartup = ravenStartup; + } + + public void Configure(IServiceCollection serviceCollection) + { + if (settings.MaintenanceMode) + { + return; + } + + serviceCollection.AddSingleton(settings); + serviceCollection.AddSingleton(documentStore); + + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddUnitOfWorkFactory(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(p => p.GetRequiredService()); + serviceCollection.AddHostedService(p => p.GetRequiredService()); + + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(p => p.GetRequiredService()); + serviceCollection.AddHostedService(p => p.GetRequiredService()); + + serviceCollection.AddCustomCheck(); + serviceCollection.AddCustomCheck(); + serviceCollection.AddCustomCheck(); + serviceCollection.AddCustomCheck(); + serviceCollection.AddCustomCheck(); + serviceCollection.AddCustomCheck(); + + serviceCollection.AddSingleton(); + + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + } + + public IPersistenceLifecycle CreateLifecycle() + { + return new RavenDbPersistenceLifecycle(ravenStartup, documentStore); + } + + public IPersistenceInstaller CreateInstaller() + { + return new RavenDbInstaller(documentStore, ravenStartup); + } + + readonly RavenStartup ravenStartup; + readonly RavenDBPersisterSettings settings; + readonly EmbeddableDocumentStore documentStore; + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs new file mode 100644 index 0000000000..640e732654 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs @@ -0,0 +1,80 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System; + using Raven.Client.Embedded; + using ServiceControl.Infrastructure.RavenDB.Expiration; + using ServiceControl.Operations; + + class RavenDbPersistenceConfiguration : IPersistenceConfiguration + { + public const string DataSpaceRemainingThresholdKey = "DataSpaceRemainingThreshold"; + const string AuditRetentionPeriodKey = "AuditRetentionPeriod"; + const string ErrorRetentionPeriodKey = "ErrorRetentionPeriod"; + const string EventsRetentionPeriodKey = "EventsRetentionPeriod"; + const string ExternalIntegrationsDispatchingBatchSizeKey = "ExternalIntegrationsDispatchingBatchSize"; + const string MaintenanceModeKey = "MaintenanceMode"; + + public PersistenceSettings CreateSettings(Func tryReadSetting) + { + T GetRequiredSetting(string key) + { + var (exists, value) = tryReadSetting(key, typeof(T)); + + if (exists) + { + return (T)value; + } + + throw new Exception($"Setting {key} of type {typeof(T)} is required"); + } + + T GetSetting(string key, T defaultValue) + { + var (exists, value) = tryReadSetting(key, typeof(T)); + + if (exists) + { + return (T)value; + } + else + { + return defaultValue; + } + } + + var settings = new RavenDBPersisterSettings + { + DatabasePath = GetSetting(RavenBootstrapper.DatabasePathKey, default), + HostName = GetSetting(RavenBootstrapper.HostNameKey, "localhost"), + DatabaseMaintenancePort = GetSetting(RavenBootstrapper.DatabaseMaintenancePortKey, RavenDBPersisterSettings.DatabaseMaintenancePortDefault), + ExposeRavenDB = GetSetting(RavenBootstrapper.ExposeRavenDBKey, false), + ExpirationProcessTimerInSeconds = GetSetting(RavenBootstrapper.ExpirationProcessTimerInSecondsKey, ExpiredDocumentsCleanerBundle.ExpirationProcessTimerInSecondsDefault), + ExpirationProcessBatchSize = GetSetting(RavenBootstrapper.ExpirationProcessBatchSizeKey, ExpiredDocumentsCleanerBundle.ExpirationProcessBatchSizeDefault), + RunCleanupBundle = GetSetting(RavenBootstrapper.RunCleanupBundleKey, true), + RunInMemory = GetSetting(RavenBootstrapper.RunInMemoryKey, false), + MinimumStorageLeftRequiredForIngestion = GetSetting(RavenBootstrapper.MinimumStorageLeftRequiredForIngestionKey, CheckMinimumStorageRequiredForIngestion.MinimumStorageLeftRequiredForIngestionDefault), + DataSpaceRemainingThreshold = GetSetting(DataSpaceRemainingThresholdKey, CheckFreeDiskSpace.DataSpaceRemainingThresholdDefault), + ErrorRetentionPeriod = GetRequiredSetting(ErrorRetentionPeriodKey), + EventsRetentionPeriod = GetSetting(EventsRetentionPeriodKey, TimeSpan.FromDays(14)), + AuditRetentionPeriod = GetSetting(AuditRetentionPeriodKey, TimeSpan.Zero), + ExternalIntegrationsDispatchingBatchSize = GetSetting(ExternalIntegrationsDispatchingBatchSizeKey, 100), + MaintenanceMode = GetSetting(MaintenanceModeKey, false), + }; + + CheckFreeDiskSpace.Validate(settings); + CheckMinimumStorageRequiredForIngestion.Validate(settings); + return settings; + } + + public IPersistence Create(PersistenceSettings settings) + { + var specificSettings = (RavenDBPersisterSettings)settings; + + var documentStore = new EmbeddableDocumentStore(); + RavenBootstrapper.Configure(documentStore, specificSettings); + + var ravenStartup = new RavenStartup(); + return new RavenDbPersistence(specificSettings, documentStore, ravenStartup); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceLifecycle.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceLifecycle.cs new file mode 100644 index 0000000000..66c4b3af3a --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceLifecycle.cs @@ -0,0 +1,62 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using NServiceBus.Logging; + using Raven.Client; + using Raven.Client.Embedded; + using ServiceControl.Persistence; + + class RavenDbPersistenceLifecycle : IPersistenceLifecycle + { + public RavenDbPersistenceLifecycle(RavenStartup ravenStartup, EmbeddableDocumentStore documentStore) + { + this.ravenStartup = ravenStartup; + this.documentStore = documentStore; + } + + public async Task Start(CancellationToken cancellationToken) + { + Logger.Info("Database initialization starting"); + documentStore.Initialize(); + Logger.Info("Database initialization complete"); + + await ravenStartup.CreateIndexesAsync(documentStore); + + Logger.Info("Testing indexes"); + await TestAllIndexesAndResetIfException(documentStore); + } + + public Task Stop(CancellationToken cancellationToken) + { + documentStore.Dispose(); + return Task.CompletedTask; + } + + static async Task TestAllIndexesAndResetIfException(IDocumentStore store) + { + foreach (var index in store.DatabaseCommands.GetStatistics().Indexes) + { + try + { + using (var session = store.OpenAsyncSession()) + { + await session.Advanced.AsyncDocumentQuery(index.Name).Take(1).ToListAsync(); + } + } + catch (Exception ex) + { + Logger.Warn($"When trying to fetch 1 document from index {index.Name} the following exception was thrown: {ex}"); + Logger.Warn($"Attempting to reset errored index: [{index.Name}] priority: {index.Priority} is valid: {index.IsInvalidIndex} indexing attempts: {index.IndexingAttempts}, failed indexing attempts:{index.IndexingErrors}"); + store.DatabaseCommands.ResetIndex(index.Name); + } + } + } + + readonly RavenStartup ravenStartup; + readonly EmbeddableDocumentStore documentStore; + + static readonly ILog Logger = LogManager.GetLogger(typeof(RavenDbPersistenceLifecycle)); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenLicense.json b/src/ServiceControl.Persistence.RavenDb5/RavenLicense.json new file mode 100644 index 0000000000..9afe8a04d1 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenLicense.json @@ -0,0 +1,16 @@ +{ +"Id": "64c6a174-3f3a-4e7d-ac5d-b3eedd801460", +"Name": "ParticularNservicebus (Israel)", +"Keys": [ +"kWznr3iTRWkzuYjFJH8IvS1LY", +"t5GJId7H/h5i4YjW2Z7YPgUoR", +"igl/wrjx4y6pmXcChQRnTu1TK", +"Q1scjkLMB2aaH9VZ5G2s4E0gz", +"08GaHnSOHpRVz6SgjFGcAqnEb", +"c0ZhNNOGsBcrOVx+KOq7+ggHs", +"MtKI8e8mCRcgiaTOkPURpAAyU", +"GKEkFCisMLQ4vMAcREjM0NRYX", +"GhufAh8AnwIgAJ8CIwBBMkMMR", +"AY4OTw9Pp8CISBiP1g=" +] +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs b/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs new file mode 100644 index 0000000000..94c4897bb2 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs @@ -0,0 +1,421 @@ +namespace ServiceControl.Persistence +{ + using System; + using System.Collections.Generic; + using System.Globalization; + using System.Linq; + using System.Linq.Expressions; + using System.Net.Http; + using System.Text; + using Raven.Client; + using Raven.Client.Linq; + using ServiceControl.MessageFailures; + using ServiceControl.Persistence.Infrastructure; + + static class RavenQueryExtensions + { + public static IRavenQueryable IncludeSystemMessagesWhere( + this IRavenQueryable source, bool includeSystemMessages) + { + if (!includeSystemMessages) + { + return source.Where(m => !m.IsSystemMessage); + } + + return source; + } + + public static IRavenQueryable Paging(this IRavenQueryable source, PagingInfo pagingInfo) + => source.Skip(pagingInfo.Offset).Take(pagingInfo.PageSize); + + public static IRavenQueryable Sort(this IRavenQueryable source, SortInfo sortInfo) + where TSource : MessagesViewIndex.SortAndFilterOptions + { + Expression> keySelector; + switch (sortInfo.Sort) + { + case "id": + case "message_id": + keySelector = m => m.MessageId; + break; + + case "message_type": + keySelector = m => m.MessageType; + break; + + case "critical_time": + keySelector = m => m.CriticalTime; + break; + + case "delivery_time": + keySelector = m => m.DeliveryTime; + break; + + case "processing_time": + keySelector = m => m.ProcessingTime; + break; + + case "processed_at": + keySelector = m => m.ProcessedAt; + break; + + case "status": + keySelector = m => m.Status; + break; + + default: + keySelector = m => m.TimeSent; + break; + } + + if (sortInfo.Direction == "asc") + { + return source.OrderBy(keySelector); + } + + return source.OrderByDescending(keySelector); + } + + public static IAsyncDocumentQuery Paging(this IAsyncDocumentQuery source, PagingInfo pagingInfo) + { + var maxResultsPerPage = pagingInfo.PageSize; + if (maxResultsPerPage < 1) + { + maxResultsPerPage = 50; + } + + var page = pagingInfo.Page; + if (page < 1) + { + page = 1; + } + + var skipResults = (page - 1) * maxResultsPerPage; + + return source.Skip(skipResults) + .Take(maxResultsPerPage); + } + + public static IAsyncDocumentQuery Sort(this IAsyncDocumentQuery source, SortInfo sortInfo) + { + var descending = true; + + var direction = sortInfo.Direction; + if (direction == "asc") + { + descending = false; + } + + string keySelector; + + var sort = sortInfo.Sort; + if (!AsyncDocumentQuerySortOptions.Contains(sort)) + { + sort = "time_sent"; + } + + switch (sort) + { + case "id": + case "message_id": + keySelector = "MessageId"; + break; + + case "message_type": + keySelector = "MessageType"; + break; + + case "status": + keySelector = "Status"; + break; + + case "modified": + keySelector = "LastModified"; + break; + + case "time_of_failure": + keySelector = "TimeOfFailure"; + break; + + default: + keySelector = "TimeSent"; + break; + } + + return source.AddOrder(keySelector, descending); + } + + + public static IAsyncDocumentQuery FilterByStatusWhere(this IAsyncDocumentQuery source, string status) + { + if (status == null) + { + return source; + } + + var filters = status.Replace(" ", string.Empty).Split(','); + var excludes = new List(); + var includes = new List(); + + foreach (var filter in filters) + { + FailedMessageStatus failedMessageStatus; + + if (filter.StartsWith("-")) + { + if (Enum.TryParse(filter.Substring(1), true, out failedMessageStatus)) + { + excludes.Add((int)failedMessageStatus); + } + + continue; + } + + if (Enum.TryParse(filter, true, out failedMessageStatus)) + { + includes.Add((int)failedMessageStatus); + } + } + + var sb = new StringBuilder(); + + sb.Append("(("); + if (includes.Count == 0) + { + sb.Append("*"); + } + else + { + sb.Append(string.Join(" OR ", includes.ToArray())); + } + + sb.Append(")"); + + if (excludes.Count > 0) + { + sb.Append(" AND NOT ("); + sb.Append(string.Join(" OR ", excludes.ToArray())); + sb.Append(")"); + } + + sb.Append(")"); + + source.AndAlso(); + source.Where($"Status: {sb}"); + + return source; + } + + + public static IAsyncDocumentQuery FilterByLastModifiedRange(this IAsyncDocumentQuery source, string modified) + { + if (modified == null) + { + return source; + } + + var filters = modified.Split(SplitChars, StringSplitOptions.None); + if (filters.Length != 2) + { + throw new Exception("Invalid modified date range, dates need to be in ISO8601 format and it needs to be a range eg. 2016-03-11T00:27:15.474Z...2016-03-16T03:27:15.474Z"); + } + + try + { + var from = DateTime.Parse(filters[0], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); + var to = DateTime.Parse(filters[1], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); + + source.AndAlso(); + source.WhereBetweenOrEqual("LastModified", from.Ticks, to.Ticks); + } + catch (Exception) + { + throw new Exception("Invalid modified date range, dates need to be in ISO8601 format and it needs to be a range eg. 2016-03-11T00:27:15.474Z...2016-03-16T03:27:15.474Z"); + } + + return source; + } + + public static IAsyncDocumentQuery FilterByQueueAddress(this IAsyncDocumentQuery source, string queueAddress) + { + if (string.IsNullOrWhiteSpace(queueAddress)) + { + return source; + } + + source.AndAlso(); + source.WhereEquals("QueueAddress", queueAddress.ToLowerInvariant()); + + return source; + } + + public static IRavenQueryable IncludeSystemMessagesWhere( + this IRavenQueryable source, HttpRequestMessage request) + { + var includeSystemMessages = request.GetQueryStringValue("include_system_messages", false); + return !includeSystemMessages ? source.Where(m => !m.IsSystemMessage) : source; + } + + public static IOrderedQueryable Paging(this IOrderedQueryable source, HttpRequestMessage request) + { + var maxResultsPerPage = request.GetQueryStringValue("per_page", 50); + if (maxResultsPerPage < 1) + { + maxResultsPerPage = 50; + } + + var page = request.GetQueryStringValue("page", 1); + + if (page < 1) + { + page = 1; + } + + var skipResults = (page - 1) * maxResultsPerPage; + + return (IOrderedQueryable)source.Skip(skipResults) + .Take(maxResultsPerPage); + } + + public static IRavenQueryable Paging(this IRavenQueryable source, HttpRequestMessage request) + { + var maxResultsPerPage = request.GetQueryStringValue("per_page", 50); + if (maxResultsPerPage < 1) + { + maxResultsPerPage = 50; + } + + var page = request.GetQueryStringValue("page", 1); + + if (page < 1) + { + page = 1; + } + + var skipResults = (page - 1) * maxResultsPerPage; + + return source.Skip(skipResults) + .Take(maxResultsPerPage); + } + + public static IRavenQueryable Sort(this IRavenQueryable source, HttpRequestMessage request, + Expression> defaultKeySelector = null, string defaultSortDirection = "desc") + where TSource : MessagesViewIndex.SortAndFilterOptions + { + var direction = request.GetQueryStringValue("direction", defaultSortDirection); + if (direction != "asc" && direction != "desc") + { + direction = defaultSortDirection; + } + + Expression> keySelector; + var sort = request.GetQueryStringValue("sort", "time_sent"); + if (!RavenQueryableSortOptions.Contains(sort)) + { + sort = "time_sent"; + } + + switch (sort) + { + case "id": + case "message_id": + keySelector = m => m.MessageId; + break; + + case "message_type": + keySelector = m => m.MessageType; + break; + + case "critical_time": + keySelector = m => m.CriticalTime; + break; + + case "delivery_time": + keySelector = m => m.DeliveryTime; + break; + + case "processing_time": + keySelector = m => m.ProcessingTime; + break; + + case "processed_at": + keySelector = m => m.ProcessedAt; + break; + + case "status": + keySelector = m => m.Status; + break; + + default: + if (defaultKeySelector == null) + { + keySelector = m => m.TimeSent; + } + else + { + keySelector = defaultKeySelector; + } + + break; + } + + if (direction == "asc") + { + return source.OrderBy(keySelector); + } + + return source.OrderByDescending(keySelector); + } + + public static T GetQueryStringValue(this HttpRequestMessage request, string key, T defaultValue = default) + { + Dictionary queryStringDictionary; + if (!request.Properties.TryGetValue("QueryStringAsDictionary", out var dictionaryAsObject)) + { + queryStringDictionary = request.GetQueryNameValuePairs().ToDictionary(x => x.Key, x => x.Value, StringComparer.OrdinalIgnoreCase); + request.Properties["QueryStringAsDictionary"] = queryStringDictionary; + } + else + { + queryStringDictionary = (Dictionary)dictionaryAsObject; + } + + queryStringDictionary.TryGetValue(key, out var value); + if (string.IsNullOrEmpty(value)) + { + return defaultValue; + } + + return (T)Convert.ChangeType(value, typeof(T)); + } + + static HashSet AsyncDocumentQuerySortOptions = new HashSet + { + "id", + "message_id", + "message_type", + "time_sent", + "status", + "modified", + "time_of_failure" + }; + + static HashSet RavenQueryableSortOptions = new HashSet + { + "processed_at", + "id", + "message_type", + "time_sent", + "critical_time", + "delivery_time", + "processing_time", + "status", + "message_id" + }; + + static string[] SplitChars = + { + "..." + }; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenQueryStatisticsExtensions.cs b/src/ServiceControl.Persistence.RavenDb5/RavenQueryStatisticsExtensions.cs new file mode 100644 index 0000000000..9be4c7b5f4 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenQueryStatisticsExtensions.cs @@ -0,0 +1,17 @@ +namespace ServiceControl.Persistence +{ + using Raven.Client; + using ServiceControl.Persistence.Infrastructure; + + static class RavenQueryStatisticsExtensions + { + public static QueryStatsInfo ToQueryStatsInfo(this RavenQueryStatistics stats) + { + return new QueryStatsInfo(stats.IndexEtag, stats.TotalResults, stats.IsStale); + } + public static QueryStatsInfo ToQueryStatsInfo(this Raven.Abstractions.Data.QueryResult queryResult) + { + return new QueryStatsInfo(queryResult.IndexEtag, queryResult.TotalResults, queryResult.IsStale); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenStartup.cs b/src/ServiceControl.Persistence.RavenDb5/RavenStartup.cs new file mode 100644 index 0000000000..7e4c22131c --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenStartup.cs @@ -0,0 +1,19 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System.Threading.Tasks; + using NServiceBus.Logging; + using Raven.Client; + using Raven.Client.Indexes; + + class RavenStartup + { + public async Task CreateIndexesAsync(IDocumentStore documentStore) + { + Logger.Info("Index creation started"); + await IndexCreation.CreateIndexesAsync(typeof(RavenBootstrapper).Assembly, documentStore); + Logger.Info("Index creation complete"); + } + + static readonly ILog Logger = LogManager.GetLogger(); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveBatch.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveBatch.cs new file mode 100644 index 0000000000..7dfc624019 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveBatch.cs @@ -0,0 +1,15 @@ +namespace ServiceControl.Recoverability +{ + using System.Collections.Generic; + + class ArchiveBatch //raven + { + public string Id { get; set; } + public List DocumentIds { get; set; } = new List(); + + public static string MakeId(string requestId, ArchiveType archiveType, int batchNumber) + { + return $"ArchiveOperations/{(int)archiveType}/{requestId}/{batchNumber}"; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveDocumentManager.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveDocumentManager.cs new file mode 100644 index 0000000000..b0ca31afc3 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveDocumentManager.cs @@ -0,0 +1,164 @@ +namespace ServiceControl.Recoverability +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading.Tasks; + using MessageFailures; + using NServiceBus.Logging; + using Raven.Abstractions.Commands; + using Raven.Abstractions.Data; + using Raven.Client; + + class ArchiveDocumentManager + { + public Task LoadArchiveOperation(IAsyncDocumentSession session, string groupId, ArchiveType archiveType) + { + return session.LoadAsync(ArchiveOperation.MakeId(groupId, archiveType)); + } + + public async Task CreateArchiveOperation(IAsyncDocumentSession session, string groupId, ArchiveType archiveType, int numberOfMessages, string groupName, int batchSize) + { + var operation = new ArchiveOperation + { + Id = ArchiveOperation.MakeId(groupId, archiveType), + RequestId = groupId, + ArchiveType = archiveType, + TotalNumberOfMessages = numberOfMessages, + NumberOfMessagesArchived = 0, + Started = DateTime.Now, + GroupName = groupName, + NumberOfBatches = (int)Math.Ceiling(numberOfMessages / (float)batchSize), + CurrentBatch = 0 + }; + + await session.StoreAsync(operation); + + var documentCount = 0; + var indexQuery = session.Query(new FailedMessages_ByGroup().IndexName); + + var docQuery = indexQuery + .Where(failure => failure.FailureGroupId == groupId) + .Where(failure => failure.Status == FailedMessageStatus.Unresolved) + .Select(document => document.Id); + + var docs = await StreamResults(session, docQuery); + + var batches = docs + .GroupBy(d => documentCount++ / batchSize); + + foreach (var batch in batches) + { + var archiveBatch = new ArchiveBatch + { + Id = ArchiveBatch.MakeId(groupId, archiveType, batch.Key), + DocumentIds = batch.ToList() + }; + + await session.StoreAsync(archiveBatch); + } + + return operation; + } + + async Task> StreamResults(IAsyncDocumentSession session, IQueryable query) + { + var results = new List(); + using (var enumerator = await session.Advanced.StreamAsync(query)) + { + while (await enumerator.MoveNextAsync()) + { + results.Add(enumerator.Current.Document); + } + } + + return results; + } + + public Task GetArchiveBatch(IAsyncDocumentSession session, string archiveOperationId, int batchNumber) + { + return session.LoadAsync($"{archiveOperationId}/{batchNumber}"); + } + + public async Task GetGroupDetails(IAsyncDocumentSession session, string groupId) + { + var group = await session.Query() + .FirstOrDefaultAsync(x => x.Id == groupId); + + return new GroupDetails + { + NumberOfMessagesInGroup = group?.Count ?? 0, + GroupName = group?.Title ?? "Undefined" + }; + } + + public async Task ArchiveMessageGroupBatch(IAsyncDocumentSession session, ArchiveBatch batch) + { + var patchCommands = batch?.DocumentIds.Select(documentId => new PatchCommandData { Key = documentId, Patches = patchRequest }); + + if (patchCommands != null) + { + await session.Advanced.DocumentStore.AsyncDatabaseCommands.BatchAsync(patchCommands); + await session.Advanced.DocumentStore.AsyncDatabaseCommands.DeleteAsync(batch.Id, null); + } + } + + public async Task WaitForIndexUpdateOfArchiveOperation(IDocumentStore store, string requestId, TimeSpan timeToWait) + { + using (var session = store.OpenAsyncSession()) + { + var indexQuery = session.Query(new FailedMessages_ByGroup().IndexName) + .Customize(x => x.WaitForNonStaleResultsAsOfNow(timeToWait)); + + var docQuery = indexQuery + .Where(failure => failure.FailureGroupId == requestId) + .Select(document => document.Id); + + try + { + await docQuery.AnyAsync(); + + return true; + } + catch + { + return false; + } + } + } + + public Task UpdateArchiveOperation(IAsyncDocumentSession session, ArchiveOperation archiveOperation) + { + return session.StoreAsync(archiveOperation); + } + + public async Task RemoveArchiveOperation(IDocumentStore store, ArchiveOperation archiveOperation) + { + using (var session = store.OpenAsyncSession()) + { + await session.Advanced.DocumentStore.AsyncDatabaseCommands.DeleteAsync(archiveOperation.Id, null); + await session.SaveChangesAsync(); + + logger.Info($"Removing ArchiveOperation {archiveOperation.Id} completed"); + } + } + + static PatchRequest[] patchRequest = + { + new PatchRequest + { + Type = PatchCommandType.Set, + Name = "Status", + Value = (int)FailedMessageStatus.Archived + } + }; + + public class GroupDetails + { + public string GroupName { get; set; } + public int NumberOfMessagesInGroup { get; set; } + } + + static ILog logger = LogManager.GetLogger(); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveOperation.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveOperation.cs new file mode 100644 index 0000000000..8fedc64233 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveOperation.cs @@ -0,0 +1,21 @@ +namespace ServiceControl.Recoverability +{ + using System; + + class ArchiveOperation // raven + { + public string Id { get; set; } + public string RequestId { get; set; } + public string GroupName { get; set; } + public ArchiveType ArchiveType { get; set; } + public int TotalNumberOfMessages { get; set; } + public int NumberOfMessagesArchived { get; set; } + public DateTime Started { get; set; } + public int NumberOfBatches { get; set; } + public int CurrentBatch { get; set; } + public static string MakeId(string requestId, ArchiveType archiveType) + { + return $"ArchiveOperations/{(int)archiveType}/{requestId}"; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveOperationExtensions.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveOperationExtensions.cs new file mode 100644 index 0000000000..8ec9667654 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveOperationExtensions.cs @@ -0,0 +1,39 @@ +namespace ServiceControl.Persistence.RavenDb.Recoverability +{ + using ServiceControl.Recoverability; + + static class ArchiveOperationExtensions + { + public static ArchiveOperation ToArchiveOperation(this InMemoryArchive a) + { + return new ArchiveOperation + { + ArchiveType = a.ArchiveType, + GroupName = a.GroupName, + Id = ArchiveOperation.MakeId(a.RequestId, a.ArchiveType), + NumberOfMessagesArchived = a.NumberOfMessagesArchived, + RequestId = a.RequestId, + Started = a.Started, + TotalNumberOfMessages = a.TotalNumberOfMessages, + NumberOfBatches = a.NumberOfBatches, + CurrentBatch = a.CurrentBatch + }; + } + + public static UnarchiveOperation ToUnarchiveOperation(this InMemoryUnarchive u) + { + return new UnarchiveOperation + { + ArchiveType = u.ArchiveType, + GroupName = u.GroupName, + Id = UnarchiveOperation.MakeId(u.RequestId, u.ArchiveType), + NumberOfMessagesUnarchived = u.NumberOfMessagesUnarchived, + RequestId = u.RequestId, + Started = u.Started, + TotalNumberOfMessages = u.TotalNumberOfMessages, + NumberOfBatches = u.NumberOfBatches, + CurrentBatch = u.CurrentBatch + }; + } + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchivingManager.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchivingManager.cs new file mode 100644 index 0000000000..2526acdbfb --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchivingManager.cs @@ -0,0 +1,112 @@ +namespace ServiceControl.Recoverability +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading.Tasks; + using Infrastructure.DomainEvents; + + class ArchivingManager + { + public ArchivingManager(IDomainEvents domainEvents, OperationsManager operationsManager) + { + this.domainEvents = domainEvents; + this.operationsManager = operationsManager; + } + + public bool IsArchiveInProgressFor(string requestId) + { + return operationsManager.ArchiveOperations.Keys.Any(key => key.EndsWith($"/{requestId}")); + } + + internal IEnumerable GetArchivalOperations() + { + return operationsManager.ArchiveOperations.Values; + } + + public bool IsOperationInProgressFor(string requestId, ArchiveType archiveType) + { + return operationsManager.IsOperationInProgressFor(requestId, archiveType); + } + + InMemoryArchive GetOrCreate(ArchiveType archiveType, string requestId) + { + if (!operationsManager.ArchiveOperations.TryGetValue(InMemoryArchive.MakeId(requestId, archiveType), out var summary)) + { + summary = new InMemoryArchive(requestId, archiveType, domainEvents); + operationsManager.ArchiveOperations[InMemoryArchive.MakeId(requestId, archiveType)] = summary; + } + + return summary; + } + + public Task StartArchiving(ArchiveOperation archiveOperation) + { + var summary = GetOrCreate(archiveOperation.ArchiveType, archiveOperation.RequestId); + + summary.TotalNumberOfMessages = archiveOperation.TotalNumberOfMessages; + summary.NumberOfMessagesArchived = archiveOperation.NumberOfMessagesArchived; + summary.Started = archiveOperation.Started; + summary.GroupName = archiveOperation.GroupName; + summary.NumberOfBatches = archiveOperation.NumberOfBatches; + summary.CurrentBatch = archiveOperation.CurrentBatch; + + return summary.Start(); + } + + public Task StartArchiving(string requestId, ArchiveType archiveType) + { + var summary = GetOrCreate(archiveType, requestId); + + summary.TotalNumberOfMessages = 0; + summary.NumberOfMessagesArchived = 0; + summary.Started = DateTime.Now; + summary.GroupName = "Undefined"; + summary.NumberOfBatches = 0; + summary.CurrentBatch = 0; + + return summary.Start(); + } + + public InMemoryArchive GetStatusForArchiveOperation(string requestId, ArchiveType archiveType) + { + operationsManager.ArchiveOperations.TryGetValue(InMemoryArchive.MakeId(requestId, archiveType), out var summary); + + return summary; + } + + public Task BatchArchived(string requestId, ArchiveType archiveType, int numberOfMessagesArchivedInBatch) + { + var summary = GetOrCreate(archiveType, requestId); + + return summary.BatchArchived(numberOfMessagesArchivedInBatch); + } + + public Task ArchiveOperationFinalizing(string requestId, ArchiveType archiveType) + { + var summary = GetOrCreate(archiveType, requestId); + return summary.FinalizeArchive(); + } + + public Task ArchiveOperationCompleted(string requestId, ArchiveType archiveType) + { + var summary = GetOrCreate(archiveType, requestId); + return summary.Complete(); + } + + public void DismissArchiveOperation(string requestId, ArchiveType archiveType) + { + RemoveArchiveOperation(requestId, archiveType); + } + + void RemoveArchiveOperation(string requestId, ArchiveType archiveType) + { + operationsManager.ArchiveOperations.Remove(InMemoryArchive.MakeId(requestId, archiveType)); + } + + IDomainEvents domainEvents; + + OperationsManager operationsManager; + + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/MessageArchiver.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/MessageArchiver.cs new file mode 100644 index 0000000000..085c64e7c5 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/MessageArchiver.cs @@ -0,0 +1,242 @@ +namespace ServiceControl.Persistence.RavenDb.Recoverability +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading.Tasks; + using NServiceBus.Logging; + using Raven.Client; + using ServiceControl.Infrastructure.DomainEvents; + using ServiceControl.Persistence.Recoverability; + using ServiceControl.Recoverability; + + class MessageArchiver : IArchiveMessages + { + public MessageArchiver(IDocumentStore store, OperationsManager operationsManager, IDomainEvents domainEvents) + { + this.store = store; + this.domainEvents = domainEvents; + this.operationsManager = operationsManager; + + archiveDocumentManager = new ArchiveDocumentManager(); + archivingManager = new ArchivingManager(domainEvents, operationsManager); + + unarchiveDocumentManager = new UnarchiveDocumentManager(); + unarchivingManager = new UnarchivingManager(domainEvents, operationsManager); + } + + public async Task ArchiveAllInGroup(string groupId) + { + logger.Info($"Archiving of {groupId} started"); + ArchiveOperation archiveOperation; + + using (var session = store.OpenAsyncSession()) + { + session.Advanced.UseOptimisticConcurrency = true; // Ensure 2 messages don't split the same operation into batches at once + + archiveOperation = await archiveDocumentManager.LoadArchiveOperation(session, groupId, ArchiveType.FailureGroup); + + if (archiveOperation == null) + { + var groupDetails = await archiveDocumentManager.GetGroupDetails(session, groupId); + if (groupDetails.NumberOfMessagesInGroup == 0) + { + logger.Warn($"No messages to archive in group {groupId}"); + return; + } + + logger.Info($"Splitting group {groupId} into batches"); + archiveOperation = await archiveDocumentManager.CreateArchiveOperation(session, groupId, ArchiveType.FailureGroup, groupDetails.NumberOfMessagesInGroup, groupDetails.GroupName, batchSize); + await session.SaveChangesAsync(); + + logger.Info($"Group {groupId} has been split into {archiveOperation.NumberOfBatches} batches"); + } + } + + await archivingManager.StartArchiving(archiveOperation); + + while (archiveOperation.CurrentBatch < archiveOperation.NumberOfBatches) + { + using (var batchSession = store.OpenAsyncSession()) + { + var nextBatch = await archiveDocumentManager.GetArchiveBatch(batchSession, archiveOperation.Id, archiveOperation.CurrentBatch); + if (nextBatch == null) + { + // We're only here in the case where Raven indexes are stale + logger.Warn($"Attempting to archive a batch ({archiveOperation.Id}/{archiveOperation.CurrentBatch}) which appears to already have been archived."); + } + else + { + logger.Info($"Archiving {nextBatch.DocumentIds.Count} messages from group {groupId} starting"); + } + + await archiveDocumentManager.ArchiveMessageGroupBatch(batchSession, nextBatch); + + await archivingManager.BatchArchived(archiveOperation.RequestId, archiveOperation.ArchiveType, nextBatch?.DocumentIds.Count ?? 0); + + archiveOperation = archivingManager.GetStatusForArchiveOperation(archiveOperation.RequestId, archiveOperation.ArchiveType).ToArchiveOperation(); + + await archiveDocumentManager.UpdateArchiveOperation(batchSession, archiveOperation); + + await batchSession.SaveChangesAsync(); + + if (nextBatch != null) + { + await domainEvents.Raise(new FailedMessageGroupBatchArchived + { + // Remove `FailedMessages/` prefix and publish pure GUIDs without Raven collection name + FailedMessagesIds = nextBatch.DocumentIds.Select(id => id.Replace("FailedMessages/", "")).ToArray() + }); + } + + if (nextBatch != null) + { + logger.Info($"Archiving of {nextBatch.DocumentIds.Count} messages from group {groupId} completed"); + } + } + } + + logger.Info($"Archiving of group {groupId} is complete. Waiting for index updates."); + await archivingManager.ArchiveOperationFinalizing(archiveOperation.RequestId, archiveOperation.ArchiveType); + if (!await archiveDocumentManager.WaitForIndexUpdateOfArchiveOperation(store, archiveOperation.RequestId, TimeSpan.FromMinutes(5)) + ) + { + logger.Warn($"Archiving group {groupId} completed but index not updated."); + } + + await archivingManager.ArchiveOperationCompleted(archiveOperation.RequestId, archiveOperation.ArchiveType); + await archiveDocumentManager.RemoveArchiveOperation(store, archiveOperation); + + await domainEvents.Raise(new FailedMessageGroupArchived + { + GroupId = groupId, + GroupName = archiveOperation.GroupName, + MessagesCount = archiveOperation.TotalNumberOfMessages, + }); + + logger.Info($"Archiving of group {groupId} completed"); + } + + public async Task UnarchiveAllInGroup(string groupId) + { + logger.Info($"Unarchiving of {groupId} started"); + UnarchiveOperation unarchiveOperation; + + using (var session = store.OpenAsyncSession()) + { + session.Advanced.UseOptimisticConcurrency = true; // Ensure 2 messages don't split the same operation into batches at once + + unarchiveOperation = await unarchiveDocumentManager.LoadUnarchiveOperation(session, groupId, ArchiveType.FailureGroup); + + if (unarchiveOperation == null) + { + var groupDetails = await unarchiveDocumentManager.GetGroupDetails(session, groupId); + if (groupDetails.NumberOfMessagesInGroup == 0) + { + logger.Warn($"No messages to unarchive in group {groupId}"); + + return; + } + + logger.Info($"Splitting group {groupId} into batches"); + unarchiveOperation = await unarchiveDocumentManager.CreateUnarchiveOperation(session, groupId, ArchiveType.FailureGroup, groupDetails.NumberOfMessagesInGroup, groupDetails.GroupName, batchSize); + await session.SaveChangesAsync(); + + logger.Info($"Group {groupId} has been split into {unarchiveOperation.NumberOfBatches} batches"); + } + } + + await unarchivingManager.StartUnarchiving(unarchiveOperation); + + while (unarchiveOperation.CurrentBatch < unarchiveOperation.NumberOfBatches) + { + using (var batchSession = store.OpenAsyncSession()) + { + var nextBatch = await unarchiveDocumentManager.GetUnarchiveBatch(batchSession, unarchiveOperation.Id, unarchiveOperation.CurrentBatch); + if (nextBatch == null) + { + // We're only here in the case where Raven indexes are stale + logger.Warn($"Attempting to unarchive a batch ({unarchiveOperation.Id}/{unarchiveOperation.CurrentBatch}) which appears to already have been archived."); + } + else + { + logger.Info($"Unarchiving {nextBatch.DocumentIds.Count} messages from group {groupId} starting"); + } + + await unarchiveDocumentManager.UnarchiveMessageGroupBatch(batchSession, nextBatch); + + await unarchivingManager.BatchUnarchived(unarchiveOperation.RequestId, unarchiveOperation.ArchiveType, nextBatch?.DocumentIds.Count ?? 0); + + unarchiveOperation = unarchivingManager.GetStatusForUnarchiveOperation(unarchiveOperation.RequestId, unarchiveOperation.ArchiveType).ToUnarchiveOperation(); + + await unarchiveDocumentManager.UpdateUnarchiveOperation(batchSession, unarchiveOperation); + + await batchSession.SaveChangesAsync(); + + if (nextBatch != null) + { + await domainEvents.Raise(new FailedMessageGroupBatchUnarchived + { + // Remove `FailedMessages/` prefix and publish pure GUIDs without Raven collection name + FailedMessagesIds = nextBatch.DocumentIds.Select(id => id.Replace("FailedMessages/", "")).ToArray() + }); + } + + if (nextBatch != null) + { + logger.Info($"Unarchiving of {nextBatch.DocumentIds.Count} messages from group {groupId} completed"); + } + } + } + + logger.Info($"Unarchiving of group {groupId} is complete. Waiting for index updates."); + await unarchivingManager.UnarchiveOperationFinalizing(unarchiveOperation.RequestId, unarchiveOperation.ArchiveType); + if (!await unarchiveDocumentManager.WaitForIndexUpdateOfUnarchiveOperation(store, unarchiveOperation.RequestId, TimeSpan.FromMinutes(5)) + ) + { + logger.Warn($"Unarchiving group {groupId} completed but index not updated."); + } + + logger.Info($"Unarchiving of group {groupId} completed"); + await unarchivingManager.UnarchiveOperationCompleted(unarchiveOperation.RequestId, unarchiveOperation.ArchiveType); + await unarchiveDocumentManager.RemoveUnarchiveOperation(store, unarchiveOperation); + + await domainEvents.Raise(new FailedMessageGroupUnarchived + { + GroupId = groupId, + GroupName = unarchiveOperation.GroupName, + MessagesCount = unarchiveOperation.TotalNumberOfMessages, + }); + } + + public bool IsOperationInProgressFor(string groupId, ArchiveType archiveType) + { + return operationsManager.IsOperationInProgressFor(groupId, archiveType); + } + + public bool IsArchiveInProgressFor(string groupId) + => archivingManager.IsArchiveInProgressFor(groupId); + + public void DismissArchiveOperation(string groupId, ArchiveType archiveType) + => archivingManager.DismissArchiveOperation(groupId, archiveType); + + public Task StartArchiving(string groupId, ArchiveType archiveType) + => archivingManager.StartArchiving(groupId, archiveType); + + public Task StartUnarchiving(string groupId, ArchiveType archiveType) + => unarchivingManager.StartUnarchiving(groupId, archiveType); + + public IEnumerable GetArchivalOperations() + => archivingManager.GetArchivalOperations(); + + readonly IDocumentStore store; + readonly OperationsManager operationsManager; + readonly IDomainEvents domainEvents; + readonly ArchiveDocumentManager archiveDocumentManager; + readonly ArchivingManager archivingManager; + readonly UnarchiveDocumentManager unarchiveDocumentManager; + readonly UnarchivingManager unarchivingManager; + static readonly ILog logger = LogManager.GetLogger(); + const int batchSize = 1000; + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveBatch.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveBatch.cs new file mode 100644 index 0000000000..d8ee487d19 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveBatch.cs @@ -0,0 +1,15 @@ +namespace ServiceControl.Recoverability +{ + using System.Collections.Generic; + + class UnarchiveBatch //raven + { + public string Id { get; set; } + public List DocumentIds { get; set; } = new List(); + + public static string MakeId(string requestId, ArchiveType archiveType, int batchNumber) + { + return $"UnarchiveOperations/{(int)archiveType}/{requestId}/{batchNumber}"; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveDocumentManager.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveDocumentManager.cs new file mode 100644 index 0000000000..2ee549bf30 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveDocumentManager.cs @@ -0,0 +1,159 @@ +namespace ServiceControl.Recoverability +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading.Tasks; + using MessageFailures; + using Raven.Abstractions.Commands; + using Raven.Abstractions.Data; + using Raven.Client; + + class UnarchiveDocumentManager + { + public Task LoadUnarchiveOperation(IAsyncDocumentSession session, string groupId, ArchiveType archiveType) + { + return session.LoadAsync(UnarchiveOperation.MakeId(groupId, archiveType)); + } + + public async Task CreateUnarchiveOperation(IAsyncDocumentSession session, string groupId, ArchiveType archiveType, int numberOfMessages, string groupName, int batchSize) + { + var operation = new UnarchiveOperation + { + Id = UnarchiveOperation.MakeId(groupId, archiveType), + RequestId = groupId, + ArchiveType = archiveType, + TotalNumberOfMessages = numberOfMessages, + NumberOfMessagesUnarchived = 0, + Started = DateTime.Now, + GroupName = groupName, + NumberOfBatches = (int)Math.Ceiling(numberOfMessages / (float)batchSize), + CurrentBatch = 0 + }; + + await session.StoreAsync(operation); + + var documentCount = 0; + var indexQuery = session.Query(new FailedMessages_ByGroup().IndexName); + + var docQuery = indexQuery + .Where(failure => failure.FailureGroupId == groupId) + .Where(failure => failure.Status == FailedMessageStatus.Archived) + .Select(document => document.Id); + + var docs = await StreamResults(session, docQuery); + + var batches = docs + .GroupBy(d => documentCount++ / batchSize); + + foreach (var batch in batches) + { + var unarchiveBatch = new UnarchiveBatch + { + Id = UnarchiveBatch.MakeId(groupId, archiveType, batch.Key), + DocumentIds = batch.ToList() + }; + + await session.StoreAsync(unarchiveBatch); + } + + return operation; + } + + async Task> StreamResults(IAsyncDocumentSession session, IQueryable query) + { + var results = new List(); + using (var enumerator = await session.Advanced.StreamAsync(query)) + { + while (await enumerator.MoveNextAsync()) + { + results.Add(enumerator.Current.Document); + } + } + + return results; + } + + public Task GetUnarchiveBatch(IAsyncDocumentSession session, string unUnarchiveOperationId, int batchNumber) + { + return session.LoadAsync($"{unUnarchiveOperationId}/{batchNumber}"); + } + + public async Task GetGroupDetails(IAsyncDocumentSession session, string groupId) + { + var group = await session.Query() + .FirstOrDefaultAsync(x => x.Id == groupId); + + return new GroupDetails + { + NumberOfMessagesInGroup = group?.Count ?? 0, + GroupName = group?.Title ?? "Undefined" + }; + } + + public async Task UnarchiveMessageGroupBatch(IAsyncDocumentSession session, UnarchiveBatch batch) + { + var patchCommands = batch?.DocumentIds.Select(documentId => new PatchCommandData { Key = documentId, Patches = patchRequest }); + + if (patchCommands != null) + { + await session.Advanced.DocumentStore.AsyncDatabaseCommands.BatchAsync(patchCommands); + await session.Advanced.DocumentStore.AsyncDatabaseCommands.DeleteAsync(batch.Id, null); + } + } + + public async Task WaitForIndexUpdateOfUnarchiveOperation(IDocumentStore store, string requestId, TimeSpan timeToWait) + { + using (var session = store.OpenAsyncSession()) + { + var indexQuery = session.Query(new FailedMessages_ByGroup().IndexName) + .Customize(x => x.WaitForNonStaleResultsAsOfNow(timeToWait)); + + var docQuery = indexQuery + .Where(failure => failure.FailureGroupId == requestId) + .Select(document => document.Id); + + try + { + await docQuery.AnyAsync(); + + return true; + } + catch + { + return false; + } + } + } + + public Task UpdateUnarchiveOperation(IAsyncDocumentSession session, UnarchiveOperation unarchiveOperation) + { + return session.StoreAsync(unarchiveOperation); + } + + public async Task RemoveUnarchiveOperation(IDocumentStore store, UnarchiveOperation unarchiveOperation) + { + using (var session = store.OpenAsyncSession()) + { + await session.Advanced.DocumentStore.AsyncDatabaseCommands.DeleteAsync(unarchiveOperation.Id, null); + await session.SaveChangesAsync(); + } + } + + static PatchRequest[] patchRequest = + { + new PatchRequest + { + Type = PatchCommandType.Set, + Name = "Status", + Value = (int)FailedMessageStatus.Unresolved + } + }; + + public class GroupDetails + { + public string GroupName { get; set; } + public int NumberOfMessagesInGroup { get; set; } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveOperation.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveOperation.cs new file mode 100644 index 0000000000..4227ce05f5 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveOperation.cs @@ -0,0 +1,21 @@ +namespace ServiceControl.Recoverability +{ + using System; + + class UnarchiveOperation // raven + { + public string Id { get; set; } + public string RequestId { get; set; } + public string GroupName { get; set; } + public ArchiveType ArchiveType { get; set; } + public int TotalNumberOfMessages { get; set; } + public int NumberOfMessagesUnarchived { get; set; } + public DateTime Started { get; set; } + public int NumberOfBatches { get; set; } + public int CurrentBatch { get; set; } + public static string MakeId(string requestId, ArchiveType archiveType) + { + return $"UnarchiveOperations/{(int)archiveType}/{requestId}"; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchivingManager.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchivingManager.cs new file mode 100644 index 0000000000..0644b29dc1 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchivingManager.cs @@ -0,0 +1,110 @@ +namespace ServiceControl.Recoverability +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading.Tasks; + using Infrastructure.DomainEvents; + + class UnarchivingManager + { + public UnarchivingManager(IDomainEvents domainEvents, OperationsManager operationsManager) + { + this.domainEvents = domainEvents; + this.operationsManager = operationsManager; + } + + public bool IsUnarchiveInProgressFor(string requestId) + { + return operationsManager.UnarchiveOperations.Keys.Any(key => key.EndsWith($"/{requestId}")); + } + + internal IEnumerable GetUnarchivalOperations() + { + return operationsManager.UnarchiveOperations.Values; + } + + public bool IsOperationInProgressFor(string requestId, ArchiveType archiveType) + { + return operationsManager.IsOperationInProgressFor(requestId, archiveType); + } + + InMemoryUnarchive GetOrCreate(ArchiveType archiveType, string requestId) + { + if (!operationsManager.UnarchiveOperations.TryGetValue(InMemoryUnarchive.MakeId(requestId, archiveType), out var summary)) + { + summary = new InMemoryUnarchive(requestId, archiveType, domainEvents); + operationsManager.UnarchiveOperations[InMemoryUnarchive.MakeId(requestId, archiveType)] = summary; + } + + return summary; + } + + public Task StartUnarchiving(UnarchiveOperation archiveOperation) + { + var summary = GetOrCreate(archiveOperation.ArchiveType, archiveOperation.RequestId); + + summary.TotalNumberOfMessages = archiveOperation.TotalNumberOfMessages; + summary.NumberOfMessagesUnarchived = archiveOperation.NumberOfMessagesUnarchived; + summary.Started = archiveOperation.Started; + summary.GroupName = archiveOperation.GroupName; + summary.NumberOfBatches = archiveOperation.NumberOfBatches; + summary.CurrentBatch = archiveOperation.CurrentBatch; + + return summary.Start(); + } + + public Task StartUnarchiving(string requestId, ArchiveType archiveType) + { + var summary = GetOrCreate(archiveType, requestId); + + summary.TotalNumberOfMessages = 0; + summary.NumberOfMessagesUnarchived = 0; + summary.Started = DateTime.Now; + summary.GroupName = "Undefined"; + summary.NumberOfBatches = 0; + summary.CurrentBatch = 0; + + return summary.Start(); + } + + public InMemoryUnarchive GetStatusForUnarchiveOperation(string requestId, ArchiveType archiveType) + { + operationsManager.UnarchiveOperations.TryGetValue(InMemoryUnarchive.MakeId(requestId, archiveType), out var summary); + + return summary; + } + + public Task BatchUnarchived(string requestId, ArchiveType archiveType, int numberOfMessagesArchivedInBatch) + { + var summary = GetOrCreate(archiveType, requestId); + + return summary.BatchUnarchived(numberOfMessagesArchivedInBatch); + } + + public Task UnarchiveOperationFinalizing(string requestId, ArchiveType archiveType) + { + var summary = GetOrCreate(archiveType, requestId); + return summary.FinalizeUnarchive(); + } + + public Task UnarchiveOperationCompleted(string requestId, ArchiveType archiveType) + { + var summary = GetOrCreate(archiveType, requestId); + return summary.Complete(); + } + + public void DismissArchiveOperation(string requestId, ArchiveType archiveType) + { + RemoveUnarchiveOperation(requestId, archiveType); + } + + void RemoveUnarchiveOperation(string requestId, ArchiveType archiveType) + { + operationsManager.UnarchiveOperations.Remove(InMemoryUnarchive.MakeId(requestId, archiveType)); + } + + IDomainEvents domainEvents; + OperationsManager operationsManager; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/GroupsDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/GroupsDataStore.cs new file mode 100644 index 0000000000..2df22233aa --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/GroupsDataStore.cs @@ -0,0 +1,60 @@ +namespace ServiceControl.Persistence.RavenDb.Recoverability +{ + using System.Collections.Generic; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Raven.Client; + using Raven.Client.Linq; + using ServiceControl.MessageFailures; + using ServiceControl.Recoverability; + + class GroupsDataStore : IGroupsDataStore + { + readonly IDocumentStore store; + + public GroupsDataStore(IDocumentStore store) + { + this.store = store; + } + + public async Task> GetFailureGroupsByClassifier(string classifier, string classifierFilter) + { + using (var session = store.OpenAsyncSession()) + { + var query = Queryable.Where(session.Query(), v => v.Type == classifier); + + if (!string.IsNullOrWhiteSpace(classifierFilter)) + { + query = query.Where(v => v.Title == classifierFilter); + } + + var groups = await query.OrderByDescending(x => x.Last) + .Take(200) + .ToListAsync(); + + var commentIds = groups.Select(x => GroupComment.MakeId(x.Id)).ToArray(); + var comments = await session.Query().Where(x => x.Id.In(commentIds)) + .ToListAsync(CancellationToken.None); + + foreach (var group in groups) + { + group.Comment = comments.FirstOrDefault(x => x.Id == GroupComment.MakeId(group.Id))?.Comment; + } + + return groups; + } + } + + public async Task GetCurrentForwardingBatch() + { + using (var session = store.OpenAsyncSession()) + { + var nowForwarding = await session.Include(r => r.RetryBatchId) + .LoadAsync(RetryBatchNowForwarding.Id); + + return nowForwarding == null ? null : await session.LoadAsync(nowForwarding.RetryBatchId); + } + } + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/RetryHistoryDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/RetryHistoryDataStore.cs new file mode 100644 index 0000000000..e25305d18e --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/RetryHistoryDataStore.cs @@ -0,0 +1,87 @@ +namespace ServiceControl.Persistence.RavenDb.Recoverability +{ + using System; + using System.Threading.Tasks; + using Raven.Client; + using ServiceControl.Recoverability; + + class RetryHistoryDataStore : IRetryHistoryDataStore + { + public RetryHistoryDataStore(IDocumentStore documentStore) + { + this.documentStore = documentStore; + } + + public async Task GetRetryHistory() + { + using (var session = documentStore.OpenAsyncSession()) + { + var id = RetryHistory.MakeId(); + var retryHistory = await session.LoadAsync(id); + + retryHistory = retryHistory ?? RetryHistory.CreateNew(); + + return retryHistory; + } + } + + public async Task RecordRetryOperationCompleted(string requestId, RetryType retryType, DateTime startTime, DateTime completionTime, + string originator, string classifier, bool messageFailed, int numberOfMessagesProcessed, DateTime lastProcessed, int retryHistoryDepth) + { + using (var session = documentStore.OpenAsyncSession()) + { + var retryHistory = await session.LoadAsync(RetryHistory.MakeId()) ?? + RetryHistory.CreateNew(); + + retryHistory.AddToUnacknowledged(new UnacknowledgedRetryOperation + { + RequestId = requestId, + RetryType = retryType, + StartTime = startTime, + CompletionTime = completionTime, + Originator = originator, + Classifier = classifier, + Failed = messageFailed, + NumberOfMessagesProcessed = numberOfMessagesProcessed, + Last = lastProcessed + }); + + retryHistory.AddToHistory(new HistoricRetryOperation + { + RequestId = requestId, + RetryType = retryType, + StartTime = startTime, + CompletionTime = completionTime, + Originator = originator, + Failed = messageFailed, + NumberOfMessagesProcessed = numberOfMessagesProcessed + }, retryHistoryDepth); + + await session.StoreAsync(retryHistory); + await session.SaveChangesAsync(); + } + } + + public async Task AcknowledgeRetryGroup(string groupId) + { + using (var session = documentStore.OpenAsyncSession()) + { + var retryHistory = await session.LoadAsync(RetryHistory.MakeId()); + if (retryHistory != null) + { + if (retryHistory.Acknowledge(groupId, RetryType.FailureGroup)) + { + await session.StoreAsync(retryHistory); + await session.SaveChangesAsync(); + + return true; + } + } + } + + return false; + } + + readonly IDocumentStore documentStore; + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/RepeatedFailuresOverTimeCircuitBreaker.cs b/src/ServiceControl.Persistence.RavenDb5/RepeatedFailuresOverTimeCircuitBreaker.cs new file mode 100644 index 0000000000..09cb5b0700 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RepeatedFailuresOverTimeCircuitBreaker.cs @@ -0,0 +1,74 @@ +namespace NServiceBus +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using Logging; + + class RepeatedFailuresOverTimeCircuitBreaker : IDisposable + { + public RepeatedFailuresOverTimeCircuitBreaker(string name, TimeSpan timeToWaitBeforeTriggering, Action triggerAction, TimeSpan delayAfterFailure) + { + this.delayAfterFailure = delayAfterFailure; + this.name = name; + this.triggerAction = triggerAction; + this.timeToWaitBeforeTriggering = timeToWaitBeforeTriggering; + + timer = new Timer(CircuitBreakerTriggered); + } + + public void Dispose() + { + //Injected + } + + public void Success() + { + var oldValue = Interlocked.Exchange(ref failureCount, 0); + + if (oldValue == 0) + { + return; + } + + timer.Change(System.Threading.Timeout.Infinite, System.Threading.Timeout.Infinite); + Logger.InfoFormat("The circuit breaker for {0} is now disarmed", name); + } + + public Task Failure(Exception exception) + { + lastException = exception; + var newValue = Interlocked.Increment(ref failureCount); + + if (newValue == 1) + { + timer.Change(timeToWaitBeforeTriggering, NoPeriodicTriggering); + Logger.WarnFormat("The circuit breaker for {0} is now in the armed state", name); + } + + return Task.Delay(delayAfterFailure); + } + + void CircuitBreakerTriggered(object state) + { + if (Interlocked.Read(ref failureCount) > 0) + { + Logger.WarnFormat("The circuit breaker for {0} will now be triggered", name); + triggerAction(lastException); + } + } + + readonly TimeSpan delayAfterFailure; + + long failureCount; + Exception lastException; + + string name; + Timer timer; + TimeSpan timeToWaitBeforeTriggering; + Action triggerAction; + + static TimeSpan NoPeriodicTriggering = TimeSpan.FromMilliseconds(-1); + static ILog Logger = LogManager.GetLogger(); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RetryBatchesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RetryBatchesDataStore.cs new file mode 100644 index 0000000000..c3c125de4f --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RetryBatchesDataStore.cs @@ -0,0 +1,96 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System; + using System.Collections.Generic; + using System.Threading.Tasks; + using MessageFailures; + using NServiceBus.Logging; + using Raven.Abstractions.Commands; + using Raven.Abstractions.Data; + using Raven.Abstractions.Exceptions; + using Raven.Client; + using ServiceControl.Recoverability; + + class RetryBatchesDataStore : IRetryBatchesDataStore + { + readonly IDocumentStore documentStore; + + static ILog Log = LogManager.GetLogger(typeof(RetryBatchesDataStore)); + + public RetryBatchesDataStore(IDocumentStore documentStore) + { + this.documentStore = documentStore; + } + + public Task CreateRetryBatchesManager() + { + var session = documentStore.OpenAsyncSession(); + var manager = new RetryBatchesManager(session); + + return Task.FromResult(manager); + } + + public async Task RecordFailedStagingAttempt(IReadOnlyCollection messages, + IReadOnlyDictionary failedMessageRetriesById, Exception e, + int maxStagingAttempts, string stagingId) + { + var commands = new ICommandData[messages.Count]; + var commandIndex = 0; + foreach (var failedMessage in messages) + { + var failedMessageRetry = failedMessageRetriesById[failedMessage.Id]; + + Log.Warn($"Attempt {1} of {maxStagingAttempts} to stage a retry message {failedMessage.UniqueMessageId} failed", e); + + commands[commandIndex] = new PatchCommandData + { + Patches = new[] + { + new PatchRequest + { + Type = PatchCommandType.Set, Name = "StageAttempts", Value = 1 + } + }, + Key = failedMessageRetry.Id + }; + + commandIndex++; + } + + + try + { + await documentStore.AsyncDatabaseCommands.BatchAsync(commands); + } + catch (ConcurrencyException) + { + Log.DebugFormat( + "Ignoring concurrency exception while incrementing staging attempt count for {0}", + stagingId); + } + } + + public async Task IncrementAttemptCounter(FailedMessageRetry message) + { + try + { + await documentStore.AsyncDatabaseCommands.PatchAsync(message.Id, + new[] + { + new PatchRequest + { + Type = PatchCommandType.Set, + Name = "StageAttempts", + Value = message.StageAttempts + 1 + } + }); + } + catch (ConcurrencyException) + { + Log.DebugFormat("Ignoring concurrency exception while incrementing staging attempt count for {0}", message.FailedMessageId); + } + } + + public Task DeleteFailedMessageRetry(string uniqueMessageId) => documentStore.AsyncDatabaseCommands.DeleteAsync(FailedMessageRetry.MakeDocumentId(uniqueMessageId), null); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RetryBatchesManager.cs b/src/ServiceControl.Persistence.RavenDb5/RetryBatchesManager.cs new file mode 100644 index 0000000000..15edba340e --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RetryBatchesManager.cs @@ -0,0 +1,58 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using MessageFailures; + using Persistence.MessageRedirects; + using Raven.Client; + using ServiceControl.Recoverability; + + class RetryBatchesManager : AbstractSessionManager, IRetryBatchesManager + { + public RetryBatchesManager(IAsyncDocumentSession session) : base(session) + { + } + + public void Delete(RetryBatch retryBatch) => Session.Delete(retryBatch); + + public void Delete(RetryBatchNowForwarding forwardingBatch) => Session.Delete(forwardingBatch); + + public Task GetFailedMessageRetries(IList stagingBatchFailureRetries) => Session.LoadAsync(stagingBatchFailureRetries); + + public void Evict(FailedMessageRetry failedMessageRetry) => Session.Advanced.Evict(failedMessageRetry); + + public Task GetFailedMessages(Dictionary.KeyCollection keys) => Session.LoadAsync(keys); + + public async Task GetRetryBatchNowForwarding() => + await Session.Include(r => r.RetryBatchId) + .LoadAsync(RetryBatchNowForwarding.Id); + + public async Task GetRetryBatch(string retryBatchId, CancellationToken cancellationToken) => + await Session.LoadAsync(retryBatchId, cancellationToken); + + public async Task GetStagingBatch() => + await Session.Query() + .Customize(q => q.Include(b => b.FailureRetries)) + .FirstOrDefaultAsync(b => b.Status == RetryBatchStatus.Staging); + + public async Task Store(RetryBatchNowForwarding retryBatchNowForwarding) => + await Session.StoreAsync(retryBatchNowForwarding, RetryBatchNowForwarding.Id); + + public async Task GetOrCreateMessageRedirectsCollection() + { + var redirects = await Session.LoadAsync(MessageRedirectsCollection.DefaultId); + + if (redirects != null) + { + redirects.ETag = Session.Advanced.GetEtagFor(redirects); + redirects.LastModified = Session.Advanced.GetMetadataFor(redirects).Value("Last-Modified"); + + return redirects; + } + + return new MessageRedirectsCollection(); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs new file mode 100644 index 0000000000..582dde65a6 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs @@ -0,0 +1,303 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Linq.Expressions; + using System.Threading.Tasks; + using MessageFailures; + using NServiceBus.Logging; + using Persistence.Infrastructure; + using Raven.Abstractions.Commands; + using Raven.Abstractions.Data; + using Raven.Abstractions.Exceptions; + using Raven.Client; + using Raven.Client.Indexes; + using Raven.Client.Linq; + using Raven.Json.Linq; + using ServiceControl.MessageFailures.Api; + using ServiceControl.Recoverability; + + class RetryDocumentDataStore : IRetryDocumentDataStore + { + readonly IDocumentStore store; + + public RetryDocumentDataStore(IDocumentStore store) + { + this.store = store; + } + + public async Task StageRetryByUniqueMessageIds(string batchDocumentId, string requestId, RetryType retryType, string[] messageIds, + DateTime startTime, + DateTime? last = null, string originator = null, string batchName = null, string classifier = null) + { + var commands = new ICommandData[messageIds.Length]; + + for (var i = 0; i < messageIds.Length; i++) + { + commands[i] = CreateFailedMessageRetryDocument(batchDocumentId, messageIds[i]); + } + + await store.AsyncDatabaseCommands.BatchAsync(commands); + } + + public async Task MoveBatchToStaging(string batchDocumentId) + { + try + { + await store.AsyncDatabaseCommands.PatchAsync(batchDocumentId, + new[] + { + new PatchRequest + { + Type = PatchCommandType.Set, + Name = "Status", + Value = (int)RetryBatchStatus.Staging, + PrevVal = (int)RetryBatchStatus.MarkingDocuments + } + }); + } + catch (ConcurrencyException) + { + log.DebugFormat("Ignoring concurrency exception while moving batch to staging {0}", batchDocumentId); + } + } + + public async Task CreateBatchDocument(string retrySessionId, string requestId, RetryType retryType, string[] failedMessageRetryIds, + string originator, + DateTime startTime, DateTime? last = null, string batchName = null, string classifier = null) + { + var batchDocumentId = RetryBatch.MakeDocumentId(Guid.NewGuid().ToString()); + using (var session = store.OpenAsyncSession()) + { + await session.StoreAsync(new RetryBatch + { + Id = batchDocumentId, + Context = batchName, + RequestId = requestId, + RetryType = retryType, + Originator = originator, + Classifier = classifier, + StartTime = startTime, + Last = last, + InitialBatchSize = failedMessageRetryIds.Length, + RetrySessionId = retrySessionId, + FailureRetries = failedMessageRetryIds, + Status = RetryBatchStatus.MarkingDocuments + }); + await session.SaveChangesAsync(); + } + + return batchDocumentId; + } + + public async Task>> QueryOrphanedBatches(string retrySessionId, DateTime cutoff) + { + using (var session = store.OpenAsyncSession()) + { + var orphanedBatches = await session.Query() + .Customize(c => c.BeforeQueryExecution(index => index.Cutoff = cutoff)) + .Where(b => b.Status == RetryBatchStatus.MarkingDocuments && b.RetrySessionId != retrySessionId) + .Statistics(out var stats) + .ToListAsync(); + + return orphanedBatches.ToQueryResult(stats); + } + } + + public async Task> QueryAvailableBatches() + { + using (var session = store.OpenAsyncSession()) + { + var results = await session.Query() + .Where(b => b.HasStagingBatches || b.HasForwardingBatches) + .ToListAsync(); + return results; + } + } + + static ICommandData CreateFailedMessageRetryDocument(string batchDocumentId, string messageId) + { + return new PatchCommandData + { + Patches = PatchRequestsEmpty, + PatchesIfMissing = new[] + { + new PatchRequest + { + Name = "FailedMessageId", + Type = PatchCommandType.Set, + Value = FailedMessageIdGenerator.MakeDocumentId(messageId) + }, + new PatchRequest + { + Name = "RetryBatchId", + Type = PatchCommandType.Set, + Value = batchDocumentId + } + }, + Key = FailedMessageRetry.MakeDocumentId(messageId), + Metadata = DefaultMetadata + }; + } + + static RavenJObject DefaultMetadata = RavenJObject.Parse($@" + {{ + ""Raven-Entity-Name"": ""{FailedMessageRetry.CollectionName}"", + ""Raven-Clr-Type"": ""{typeof(FailedMessageRetry).AssemblyQualifiedName}"" + }}"); + + static PatchRequest[] PatchRequestsEmpty = Array.Empty(); + + static ILog log = LogManager.GetLogger(typeof(RetryDocumentDataStore)); + + + + public async Task GetBatchesForAll(DateTime cutoff, Func callback) + { + // StartRetryForIndex("All", RetryType.All, DateTime.UtcNow, originator: "all messages"); + //public void StartRetryForIndex(string requestId, RetryType retryType, DateTime startTime, Expression> filter = null, string originator = null, string classifier = null) + //StartRetryForIndex(endpoint, RetryType.AllForEndpoint, DateTime.UtcNow, m => m.ReceivingEndpointName == endpoint, $"all messages for endpoint {endpoint}"); + + var x = new IndexBasedBulkRetryRequest( + "All", + RetryType.All, + originator: "all messages", + classifier: null, + cutoff, + filter: null + ); + + using (var session = store.OpenAsyncSession()) + using (var stream = await x.GetDocuments(session)) + { + while (await stream.MoveNextAsync()) + { + var current = stream.Current.Document; + await callback(current.UniqueMessageId, current.LatestTimeOfFailure); + } + } + } + + public async Task GetBatchesForEndpoint(DateTime cutoff, string endpoint, Func callback) + { + //ForIndex + //StartRetryForIndex(endpoint, RetryType.AllForEndpoint, DateTime.UtcNow, m => m.ReceivingEndpointName == endpoint, $"all messages for endpoint {endpoint}"); + + var x = new IndexBasedBulkRetryRequest( + endpoint, + RetryType.AllForEndpoint, + originator: $"all messages for endpoint {endpoint}", + classifier: null, + cutoff, + filter: m => m.ReceivingEndpointName == endpoint + ); + + using (var session = store.OpenAsyncSession()) + using (var stream = await x.GetDocuments(session)) + { + while (await stream.MoveNextAsync()) + { + var current = stream.Current.Document; + await callback(current.UniqueMessageId, current.LatestTimeOfFailure); + } + } + } + + public async Task GetBatchesForFailedQueueAddress(DateTime cutoff, string failedQueueAddress, FailedMessageStatus status, Func callback) + { + //ForIndex + //StartRetryForIndex(failedQueueAddress, RetryType.ByQueueAddress, DateTime.UtcNow, m => m.QueueAddress == failedQueueAddress && m.Status == status, ); + + var x = new IndexBasedBulkRetryRequest( + failedQueueAddress, + RetryType.AllForEndpoint, + originator: $"all messages for failed queue address '{failedQueueAddress}'", + classifier: null, + cutoff, + filter: m => m.QueueAddress == failedQueueAddress && m.Status == status + ); + + using (var session = store.OpenAsyncSession()) + using (var stream = await x.GetDocuments(session)) + { + while (await stream.MoveNextAsync()) + { + var current = stream.Current.Document; + await callback(current.UniqueMessageId, current.LatestTimeOfFailure); + } + } + } + + public async Task GetBatchesForFailureGroup(string groupId, string groupTitle, string groupType, DateTime cutoff, Func callback) + { + //retries.StartRetryForIndex(message.GroupId, RetryType.FailureGroup, started, x => x.FailureGroupId == message.GroupId, originator, group?.Type); + + var x = new IndexBasedBulkRetryRequest( + groupId, + RetryType.AllForEndpoint, + originator: groupTitle, + classifier: groupType, + cutoff, + filter: m => m.FailureGroupId == groupId + ); + + using (var session = store.OpenAsyncSession()) + using (var stream = await x.GetDocuments(session)) + { + while (await stream.MoveNextAsync()) + { + var current = stream.Current.Document; + await callback(current.UniqueMessageId, current.LatestTimeOfFailure); + } + } + } + + public async Task QueryFailureGroupViewOnGroupId(string groupId) + { + using (var session = store.OpenAsyncSession()) + { + var group = await session.Query() + .FirstOrDefaultAsync(x => x.Id == groupId); + return group; + } + } + + class IndexBasedBulkRetryRequest + where TIndex : AbstractIndexCreationTask, new() + where TType : IHaveStatus + { + public IndexBasedBulkRetryRequest(string requestId, RetryType retryType, string originator, string classifier, DateTime startTime, Expression> filter) + { + RequestId = requestId; + RetryType = retryType; + Originator = originator; + this.filter = filter; + StartTime = startTime; + Classifier = classifier; + } + + public string RequestId { get; set; } + public RetryType RetryType { get; set; } + public string Originator { get; set; } + public string Classifier { get; set; } + public DateTime StartTime { get; set; } + + public Task>> GetDocuments(IAsyncDocumentSession session) + { + var query = session.Query(); + + query = query.Where(d => d.Status == FailedMessageStatus.Unresolved); + + if (filter != null) + { + query = query.Where(filter); + } + + return session.Advanced.StreamAsync(query.TransformWith()); + } + + readonly Expression> filter; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/SagaAudit/AuditRetentionCustomCheck.cs b/src/ServiceControl.Persistence.RavenDb5/SagaAudit/AuditRetentionCustomCheck.cs new file mode 100644 index 0000000000..e671a462df --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/SagaAudit/AuditRetentionCustomCheck.cs @@ -0,0 +1,50 @@ + +namespace ServiceControl.Persistence.RavenDb.SagaAudit +{ + using System; + using System.Threading.Tasks; + using NServiceBus.CustomChecks; + using Raven.Client; + using ServiceControl.SagaAudit; + + // This custom check stays in the Raven3.5 persister becuase the Raven5 persister will not store audit data + class AuditRetentionCustomCheck : CustomCheck + { + readonly IDocumentStore documentStore; + readonly bool auditRetentionPeriodIsSet; + + public AuditRetentionCustomCheck( + IDocumentStore documentStore, + RavenDBPersisterSettings settings, + TimeSpan? repeatAfter = null + ) + : base("Saga Audit Data Retention", "ServiceControl Health", repeatAfter.HasValue ? repeatAfter : TimeSpan.FromHours(1)) + { + this.documentStore = documentStore; + auditRetentionPeriodIsSet = settings.AuditRetentionPeriod != default; + } + + public override async Task PerformCheck() + { + if (auditRetentionPeriodIsSet) + { + return CheckResult.Pass; + } + + if (await DetectSagaAuditData()) + { + return CheckResult.Failed("Saga snapshot data detected without an audit retention period configured. If saga audit data is allowed to accumulate, it can result in degraded performance. Visit https://docs.particular.net/search?q=servicecontrol+troubleshooting for more information."); + } + + return CheckResult.Pass; + } + + async Task DetectSagaAuditData() + { + using (var session = documentStore.OpenAsyncSession()) + { + return await session.Query().AnyAsync(); + } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaAuditDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaAuditDataStore.cs new file mode 100644 index 0000000000..69af8b1da6 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaAuditDataStore.cs @@ -0,0 +1,45 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System; + using System.Threading.Tasks; + using Raven.Client; + using ServiceControl.Persistence.Infrastructure; + using ServiceControl.SagaAudit; + + class SagaAuditDataStore : ISagaAuditDataStore + { + public SagaAuditDataStore(IDocumentStore store) + { + this.store = store; + } + + public async Task StoreSnapshot(SagaSnapshot sagaSnapshot) + { + using (var session = store.OpenAsyncSession()) + { + await session.StoreAsync(sagaSnapshot); + await session.SaveChangesAsync(); + } + } + + public async Task> GetSagaById(Guid sagaId) + { + using (var session = store.OpenAsyncSession()) + { + var sagaHistory = await + session.Query() + .Statistics(out var stats) + .SingleOrDefaultAsync(x => x.SagaId == sagaId); + + if (sagaHistory == null) + { + return QueryResult.Empty(); + } + + return new QueryResult(sagaHistory, new QueryStatsInfo(stats.IndexEtag, stats.TotalResults, stats.IsStale)); + } + } + + readonly IDocumentStore store; + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaListIndex.cs b/src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaListIndex.cs new file mode 100644 index 0000000000..73961a8b79 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaListIndex.cs @@ -0,0 +1,45 @@ +namespace ServiceControl.SagaAudit +{ + using System; + using System.Linq; + using Raven.Client.Indexes; + + class SagaListIndex : AbstractMultiMapIndexCreationTask + { + public SagaListIndex() + { + AddMap(docs => from doc in docs + select new Result + { + Id = doc.SagaId, + Uri = "api/sagas/" + doc.SagaId, + SagaType = doc.SagaType + }); + AddMap(docs => from doc in docs + select new Result + { + Id = doc.SagaId, + Uri = "api/sagas/" + doc.SagaId, + SagaType = doc.SagaType + }); + + Reduce = results => from result in results + group result by result.Id + into g + let first = g.First() + select new Result + { + Id = g.Key, + Uri = first.Uri, + SagaType = first.SagaType + }; + } + + public class Result + { + public Guid Id { get; set; } + public string Uri { get; set; } + public string SagaType { get; set; } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/ServiceControl.Persistence.RavenDb5.csproj b/src/ServiceControl.Persistence.RavenDb5/ServiceControl.Persistence.RavenDb5.csproj new file mode 100644 index 0000000000..01b5a47ea4 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/ServiceControl.Persistence.RavenDb5.csproj @@ -0,0 +1,28 @@ + + + + net472 + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Transactions/RavenTransactionalDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/Transactions/RavenTransactionalDataStore.cs new file mode 100644 index 0000000000..caa69201b3 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Transactions/RavenTransactionalDataStore.cs @@ -0,0 +1,14 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System.Threading.Tasks; + using Raven.Client; + + abstract class AbstractSessionManager : IDataSessionManager + { + protected IAsyncDocumentSession Session { get; } + + protected AbstractSessionManager(IAsyncDocumentSession session) => Session = session; + public Task SaveChanges() => Session.SaveChangesAsync(); + public void Dispose() => Session.Dispose(); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs b/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs new file mode 100644 index 0000000000..157a7ce15e --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs @@ -0,0 +1,49 @@ +namespace ServiceControl.MessageFailures.Api +{ + using System; + using System.Linq; + using Raven.Client.Indexes; + + class FailedMessageViewTransformer : AbstractTransformerCreationTask + { + public FailedMessageViewTransformer() + { + TransformResults = failures => from failure in failures + let rec = failure.ProcessingAttempts.Last() + let edited = rec.Headers["ServiceControl.EditOf"] != null + select new + { + Id = failure.UniqueMessageId, + MessageType = rec.MessageMetadata["MessageType"], + IsSystemMessage = (bool)rec.MessageMetadata["IsSystemMessage"], + SendingEndpoint = rec.MessageMetadata["SendingEndpoint"], + ReceivingEndpoint = rec.MessageMetadata["ReceivingEndpoint"], + TimeSent = (DateTime?)rec.MessageMetadata["TimeSent"], + MessageId = rec.MessageMetadata["MessageId"], + rec.FailureDetails.Exception, + QueueAddress = rec.FailureDetails.AddressOfFailingEndpoint, + NumberOfProcessingAttempts = failure.ProcessingAttempts.Count, + failure.Status, + rec.FailureDetails.TimeOfFailure, + LastModified = MetadataFor(failure)["Last-Modified"].Value(), + Edited = edited, + EditOf = edited ? rec.Headers["ServiceControl.EditOf"] : "" + }; + } + + public static string Name + { + get + { + if (transformerName == null) + { + transformerName = new FailedMessageViewTransformer().TransformerName; + } + + return transformerName; + } + } + + static string transformerName; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesBodyTransformer.cs b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesBodyTransformer.cs new file mode 100644 index 0000000000..18a53ae86b --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesBodyTransformer.cs @@ -0,0 +1,59 @@ +namespace ServiceControl.CompositeViews.Messages +{ + using System.Collections.Generic; + using System.Linq; + using Raven.Client.Indexes; + using ServiceControl.MessageFailures; + + class MessagesBodyTransformer : AbstractTransformerCreationTask + { + public MessagesBodyTransformer() + { + TransformResults = messages => from message in messages + let metadata = message.ProcessingAttempts != null + ? message.ProcessingAttempts.Last().MessageMetadata + : message.MessageMetadata + let body = message.ProcessingAttempts != null + ? message.ProcessingAttempts.Last().Body ?? metadata["Body"] + : metadata["Body"] + select new + { + MessageId = metadata["MessageId"], + Body = body, + BodySize = (int)metadata["ContentLength"], + ContentType = metadata["ContentType"], + BodyNotStored = (bool)metadata["BodyNotStored"] + }; + } + + public static string Name + { + get + { + if (transformerName == null) + { + transformerName = new MessagesBodyTransformer().TransformerName; + } + + return transformerName; + } + } + + static string transformerName; + + public class Input + { + public Dictionary MessageMetadata { get; set; } + public List ProcessingAttempts { get; set; } + } + + public class Result + { + public string MessageId { get; set; } + public string Body { get; set; } + public string ContentType { get; set; } + public int BodySize { get; set; } + public bool BodyNotStored { get; set; } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs new file mode 100644 index 0000000000..98047d961b --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs @@ -0,0 +1,76 @@ +namespace ServiceControl.CompositeViews.Messages +{ + using System; + using System.Collections.Generic; + using System.Linq; + using MessageFailures; + using Raven.Client.Indexes; + using ServiceControl.Persistence; + + class MessagesViewTransformer : AbstractTransformerCreationTask + { + public MessagesViewTransformer() + { + TransformResults = messages => from message in messages + let metadata = + message.ProcessingAttempts != null + ? message.ProcessingAttempts.Last().MessageMetadata + : message.MessageMetadata + let headers = + message.ProcessingAttempts != null ? message.ProcessingAttempts.Last().Headers : message.Headers + let processedAt = + message.ProcessingAttempts != null + ? message.ProcessingAttempts.Last().AttemptedAt + : message.ProcessedAt + let status = + message.ProcessingAttempts == null + ? !(bool)message.MessageMetadata["IsRetried"] + ? MessageStatus.Successful + : MessageStatus.ResolvedSuccessfully + : message.Status == FailedMessageStatus.Resolved + ? MessageStatus.ResolvedSuccessfully + : message.Status == FailedMessageStatus.RetryIssued + ? MessageStatus.RetryIssued + : message.Status == FailedMessageStatus.Archived + ? MessageStatus.ArchivedFailure + : message.ProcessingAttempts.Count == 1 + ? MessageStatus.Failed + : MessageStatus.RepeatedFailure + select new + { + Id = message.UniqueMessageId, + MessageId = metadata["MessageId"], + MessageType = metadata["MessageType"], + SendingEndpoint = metadata["SendingEndpoint"], + ReceivingEndpoint = metadata["ReceivingEndpoint"], + TimeSent = (DateTime?)metadata["TimeSent"], + ProcessedAt = processedAt, + CriticalTime = (TimeSpan)metadata["CriticalTime"], + ProcessingTime = (TimeSpan)metadata["ProcessingTime"], + DeliveryTime = (TimeSpan)metadata["DeliveryTime"], + IsSystemMessage = (bool)metadata["IsSystemMessage"], + ConversationId = metadata["ConversationId"], + //the reason the we need to use a KeyValuePair is that raven seems to interpret the values and convert them + // to real types. In this case it was the NServiceBus.Temporary.DelayDeliveryWith header to was converted to a timespan + Headers = headers.Select(header => new KeyValuePair(header.Key, header.Value)), + Status = status, + MessageIntent = metadata["MessageIntent"], + BodyUrl = metadata["BodyUrl"], + BodySize = (int)metadata["ContentLength"], + InvokedSagas = metadata["InvokedSagas"], + OriginatesFromSaga = metadata["OriginatesFromSaga"] + }; + } + + public class Input + { + public string Id { get; set; } + public string UniqueMessageId { get; set; } + public DateTime ProcessedAt { get; set; } + public Dictionary Headers { get; set; } + public Dictionary MessageMetadata { get; set; } + public List ProcessingAttempts { get; set; } + public FailedMessageStatus Status { get; set; } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs new file mode 100644 index 0000000000..d6b4d9bcaa --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs @@ -0,0 +1,28 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System.Collections.Concurrent; + using System.Threading.Tasks; + using Raven.Abstractions.Commands; + using Raven.Client; + using ServiceControl.Persistence.UnitOfWork; + + class RavenDbIngestionUnitOfWork : IngestionUnitOfWorkBase + { + readonly IDocumentStore store; + readonly ConcurrentBag commands; + + public RavenDbIngestionUnitOfWork(IDocumentStore store) + { + this.store = store; + commands = new ConcurrentBag(); + Monitoring = new RavenDbMonitoringIngestionUnitOfWork(this); + Recoverability = new RavenDbRecoverabilityIngestionUnitOfWork(this); + } + + internal void AddCommand(ICommandData command) => commands.Add(command); + + public override Task Complete() => + // not really interested in the batch results since a batch is atomic + store.AsyncDatabaseCommands.BatchAsync(commands); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWorkFactory.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWorkFactory.cs new file mode 100644 index 0000000000..3fe030ec17 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWorkFactory.cs @@ -0,0 +1,26 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System.Threading.Tasks; + using Raven.Client; + using ServiceControl.Persistence.UnitOfWork; + + class RavenDbIngestionUnitOfWorkFactory : IIngestionUnitOfWorkFactory + { + readonly IDocumentStore store; + readonly MinimumRequiredStorageState customCheckState; + + public RavenDbIngestionUnitOfWorkFactory(IDocumentStore store, MinimumRequiredStorageState customCheckState) + { + this.store = store; + this.customCheckState = customCheckState; + } + + public ValueTask StartNew() + => new ValueTask(new RavenDbIngestionUnitOfWork(store)); + + public bool CanIngestMore() + { + return customCheckState.CanIngestMore; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs new file mode 100644 index 0000000000..f3ad877561 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs @@ -0,0 +1,42 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System.Threading.Tasks; + using Raven.Abstractions.Commands; + using Raven.Json.Linq; + using ServiceControl.Persistence.UnitOfWork; + + class RavenDbMonitoringIngestionUnitOfWork : IMonitoringIngestionUnitOfWork + { + RavenDbIngestionUnitOfWork parentUnitOfWork; + + public RavenDbMonitoringIngestionUnitOfWork(RavenDbIngestionUnitOfWork parentUnitOfWork) + { + this.parentUnitOfWork = parentUnitOfWork; + } + + public Task RecordKnownEndpoint(KnownEndpoint knownEndpoint) + { + parentUnitOfWork.AddCommand(CreateKnownEndpointsPutCommand(knownEndpoint)); + return Task.CompletedTask; + } + + static PutCommandData CreateKnownEndpointsPutCommand(KnownEndpoint endpoint) => new PutCommandData + { + Document = RavenJObject.FromObject(endpoint), + Etag = null, + Key = endpoint.Id.ToString(), + Metadata = KnownEndpointMetadata + }; + + static RavenDbMonitoringIngestionUnitOfWork() + { + KnownEndpointMetadata = RavenJObject.Parse($@" + {{ + ""Raven-Entity-Name"": ""{KnownEndpoint.CollectionName}"", + ""Raven-Clr-Type"": ""{typeof(KnownEndpoint).AssemblyQualifiedName}"" + }}"); + } + + static readonly RavenJObject KnownEndpointMetadata; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs new file mode 100644 index 0000000000..6ec853c166 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs @@ -0,0 +1,137 @@ +namespace ServiceControl.Persistence.RavenDb +{ + using System.Collections.Generic; + using System.Threading.Tasks; + using Raven.Abstractions.Commands; + using Raven.Abstractions.Data; + using Raven.Abstractions.Extensions; + using Raven.Imports.Newtonsoft.Json; + using Raven.Json.Linq; + using ServiceControl.MessageFailures; + using ServiceControl.Persistence.UnitOfWork; + using ServiceControl.Recoverability; + + class RavenDbRecoverabilityIngestionUnitOfWork : IRecoverabilityIngestionUnitOfWork + { + RavenDbIngestionUnitOfWork parentUnitOfWork; + + public RavenDbRecoverabilityIngestionUnitOfWork(RavenDbIngestionUnitOfWork parentUnitOfWork) + { + this.parentUnitOfWork = parentUnitOfWork; + } + + public Task RecordFailedProcessingAttempt( + string uniqueMessageId, + FailedMessage.ProcessingAttempt processingAttempt, + List groups) + { + parentUnitOfWork.AddCommand( + CreateFailedMessagesPatchCommand(uniqueMessageId, processingAttempt, groups) + ); + return Task.CompletedTask; + } + + public Task RecordSuccessfulRetry(string retriedMessageUniqueId) + { + var failedMessageDocumentId = FailedMessageIdGenerator.MakeDocumentId(retriedMessageUniqueId); + var failedMessageRetryDocumentId = FailedMessageRetry.MakeDocumentId(retriedMessageUniqueId); + + parentUnitOfWork.AddCommand(new PatchCommandData + { + Key = failedMessageDocumentId, + Patches = new[] + { + new PatchRequest {Type = PatchCommandType.Set, Name = nameof(FailedMessage.Status), Value = (int)FailedMessageStatus.Resolved} + } + }); + + parentUnitOfWork.AddCommand(new DeleteCommandData { Key = failedMessageRetryDocumentId }); + return Task.CompletedTask; + } + + ICommandData CreateFailedMessagesPatchCommand(string uniqueMessageId, FailedMessage.ProcessingAttempt processingAttempt, + List groups) + { + var documentId = FailedMessageIdGenerator.MakeDocumentId(uniqueMessageId); + + var serializedGroups = RavenJToken.FromObject(groups); + var serializedAttempt = RavenJToken.FromObject(processingAttempt, Serializer); + + //HINT: RavenDB 3.5 is using Lodash v4.13.1 to provide javascript utility functions + // https://ravendb.net/docs/article-page/3.5/csharp/client-api/commands/patches/how-to-use-javascript-to-patch-your-documents#methods-objects-and-variables + return new ScriptedPatchCommandData + { + Key = documentId, + Patch = new ScriptedPatchRequest + { + Script = $@"this.{nameof(FailedMessage.Status)} = status; + this.{nameof(FailedMessage.FailureGroups)} = failureGroups; + + var newAttempts = this.{nameof(FailedMessage.ProcessingAttempts)}; + + //De-duplicate attempts by AttemptedAt value + + var duplicateIndex = _.findIndex(this.{nameof(FailedMessage.ProcessingAttempts)}, function(a){{ + return a.{nameof(FailedMessage.ProcessingAttempt.AttemptedAt)} === attempt.{nameof(FailedMessage.ProcessingAttempt.AttemptedAt)}; + }}); + + if(duplicateIndex === -1){{ + newAttempts = _.union(newAttempts, [attempt]); + }} + + //Trim to the latest MaxProcessingAttempts + + newAttempts = _.sortBy(newAttempts, function(a) {{ + return a.{nameof(FailedMessage.ProcessingAttempt.AttemptedAt)}; + }}); + + if(newAttempts.length > {MaxProcessingAttempts}) + {{ + newAttempts = _.slice(newAttempts, newAttempts.length - {MaxProcessingAttempts}, newAttempts.length); + }} + + this.{nameof(FailedMessage.ProcessingAttempts)} = newAttempts; + ", + Values = new Dictionary + { + {"status", (int)FailedMessageStatus.Unresolved}, + {"failureGroups", serializedGroups}, + {"attempt", serializedAttempt} + }, + }, + PatchIfMissing = new ScriptedPatchRequest + { + Script = $@"this.{nameof(FailedMessage.Status)} = status; + this.{nameof(FailedMessage.FailureGroups)} = failureGroups; + this.{nameof(FailedMessage.ProcessingAttempts)} = [attempt]; + this.{nameof(FailedMessage.UniqueMessageId)} = uniqueMessageId; + ", + Values = new Dictionary + { + {"status", (int)FailedMessageStatus.Unresolved}, + {"failureGroups", serializedGroups}, + {"attempt", serializedAttempt}, + {"uniqueMessageId", uniqueMessageId} + } + }, + Metadata = FailedMessageMetadata + }; + } + + static RavenDbRecoverabilityIngestionUnitOfWork() + { + Serializer = JsonExtensions.CreateDefaultJsonSerializer(); + Serializer.TypeNameHandling = TypeNameHandling.Auto; + + FailedMessageMetadata = RavenJObject.Parse($@" + {{ + ""Raven-Entity-Name"": ""{FailedMessageIdGenerator.CollectionName}"", + ""Raven-Clr-Type"": ""{typeof(FailedMessage).AssemblyQualifiedName}"" + }}"); + } + + static int MaxProcessingAttempts = 10; + static readonly RavenJObject FailedMessageMetadata; + static readonly JsonSerializer Serializer; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/WaitHandleExtensions.cs b/src/ServiceControl.Persistence.RavenDb5/WaitHandleExtensions.cs new file mode 100644 index 0000000000..5535209353 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/WaitHandleExtensions.cs @@ -0,0 +1,44 @@ +namespace ServiceBus.Management.Infrastructure.Extensions +{ + using System; + using System.Threading; + using System.Threading.Tasks; + + static class WaitHandleExtensions + { + public static async Task WaitOneAsync(this WaitHandle handle, int millisecondsTimeout, CancellationToken cancellationToken = default) + { + RegisteredWaitHandle registeredHandle = null; + var tokenRegistration = default(CancellationTokenRegistration); + try + { + var tcs = new TaskCompletionSource(); + registeredHandle = ThreadPool.RegisterWaitForSingleObject( + handle, + (state, timedOut) => ((TaskCompletionSource)state).TrySetResult(!timedOut), + tcs, + millisecondsTimeout, + true); + tokenRegistration = cancellationToken.Register( + state => ((TaskCompletionSource)state).TrySetCanceled(), + tcs); + return await tcs.Task; + } + finally + { + registeredHandle?.Unregister(null); + tokenRegistration.Dispose(); + } + } + + public static Task WaitOneAsync(this WaitHandle handle, TimeSpan timeout, CancellationToken cancellationToken = default) + { + return handle.WaitOneAsync((int)timeout.TotalMilliseconds, cancellationToken); + } + + public static Task WaitOneAsync(this WaitHandle handle, CancellationToken cancellationToken = default) + { + return handle.WaitOneAsync(Timeout.Infinite, cancellationToken); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/persistence.manifest b/src/ServiceControl.Persistence.RavenDb5/persistence.manifest new file mode 100644 index 0000000000..4dace6b01e --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/persistence.manifest @@ -0,0 +1,24 @@ +{ + "Version": "1.0.0", + "Name": "RavenDB5", + "DisplayName": "RavenDB 5", + "Description": "RavenDB 5 ServiceControl persister", + "TypeName": "ServiceControl.Persistence.RavenDb.RavenDbPersistenceConfiguration, ServiceControl.Persistence.RavenDb5", + "Settings": [ + { + "Name": "ServiceControl/DBPath", + "Mandatory": true + }, + { + "Name": "ServiceControl/HostName", + "Mandatory": true + }, + { + "Name": "ServiceControl/DatabaseMaintenancePort", + "Mandatory": true + } + ], + "SettingsWithPathsToCleanup": [ + "ServiceControl/DBPath" + ] +} diff --git a/src/ServiceControl.sln b/src/ServiceControl.sln index dfae66114e..b4779e2aa7 100644 --- a/src/ServiceControl.sln +++ b/src/ServiceControl.sln @@ -181,6 +181,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ServiceControl.Persistence. EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ServiceControl.AcceptanceTests.RavenDB", "ServiceControl.AcceptanceTests.RavenDB\ServiceControl.AcceptanceTests.RavenDB.csproj", "{33D5D084-FABB-4F65-A046-1C6626DF9AFB}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ServiceControl.Persistence.RavenDb5", "ServiceControl.Persistence.RavenDb5\ServiceControl.Persistence.RavenDb5.csproj", "{84627DC0-7B43-480D-80CF-A4DDA514A4EE}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1015,6 +1017,18 @@ Global {33D5D084-FABB-4F65-A046-1C6626DF9AFB}.Release|x64.Build.0 = Release|Any CPU {33D5D084-FABB-4F65-A046-1C6626DF9AFB}.Release|x86.ActiveCfg = Release|Any CPU {33D5D084-FABB-4F65-A046-1C6626DF9AFB}.Release|x86.Build.0 = Release|Any CPU + {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Debug|x64.ActiveCfg = Debug|Any CPU + {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Debug|x64.Build.0 = Debug|Any CPU + {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Debug|x86.ActiveCfg = Debug|Any CPU + {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Debug|x86.Build.0 = Debug|Any CPU + {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Release|Any CPU.Build.0 = Release|Any CPU + {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Release|x64.ActiveCfg = Release|Any CPU + {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Release|x64.Build.0 = Release|Any CPU + {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Release|x86.ActiveCfg = Release|Any CPU + {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1097,6 +1111,7 @@ Global {B93B8B7E-BEAF-4645-AA5C-2A1CADEC3D4B} = {9AF9D3C7-E859-451B-BA4D-B954D289213A} {F2BD40E4-A077-429A-8A22-1C80AFC240E8} = {350F72AB-142D-4AAD-9EF1-1A83DC991D87} {33D5D084-FABB-4F65-A046-1C6626DF9AFB} = {350F72AB-142D-4AAD-9EF1-1A83DC991D87} + {84627DC0-7B43-480D-80CF-A4DDA514A4EE} = {9B52418E-BF18-4D25-BE17-4B56D3FB1154} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {3B9E5B72-F580-465A-A22C-2D2148AF4EB4} From 1639528a2919fcd22592aeb8ceeb0ddf6a256bfb Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 6 Sep 2023 14:57:45 +0200 Subject: [PATCH 02/78] Artifact path to new RavenDB5 folder + RavenDB package --- .../ServiceControl.Persistence.RavenDb5.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/ServiceControl.Persistence.RavenDb5.csproj b/src/ServiceControl.Persistence.RavenDb5/ServiceControl.Persistence.RavenDb5.csproj index 01b5a47ea4..444aaf313a 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ServiceControl.Persistence.RavenDb5.csproj +++ b/src/ServiceControl.Persistence.RavenDb5/ServiceControl.Persistence.RavenDb5.csproj @@ -11,7 +11,7 @@ - + @@ -22,7 +22,7 @@ - + \ No newline at end of file From e96b3b300d0e96044d06338dfb354706f09c6e3c Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 6 Sep 2023 15:24:21 +0200 Subject: [PATCH 03/78] Added missing namespace, tried updating most indexes --- ...iledMessages_UniqueMessageIdAndTimeOfFailures.cs | 2 +- .../CustomChecks/CheckRavenDBIndexErrors.cs | 2 +- .../CustomChecks/CheckRavenDBIndexLag.cs | 5 +++-- .../CustomChecks/FailedAuditImportCustomCheck.cs | 2 +- .../Editing/EditFailedMessageManager.cs | 3 +-- .../Editing/NotificationsManager.cs | 2 +- .../ErrorMessagesDataStore.cs | 13 ++++++++----- .../EventLogDataStore.cs | 1 + .../Expiration/AuditMessageCleaner.cs | 9 +++------ .../Expiration/ErrorMessageCleaner.cs | 8 +++----- .../Expiration/ExpiryErrorMessageIndex.cs | 4 +--- .../Expiration/ExpiryEventLogItemsIndex.cs | 4 +--- .../Expiration/ExpiryProcessedMessageIndex.cs | 4 +--- .../Extensions.cs | 1 + .../ExternalIntegrationRequestsDataStore.cs | 4 +++- .../FailedErrorImportDataStore.cs | 3 ++- .../FailedMessageViewIndexNotifications.cs | 1 + .../FailedMessages/FailedMessageReclassifier.cs | 11 +++++++---- .../Indexes/ArchivedGroupsViewIndex.cs | 4 +--- .../Indexes/CustomChecksIndex.cs | 4 +--- .../Indexes/ExpirySagaAuditIndex.cs | 4 +--- .../Indexes/FailedErrorImportIndex.cs | 4 +--- .../Indexes/FailedMessageFacetsIndex.cs | 11 ++++------- .../Indexes/FailedMessageRetries_ByBatch.cs | 4 +--- .../Indexes/FailedMessageViewIndex.cs | 4 +--- .../Indexes/FailedMessages_ByGroup.cs | 5 +---- .../Indexes/FailureGroupsViewIndex.cs | 4 +--- .../Indexes/GroupCommentIndex.cs | 4 +--- .../Indexes/ImportFailedAudits.cs | 4 +--- .../Indexes/KnownEndpointIndex.cs | 4 +--- .../Indexes/MessagesViewIndex.cs | 7 ++----- .../Indexes/QueueAddressIndex.cs | 4 +--- .../Indexes/RetryBatches_ByStatusAndSession.cs | 4 +--- .../RetryBatches_ByStatus_ReduceInitialBatchSize.cs | 4 +--- .../Indexes/SagaDetailsIndex.cs | 2 +- .../Indexes/SagaHistoryCleaner.cs | 3 +++ .../QueueAddressStore.cs | 5 +++-- .../RavenAttachmentsBodyStorage.cs | 5 +++-- .../RavenDbCustomCheckDataStore.cs | 6 ++++-- .../RavenDbInstaller.cs | 2 +- .../RavenDbMonitoringDataStore.cs | 3 ++- .../RavenDbPersistence.cs | 5 +++-- .../RavenDbPersistenceConfiguration.cs | 2 +- .../RavenDbPersistenceLifecycle.cs | 3 ++- .../RavenQueryExtensions.cs | 6 ++++-- .../RavenStartup.cs | 2 ++ .../Archiving/UnarchiveDocumentManager.cs | 7 ++++--- .../Recoverability/GroupsDataStore.cs | 6 ++++-- .../Recoverability/RetryHistoryDataStore.cs | 2 +- .../RetryBatchesDataStore.cs | 6 +++++- .../RetryBatchesManager.cs | 3 ++- .../RetryDocumentDataStore.cs | 7 +++++++ .../SagaAudit/AuditRetentionCustomCheck.cs | 2 +- .../SagaAudit/SagaAuditDataStore.cs | 3 ++- .../SagaAudit/SagaListIndex.cs | 2 +- .../Transactions/RavenTransactionalDataStore.cs | 1 + .../Transformers/FailedMessageViewTransformer.cs | 2 +- .../Transformers/MessagesBodyTransformer.cs | 2 +- .../Transformers/MessagesViewTransformer.cs | 2 +- .../UnitOfWork/RavenDbIngestionUnitOfWork.cs | 4 +++- .../UnitOfWork/RavenDbIngestionUnitOfWorkFactory.cs | 3 ++- .../RavenDbMonitoringIngestionUnitOfWork.cs | 3 ++- .../RavenDbRecoverabilityIngestionUnitOfWork.cs | 8 +++++--- 63 files changed, 132 insertions(+), 129 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/BulkRetry/FailedMessages_UniqueMessageIdAndTimeOfFailures.cs b/src/ServiceControl.Persistence.RavenDb5/BulkRetry/FailedMessages_UniqueMessageIdAndTimeOfFailures.cs index ba15cb1158..fbad94325f 100644 --- a/src/ServiceControl.Persistence.RavenDb5/BulkRetry/FailedMessages_UniqueMessageIdAndTimeOfFailures.cs +++ b/src/ServiceControl.Persistence.RavenDb5/BulkRetry/FailedMessages_UniqueMessageIdAndTimeOfFailures.cs @@ -5,7 +5,7 @@ namespace ServiceControl.Recoverability using MessageFailures; using Raven.Client.Indexes; - class FailedMessages_UniqueMessageIdAndTimeOfFailures : AbstractTransformerCreationTask + class FailedMessages_UniqueMessageIdAndTimeOfFailures : AbstractTransformerCreationTask // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers { public FailedMessages_UniqueMessageIdAndTimeOfFailures() { diff --git a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexErrors.cs b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexErrors.cs index 27f038687d..d79a23ab70 100644 --- a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexErrors.cs +++ b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexErrors.cs @@ -6,7 +6,7 @@ using System.Threading.Tasks; using NServiceBus.CustomChecks; using NServiceBus.Logging; - using Raven.Client; + using Raven.Client.Documents; class CheckRavenDBIndexErrors : CustomCheck { diff --git a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexLag.cs b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexLag.cs index f138e0efa0..f39c687a4e 100644 --- a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexLag.cs +++ b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexLag.cs @@ -6,8 +6,9 @@ using System.Threading.Tasks; using NServiceBus.CustomChecks; using NServiceBus.Logging; - using Raven.Client; - using Raven.Abstractions.Data; + using Raven.Client.Documents; + using Raven.Client.Documents.Indexes; + using Raven.Client.Documents.Operations; using CustomCheck = NServiceBus.CustomChecks.CustomCheck; class CheckRavenDBIndexLag : CustomCheck diff --git a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/FailedAuditImportCustomCheck.cs b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/FailedAuditImportCustomCheck.cs index bada097bf3..58b0648761 100644 --- a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/FailedAuditImportCustomCheck.cs +++ b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/FailedAuditImportCustomCheck.cs @@ -4,7 +4,7 @@ using System.Threading.Tasks; using NServiceBus.CustomChecks; using NServiceBus.Logging; - using Raven.Client; + using Raven.Client.Documents; class FailedAuditImportCustomCheck : CustomCheck { diff --git a/src/ServiceControl.Persistence.RavenDb5/Editing/EditFailedMessageManager.cs b/src/ServiceControl.Persistence.RavenDb5/Editing/EditFailedMessageManager.cs index 3664c36fb8..cde27ab0d7 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Editing/EditFailedMessageManager.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Editing/EditFailedMessageManager.cs @@ -2,10 +2,9 @@ { using System; using System.Threading.Tasks; - using Raven.Abstractions.Data; - using Raven.Client; using ServiceControl.MessageFailures; using ServiceControl.Persistence.Recoverability.Editing; + using Raven.Client.Documents.Session; class EditFailedMessageManager : AbstractSessionManager, IEditFailedMessagesManager { diff --git a/src/ServiceControl.Persistence.RavenDb5/Editing/NotificationsManager.cs b/src/ServiceControl.Persistence.RavenDb5/Editing/NotificationsManager.cs index dd4d8fc4b3..8857860ad9 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Editing/NotificationsManager.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Editing/NotificationsManager.cs @@ -3,7 +3,7 @@ using System; using System.Threading.Tasks; using Notifications; - using Raven.Client; + using Raven.Client.Documents.Session; class NotificationsManager : AbstractSessionManager, INotificationsManager { diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index 2de540abd0..25f1afce5a 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -9,17 +9,20 @@ using CompositeViews.Messages; using Editing; using NServiceBus.Logging; - using Raven.Abstractions.Data; - using Raven.Abstractions.Extensions; - using Raven.Client; - using Raven.Client.Linq; - using Raven.Json.Linq; using ServiceControl.EventLog; using ServiceControl.MessageFailures; using ServiceControl.MessageFailures.Api; using ServiceControl.Operations; using ServiceControl.Persistence.Infrastructure; using ServiceControl.Recoverability; + using Raven.Abstractions.Data; + using Raven.Abstractions.Extensions; + using Raven.Client; + using Raven.Client.Documents; + using Raven.Client.Documents.Queries; + using Raven.Client.Documents.Session; + using Raven.Client.Linq; + using Raven.Json.Linq; class ErrorMessagesDataStore : IErrorMessageDataStore { diff --git a/src/ServiceControl.Persistence.RavenDb5/EventLogDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/EventLogDataStore.cs index 6fbafb64f6..88163218a1 100644 --- a/src/ServiceControl.Persistence.RavenDb5/EventLogDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/EventLogDataStore.cs @@ -5,6 +5,7 @@ using EventLog; using Persistence.Infrastructure; using Raven.Client; + using Raven.Client.Documents; using Raven.Client.Linq; class EventLogDataStore : IEventLogDataStore diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/AuditMessageCleaner.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/AuditMessageCleaner.cs index 74a9d3b3af..ca29378314 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Expiration/AuditMessageCleaner.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Expiration/AuditMessageCleaner.cs @@ -6,12 +6,9 @@ using System.Linq; using System.Threading; using NServiceBus.Logging; - using Raven.Abstractions; - using Raven.Abstractions.Commands; - using Raven.Abstractions.Data; - using Raven.Abstractions.Exceptions; - using Raven.Database; - using Raven.Json.Linq; + using Raven.Client.Documents.Commands.Batches; + using Raven.Client.Documents.Queries; + using Raven.Client.Util; static class AuditMessageCleaner { diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/ErrorMessageCleaner.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/ErrorMessageCleaner.cs index 8e91475081..cd73810cdf 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Expiration/ErrorMessageCleaner.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Expiration/ErrorMessageCleaner.cs @@ -6,13 +6,11 @@ using System.Linq; using System.Threading; using NServiceBus.Logging; - using Raven.Abstractions; - using Raven.Abstractions.Commands; - using Raven.Abstractions.Data; - using Raven.Abstractions.Exceptions; - using Raven.Database; using ServiceControl.MessageFailures; using ServiceControl.Recoverability; + using Raven.Client.Documents.Commands.Batches; + using Raven.Client.Documents.Queries; + using Raven.Client.Util; static class ErrorMessageCleaner { diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryErrorMessageIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryErrorMessageIndex.cs index 39f18df6c8..c56217b921 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryErrorMessageIndex.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryErrorMessageIndex.cs @@ -3,7 +3,7 @@ namespace ServiceControl.Infrastructure.RavenDB.Expiration using System; using System.Linq; using MessageFailures; - using Raven.Client.Indexes; + using Raven.Client.Documents.Indexes; class ExpiryErrorMessageIndex : AbstractIndexCreationTask { @@ -16,8 +16,6 @@ public ExpiryErrorMessageIndex() message.Status, LastModified = MetadataFor(message).Value("Last-Modified").Ticks }; - - DisableInMemoryIndexing = true; } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryEventLogItemsIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryEventLogItemsIndex.cs index 13ebc15cfe..8ae371def3 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryEventLogItemsIndex.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryEventLogItemsIndex.cs @@ -3,7 +3,7 @@ namespace ServiceControl.Infrastructure.RavenDB.Expiration using System; using System.Linq; using EventLog; - using Raven.Client.Indexes; + using Raven.Client.Documents.Indexes; class ExpiryEventLogItemsIndex : AbstractIndexCreationTask { @@ -14,8 +14,6 @@ public ExpiryEventLogItemsIndex() { LastModified = MetadataFor(message).Value("Last-Modified").Ticks }; - - DisableInMemoryIndexing = true; } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryProcessedMessageIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryProcessedMessageIndex.cs index 3d6e24725b..fc80629418 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryProcessedMessageIndex.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryProcessedMessageIndex.cs @@ -2,7 +2,7 @@ namespace ServiceControl.Infrastructure.RavenDB.Expiration { using System.Linq; using MessageAuditing; - using Raven.Client.Indexes; + using Raven.Client.Documents.Indexes; class ExpiryProcessedMessageIndex : AbstractIndexCreationTask { @@ -13,8 +13,6 @@ public ExpiryProcessedMessageIndex() { ProcessedAt = message.ProcessedAt.Ticks }; - - DisableInMemoryIndexing = true; } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Extensions.cs b/src/ServiceControl.Persistence.RavenDb5/Extensions.cs index 5f2184db8f..e328daf39e 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Extensions.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Extensions.cs @@ -3,6 +3,7 @@ using System; using System.Threading; using Raven.Abstractions.Data; + using Raven.Client.Documents.Queries; using Raven.Database; using Raven.Json.Linq; diff --git a/src/ServiceControl.Persistence.RavenDb5/ExternalIntegrationRequestsDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ExternalIntegrationRequestsDataStore.cs index ebb0db036d..c9af28741b 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ExternalIntegrationRequestsDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ExternalIntegrationRequestsDataStore.cs @@ -10,10 +10,12 @@ using Microsoft.Extensions.Hosting; using NServiceBus; using NServiceBus.Logging; + using ServiceBus.Management.Infrastructure.Extensions; using Raven.Abstractions.Data; using Raven.Client; + using Raven.Client.Documents; + using Raven.Client.Documents.Changes; using Raven.Client.Linq; - using ServiceBus.Management.Infrastructure.Extensions; class ExternalIntegrationRequestsDataStore : IExternalIntegrationRequestsDataStore diff --git a/src/ServiceControl.Persistence.RavenDb5/FailedErrorImportDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/FailedErrorImportDataStore.cs index 3d96697bd1..52b3d6027b 100644 --- a/src/ServiceControl.Persistence.RavenDb5/FailedErrorImportDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/FailedErrorImportDataStore.cs @@ -4,8 +4,9 @@ using System.Threading; using System.Threading.Tasks; using NServiceBus.Logging; - using Raven.Client; using ServiceControl.Operations; + using Raven.Client; + using Raven.Client.Documents; class FailedErrorImportDataStore : IFailedErrorImportDataStore { diff --git a/src/ServiceControl.Persistence.RavenDb5/FailedMessageViewIndexNotifications.cs b/src/ServiceControl.Persistence.RavenDb5/FailedMessageViewIndexNotifications.cs index 1ab4388de0..92a5945fdc 100644 --- a/src/ServiceControl.Persistence.RavenDb5/FailedMessageViewIndexNotifications.cs +++ b/src/ServiceControl.Persistence.RavenDb5/FailedMessageViewIndexNotifications.cs @@ -8,6 +8,7 @@ using NServiceBus.Logging; using Persistence; using Raven.Client; + using Raven.Client.Documents; class FailedMessageViewIndexNotifications : IFailedMessageViewIndexNotifications diff --git a/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs b/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs index 3da00aee79..97c52c1459 100644 --- a/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs +++ b/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs @@ -7,14 +7,17 @@ using System.Threading.Tasks; using Microsoft.Extensions.Hosting; using NServiceBus.Logging; - using Raven.Abstractions.Data; - using Raven.Abstractions.Exceptions; - using Raven.Client; - using Raven.Json.Linq; using ServiceControl.MessageFailures; using ServiceControl.MessageFailures.Api; using ServiceControl.Persistence.Infrastructure; using ServiceControl.Recoverability; + using Raven.Abstractions.Data; + using Raven.Abstractions.Exceptions; + using Raven.Client; + using Raven.Client.Documents; + using Raven.Client.Documents.Operations; + using Raven.Client.Exceptions; + using Raven.Json.Linq; class FailedMessageReclassifier : IReclassifyFailedMessages { diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/ArchivedGroupsViewIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/ArchivedGroupsViewIndex.cs index 6de54efd63..5d373b119b 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/ArchivedGroupsViewIndex.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/ArchivedGroupsViewIndex.cs @@ -2,7 +2,7 @@ namespace ServiceControl.Recoverability { using System.Linq; using MessageFailures; - using Raven.Client.Indexes; + using Raven.Client.Documents.Indexes; class ArchivedGroupsViewIndex : AbstractIndexCreationTask { @@ -39,8 +39,6 @@ into g Last = g.Max(x => x.Last), Type = g.Key.Type }; - - DisableInMemoryIndexing = true; } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/CustomChecksIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/CustomChecksIndex.cs index 83a96221c4..7dd1833657 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/CustomChecksIndex.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/CustomChecksIndex.cs @@ -1,8 +1,8 @@ namespace ServiceControl.Persistence { using System.Linq; - using Raven.Client.Indexes; using ServiceControl.Contracts.CustomChecks; + using Raven.Client.Documents.Indexes; class CustomChecksIndex : AbstractIndexCreationTask { @@ -16,8 +16,6 @@ public CustomChecksIndex() Category = cc.Category, CustomCheckId = cc.CustomCheckId }; - - DisableInMemoryIndexing = true; } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/ExpirySagaAuditIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/ExpirySagaAuditIndex.cs index fb6e83706c..5c9ecff7ff 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/ExpirySagaAuditIndex.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/ExpirySagaAuditIndex.cs @@ -2,7 +2,7 @@ namespace ServiceControl.SagaAudit { using System; using System.Linq; - using Raven.Client.Indexes; + using Raven.Client.Documents.Indexes; class ExpirySagaAuditIndex : AbstractMultiMapIndexCreationTask { @@ -19,8 +19,6 @@ public ExpirySagaAuditIndex() { LastModified = MetadataFor(sagaHistory).Value("Last-Modified").Ticks }); - - DisableInMemoryIndexing = true; } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedErrorImportIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedErrorImportIndex.cs index 0816d9ca4e..653e2f0f96 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedErrorImportIndex.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedErrorImportIndex.cs @@ -1,7 +1,7 @@ namespace ServiceControl.Operations { using System.Linq; - using Raven.Client.Indexes; + using Raven.Client.Documents.Indexes; class FailedErrorImportIndex : AbstractIndexCreationTask { @@ -13,8 +13,6 @@ public FailedErrorImportIndex() Id = cc.Id, Message = cc.Message }; - - DisableInMemoryIndexing = true; } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageFacetsIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageFacetsIndex.cs index 6ecba0affc..5335c9f8d9 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageFacetsIndex.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageFacetsIndex.cs @@ -1,9 +1,8 @@ namespace ServiceControl.MessageFailures.Api { using System.Linq; - using Raven.Abstractions.Indexing; - using Raven.Client.Indexes; using ServiceControl.Operations; + using Raven.Client.Documents.Indexes; class FailedMessageFacetsIndex : AbstractIndexCreationTask { @@ -19,11 +18,9 @@ public FailedMessageFacetsIndex() MessageType = failure.ProcessingAttempts.Last().MessageMetadata["MessageType"] }; - Index("Name", FieldIndexing.NotAnalyzed); //to avoid lower casing - Index("Host", FieldIndexing.NotAnalyzed); //to avoid lower casing - Index("MessageType", FieldIndexing.NotAnalyzed); //to avoid lower casing - - DisableInMemoryIndexing = true; + Index("Name", FieldIndexing.Exact); //to avoid lower casing + Index("Host", FieldIndexing.Exact); //to avoid lower casing + Index("MessageType", FieldIndexing.Exact); //to avoid lower casing } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageRetries_ByBatch.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageRetries_ByBatch.cs index 2ff6a5d4ba..3a782c6a4e 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageRetries_ByBatch.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageRetries_ByBatch.cs @@ -1,7 +1,7 @@ namespace ServiceControl.Persistence { using System.Linq; - using Raven.Client.Indexes; + using Raven.Client.Documents.Indexes; using ServiceControl.Recoverability; class FailedMessageRetries_ByBatch : AbstractIndexCreationTask @@ -13,8 +13,6 @@ public FailedMessageRetries_ByBatch() { doc.RetryBatchId }; - - DisableInMemoryIndexing = true; } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageViewIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageViewIndex.cs index 11e3514eec..81b4d2529b 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageViewIndex.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageViewIndex.cs @@ -2,8 +2,8 @@ namespace ServiceControl.MessageFailures.Api { using System; using System.Linq; - using Raven.Client.Indexes; using ServiceControl.Operations; + using Raven.Client.Documents.Indexes; class FailedMessageViewIndex : AbstractIndexCreationTask { @@ -22,8 +22,6 @@ public FailedMessageViewIndex() processingAttemptsLast.FailureDetails.TimeOfFailure, LastModified = MetadataFor(message).Value("Last-Modified").Ticks }; - - DisableInMemoryIndexing = true; } public class SortAndFilterOptions : IHaveStatus diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessages_ByGroup.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessages_ByGroup.cs index d7b96da4d2..a6555f67b3 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessages_ByGroup.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessages_ByGroup.cs @@ -3,8 +3,7 @@ namespace ServiceControl.Recoverability using System; using System.Linq; using MessageFailures; - using Raven.Abstractions.Indexing; - using Raven.Client.Indexes; + using Raven.Client.Documents.Indexes; class FailedMessages_ByGroup : AbstractIndexCreationTask { @@ -27,8 +26,6 @@ from failureGroup in doc.FailureGroups }; StoreAllFields(FieldStorage.Yes); - - DisableInMemoryIndexing = true; } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailureGroupsViewIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailureGroupsViewIndex.cs index 47f2402d7c..104bff6816 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailureGroupsViewIndex.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailureGroupsViewIndex.cs @@ -2,7 +2,7 @@ namespace ServiceControl.Recoverability { using System.Linq; using MessageFailures; - using Raven.Client.Indexes; + using Raven.Client.Documents.Indexes; class FailureGroupsViewIndex : AbstractIndexCreationTask { @@ -39,8 +39,6 @@ into g Last = g.Max(x => x.Last), Type = g.Key.Type }; - - DisableInMemoryIndexing = true; } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/GroupCommentIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/GroupCommentIndex.cs index e5b86901c6..3bfb3ea353 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/GroupCommentIndex.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/GroupCommentIndex.cs @@ -2,15 +2,13 @@ namespace ServiceControl.Recoverability { using System.Linq; using MessageFailures; - using Raven.Client.Indexes; + using Raven.Client.Documents.Indexes; class GroupCommentIndex : AbstractIndexCreationTask { public GroupCommentIndex() { Map = docs => docs.Select(gc => new GroupComment { Id = gc.Id, Comment = gc.Comment }); - - DisableInMemoryIndexing = true; } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/ImportFailedAudits.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/ImportFailedAudits.cs index 10b8970b81..23c9837271 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/ImportFailedAudits.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/ImportFailedAudits.cs @@ -1,7 +1,7 @@ namespace ServiceControl.Operations { using System.Linq; - using Raven.Client.Indexes; + using Raven.Client.Documents.Indexes; class FailedAuditImportIndex : AbstractIndexCreationTask { @@ -13,8 +13,6 @@ public FailedAuditImportIndex() Id = cc.Id, Message = cc.Message }; - - DisableInMemoryIndexing = true; } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/KnownEndpointIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/KnownEndpointIndex.cs index 297724d957..1b25b507c2 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/KnownEndpointIndex.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/KnownEndpointIndex.cs @@ -1,7 +1,7 @@ namespace ServiceControl.Persistence { using System.Linq; - using Raven.Client.Indexes; + using Raven.Client.Documents.Indexes; class KnownEndpointIndex : AbstractIndexCreationTask { @@ -16,8 +16,6 @@ public KnownEndpointIndex() message.Monitored, message.HasTemporaryId }; - - DisableInMemoryIndexing = true; } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/MessagesViewIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/MessagesViewIndex.cs index 3b3acd9476..195f3dfe97 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/MessagesViewIndex.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/MessagesViewIndex.cs @@ -3,11 +3,10 @@ namespace ServiceControl.Persistence using System; using System.Linq; using Lucene.Net.Analysis.Standard; - using Raven.Abstractions.Indexing; - using Raven.Client.Indexes; using ServiceControl.MessageAuditing; using ServiceControl.MessageFailures; using ServiceControl.Operations; + using Raven.Client.Documents.Indexes; class MessagesViewIndex : AbstractMultiMapIndexCreationTask { @@ -54,11 +53,9 @@ public MessagesViewIndex() ConversationId = (string)last.MessageMetadata["ConversationId"] }); - Index(x => x.Query, FieldIndexing.Analyzed); + Index(x => x.Query, FieldIndexing.Search); Analyze(x => x.Query, typeof(StandardAnalyzer).AssemblyQualifiedName); - - DisableInMemoryIndexing = true; } public class SortAndFilterOptions diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/QueueAddressIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/QueueAddressIndex.cs index 5a31449809..b106c9bdcb 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/QueueAddressIndex.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/QueueAddressIndex.cs @@ -1,7 +1,7 @@ namespace ServiceControl.MessageFailures.Api { using System.Linq; - using Raven.Client.Indexes; + using Raven.Client.Documents.Indexes; class QueueAddressIndex : AbstractIndexCreationTask { @@ -23,8 +23,6 @@ into g PhysicalAddress = g.Key, FailedMessageCount = g.Sum(m => m.FailedMessageCount) }; - - DisableInMemoryIndexing = true; } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/RetryBatches_ByStatusAndSession.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/RetryBatches_ByStatusAndSession.cs index 70ad6da482..f63a0d43aa 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/RetryBatches_ByStatusAndSession.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/RetryBatches_ByStatusAndSession.cs @@ -1,7 +1,7 @@ namespace ServiceControl.Persistence { using System.Linq; - using Raven.Client.Indexes; + using Raven.Client.Documents.Indexes; class RetryBatches_ByStatusAndSession : AbstractIndexCreationTask { @@ -13,8 +13,6 @@ public RetryBatches_ByStatusAndSession() doc.RetrySessionId, doc.Status }; - - DisableInMemoryIndexing = true; } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/RetryBatches_ByStatus_ReduceInitialBatchSize.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/RetryBatches_ByStatus_ReduceInitialBatchSize.cs index c20490177b..e8dd133c0d 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/RetryBatches_ByStatus_ReduceInitialBatchSize.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/RetryBatches_ByStatus_ReduceInitialBatchSize.cs @@ -1,7 +1,7 @@ namespace ServiceControl.Persistence { using System.Linq; - using Raven.Client.Indexes; + using Raven.Client.Documents.Indexes; class RetryBatches_ByStatus_ReduceInitialBatchSize : AbstractIndexCreationTask { @@ -40,8 +40,6 @@ into g Last = g.Max(x => x.Last), g.First().Classifier }; - - DisableInMemoryIndexing = true; } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/SagaDetailsIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/SagaDetailsIndex.cs index 36ecd37a53..82524479a3 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/SagaDetailsIndex.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/SagaDetailsIndex.cs @@ -1,7 +1,7 @@ namespace ServiceControl.SagaAudit { using System.Linq; - using Raven.Client.Indexes; + using Raven.Client.Documents.Indexes; class SagaDetailsIndex : AbstractMultiMapIndexCreationTask { diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/SagaHistoryCleaner.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/SagaHistoryCleaner.cs index e0b6782807..019ce34398 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/SagaHistoryCleaner.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/SagaHistoryCleaner.cs @@ -10,6 +10,9 @@ using Raven.Abstractions.Commands; using Raven.Abstractions.Data; using Raven.Abstractions.Exceptions; + using Raven.Client.Documents.Commands.Batches; + using Raven.Client.Documents.Queries; + using Raven.Client.Util; using Raven.Database; using ServiceControl.Infrastructure.RavenDB; diff --git a/src/ServiceControl.Persistence.RavenDb5/QueueAddressStore.cs b/src/ServiceControl.Persistence.RavenDb5/QueueAddressStore.cs index c951bb94d7..bb8209c05b 100644 --- a/src/ServiceControl.Persistence.RavenDb5/QueueAddressStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/QueueAddressStore.cs @@ -3,11 +3,12 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; - using Raven.Client; - using Raven.Client.Linq; using ServiceControl.MessageFailures; using ServiceControl.MessageFailures.Api; using ServiceControl.Persistence.Infrastructure; + using Raven.Client; + using Raven.Client.Documents; + using Raven.Client.Linq; class QueueAddressStore : IQueueAddressStore { diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs b/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs index 8fdc936d17..c60956ab96 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs @@ -3,10 +3,11 @@ using System; using System.IO; using System.Threading.Tasks; - using Raven.Client; - using Raven.Json.Linq; using ServiceControl.CompositeViews.Messages; using ServiceControl.Persistence; + using Raven.Client; + using Raven.Client.Documents; + using Raven.Json.Linq; // For Raven5, look at how the Audit instance is implementing this, as Attachments won't exist // and there will be no need for a fallback method on a new persistence diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs index 483e3b3f01..ff39aef332 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs @@ -4,11 +4,13 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; - using Raven.Client; - using Raven.Client.Linq; using ServiceControl.Contracts.CustomChecks; using ServiceControl.Persistence; using ServiceControl.Persistence.Infrastructure; + using Raven.Client; + using Raven.Client.Documents; + using Raven.Client.Documents.Linq; + using Raven.Client.Linq; class RavenDbCustomCheckDataStore : ICustomChecksDataStore { diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbInstaller.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbInstaller.cs index c940a0448d..867329669e 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbInstaller.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbInstaller.cs @@ -3,8 +3,8 @@ using System.Threading; using System.Threading.Tasks; using NServiceBus.Logging; - using Raven.Client.Embedded; using ServiceControl.Infrastructure.RavenDB; + using Raven.Client.Embedded; class RavenDbInstaller : IPersistenceInstaller { diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbMonitoringDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbMonitoringDataStore.cs index f04a89718b..e95fe73d23 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbMonitoringDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbMonitoringDataStore.cs @@ -4,9 +4,10 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; - using Raven.Client; using ServiceControl.Operations; using ServiceControl.Persistence; + using Raven.Client; + using Raven.Client.Documents; class RavenDbMonitoringDataStore : IMonitoringDataStore { diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs index bab32ac5bd..b2a1084356 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs @@ -3,8 +3,6 @@ using MessageRedirects; using Microsoft.Extensions.DependencyInjection; using Persistence.Recoverability; - using Raven.Client; - using Raven.Client.Embedded; using Recoverability; using ServiceControl.CustomChecks; using ServiceControl.Infrastructure.RavenDB.Subscriptions; @@ -16,6 +14,9 @@ using ServiceControl.Persistence.RavenDb.SagaAudit; using ServiceControl.Persistence.UnitOfWork; using ServiceControl.Recoverability; + using Raven.Client; + using Raven.Client.Documents; + using Raven.Client.Embedded; class RavenDbPersistence : IPersistence { diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs index 640e732654..2e6ec8bc3b 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs @@ -1,9 +1,9 @@ namespace ServiceControl.Persistence.RavenDb { using System; - using Raven.Client.Embedded; using ServiceControl.Infrastructure.RavenDB.Expiration; using ServiceControl.Operations; + using Raven.Client.Embedded; class RavenDbPersistenceConfiguration : IPersistenceConfiguration { diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceLifecycle.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceLifecycle.cs index 66c4b3af3a..73fe1be23f 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceLifecycle.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceLifecycle.cs @@ -4,9 +4,10 @@ using System.Threading; using System.Threading.Tasks; using NServiceBus.Logging; + using ServiceControl.Persistence; using Raven.Client; + using Raven.Client.Documents; using Raven.Client.Embedded; - using ServiceControl.Persistence; class RavenDbPersistenceLifecycle : IPersistenceLifecycle { diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs b/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs index 94c4897bb2..e56b2451a9 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs @@ -7,10 +7,12 @@ namespace ServiceControl.Persistence using System.Linq.Expressions; using System.Net.Http; using System.Text; - using Raven.Client; - using Raven.Client.Linq; using ServiceControl.MessageFailures; using ServiceControl.Persistence.Infrastructure; + using Raven.Client; + using Raven.Client.Documents.Linq; + using Raven.Client.Documents.Session; + using Raven.Client.Linq; static class RavenQueryExtensions { diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenStartup.cs b/src/ServiceControl.Persistence.RavenDb5/RavenStartup.cs index 7e4c22131c..8992e67346 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenStartup.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenStartup.cs @@ -3,6 +3,8 @@ using System.Threading.Tasks; using NServiceBus.Logging; using Raven.Client; + using Raven.Client.Documents; + using Raven.Client.Documents.Indexes; using Raven.Client.Indexes; class RavenStartup diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveDocumentManager.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveDocumentManager.cs index 2ee549bf30..a00efefafe 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveDocumentManager.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveDocumentManager.cs @@ -5,9 +5,10 @@ using System.Linq; using System.Threading.Tasks; using MessageFailures; - using Raven.Abstractions.Commands; - using Raven.Abstractions.Data; - using Raven.Client; + using Raven.Client.Documents; + using Raven.Client.Documents.Commands.Batches; + using Raven.Client.Documents.Operations; + using Raven.Client.Documents.Session; class UnarchiveDocumentManager { diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/GroupsDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/GroupsDataStore.cs index 2df22233aa..217eba1bcd 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Recoverability/GroupsDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/GroupsDataStore.cs @@ -4,10 +4,12 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; - using Raven.Client; - using Raven.Client.Linq; using ServiceControl.MessageFailures; using ServiceControl.Recoverability; + using Raven.Client; + using Raven.Client.Documents; + using Raven.Client.Documents.Linq; + using Raven.Client.Linq; class GroupsDataStore : IGroupsDataStore { diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/RetryHistoryDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/RetryHistoryDataStore.cs index e25305d18e..fed4187a33 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Recoverability/RetryHistoryDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/RetryHistoryDataStore.cs @@ -2,8 +2,8 @@ { using System; using System.Threading.Tasks; - using Raven.Client; using ServiceControl.Recoverability; + using Raven.Client.Documents; class RetryHistoryDataStore : IRetryHistoryDataStore { diff --git a/src/ServiceControl.Persistence.RavenDb5/RetryBatchesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RetryBatchesDataStore.cs index c3c125de4f..5e04f2f4a1 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RetryBatchesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RetryBatchesDataStore.cs @@ -5,11 +5,15 @@ using System.Threading.Tasks; using MessageFailures; using NServiceBus.Logging; + using ServiceControl.Recoverability; using Raven.Abstractions.Commands; using Raven.Abstractions.Data; using Raven.Abstractions.Exceptions; using Raven.Client; - using ServiceControl.Recoverability; + using Raven.Client.Documents; + using Raven.Client.Documents.Commands.Batches; + using Raven.Client.Documents.Operations; + using Raven.Client.Exceptions; class RetryBatchesDataStore : IRetryBatchesDataStore { diff --git a/src/ServiceControl.Persistence.RavenDb5/RetryBatchesManager.cs b/src/ServiceControl.Persistence.RavenDb5/RetryBatchesManager.cs index 15edba340e..94595b5c0f 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RetryBatchesManager.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RetryBatchesManager.cs @@ -6,8 +6,9 @@ using System.Threading.Tasks; using MessageFailures; using Persistence.MessageRedirects; - using Raven.Client; using ServiceControl.Recoverability; + using Raven.Client; + using Raven.Client.Documents.Session; class RetryBatchesManager : AbstractSessionManager, IRetryBatchesManager { diff --git a/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs index 582dde65a6..19621322f8 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs @@ -12,6 +12,13 @@ using Raven.Abstractions.Data; using Raven.Abstractions.Exceptions; using Raven.Client; + using Raven.Client.Documents; + using Raven.Client.Documents.Commands; + using Raven.Client.Documents.Commands.Batches; + using Raven.Client.Documents.Indexes; + using Raven.Client.Documents.Operations; + using Raven.Client.Documents.Session; + using Raven.Client.Exceptions; using Raven.Client.Indexes; using Raven.Client.Linq; using Raven.Json.Linq; diff --git a/src/ServiceControl.Persistence.RavenDb5/SagaAudit/AuditRetentionCustomCheck.cs b/src/ServiceControl.Persistence.RavenDb5/SagaAudit/AuditRetentionCustomCheck.cs index e671a462df..d4d2155f11 100644 --- a/src/ServiceControl.Persistence.RavenDb5/SagaAudit/AuditRetentionCustomCheck.cs +++ b/src/ServiceControl.Persistence.RavenDb5/SagaAudit/AuditRetentionCustomCheck.cs @@ -4,8 +4,8 @@ namespace ServiceControl.Persistence.RavenDb.SagaAudit using System; using System.Threading.Tasks; using NServiceBus.CustomChecks; - using Raven.Client; using ServiceControl.SagaAudit; + using Raven.Client.Documents; // This custom check stays in the Raven3.5 persister becuase the Raven5 persister will not store audit data class AuditRetentionCustomCheck : CustomCheck diff --git a/src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaAuditDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaAuditDataStore.cs index 69af8b1da6..eb4732a373 100644 --- a/src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaAuditDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaAuditDataStore.cs @@ -2,9 +2,10 @@ { using System; using System.Threading.Tasks; - using Raven.Client; using ServiceControl.Persistence.Infrastructure; using ServiceControl.SagaAudit; + using Raven.Client; + using Raven.Client.Documents; class SagaAuditDataStore : ISagaAuditDataStore { diff --git a/src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaListIndex.cs b/src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaListIndex.cs index 73961a8b79..50ac2e4533 100644 --- a/src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaListIndex.cs +++ b/src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaListIndex.cs @@ -2,7 +2,7 @@ namespace ServiceControl.SagaAudit { using System; using System.Linq; - using Raven.Client.Indexes; + using Raven.Client.Documents.Indexes; class SagaListIndex : AbstractMultiMapIndexCreationTask { diff --git a/src/ServiceControl.Persistence.RavenDb5/Transactions/RavenTransactionalDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/Transactions/RavenTransactionalDataStore.cs index caa69201b3..f67b84c65c 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Transactions/RavenTransactionalDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Transactions/RavenTransactionalDataStore.cs @@ -2,6 +2,7 @@ { using System.Threading.Tasks; using Raven.Client; + using Raven.Client.Documents.Session; abstract class AbstractSessionManager : IDataSessionManager { diff --git a/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs b/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs index 157a7ce15e..a22983e853 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs @@ -4,7 +4,7 @@ using System.Linq; using Raven.Client.Indexes; - class FailedMessageViewTransformer : AbstractTransformerCreationTask + class FailedMessageViewTransformer : AbstractTransformerCreationTask // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers { public FailedMessageViewTransformer() { diff --git a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesBodyTransformer.cs b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesBodyTransformer.cs index 18a53ae86b..71c4a182bf 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesBodyTransformer.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesBodyTransformer.cs @@ -5,7 +5,7 @@ namespace ServiceControl.CompositeViews.Messages using Raven.Client.Indexes; using ServiceControl.MessageFailures; - class MessagesBodyTransformer : AbstractTransformerCreationTask + class MessagesBodyTransformer : AbstractTransformerCreationTask // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers { public MessagesBodyTransformer() { diff --git a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs index 98047d961b..f7889158d4 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs @@ -7,7 +7,7 @@ namespace ServiceControl.CompositeViews.Messages using Raven.Client.Indexes; using ServiceControl.Persistence; - class MessagesViewTransformer : AbstractTransformerCreationTask + class MessagesViewTransformer : AbstractTransformerCreationTask // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers { public MessagesViewTransformer() { diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs index d6b4d9bcaa..d0822a52a6 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs @@ -2,9 +2,11 @@ { using System.Collections.Concurrent; using System.Threading.Tasks; + using ServiceControl.Persistence.UnitOfWork; using Raven.Abstractions.Commands; using Raven.Client; - using ServiceControl.Persistence.UnitOfWork; + using Raven.Client.Documents; + using Raven.Client.Documents.Commands.Batches; class RavenDbIngestionUnitOfWork : IngestionUnitOfWorkBase { diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWorkFactory.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWorkFactory.cs index 3fe030ec17..60d64a8e2c 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWorkFactory.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWorkFactory.cs @@ -1,8 +1,9 @@ namespace ServiceControl.Persistence.RavenDb { using System.Threading.Tasks; - using Raven.Client; using ServiceControl.Persistence.UnitOfWork; + using Raven.Client; + using Raven.Client.Documents; class RavenDbIngestionUnitOfWorkFactory : IIngestionUnitOfWorkFactory { diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs index f3ad877561..63f8983879 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs @@ -1,9 +1,10 @@ namespace ServiceControl.Persistence.RavenDb { using System.Threading.Tasks; + using ServiceControl.Persistence.UnitOfWork; using Raven.Abstractions.Commands; + using Raven.Client.Documents.Commands.Batches; using Raven.Json.Linq; - using ServiceControl.Persistence.UnitOfWork; class RavenDbMonitoringIngestionUnitOfWork : IMonitoringIngestionUnitOfWork { diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs index 6ec853c166..2bb6817489 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs @@ -2,14 +2,16 @@ { using System.Collections.Generic; using System.Threading.Tasks; + using ServiceControl.MessageFailures; + using ServiceControl.Persistence.UnitOfWork; + using ServiceControl.Recoverability; using Raven.Abstractions.Commands; using Raven.Abstractions.Data; using Raven.Abstractions.Extensions; + using Raven.Client.Documents.Commands.Batches; + using Raven.Client.Documents.Operations; using Raven.Imports.Newtonsoft.Json; using Raven.Json.Linq; - using ServiceControl.MessageFailures; - using ServiceControl.Persistence.UnitOfWork; - using ServiceControl.Recoverability; class RavenDbRecoverabilityIngestionUnitOfWork : IRecoverabilityIngestionUnitOfWork { From 6f18f1aa070a3fd460e226abf03e991cf32a6a00 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 6 Sep 2023 15:45:21 +0200 Subject: [PATCH 04/78] Raven.Imports.Newtonsoft.Json replaced with Newtonsoft.Json.Linq --- .../ErrorMessagesDataStore.cs | 4 ++-- .../Expiration/AuditMessageCleaner.cs | 5 ++--- .../Expiration/ErrorMessageCleaner.cs | 2 -- src/ServiceControl.Persistence.RavenDb5/Extensions.cs | 4 ++-- .../FailedMessages/FailedMessageReclassifier.cs | 4 ++-- .../Infrastructure/MigratedTypeAwareBinder.cs | 2 +- .../Infrastructure/Subscriptions/LegacyAddress.cs | 5 ++--- .../Subscriptions/MessageTypeConverter.cs | 2 +- .../Infrastructure/Subscriptions/Subscription.cs | 2 +- .../SubscriptionsLegacyAddressConverter.cs | 10 +++++----- .../RavenAttachmentsBodyStorage.cs | 4 ++-- .../RetryDocumentDataStore.cs | 5 ++--- .../UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs | 8 ++++---- .../RavenDbRecoverabilityIngestionUnitOfWork.cs | 8 ++++---- 14 files changed, 30 insertions(+), 35 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index 25f1afce5a..df9dd46ea5 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -8,6 +8,7 @@ using System.Threading.Tasks; using CompositeViews.Messages; using Editing; + using Newtonsoft.Json.Linq; using NServiceBus.Logging; using ServiceControl.EventLog; using ServiceControl.MessageFailures; @@ -22,7 +23,6 @@ using Raven.Client.Documents.Queries; using Raven.Client.Documents.Session; using Raven.Client.Linq; - using Raven.Json.Linq; class ErrorMessagesDataStore : IErrorMessageDataStore { @@ -703,7 +703,7 @@ public async Task FetchFromFailedMessage(string uniqueMessageId) var results = await documentStore.AsyncDatabaseCommands.GetAsync(new[] { documentId }, null, transformer: MessagesBodyTransformer.Name); - string resultBody = ((results.Results?.SingleOrDefault()?["$values"] as RavenJArray)?.SingleOrDefault() as RavenJObject) + string resultBody = ((results.Results?.SingleOrDefault()?["$values"] as JArray)?.SingleOrDefault() as JObject) ?.ToObject()?.Body; if (resultBody != null) diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/AuditMessageCleaner.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/AuditMessageCleaner.cs index ca29378314..43a303e6ac 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Expiration/AuditMessageCleaner.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Expiration/AuditMessageCleaner.cs @@ -5,6 +5,7 @@ using System.Diagnostics; using System.Linq; using System.Threading; + using Newtonsoft.Json.Linq; using NServiceBus.Logging; using Raven.Client.Documents.Commands.Batches; using Raven.Client.Documents.Queries; @@ -110,9 +111,7 @@ public static void Clean(int deletionBatchSize, DocumentDatabase database, DateT for (var idx = s; idx <= e; idx++) { //We want to continue using attachments for now -#pragma warning disable 618 accessor.Attachments.DeleteAttachment(att[idx], null); -#pragma warning restore 618 deleted++; } }); @@ -140,7 +139,7 @@ public static void Clean(int deletionBatchSize, DocumentDatabase database, DateT } } - static bool TryGetBodyId(RavenJObject doc, out string bodyId) + static bool TryGetBodyId(JObject doc, out string bodyId) { bodyId = null; if (doc.Value("MessageMetadata.BodyNotStored")) diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/ErrorMessageCleaner.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/ErrorMessageCleaner.cs index cd73810cdf..20581adc19 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Expiration/ErrorMessageCleaner.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Expiration/ErrorMessageCleaner.cs @@ -121,9 +121,7 @@ public static void Clean(int deletionBatchSize, DocumentDatabase database, DateT for (var idx = s; idx <= e; idx++) { //We want to continue using attachments for now -#pragma warning disable 618 accessor.Attachments.DeleteAttachment("messagebodies/" + attachments[idx], null); -#pragma warning restore 618 deleted++; } }); diff --git a/src/ServiceControl.Persistence.RavenDb5/Extensions.cs b/src/ServiceControl.Persistence.RavenDb5/Extensions.cs index e328daf39e..414d93e13e 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Extensions.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Extensions.cs @@ -2,14 +2,14 @@ { using System; using System.Threading; + using Newtonsoft.Json.Linq; using Raven.Abstractions.Data; using Raven.Client.Documents.Queries; using Raven.Database; - using Raven.Json.Linq; static class Extensions { - public static void Query(this DocumentDatabase db, string index, IndexQuery query, Action onItem, TState state, CancellationToken cancellationToken = default) + public static void Query(this DocumentDatabase db, string index, IndexQuery query, Action onItem, TState state, CancellationToken cancellationToken = default) { var results = db.Queries.Query(index, query, cancellationToken); foreach (var doc in results.Results) diff --git a/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs b/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs index 97c52c1459..2e25478993 100644 --- a/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs +++ b/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs @@ -6,6 +6,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Hosting; + using Newtonsoft.Json.Linq; using NServiceBus.Logging; using ServiceControl.MessageFailures; using ServiceControl.MessageFailures.Api; @@ -17,7 +18,6 @@ using Raven.Client.Documents; using Raven.Client.Documents.Operations; using Raven.Client.Exceptions; - using Raven.Json.Linq; class FailedMessageReclassifier : IReclassifyFailedMessages { @@ -103,7 +103,7 @@ int ReclassifyBatch(IDocumentStore store, IEnumerable { - var failureGroups = GetClassificationGroups(doc.Item2, classifiers).Select(RavenJObject.FromObject); + var failureGroups = GetClassificationGroups(doc.Item2, classifiers).Select(JObject.FromObject); try { diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/MigratedTypeAwareBinder.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/MigratedTypeAwareBinder.cs index 0dcaa04fd0..1ea5d67207 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/MigratedTypeAwareBinder.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/MigratedTypeAwareBinder.cs @@ -2,7 +2,7 @@ { using System; using System.Linq; - using Raven.Imports.Newtonsoft.Json.Serialization; + using Newtonsoft.Json.Serialization; using SagaAudit; using ServiceControl.Contracts.CustomChecks; using ServiceControl.Contracts.Operations; diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/LegacyAddress.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/LegacyAddress.cs index 81e3f1677b..026bb13933 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/LegacyAddress.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/LegacyAddress.cs @@ -2,8 +2,7 @@ { using System; using System.Collections.Generic; - using Raven.Imports.Newtonsoft.Json.Linq; - using Raven.Json.Linq; + using Newtonsoft.Json.Linq; class LegacyAddress { @@ -54,7 +53,7 @@ public static LegacyAddress ConvertToLegacyAddress(SubscriptionClient client) }; } - public static string ParseToString(Func tokenSelector) + public static string ParseToString(Func tokenSelector) { var token = tokenSelector(); diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/MessageTypeConverter.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/MessageTypeConverter.cs index 2805858866..9cc3ea9d0b 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/MessageTypeConverter.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/MessageTypeConverter.cs @@ -1,8 +1,8 @@ namespace ServiceControl.Infrastructure.RavenDB.Subscriptions { using System; + using Newtonsoft.Json; using NServiceBus.Unicast.Subscriptions; - using Raven.Imports.Newtonsoft.Json; class MessageTypeConverter : JsonConverter { diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/Subscription.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/Subscription.cs index 542d7e6bf0..f5feba6157 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/Subscription.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/Subscription.cs @@ -1,8 +1,8 @@ namespace ServiceControl.Infrastructure.RavenDB.Subscriptions { using System.Collections.Generic; + using Newtonsoft.Json; using NServiceBus.Unicast.Subscriptions; - using Raven.Imports.Newtonsoft.Json; class Subscription { diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/SubscriptionsLegacyAddressConverter.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/SubscriptionsLegacyAddressConverter.cs index 8b6f7cb5c9..67b80e2502 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/SubscriptionsLegacyAddressConverter.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/SubscriptionsLegacyAddressConverter.cs @@ -1,12 +1,12 @@ namespace ServiceControl.Infrastructure.RavenDB.Subscriptions { using System.Linq; + using Newtonsoft.Json.Linq; using Raven.Client.Listeners; - using Raven.Json.Linq; class SubscriptionsLegacyAddressConverter : IDocumentConversionListener { - public void BeforeConversionToDocument(string key, object entity, RavenJObject metadata) + public void BeforeConversionToDocument(string key, object entity, JObject metadata) { if (!(entity is Subscription subscription)) { @@ -18,15 +18,15 @@ public void BeforeConversionToDocument(string key, object entity, RavenJObject m subscription.LegacySubscriptions.AddRange(converted); } - public void AfterConversionToDocument(string key, object entity, RavenJObject document, RavenJObject metadata) + public void AfterConversionToDocument(string key, object entity, JObject document, JObject metadata) { } - public void BeforeConversionToEntity(string key, RavenJObject document, RavenJObject metadata) + public void BeforeConversionToEntity(string key, JObject document, JObject metadata) { } - public void AfterConversionToEntity(string key, RavenJObject document, RavenJObject metadata, object entity) + public void AfterConversionToEntity(string key, JObject document, JObject metadata, object entity) { if (!(entity is Subscription subscription)) { diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs b/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs index c60956ab96..e167eb510f 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs @@ -3,11 +3,11 @@ using System; using System.IO; using System.Threading.Tasks; + using Newtonsoft.Json.Linq; using ServiceControl.CompositeViews.Messages; using ServiceControl.Persistence; using Raven.Client; using Raven.Client.Documents; - using Raven.Json.Linq; // For Raven5, look at how the Audit instance is implementing this, as Attachments won't exist // and there will be no need for a fallback method on a new persistence @@ -26,7 +26,7 @@ public Task Store(string bodyId, string contentType, int bodySize, Stream bodySt //We want to continue using attachments for now #pragma warning disable 618 return documentStore.AsyncDatabaseCommands.PutAttachmentAsync(id, null, bodyStream, - new RavenJObject + new JObject #pragma warning restore 618 { {"ContentType", contentType}, diff --git a/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs index 19621322f8..488bd393ff 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs @@ -6,6 +6,7 @@ using System.Linq.Expressions; using System.Threading.Tasks; using MessageFailures; + using Newtonsoft.Json.Linq; using NServiceBus.Logging; using Persistence.Infrastructure; using Raven.Abstractions.Commands; @@ -19,9 +20,7 @@ using Raven.Client.Documents.Operations; using Raven.Client.Documents.Session; using Raven.Client.Exceptions; - using Raven.Client.Indexes; using Raven.Client.Linq; - using Raven.Json.Linq; using ServiceControl.MessageFailures.Api; using ServiceControl.Recoverability; @@ -148,7 +147,7 @@ static ICommandData CreateFailedMessageRetryDocument(string batchDocumentId, str }; } - static RavenJObject DefaultMetadata = RavenJObject.Parse($@" + static JObject DefaultMetadata = JObject.Parse($@" {{ ""Raven-Entity-Name"": ""{FailedMessageRetry.CollectionName}"", ""Raven-Clr-Type"": ""{typeof(FailedMessageRetry).AssemblyQualifiedName}"" diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs index 63f8983879..006136e469 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs @@ -1,10 +1,10 @@ namespace ServiceControl.Persistence.RavenDb { using System.Threading.Tasks; + using Newtonsoft.Json.Linq; using ServiceControl.Persistence.UnitOfWork; using Raven.Abstractions.Commands; using Raven.Client.Documents.Commands.Batches; - using Raven.Json.Linq; class RavenDbMonitoringIngestionUnitOfWork : IMonitoringIngestionUnitOfWork { @@ -23,7 +23,7 @@ public Task RecordKnownEndpoint(KnownEndpoint knownEndpoint) static PutCommandData CreateKnownEndpointsPutCommand(KnownEndpoint endpoint) => new PutCommandData { - Document = RavenJObject.FromObject(endpoint), + Document = JObject.FromObject(endpoint), Etag = null, Key = endpoint.Id.ToString(), Metadata = KnownEndpointMetadata @@ -31,13 +31,13 @@ public Task RecordKnownEndpoint(KnownEndpoint knownEndpoint) static RavenDbMonitoringIngestionUnitOfWork() { - KnownEndpointMetadata = RavenJObject.Parse($@" + KnownEndpointMetadata = JObject.Parse($@" {{ ""Raven-Entity-Name"": ""{KnownEndpoint.CollectionName}"", ""Raven-Clr-Type"": ""{typeof(KnownEndpoint).AssemblyQualifiedName}"" }}"); } - static readonly RavenJObject KnownEndpointMetadata; + static readonly JObject KnownEndpointMetadata; } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs index 2bb6817489..5d84f2f492 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs @@ -2,6 +2,8 @@ { using System.Collections.Generic; using System.Threading.Tasks; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; using ServiceControl.MessageFailures; using ServiceControl.Persistence.UnitOfWork; using ServiceControl.Recoverability; @@ -10,8 +12,6 @@ using Raven.Abstractions.Extensions; using Raven.Client.Documents.Commands.Batches; using Raven.Client.Documents.Operations; - using Raven.Imports.Newtonsoft.Json; - using Raven.Json.Linq; class RavenDbRecoverabilityIngestionUnitOfWork : IRecoverabilityIngestionUnitOfWork { @@ -125,7 +125,7 @@ static RavenDbRecoverabilityIngestionUnitOfWork() Serializer = JsonExtensions.CreateDefaultJsonSerializer(); Serializer.TypeNameHandling = TypeNameHandling.Auto; - FailedMessageMetadata = RavenJObject.Parse($@" + FailedMessageMetadata = JObject.Parse($@" {{ ""Raven-Entity-Name"": ""{FailedMessageIdGenerator.CollectionName}"", ""Raven-Clr-Type"": ""{typeof(FailedMessage).AssemblyQualifiedName}"" @@ -133,7 +133,7 @@ static RavenDbRecoverabilityIngestionUnitOfWork() } static int MaxProcessingAttempts = 10; - static readonly RavenJObject FailedMessageMetadata; + static readonly JObject FailedMessageMetadata; static readonly JsonSerializer Serializer; } } \ No newline at end of file From 5aa4723895b644b555a31496563e795666247fc2 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 6 Sep 2023 15:53:12 +0200 Subject: [PATCH 05/78] More namespaces --- .../FailedMessages_UniqueMessageIdAndTimeOfFailures.cs | 1 - .../ErrorMessagesDataStore.cs | 3 --- .../Expiration/EventLogItemsCleaner.cs | 8 +++----- .../Expiration/ExpiredDocumentsCleaner.cs | 3 +-- src/ServiceControl.Persistence.RavenDb5/Extensions.cs | 2 -- .../ExternalIntegrationRequestsDataStore.cs | 3 --- .../FailedMessages/FailedMessageReclassifier.cs | 4 +--- .../Indexes/SagaHistoryCleaner.cs | 5 ----- ...pointsWithTemporaryIdsThatAreDuplicateDataMigration.cs | 2 +- .../RavenBootstrapper.cs | 3 --- src/ServiceControl.Persistence.RavenDb5/RavenStartup.cs | 1 - .../Recoverability/Archiving/ArchiveDocumentManager.cs | 7 ++++--- .../RetryBatchesDataStore.cs | 3 --- .../RetryDocumentDataStore.cs | 4 ---- .../Transformers/FailedMessageViewTransformer.cs | 1 - .../Transformers/MessagesBodyTransformer.cs | 1 - .../Transformers/MessagesViewTransformer.cs | 1 - .../UnitOfWork/RavenDbIngestionUnitOfWork.cs | 1 - .../UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs | 1 - .../RavenDbRecoverabilityIngestionUnitOfWork.cs | 3 --- 20 files changed, 10 insertions(+), 47 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/BulkRetry/FailedMessages_UniqueMessageIdAndTimeOfFailures.cs b/src/ServiceControl.Persistence.RavenDb5/BulkRetry/FailedMessages_UniqueMessageIdAndTimeOfFailures.cs index fbad94325f..c06a7b3a6e 100644 --- a/src/ServiceControl.Persistence.RavenDb5/BulkRetry/FailedMessages_UniqueMessageIdAndTimeOfFailures.cs +++ b/src/ServiceControl.Persistence.RavenDb5/BulkRetry/FailedMessages_UniqueMessageIdAndTimeOfFailures.cs @@ -3,7 +3,6 @@ namespace ServiceControl.Recoverability using System; using System.Linq; using MessageFailures; - using Raven.Client.Indexes; class FailedMessages_UniqueMessageIdAndTimeOfFailures : AbstractTransformerCreationTask // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers { diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index df9dd46ea5..abaece8b1f 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -16,13 +16,10 @@ using ServiceControl.Operations; using ServiceControl.Persistence.Infrastructure; using ServiceControl.Recoverability; - using Raven.Abstractions.Data; - using Raven.Abstractions.Extensions; using Raven.Client; using Raven.Client.Documents; using Raven.Client.Documents.Queries; using Raven.Client.Documents.Session; - using Raven.Client.Linq; class ErrorMessagesDataStore : IErrorMessageDataStore { diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/EventLogItemsCleaner.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/EventLogItemsCleaner.cs index 3e037c35cb..8c4f4cf1ea 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Expiration/EventLogItemsCleaner.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Expiration/EventLogItemsCleaner.cs @@ -6,11 +6,9 @@ using System.Linq; using System.Threading; using NServiceBus.Logging; - using Raven.Abstractions; - using Raven.Abstractions.Commands; - using Raven.Abstractions.Data; - using Raven.Abstractions.Exceptions; - using Raven.Database; + using Raven.Client.Documents.Commands.Batches; + using Raven.Client.Documents.Queries; + using Raven.Client.Util; static class EventLogItemsCleaner { diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiredDocumentsCleaner.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiredDocumentsCleaner.cs index ec610a1bc1..459f69b4ef 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiredDocumentsCleaner.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiredDocumentsCleaner.cs @@ -5,8 +5,7 @@ using System.Threading.Tasks; using BackgroundTasks; using NServiceBus.Logging; - using Raven.Abstractions; - using Raven.Database; + using Raven.Client.Util; using SagaAudit; class ExpiredDocumentsCleaner diff --git a/src/ServiceControl.Persistence.RavenDb5/Extensions.cs b/src/ServiceControl.Persistence.RavenDb5/Extensions.cs index 414d93e13e..aaab921183 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Extensions.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Extensions.cs @@ -3,9 +3,7 @@ using System; using System.Threading; using Newtonsoft.Json.Linq; - using Raven.Abstractions.Data; using Raven.Client.Documents.Queries; - using Raven.Database; static class Extensions { diff --git a/src/ServiceControl.Persistence.RavenDb5/ExternalIntegrationRequestsDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ExternalIntegrationRequestsDataStore.cs index c9af28741b..ad7685e6cd 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ExternalIntegrationRequestsDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ExternalIntegrationRequestsDataStore.cs @@ -11,11 +11,8 @@ using NServiceBus; using NServiceBus.Logging; using ServiceBus.Management.Infrastructure.Extensions; - using Raven.Abstractions.Data; - using Raven.Client; using Raven.Client.Documents; using Raven.Client.Documents.Changes; - using Raven.Client.Linq; class ExternalIntegrationRequestsDataStore : IExternalIntegrationRequestsDataStore diff --git a/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs b/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs index 2e25478993..960fed6644 100644 --- a/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs +++ b/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs @@ -12,8 +12,6 @@ using ServiceControl.MessageFailures.Api; using ServiceControl.Persistence.Infrastructure; using ServiceControl.Recoverability; - using Raven.Abstractions.Data; - using Raven.Abstractions.Exceptions; using Raven.Client; using Raven.Client.Documents; using Raven.Client.Documents.Operations; @@ -114,7 +112,7 @@ int ReclassifyBatch(IDocumentStore store, IEnumerable // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers { diff --git a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesBodyTransformer.cs b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesBodyTransformer.cs index 71c4a182bf..717a0294a7 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesBodyTransformer.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesBodyTransformer.cs @@ -2,7 +2,6 @@ namespace ServiceControl.CompositeViews.Messages { using System.Collections.Generic; using System.Linq; - using Raven.Client.Indexes; using ServiceControl.MessageFailures; class MessagesBodyTransformer : AbstractTransformerCreationTask // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers diff --git a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs index f7889158d4..733fca57fa 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs @@ -4,7 +4,6 @@ namespace ServiceControl.CompositeViews.Messages using System.Collections.Generic; using System.Linq; using MessageFailures; - using Raven.Client.Indexes; using ServiceControl.Persistence; class MessagesViewTransformer : AbstractTransformerCreationTask // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs index d0822a52a6..09002e08de 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs @@ -3,7 +3,6 @@ using System.Collections.Concurrent; using System.Threading.Tasks; using ServiceControl.Persistence.UnitOfWork; - using Raven.Abstractions.Commands; using Raven.Client; using Raven.Client.Documents; using Raven.Client.Documents.Commands.Batches; diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs index 006136e469..2a46dec4ca 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs @@ -3,7 +3,6 @@ using System.Threading.Tasks; using Newtonsoft.Json.Linq; using ServiceControl.Persistence.UnitOfWork; - using Raven.Abstractions.Commands; using Raven.Client.Documents.Commands.Batches; class RavenDbMonitoringIngestionUnitOfWork : IMonitoringIngestionUnitOfWork diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs index 5d84f2f492..0262508126 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs @@ -7,9 +7,6 @@ using ServiceControl.MessageFailures; using ServiceControl.Persistence.UnitOfWork; using ServiceControl.Recoverability; - using Raven.Abstractions.Commands; - using Raven.Abstractions.Data; - using Raven.Abstractions.Extensions; using Raven.Client.Documents.Commands.Batches; using Raven.Client.Documents.Operations; From ed0c11373e0f0d55e9932098a8d314318ad08822 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 6 Sep 2023 15:58:28 +0200 Subject: [PATCH 06/78] more namespaces --- src/ServiceControl.Persistence.RavenDb5/EventLogDataStore.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/EventLogDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/EventLogDataStore.cs index 88163218a1..1294346cf7 100644 --- a/src/ServiceControl.Persistence.RavenDb5/EventLogDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/EventLogDataStore.cs @@ -6,7 +6,6 @@ using Persistence.Infrastructure; using Raven.Client; using Raven.Client.Documents; - using Raven.Client.Linq; class EventLogDataStore : IEventLogDataStore { From 6b170083c384a671d6fbcbfdeb866499b79327dc Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 6 Sep 2023 15:58:46 +0200 Subject: [PATCH 07/78] RavenQueryStatistics --- .../Extensions/QueryResultConvert.cs | 4 ++-- .../RavenQueryStatisticsExtensions.cs | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/Extensions/QueryResultConvert.cs b/src/ServiceControl.Persistence.RavenDb5/Extensions/QueryResultConvert.cs index 9d47302837..ba7100e879 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Extensions/QueryResultConvert.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Extensions/QueryResultConvert.cs @@ -2,11 +2,11 @@ { using System.Collections.Generic; using Persistence.Infrastructure; - using Raven.Client; + using Raven.Client.Documents.Session; static class QueryResultConvert { - public static QueryResult> ToQueryResult(this IList result, RavenQueryStatistics stats) + public static QueryResult> ToQueryResult(this IList result, QueryStatistics stats) where T : class { return new QueryResult>(result, stats.ToQueryStatsInfo()); diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenQueryStatisticsExtensions.cs b/src/ServiceControl.Persistence.RavenDb5/RavenQueryStatisticsExtensions.cs index 9be4c7b5f4..b15fda923e 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenQueryStatisticsExtensions.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenQueryStatisticsExtensions.cs @@ -1,11 +1,12 @@ namespace ServiceControl.Persistence { using Raven.Client; + using Raven.Client.Documents.Session; using ServiceControl.Persistence.Infrastructure; static class RavenQueryStatisticsExtensions { - public static QueryStatsInfo ToQueryStatsInfo(this RavenQueryStatistics stats) + public static QueryStatsInfo ToQueryStatsInfo(this QueryStatistics stats) { return new QueryStatsInfo(stats.IndexEtag, stats.TotalResults, stats.IsStale); } From 91f73b4ecc707f844cb563f83489d6bd2bda48a3 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 6 Sep 2023 16:00:37 +0200 Subject: [PATCH 08/78] .As --- .../FailedMessages/FailedMessageReclassifier.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs b/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs index 960fed6644..f977542551 100644 --- a/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs +++ b/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs @@ -57,7 +57,7 @@ public async Task ReclassifyFailedMessages(bool force) var totalMessagesReclassified = 0; - using (var stream = await session.Advanced.StreamAsync(query.As()) + using (var stream = await session.Advanced.StreamAsync(query.OfType()) ) { while (!abort && await stream.MoveNextAsync()) From b80271d9fab2533d992a06e69bd95dd696e54ec3 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 6 Sep 2023 16:09:24 +0200 Subject: [PATCH 09/78] WhereBetween is now INCLUSIVE where previously it was EXCLUSIVE --- .../ErrorMessagesDataStore.cs | 4 ++-- .../RavenQueryExtensions.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index abaece8b1f..79442d4428 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -546,7 +546,7 @@ public async Task ProcessPendingRetries(DateTime periodFrom, DateTime periodTo, .AsyncDocumentQuery() .WhereEquals("Status", (int)FailedMessageStatus.RetryIssued) .AndAlso() - .WhereBetweenOrEqual("LastModified", periodFrom.Ticks, periodTo.Ticks); + .WhereBetween("LastModified", periodFrom.Ticks, periodTo.Ticks); if (!string.IsNullOrWhiteSpace(queueAddress)) { @@ -676,7 +676,7 @@ public async Task GetRetryPendingMessages(DateTime from, DateTime to, .AsyncDocumentQuery() .WhereEquals("Status", (int)FailedMessageStatus.RetryIssued) .AndAlso() - .WhereBetweenOrEqual(options => options.LastModified, from.Ticks, to.Ticks) + .WhereBetween(options => options.LastModified, from.Ticks, to.Ticks) .AndAlso() .WhereEquals(o => o.QueueAddress, queueAddress) .SetResultTransformer(FailedMessageViewTransformer.Name) diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs b/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs index e56b2451a9..215291ca86 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs @@ -228,7 +228,7 @@ public static IAsyncDocumentQuery FilterByLastModifiedRange(this IAsyncDoc var to = DateTime.Parse(filters[1], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); source.AndAlso(); - source.WhereBetweenOrEqual("LastModified", from.Ticks, to.Ticks); + source.WhereBetween("LastModified", from.Ticks, to.Ticks); } catch (Exception) { From d2687dd85f04e599492c472f7a3230e4f55f06af Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 6 Sep 2023 16:18:07 +0200 Subject: [PATCH 10/78] namespaces --- .../ErrorMessagesDataStore.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index 79442d4428..c72fc8add9 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -20,6 +20,7 @@ using Raven.Client.Documents; using Raven.Client.Documents.Queries; using Raven.Client.Documents.Session; + using Raven.Client.Documents.Queries.Facets; class ErrorMessagesDataStore : IErrorMessageDataStore { From 8644e4dace75f2232e666eea8dd269a48033446c Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 6 Sep 2023 16:33:16 +0200 Subject: [PATCH 11/78] Breaking change header meta data key changes --- .../ErrorMessagesDataStore.cs | 2 +- .../Expiration/AuditMessageCleaner.cs | 4 ++-- .../Expiration/ErrorMessageCleaner.cs | 4 ++-- .../Expiration/EventLogItemsCleaner.cs | 4 ++-- .../Expiration/ExpiryErrorMessageIndex.cs | 2 +- .../Expiration/ExpiryEventLogItemsIndex.cs | 2 +- .../Indexes/ExpirySagaAuditIndex.cs | 4 ++-- .../Indexes/FailedMessageViewIndex.cs | 2 +- .../Indexes/FailedMessages_ByGroup.cs | 2 +- .../Indexes/SagaHistoryCleaner.cs | 4 ++-- .../MessageRedirects/MessageRedirectsDataStore.cs | 6 +++--- .../RetryBatchesManager.cs | 2 +- .../RetryDocumentDataStore.cs | 2 +- .../Transformers/FailedMessageViewTransformer.cs | 2 +- .../UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs | 2 +- .../UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs | 2 +- 16 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index c72fc8add9..e3ca468597 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -408,7 +408,7 @@ static FailedMessageView Map(FailedMessage message, IAsyncDocumentSession sessio NumberOfProcessingAttempts = message.ProcessingAttempts.Count, Status = message.Status, TimeOfFailure = failureDetails.TimeOfFailure, - LastModified = session.Advanced.GetMetadataFor(message)["Last-Modified"].Value(), + LastModified = session.Advanced.GetMetadataFor(message)["@last-modified"].Value(), Edited = wasEdited, EditOf = wasEdited ? message.ProcessingAttempts.Last().Headers["ServiceControl.EditOf"] : "" }; diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/AuditMessageCleaner.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/AuditMessageCleaner.cs index 43a303e6ac..71f47bec22 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Expiration/AuditMessageCleaner.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Expiration/AuditMessageCleaner.cs @@ -32,7 +32,7 @@ public static void Clean(int deletionBatchSize, DocumentDatabase database, DateT Query = $"ProcessedAt:[* TO {expiryThreshold.Ticks}]", FieldsToFetch = new[] { - "__document_id", + "@id", "MessageMetadata.MessageId", "MessageMetadata.BodyNotStored" }, @@ -48,7 +48,7 @@ public static void Clean(int deletionBatchSize, DocumentDatabase database, DateT database.Query(indexName, query, (doc, state) => { - var id = doc.Value("__document_id"); + var id = doc.Value("@id"); if (string.IsNullOrEmpty(id)) { return; diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/ErrorMessageCleaner.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/ErrorMessageCleaner.cs index 20581adc19..863fba192a 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Expiration/ErrorMessageCleaner.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Expiration/ErrorMessageCleaner.cs @@ -40,7 +40,7 @@ public static void Clean(int deletionBatchSize, DocumentDatabase database, DateT Query = $"Status:[2 TO 4] AND LastModified:[* TO {expiryThreshold.Ticks}]", FieldsToFetch = new[] { - "__document_id", + "@id", "ProcessingAttempts[0].MessageId" }, SortedFields = new[] @@ -55,7 +55,7 @@ public static void Clean(int deletionBatchSize, DocumentDatabase database, DateT database.Query(indexName, query, (doc, state) => { - var id = doc.Value("__document_id"); + var id = doc.Value("@id"); if (string.IsNullOrEmpty(id)) { return; diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/EventLogItemsCleaner.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/EventLogItemsCleaner.cs index 8c4f4cf1ea..76beb5f8c0 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Expiration/EventLogItemsCleaner.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Expiration/EventLogItemsCleaner.cs @@ -29,7 +29,7 @@ public static void Clean(int deletionBatchSize, DocumentDatabase database, DateT Query = $"LastModified:[* TO {expiryThreshold.Ticks}]", FieldsToFetch = new[] { - "__document_id" + "@id" }, SortedFields = new[] { @@ -43,7 +43,7 @@ public static void Clean(int deletionBatchSize, DocumentDatabase database, DateT database.Query(indexName, query, (doc, commands) => { - var id = doc.Value("__document_id"); + var id = doc.Value("@id"); if (string.IsNullOrEmpty(id)) { return; diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryErrorMessageIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryErrorMessageIndex.cs index c56217b921..f75e41f1e7 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryErrorMessageIndex.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryErrorMessageIndex.cs @@ -14,7 +14,7 @@ public ExpiryErrorMessageIndex() select new { message.Status, - LastModified = MetadataFor(message).Value("Last-Modified").Ticks + LastModified = MetadataFor(message).Value("@last-modified").Ticks }; } } diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryEventLogItemsIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryEventLogItemsIndex.cs index 8ae371def3..72038d02ff 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryEventLogItemsIndex.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryEventLogItemsIndex.cs @@ -12,7 +12,7 @@ public ExpiryEventLogItemsIndex() Map = messages => from message in messages select new { - LastModified = MetadataFor(message).Value("Last-Modified").Ticks + LastModified = MetadataFor(message).Value("@last-modified").Ticks }; } } diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/ExpirySagaAuditIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/ExpirySagaAuditIndex.cs index 5c9ecff7ff..2677160995 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/ExpirySagaAuditIndex.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/ExpirySagaAuditIndex.cs @@ -11,13 +11,13 @@ public ExpirySagaAuditIndex() AddMap(messages => from message in messages select new { - LastModified = MetadataFor(message).Value("Last-Modified").Ticks + LastModified = MetadataFor(message).Value("@last-modified").Ticks }); AddMap(sagaHistories => from sagaHistory in sagaHistories select new { - LastModified = MetadataFor(sagaHistory).Value("Last-Modified").Ticks + LastModified = MetadataFor(sagaHistory).Value("@last-modified").Ticks }); } } diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageViewIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageViewIndex.cs index 81b4d2529b..b390b2c5cb 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageViewIndex.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageViewIndex.cs @@ -20,7 +20,7 @@ public FailedMessageViewIndex() ReceivingEndpointName = ((EndpointDetails)processingAttemptsLast.MessageMetadata["ReceivingEndpoint"]).Name, QueueAddress = processingAttemptsLast.FailureDetails.AddressOfFailingEndpoint, processingAttemptsLast.FailureDetails.TimeOfFailure, - LastModified = MetadataFor(message).Value("Last-Modified").Ticks + LastModified = MetadataFor(message).Value("@last-modified").Ticks }; } diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessages_ByGroup.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessages_ByGroup.cs index a6555f67b3..7fbe6207f4 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessages_ByGroup.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessages_ByGroup.cs @@ -22,7 +22,7 @@ from failureGroup in doc.FailureGroups MessageType = (string)processingAttemptsLast.MessageMetadata["MessageType"], TimeSent = (DateTime)processingAttemptsLast.MessageMetadata["TimeSent"], TimeOfFailure = processingAttemptsLast.FailureDetails.TimeOfFailure, - LastModified = MetadataFor(doc).Value("Last-Modified").Ticks + LastModified = MetadataFor(doc).Value("@last-modified").Ticks }; StoreAllFields(FieldStorage.Yes); diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/SagaHistoryCleaner.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/SagaHistoryCleaner.cs index 8338cffeea..56ac0c4b3b 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/SagaHistoryCleaner.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/SagaHistoryCleaner.cs @@ -30,7 +30,7 @@ public static void Clean(int deletionBatchSize, DocumentDatabase database, DateT Query = $"LastModified:[* TO {expiryThreshold.Ticks}]", FieldsToFetch = new[] { - "__document_id" + "@id" }, SortedFields = new[] { @@ -44,7 +44,7 @@ public static void Clean(int deletionBatchSize, DocumentDatabase database, DateT database.Query(indexName, query, (doc, commands) => { - var id = doc.Value("__document_id"); + var id = doc.Value("@id"); if (string.IsNullOrEmpty(id)) { return; diff --git a/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs index 842342cb8d..fbb7546440 100644 --- a/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs @@ -2,7 +2,7 @@ { using System; using System.Threading.Tasks; - using Raven.Client; + using Raven.Client.Documents; using ServiceControl.Persistence.MessageRedirects; class MessageRedirectsDataStore : IMessageRedirectsDataStore @@ -22,8 +22,8 @@ public async Task GetOrCreate() if (redirects != null) { - redirects.ETag = session.Advanced.GetEtagFor(redirects); - redirects.LastModified = session.Advanced.GetMetadataFor(redirects).Value("Last-Modified"); + redirects.ETag = session.Advanced.GetChangeVectorFor(redirects).; + redirects.LastModified = session.Advanced.GetMetadataFor(redirects).Value("@last-modified"); return redirects; } diff --git a/src/ServiceControl.Persistence.RavenDb5/RetryBatchesManager.cs b/src/ServiceControl.Persistence.RavenDb5/RetryBatchesManager.cs index 94595b5c0f..a2e090d5b3 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RetryBatchesManager.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RetryBatchesManager.cs @@ -48,7 +48,7 @@ public async Task GetOrCreateMessageRedirectsCollect if (redirects != null) { redirects.ETag = Session.Advanced.GetEtagFor(redirects); - redirects.LastModified = Session.Advanced.GetMetadataFor(redirects).Value("Last-Modified"); + redirects.LastModified = Session.Advanced.GetMetadataFor(redirects).Value("@last-modified"); return redirects; } diff --git a/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs index 399f231eb2..8b6b791423 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs @@ -145,7 +145,7 @@ static ICommandData CreateFailedMessageRetryDocument(string batchDocumentId, str static JObject DefaultMetadata = JObject.Parse($@" {{ - ""Raven-Entity-Name"": ""{FailedMessageRetry.CollectionName}"", + ""@collection"": ""{FailedMessageRetry.CollectionName}"", ""Raven-Clr-Type"": ""{typeof(FailedMessageRetry).AssemblyQualifiedName}"" }}"); diff --git a/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs b/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs index a187102549..c77ba1c092 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs @@ -24,7 +24,7 @@ public FailedMessageViewTransformer() NumberOfProcessingAttempts = failure.ProcessingAttempts.Count, failure.Status, rec.FailureDetails.TimeOfFailure, - LastModified = MetadataFor(failure)["Last-Modified"].Value(), + LastModified = MetadataFor(failure)["@last-modified"].Value(), Edited = edited, EditOf = edited ? rec.Headers["ServiceControl.EditOf"] : "" }; diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs index 2a46dec4ca..2462957f21 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs @@ -32,7 +32,7 @@ static RavenDbMonitoringIngestionUnitOfWork() { KnownEndpointMetadata = JObject.Parse($@" {{ - ""Raven-Entity-Name"": ""{KnownEndpoint.CollectionName}"", + ""@collection"": ""{KnownEndpoint.CollectionName}"", ""Raven-Clr-Type"": ""{typeof(KnownEndpoint).AssemblyQualifiedName}"" }}"); } diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs index 0262508126..631ac8397a 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs @@ -124,7 +124,7 @@ static RavenDbRecoverabilityIngestionUnitOfWork() FailedMessageMetadata = JObject.Parse($@" {{ - ""Raven-Entity-Name"": ""{FailedMessageIdGenerator.CollectionName}"", + ""@collection"": ""{FailedMessageIdGenerator.CollectionName}"", ""Raven-Clr-Type"": ""{typeof(FailedMessage).AssemblyQualifiedName}"" }}"); } From 0becd8a3c2feb42b8be957a6b314bd88f9db89c2 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 6 Sep 2023 16:59:33 +0200 Subject: [PATCH 12/78] .Value --- .../ErrorMessagesDataStore.cs | 2 +- .../MessageRedirects/MessageRedirectsDataStore.cs | 4 ++-- .../MetadataExtensions.cs | 13 ++++++++++++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index e3ca468597..ec66005082 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -401,7 +401,7 @@ static FailedMessageView Map(FailedMessage message, IAsyncDocumentSession sessio Id = message.UniqueMessageId, MessageType = metadata.GetAsStringOrNull("MessageType"), IsSystemMessage = metadata.GetOrDefault("IsSystemMessage"), - TimeSent = metadata.GetAsNullableDatetime("TimeSent"), + TimeSent = metadata.GetAsNullableDateTime("TimeSent"), MessageId = metadata.GetAsStringOrNull("MessageId"), Exception = failureDetails.Exception, QueueAddress = failureDetails.AddressOfFailingEndpoint, diff --git a/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs index fbb7546440..e28edd9024 100644 --- a/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs @@ -22,8 +22,8 @@ public async Task GetOrCreate() if (redirects != null) { - redirects.ETag = session.Advanced.GetChangeVectorFor(redirects).; - redirects.LastModified = session.Advanced.GetMetadataFor(redirects).Value("@last-modified"); + redirects.ETag = session.Advanced.GetChangeVectorFor(redirects); + redirects.LastModified = session.Advanced.GetMetadataFor(redirects).GetAsDateTime("@last-modified"); return redirects; } diff --git a/src/ServiceControl.Persistence.RavenDb5/MetadataExtensions.cs b/src/ServiceControl.Persistence.RavenDb5/MetadataExtensions.cs index ddfb3ce644..528691f129 100644 --- a/src/ServiceControl.Persistence.RavenDb5/MetadataExtensions.cs +++ b/src/ServiceControl.Persistence.RavenDb5/MetadataExtensions.cs @@ -25,7 +25,7 @@ public static string GetAsStringOrNull(this IDictionary metadata return null; } - public static DateTime? GetAsNullableDatetime(this IDictionary metadata, string key) + public static DateTime? GetAsNullableDateTime(this IDictionary metadata, string key) { var datetimeAsString = metadata.GetAsStringOrNull(key); @@ -36,5 +36,16 @@ public static string GetAsStringOrNull(this IDictionary metadata return null; } + + public static DateTime GetAsDateTime(this IDictionary metadata, string key) + { + var datetimeAsString = metadata.GetAsStringOrNull(key); + + if (datetimeAsString != null) + { + return DateTime.Parse(datetimeAsString); + } + } + } } \ No newline at end of file From fd77adc07d04dc3f32ea3a07830f1955d80b2afc Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 7 Sep 2023 14:20:02 -0500 Subject: [PATCH 13/78] Raven5 will not store Audit or saga history, no need to clean them --- .../Expiration/AuditMessageCleaner.cs | 162 ------------------ .../Expiration/ExpiredDocumentsCleaner.cs | 13 -- .../Indexes/SagaHistoryCleaner.cs | 110 ------------ 3 files changed, 285 deletions(-) delete mode 100644 src/ServiceControl.Persistence.RavenDb5/Expiration/AuditMessageCleaner.cs delete mode 100644 src/ServiceControl.Persistence.RavenDb5/Indexes/SagaHistoryCleaner.cs diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/AuditMessageCleaner.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/AuditMessageCleaner.cs deleted file mode 100644 index 71f47bec22..0000000000 --- a/src/ServiceControl.Persistence.RavenDb5/Expiration/AuditMessageCleaner.cs +++ /dev/null @@ -1,162 +0,0 @@ -namespace ServiceControl.Infrastructure.RavenDB.Expiration -{ - using System; - using System.Collections.Generic; - using System.Diagnostics; - using System.Linq; - using System.Threading; - using Newtonsoft.Json.Linq; - using NServiceBus.Logging; - using Raven.Client.Documents.Commands.Batches; - using Raven.Client.Documents.Queries; - using Raven.Client.Util; - - static class AuditMessageCleaner - { - public static void Clean(int deletionBatchSize, DocumentDatabase database, DateTime expiryThreshold, CancellationToken cancellationToken = default) - { - var stopwatch = Stopwatch.StartNew(); - var items = new List(deletionBatchSize); - var attachments = new List(deletionBatchSize); - var itemsAndAttachements = Tuple.Create(items, attachments); - var indexName = new ExpiryProcessedMessageIndex().IndexName; - - try - { - var query = new IndexQuery - { - Start = 0, - PageSize = deletionBatchSize, - Cutoff = SystemTime.UtcNow, - DisableCaching = true, - Query = $"ProcessedAt:[* TO {expiryThreshold.Ticks}]", - FieldsToFetch = new[] - { - "@id", - "MessageMetadata.MessageId", - "MessageMetadata.BodyNotStored" - }, - SortedFields = new[] - { - new SortedField("ProcessedAt") - { - Field = "ProcessedAt", - Descending = false - } - } - }; - - database.Query(indexName, query, (doc, state) => - { - var id = doc.Value("@id"); - if (string.IsNullOrEmpty(id)) - { - return; - } - - state.Item1.Add(new DeleteCommandData - { - Key = id - }); - - if (TryGetBodyId(doc, out var bodyId)) - { - state.Item2.Add(bodyId); - } - }, - itemsAndAttachements, cancellationToken); - } - catch (IndexDisabledException ex) - { - logger.Error($"Unable to cleanup audit messages. The index ${indexName} was disabled.", ex); - return; - } - catch (OperationCanceledException) - { - logger.Info("Cleanup operation cancelled"); - return; - } - - if (cancellationToken.IsCancellationRequested) - { - return; - } - - var deletedAuditDocuments = Chunker.ExecuteInChunks(items.Count, (itemsForBatch, db, s, e) => - { - if (logger.IsDebugEnabled) - { - logger.Debug($"Batching deletion of {s}-{e} audit documents."); - } - - var results = db.Batch(itemsForBatch.GetRange(s, e - s + 1), CancellationToken.None); - if (logger.IsDebugEnabled) - { - logger.Debug($"Batching deletion of {s}-{e} audit documents completed."); - } - - return results.Count(x => x.Deleted == true); - }, items, database, cancellationToken); - - var deletedAttachments = Chunker.ExecuteInChunks(attachments.Count, (att, db, s, e) => - { - var deleted = 0; - if (logger.IsDebugEnabled) - { - logger.Debug($"Batching deletion of {s}-{e} attachment audit documents."); - } - - db.TransactionalStorage.Batch(accessor => - { - for (var idx = s; idx <= e; idx++) - { - //We want to continue using attachments for now - accessor.Attachments.DeleteAttachment(att[idx], null); - deleted++; - } - }); - if (logger.IsDebugEnabled) - { - logger.Debug($"Batching deletion of {s}-{e} attachment audit documents completed."); - } - - return deleted; - }, attachments, database, cancellationToken); - - if (deletedAttachments + deletedAuditDocuments == 0) - { - if (logger.IsDebugEnabled) - { - logger.Debug("No expired audit documents found"); - } - } - else - { - if (logger.IsDebugEnabled) - { - logger.Debug($"Deleted {deletedAuditDocuments} expired audit documents and {deletedAttachments} message body attachments. Batch execution took {stopwatch.ElapsedMilliseconds} ms"); - } - } - } - - static bool TryGetBodyId(JObject doc, out string bodyId) - { - bodyId = null; - if (doc.Value("MessageMetadata.BodyNotStored")) - { - return false; - } - - var messageId = doc.Value("MessageMetadata.MessageId"); - if (messageId == null) - { - return false; - } - - bodyId = $"messagebodies/{messageId}"; - return true; - } - - static ILog logger = LogManager.GetLogger(typeof(AuditMessageCleaner)); - } -} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiredDocumentsCleaner.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiredDocumentsCleaner.cs index 459f69b4ef..22c9d8fa5e 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiredDocumentsCleaner.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiredDocumentsCleaner.cs @@ -6,7 +6,6 @@ using BackgroundTasks; using NServiceBus.Logging; using Raven.Client.Util; - using SagaAudit; class ExpiredDocumentsCleaner { @@ -28,18 +27,6 @@ public static Task RunCleanup(int deletionBatchSize, Do } EventLogItemsCleaner.Clean(deletionBatchSize, database, threshold, cancellationToken); - if (settings.AuditRetentionPeriod.HasValue) - { - threshold = SystemTime.UtcNow.Add(settings.AuditRetentionPeriod.Value); - - if (logger.IsDebugEnabled) - { - logger.Debug($"Trying to find expired ProcessedMessage and SagaHistory documents to delete (with threshold {threshold.ToString(Default.DateTimeFormatsToWrite, CultureInfo.InvariantCulture)})"); - } - AuditMessageCleaner.Clean(deletionBatchSize, database, threshold, cancellationToken); - SagaHistoryCleaner.Clean(deletionBatchSize, database, threshold, cancellationToken); - } - return Task.FromResult(TimerJobExecutionResult.ScheduleNextExecution); } diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/SagaHistoryCleaner.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/SagaHistoryCleaner.cs deleted file mode 100644 index 56ac0c4b3b..0000000000 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/SagaHistoryCleaner.cs +++ /dev/null @@ -1,110 +0,0 @@ -namespace ServiceControl.SagaAudit -{ - using System; - using System.Collections.Generic; - using System.Diagnostics; - using System.Linq; - using System.Threading; - using NServiceBus.Logging; - using Raven.Client.Documents.Commands.Batches; - using Raven.Client.Documents.Queries; - using Raven.Client.Util; - using ServiceControl.Infrastructure.RavenDB; - - static class SagaHistoryCleaner - { - public static void Clean(int deletionBatchSize, DocumentDatabase database, DateTime expiryThreshold, CancellationToken cancellationToken = default) - { - var stopwatch = Stopwatch.StartNew(); - var items = new List(deletionBatchSize); - var indexName = new ExpirySagaAuditIndex().IndexName; - - try - { - var query = new IndexQuery - { - Start = 0, - DisableCaching = true, - Cutoff = SystemTime.UtcNow, - PageSize = deletionBatchSize, - Query = $"LastModified:[* TO {expiryThreshold.Ticks}]", - FieldsToFetch = new[] - { - "@id" - }, - SortedFields = new[] - { - new SortedField("LastModified") - { - Field = "LastModified", - Descending = false - } - } - }; - - database.Query(indexName, query, (doc, commands) => - { - var id = doc.Value("@id"); - if (string.IsNullOrEmpty(id)) - { - return; - } - - commands.Add(new DeleteCommandData - { - Key = id - }); - }, - items, cancellationToken); - } - catch (IndexDisabledException ex) - { - logger.Error($"Unable to cleanup saga history. The index ${indexName} was disabled.", ex); - return; - } - catch (OperationCanceledException) - { - logger.Info("Cleanup operation cancelled"); - return; - } - - if (cancellationToken.IsCancellationRequested) - { - return; - } - - var deletionCount = Chunker.ExecuteInChunks(items.Count, (itemsForBatch, db, s, e) => - { - if (logger.IsDebugEnabled) - { - logger.Debug($"Batching deletion of {s}-{e} saga history documents."); - } - - var results = db.Batch(itemsForBatch.GetRange(s, e - s + 1), CancellationToken.None); - if (logger.IsDebugEnabled) - { - logger.Debug($"Batching deletion of {s}-{e} saga history documents completed."); - } - - return results.Count(x => x.Deleted == true); - }, items, database, cancellationToken); - - if (deletionCount == 0) - { - if (logger.IsDebugEnabled) - { - logger.Debug("No expired saga history documents found"); - } - } - else - { - if (logger.IsDebugEnabled) - { - logger.Debug($"Deleted {deletionCount} expired saga history documents. Batch execution took {stopwatch.ElapsedMilliseconds} ms"); - } - } - } - - static ILog logger = LogManager.GetLogger(typeof(SagaHistoryCleaner)); - } -} From fc6985dc0a9442deac10b9715beff6dbe36ff081 Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 7 Sep 2023 14:25:32 -0500 Subject: [PATCH 14/78] No saga audit details in Raven5 --- .../Indexes/SagaDetailsIndex.cs | 57 ------------------- .../SagaAudit/SagaAuditDataStore.cs | 36 +----------- 2 files changed, 2 insertions(+), 91 deletions(-) delete mode 100644 src/ServiceControl.Persistence.RavenDb5/Indexes/SagaDetailsIndex.cs diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/SagaDetailsIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/SagaDetailsIndex.cs deleted file mode 100644 index 82524479a3..0000000000 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/SagaDetailsIndex.cs +++ /dev/null @@ -1,57 +0,0 @@ -namespace ServiceControl.SagaAudit -{ - using System.Linq; - using Raven.Client.Documents.Indexes; - - class SagaDetailsIndex : AbstractMultiMapIndexCreationTask - { - public SagaDetailsIndex() - { - AddMap(docs => from doc in docs - select new - { - doc.SagaId, - Id = doc.SagaId, - doc.SagaType, - Changes = new[] - { - new SagaStateChange - { - Endpoint = doc.Endpoint, - FinishTime = doc.FinishTime, - InitiatingMessage = doc.InitiatingMessage, - OutgoingMessages = doc.OutgoingMessages, - StartTime = doc.StartTime, - StateAfterChange = doc.StateAfterChange, - Status = doc.Status - } - } - }); - - //Legacy so we still scan old sagahistories - AddMap(docs => from doc in docs - select new - { - doc.SagaId, - Id = doc.SagaId, - doc.SagaType, - doc.Changes - } - ); - - Reduce = results => from result in results - group result by result.SagaId - into g - let first = g.First() - select new SagaHistory - { - Id = first.SagaId, - SagaId = first.SagaId, - SagaType = first.SagaType, - Changes = g.SelectMany(x => x.Changes) - .OrderByDescending(x => x.FinishTime) - .ToList() - }; - } - } -} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaAuditDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaAuditDataStore.cs index eb4732a373..37472912c9 100644 --- a/src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaAuditDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaAuditDataStore.cs @@ -4,43 +4,11 @@ using System.Threading.Tasks; using ServiceControl.Persistence.Infrastructure; using ServiceControl.SagaAudit; - using Raven.Client; - using Raven.Client.Documents; class SagaAuditDataStore : ISagaAuditDataStore { - public SagaAuditDataStore(IDocumentStore store) - { - this.store = store; - } + public Task StoreSnapshot(SagaSnapshot sagaSnapshot) => throw new NotImplementedException(); - public async Task StoreSnapshot(SagaSnapshot sagaSnapshot) - { - using (var session = store.OpenAsyncSession()) - { - await session.StoreAsync(sagaSnapshot); - await session.SaveChangesAsync(); - } - } - - public async Task> GetSagaById(Guid sagaId) - { - using (var session = store.OpenAsyncSession()) - { - var sagaHistory = await - session.Query() - .Statistics(out var stats) - .SingleOrDefaultAsync(x => x.SagaId == sagaId); - - if (sagaHistory == null) - { - return QueryResult.Empty(); - } - - return new QueryResult(sagaHistory, new QueryStatsInfo(stats.IndexEtag, stats.TotalResults, stats.IsStale)); - } - } - - readonly IDocumentStore store; + public Task> GetSagaById(Guid sagaId) => Task.FromResult(QueryResult.Empty()); } } From 193f9709a0f73f0644e1e7dffd528023ab63fdac Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 7 Sep 2023 14:27:50 -0500 Subject: [PATCH 15/78] No SagaHistory/SagaSnapshot or Audit Retention in Raven5 --- .../SagaAudit/AuditRetentionCustomCheck.cs | 50 ------------------- .../SagaAudit/SagaListIndex.cs | 45 ----------------- 2 files changed, 95 deletions(-) delete mode 100644 src/ServiceControl.Persistence.RavenDb5/SagaAudit/AuditRetentionCustomCheck.cs delete mode 100644 src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaListIndex.cs diff --git a/src/ServiceControl.Persistence.RavenDb5/SagaAudit/AuditRetentionCustomCheck.cs b/src/ServiceControl.Persistence.RavenDb5/SagaAudit/AuditRetentionCustomCheck.cs deleted file mode 100644 index d4d2155f11..0000000000 --- a/src/ServiceControl.Persistence.RavenDb5/SagaAudit/AuditRetentionCustomCheck.cs +++ /dev/null @@ -1,50 +0,0 @@ - -namespace ServiceControl.Persistence.RavenDb.SagaAudit -{ - using System; - using System.Threading.Tasks; - using NServiceBus.CustomChecks; - using ServiceControl.SagaAudit; - using Raven.Client.Documents; - - // This custom check stays in the Raven3.5 persister becuase the Raven5 persister will not store audit data - class AuditRetentionCustomCheck : CustomCheck - { - readonly IDocumentStore documentStore; - readonly bool auditRetentionPeriodIsSet; - - public AuditRetentionCustomCheck( - IDocumentStore documentStore, - RavenDBPersisterSettings settings, - TimeSpan? repeatAfter = null - ) - : base("Saga Audit Data Retention", "ServiceControl Health", repeatAfter.HasValue ? repeatAfter : TimeSpan.FromHours(1)) - { - this.documentStore = documentStore; - auditRetentionPeriodIsSet = settings.AuditRetentionPeriod != default; - } - - public override async Task PerformCheck() - { - if (auditRetentionPeriodIsSet) - { - return CheckResult.Pass; - } - - if (await DetectSagaAuditData()) - { - return CheckResult.Failed("Saga snapshot data detected without an audit retention period configured. If saga audit data is allowed to accumulate, it can result in degraded performance. Visit https://docs.particular.net/search?q=servicecontrol+troubleshooting for more information."); - } - - return CheckResult.Pass; - } - - async Task DetectSagaAuditData() - { - using (var session = documentStore.OpenAsyncSession()) - { - return await session.Query().AnyAsync(); - } - } - } -} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaListIndex.cs b/src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaListIndex.cs deleted file mode 100644 index 50ac2e4533..0000000000 --- a/src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaListIndex.cs +++ /dev/null @@ -1,45 +0,0 @@ -namespace ServiceControl.SagaAudit -{ - using System; - using System.Linq; - using Raven.Client.Documents.Indexes; - - class SagaListIndex : AbstractMultiMapIndexCreationTask - { - public SagaListIndex() - { - AddMap(docs => from doc in docs - select new Result - { - Id = doc.SagaId, - Uri = "api/sagas/" + doc.SagaId, - SagaType = doc.SagaType - }); - AddMap(docs => from doc in docs - select new Result - { - Id = doc.SagaId, - Uri = "api/sagas/" + doc.SagaId, - SagaType = doc.SagaType - }); - - Reduce = results => from result in results - group result by result.Id - into g - let first = g.First() - select new Result - { - Id = g.Key, - Uri = first.Uri, - SagaType = first.SagaType - }; - } - - public class Result - { - public Guid Id { get; set; } - public string Uri { get; set; } - public string SagaType { get; set; } - } - } -} \ No newline at end of file From b37efcca71ce28656ad1fc783974e67e2d60a732 Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 7 Sep 2023 14:30:54 -0500 Subject: [PATCH 16/78] No audits in Raven5 means no failed audits either --- .../FailedAuditImportCustomCheck.cs | 43 ------------------- .../Indexes/ImportFailedAudits.cs | 18 -------- .../RavenDbPersistence.cs | 9 ++-- 3 files changed, 3 insertions(+), 67 deletions(-) delete mode 100644 src/ServiceControl.Persistence.RavenDb5/CustomChecks/FailedAuditImportCustomCheck.cs delete mode 100644 src/ServiceControl.Persistence.RavenDb5/Indexes/ImportFailedAudits.cs diff --git a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/FailedAuditImportCustomCheck.cs b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/FailedAuditImportCustomCheck.cs deleted file mode 100644 index 58b0648761..0000000000 --- a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/FailedAuditImportCustomCheck.cs +++ /dev/null @@ -1,43 +0,0 @@ -namespace ServiceControl.Operations -{ - using System; - using System.Threading.Tasks; - using NServiceBus.CustomChecks; - using NServiceBus.Logging; - using Raven.Client.Documents; - - class FailedAuditImportCustomCheck : CustomCheck - { - public FailedAuditImportCustomCheck(IDocumentStore store) - : base("Audit Message Ingestion", "ServiceControl Health", TimeSpan.FromHours(1)) - { - this.store = store; - } - - public override async Task PerformCheck() - { - using (var session = store.OpenAsyncSession()) - { - var query = session.Query(); - using (var ie = await session.Advanced.StreamAsync(query) - ) - { - if (await ie.MoveNextAsync()) - { - Logger.Warn(message); - return CheckResult.Failed(message); - } - } - } - - return CheckResult.Pass; - } - - readonly IDocumentStore store; - - const string message = @"One or more audit messages have failed to import properly into ServiceControl and have been stored in the ServiceControl database. -The import of these messages could have failed for a number of reasons and ServiceControl is not able to automatically reimport them. For guidance on how to resolve this see https://docs.particular.net/servicecontrol/import-failed-messages"; - - static readonly ILog Logger = LogManager.GetLogger(typeof(FailedAuditImportCustomCheck)); - } -} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/ImportFailedAudits.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/ImportFailedAudits.cs deleted file mode 100644 index 23c9837271..0000000000 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/ImportFailedAudits.cs +++ /dev/null @@ -1,18 +0,0 @@ -namespace ServiceControl.Operations -{ - using System.Linq; - using Raven.Client.Documents.Indexes; - - class FailedAuditImportIndex : AbstractIndexCreationTask - { - public FailedAuditImportIndex() - { - Map = docs => from cc in docs - select new FailedAuditImport - { - Id = cc.Id, - Message = cc.Message - }; - } - } -} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs index b2a1084356..83cf0736f7 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs @@ -3,6 +3,9 @@ using MessageRedirects; using Microsoft.Extensions.DependencyInjection; using Persistence.Recoverability; + using Raven.Client; + using Raven.Client.Documents; + using Raven.Client.Embedded; using Recoverability; using ServiceControl.CustomChecks; using ServiceControl.Infrastructure.RavenDB.Subscriptions; @@ -11,12 +14,8 @@ using ServiceControl.Operations.BodyStorage; using ServiceControl.Operations.BodyStorage.RavenAttachments; using ServiceControl.Persistence.MessageRedirects; - using ServiceControl.Persistence.RavenDb.SagaAudit; using ServiceControl.Persistence.UnitOfWork; using ServiceControl.Recoverability; - using Raven.Client; - using Raven.Client.Documents; - using Raven.Client.Embedded; class RavenDbPersistence : IPersistence { @@ -54,9 +53,7 @@ public void Configure(IServiceCollection serviceCollection) serviceCollection.AddCustomCheck(); serviceCollection.AddCustomCheck(); - serviceCollection.AddCustomCheck(); serviceCollection.AddCustomCheck(); - serviceCollection.AddCustomCheck(); serviceCollection.AddCustomCheck(); serviceCollection.AddSingleton(); From d59bb8fe959a6636c56593469464d524efd0aed3 Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 7 Sep 2023 15:08:44 -0500 Subject: [PATCH 17/78] Change CheckRavenDBIndexLag custom check given database reports different info now --- .../CustomChecks/CheckRavenDBIndexLag.cs | 47 ++++++++++--------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexLag.cs b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexLag.cs index f39c687a4e..b45f9d21e6 100644 --- a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexLag.cs +++ b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexLag.cs @@ -7,7 +7,6 @@ using NServiceBus.CustomChecks; using NServiceBus.Logging; using Raven.Client.Documents; - using Raven.Client.Documents.Indexes; using Raven.Client.Documents.Operations; using CustomCheck = NServiceBus.CustomChecks.CustomCheck; @@ -19,9 +18,9 @@ public CheckRavenDBIndexLag(IDocumentStore store) this.store = store; } - public override Task PerformCheck() + public override async Task PerformCheck() { - var statistics = store.DatabaseCommands.GetStatistics(); + var statistics = await store.Maintenance.SendAsync(new GetStatisticsOperation()); var indexes = statistics.Indexes.OrderBy(x => x.Name).ToArray(); CreateDiagnosticsLogEntry(statistics, indexes); @@ -36,32 +35,33 @@ public override Task PerformCheck() return CheckResult.Pass; } - static int CheckAndReportIndexesWithTooMuchIndexLag(IndexStats[] indexes) + static int CheckAndReportIndexesWithTooMuchIndexLag(IndexInformation[] indexes) { int indexCountWithTooMuchLag = 0; foreach (var indexStats in indexes) { - // IndexingLag is the number of documents that the index is behind, it is not a time unit. - var indexLag = indexStats.IndexingLag.GetValueOrDefault(); - indexLag = Math.Abs(indexLag); - - if (indexLag > IndexLagThresholdError) - { - indexCountWithTooMuchLag++; - Log.Error($"Index [{indexStats.Name}] IndexingLag {indexLag:n0} is above error threshold ({IndexLagThresholdError:n0}). Launch in maintenance mode to let indexes catch up."); - } - else if (indexLag > IndexLagThresholdWarning) + if (indexStats.LastIndexingTime.HasValue) { - indexCountWithTooMuchLag++; - Log.Warn($"Index [{indexStats.Name}] IndexingLag {indexLag:n0} is above warning threshold ({IndexLagThresholdWarning:n0}). Launch in maintenance mode to let indexes catch up."); + var indexLag = DateTime.UtcNow - indexStats.LastIndexingTime.Value; + + if (indexLag > IndexLagThresholdError) + { + indexCountWithTooMuchLag++; + Log.Error($"Index [{indexStats.Name}] IndexingLag {indexLag:n0} is above error threshold ({IndexLagThresholdError:n0}). Launch in maintenance mode to let indexes catch up."); + } + else if (indexLag > IndexLagThresholdWarning) + { + indexCountWithTooMuchLag++; + Log.Warn($"Index [{indexStats.Name}] IndexingLag {indexLag:n0} is above warning threshold ({IndexLagThresholdWarning:n0}). Launch in maintenance mode to let indexes catch up."); + } } } return indexCountWithTooMuchLag; } - static void CreateDiagnosticsLogEntry(DatabaseStatistics statistics, IndexStats[] indexes) + static void CreateDiagnosticsLogEntry(DatabaseStatistics statistics, IndexInformation[] indexes) { if (!Log.IsDebugEnabled) { @@ -70,19 +70,20 @@ static void CreateDiagnosticsLogEntry(DatabaseStatistics statistics, IndexStats[ var report = new StringBuilder(); report.AppendLine("Internal RavenDB index health report:"); + report.AppendLine($"- DB Size: {statistics.SizeOnDisk.HumaneSize}"); + report.AppendLine($"- LastIndexingTime {statistics.LastIndexingTime:u}"); foreach (var indexStats in indexes) { - // IndexingLag is the number of documents that the index is behind, it is not a time unit. - var indexLag = indexStats.IndexingLag.GetValueOrDefault(); - indexLag = Math.Abs(indexLag); - report.AppendLine($"- Index [{indexStats.Name,-44}] Stale: {statistics.StaleIndexes.Contains(indexStats.Name),-5}, Lag: {indexLag,9:n0}, Valid: {indexStats.IsInvalidIndex,-5}, LastIndexed: {indexStats.LastIndexedTimestamp:u}, LastIndexing: {indexStats.LastIndexingTime:u}"); + report.AppendLine($"- Index [{indexStats.Name,-44}] State: {indexStats.State}, Stale: {indexStats.IsStale,-5}, Priority: {indexStats.Priority,-6}, LastIndexingTime: {indexStats.LastIndexingTime:u}"); } Log.Debug(report.ToString()); } - const int IndexLagThresholdWarning = 10000; - const int IndexLagThresholdError = 100000; + // TODO: RavenDB 3.5 had IndexLag thresholds that were number of document writes, and I converted to times. Revisit these numbers before shipping + // For IndexLag as document writes, 10k was a warning, 100k was an error. These TimeSpans assume same # of writes / 250 writes/sec + static readonly TimeSpan IndexLagThresholdWarning = TimeSpan.FromSeconds(40); // Assuming 10_000 writes at 250 writes/sec + static readonly TimeSpan IndexLagThresholdError = TimeSpan.FromSeconds(400); // Assuming 100_000 writes at 250 writes/sec static readonly ILog Log = LogManager.GetLogger(); readonly IDocumentStore store; From 4aba28c522bfaffc689e7426c19bf32a015748bf Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 7 Sep 2023 15:14:12 -0500 Subject: [PATCH 18/78] More SagaAudit stuff --- .../Indexes/ExpirySagaAuditIndex.cs | 24 ------------------- .../Infrastructure/MigratedTypeAwareBinder.cs | 8 ------- .../RavenDbPersistence.cs | 2 +- ... => NoImplementationSagaAuditDataStore.cs} | 2 +- 4 files changed, 2 insertions(+), 34 deletions(-) delete mode 100644 src/ServiceControl.Persistence.RavenDb5/Indexes/ExpirySagaAuditIndex.cs rename src/ServiceControl.Persistence.RavenDb5/SagaAudit/{SagaAuditDataStore.cs => NoImplementationSagaAuditDataStore.cs} (86%) diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/ExpirySagaAuditIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/ExpirySagaAuditIndex.cs deleted file mode 100644 index 2677160995..0000000000 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/ExpirySagaAuditIndex.cs +++ /dev/null @@ -1,24 +0,0 @@ -namespace ServiceControl.SagaAudit -{ - using System; - using System.Linq; - using Raven.Client.Documents.Indexes; - - class ExpirySagaAuditIndex : AbstractMultiMapIndexCreationTask - { - public ExpirySagaAuditIndex() - { - AddMap(messages => from message in messages - select new - { - LastModified = MetadataFor(message).Value("@last-modified").Ticks - }); - - AddMap(sagaHistories => from sagaHistory in sagaHistories - select new - { - LastModified = MetadataFor(sagaHistory).Value("@last-modified").Ticks - }); - } - } -} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/MigratedTypeAwareBinder.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/MigratedTypeAwareBinder.cs index 1ea5d67207..70257f7ffe 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/MigratedTypeAwareBinder.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/MigratedTypeAwareBinder.cs @@ -3,7 +3,6 @@ using System; using System.Linq; using Newtonsoft.Json.Serialization; - using SagaAudit; using ServiceControl.Contracts.CustomChecks; using ServiceControl.Contracts.Operations; using ServiceControl.MessageAuditing; @@ -17,11 +16,6 @@ class MigratedTypeAwareBinder : DefaultSerializationBinder { public override Type BindToType(string assemblyName, string typeName) { - if (typeName == "ServiceControl.SagaAudit.SagaInfo" && assemblyName == "ServiceControl") - { - return typeof(SagaInfo); - } - var className = GetClassName(typeName); switch (className) { @@ -53,8 +47,6 @@ public override Type BindToType(string assemblyName, string typeName) return typeof(RetryBatch); case nameof(RetryBatchNowForwarding): return typeof(RetryBatchNowForwarding); - case nameof(SagaInfo): - return typeof(SagaInfo); default: return base.BindToType(assemblyName, typeName); } diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs index 83cf0736f7..b83ff20171 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs @@ -72,7 +72,7 @@ public void Configure(IServiceCollection serviceCollection) serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); - serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); } diff --git a/src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaAuditDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/SagaAudit/NoImplementationSagaAuditDataStore.cs similarity index 86% rename from src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaAuditDataStore.cs rename to src/ServiceControl.Persistence.RavenDb5/SagaAudit/NoImplementationSagaAuditDataStore.cs index 37472912c9..0ee0bf02e7 100644 --- a/src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaAuditDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/SagaAudit/NoImplementationSagaAuditDataStore.cs @@ -5,7 +5,7 @@ using ServiceControl.Persistence.Infrastructure; using ServiceControl.SagaAudit; - class SagaAuditDataStore : ISagaAuditDataStore + class NoImplementationSagaAuditDataStore : ISagaAuditDataStore { public Task StoreSnapshot(SagaSnapshot sagaSnapshot) => throw new NotImplementedException(); From ca58b0c8a571b241f12d8c4b6a97bc60d44e3a7e Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 7 Sep 2023 15:18:32 -0500 Subject: [PATCH 19/78] No such thing as bundles. Have to use https://ravendb.net/docs/article-page/5.4/csharp/server/extensions/expiration --- .../ExpiredDocumentsCleanerBundle.cs | 125 ------------------ .../RavenBootstrapper.cs | 8 -- .../RavenDBPersisterSettings.cs | 4 - .../RavenDbPersistenceConfiguration.cs | 5 +- 4 files changed, 1 insertion(+), 141 deletions(-) delete mode 100644 src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiredDocumentsCleanerBundle.cs diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiredDocumentsCleanerBundle.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiredDocumentsCleanerBundle.cs deleted file mode 100644 index bcaf23c33b..0000000000 --- a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiredDocumentsCleanerBundle.cs +++ /dev/null @@ -1,125 +0,0 @@ -namespace ServiceControl.Infrastructure.RavenDB.Expiration -{ - using System; - using System.ComponentModel.Composition; - using System.Threading.Tasks; - using BackgroundTasks; - using NServiceBus.Logging; - using Persistence.RavenDb; - using Raven.Database; - using Raven.Database.Plugins; - - [InheritedExport(typeof(IStartupTask))] - [ExportMetadata("Bundle", "customDocumentExpiration")] - class ExpiredDocumentsCleanerBundle : IStartupTask, IDisposable - { - // TODO: Ensure that the timers are started when the persister starts! - - public void Dispose() - { - lock (this) - { - if (timer == null) - { - return; - } - - var stopTask = timer.Stop(); - var delayTask = Task.Delay(TimeSpan.FromSeconds(30)); - var composite = Task.WhenAny(stopTask, delayTask); - - var finishedTask = composite.GetAwaiter().GetResult(); - if (finishedTask == delayTask) - { - Logger.Error("Cleanup process did not finish on time. Forcing shutdown."); - } - else - { - Logger.Info("Expired documents cleanup process stopped."); - } - - timer = null; - } - } - - public void Execute(DocumentDatabase database) - { - var persistenceSettings = RavenBootstrapper.Settings; - - if (ExpirationProcessTimerInSeconds == 0) - { - return; - } - - var due = TimeSpan.FromSeconds(ExpirationProcessTimerInSeconds); - var deletionBatchSize = ExpirationProcessBatchSize; - - Logger.Info($"Running deletion of expired documents every {ExpirationProcessTimerInSeconds} seconds"); - Logger.Info($"Deletion batch size set to {deletionBatchSize}"); - Logger.Info($"Retention period for errors is {persistenceSettings.ErrorRetentionPeriod}"); - - var auditRetention = persistenceSettings.AuditRetentionPeriod; - - if (auditRetention.HasValue) - { - Logger.InfoFormat("Retention period for audits and saga history is {0}", persistenceSettings.AuditRetentionPeriod); - } - - timer = new TimerJob( - token => ExpiredDocumentsCleaner.RunCleanup(deletionBatchSize, database, persistenceSettings, token), due, due, e => { Logger.Error("Error when trying to find expired documents", e); }); - } - - int ExpirationProcessTimerInSeconds - { - get - { - var expirationProcessTimerInSeconds = settings.ExpirationProcessTimerInSeconds; - - if (expirationProcessTimerInSeconds < 0) - { - Logger.Error($"ExpirationProcessTimerInSeconds cannot be negative. Defaulting to {ExpirationProcessTimerInSecondsDefault}"); - return ExpirationProcessTimerInSecondsDefault; - } - - if (expirationProcessTimerInSeconds > TimeSpan.FromHours(3).TotalSeconds) - { - Logger.Error($"ExpirationProcessTimerInSeconds cannot be larger than {TimeSpan.FromHours(3).TotalSeconds}. Defaulting to {ExpirationProcessTimerInSecondsDefault}"); - return ExpirationProcessTimerInSecondsDefault; - } - - return expirationProcessTimerInSeconds; - } - } - - public int ExpirationProcessBatchSize - { - get - { - var expirationProcessBatchSize = settings.ExpirationProcessBatchSize; - - if (expirationProcessBatchSize < 1) - { - Logger.Error($"ExpirationProcessBatchSize cannot be less than 1. Defaulting to {ExpirationProcessBatchSizeDefault}"); - return ExpirationProcessBatchSizeDefault; - } - - if (expirationProcessBatchSize < ExpirationProcessBatchSizeMinimum) - { - Logger.Error($"ExpirationProcessBatchSize cannot be less than {ExpirationProcessBatchSizeMinimum}. Defaulting to {ExpirationProcessBatchSizeDefault}"); - return ExpirationProcessBatchSizeDefault; - } - - return expirationProcessBatchSize; - } - } - - public const int ExpirationProcessTimerInSecondsDefault = 600; - public const int ExpirationProcessBatchSizeDefault = 65512; - const int ExpirationProcessBatchSizeMinimum = 10240; - - readonly RavenDBPersisterSettings settings = RavenBootstrapper.Settings; - - static readonly ILog Logger = LogManager.GetLogger(typeof(ExpiredDocumentsCleanerBundle)); - TimerJob timer; - } -} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs b/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs index 64f07ddf32..218685fe74 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs @@ -18,7 +18,6 @@ static class RavenBootstrapper public const string ExposeRavenDBKey = "ExposeRavenDB"; public const string ExpirationProcessTimerInSecondsKey = "ExpirationProcessTimerInSeconds"; public const string ExpirationProcessBatchSizeKey = "ExpirationProcessBatchSize"; - public const string RunCleanupBundleKey = "RavenDB35/RunCleanupBundle"; public const string RunInMemoryKey = "RavenDB35/RunInMemory"; public const string MinimumStorageLeftRequiredForIngestionKey = "MinimumStorageLeftRequiredForIngestion"; @@ -72,13 +71,6 @@ public static void Configure(EmbeddableDocumentStore documentStore, RavenDBPersi documentStore.Configuration.Settings["Raven/AnonymousAccess"] = "Admin"; documentStore.Configuration.Settings["Raven/Licensing/AllowAdminAnonymousAccessForCommercialUse"] = "true"; - var runCleanupBundle = settings.RunCleanupBundle; - - if (runCleanupBundle) - { - documentStore.Configuration.Settings.Add("Raven/ActiveBundles", "CustomDocumentExpiration"); - } - documentStore.Configuration.DisableClusterDiscovery = true; documentStore.Configuration.ResetIndexOnUncleanShutdown = true; diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs index a15e1e4e3f..76dcb15ca7 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs @@ -1,5 +1,4 @@ using System; -using ServiceControl.Infrastructure.RavenDB.Expiration; using ServiceControl.Operations; using ServiceControl.Persistence; @@ -9,9 +8,6 @@ class RavenDBPersisterSettings : PersistenceSettings public int DatabaseMaintenancePort { get; set; } = DatabaseMaintenancePortDefault; public string DatabaseMaintenanceUrl => $"http://{HostName}:{DatabaseMaintenancePort}"; public bool ExposeRavenDB { get; set; } - public int ExpirationProcessTimerInSeconds { get; set; } = ExpiredDocumentsCleanerBundle.ExpirationProcessTimerInSecondsDefault; - public int ExpirationProcessBatchSize { get; set; } = ExpiredDocumentsCleanerBundle.ExpirationProcessBatchSizeDefault; - public bool RunCleanupBundle { get; set; } public bool RunInMemory { get; set; } public int MinimumStorageLeftRequiredForIngestion { get; set; } = CheckMinimumStorageRequiredForIngestion.MinimumStorageLeftRequiredForIngestionDefault; public int DataSpaceRemainingThreshold { get; set; } = CheckFreeDiskSpace.DataSpaceRemainingThresholdDefault; diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs index 2e6ec8bc3b..5609585c5f 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs @@ -1,9 +1,9 @@ namespace ServiceControl.Persistence.RavenDb { using System; + using Raven.Client.Embedded; using ServiceControl.Infrastructure.RavenDB.Expiration; using ServiceControl.Operations; - using Raven.Client.Embedded; class RavenDbPersistenceConfiguration : IPersistenceConfiguration { @@ -48,9 +48,6 @@ T GetSetting(string key, T defaultValue) HostName = GetSetting(RavenBootstrapper.HostNameKey, "localhost"), DatabaseMaintenancePort = GetSetting(RavenBootstrapper.DatabaseMaintenancePortKey, RavenDBPersisterSettings.DatabaseMaintenancePortDefault), ExposeRavenDB = GetSetting(RavenBootstrapper.ExposeRavenDBKey, false), - ExpirationProcessTimerInSeconds = GetSetting(RavenBootstrapper.ExpirationProcessTimerInSecondsKey, ExpiredDocumentsCleanerBundle.ExpirationProcessTimerInSecondsDefault), - ExpirationProcessBatchSize = GetSetting(RavenBootstrapper.ExpirationProcessBatchSizeKey, ExpiredDocumentsCleanerBundle.ExpirationProcessBatchSizeDefault), - RunCleanupBundle = GetSetting(RavenBootstrapper.RunCleanupBundleKey, true), RunInMemory = GetSetting(RavenBootstrapper.RunInMemoryKey, false), MinimumStorageLeftRequiredForIngestion = GetSetting(RavenBootstrapper.MinimumStorageLeftRequiredForIngestionKey, CheckMinimumStorageRequiredForIngestion.MinimumStorageLeftRequiredForIngestionDefault), DataSpaceRemainingThreshold = GetSetting(DataSpaceRemainingThresholdKey, CheckFreeDiskSpace.DataSpaceRemainingThresholdDefault), From b5dca28a9d7e48d6a3daf4137dec34d72dc128ed Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 7 Sep 2023 15:50:49 -0500 Subject: [PATCH 20/78] Try to fix migration (which may not even be needed) --- .../Extensions.cs | 13 +++++++++++++ .../Infrastructure/Migrations/IDataMigration.cs | 2 +- ...WithTemporaryIdsThatAreDuplicateDataMigration.cs | 9 +++++++-- .../QueueAddressStore.cs | 8 ++++---- 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/Extensions.cs b/src/ServiceControl.Persistence.RavenDb5/Extensions.cs index aaab921183..e77d23b740 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Extensions.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Extensions.cs @@ -3,6 +3,7 @@ using System; using System.Threading; using Newtonsoft.Json.Linq; + using Raven.Client.Documents.Conventions; using Raven.Client.Documents.Queries; static class Extensions @@ -15,5 +16,17 @@ public static void Query(this DocumentDatabase db, string index, IndexQu onItem(doc, state); } } + + // TODO: This polyfill of RavenDB 3.5 is a guess based loosely on https://github.com/ravendb/ravendb/blob/v3.5/Raven.Client.Lightweight/Document/DocumentConvention.cs#L151 + public static string DefaultFindFullDocumentKeyFromNonStringIdentifier(this DocumentConventions conventions, T id, Type collectionType, bool allowNull) + { + if (allowNull && id.Equals(default(T))) + { + return null; + } + + var collectionName = conventions.FindCollectionName(collectionType); + return $"{collectionName}{conventions.IdentityPartsSeparator}{id}"; + } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/IDataMigration.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/IDataMigration.cs index 006e2e7c8f..25fe4a0511 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/IDataMigration.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/IDataMigration.cs @@ -1,7 +1,7 @@ namespace ServiceControl.Infrastructure.RavenDB { using System.Threading.Tasks; - using Raven.Client; + using Raven.Client.Documents; interface IDataMigration { diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration.cs index 5f87e6b50f..0b3f0253d4 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration.cs @@ -2,10 +2,11 @@ { using System.Linq; using System.Threading.Tasks; - using Raven.Client; using Raven.Client.Documents; + using Raven.Client.Documents.Commands; using ServiceControl.Persistence; + // TODO: I don't know if we can delete this because no prior Raven5 database will exist, or if it's an ongoing need to purge these things on every startup class PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration : IDataMigration { public Task Migrate(IDocumentStore store) @@ -21,7 +22,11 @@ public Task Migrate(IDocumentStore store) //If we have knowEndpoints with non temp ids, we should delete all temp ids ones. if (fixedIdsCount > 0) { - knownEndpoints.Where(e => e.HasTemporaryId).ForEach(k => { store.DatabaseCommands.Delete(store.Conventions.DefaultFindFullDocumentKeyFromNonStringIdentifier(k.Id, typeof(KnownEndpoint), false), null); }); + foreach (var key in knownEndpoints.Where(e => e.HasTemporaryId)) + { + var documentId = store.Conventions.DefaultFindFullDocumentKeyFromNonStringIdentifier(key.Id, typeof(KnownEndpoint), false); + session.Advanced.RequestExecutor.Execute(new DeleteDocumentCommand(documentId, null), session.Advanced.Context); + } } } } diff --git a/src/ServiceControl.Persistence.RavenDb5/QueueAddressStore.cs b/src/ServiceControl.Persistence.RavenDb5/QueueAddressStore.cs index bb8209c05b..320d0bbe8d 100644 --- a/src/ServiceControl.Persistence.RavenDb5/QueueAddressStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/QueueAddressStore.cs @@ -3,12 +3,12 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; - using ServiceControl.MessageFailures; - using ServiceControl.MessageFailures.Api; - using ServiceControl.Persistence.Infrastructure; using Raven.Client; using Raven.Client.Documents; using Raven.Client.Linq; + using ServiceControl.MessageFailures; + using ServiceControl.MessageFailures.Api; + using ServiceControl.Persistence.Infrastructure; class QueueAddressStore : IQueueAddressStore { @@ -41,9 +41,9 @@ public async Task>> GetAddressesBySearchTerm(str var failedMessageQueues = await session .Query() .Statistics(out var stats) + .Paging(pagingInfo) .Where(q => q.PhysicalAddress.StartsWith(search)) .OrderBy(q => q.PhysicalAddress) - .Paging(pagingInfo) .ToListAsync(); var result = new QueryResult>(failedMessageQueues, stats.ToQueryStatsInfo()); From 88153936d53c686d32f1259e943ae844158a7082 Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 7 Sep 2023 16:05:38 -0500 Subject: [PATCH 21/78] Adding ExpirationProcessTimerInSeconds config back in, we'll need it for the Raven5 document expiration config --- .../RavenDBPersisterSettings.cs | 1 + .../RavenDbPersistenceConfiguration.cs | 1 + 2 files changed, 2 insertions(+) diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs index 76dcb15ca7..da3691b38d 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs @@ -8,6 +8,7 @@ class RavenDBPersisterSettings : PersistenceSettings public int DatabaseMaintenancePort { get; set; } = DatabaseMaintenancePortDefault; public string DatabaseMaintenanceUrl => $"http://{HostName}:{DatabaseMaintenancePort}"; public bool ExposeRavenDB { get; set; } + public int ExpirationProcessTimerInSeconds { get; set; } = ExpiredDocumentsCleanerBundle.ExpirationProcessTimerInSecondsDefault; public bool RunInMemory { get; set; } public int MinimumStorageLeftRequiredForIngestion { get; set; } = CheckMinimumStorageRequiredForIngestion.MinimumStorageLeftRequiredForIngestionDefault; public int DataSpaceRemainingThreshold { get; set; } = CheckFreeDiskSpace.DataSpaceRemainingThresholdDefault; diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs index 5609585c5f..f2eb8ed3f5 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs @@ -48,6 +48,7 @@ T GetSetting(string key, T defaultValue) HostName = GetSetting(RavenBootstrapper.HostNameKey, "localhost"), DatabaseMaintenancePort = GetSetting(RavenBootstrapper.DatabaseMaintenancePortKey, RavenDBPersisterSettings.DatabaseMaintenancePortDefault), ExposeRavenDB = GetSetting(RavenBootstrapper.ExposeRavenDBKey, false), + ExpirationProcessTimerInSeconds = GetSetting(RavenBootstrapper.ExpirationProcessTimerInSecondsKey, ExpiredDocumentsCleanerBundle.ExpirationProcessTimerInSecondsDefault), RunInMemory = GetSetting(RavenBootstrapper.RunInMemoryKey, false), MinimumStorageLeftRequiredForIngestion = GetSetting(RavenBootstrapper.MinimumStorageLeftRequiredForIngestionKey, CheckMinimumStorageRequiredForIngestion.MinimumStorageLeftRequiredForIngestionDefault), DataSpaceRemainingThreshold = GetSetting(DataSpaceRemainingThresholdKey, CheckFreeDiskSpace.DataSpaceRemainingThresholdDefault), From f36d892b5a91be62a391f70cf52c0da7a67e6671 Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 7 Sep 2023 16:15:46 -0500 Subject: [PATCH 22/78] Remove all the manual expiration stuff --- .../Expiration/ErrorMessageCleaner.cs | 170 ------------------ .../Expiration/EventLogItemsCleaner.cs | 110 ------------ .../Expiration/ExpiredDocumentsCleaner.cs | 35 ---- .../Expiration/ExpiryErrorMessageIndex.cs | 21 --- .../Expiration/ExpiryEventLogItemsIndex.cs | 19 -- .../Expiration/ExpiryProcessedMessageIndex.cs | 18 -- .../RavenDbPersistenceConfiguration.cs | 3 +- 7 files changed, 1 insertion(+), 375 deletions(-) delete mode 100644 src/ServiceControl.Persistence.RavenDb5/Expiration/ErrorMessageCleaner.cs delete mode 100644 src/ServiceControl.Persistence.RavenDb5/Expiration/EventLogItemsCleaner.cs delete mode 100644 src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiredDocumentsCleaner.cs delete mode 100644 src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryErrorMessageIndex.cs delete mode 100644 src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryEventLogItemsIndex.cs delete mode 100644 src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryProcessedMessageIndex.cs diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/ErrorMessageCleaner.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/ErrorMessageCleaner.cs deleted file mode 100644 index 863fba192a..0000000000 --- a/src/ServiceControl.Persistence.RavenDb5/Expiration/ErrorMessageCleaner.cs +++ /dev/null @@ -1,170 +0,0 @@ -namespace ServiceControl.Infrastructure.RavenDB.Expiration -{ - using System; - using System.Collections.Generic; - using System.Diagnostics; - using System.Linq; - using System.Threading; - using NServiceBus.Logging; - using ServiceControl.MessageFailures; - using ServiceControl.Recoverability; - using Raven.Client.Documents.Commands.Batches; - using Raven.Client.Documents.Queries; - using Raven.Client.Util; - - static class ErrorMessageCleaner - { - public static void Clean(int deletionBatchSize, DocumentDatabase database, DateTime expiryThreshold, CancellationToken cancellationToken = default) - { - var stopwatch = Stopwatch.StartNew(); - var items = new List(deletionBatchSize); - var attachments = new List(deletionBatchSize); - var failedRetryItems = new List(deletionBatchSize); - var indexName = new ExpiryErrorMessageIndex().IndexName; - - var itemsAndAttachements = new - { - items, - attachments, - failedRetryItems - }; - - try - { - var query = new IndexQuery - { - Start = 0, - PageSize = deletionBatchSize, - Cutoff = SystemTime.UtcNow, - DisableCaching = true, - Query = $"Status:[2 TO 4] AND LastModified:[* TO {expiryThreshold.Ticks}]", - FieldsToFetch = new[] - { - "@id", - "ProcessingAttempts[0].MessageId" - }, - SortedFields = new[] - { - new SortedField("LastModified") - { - Field = "LastModified", - Descending = false - } - } - }; - - database.Query(indexName, query, (doc, state) => - { - var id = doc.Value("@id"); - if (string.IsNullOrEmpty(id)) - { - return; - } - - var failedMessageRetryId = FailedMessageRetry.MakeDocumentId(FailedMessageIdGenerator.GetMessageIdFromDocumentId(id)); - state.failedRetryItems.Add(new DeleteCommandData - { - Key = failedMessageRetryId - }); - - state.items.Add(new DeleteCommandData - { - Key = id - }); - var bodyid = doc.Value("ProcessingAttempts[0].MessageId"); - state.attachments.Add(bodyid); - }, - itemsAndAttachements, cancellationToken); - } - catch (IndexDisabledException ex) - { - logger.Error($"Unable to cleanup error messages. The index ${indexName} was disabled.", ex); - return; - } - catch (OperationCanceledException) - { - logger.Info("Cleanup operation cancelled"); - return; - } - - if (cancellationToken.IsCancellationRequested) - { - return; - } - - var deletedFailedMessageRetry = Chunker.ExecuteInChunks(failedRetryItems.Count, (itemsForBatch, db, s, e) => - { - if (logger.IsDebugEnabled) - { - logger.Debug($"Batching deletion of {s}-{e} FailedMessageRetry documents."); - } - - var results = db.Batch(itemsForBatch.GetRange(s, e - s + 1), CancellationToken.None); - if (logger.IsDebugEnabled) - { - logger.Debug($"Batching deletion of {s}-{e} FailedMessageRetry documents completed."); - } - - return results.Count(x => x.Deleted == true); - }, failedRetryItems, database, cancellationToken); - - var deletedAttachments = Chunker.ExecuteInChunks(attachments.Count, (atts, db, s, e) => - { - var deleted = 0; - if (logger.IsDebugEnabled) - { - logger.Debug($"Batching deletion of {s}-{e} attachment error documents."); - } - - db.TransactionalStorage.Batch(accessor => - { - for (var idx = s; idx <= e; idx++) - { - //We want to continue using attachments for now - accessor.Attachments.DeleteAttachment("messagebodies/" + attachments[idx], null); - deleted++; - } - }); - if (logger.IsDebugEnabled) - { - logger.Debug($"Batching deletion of {s}-{e} attachment error documents completed."); - } - - return deleted; - }, attachments, database, cancellationToken); - - var deletedFailedMessage = Chunker.ExecuteInChunks(items.Count, (itemsForBatch, db, s, e) => - { - if (logger.IsDebugEnabled) - { - logger.Debug($"Batching deletion of {s}-{e} error documents."); - } - - var results = db.Batch(itemsForBatch.GetRange(s, e - s + 1), CancellationToken.None); - if (logger.IsDebugEnabled) - { - logger.Debug($"Batching deletion of {s}-{e} error documents completed."); - } - - return results.Count(x => x.Deleted == true); - }, items, database, cancellationToken); - - if (deletedFailedMessage + deletedAttachments + deletedFailedMessageRetry == 0) - { - if (logger.IsDebugEnabled) - { - logger.Debug("No expired error documents found"); - } - } - else - { - if (logger.IsDebugEnabled) - { - logger.Debug($"Deleted {deletedFailedMessage} expired error documents and {deletedAttachments} message body attachments. Batch execution took {stopwatch.ElapsedMilliseconds} ms"); - } - } - } - - static ILog logger = LogManager.GetLogger(typeof(ErrorMessageCleaner)); - } -} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/EventLogItemsCleaner.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/EventLogItemsCleaner.cs deleted file mode 100644 index 76beb5f8c0..0000000000 --- a/src/ServiceControl.Persistence.RavenDb5/Expiration/EventLogItemsCleaner.cs +++ /dev/null @@ -1,110 +0,0 @@ -namespace ServiceControl.Infrastructure.RavenDB.Expiration -{ - using System; - using System.Collections.Generic; - using System.Diagnostics; - using System.Linq; - using System.Threading; - using NServiceBus.Logging; - using Raven.Client.Documents.Commands.Batches; - using Raven.Client.Documents.Queries; - using Raven.Client.Util; - - static class EventLogItemsCleaner - { - public static void Clean(int deletionBatchSize, DocumentDatabase database, DateTime expiryThreshold, CancellationToken cancellationToken = default) - { - var stopwatch = Stopwatch.StartNew(); - var items = new List(deletionBatchSize); - var indexName = new ExpiryEventLogItemsIndex().IndexName; - - try - { - var query = new IndexQuery - { - Start = 0, - DisableCaching = true, - Cutoff = SystemTime.UtcNow, - PageSize = deletionBatchSize, - Query = $"LastModified:[* TO {expiryThreshold.Ticks}]", - FieldsToFetch = new[] - { - "@id" - }, - SortedFields = new[] - { - new SortedField("LastModified") - { - Field = "LastModified", - Descending = false - } - } - }; - - database.Query(indexName, query, (doc, commands) => - { - var id = doc.Value("@id"); - if (string.IsNullOrEmpty(id)) - { - return; - } - - commands.Add(new DeleteCommandData - { - Key = id - }); - }, - items, cancellationToken); - } - catch (IndexDisabledException ex) - { - logger.Error($"Unable to cleanup event log items. The index ${indexName} was disabled.", ex); - return; - } - catch (OperationCanceledException) - { - logger.Info("Cleanup operation cancelled"); - return; - } - - if (cancellationToken.IsCancellationRequested) - { - return; - } - - var deletionCount = Chunker.ExecuteInChunks(items.Count, (itemsForBatch, db, s, e) => - { - if (logger.IsDebugEnabled) - { - logger.Debug($"Batching deletion of {s}-{e} event log documents."); - } - - var results = db.Batch(itemsForBatch.GetRange(s, e - s + 1), CancellationToken.None); - - if (logger.IsDebugEnabled) - { - logger.Debug($"Batching deletion of {s}-{e} event log documents completed."); - } - - return results.Count(x => x.Deleted == true); - }, items, database, cancellationToken); - - if (deletionCount == 0) - { - if (logger.IsDebugEnabled) - { - logger.Debug("No expired event log documents found"); - } - } - else - { - if (logger.IsDebugEnabled) - { - logger.Debug($"Deleted {deletionCount} expired event log documents. Batch execution took {stopwatch.ElapsedMilliseconds} ms"); - } - } - } - - static ILog logger = LogManager.GetLogger(typeof(EventLogItemsCleaner)); - } -} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiredDocumentsCleaner.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiredDocumentsCleaner.cs deleted file mode 100644 index 22c9d8fa5e..0000000000 --- a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiredDocumentsCleaner.cs +++ /dev/null @@ -1,35 +0,0 @@ -namespace ServiceControl.Infrastructure.RavenDB.Expiration -{ - using System.Globalization; - using System.Threading; - using System.Threading.Tasks; - using BackgroundTasks; - using NServiceBus.Logging; - using Raven.Client.Util; - - class ExpiredDocumentsCleaner - { - public static Task RunCleanup(int deletionBatchSize, DocumentDatabase database, RavenDBPersisterSettings settings, CancellationToken cancellationToken = default) - { - var threshold = SystemTime.UtcNow.Add(settings.ErrorRetentionPeriod); - - if (logger.IsDebugEnabled) - { - logger.Debug($"Trying to find expired FailedMessage documents to delete (with threshold {threshold.ToString(Default.DateTimeFormatsToWrite, CultureInfo.InvariantCulture)})"); - } - ErrorMessageCleaner.Clean(deletionBatchSize, database, threshold, cancellationToken); - - threshold = SystemTime.UtcNow.Add(settings.EventsRetentionPeriod); - - if (logger.IsDebugEnabled) - { - logger.Debug($"Trying to find expired EventLogItem documents to delete (with threshold {threshold.ToString(Default.DateTimeFormatsToWrite, CultureInfo.InvariantCulture)})"); - } - EventLogItemsCleaner.Clean(deletionBatchSize, database, threshold, cancellationToken); - - return Task.FromResult(TimerJobExecutionResult.ScheduleNextExecution); - } - - static ILog logger = LogManager.GetLogger(typeof(ExpiredDocumentsCleaner)); - } -} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryErrorMessageIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryErrorMessageIndex.cs deleted file mode 100644 index f75e41f1e7..0000000000 --- a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryErrorMessageIndex.cs +++ /dev/null @@ -1,21 +0,0 @@ -namespace ServiceControl.Infrastructure.RavenDB.Expiration -{ - using System; - using System.Linq; - using MessageFailures; - using Raven.Client.Documents.Indexes; - - class ExpiryErrorMessageIndex : AbstractIndexCreationTask - { - public ExpiryErrorMessageIndex() - { - Map = messages => from message in messages - where message.Status != FailedMessageStatus.Unresolved - select new - { - message.Status, - LastModified = MetadataFor(message).Value("@last-modified").Ticks - }; - } - } -} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryEventLogItemsIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryEventLogItemsIndex.cs deleted file mode 100644 index 72038d02ff..0000000000 --- a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryEventLogItemsIndex.cs +++ /dev/null @@ -1,19 +0,0 @@ -namespace ServiceControl.Infrastructure.RavenDB.Expiration -{ - using System; - using System.Linq; - using EventLog; - using Raven.Client.Documents.Indexes; - - class ExpiryEventLogItemsIndex : AbstractIndexCreationTask - { - public ExpiryEventLogItemsIndex() - { - Map = messages => from message in messages - select new - { - LastModified = MetadataFor(message).Value("@last-modified").Ticks - }; - } - } -} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryProcessedMessageIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryProcessedMessageIndex.cs deleted file mode 100644 index fc80629418..0000000000 --- a/src/ServiceControl.Persistence.RavenDb5/Expiration/ExpiryProcessedMessageIndex.cs +++ /dev/null @@ -1,18 +0,0 @@ -namespace ServiceControl.Infrastructure.RavenDB.Expiration -{ - using System.Linq; - using MessageAuditing; - using Raven.Client.Documents.Indexes; - - class ExpiryProcessedMessageIndex : AbstractIndexCreationTask - { - public ExpiryProcessedMessageIndex() - { - Map = messages => from message in messages - select new - { - ProcessedAt = message.ProcessedAt.Ticks - }; - } - } -} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs index f2eb8ed3f5..58b5cd8ee9 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs @@ -2,7 +2,6 @@ { using System; using Raven.Client.Embedded; - using ServiceControl.Infrastructure.RavenDB.Expiration; using ServiceControl.Operations; class RavenDbPersistenceConfiguration : IPersistenceConfiguration @@ -48,7 +47,7 @@ T GetSetting(string key, T defaultValue) HostName = GetSetting(RavenBootstrapper.HostNameKey, "localhost"), DatabaseMaintenancePort = GetSetting(RavenBootstrapper.DatabaseMaintenancePortKey, RavenDBPersisterSettings.DatabaseMaintenancePortDefault), ExposeRavenDB = GetSetting(RavenBootstrapper.ExposeRavenDBKey, false), - ExpirationProcessTimerInSeconds = GetSetting(RavenBootstrapper.ExpirationProcessTimerInSecondsKey, ExpiredDocumentsCleanerBundle.ExpirationProcessTimerInSecondsDefault), + ExpirationProcessTimerInSeconds = GetSetting(RavenBootstrapper.ExpirationProcessTimerInSecondsKey, 600), RunInMemory = GetSetting(RavenBootstrapper.RunInMemoryKey, false), MinimumStorageLeftRequiredForIngestion = GetSetting(RavenBootstrapper.MinimumStorageLeftRequiredForIngestionKey, CheckMinimumStorageRequiredForIngestion.MinimumStorageLeftRequiredForIngestionDefault), DataSpaceRemainingThreshold = GetSetting(DataSpaceRemainingThresholdKey, CheckFreeDiskSpace.DataSpaceRemainingThresholdDefault), From e3475303bbcfb69ce77ae98a46f056b2ac8cf2aa Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Fri, 8 Sep 2023 09:17:03 +0200 Subject: [PATCH 23/78] integration events dispatcher --- .../ExternalIntegrationRequestsDataStore.cs | 6 ++---- .../RavenQueryExtensions.cs | 1 - .../Recoverability/Archiving/MessageArchiver.cs | 1 + 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/ExternalIntegrationRequestsDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ExternalIntegrationRequestsDataStore.cs index ad7685e6cd..b9306f2508 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ExternalIntegrationRequestsDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ExternalIntegrationRequestsDataStore.cs @@ -138,8 +138,8 @@ async Task TryDispatchEventBatch() if (awaitingDispatching.Count == 0) { - // If the index hasn't caught up, try again - return stats.IndexEtag.CompareTo(latestEtag) < 0; + //TODO: this should ensure we query again if the result is potentially stale + return stats.IsStale; } var allContexts = awaitingDispatching.Select(r => r.DispatchContext).ToArray(); @@ -169,7 +169,6 @@ public Task StartAsync(CancellationToken cancellationToken) .Where(c => c.Type == DocumentChangeTypes.Put) .Subscribe(d => { - latestEtag = Etag.Max(d.Etag, latestEtag); signal.Set(); }); @@ -207,7 +206,6 @@ public async ValueTask DisposeAsync() readonly RepeatedFailuresOverTimeCircuitBreaker circuitBreaker; IDisposable subscription; - Etag latestEtag = Etag.Empty; Task task; ManualResetEventSlim signal = new ManualResetEventSlim(); Func callback; diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs b/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs index 215291ca86..fd53d78104 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs @@ -12,7 +12,6 @@ namespace ServiceControl.Persistence using Raven.Client; using Raven.Client.Documents.Linq; using Raven.Client.Documents.Session; - using Raven.Client.Linq; static class RavenQueryExtensions { diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/MessageArchiver.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/MessageArchiver.cs index 085c64e7c5..244c149879 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/MessageArchiver.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/MessageArchiver.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using NServiceBus.Logging; using Raven.Client; + using Raven.Client.Documents; using ServiceControl.Infrastructure.DomainEvents; using ServiceControl.Persistence.Recoverability; using ServiceControl.Recoverability; From 9b8a82a860023a08d846c4e740471394370074b4 Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Fri, 8 Sep 2023 09:19:35 +0200 Subject: [PATCH 24/78] small comment --- .../ExternalIntegrationRequestsDataStore.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ServiceControl.Persistence.RavenDb5/ExternalIntegrationRequestsDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ExternalIntegrationRequestsDataStore.cs index b9306f2508..0ea74e7346 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ExternalIntegrationRequestsDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ExternalIntegrationRequestsDataStore.cs @@ -139,6 +139,7 @@ async Task TryDispatchEventBatch() if (awaitingDispatching.Count == 0) { //TODO: this should ensure we query again if the result is potentially stale + // if ☝️ is not true we will need to use/parse the ChangeVector when document is written and compare to ResultEtag return stats.IsStale; } From 3e2e495788728b231d58d437399553dcedd1f3f3 Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Fri, 8 Sep 2023 09:28:35 +0200 Subject: [PATCH 25/78] etag in querystatsinfo --- .../RavenQueryStatisticsExtensions.cs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenQueryStatisticsExtensions.cs b/src/ServiceControl.Persistence.RavenDb5/RavenQueryStatisticsExtensions.cs index b15fda923e..4665d755f1 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenQueryStatisticsExtensions.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenQueryStatisticsExtensions.cs @@ -8,11 +8,13 @@ static class RavenQueryStatisticsExtensions { public static QueryStatsInfo ToQueryStatsInfo(this QueryStatistics stats) { - return new QueryStatsInfo(stats.IndexEtag, stats.TotalResults, stats.IsStale); - } - public static QueryStatsInfo ToQueryStatsInfo(this Raven.Abstractions.Data.QueryResult queryResult) - { - return new QueryStatsInfo(queryResult.IndexEtag, queryResult.TotalResults, queryResult.IsStale); + return new QueryStatsInfo($"{stats.ResultEtag}", stats.TotalResults, stats.IsStale); } + + //TODO: This method can likely be removed + //public static QueryStatsInfo ToQueryStatsInfo(this Raven.Abstractions.Data.QueryResult queryResult) + //{ + // return new QueryStatsInfo(queryResult.IndexEtag, queryResult.TotalResults, queryResult.IsStale); + //} } } \ No newline at end of file From 16712947fdf7c2403449f2e84ecbcc2c351ef23e Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Fri, 8 Sep 2023 09:42:30 +0200 Subject: [PATCH 26/78] subscription storage --- .../RavenDbSubscriptionStorage.cs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/RavenDbSubscriptionStorage.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/RavenDbSubscriptionStorage.cs index a64bd817c3..b56b86263c 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/RavenDbSubscriptionStorage.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/RavenDbSubscriptionStorage.cs @@ -14,6 +14,9 @@ using NServiceBus.Unicast.Subscriptions; using NServiceBus.Unicast.Subscriptions.MessageDrivenSubscriptions; using Raven.Client; + using Raven.Client.Documents; + using Raven.Client.Documents.Commands; + using Raven.Client.Documents.Session; using ServiceControl.Persistence; using ServiceControl.Persistence.RavenDb.Infrastructure; @@ -211,16 +214,14 @@ static async Task MigrateSubscriptions(IAsyncDocumentSession sess var subscriptions = new Subscriptions(); - using (var stream = await session.Advanced.StreamAsync("Subscriptions") - ) + var stream = await session.Advanced.StreamAsync("Subscriptions"); + + while (await stream.MoveNextAsync()) { - while (await stream.MoveNextAsync()) - { - var existingSubscription = stream.Current.Document; - existingSubscription.Subscribers.Remove(localClient); - subscriptions.All.Add(existingSubscription.Id.Replace("Subscriptions/", string.Empty), existingSubscription); - await session.Advanced.DocumentStore.AsyncDatabaseCommands.DeleteAsync(stream.Current.Key, null); - } + var existingSubscription = stream.Current.Document; + existingSubscription.Subscribers.Remove(localClient); + subscriptions.All.Add(existingSubscription.Id.Replace("Subscriptions/", string.Empty), existingSubscription); + await session.Advanced.RequestExecutor.ExecuteAsync(new DeleteDocumentCommand(stream.Current.Id, null), session.Advanced.Context); } await session.StoreAsync(subscriptions, Subscriptions.SingleDocumentId); From 93f1931898201eda4aaed65d5c19d0e9d1c5d419 Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Fri, 8 Sep 2023 09:53:53 +0200 Subject: [PATCH 27/78] custom check data store --- .../QueueAddressStore.cs | 1 - .../RavenDbCustomCheckDataStore.cs | 13 +++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/QueueAddressStore.cs b/src/ServiceControl.Persistence.RavenDb5/QueueAddressStore.cs index 320d0bbe8d..de271501ea 100644 --- a/src/ServiceControl.Persistence.RavenDb5/QueueAddressStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/QueueAddressStore.cs @@ -5,7 +5,6 @@ using System.Threading.Tasks; using Raven.Client; using Raven.Client.Documents; - using Raven.Client.Linq; using ServiceControl.MessageFailures; using ServiceControl.MessageFailures.Api; using ServiceControl.Persistence.Infrastructure; diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs index ff39aef332..4724f03edb 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs @@ -2,15 +2,16 @@ { using System; using System.Collections.Generic; - using System.Linq; using System.Threading.Tasks; using ServiceControl.Contracts.CustomChecks; using ServiceControl.Persistence; using ServiceControl.Persistence.Infrastructure; using Raven.Client; using Raven.Client.Documents; + using Raven.Client.Documents.Commands; using Raven.Client.Documents.Linq; - using Raven.Client.Linq; + using Raven.Client.Documents.Session; + using ServiceControl.Infrastructure.RavenDB; class RavenDbCustomCheckDataStore : ICustomChecksDataStore { @@ -69,13 +70,17 @@ public async Task>> GetStats(PagingInfo paging, s .Paging(paging) .ToListAsync(); - return new QueryResult>(results, new QueryStatsInfo(stats.IndexEtag, stats.TotalResults, stats.IsStale)); + return new QueryResult>(results, new QueryStatsInfo($"{stats.ResultEtag}", stats.TotalResults, stats.IsStale)); } } public async Task DeleteCustomCheck(Guid id) { - await store.AsyncDatabaseCommands.DeleteAsync(store.Conventions.DefaultFindFullDocumentKeyFromNonStringIdentifier(id, typeof(CustomCheck), false), null); + var documentId = store.Conventions.DefaultFindFullDocumentKeyFromNonStringIdentifier(id, typeof(CustomCheck), false); + using (var session = store.OpenAsyncSession(new SessionOptions {NoTracking = true, NoCaching = true})) + { + await session.Advanced.RequestExecutor.ExecuteAsync(new DeleteDocumentCommand(documentId, null), session.Advanced.Context); + } } public async Task GetNumberOfFailedChecks() From 0bc9d54407575800753e909a6ddb07860378189a Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Fri, 8 Sep 2023 12:18:02 +0200 Subject: [PATCH 28/78] ravendb5 configuration and setup --- .../DatabaseSetup.cs | 88 ++++++++++ .../EmbeddedDatabase.cs | 164 ++++++++++++++++++ .../RavenBootstrapper.cs | 4 +- .../RavenDBPersisterSettings.cs | 12 +- .../RavenDbEmbeddedPersistenceLifecycle.cs | 46 +++++ .../RavenDbExternalPersistenceLifecycle.cs | 64 +++++++ .../RavenDbInstaller.cs | 36 +--- .../RavenDbPersistence.cs | 18 +- .../RavenDbPersistenceConfiguration.cs | 12 +- .../RavenDbPersistenceLifecycle.cs | 63 ------- .../Persistence/PersistenceFactory.cs | 1 - 11 files changed, 400 insertions(+), 108 deletions(-) create mode 100644 src/ServiceControl.Persistence.RavenDb5/DatabaseSetup.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/EmbeddedDatabase.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/RavenDbEmbeddedPersistenceLifecycle.cs create mode 100644 src/ServiceControl.Persistence.RavenDb5/RavenDbExternalPersistenceLifecycle.cs delete mode 100644 src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceLifecycle.cs diff --git a/src/ServiceControl.Persistence.RavenDb5/DatabaseSetup.cs b/src/ServiceControl.Persistence.RavenDb5/DatabaseSetup.cs new file mode 100644 index 0000000000..0b53d5782c --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/DatabaseSetup.cs @@ -0,0 +1,88 @@ +namespace ServiceControl.Persistence.RavenDb5 +{ + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using Raven.Client.Documents; + using Raven.Client.Documents.Indexes; + using Raven.Client.Documents.Operations; + using Raven.Client.Documents.Operations.Expiration; + using Raven.Client.Exceptions; + using Raven.Client.Exceptions.Database; + using Raven.Client.ServerWide; + using Raven.Client.ServerWide.Operations; + using ServiceControl.MessageFailures.Api; + using ServiceControl.Operations; + using ServiceControl.Persistence; + using ServiceControl.Recoverability; + + class DatabaseSetup + { + public DatabaseSetup(RavenDBPersisterSettings settings) + { + this.settings = settings; + } + + public async Task Execute(IDocumentStore documentStore, CancellationToken cancellationToken) + { + try + { + await documentStore.Maintenance.ForDatabase(settings.DatabaseName).SendAsync(new GetStatisticsOperation(), cancellationToken); + } + catch (DatabaseDoesNotExistException) + { + try + { + await documentStore.Maintenance.Server + .SendAsync(new CreateDatabaseOperation(new DatabaseRecord(settings.DatabaseName)), cancellationToken); + } + catch (ConcurrencyException) + { + // The database was already created before calling CreateDatabaseOperation + } + } + + var indexList = new List { + new ArchivedGroupsViewIndex(), + new CustomChecksIndex(), + new FailedErrorImportIndex(), + new FailedMessageFacetsIndex(), + new FailedMessageRetries_ByBatch(), + new FailedMessageViewIndex(), + new FailureGroupsViewIndex(), + new GroupCommentIndex(), + new KnownEndpointIndex(), + new MessagesViewIndex(), + new QueueAddressIndex(), + new RetryBatches_ByStatusAndSession(), + new RetryBatches_ByStatus_ReduceInitialBatchSize() + + }; + + //TODO: Handle full text search + //if (settings.EnableFullTextSearch) + //{ + // indexList.Add(new MessagesViewIndexWithFullTextSearch()); + // await documentStore.Maintenance.SendAsync(new DeleteIndexOperation("MessagesViewIndex"), cancellationToken); + //} + //else + //{ + // indexList.Add(new MessagesViewIndex()); + // await documentStore.Maintenance + // .SendAsync(new DeleteIndexOperation("MessagesViewIndexWithFullTextSearch"), cancellationToken); + //} + + await IndexCreation.CreateIndexesAsync(indexList, documentStore, null, null, cancellationToken); + + var expirationConfig = new ExpirationConfiguration + { + Disabled = false, + DeleteFrequencyInSec = settings.ExpirationProcessTimerInSeconds + }; + + await documentStore.Maintenance.SendAsync(new ConfigureExpirationOperation(expirationConfig), cancellationToken); + } + + readonly RavenDBPersisterSettings settings; + } +} diff --git a/src/ServiceControl.Persistence.RavenDb5/EmbeddedDatabase.cs b/src/ServiceControl.Persistence.RavenDb5/EmbeddedDatabase.cs new file mode 100644 index 0000000000..961be13de4 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/EmbeddedDatabase.cs @@ -0,0 +1,164 @@ +namespace ServiceControl.Persistence.RavenDb5 +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.IO; + using System.Threading; + using System.Threading.Tasks; + using NServiceBus.Logging; + using Raven.Client.Documents; + using Raven.Client.Documents.Conventions; + using Raven.Embedded; + + public class EmbeddedDatabase : IDisposable + { + EmbeddedDatabase(RavenDBPersisterSettings configuration) + { + this.configuration = configuration; + ServerUrl = configuration.ServerUrl; + } + + public string ServerUrl { get; private set; } + + static (string LicenseFileName, string ServerDirectory) GetRavenLicenseFileNameAndServerDirectory() + { + var licenseFileName = "RavenLicense.json"; + var localRavenLicense = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, licenseFileName); + if (File.Exists(localRavenLicense)) + { + return (localRavenLicense, null); + } + + //TODO: refactor this to extract the folder name to a constant + localRavenLicense = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Persisters", "RavenDB5", licenseFileName); + if (!File.Exists(localRavenLicense)) + { + throw new Exception($"RavenDB license not found. Make sure the RavenDB license file, '{licenseFileName}', " + + $"is stored in the '{AppDomain.CurrentDomain.BaseDirectory}' folder or in the 'Persisters/RavenDB5' subfolder."); + } + + // By default RavenDB 5 searches its binaries in the RavenDBServer right below the BaseDirectory. + // If we're loading from Persisters/RavenDB5 we also have to signal RavenDB where are binaries + var serverDirectory = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Persisters", "RavenDB5", "RavenDBServer"); + + return (localRavenLicense, serverDirectory); + } + + internal static EmbeddedDatabase Start(RavenDBPersisterSettings settings) + { + var licenseFileNameAndServerDirectory = GetRavenLicenseFileNameAndServerDirectory(); + + var nugetPackagesPath = Path.Combine(settings.DatabasePath, "Packages", "NuGet"); + + logger.InfoFormat("Loading RavenDB license from {0}", licenseFileNameAndServerDirectory.LicenseFileName); + var serverOptions = new ServerOptions + { + CommandLineArgs = new List + { + $"--License.Path=\"{licenseFileNameAndServerDirectory.LicenseFileName}\"", + $"--Logs.Mode={settings.LogsMode}", + // HINT: If this is not set, then Raven will pick a default location relative to the server binaries + // See https://github.com/ravendb/ravendb/issues/15694 + $"--Indexing.NuGetPackagesPath=\"{nugetPackagesPath}\"" + }, + AcceptEula = true, + DataDirectory = settings.DatabasePath, + ServerUrl = settings.ServerUrl, + LogsPath = settings.LogPath + }; + + if (!string.IsNullOrWhiteSpace(licenseFileNameAndServerDirectory.ServerDirectory)) + { + serverOptions.ServerDirectory = licenseFileNameAndServerDirectory.ServerDirectory; + } + + var embeddedDatabase = new EmbeddedDatabase(settings); + + embeddedDatabase.Start(serverOptions); + + return embeddedDatabase; + } + + void Start(ServerOptions serverOptions) + { + EmbeddedServer.Instance.ServerProcessExited += (sender, args) => + { + if (sender is Process process && process.HasExited && process.ExitCode != 0) + { + logger.Warn($"RavenDB server process exited unexpectedly with exitCode: {process.ExitCode}. Process will be restarted."); + + restartRequired = true; + } + }; + + EmbeddedServer.Instance.StartServer(serverOptions); + + var _ = Task.Run(async () => + { + while (!shutdownTokenSource.IsCancellationRequested) + { + try + { + await Task.Delay(delayBetweenRestarts, shutdownTokenSource.Token); + + if (restartRequired) + { + logger.Info("Restarting RavenDB server process"); + + await EmbeddedServer.Instance.RestartServerAsync(); + restartRequired = false; + + logger.Info("RavenDB server process restarted successfully."); + } + } + catch (OperationCanceledException) + { + //no-op + } + catch (Exception e) + { + logger.Fatal($"RavenDB server restart failed. Restart will be retried in {delayBetweenRestarts}.", e); + } + } + }); + } + + public async Task Connect(CancellationToken cancellationToken) + { + var dbOptions = new DatabaseOptions(configuration.DatabaseName) + { + Conventions = new DocumentConventions + { + SaveEnumsAsIntegers = true + } + }; + + //TODO: copied from Audit. In Audit FindClrType so I guess this is not needed. Confirm and remove + //if (configuration.FindClrType != null) + //{ + // dbOptions.Conventions.FindClrType += configuration.FindClrType; + //} + + var store = await EmbeddedServer.Instance.GetDocumentStoreAsync(dbOptions, cancellationToken); + + var databaseSetup = new DatabaseSetup(configuration); + await databaseSetup.Execute(store, cancellationToken); + + return store; + } + + public void Dispose() + { + shutdownTokenSource.Cancel(); + EmbeddedServer.Instance?.Dispose(); + } + + CancellationTokenSource shutdownTokenSource = new CancellationTokenSource(); + bool restartRequired; + readonly RavenDBPersisterSettings configuration; + + static TimeSpan delayBetweenRestarts = TimeSpan.FromSeconds(60); + static readonly ILog logger = LogManager.GetLogger(); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs b/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs index 218685fe74..227498177e 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs @@ -17,10 +17,10 @@ static class RavenBootstrapper public const string DatabaseMaintenancePortKey = "DatabaseMaintenancePort"; public const string ExposeRavenDBKey = "ExposeRavenDB"; public const string ExpirationProcessTimerInSecondsKey = "ExpirationProcessTimerInSeconds"; - public const string ExpirationProcessBatchSizeKey = "ExpirationProcessBatchSize"; public const string RunInMemoryKey = "RavenDB35/RunInMemory"; + public const string ConnectionStringKey = "RavenDB5/ConnectionString"; public const string MinimumStorageLeftRequiredForIngestionKey = "MinimumStorageLeftRequiredForIngestion"; - + public const string DatabaseNameKey = "RavenDB5/DatabaseName"; public static RavenDBPersisterSettings Settings { get; private set; } diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs index da3691b38d..6e597ffdaf 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs @@ -8,7 +8,7 @@ class RavenDBPersisterSettings : PersistenceSettings public int DatabaseMaintenancePort { get; set; } = DatabaseMaintenancePortDefault; public string DatabaseMaintenanceUrl => $"http://{HostName}:{DatabaseMaintenancePort}"; public bool ExposeRavenDB { get; set; } - public int ExpirationProcessTimerInSeconds { get; set; } = ExpiredDocumentsCleanerBundle.ExpirationProcessTimerInSecondsDefault; + public int ExpirationProcessTimerInSeconds { get; set; } = ExpirationProcessTimerInSecondsDefault; public bool RunInMemory { get; set; } public int MinimumStorageLeftRequiredForIngestion { get; set; } = CheckMinimumStorageRequiredForIngestion.MinimumStorageLeftRequiredForIngestionDefault; public int DataSpaceRemainingThreshold { get; set; } = CheckFreeDiskSpace.DataSpaceRemainingThresholdDefault; @@ -16,6 +16,16 @@ class RavenDBPersisterSettings : PersistenceSettings public TimeSpan EventsRetentionPeriod { get; set; } public TimeSpan? AuditRetentionPeriod { get; set; } public int ExternalIntegrationsDispatchingBatchSize { get; set; } = 100; + + //TODO: these are newly added settings, we should remove any duplication + public string ServerUrl { get; set; } + public string ConnectionString { get; set; } + public bool UseEmbeddedServer => !string.IsNullOrWhiteSpace(ConnectionString); + public string LogPath { get; set; } + public string LogsMode { get; set; } + public string DatabaseName { get; set; } = DatabaseNameDefault; + public const string DatabaseNameDefault = "audit"; public const int DatabaseMaintenancePortDefault = 33334; + public const int ExpirationProcessTimerInSecondsDefault = 600; } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbEmbeddedPersistenceLifecycle.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbEmbeddedPersistenceLifecycle.cs new file mode 100644 index 0000000000..7df1fa9489 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbEmbeddedPersistenceLifecycle.cs @@ -0,0 +1,46 @@ +namespace ServiceControl.Persistence.RavenDb5 +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using Raven.Client.Documents; + using ServiceControl.Persistence; + + class RavenDbEmbeddedPersistenceLifecycle : IPersistenceLifecycle + { + public RavenDbEmbeddedPersistenceLifecycle(RavenDBPersisterSettings databaseConfiguration) + { + this.databaseConfiguration = databaseConfiguration; + } + + public IDocumentStore GetDocumentStore() + { + if (documentStore == null) + { + throw new InvalidOperationException("Document store is not available until the persistence have been started"); + } + + return documentStore; + } + + public async Task Start(CancellationToken cancellationToken) + { + database = EmbeddedDatabase.Start(databaseConfiguration); + + documentStore = await database.Connect(cancellationToken); + } + + public Task Stop(CancellationToken cancellationToken) + { + documentStore?.Dispose(); + database?.Dispose(); + + return Task.CompletedTask; + } + + IDocumentStore documentStore; + EmbeddedDatabase database; + + readonly RavenDBPersisterSettings databaseConfiguration; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbExternalPersistenceLifecycle.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbExternalPersistenceLifecycle.cs new file mode 100644 index 0000000000..910afddeb1 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbExternalPersistenceLifecycle.cs @@ -0,0 +1,64 @@ +namespace ServiceControl.Persistence.RavenDb5 +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using Raven.Client.Documents; + using Raven.Client.Documents.Conventions; + using ServiceControl.Persistence; + + class RavenDbExternalPersistenceLifecycle : IPersistenceLifecycle + { + public RavenDbExternalPersistenceLifecycle(RavenDBPersisterSettings settings) + { + this.settings = settings; + } + + public IDocumentStore GetDocumentStore() + { + if (documentStore == null) + { + throw new InvalidOperationException("Document store is not available until the persistence have been started"); + } + + return documentStore; + } + + public async Task Start(CancellationToken cancellationToken) + { + var store = new DocumentStore + { + Database = settings.DatabaseName, + Urls = new[] { settings.ConnectionString }, + Conventions = new DocumentConventions + { + SaveEnumsAsIntegers = true + } + }; + + //TODO: copied from Audit, not sure if needed (never assigned). Check and remove + //if (settings.FindClrType != null) + //{ + // store.Conventions.FindClrType += settings.FindClrType; + //} + + store.Initialize(); + + documentStore = store; + + var databaseSetup = new DatabaseSetup(settings); + await databaseSetup.Execute(store, cancellationToken).ConfigureAwait(false); + } + + public Task Stop(CancellationToken cancellationToken) + { + documentStore?.Dispose(); + + return Task.CompletedTask; + } + + IDocumentStore documentStore; + + readonly RavenDBPersisterSettings settings; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbInstaller.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbInstaller.cs index 867329669e..296df56084 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbInstaller.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbInstaller.cs @@ -1,40 +1,22 @@ -namespace ServiceControl.Persistence.RavenDb +namespace ServiceControl.Persistence.RavenDb5 { using System.Threading; using System.Threading.Tasks; - using NServiceBus.Logging; - using ServiceControl.Infrastructure.RavenDB; - using Raven.Client.Embedded; + using ServiceControl.Persistence; class RavenDbInstaller : IPersistenceInstaller { - public RavenDbInstaller(EmbeddableDocumentStore documentStore, RavenStartup ravenStartup) + public RavenDbInstaller(IPersistenceLifecycle lifecycle) { - this.documentStore = documentStore; - this.ravenStartup = ravenStartup; + this.lifecycle = lifecycle; } - public async Task Install(CancellationToken cancellationToken = default) + public async Task Install(CancellationToken cancellationToken) { - Logger.Info("Database initialization starting"); - documentStore.Initialize(); - Logger.Info("Database initialization complete"); - - Logger.Info("Index creation started"); - await ravenStartup.CreateIndexesAsync(documentStore); - Logger.Info("Index creation complete"); - - Logger.Info("Data migrations starting"); - var purgeTempIdKnownEndpoints = new PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration(); - await purgeTempIdKnownEndpoints.Migrate(documentStore); - Logger.Info("Data migrations complete"); - + await lifecycle.Start(cancellationToken); + await lifecycle.Stop(cancellationToken); } - readonly EmbeddableDocumentStore documentStore; - readonly RavenStartup ravenStartup; - - static readonly ILog Logger = LogManager.GetLogger(typeof(RavenDbPersistenceLifecycle)); - + readonly IPersistenceLifecycle lifecycle; } -} +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs index b83ff20171..74cbab97b0 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs @@ -3,9 +3,8 @@ using MessageRedirects; using Microsoft.Extensions.DependencyInjection; using Persistence.Recoverability; - using Raven.Client; using Raven.Client.Documents; - using Raven.Client.Embedded; + using RavenDb5; using Recoverability; using ServiceControl.CustomChecks; using ServiceControl.Infrastructure.RavenDB.Subscriptions; @@ -19,11 +18,9 @@ class RavenDbPersistence : IPersistence { - public RavenDbPersistence(RavenDBPersisterSettings settings, EmbeddableDocumentStore documentStore, RavenStartup ravenStartup) + public RavenDbPersistence(RavenDBPersisterSettings settings) { this.settings = settings; - this.documentStore = documentStore; - this.ravenStartup = ravenStartup; } public void Configure(IServiceCollection serviceCollection) @@ -78,16 +75,19 @@ public void Configure(IServiceCollection serviceCollection) public IPersistenceLifecycle CreateLifecycle() { - return new RavenDbPersistenceLifecycle(ravenStartup, documentStore); + if (settings.UseEmbeddedServer) + { + return new RavenDbEmbeddedPersistenceLifecycle(settings); + } + + return new RavenDbExternalPersistenceLifecycle(settings); } public IPersistenceInstaller CreateInstaller() { - return new RavenDbInstaller(documentStore, ravenStartup); + return new RavenDbInstaller(CreateLifecycle()); } - readonly RavenStartup ravenStartup; readonly RavenDBPersisterSettings settings; - readonly EmbeddableDocumentStore documentStore; } } diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs index 58b5cd8ee9..10e73dd5d7 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs @@ -1,7 +1,6 @@ namespace ServiceControl.Persistence.RavenDb { using System; - using Raven.Client.Embedded; using ServiceControl.Operations; class RavenDbPersistenceConfiguration : IPersistenceConfiguration @@ -43,6 +42,8 @@ T GetSetting(string key, T defaultValue) var settings = new RavenDBPersisterSettings { + ConnectionString = GetSetting(RavenBootstrapper.ConnectionStringKey, default), + DatabaseName = GetSetting(RavenBootstrapper.DatabaseNameKey, RavenDBPersisterSettings.DatabaseNameDefault), DatabasePath = GetSetting(RavenBootstrapper.DatabasePathKey, default), HostName = GetSetting(RavenBootstrapper.HostNameKey, "localhost"), DatabaseMaintenancePort = GetSetting(RavenBootstrapper.DatabaseMaintenancePortKey, RavenDBPersisterSettings.DatabaseMaintenancePortDefault), @@ -67,11 +68,12 @@ public IPersistence Create(PersistenceSettings settings) { var specificSettings = (RavenDBPersisterSettings)settings; - var documentStore = new EmbeddableDocumentStore(); - RavenBootstrapper.Configure(documentStore, specificSettings); + //var documentStore = new EmbeddableDocumentStore(); + //RavenBootstrapper.Configure(documentStore, specificSettings); - var ravenStartup = new RavenStartup(); - return new RavenDbPersistence(specificSettings, documentStore, ravenStartup); + //var ravenStartup = new RavenStartup(); + + return new RavenDbPersistence(specificSettings); } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceLifecycle.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceLifecycle.cs deleted file mode 100644 index 73fe1be23f..0000000000 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceLifecycle.cs +++ /dev/null @@ -1,63 +0,0 @@ -namespace ServiceControl.Persistence.RavenDb -{ - using System; - using System.Threading; - using System.Threading.Tasks; - using NServiceBus.Logging; - using ServiceControl.Persistence; - using Raven.Client; - using Raven.Client.Documents; - using Raven.Client.Embedded; - - class RavenDbPersistenceLifecycle : IPersistenceLifecycle - { - public RavenDbPersistenceLifecycle(RavenStartup ravenStartup, EmbeddableDocumentStore documentStore) - { - this.ravenStartup = ravenStartup; - this.documentStore = documentStore; - } - - public async Task Start(CancellationToken cancellationToken) - { - Logger.Info("Database initialization starting"); - documentStore.Initialize(); - Logger.Info("Database initialization complete"); - - await ravenStartup.CreateIndexesAsync(documentStore); - - Logger.Info("Testing indexes"); - await TestAllIndexesAndResetIfException(documentStore); - } - - public Task Stop(CancellationToken cancellationToken) - { - documentStore.Dispose(); - return Task.CompletedTask; - } - - static async Task TestAllIndexesAndResetIfException(IDocumentStore store) - { - foreach (var index in store.DatabaseCommands.GetStatistics().Indexes) - { - try - { - using (var session = store.OpenAsyncSession()) - { - await session.Advanced.AsyncDocumentQuery(index.Name).Take(1).ToListAsync(); - } - } - catch (Exception ex) - { - Logger.Warn($"When trying to fetch 1 document from index {index.Name} the following exception was thrown: {ex}"); - Logger.Warn($"Attempting to reset errored index: [{index.Name}] priority: {index.Priority} is valid: {index.IsInvalidIndex} indexing attempts: {index.IndexingAttempts}, failed indexing attempts:{index.IndexingErrors}"); - store.DatabaseCommands.ResetIndex(index.Name); - } - } - } - - readonly RavenStartup ravenStartup; - readonly EmbeddableDocumentStore documentStore; - - static readonly ILog Logger = LogManager.GetLogger(typeof(RavenDbPersistenceLifecycle)); - } -} \ No newline at end of file diff --git a/src/ServiceControl/Persistence/PersistenceFactory.cs b/src/ServiceControl/Persistence/PersistenceFactory.cs index 9345e97b56..80dfdb4bb2 100644 --- a/src/ServiceControl/Persistence/PersistenceFactory.cs +++ b/src/ServiceControl/Persistence/PersistenceFactory.cs @@ -3,7 +3,6 @@ namespace ServiceControl.Persistence using System; using System.IO; using System.Linq; - using System.Web.Hosting; using ServiceBus.Management.Infrastructure.Settings; static class PersistenceFactory From 68f0973f82b719fa158437e7003d442eb43b3bdd Mon Sep 17 00:00:00 2001 From: David Boike Date: Fri, 8 Sep 2023 14:54:36 -0500 Subject: [PATCH 29/78] Kill unsupported transformer, talk about complicated --- ...ssages_UniqueMessageIdAndTimeOfFailures.cs | 26 --- .../RetryDocumentDataStore.cs | 160 +++++++----------- ...ServiceControl.Persistence.RavenDb5.csproj | 1 + 3 files changed, 66 insertions(+), 121 deletions(-) delete mode 100644 src/ServiceControl.Persistence.RavenDb5/BulkRetry/FailedMessages_UniqueMessageIdAndTimeOfFailures.cs diff --git a/src/ServiceControl.Persistence.RavenDb5/BulkRetry/FailedMessages_UniqueMessageIdAndTimeOfFailures.cs b/src/ServiceControl.Persistence.RavenDb5/BulkRetry/FailedMessages_UniqueMessageIdAndTimeOfFailures.cs deleted file mode 100644 index c06a7b3a6e..0000000000 --- a/src/ServiceControl.Persistence.RavenDb5/BulkRetry/FailedMessages_UniqueMessageIdAndTimeOfFailures.cs +++ /dev/null @@ -1,26 +0,0 @@ -namespace ServiceControl.Recoverability -{ - using System; - using System.Linq; - using MessageFailures; - - class FailedMessages_UniqueMessageIdAndTimeOfFailures : AbstractTransformerCreationTask // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers - { - public FailedMessages_UniqueMessageIdAndTimeOfFailures() - { - TransformResults = failedMessages => from failedMessage in failedMessages - select new - { - failedMessage.UniqueMessageId, - LatestTimeOfFailure = failedMessage.ProcessingAttempts.Max(x => x.FailureDetails.TimeOfFailure) - }; - } - - public struct Result - { - public string UniqueMessageId { get; set; } - - public DateTime LatestTimeOfFailure { get; set; } - } - } -} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs index 8b6b791423..32c15e560b 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs @@ -3,19 +3,15 @@ using System; using System.Collections.Generic; using System.Linq; - using System.Linq.Expressions; using System.Threading.Tasks; using MessageFailures; using Newtonsoft.Json.Linq; using NServiceBus.Logging; using Persistence.Infrastructure; - using Raven.Client; using Raven.Client.Documents; - using Raven.Client.Documents.Commands; using Raven.Client.Documents.Commands.Batches; - using Raven.Client.Documents.Indexes; + using Raven.Client.Documents.Linq; using Raven.Client.Documents.Operations; - using Raven.Client.Documents.Session; using Raven.Client.Exceptions; using ServiceControl.MessageFailures.Api; using ServiceControl.Recoverability; @@ -153,7 +149,11 @@ static ICommandData CreateFailedMessageRetryDocument(string batchDocumentId, str static ILog log = LogManager.GetLogger(typeof(RetryDocumentDataStore)); - + // TODO: Verify Stream queries in this file, which were the result of joining overly-complex IndexBasedBulkRetryRequest + // which was in this file, as well as the FailedMessages_UniqueMessageIdAndTimeOfFailures transformer, since transformers + // are not supported in RavenDB 5. I don't know what all the other properties of IndexBasedBulkRetryRequest were ever for, + // since they weren't used in this class. I also don't know what the other comments that were in each streaming query method + // were for either. public async Task GetBatchesForAll(DateTime cutoff, Func callback) { @@ -161,22 +161,23 @@ public async Task GetBatchesForAll(DateTime cutoff, Func //public void StartRetryForIndex(string requestId, RetryType retryType, DateTime startTime, Expression> filter = null, string originator = null, string classifier = null) //StartRetryForIndex(endpoint, RetryType.AllForEndpoint, DateTime.UtcNow, m => m.ReceivingEndpointName == endpoint, $"all messages for endpoint {endpoint}"); - var x = new IndexBasedBulkRetryRequest( - "All", - RetryType.All, - originator: "all messages", - classifier: null, - cutoff, - filter: null - ); - using (var session = store.OpenAsyncSession()) - using (var stream = await x.GetDocuments(session)) { - while (await stream.MoveNextAsync()) + var query = session.Query() + .Where(d => d.Status == FailedMessageStatus.Unresolved) + .Select(m => new + { + UniqueMessageId = m.MessageId, + LatestTimeOfFailure = m.TimeOfFailure + }); + + await using (var stream = await session.Advanced.StreamAsync(query)) { - var current = stream.Current.Document; - await callback(current.UniqueMessageId, current.LatestTimeOfFailure); + while (await stream.MoveNextAsync()) + { + var current = stream.Current.Document; + await callback(current.UniqueMessageId, current.LatestTimeOfFailure); + } } } } @@ -186,22 +187,24 @@ public async Task GetBatchesForEndpoint(DateTime cutoff, string endpoint, Func //StartRetryForIndex(endpoint, RetryType.AllForEndpoint, DateTime.UtcNow, m => m.ReceivingEndpointName == endpoint, $"all messages for endpoint {endpoint}"); - var x = new IndexBasedBulkRetryRequest( - endpoint, - RetryType.AllForEndpoint, - originator: $"all messages for endpoint {endpoint}", - classifier: null, - cutoff, - filter: m => m.ReceivingEndpointName == endpoint - ); - using (var session = store.OpenAsyncSession()) - using (var stream = await x.GetDocuments(session)) { - while (await stream.MoveNextAsync()) + var query = session.Query() + .Where(d => d.Status == FailedMessageStatus.Unresolved) + .Where(m => m.ReceivingEndpointName == endpoint) + .Select(m => new + { + UniqueMessageId = m.MessageId, + LatestTimeOfFailure = m.TimeOfFailure + }); + + await using (var stream = await session.Advanced.StreamAsync(query)) { - var current = stream.Current.Document; - await callback(current.UniqueMessageId, current.LatestTimeOfFailure); + while (await stream.MoveNextAsync()) + { + var current = stream.Current.Document; + await callback(current.UniqueMessageId, current.LatestTimeOfFailure); + } } } } @@ -211,22 +214,24 @@ public async Task GetBatchesForFailedQueueAddress(DateTime cutoff, string failed //ForIndex //StartRetryForIndex(failedQueueAddress, RetryType.ByQueueAddress, DateTime.UtcNow, m => m.QueueAddress == failedQueueAddress && m.Status == status, ); - var x = new IndexBasedBulkRetryRequest( - failedQueueAddress, - RetryType.AllForEndpoint, - originator: $"all messages for failed queue address '{failedQueueAddress}'", - classifier: null, - cutoff, - filter: m => m.QueueAddress == failedQueueAddress && m.Status == status - ); - using (var session = store.OpenAsyncSession()) - using (var stream = await x.GetDocuments(session)) { - while (await stream.MoveNextAsync()) + var query = session.Query() + .Where(d => d.Status == FailedMessageStatus.Unresolved) + .Where(m => m.QueueAddress == failedQueueAddress && m.Status == status) + .Select(m => new + { + UniqueMessageId = m.MessageId, + LatestTimeOfFailure = m.TimeOfFailure + }); + + await using (var stream = await session.Advanced.StreamAsync(query)) { - var current = stream.Current.Document; - await callback(current.UniqueMessageId, current.LatestTimeOfFailure); + while (await stream.MoveNextAsync()) + { + var current = stream.Current.Document; + await callback(current.UniqueMessageId, current.LatestTimeOfFailure); + } } } } @@ -235,22 +240,24 @@ public async Task GetBatchesForFailureGroup(string groupId, string groupTitle, s { //retries.StartRetryForIndex(message.GroupId, RetryType.FailureGroup, started, x => x.FailureGroupId == message.GroupId, originator, group?.Type); - var x = new IndexBasedBulkRetryRequest( - groupId, - RetryType.AllForEndpoint, - originator: groupTitle, - classifier: groupType, - cutoff, - filter: m => m.FailureGroupId == groupId - ); - using (var session = store.OpenAsyncSession()) - using (var stream = await x.GetDocuments(session)) { - while (await stream.MoveNextAsync()) + var query = session.Query() + .Where(d => d.Status == FailedMessageStatus.Unresolved) + .Where(m => m.FailureGroupId == groupId) + .Select(m => new + { + UniqueMessageId = m.MessageId, + LatestTimeOfFailure = m.TimeOfFailure + }); + + await using (var stream = await session.Advanced.StreamAsync(query)) { - var current = stream.Current.Document; - await callback(current.UniqueMessageId, current.LatestTimeOfFailure); + while (await stream.MoveNextAsync()) + { + var current = stream.Current.Document; + await callback(current.UniqueMessageId, current.LatestTimeOfFailure); + } } } } @@ -264,42 +271,5 @@ public async Task QueryFailureGroupViewOnGroupId(string groupI return group; } } - - class IndexBasedBulkRetryRequest - where TIndex : AbstractIndexCreationTask, new() - where TType : IHaveStatus - { - public IndexBasedBulkRetryRequest(string requestId, RetryType retryType, string originator, string classifier, DateTime startTime, Expression> filter) - { - RequestId = requestId; - RetryType = retryType; - Originator = originator; - this.filter = filter; - StartTime = startTime; - Classifier = classifier; - } - - public string RequestId { get; set; } - public RetryType RetryType { get; set; } - public string Originator { get; set; } - public string Classifier { get; set; } - public DateTime StartTime { get; set; } - - public Task>> GetDocuments(IAsyncDocumentSession session) - { - var query = session.Query(); - - query = query.Where(d => d.Status == FailedMessageStatus.Unresolved); - - if (filter != null) - { - query = query.Where(filter); - } - - return session.Advanced.StreamAsync(query.TransformWith()); - } - - readonly Expression> filter; - } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/ServiceControl.Persistence.RavenDb5.csproj b/src/ServiceControl.Persistence.RavenDb5/ServiceControl.Persistence.RavenDb5.csproj index 444aaf313a..85fe63bef3 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ServiceControl.Persistence.RavenDb5.csproj +++ b/src/ServiceControl.Persistence.RavenDb5/ServiceControl.Persistence.RavenDb5.csproj @@ -2,6 +2,7 @@ net472 + 8.0 From 4a4d7500a7c3c6423ddb568a581a33b9c6a53a81 Mon Sep 17 00:00:00 2001 From: David Boike Date: Fri, 8 Sep 2023 14:55:48 -0500 Subject: [PATCH 30/78] Couple unused usings, rest of file fine --- .../Recoverability/GroupsDataStore.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/GroupsDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/GroupsDataStore.cs index 217eba1bcd..49577f38ad 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Recoverability/GroupsDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/GroupsDataStore.cs @@ -4,12 +4,10 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; - using ServiceControl.MessageFailures; - using ServiceControl.Recoverability; - using Raven.Client; using Raven.Client.Documents; using Raven.Client.Documents.Linq; - using Raven.Client.Linq; + using ServiceControl.MessageFailures; + using ServiceControl.Recoverability; class GroupsDataStore : IGroupsDataStore { From 05cd8f3167baa068a0b3427d35be5bb4e99950b8 Mon Sep 17 00:00:00 2001 From: David Boike Date: Fri, 8 Sep 2023 15:00:20 -0500 Subject: [PATCH 31/78] Won't be any legacy subscriptions stored in a brand new Raven5 DB --- .../Subscriptions/LegacyAddress.cs | 97 ------------------- .../Subscriptions/Subscription.cs | 31 +----- .../SubscriptionsLegacyAddressConverter.cs | 50 ---------- .../RavenBootstrapper.cs | 1 - 4 files changed, 1 insertion(+), 178 deletions(-) delete mode 100644 src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/LegacyAddress.cs delete mode 100644 src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/SubscriptionsLegacyAddressConverter.cs diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/LegacyAddress.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/LegacyAddress.cs deleted file mode 100644 index 026bb13933..0000000000 --- a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/LegacyAddress.cs +++ /dev/null @@ -1,97 +0,0 @@ -namespace ServiceControl.Infrastructure.RavenDB.Subscriptions -{ - using System; - using System.Collections.Generic; - using Newtonsoft.Json.Linq; - - class LegacyAddress - { - public string Queue { get; set; } - public string Machine { get; set; } - public static List ParseMultipleToSubscriptionClient(List addresses) => addresses.ConvertAll(ParseToSubscriptionClient); - public static List ConvertMultipleToLegacyAddress(List subscriptions) => subscriptions.ConvertAll(ConvertToLegacyAddress); - - public static SubscriptionClient ParseToSubscriptionClient(LegacyAddress address) - { - var queue = address.Queue; - var machine = address.Machine; - - // Previously known as IgnoreMachineName (for brokers) - if (string.IsNullOrEmpty(machine)) - { - return new SubscriptionClient - { - TransportAddress = queue, - Endpoint = null - }; - } - - return new SubscriptionClient - { - TransportAddress = queue + "@" + machine, - Endpoint = null - }; - } - - public static LegacyAddress ConvertToLegacyAddress(SubscriptionClient client) - { - var atIndex = client.TransportAddress?.IndexOf("@", StringComparison.InvariantCulture); - - if (atIndex.HasValue && atIndex.Value > -1) - { - return new LegacyAddress - { - Queue = client.TransportAddress.Substring(0, atIndex.Value), - Machine = client.TransportAddress.Substring(atIndex.Value + 1) - }; - } - - return new LegacyAddress - { - Queue = client.TransportAddress, - Machine = null - }; - } - - public static string ParseToString(Func tokenSelector) - { - var token = tokenSelector(); - - // When we have the new timeout data we just return the value - if (token.Type == JTokenType.String) - { - return token.Value(); - } - - var queue = token.Value("Queue"); - var machine = token.Value("Machine"); - - // Previously known as IgnoreMachineName (for brokers) - if (string.IsNullOrEmpty(machine)) - { - return queue; - } - - return queue + "@" + machine; - } - - public override bool Equals(object obj) - { - if (obj is null) - { - return false; - } - - if (ReferenceEquals(this, obj)) - { - return true; - } - - return obj is LegacyAddress address && Equals(address); - } - - bool Equals(LegacyAddress obj) => string.Equals(Queue, obj.Queue) && string.Equals(Machine, obj.Machine); - - public override int GetHashCode() => Queue.GetHashCode() ^ Machine.GetHashCode(); - } -} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/Subscription.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/Subscription.cs index f5feba6157..51a1d8e541 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/Subscription.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/Subscription.cs @@ -11,35 +11,6 @@ class Subscription [JsonConverter(typeof(MessageTypeConverter))] public MessageType MessageType { get; set; } - public List Subscribers - { - get - { - if (subscribers == null) - { - subscribers = new List(); - } - - return subscribers; - } - set => subscribers = value; - } - - [JsonProperty("Clients")] - public List LegacySubscriptions - { - get - { - if (legacySubscriptions == null) - { - legacySubscriptions = new List(); - } - - return legacySubscriptions; - } - } - - List subscribers; - List legacySubscriptions; + public List Subscribers { get; set; } = new List(); } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/SubscriptionsLegacyAddressConverter.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/SubscriptionsLegacyAddressConverter.cs deleted file mode 100644 index 67b80e2502..0000000000 --- a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/SubscriptionsLegacyAddressConverter.cs +++ /dev/null @@ -1,50 +0,0 @@ -namespace ServiceControl.Infrastructure.RavenDB.Subscriptions -{ - using System.Linq; - using Newtonsoft.Json.Linq; - using Raven.Client.Listeners; - - class SubscriptionsLegacyAddressConverter : IDocumentConversionListener - { - public void BeforeConversionToDocument(string key, object entity, JObject metadata) - { - if (!(entity is Subscription subscription)) - { - return; - } - - var converted = LegacyAddress.ConvertMultipleToLegacyAddress(subscription.Subscribers); - subscription.LegacySubscriptions.Clear(); - subscription.LegacySubscriptions.AddRange(converted); - } - - public void AfterConversionToDocument(string key, object entity, JObject document, JObject metadata) - { - } - - public void BeforeConversionToEntity(string key, JObject document, JObject metadata) - { - } - - public void AfterConversionToEntity(string key, JObject document, JObject metadata, object entity) - { - if (!(entity is Subscription subscription)) - { - return; - } - - var clients = document["Clients"]; - - if (clients != null) - { - var converted = LegacyAddress.ParseMultipleToSubscriptionClient(subscription.LegacySubscriptions); - - var legacySubscriptions = converted.Except(subscription.Subscribers).ToArray(); - foreach (var legacySubscription in legacySubscriptions) - { - subscription.Subscribers.Add(legacySubscription); - } - } - } - } -} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs b/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs index 227498177e..4ccad009f4 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs @@ -47,7 +47,6 @@ public static void Configure(EmbeddableDocumentStore documentStore, RavenDBPersi documentStore.DataDirectory = dbPath; documentStore.Configuration.CompiledIndexCacheDirectory = dbPath; - documentStore.Listeners.RegisterListener(new SubscriptionsLegacyAddressConverter()); } var exposeRavenDB = settings.ExposeRavenDB; From 794003e95ca34ed70a1eb6381ab435dbea9a35b5 Mon Sep 17 00:00:00 2001 From: David Boike Date: Fri, 8 Sep 2023 15:03:24 -0500 Subject: [PATCH 32/78] Believe this was to support the expiration bundle, could be indexes but that should be handled separate anyway --- src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs b/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs index 4ccad009f4..19aa3c854f 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs @@ -1,7 +1,6 @@ namespace ServiceControl.Persistence.RavenDb { using System; - using System.ComponentModel.Composition.Hosting; using System.Globalization; using System.IO; using System.Runtime.Serialization; @@ -92,7 +91,6 @@ public static void Configure(EmbeddableDocumentStore documentStore, RavenDBPersi : hostName; documentStore.Conventions.SaveEnumsAsIntegers = true; documentStore.Conventions.CustomizeJsonSerializer = serializer => serializer.Binder = MigratedTypeAwareBinder; - documentStore.Configuration.Catalog.Catalogs.Add(new AssemblyCatalog(typeof(RavenBootstrapper).Assembly)); documentStore.Conventions.FindClrType = (id, doc, metadata) => { var clrtype = metadata.Value(Constants.RavenClrType); From 23780e8c4e6d4b92b9434af09be7497c71ea01e3 Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Mon, 11 Sep 2023 10:55:57 +0200 Subject: [PATCH 33/78] cleanup RavenBootstrapper --- .../EmbeddedDatabase.cs | 71 ++++++ .../RavenBootstrapper.cs | 203 ------------------ 2 files changed, 71 insertions(+), 203 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/EmbeddedDatabase.cs b/src/ServiceControl.Persistence.RavenDb5/EmbeddedDatabase.cs index 961be13de4..536d5247fc 100644 --- a/src/ServiceControl.Persistence.RavenDb5/EmbeddedDatabase.cs +++ b/src/ServiceControl.Persistence.RavenDb5/EmbeddedDatabase.cs @@ -3,9 +3,11 @@ using System; using System.Collections.Generic; using System.Diagnostics; + using System.Globalization; using System.IO; using System.Threading; using System.Threading.Tasks; + using ByteSizeLib; using NServiceBus.Logging; using Raven.Client.Documents; using Raven.Client.Documents.Conventions; @@ -77,6 +79,8 @@ internal static EmbeddedDatabase Start(RavenDBPersisterSettings settings) embeddedDatabase.Start(serverOptions); + RecordStartup(settings); + return embeddedDatabase; } @@ -154,6 +158,73 @@ public void Dispose() EmbeddedServer.Instance?.Dispose(); } + static void RecordStartup(RavenDBPersisterSettings settings) + { + var dataSize = DataSize(settings); + var folderSize = FolderSize(settings); + + var startupMessage = $@" +------------------------------------------------------------- +Database Size: {ByteSize.FromBytes(dataSize).ToString("#.##", CultureInfo.InvariantCulture)} +Database Folder Size: {ByteSize.FromBytes(folderSize).ToString("#.##", CultureInfo.InvariantCulture)} +-------------------------------------------------------------"; + + logger.Info(startupMessage); + } + + static long DataSize(RavenDBPersisterSettings settings) + { + var datafilePath = Path.Combine(settings.DatabasePath, "data"); + + try + { + var info = new FileInfo(datafilePath); + if (!info.Exists) + { + return -1; + } + return info.Length; + } + catch + { + return -1; + } + } + + static long FolderSize(RavenDBPersisterSettings settings) + { + try + { + var dir = new DirectoryInfo(settings.DatabasePath); + var dirSize = DirSize(dir); + return dirSize; + } + catch + { + return -1; + } + } + + static long DirSize(DirectoryInfo d) + { + long size = 0; + if (d.Exists) + { + FileInfo[] fis = d.GetFiles(); + foreach (FileInfo fi in fis) + { + size += fi.Length; + } + + DirectoryInfo[] dis = d.GetDirectories(); + foreach (DirectoryInfo di in dis) + { + size += DirSize(di); + } + } + + return size; + } CancellationTokenSource shutdownTokenSource = new CancellationTokenSource(); bool restartRequired; readonly RavenDBPersisterSettings configuration; diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs b/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs index 19aa3c854f..c4f721cae5 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs @@ -1,14 +1,5 @@ namespace ServiceControl.Persistence.RavenDb { - using System; - using System.Globalization; - using System.IO; - using System.Runtime.Serialization; - using ByteSizeLib; - using NServiceBus.Logging; - using ServiceControl.Infrastructure.RavenDB; - using ServiceControl.Infrastructure.RavenDB.Subscriptions; - static class RavenBootstrapper { public const string DatabasePathKey = "DbPath"; @@ -20,199 +11,5 @@ static class RavenBootstrapper public const string ConnectionStringKey = "RavenDB5/ConnectionString"; public const string MinimumStorageLeftRequiredForIngestionKey = "MinimumStorageLeftRequiredForIngestion"; public const string DatabaseNameKey = "RavenDB5/DatabaseName"; - - public static RavenDBPersisterSettings Settings { get; private set; } - - public static void Configure(EmbeddableDocumentStore documentStore, RavenDBPersisterSettings settings) - { - Settings = settings; - - var runInMemory = settings.RunInMemory; - - if (runInMemory) - { - documentStore.RunInMemory = true; - } - else - { - var dbPath = settings.DatabasePath; - - if (string.IsNullOrEmpty(dbPath)) - { - throw new InvalidOperationException($"{DatabasePathKey} is mandatory"); - } - - Directory.CreateDirectory(dbPath); - - documentStore.DataDirectory = dbPath; - documentStore.Configuration.CompiledIndexCacheDirectory = dbPath; - } - - var exposeRavenDB = settings.ExposeRavenDB; - - documentStore.UseEmbeddedHttpServer = settings.MaintenanceMode || exposeRavenDB; - documentStore.EnlistInDistributedTransactions = false; - - var localRavenLicense = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "RavenLicense.xml"); - if (File.Exists(localRavenLicense)) - { - Logger.InfoFormat("Loading RavenDB license found from {0}", localRavenLicense); - documentStore.Configuration.Settings["Raven/License"] = ReadAllTextWithoutLocking(localRavenLicense); - } - else - { - Logger.InfoFormat("Loading Embedded RavenDB license"); - documentStore.Configuration.Settings["Raven/License"] = ReadLicense(); - } - - //This is affects only remote access to the database in maintenance mode and enables access without authentication - documentStore.Configuration.Settings["Raven/AnonymousAccess"] = "Admin"; - documentStore.Configuration.Settings["Raven/Licensing/AllowAdminAnonymousAccessForCommercialUse"] = "true"; - - documentStore.Configuration.DisableClusterDiscovery = true; - documentStore.Configuration.ResetIndexOnUncleanShutdown = true; - - if (settings.DatabaseMaintenancePort == 0) - { - throw new Exception($"{DatabaseMaintenancePortKey} is mandatory."); - } - - documentStore.Configuration.Port = settings.DatabaseMaintenancePort; - - if (string.IsNullOrEmpty(settings.HostName)) - { - throw new Exception($"{HostNameKey} is mandatory."); - } - - var hostName = settings.HostName; - - documentStore.Configuration.HostName = hostName == "*" || hostName == "+" - ? "localhost" - : hostName; - documentStore.Conventions.SaveEnumsAsIntegers = true; - documentStore.Conventions.CustomizeJsonSerializer = serializer => serializer.Binder = MigratedTypeAwareBinder; - documentStore.Conventions.FindClrType = (id, doc, metadata) => - { - var clrtype = metadata.Value(Constants.RavenClrType); - - // The CLR type cannot be assumed to be always there - if (clrtype == null) - { - return null; - } - - if (clrtype.EndsWith(".Subscription, NServiceBus.Core")) - { - clrtype = ReflectionUtil.GetFullNameWithoutVersionInformation(typeof(Subscription)); - } - else if (clrtype.EndsWith(".Subscription, NServiceBus.RavenDB")) - { - clrtype = ReflectionUtil.GetFullNameWithoutVersionInformation(typeof(Subscription)); - } - - return clrtype; - }; - - if (settings.MaintenanceMode) - { - Logger.InfoFormat($"RavenDB is now accepting requests on {settings.DatabaseMaintenanceUrl}"); - } - - if (settings.RunInMemory == false) - { - RecordStartup(); - } - } - - static void RecordStartup() - { - var dataSize = DataSize(); - var folderSize = FolderSize(); - - var startupMessage = $@" -------------------------------------------------------------- -Database Size: {ByteSize.FromBytes(dataSize).ToString("#.##", CultureInfo.InvariantCulture)} -Database Folder Size: {ByteSize.FromBytes(folderSize).ToString("#.##", CultureInfo.InvariantCulture)} --------------------------------------------------------------"; - - Logger.Info(startupMessage); - } - - public static string ReadLicense() - { - using (var resourceStream = typeof(RavenBootstrapper).Assembly.GetManifestResourceStream("ServiceControl.Persistence.RavenDb.RavenLicense.xml")) - using (var reader = new StreamReader(resourceStream)) - { - return reader.ReadToEnd(); - } - } - - static string ReadAllTextWithoutLocking(string path) - { - using (var fileStream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) - using (var textReader = new StreamReader(fileStream)) - { - return textReader.ReadToEnd(); - } - } - - static long DataSize() - { - var datafilePath = Path.Combine(Settings.DatabasePath, "data"); - - try - { - var info = new FileInfo(datafilePath); - if (!info.Exists) - { - return -1; - } - return info.Length; - } - catch - { - return -1; - } - } - - static long FolderSize() - { - try - { - var dir = new DirectoryInfo(Settings.DatabasePath); - var dirSize = DirSize(dir); - return dirSize; - } - catch - { - return -1; - } - } - - static long DirSize(DirectoryInfo d) - { - long size = 0; - if (d.Exists) - { - FileInfo[] fis = d.GetFiles(); - foreach (FileInfo fi in fis) - { - size += fi.Length; - } - - DirectoryInfo[] dis = d.GetDirectories(); - foreach (DirectoryInfo di in dis) - { - size += DirSize(di); - } - } - - return size; - } - - static readonly SerializationBinder MigratedTypeAwareBinder = new MigratedTypeAwareBinder(); - - static readonly ILog Logger = LogManager.GetLogger(typeof(RavenBootstrapper)); - } } From 18757ca35a36e46e871efcb4a985d0248f2dc018 Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Mon, 11 Sep 2023 11:03:23 +0200 Subject: [PATCH 34/78] ravendb5 lifecycle creation and resolution --- .../RavenDbPersistence.cs | 5 +--- .../RavenDbEmbeddedPersistenceLifecycle.cs | 1 + .../RavenDbPersistence.cs | 24 +++++++++++++++---- .../IPersistence.cs | 2 +- .../PersistenceHostBuilderExtensions.cs | 5 ++-- 5 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb/RavenDbPersistence.cs b/src/ServiceControl.Persistence.RavenDb/RavenDbPersistence.cs index bab32ac5bd..c80cb622a5 100644 --- a/src/ServiceControl.Persistence.RavenDb/RavenDbPersistence.cs +++ b/src/ServiceControl.Persistence.RavenDb/RavenDbPersistence.cs @@ -78,10 +78,7 @@ public void Configure(IServiceCollection serviceCollection) serviceCollection.AddSingleton(); } - public IPersistenceLifecycle CreateLifecycle() - { - return new RavenDbPersistenceLifecycle(ravenStartup, documentStore); - } + public void ConfigureLifecycle(IServiceCollection serviceCollection) => serviceCollection.AddSingleton(new RavenDbPersistenceLifecycle(ravenStartup, documentStore)); public IPersistenceInstaller CreateInstaller() { diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbEmbeddedPersistenceLifecycle.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbEmbeddedPersistenceLifecycle.cs index 7df1fa9489..1e40c3a5f3 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbEmbeddedPersistenceLifecycle.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbEmbeddedPersistenceLifecycle.cs @@ -3,6 +3,7 @@ using System; using System.Threading; using System.Threading.Tasks; + using Microsoft.Extensions.Hosting; using Raven.Client.Documents; using ServiceControl.Persistence; diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs index 74cbab97b0..6cd11d1000 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs @@ -31,7 +31,8 @@ public void Configure(IServiceCollection serviceCollection) } serviceCollection.AddSingleton(settings); - serviceCollection.AddSingleton(documentStore); + + ConfigureLifecycle(serviceCollection); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); @@ -73,19 +74,32 @@ public void Configure(IServiceCollection serviceCollection) serviceCollection.AddSingleton(); } - public IPersistenceLifecycle CreateLifecycle() + public void ConfigureLifecycle(IServiceCollection serviceCollection) { if (settings.UseEmbeddedServer) { - return new RavenDbEmbeddedPersistenceLifecycle(settings); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(_ => new RavenDbEmbeddedPersistenceLifecycle(settings).GetDocumentStore()); + + return; } - return new RavenDbExternalPersistenceLifecycle(settings); + var external = new RavenDbExternalPersistenceLifecycle(settings); + + serviceCollection.AddSingleton(external); + serviceCollection.AddSingleton(_ => external.GetDocumentStore()); } public IPersistenceInstaller CreateInstaller() { - return new RavenDbInstaller(CreateLifecycle()); + var serviceCollection = new ServiceCollection(); + + serviceCollection.AddSingleton(settings); + ConfigureLifecycle(serviceCollection); + + var lifecycle = serviceCollection.BuildServiceProvider().GetRequiredService(); + + return new RavenDbInstaller(lifecycle); } readonly RavenDBPersisterSettings settings; diff --git a/src/ServiceControl.Persistence/IPersistence.cs b/src/ServiceControl.Persistence/IPersistence.cs index 2a4bef4cdd..b974f1b261 100644 --- a/src/ServiceControl.Persistence/IPersistence.cs +++ b/src/ServiceControl.Persistence/IPersistence.cs @@ -6,6 +6,6 @@ public interface IPersistence { void Configure(IServiceCollection serviceCollection); IPersistenceInstaller CreateInstaller(); - IPersistenceLifecycle CreateLifecycle(); + void ConfigureLifecycle(IServiceCollection serviceCollection); } } diff --git a/src/ServiceControl/Persistence/PersistenceHostBuilderExtensions.cs b/src/ServiceControl/Persistence/PersistenceHostBuilderExtensions.cs index ef34ba16d2..02b1300e81 100644 --- a/src/ServiceControl/Persistence/PersistenceHostBuilderExtensions.cs +++ b/src/ServiceControl/Persistence/PersistenceHostBuilderExtensions.cs @@ -20,9 +20,10 @@ public static IHostBuilder SetupPersistence(this IHostBuilder hostBuilder, Setti public static void CreatePersisterLifecyle(IServiceCollection serviceCollection, IPersistence persistence) { - var lifecycle = persistence.CreateLifecycle(); + persistence.ConfigureLifecycle(serviceCollection); + // lifecycle needs to be started before any other hosted service - serviceCollection.AddHostedService(_ => new PersistenceLifecycleHostedService(lifecycle)); + serviceCollection.AddHostedService(); persistence.Configure(serviceCollection); } } From d7f18edecd78f249a2c1828dcdea148882ca4ad3 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Mon, 11 Sep 2023 14:30:30 +0200 Subject: [PATCH 35/78] =?UTF-8?q?=F0=9F=A9=B9=20Removed=20`MessagesBodyTra?= =?UTF-8?q?nsformer`,=20not=20needed=20as=20message=20bodies=20are=20alway?= =?UTF-8?q?s=20stored=20as=20attachments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ErrorMessagesDataStore.cs | 16 +-- .../RavenAttachmentsBodyStorage.cs | 117 +++++------------- .../Transformers/MessagesBodyTransformer.cs | 58 --------- 3 files changed, 34 insertions(+), 157 deletions(-) delete mode 100644 src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesBodyTransformer.cs diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index ec66005082..0cb0fba2f5 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -695,21 +695,9 @@ public async Task GetRetryPendingMessages(DateTime from, DateTime to, return ids.ToArray(); } - public async Task FetchFromFailedMessage(string uniqueMessageId) + public Task FetchFromFailedMessage(string uniqueMessageId) { - string documentId = FailedMessageIdGenerator.MakeDocumentId(uniqueMessageId); - var results = await documentStore.AsyncDatabaseCommands.GetAsync(new[] { documentId }, null, - transformer: MessagesBodyTransformer.Name); - - string resultBody = ((results.Results?.SingleOrDefault()?["$values"] as JArray)?.SingleOrDefault() as JObject) - ?.ToObject()?.Body; - - if (resultBody != null) - { - return Encoding.UTF8.GetBytes(resultBody); - } - - return null; + throw new NotSupportedException("Body not stored embedded"); } public async Task StoreEventLogItem(EventLogItem logItem) diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs b/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs index e167eb510f..5a65756a95 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs @@ -1,18 +1,16 @@ namespace ServiceControl.Operations.BodyStorage.RavenAttachments { - using System; using System.IO; using System.Threading.Tasks; - using Newtonsoft.Json.Linq; - using ServiceControl.CompositeViews.Messages; - using ServiceControl.Persistence; - using Raven.Client; using Raven.Client.Documents; + using Raven.Client.Documents.Operations.Attachments; - // For Raven5, look at how the Audit instance is implementing this, as Attachments won't exist - // and there will be no need for a fallback method on a new persistence + // TODO: For Raven5, look at how the Audit instance is implementing this, as Attachments won't exist + // and there will be no need for a fallback method on a new persistence + // Ramon: Don't understand the comment, audit RavenDB 5 is using attachments.... class RavenAttachmentsBodyStorage : IBodyStorage { + const string AttachmentName = "body"; readonly IDocumentStore documentStore; public RavenAttachmentsBodyStorage(IDocumentStore documentStore) @@ -20,94 +18,43 @@ public RavenAttachmentsBodyStorage(IDocumentStore documentStore) this.documentStore = documentStore; } - public Task Store(string bodyId, string contentType, int bodySize, Stream bodyStream) + public async Task Store(string messageId, string contentType, int bodySize, Stream bodyStream) { - var id = MessageBodyIdGenerator.MakeDocumentId(bodyId); - //We want to continue using attachments for now -#pragma warning disable 618 - return documentStore.AsyncDatabaseCommands.PutAttachmentAsync(id, null, bodyStream, - new JObject -#pragma warning restore 618 - { - {"ContentType", contentType}, - {"ContentLength", bodySize} - }); + // var id = MessageBodyIdGenerator.MakeDocumentId(messageId); // TODO: Not needed? Not used by audit + + using var session = documentStore.OpenAsyncSession(); + + // Following is possible to but not documented in the Raven docs. + //session.Advanced.Attachments.Store(messageId,"body",bodyStream,contentType); + // https://ravendb.net/docs/article-page/5.4/csharp/client-api/operations/attachments/get-attachment + _ = await documentStore.Operations.SendAsync( + new PutAttachmentOperation(messageId, + AttachmentName, + bodyStream, + contentType)); } - // The RavenDB5 implementation, like the Audit instance, will not use Attachments - public async Task TryFetch(string bodyId) + public async Task TryFetch(string messageId) { - bodyId = bodyId?.Replace("/", @"\"); + //var messageId = MessageBodyIdGenerator.MakeDocumentId(bodyId); // TODO: Not needed? Not used by audit - // First try loading from index + using var session = documentStore.OpenAsyncSession(); - using (var session = documentStore.OpenAsyncSession()) - { - var message = await session.Query() - .Statistics(out var stats) - .TransformWith() - .FirstOrDefaultAsync(f => f.MessageId == bodyId); - - if (message != null) - { - if (message.Body == null && message.BodyNotStored) - { - return new MessageBodyStreamResult { HasResult = false }; - } - - if (message.Body != null) - { - var stream = new MemoryStream(); - var writer = new StreamWriter(stream); - // ReSharper disable once MethodHasAsyncOverload, operations are MemoryStream so do not require async overloads - writer.Write(message.Body); - // ReSharper disable once MethodHasAsyncOverload, operations are MemoryStream so do not require async overloads - writer.Flush(); - stream.Position = 0; - - return new MessageBodyStreamResult - { - HasResult = true, - Stream = stream, - ContentType = message.ContentType, - BodySize = message.BodySize, - Etag = stats.IndexEtag - }; - } - else if (message.BodySize == 0) - { - return new MessageBodyStreamResult - { - HasResult = true, - Stream = new MemoryStream(Array.Empty()), - ContentType = message.ContentType, - BodySize = message.BodySize, - Etag = stats.IndexEtag - }; - } - } - } - - var id = MessageBodyIdGenerator.MakeDocumentId(bodyId); + var result = await session.Advanced.Attachments.GetAsync(messageId, AttachmentName); - //We want to continue using attachments for now -#pragma warning disable 618 - var attachment = await documentStore.AsyncDatabaseCommands.GetAttachmentAsync(id); -#pragma warning restore 618 - - if (attachment != null) + if (result == null) { - return new MessageBodyStreamResult - { - HasResult = true, - Stream = (MemoryStream)attachment.Data(), - ContentType = attachment.Metadata["ContentType"].Value(), - BodySize = attachment.Metadata["ContentLength"].Value(), - Etag = attachment.Etag - }; + return null; } - return null; + return new MessageBodyStreamResult + { + HasResult = true, + Stream = (MemoryStream)result.Stream, + ContentType = result.Details.ContentType, + BodySize = (int)result.Details.Size, + Etag = result.Details.ChangeVector + }; } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesBodyTransformer.cs b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesBodyTransformer.cs deleted file mode 100644 index 717a0294a7..0000000000 --- a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesBodyTransformer.cs +++ /dev/null @@ -1,58 +0,0 @@ -namespace ServiceControl.CompositeViews.Messages -{ - using System.Collections.Generic; - using System.Linq; - using ServiceControl.MessageFailures; - - class MessagesBodyTransformer : AbstractTransformerCreationTask // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers - { - public MessagesBodyTransformer() - { - TransformResults = messages => from message in messages - let metadata = message.ProcessingAttempts != null - ? message.ProcessingAttempts.Last().MessageMetadata - : message.MessageMetadata - let body = message.ProcessingAttempts != null - ? message.ProcessingAttempts.Last().Body ?? metadata["Body"] - : metadata["Body"] - select new - { - MessageId = metadata["MessageId"], - Body = body, - BodySize = (int)metadata["ContentLength"], - ContentType = metadata["ContentType"], - BodyNotStored = (bool)metadata["BodyNotStored"] - }; - } - - public static string Name - { - get - { - if (transformerName == null) - { - transformerName = new MessagesBodyTransformer().TransformerName; - } - - return transformerName; - } - } - - static string transformerName; - - public class Input - { - public Dictionary MessageMetadata { get; set; } - public List ProcessingAttempts { get; set; } - } - - public class Result - { - public string MessageId { get; set; } - public string Body { get; set; } - public string ContentType { get; set; } - public int BodySize { get; set; } - public bool BodyNotStored { get; set; } - } - } -} \ No newline at end of file From eb9b804bfe4e2ee22bc73a0b00d6aa16e6167772 Mon Sep 17 00:00:00 2001 From: David Boike Date: Mon, 11 Sep 2023 10:21:22 -0500 Subject: [PATCH 36/78] Simplify MessagesViewIndex to not be multi-map since there will be no ProcessedMessage (audit) documents - can rename later --- .../Indexes/MessagesViewIndex.cs | 69 +++++++------------ 1 file changed, 26 insertions(+), 43 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/MessagesViewIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/MessagesViewIndex.cs index 195f3dfe97..9ac54b009d 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/MessagesViewIndex.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/MessagesViewIndex.cs @@ -3,55 +3,38 @@ namespace ServiceControl.Persistence using System; using System.Linq; using Lucene.Net.Analysis.Standard; - using ServiceControl.MessageAuditing; + using Raven.Client.Documents.Indexes; using ServiceControl.MessageFailures; using ServiceControl.Operations; - using Raven.Client.Documents.Indexes; - class MessagesViewIndex : AbstractMultiMapIndexCreationTask + // TODO: Consider renaming to FailedMessageIndex + class MessagesViewIndex : AbstractIndexCreationTask { public MessagesViewIndex() { - AddMap(messages => from message in messages - select new SortAndFilterOptions - { - MessageId = (string)message.MessageMetadata["MessageId"], - MessageType = (string)message.MessageMetadata["MessageType"], - IsSystemMessage = (bool)message.MessageMetadata["IsSystemMessage"], - Status = (bool)message.MessageMetadata["IsRetried"] ? MessageStatus.ResolvedSuccessfully : MessageStatus.Successful, - TimeSent = (DateTime)message.MessageMetadata["TimeSent"], - ProcessedAt = message.ProcessedAt, - ReceivingEndpointName = ((EndpointDetails)message.MessageMetadata["ReceivingEndpoint"]).Name, - CriticalTime = (TimeSpan?)message.MessageMetadata["CriticalTime"], - ProcessingTime = (TimeSpan?)message.MessageMetadata["ProcessingTime"], - DeliveryTime = (TimeSpan?)message.MessageMetadata["DeliveryTime"], - Query = message.MessageMetadata.Select(_ => _.Value.ToString()).Union(new[] { string.Join(" ", message.Headers.Select(x => x.Value)) }).ToArray(), - ConversationId = (string)message.MessageMetadata["ConversationId"] - }); - - AddMap(messages => from message in messages - let last = message.ProcessingAttempts.Last() - select new SortAndFilterOptions - { - MessageId = last.MessageId, - MessageType = (string)last.MessageMetadata["MessageType"], - IsSystemMessage = (bool)last.MessageMetadata["IsSystemMessage"], - Status = message.Status == FailedMessageStatus.Archived - ? MessageStatus.ArchivedFailure - : message.Status == FailedMessageStatus.Resolved - ? MessageStatus.ResolvedSuccessfully - : message.ProcessingAttempts.Count == 1 - ? MessageStatus.Failed - : MessageStatus.RepeatedFailure, - TimeSent = (DateTime)last.MessageMetadata["TimeSent"], - ProcessedAt = last.AttemptedAt, - ReceivingEndpointName = ((EndpointDetails)last.MessageMetadata["ReceivingEndpoint"]).Name, - CriticalTime = null, - ProcessingTime = null, - DeliveryTime = null, - Query = last.MessageMetadata.Select(_ => _.Value.ToString()).Union(new[] { string.Join(" ", last.Headers.Select(x => x.Value)) }).ToArray(), - ConversationId = (string)last.MessageMetadata["ConversationId"] - }); + Map = messages => from message in messages + let last = message.ProcessingAttempts.Last() + select new SortAndFilterOptions + { + MessageId = last.MessageId, + MessageType = (string)last.MessageMetadata["MessageType"], + IsSystemMessage = (bool)last.MessageMetadata["IsSystemMessage"], + Status = message.Status == FailedMessageStatus.Archived + ? MessageStatus.ArchivedFailure + : message.Status == FailedMessageStatus.Resolved + ? MessageStatus.ResolvedSuccessfully + : message.ProcessingAttempts.Count == 1 + ? MessageStatus.Failed + : MessageStatus.RepeatedFailure, + TimeSent = (DateTime)last.MessageMetadata["TimeSent"], + ProcessedAt = last.AttemptedAt, + ReceivingEndpointName = ((EndpointDetails)last.MessageMetadata["ReceivingEndpoint"]).Name, + CriticalTime = null, + ProcessingTime = null, + DeliveryTime = null, + Query = last.MessageMetadata.Select(_ => _.Value.ToString()).Union(new[] { string.Join(" ", last.Headers.Select(x => x.Value)) }).ToArray(), + ConversationId = (string)last.MessageMetadata["ConversationId"] + }; Index(x => x.Query, FieldIndexing.Search); From a46aaf967c61debb6bd2fc7cd06d60be79699aff Mon Sep 17 00:00:00 2001 From: David Boike Date: Mon, 11 Sep 2023 11:45:05 -0500 Subject: [PATCH 37/78] These methods return Audit data and probably shouldn't be in the ErrorMessagesDataStore in the first place --- .../ErrorMessagesDataStore.cs | 22 ---- .../ErrorMessagesDataStore.cs | 118 ++++-------------- .../IErrorMessageDatastore.cs | 1 - 3 files changed, 23 insertions(+), 118 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb/ErrorMessagesDataStore.cs index 2de540abd0..261413ed30 100644 --- a/src/ServiceControl.Persistence.RavenDb/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb/ErrorMessagesDataStore.cs @@ -136,28 +136,6 @@ SortInfo sortInfo } } - public async Task>> GetAllMessagesForEndpoint( - string searchTerms, - string receivingEndpointName, - PagingInfo pagingInfo, - SortInfo sortInfo - ) - { - using (var session = documentStore.OpenAsyncSession()) - { - var results = await session.Query() - .Statistics(out var stats) - .Search(x => x.Query, searchTerms) - .Where(m => m.ReceivingEndpointName == receivingEndpointName) - .Sort(sortInfo) - .Paging(pagingInfo) - .TransformWith() - .ToListAsync(); - - return new QueryResult>(results, stats.ToQueryStatsInfo()); - } - } - public async Task FailedMessageFetch(string failedMessageId) { using (var session = documentStore.OpenAsyncSession()) diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index 0cb0fba2f5..dc5ea2f380 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -4,23 +4,23 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; - using System.Text; using System.Threading.Tasks; using CompositeViews.Messages; using Editing; using Newtonsoft.Json.Linq; using NServiceBus.Logging; + using Raven.Client.Documents; + using Raven.Client.Documents.Linq; + using Raven.Client.Documents.Queries; + using Raven.Client.Documents.Queries.Facets; + using Raven.Client.Documents.Session; using ServiceControl.EventLog; using ServiceControl.MessageFailures; using ServiceControl.MessageFailures.Api; using ServiceControl.Operations; + using ServiceControl.Persistence; using ServiceControl.Persistence.Infrastructure; using ServiceControl.Recoverability; - using Raven.Client; - using Raven.Client.Documents; - using Raven.Client.Documents.Queries; - using Raven.Client.Documents.Session; - using Raven.Client.Documents.Queries.Facets; class ErrorMessagesDataStore : IErrorMessageDataStore { @@ -32,131 +32,59 @@ public ErrorMessagesDataStore(IDocumentStore documentStore) } - public async Task>> GetAllMessages( + static Task>> EmptyQueryResult() => Task.FromResult(new QueryResult>(new List(), new QueryStatsInfo())); + + // TODO: This is for audit messages, should probably not be on IErrorMessageDataStore + public Task>> GetAllMessages( PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages ) { - using (var session = documentStore.OpenAsyncSession()) - { - var results = await session.Query() - .IncludeSystemMessagesWhere(includeSystemMessages) - .Statistics(out var stats) - .Sort(sortInfo) - .Paging(pagingInfo) - .TransformWith() - .ToListAsync(); - - return new QueryResult>(results, stats.ToQueryStatsInfo()); - } + return EmptyQueryResult(); } - public async Task>> GetAllMessagesForEndpoint( + // TODO: This is for audit messages, should probably not be on IErrorMessageDataStore + public Task>> GetAllMessagesForEndpoint( string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages ) { - using (var session = documentStore.OpenAsyncSession()) - { - var results = await session.Query() - .IncludeSystemMessagesWhere(includeSystemMessages) - .Where(m => m.ReceivingEndpointName == endpointName) - .Statistics(out var stats) - .Sort(sortInfo) - .Paging(pagingInfo) - .TransformWith() - .ToListAsync(); - - return new QueryResult>(results, stats.ToQueryStatsInfo()); - } + return EmptyQueryResult(); } - public async Task>> SearchEndpointMessages( + // TODO: This is for audit messages, should probably not be on IErrorMessageDataStore + public Task>> SearchEndpointMessages( string endpointName, string searchKeyword, PagingInfo pagingInfo, SortInfo sortInfo ) { - using (var session = documentStore.OpenAsyncSession()) - { - var results = await session.Query() - .Statistics(out var stats) - .Search(x => x.Query, searchKeyword) - .Where(m => m.ReceivingEndpointName == endpointName) - .Sort(sortInfo) - .Paging(pagingInfo) - .TransformWith() - .ToListAsync(); - - return new QueryResult>(results, stats.ToQueryStatsInfo()); - } + return EmptyQueryResult(); } - public async Task>> GetAllMessagesByConversation( + // TODO: This is for audit messages, should probably not be on IErrorMessageDataStore + public Task>> GetAllMessagesByConversation( string conversationId, PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages ) { - using (var session = documentStore.OpenAsyncSession()) - { - var results = await session.Query() - .Statistics(out var stats) - .Where(m => m.ConversationId == conversationId) - .Sort(sortInfo) - .Paging(pagingInfo) - .TransformWith() - .ToListAsync(); - - return new QueryResult>(results, stats.ToQueryStatsInfo()); - } - } - - public async Task>> GetAllMessagesForSearch( - string searchTerms, - PagingInfo pagingInfo, - SortInfo sortInfo - ) - { - using (var session = documentStore.OpenAsyncSession()) - { - var results = await session.Query() - .Statistics(out var stats) - .Search(x => x.Query, searchTerms) - .Sort(sortInfo) - .Paging(pagingInfo) - .TransformWith() - .ToListAsync(); - - return new QueryResult>(results, stats.ToQueryStatsInfo()); - } + return EmptyQueryResult(); } - public async Task>> GetAllMessagesForEndpoint( + // TODO: This is for audit messages, should probably not be on IErrorMessageDataStore + public Task>> GetAllMessagesForSearch( string searchTerms, - string receivingEndpointName, PagingInfo pagingInfo, SortInfo sortInfo ) { - using (var session = documentStore.OpenAsyncSession()) - { - var results = await session.Query() - .Statistics(out var stats) - .Search(x => x.Query, searchTerms) - .Where(m => m.ReceivingEndpointName == receivingEndpointName) - .Sort(sortInfo) - .Paging(pagingInfo) - .TransformWith() - .ToListAsync(); - - return new QueryResult>(results, stats.ToQueryStatsInfo()); - } + return EmptyQueryResult(); } public async Task FailedMessageFetch(string failedMessageId) diff --git a/src/ServiceControl.Persistence/IErrorMessageDatastore.cs b/src/ServiceControl.Persistence/IErrorMessageDatastore.cs index 910783554e..865d19e208 100644 --- a/src/ServiceControl.Persistence/IErrorMessageDatastore.cs +++ b/src/ServiceControl.Persistence/IErrorMessageDatastore.cs @@ -17,7 +17,6 @@ public interface IErrorMessageDataStore Task>> GetAllMessagesForEndpoint(string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages); Task>> GetAllMessagesByConversation(string conversationId, PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages); Task>> GetAllMessagesForSearch(string searchTerms, PagingInfo pagingInfo, SortInfo sortInfo); - Task>> GetAllMessagesForEndpoint(string searchTerms, string receivingEndpointName, PagingInfo pagingInfo, SortInfo sortInfo); Task FailedMessageFetch(string failedMessageId); Task FailedMessageMarkAsArchived(string failedMessageId); Task FailedMessagesFetch(Guid[] ids); From dc7e5fcfbbdf54c257c5388c1ece9c2af2f945f8 Mon Sep 17 00:00:00 2001 From: David Boike Date: Mon, 11 Sep 2023 11:48:51 -0500 Subject: [PATCH 38/78] MessagesViewTransformer not actually used --- .../Transformers/MessagesViewTransformer.cs | 75 ------------------- 1 file changed, 75 deletions(-) delete mode 100644 src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs diff --git a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs deleted file mode 100644 index 733fca57fa..0000000000 --- a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs +++ /dev/null @@ -1,75 +0,0 @@ -namespace ServiceControl.CompositeViews.Messages -{ - using System; - using System.Collections.Generic; - using System.Linq; - using MessageFailures; - using ServiceControl.Persistence; - - class MessagesViewTransformer : AbstractTransformerCreationTask // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers - { - public MessagesViewTransformer() - { - TransformResults = messages => from message in messages - let metadata = - message.ProcessingAttempts != null - ? message.ProcessingAttempts.Last().MessageMetadata - : message.MessageMetadata - let headers = - message.ProcessingAttempts != null ? message.ProcessingAttempts.Last().Headers : message.Headers - let processedAt = - message.ProcessingAttempts != null - ? message.ProcessingAttempts.Last().AttemptedAt - : message.ProcessedAt - let status = - message.ProcessingAttempts == null - ? !(bool)message.MessageMetadata["IsRetried"] - ? MessageStatus.Successful - : MessageStatus.ResolvedSuccessfully - : message.Status == FailedMessageStatus.Resolved - ? MessageStatus.ResolvedSuccessfully - : message.Status == FailedMessageStatus.RetryIssued - ? MessageStatus.RetryIssued - : message.Status == FailedMessageStatus.Archived - ? MessageStatus.ArchivedFailure - : message.ProcessingAttempts.Count == 1 - ? MessageStatus.Failed - : MessageStatus.RepeatedFailure - select new - { - Id = message.UniqueMessageId, - MessageId = metadata["MessageId"], - MessageType = metadata["MessageType"], - SendingEndpoint = metadata["SendingEndpoint"], - ReceivingEndpoint = metadata["ReceivingEndpoint"], - TimeSent = (DateTime?)metadata["TimeSent"], - ProcessedAt = processedAt, - CriticalTime = (TimeSpan)metadata["CriticalTime"], - ProcessingTime = (TimeSpan)metadata["ProcessingTime"], - DeliveryTime = (TimeSpan)metadata["DeliveryTime"], - IsSystemMessage = (bool)metadata["IsSystemMessage"], - ConversationId = metadata["ConversationId"], - //the reason the we need to use a KeyValuePair is that raven seems to interpret the values and convert them - // to real types. In this case it was the NServiceBus.Temporary.DelayDeliveryWith header to was converted to a timespan - Headers = headers.Select(header => new KeyValuePair(header.Key, header.Value)), - Status = status, - MessageIntent = metadata["MessageIntent"], - BodyUrl = metadata["BodyUrl"], - BodySize = (int)metadata["ContentLength"], - InvokedSagas = metadata["InvokedSagas"], - OriginatesFromSaga = metadata["OriginatesFromSaga"] - }; - } - - public class Input - { - public string Id { get; set; } - public string UniqueMessageId { get; set; } - public DateTime ProcessedAt { get; set; } - public Dictionary Headers { get; set; } - public Dictionary MessageMetadata { get; set; } - public List ProcessingAttempts { get; set; } - public FailedMessageStatus Status { get; set; } - } - } -} \ No newline at end of file From 3797cd1031419ac452e326e01c95a0488d4afa1b Mon Sep 17 00:00:00 2001 From: David Boike Date: Mon, 11 Sep 2023 12:02:51 -0500 Subject: [PATCH 39/78] ErrorMessagesDataStore minor API fixes --- .../ErrorMessagesDataStore.cs | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index dc5ea2f380..d87404fb24 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -91,7 +91,7 @@ public async Task FailedMessageFetch(string failedMessageId) { using (var session = documentStore.OpenAsyncSession()) { - return await session.LoadAsync(new Guid(failedMessageId)); + return await session.LoadAsync(failedMessageId); } } @@ -99,7 +99,7 @@ public async Task FailedMessageMarkAsArchived(string failedMessageId) { using (var session = documentStore.OpenAsyncSession()) { - var failedMessage = await session.LoadAsync(new Guid(failedMessageId)); + var failedMessage = await session.LoadAsync(failedMessageId); if (failedMessage.Status != FailedMessageStatus.Archived) { @@ -114,8 +114,8 @@ public async Task FailedMessagesFetch(Guid[] ids) { using (var session = documentStore.OpenAsyncSession()) { - var results = await session.LoadAsync(ids.Cast()); - return results.Where(x => x != null).ToArray(); + var results = await session.LoadAsync(ids.Select(g => g.ToString())); + return results.Values.Where(x => x != null).ToArray(); } } @@ -246,27 +246,26 @@ public async Task> ErrorsSummary() using (var session = documentStore.OpenAsyncSession()) { var facetResults = await session.Query() - .ToFacetsAsync(new List + .AggregateBy(new List { new Facet { - Name = "Name", - DisplayName = "Endpoints" + FieldName = "Name", + DisplayFieldName = "Endpoints" }, new Facet { - Name = "Host", - DisplayName = "Hosts" + FieldName = "Host", + DisplayFieldName = "Hosts" }, new Facet { - Name = "MessageType", - DisplayName = "Message types" + FieldName = "MessageType", + DisplayFieldName = "Message types" } - }); + }).ExecuteAsync(); var results = facetResults - .Results .ToDictionary( x => x.Key, x => (object)x.Value @@ -280,7 +279,7 @@ public async Task ErrorBy(Guid failedMessageId) { using (var session = documentStore.OpenAsyncSession()) { - var message = await session.LoadAsync(failedMessageId); + var message = await session.LoadAsync(failedMessageId.ToString()); return message; } } @@ -306,7 +305,7 @@ public async Task ErrorLastBy(Guid failedMessageId) { using (var session = documentStore.OpenAsyncSession()) { - var message = await session.LoadAsync(failedMessageId); + var message = await session.LoadAsync(failedMessageId.ToString()); if (message == null) { return null; @@ -452,7 +451,7 @@ public async Task MarkMessageAsResolved(string failedMessageId) { session.Advanced.UseOptimisticConcurrency = true; - var failedMessage = await session.LoadAsync(new Guid(failedMessageId)); + var failedMessage = await session.LoadAsync(failedMessageId); if (failedMessage == null) { From 9b58da47dac3e39d2c24079c5140e5e63a2adf79 Mon Sep 17 00:00:00 2001 From: David Boike Date: Mon, 11 Sep 2023 12:36:19 -0500 Subject: [PATCH 40/78] QueryResultAysnc() => GetQueryResultAsync() --- .../ErrorMessagesDataStore.cs | 4 ++-- .../RavenQueryExtensions.cs | 5 ++--- .../RavenQueryStatisticsExtensions.cs | 10 ++++------ 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index d87404fb24..3ddc771a27 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -208,7 +208,7 @@ string queueAddress .FilterByStatusWhere(status) .FilterByLastModifiedRange(modified) .FilterByQueueAddress(queueAddress) - .QueryResultAsync(); + .GetQueryResultAsync(); return stats.ToQueryStatsInfo(); } @@ -423,7 +423,7 @@ public async Task GetGroupErrorsCount(string groupId, string sta .WhereEquals(view => view.FailureGroupId, groupId) .FilterByStatusWhere(status) .FilterByLastModifiedRange(modified) - .QueryResultAsync(); + .GetQueryResultAsync(); return queryResult.ToQueryStatsInfo(); } diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs b/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs index fd53d78104..0d04ab9904 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs @@ -7,11 +7,10 @@ namespace ServiceControl.Persistence using System.Linq.Expressions; using System.Net.Http; using System.Text; - using ServiceControl.MessageFailures; - using ServiceControl.Persistence.Infrastructure; - using Raven.Client; using Raven.Client.Documents.Linq; using Raven.Client.Documents.Session; + using ServiceControl.MessageFailures; + using ServiceControl.Persistence.Infrastructure; static class RavenQueryExtensions { diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenQueryStatisticsExtensions.cs b/src/ServiceControl.Persistence.RavenDb5/RavenQueryStatisticsExtensions.cs index 4665d755f1..fa5746ccfa 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenQueryStatisticsExtensions.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenQueryStatisticsExtensions.cs @@ -1,6 +1,5 @@ namespace ServiceControl.Persistence { - using Raven.Client; using Raven.Client.Documents.Session; using ServiceControl.Persistence.Infrastructure; @@ -11,10 +10,9 @@ public static QueryStatsInfo ToQueryStatsInfo(this QueryStatistics stats) return new QueryStatsInfo($"{stats.ResultEtag}", stats.TotalResults, stats.IsStale); } - //TODO: This method can likely be removed - //public static QueryStatsInfo ToQueryStatsInfo(this Raven.Abstractions.Data.QueryResult queryResult) - //{ - // return new QueryStatsInfo(queryResult.IndexEtag, queryResult.TotalResults, queryResult.IsStale); - //} + public static QueryStatsInfo ToQueryStatsInfo(this Raven.Client.Documents.Queries.QueryResult queryResult) + { + return new QueryStatsInfo(queryResult.ResultEtag.ToString(), queryResult.TotalResults, queryResult.IsStale); + } } } \ No newline at end of file From 6e522500448b7a456feb41ccc9909118f812d8ec Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Tue, 12 Sep 2023 09:27:08 +0200 Subject: [PATCH 41/78] index error custom check --- .../CustomChecks/CheckRavenDBIndexErrors.cs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexErrors.cs b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexErrors.cs index d79a23ab70..0bfb537b5e 100644 --- a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexErrors.cs +++ b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexErrors.cs @@ -1,12 +1,12 @@ namespace ServiceControl { using System; - using System.Linq; using System.Text; using System.Threading.Tasks; using NServiceBus.CustomChecks; using NServiceBus.Logging; using Raven.Client.Documents; + using Raven.Client.Documents.Operations.Indexes; class CheckRavenDBIndexErrors : CustomCheck { @@ -18,10 +18,9 @@ public CheckRavenDBIndexErrors(IDocumentStore store) public override Task PerformCheck() { - var statistics = store.DatabaseCommands.GetStatistics(); - var indexes = statistics.Indexes.OrderBy(x => x.Name).ToArray(); - - if (statistics.Errors.Length == 0) + var indexErrors = store.Maintenance.Send(new GetIndexErrorsOperation()); + + if (indexErrors.Length == 0) { return CheckResult.Pass; } @@ -29,9 +28,12 @@ public override Task PerformCheck() var text = new StringBuilder(); text.AppendLine("Detected RavenDB index errors, please start maintenance mode and resolve the following issues:"); - foreach (var indexError in statistics.Errors) + foreach (var indexError in indexErrors) { - text.AppendLine($"- Index [{indexError.IndexName}] error: {indexError.Error} (Action: {indexError.Action}, Doc: {indexError.Document}, At: {indexError.Timestamp})"); + foreach (var indexingError in indexError.Errors) + { + text.AppendLine($"- Index [{indexError.Name}] error: {indexError.Name} (Action: {indexingError.Action}, Doc: {indexingError.Document}, At: {indexingError.Timestamp})"); + } } text.AppendLine().AppendLine("See: https://docs.particular.net/search?q=servicecontrol+troubleshooting"); From 83c9da590a7bdbc25a15a91495bc48926add91b9 Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Tue, 12 Sep 2023 09:27:40 +0200 Subject: [PATCH 42/78] removing lastmodified extension --- .../Extensions.cs | 16 ++++++++-------- .../MessageRedirectsDataStore.cs | 3 ++- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/Extensions.cs b/src/ServiceControl.Persistence.RavenDb5/Extensions.cs index e77d23b740..55f2936ace 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Extensions.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Extensions.cs @@ -8,14 +8,14 @@ static class Extensions { - public static void Query(this DocumentDatabase db, string index, IndexQuery query, Action onItem, TState state, CancellationToken cancellationToken = default) - { - var results = db.Queries.Query(index, query, cancellationToken); - foreach (var doc in results.Results) - { - onItem(doc, state); - } - } + //public static void Query(this DocumentDatabase db, string index, IndexQuery query, Action onItem, TState state, CancellationToken cancellationToken = default) + //{ + // var results = db.Queries.Query(index, query, cancellationToken); + // foreach (var doc in results.Results) + // { + // onItem(doc, state); + // } + //} // TODO: This polyfill of RavenDB 3.5 is a guess based loosely on https://github.com/ravendb/ravendb/blob/v3.5/Raven.Client.Lightweight/Document/DocumentConvention.cs#L151 public static string DefaultFindFullDocumentKeyFromNonStringIdentifier(this DocumentConventions conventions, T id, Type collectionType, bool allowNull) diff --git a/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs index e28edd9024..cc9ac0f7db 100644 --- a/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs @@ -3,6 +3,7 @@ using System; using System.Threading.Tasks; using Raven.Client.Documents; + using Raven.Client.Documents.Queries; using ServiceControl.Persistence.MessageRedirects; class MessageRedirectsDataStore : IMessageRedirectsDataStore @@ -23,7 +24,7 @@ public async Task GetOrCreate() if (redirects != null) { redirects.ETag = session.Advanced.GetChangeVectorFor(redirects); - redirects.LastModified = session.Advanced.GetMetadataFor(redirects).GetAsDateTime("@last-modified"); + redirects.LastModified = RavenQuery.LastModified(redirects); return redirects; } From 3a9a5fbfb334aa9bb3a614fd6e93c39d087eb139 Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Tue, 12 Sep 2023 09:28:08 +0200 Subject: [PATCH 43/78] commenting out transformers --- .../MetadataExtensions.cs | 11 ----- .../RavenDbPersistence.cs | 2 +- .../FailedMessageViewTransformer.cs | 47 ++++++++++--------- 3 files changed, 25 insertions(+), 35 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/MetadataExtensions.cs b/src/ServiceControl.Persistence.RavenDb5/MetadataExtensions.cs index 528691f129..7ac89dd9a4 100644 --- a/src/ServiceControl.Persistence.RavenDb5/MetadataExtensions.cs +++ b/src/ServiceControl.Persistence.RavenDb5/MetadataExtensions.cs @@ -36,16 +36,5 @@ public static string GetAsStringOrNull(this IDictionary metadata return null; } - - public static DateTime GetAsDateTime(this IDictionary metadata, string key) - { - var datetimeAsString = metadata.GetAsStringOrNull(key); - - if (datetimeAsString != null) - { - return DateTime.Parse(datetimeAsString); - } - } - } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs index 6cd11d1000..acba5555d3 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs @@ -93,7 +93,7 @@ public void ConfigureLifecycle(IServiceCollection serviceCollection) public IPersistenceInstaller CreateInstaller() { var serviceCollection = new ServiceCollection(); - + serviceCollection.AddSingleton(settings); ConfigureLifecycle(serviceCollection); diff --git a/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs b/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs index c77ba1c092..d4c37a9d61 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs @@ -3,31 +3,31 @@ using System; using System.Linq; - class FailedMessageViewTransformer : AbstractTransformerCreationTask // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers + class FailedMessageViewTransformer //: AbstractTransformerCreationTask // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers { public FailedMessageViewTransformer() { - TransformResults = failures => from failure in failures - let rec = failure.ProcessingAttempts.Last() - let edited = rec.Headers["ServiceControl.EditOf"] != null - select new - { - Id = failure.UniqueMessageId, - MessageType = rec.MessageMetadata["MessageType"], - IsSystemMessage = (bool)rec.MessageMetadata["IsSystemMessage"], - SendingEndpoint = rec.MessageMetadata["SendingEndpoint"], - ReceivingEndpoint = rec.MessageMetadata["ReceivingEndpoint"], - TimeSent = (DateTime?)rec.MessageMetadata["TimeSent"], - MessageId = rec.MessageMetadata["MessageId"], - rec.FailureDetails.Exception, - QueueAddress = rec.FailureDetails.AddressOfFailingEndpoint, - NumberOfProcessingAttempts = failure.ProcessingAttempts.Count, - failure.Status, - rec.FailureDetails.TimeOfFailure, - LastModified = MetadataFor(failure)["@last-modified"].Value(), - Edited = edited, - EditOf = edited ? rec.Headers["ServiceControl.EditOf"] : "" - }; + //TransformResults = failures => from failure in failures + // let rec = failure.ProcessingAttempts.Last() + // let edited = rec.Headers["ServiceControl.EditOf"] != null + // select new + // { + // Id = failure.UniqueMessageId, + // MessageType = rec.MessageMetadata["MessageType"], + // IsSystemMessage = (bool)rec.MessageMetadata["IsSystemMessage"], + // SendingEndpoint = rec.MessageMetadata["SendingEndpoint"], + // ReceivingEndpoint = rec.MessageMetadata["ReceivingEndpoint"], + // TimeSent = (DateTime?)rec.MessageMetadata["TimeSent"], + // MessageId = rec.MessageMetadata["MessageId"], + // rec.FailureDetails.Exception, + // QueueAddress = rec.FailureDetails.AddressOfFailingEndpoint, + // NumberOfProcessingAttempts = failure.ProcessingAttempts.Count, + // failure.Status, + // rec.FailureDetails.TimeOfFailure, + // LastModified = MetadataFor(failure)["@last-modified"].Value(), + // Edited = edited, + // EditOf = edited ? rec.Headers["ServiceControl.EditOf"] : "" + // }; } public static string Name @@ -36,7 +36,8 @@ public static string Name { if (transformerName == null) { - transformerName = new FailedMessageViewTransformer().TransformerName; + return string.Empty; + //transformerName = new FailedMessageViewTransformer().TransformerName; } return transformerName; From 69df870c74971a66d28c841072be03d3121d8561 Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Tue, 12 Sep 2023 10:07:34 +0200 Subject: [PATCH 44/78] RavenDbRecoverabilityIngestionUnitOfWork --- .../UnitOfWork/RavenDbIngestionUnitOfWork.cs | 14 ++++++--- ...avenDbRecoverabilityIngestionUnitOfWork.cs | 31 +++++++------------ 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs index 09002e08de..86b3233801 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs @@ -3,7 +3,6 @@ using System.Collections.Concurrent; using System.Threading.Tasks; using ServiceControl.Persistence.UnitOfWork; - using Raven.Client; using Raven.Client.Documents; using Raven.Client.Documents.Commands.Batches; @@ -22,8 +21,15 @@ public RavenDbIngestionUnitOfWork(IDocumentStore store) internal void AddCommand(ICommandData command) => commands.Add(command); - public override Task Complete() => - // not really interested in the batch results since a batch is atomic - store.AsyncDatabaseCommands.BatchAsync(commands); + public override async Task Complete() + { + using (var session = store.OpenAsyncSession()) + { + // not really interested in the batch results since a batch is atomic + session.Advanced.Defer(commands.ToArray()); + + await session.SaveChangesAsync(); + } + } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs index 631ac8397a..590d9141d1 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs @@ -35,16 +35,12 @@ public Task RecordSuccessfulRetry(string retriedMessageUniqueId) var failedMessageDocumentId = FailedMessageIdGenerator.MakeDocumentId(retriedMessageUniqueId); var failedMessageRetryDocumentId = FailedMessageRetry.MakeDocumentId(retriedMessageUniqueId); - parentUnitOfWork.AddCommand(new PatchCommandData + parentUnitOfWork.AddCommand(new PatchCommandData(failedMessageDocumentId, null, new PatchRequest { - Key = failedMessageDocumentId, - Patches = new[] - { - new PatchRequest {Type = PatchCommandType.Set, Name = nameof(FailedMessage.Status), Value = (int)FailedMessageStatus.Resolved} - } - }); + Script = $@"this.nameof(FailedMessage.Status) = {(int)FailedMessageStatus.Resolved};" + })); - parentUnitOfWork.AddCommand(new DeleteCommandData { Key = failedMessageRetryDocumentId }); + parentUnitOfWork.AddCommand(new DeleteCommandData(failedMessageRetryDocumentId, null)); return Task.CompletedTask; } @@ -53,15 +49,12 @@ ICommandData CreateFailedMessagesPatchCommand(string uniqueMessageId, FailedMess { var documentId = FailedMessageIdGenerator.MakeDocumentId(uniqueMessageId); - var serializedGroups = RavenJToken.FromObject(groups); - var serializedAttempt = RavenJToken.FromObject(processingAttempt, Serializer); + var serializedGroups = JToken.FromObject(groups); + var serializedAttempt = JToken.FromObject(processingAttempt, Serializer); //HINT: RavenDB 3.5 is using Lodash v4.13.1 to provide javascript utility functions // https://ravendb.net/docs/article-page/3.5/csharp/client-api/commands/patches/how-to-use-javascript-to-patch-your-documents#methods-objects-and-variables - return new ScriptedPatchCommandData - { - Key = documentId, - Patch = new ScriptedPatchRequest + return new PatchCommandData(documentId, null, new PatchRequest { Script = $@"this.{nameof(FailedMessage.Status)} = status; this.{nameof(FailedMessage.FailureGroups)} = failureGroups; @@ -98,12 +91,13 @@ ICommandData CreateFailedMessagesPatchCommand(string uniqueMessageId, FailedMess {"attempt", serializedAttempt} }, }, - PatchIfMissing = new ScriptedPatchRequest + patchIfMissing: new PatchRequest { Script = $@"this.{nameof(FailedMessage.Status)} = status; this.{nameof(FailedMessage.FailureGroups)} = failureGroups; this.{nameof(FailedMessage.ProcessingAttempts)} = [attempt]; this.{nameof(FailedMessage.UniqueMessageId)} = uniqueMessageId; + this.@metadata = {FailedMessageMetadata} ", Values = new Dictionary { @@ -112,14 +106,13 @@ ICommandData CreateFailedMessagesPatchCommand(string uniqueMessageId, FailedMess {"attempt", serializedAttempt}, {"uniqueMessageId", uniqueMessageId} } - }, - Metadata = FailedMessageMetadata - }; + }); } static RavenDbRecoverabilityIngestionUnitOfWork() { - Serializer = JsonExtensions.CreateDefaultJsonSerializer(); + //TODO: check if this actually works + Serializer = JsonSerializer.CreateDefault(); Serializer.TypeNameHandling = TypeNameHandling.Auto; FailedMessageMetadata = JObject.Parse($@" From fbf9efe35fa2dbe48cc87862b40bf5fd150c2c85 Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Tue, 12 Sep 2023 10:08:15 +0200 Subject: [PATCH 45/78] missing bit --- .../UnitOfWork/RavenDbIngestionUnitOfWork.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs index 86b3233801..f1f67516c3 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs @@ -27,7 +27,6 @@ public override async Task Complete() { // not really interested in the batch results since a batch is atomic session.Advanced.Defer(commands.ToArray()); - await session.SaveChangesAsync(); } } From b2a460331455dcb672088d109cec1b63f5035fa9 Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Tue, 12 Sep 2023 11:38:54 +0200 Subject: [PATCH 46/78] UnarchiveDocumentManager --- .../Editing/EditFailedMessageManager.cs | 2 +- .../Archiving/UnarchiveDocumentManager.cs | 20 ++++++------------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/Editing/EditFailedMessageManager.cs b/src/ServiceControl.Persistence.RavenDb5/Editing/EditFailedMessageManager.cs index cde27ab0d7..1f87798ea0 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Editing/EditFailedMessageManager.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Editing/EditFailedMessageManager.cs @@ -40,7 +40,7 @@ public Task SetCurrentEditingMessageId(string editingMessageId) Id = FailedMessageEdit.MakeDocumentId(failedMessage.UniqueMessageId), FailedMessageId = failedMessage.Id, EditId = editingMessageId - }, Etag.Empty); + }); } public Task SetFailedMessageAsResolved() diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveDocumentManager.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveDocumentManager.cs index a00efefafe..1967f361c5 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveDocumentManager.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveDocumentManager.cs @@ -94,12 +94,12 @@ public async Task GetGroupDetails(IAsyncDocumentSession session, s public async Task UnarchiveMessageGroupBatch(IAsyncDocumentSession session, UnarchiveBatch batch) { - var patchCommands = batch?.DocumentIds.Select(documentId => new PatchCommandData { Key = documentId, Patches = patchRequest }); + var patchCommands = batch?.DocumentIds.Select(documentId => new PatchCommandData(documentId, null, patchRequest)); if (patchCommands != null) { - await session.Advanced.DocumentStore.AsyncDatabaseCommands.BatchAsync(patchCommands); - await session.Advanced.DocumentStore.AsyncDatabaseCommands.DeleteAsync(batch.Id, null); + session.Advanced.Defer(patchCommands.ToArray()); + session.Advanced.Defer(new DeleteCommandData(batch.Id, null)); } } @@ -108,7 +108,7 @@ public async Task WaitForIndexUpdateOfUnarchiveOperation(IDocumentStore st using (var session = store.OpenAsyncSession()) { var indexQuery = session.Query(new FailedMessages_ByGroup().IndexName) - .Customize(x => x.WaitForNonStaleResultsAsOfNow(timeToWait)); + .Customize(x => x.WaitForNonStaleResults(timeToWait)); var docQuery = indexQuery .Where(failure => failure.FailureGroupId == requestId) @@ -136,20 +136,12 @@ public async Task RemoveUnarchiveOperation(IDocumentStore store, UnarchiveOperat { using (var session = store.OpenAsyncSession()) { - await session.Advanced.DocumentStore.AsyncDatabaseCommands.DeleteAsync(unarchiveOperation.Id, null); + session.Advanced.Defer(new DeleteCommandData(unarchiveOperation.Id, null)); await session.SaveChangesAsync(); } } - static PatchRequest[] patchRequest = - { - new PatchRequest - { - Type = PatchCommandType.Set, - Name = "Status", - Value = (int)FailedMessageStatus.Unresolved - } - }; + static PatchRequest patchRequest = new PatchRequest {Script = $@"this.Status = {(int)FailedMessageStatus.Unresolved}"}; public class GroupDetails { From b45c8fb892276f9493e3674b1945063923e94da0 Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Tue, 12 Sep 2023 11:41:12 +0200 Subject: [PATCH 47/78] ArchiveDocumentManager --- .../Archiving/ArchiveDocumentManager.cs | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveDocumentManager.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveDocumentManager.cs index 0ac345f850..7be6159136 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveDocumentManager.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveDocumentManager.cs @@ -95,12 +95,12 @@ public async Task GetGroupDetails(IAsyncDocumentSession session, s public async Task ArchiveMessageGroupBatch(IAsyncDocumentSession session, ArchiveBatch batch) { - var patchCommands = batch?.DocumentIds.Select(documentId => new PatchCommandData { Key = documentId, Patches = patchRequest }); + var patchCommands = batch?.DocumentIds.Select(documentId => new PatchCommandData(documentId, null, patchRequest); if (patchCommands != null) { - await session.Advanced.DocumentStore.AsyncDatabaseCommands.BatchAsync(patchCommands); - await session.Advanced.DocumentStore.AsyncDatabaseCommands.DeleteAsync(batch.Id, null); + session.Advanced.Defer(patchCommands.ToArray());; + session.Advanced.Defer(new DeleteCommandData(batch.Id, null)); } } @@ -109,7 +109,7 @@ public async Task WaitForIndexUpdateOfArchiveOperation(IDocumentStore stor using (var session = store.OpenAsyncSession()) { var indexQuery = session.Query(new FailedMessages_ByGroup().IndexName) - .Customize(x => x.WaitForNonStaleResultsAsOfNow(timeToWait)); + .Customize(x => x.WaitForNonStaleResults(timeToWait)); var docQuery = indexQuery .Where(failure => failure.FailureGroupId == requestId) @@ -137,22 +137,14 @@ public async Task RemoveArchiveOperation(IDocumentStore store, ArchiveOperation { using (var session = store.OpenAsyncSession()) { - await session.Advanced.DocumentStore.AsyncDatabaseCommands.DeleteAsync(archiveOperation.Id, null); + session.Advanced.Defer(new DeleteCommandData(archiveOperation.Id, null)); await session.SaveChangesAsync(); logger.Info($"Removing ArchiveOperation {archiveOperation.Id} completed"); } } - static PatchRequest[] patchRequest = - { - new PatchRequest - { - Type = PatchCommandType.Set, - Name = "Status", - Value = (int)FailedMessageStatus.Archived - } - }; + static PatchRequest patchRequest = new PatchRequest {Script = @$"this.Status = {(int)FailedMessageStatus.Archived}"}; public class GroupDetails { From b4ad69761e7927031e4ef0678e1866f3b1dcca9a Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Tue, 12 Sep 2023 11:55:10 +0200 Subject: [PATCH 48/78] RavenDbMonitoringIngestionUnitOfWork and async fixes --- .../Archiving/ArchiveDocumentManager.cs | 6 +++--- .../Recoverability/Archiving/MessageArchiver.cs | 4 ++-- .../Archiving/UnarchiveDocumentManager.cs | 4 ++-- .../RavenDbMonitoringIngestionUnitOfWork.cs | 17 +++++++++++------ 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveDocumentManager.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveDocumentManager.cs index 7be6159136..f8a1522de3 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveDocumentManager.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveDocumentManager.cs @@ -65,7 +65,7 @@ public async Task CreateArchiveOperation(IAsyncDocumentSession async Task> StreamResults(IAsyncDocumentSession session, IQueryable query) { var results = new List(); - using (var enumerator = await session.Advanced.StreamAsync(query)) + await using (var enumerator = await session.Advanced.StreamAsync(query)) { while (await enumerator.MoveNextAsync()) { @@ -93,9 +93,9 @@ public async Task GetGroupDetails(IAsyncDocumentSession session, s }; } - public async Task ArchiveMessageGroupBatch(IAsyncDocumentSession session, ArchiveBatch batch) + public void ArchiveMessageGroupBatch(IAsyncDocumentSession session, ArchiveBatch batch) { - var patchCommands = batch?.DocumentIds.Select(documentId => new PatchCommandData(documentId, null, patchRequest); + var patchCommands = batch?.DocumentIds.Select(documentId => new PatchCommandData(documentId, null, patchRequest)); if (patchCommands != null) { diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/MessageArchiver.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/MessageArchiver.cs index 244c149879..bdeba84603 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/MessageArchiver.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/MessageArchiver.cs @@ -71,7 +71,7 @@ public async Task ArchiveAllInGroup(string groupId) logger.Info($"Archiving {nextBatch.DocumentIds.Count} messages from group {groupId} starting"); } - await archiveDocumentManager.ArchiveMessageGroupBatch(batchSession, nextBatch); + archiveDocumentManager.ArchiveMessageGroupBatch(batchSession, nextBatch); await archivingManager.BatchArchived(archiveOperation.RequestId, archiveOperation.ArchiveType, nextBatch?.DocumentIds.Count ?? 0); @@ -164,7 +164,7 @@ public async Task UnarchiveAllInGroup(string groupId) logger.Info($"Unarchiving {nextBatch.DocumentIds.Count} messages from group {groupId} starting"); } - await unarchiveDocumentManager.UnarchiveMessageGroupBatch(batchSession, nextBatch); + unarchiveDocumentManager.UnarchiveMessageGroupBatch(batchSession, nextBatch); await unarchivingManager.BatchUnarchived(unarchiveOperation.RequestId, unarchiveOperation.ArchiveType, nextBatch?.DocumentIds.Count ?? 0); diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveDocumentManager.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveDocumentManager.cs index 1967f361c5..0c2ae5a09f 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveDocumentManager.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveDocumentManager.cs @@ -64,7 +64,7 @@ public async Task CreateUnarchiveOperation(IAsyncDocumentSes async Task> StreamResults(IAsyncDocumentSession session, IQueryable query) { var results = new List(); - using (var enumerator = await session.Advanced.StreamAsync(query)) + await using (var enumerator = await session.Advanced.StreamAsync(query)) { while (await enumerator.MoveNextAsync()) { @@ -92,7 +92,7 @@ public async Task GetGroupDetails(IAsyncDocumentSession session, s }; } - public async Task UnarchiveMessageGroupBatch(IAsyncDocumentSession session, UnarchiveBatch batch) + public void UnarchiveMessageGroupBatch(IAsyncDocumentSession session, UnarchiveBatch batch) { var patchCommands = batch?.DocumentIds.Select(documentId => new PatchCommandData(documentId, null, patchRequest)); diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs index 2462957f21..e3a3114b93 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs @@ -4,6 +4,7 @@ using Newtonsoft.Json.Linq; using ServiceControl.Persistence.UnitOfWork; using Raven.Client.Documents.Commands.Batches; + using Raven.Client.Documents.Operations; class RavenDbMonitoringIngestionUnitOfWork : IMonitoringIngestionUnitOfWork { @@ -20,13 +21,17 @@ public Task RecordKnownEndpoint(KnownEndpoint knownEndpoint) return Task.CompletedTask; } - static PutCommandData CreateKnownEndpointsPutCommand(KnownEndpoint endpoint) => new PutCommandData + static PatchCommandData CreateKnownEndpointsPutCommand(KnownEndpoint endpoint) { - Document = JObject.FromObject(endpoint), - Etag = null, - Key = endpoint.Id.ToString(), - Metadata = KnownEndpointMetadata - }; + var document = JObject.FromObject(endpoint); + document["@metadata"] = KnownEndpointMetadata; + + return new PatchCommandData(endpoint.Id.ToString(), null, new PatchRequest + { + //TODO: check if this works + Script = $"put('{KnownEndpoint.CollectionName}/', {document}" + }); + } static RavenDbMonitoringIngestionUnitOfWork() { From 22b572dbc0aa37c2b74c592981ee3d430af219c3 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Tue, 12 Sep 2023 13:06:07 +0200 Subject: [PATCH 49/78] =?UTF-8?q?=E2=9A=9C=EF=B8=8FCompiler=20hints?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../CustomChecks/CheckRavenDBIndexErrors.cs | 4 ++-- .../CustomChecks/CheckRavenDBIndexErrors.cs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb/CustomChecks/CheckRavenDBIndexErrors.cs b/src/ServiceControl.Persistence.RavenDb/CustomChecks/CheckRavenDBIndexErrors.cs index 27f038687d..c076725f57 100644 --- a/src/ServiceControl.Persistence.RavenDb/CustomChecks/CheckRavenDBIndexErrors.cs +++ b/src/ServiceControl.Persistence.RavenDb/CustomChecks/CheckRavenDBIndexErrors.cs @@ -41,8 +41,8 @@ public override Task PerformCheck() return CheckResult.Failed(message); } - static ILog Logger = LogManager.GetLogger(); + static readonly ILog Logger = LogManager.GetLogger(); - IDocumentStore store; + readonly IDocumentStore store; } } diff --git a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexErrors.cs b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexErrors.cs index 0bfb537b5e..83156e61d8 100644 --- a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexErrors.cs +++ b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexErrors.cs @@ -19,7 +19,7 @@ public CheckRavenDBIndexErrors(IDocumentStore store) public override Task PerformCheck() { var indexErrors = store.Maintenance.Send(new GetIndexErrorsOperation()); - + if (indexErrors.Length == 0) { return CheckResult.Pass; @@ -43,8 +43,8 @@ public override Task PerformCheck() return CheckResult.Failed(message); } - static ILog Logger = LogManager.GetLogger(); + static readonly ILog Logger = LogManager.GetLogger(); - IDocumentStore store; + readonly IDocumentStore store; } } From d08fe262e996a6580c23e352d457183213d8efe2 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Tue, 12 Sep 2023 13:06:49 +0200 Subject: [PATCH 50/78] =?UTF-8?q?=F0=9F=A9=B9=20ErrorMessagesDataStore?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ErrorMessagesDataStore.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index 3ddc771a27..45200165ba 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -335,7 +335,7 @@ static FailedMessageView Map(FailedMessage message, IAsyncDocumentSession sessio NumberOfProcessingAttempts = message.ProcessingAttempts.Count, Status = message.Status, TimeOfFailure = failureDetails.TimeOfFailure, - LastModified = session.Advanced.GetMetadataFor(message)["@last-modified"].Value(), + LastModified = session.Advanced.GetLastModifiedFor(message).Value, Edited = wasEdited, EditOf = wasEdited ? message.ProcessingAttempts.Last().Headers["ServiceControl.EditOf"] : "" }; @@ -539,7 +539,7 @@ class DocumentPatchResult public async Task<(string[] ids, int count)> UnArchiveMessages(IEnumerable failedMessageIds) { - FailedMessage[] failedMessages; + Dictionary failedMessages; using (var session = documentStore.OpenAsyncSession()) { @@ -549,7 +549,7 @@ class DocumentPatchResult failedMessages = await session.LoadAsync(documentIds); - foreach (var failedMessage in failedMessages) + foreach (var failedMessage in failedMessages.Values) { if (failedMessage.Status == FailedMessageStatus.Archived) { @@ -561,8 +561,8 @@ class DocumentPatchResult } return ( - failedMessages.Select(x => x.UniqueMessageId).ToArray(), - failedMessages.Length + failedMessages.Values.Select(x => x.UniqueMessageId).ToArray(), // TODO: (ramon) I don't think we can use Keys here as UniqueMessageId is something different than failedMessageId right? + failedMessages.Count ); } From 0abd1d1a50df56f65669ec6393fb9897f2fed4da Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Tue, 12 Sep 2023 14:18:22 +0200 Subject: [PATCH 51/78] =?UTF-8?q?=F0=9F=A9=B9=20Just=20churning=20through?= =?UTF-8?q?=20the=20errors=20and=20small=20non-complex=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../EventLogDataStore.cs | 4 +- .../KnownEndpointIdGenerator.cs | 7 + .../EventLogDataStore.cs | 8 +- .../FailedErrorImportDataStore.cs | 52 +++---- .../FailedMessageReclassifier.cs | 3 +- .../QueueAddressStore.cs | 41 +++--- .../RavenDBPersisterSettings.cs | 2 +- .../RavenDbCustomCheckDataStore.cs | 47 +++---- .../RavenDbMonitoringDataStore.cs | 127 ++++++++---------- .../RavenDbPersistence.cs | 5 +- .../RavenQueryExtensions.cs | 6 +- .../Archiving/ArchiveDocumentManager.cs | 4 +- .../Archiving/UnarchiveDocumentManager.cs | 14 +- .../Recoverability/RetryHistoryDataStore.cs | 85 ++++++------ .../RetryBatchesDataStore.cs | 3 +- .../RetryBatchesManager.cs | 21 ++- ...avenDbRecoverabilityIngestionUnitOfWork.cs | 8 +- 17 files changed, 202 insertions(+), 235 deletions(-) create mode 100644 src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/KnownEndpointIdGenerator.cs diff --git a/src/ServiceControl.Persistence.RavenDb/EventLogDataStore.cs b/src/ServiceControl.Persistence.RavenDb/EventLogDataStore.cs index 6fbafb64f6..6299ffda67 100644 --- a/src/ServiceControl.Persistence.RavenDb/EventLogDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb/EventLogDataStore.cs @@ -29,7 +29,9 @@ public async Task Add(EventLogItem logItem) { using (var session = documentStore.OpenAsyncSession()) { - var results = await session.Query().Statistics(out var stats) + var results = await session + .Query() + .Statistics(out var stats) .OrderByDescending(p => p.RaisedAt) .Paging(pagingInfo) .ToListAsync(); diff --git a/src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/KnownEndpointIdGenerator.cs b/src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/KnownEndpointIdGenerator.cs new file mode 100644 index 0000000000..ea368580af --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/KnownEndpointIdGenerator.cs @@ -0,0 +1,7 @@ +using System; + +static class KnownEndpointIdGenerator +{ + const string CollectionName = "KnownEndpoint"; + public static string MakeDocumentId(Guid endpointId) => $"{CollectionName}/{endpointId}"; +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/EventLogDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/EventLogDataStore.cs index 1294346cf7..597f52958b 100644 --- a/src/ServiceControl.Persistence.RavenDb5/EventLogDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/EventLogDataStore.cs @@ -4,8 +4,8 @@ using System.Threading.Tasks; using EventLog; using Persistence.Infrastructure; - using Raven.Client; using Raven.Client.Documents; + using Raven.Client.Documents.Session; class EventLogDataStore : IEventLogDataStore { @@ -29,8 +29,10 @@ public async Task Add(EventLogItem logItem) { using (var session = documentStore.OpenAsyncSession()) { - var results = await session.Query().Statistics(out var stats) - .OrderByDescending(p => p.RaisedAt) + var results = await session + .Query() + .Statistics(out var stats) + .OrderByDescending(p => p.RaisedAt, OrderingType.Double) .Paging(pagingInfo) .ToListAsync(); diff --git a/src/ServiceControl.Persistence.RavenDb5/FailedErrorImportDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/FailedErrorImportDataStore.cs index 52b3d6027b..0668256e27 100644 --- a/src/ServiceControl.Persistence.RavenDb5/FailedErrorImportDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/FailedErrorImportDataStore.cs @@ -5,7 +5,6 @@ using System.Threading.Tasks; using NServiceBus.Logging; using ServiceControl.Operations; - using Raven.Client; using Raven.Client.Documents; class FailedErrorImportDataStore : IFailedErrorImportDataStore @@ -26,34 +25,31 @@ public async Task ProcessFailedErrorImports(Func p using (var session = store.OpenAsyncSession()) { var query = session.Query(); - using (var stream = await session.Advanced.StreamAsync(query, cancellationToken) - ) + await using var stream = await session.Advanced.StreamAsync(query, cancellationToken); + while (!cancellationToken.IsCancellationRequested && await stream.MoveNextAsync()) { - while (!cancellationToken.IsCancellationRequested && await stream.MoveNextAsync()) + var transportMessage = stream.Current.Document.Message; + try { - var transportMessage = stream.Current.Document.Message; - try - { - await processMessage(transportMessage); + await processMessage(transportMessage); - await store.AsyncDatabaseCommands.DeleteAsync(stream.Current.Key, null, cancellationToken); - succeeded++; + await store.AsyncDatabaseCommands.DeleteAsync(stream.Current.Key, null, cancellationToken); + succeeded++; - if (Logger.IsDebugEnabled) - { - Logger.Debug($"Successfully re-imported failed error message {transportMessage.Id}."); - } - } - catch (OperationCanceledException) - { - // no-op - } - catch (Exception e) + if (Logger.IsDebugEnabled) { - Logger.Error($"Error while attempting to re-import failed error message {transportMessage.Id}.", e); - failed++; + Logger.Debug($"Successfully re-imported failed error message {transportMessage.Id}."); } } + catch (OperationCanceledException) + { + // no-op + } + catch (Exception e) + { + Logger.Error($"Error while attempting to re-import failed error message {transportMessage.Id}.", e); + failed++; + } } } @@ -67,14 +63,10 @@ public async Task ProcessFailedErrorImports(Func p public async Task QueryContainsFailedImports() { - using (var session = store.OpenAsyncSession()) - { - var query = session.Query(); - using (var ie = await session.Advanced.StreamAsync(query)) - { - return await ie.MoveNextAsync(); - } - } + using var session = store.OpenAsyncSession(); + var query = session.Query(); + await using var ie = await session.Advanced.StreamAsync(query); + return await ie.MoveNextAsync(); } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs b/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs index f977542551..04a69569fc 100644 --- a/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs +++ b/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs @@ -57,8 +57,7 @@ public async Task ReclassifyFailedMessages(bool force) var totalMessagesReclassified = 0; - using (var stream = await session.Advanced.StreamAsync(query.OfType()) - ) + await using (var stream = await session.Advanced.StreamAsync(query.OfType())) { while (!abort && await stream.MoveNextAsync()) { diff --git a/src/ServiceControl.Persistence.RavenDb5/QueueAddressStore.cs b/src/ServiceControl.Persistence.RavenDb5/QueueAddressStore.cs index de271501ea..764ad9cdcc 100644 --- a/src/ServiceControl.Persistence.RavenDb5/QueueAddressStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/QueueAddressStore.cs @@ -3,7 +3,6 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; - using Raven.Client; using Raven.Client.Documents; using ServiceControl.MessageFailures; using ServiceControl.MessageFailures.Api; @@ -20,34 +19,30 @@ public QueueAddressStore(IDocumentStore documentStore) public async Task>> GetAddresses(PagingInfo pagingInfo) { - using (var session = documentStore.OpenAsyncSession()) - { - var addresses = await session - .Query() - .Statistics(out var stats) - .Paging(pagingInfo) - .ToListAsync(); + using var session = documentStore.OpenAsyncSession(); + var addresses = await session + .Query() + .Statistics(out var stats) + .Paging(pagingInfo) + .ToListAsync(); - var result = new QueryResult>(addresses, stats.ToQueryStatsInfo()); - return result; - } + var result = new QueryResult>(addresses, stats.ToQueryStatsInfo()); + return result; } public async Task>> GetAddressesBySearchTerm(string search, PagingInfo pagingInfo) { - using (var session = documentStore.OpenAsyncSession()) - { - var failedMessageQueues = await session - .Query() - .Statistics(out var stats) - .Paging(pagingInfo) - .Where(q => q.PhysicalAddress.StartsWith(search)) - .OrderBy(q => q.PhysicalAddress) - .ToListAsync(); + using var session = documentStore.OpenAsyncSession(); + var failedMessageQueues = await session + .Query() + .Statistics(out var stats) + .Paging(pagingInfo) + .Where(q => q.PhysicalAddress.StartsWith(search)) + .OrderBy(q => q.PhysicalAddress) + .ToListAsync(); - var result = new QueryResult>(failedMessageQueues, stats.ToQueryStatsInfo()); - return result; - } + var result = new QueryResult>(failedMessageQueues, stats.ToQueryStatsInfo()); + return result; } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs index 6e597ffdaf..7a2fbfbf13 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs @@ -16,7 +16,7 @@ class RavenDBPersisterSettings : PersistenceSettings public TimeSpan EventsRetentionPeriod { get; set; } public TimeSpan? AuditRetentionPeriod { get; set; } public int ExternalIntegrationsDispatchingBatchSize { get; set; } = 100; - + //TODO: these are newly added settings, we should remove any duplication public string ServerUrl { get; set; } public string ConnectionString { get; set; } diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs index 4724f03edb..f7dc82eb6c 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs @@ -6,7 +6,6 @@ using ServiceControl.Contracts.CustomChecks; using ServiceControl.Persistence; using ServiceControl.Persistence.Infrastructure; - using Raven.Client; using Raven.Client.Documents; using Raven.Client.Documents.Commands; using Raven.Client.Documents.Linq; @@ -27,19 +26,13 @@ public async Task UpdateCustomCheckStatus(CustomCheckDetail de using (var session = store.OpenAsyncSession()) { - var customCheck = await session.LoadAsync(id); + var customCheck = await session.LoadAsync(id.ToString()); if (customCheck == null || (customCheck.Status == Status.Fail && !detail.HasFailed) || (customCheck.Status == Status.Pass && detail.HasFailed)) { - if (customCheck == null) - { - customCheck = new CustomCheck - { - Id = id - }; - } + customCheck ??= new CustomCheck { Id = id }; status = CheckStateChange.Changed; } @@ -59,38 +52,32 @@ public async Task UpdateCustomCheckStatus(CustomCheckDetail de public async Task>> GetStats(PagingInfo paging, string status = null) { - using (var session = store.OpenAsyncSession()) - { - var query = - session.Query().Statistics(out var stats); + using var session = store.OpenAsyncSession(); + var query = + session.Query().Statistics(out var stats); - query = AddStatusFilter(query, status); + query = AddStatusFilter(query, status); - var results = await query - .Paging(paging) - .ToListAsync(); + var results = await query + .Paging(paging) + .ToListAsync(); - return new QueryResult>(results, new QueryStatsInfo($"{stats.ResultEtag}", stats.TotalResults, stats.IsStale)); - } + return new QueryResult>(results, new QueryStatsInfo($"{stats.ResultEtag}", stats.TotalResults, stats.IsStale)); } public async Task DeleteCustomCheck(Guid id) { var documentId = store.Conventions.DefaultFindFullDocumentKeyFromNonStringIdentifier(id, typeof(CustomCheck), false); - using (var session = store.OpenAsyncSession(new SessionOptions {NoTracking = true, NoCaching = true})) - { - await session.Advanced.RequestExecutor.ExecuteAsync(new DeleteDocumentCommand(documentId, null), session.Advanced.Context); - } + using var session = store.OpenAsyncSession(new SessionOptions { NoTracking = true, NoCaching = true }); + await session.Advanced.RequestExecutor.ExecuteAsync(new DeleteDocumentCommand(documentId, null), session.Advanced.Context); } public async Task GetNumberOfFailedChecks() { - using (var session = store.OpenAsyncSession()) - { - var failedCustomCheckCount = await session.Query().CountAsync(p => p.Status == Status.Fail); + using var session = store.OpenAsyncSession(); + var failedCustomCheckCount = await session.Query().CountAsync(p => p.Status == Status.Fail); - return failedCustomCheckCount; - } + return failedCustomCheckCount; } static IRavenQueryable AddStatusFilter(IRavenQueryable query, string status) @@ -113,8 +100,6 @@ static IRavenQueryable AddStatusFilter(IRavenQueryable return query; } - public Task Setup() => Task.CompletedTask; - - IDocumentStore store; + readonly IDocumentStore store; } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbMonitoringDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbMonitoringDataStore.cs index e95fe73d23..3d6613bda2 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbMonitoringDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbMonitoringDataStore.cs @@ -2,11 +2,9 @@ { using System; using System.Collections.Generic; - using System.Linq; using System.Threading.Tasks; using ServiceControl.Operations; using ServiceControl.Persistence; - using Raven.Client; using Raven.Client.Documents; class RavenDbMonitoringDataStore : IMonitoringDataStore @@ -20,113 +18,102 @@ public async Task CreateIfNotExists(EndpointDetails endpoint) { var id = endpoint.GetDeterministicId(); - using (var session = store.OpenAsyncSession()) - { - var knownEndpoint = await session.LoadAsync(id); + using var session = store.OpenAsyncSession(); - if (knownEndpoint != null) - { - return; - } + var knownEndpoint = await session.LoadAsync(id.ToString()); - knownEndpoint = new KnownEndpoint - { - Id = id, - EndpointDetails = endpoint, - HostDisplayName = endpoint.Host, - Monitored = false - }; + if (knownEndpoint != null) + { + return; + } - await session.StoreAsync(knownEndpoint); + knownEndpoint = new KnownEndpoint + { + Id = id, + EndpointDetails = endpoint, + HostDisplayName = endpoint.Host, + Monitored = false + }; - await session.SaveChangesAsync(); - } + await session.StoreAsync(knownEndpoint); + + await session.SaveChangesAsync(); } public async Task CreateOrUpdate(EndpointDetails endpoint, IEndpointInstanceMonitoring endpointInstanceMonitoring) { var id = endpoint.GetDeterministicId(); - using (var session = store.OpenAsyncSession()) - { - var knownEndpoint = await session.LoadAsync(id); + using var session = store.OpenAsyncSession(); - if (knownEndpoint == null) - { - knownEndpoint = new KnownEndpoint - { - Id = id, - EndpointDetails = endpoint, - HostDisplayName = endpoint.Host, - Monitored = true - }; - - await session.StoreAsync(knownEndpoint); - } - else + var knownEndpoint = await session.LoadAsync(id.ToString()); + + if (knownEndpoint == null) + { + knownEndpoint = new KnownEndpoint { - knownEndpoint.Monitored = endpointInstanceMonitoring.IsMonitored(id); - } + Id = id, + EndpointDetails = endpoint, + HostDisplayName = endpoint.Host, + Monitored = true + }; - await session.SaveChangesAsync(); + await session.StoreAsync(knownEndpoint); + } + else + { + knownEndpoint.Monitored = endpointInstanceMonitoring.IsMonitored(id); } + + await session.SaveChangesAsync(); } public async Task UpdateEndpointMonitoring(EndpointDetails endpoint, bool isMonitored) { var id = endpoint.GetDeterministicId(); - using (var session = store.OpenAsyncSession()) - { - var knownEndpoint = await session.LoadAsync(id); + using var session = store.OpenAsyncSession(); - if (knownEndpoint != null) - { - knownEndpoint.Monitored = isMonitored; + var knownEndpoint = await session.LoadAsync(id.ToString()); + + if (knownEndpoint != null) + { + knownEndpoint.Monitored = isMonitored; - await session.SaveChangesAsync(); - } + await session.SaveChangesAsync(); } } public async Task WarmupMonitoringFromPersistence(IEndpointInstanceMonitoring endpointInstanceMonitoring) { - using (var session = store.OpenAsyncSession()) + using var session = store.OpenAsyncSession(); + await using var endpointsEnumerator = await session.Advanced.StreamAsync(session.Query()); + + while (await endpointsEnumerator.MoveNextAsync()) { - using (var endpointsEnumerator = await session.Advanced.StreamAsync(session.Query()) - ) - { - while (await endpointsEnumerator.MoveNextAsync()) - { - var endpoint = endpointsEnumerator.Current.Document; + var endpoint = endpointsEnumerator.Current.Document; - endpointInstanceMonitoring.DetectEndpointFromPersistentStore(endpoint.EndpointDetails, endpoint.Monitored); - } - } + endpointInstanceMonitoring.DetectEndpointFromPersistentStore(endpoint.EndpointDetails, endpoint.Monitored); } } public async Task Delete(Guid endpointId) { - using (var session = store.OpenAsyncSession()) - { - session.Delete(endpointId); - await session.SaveChangesAsync(); - } + using var session = store.OpenAsyncSession(); + session.Delete(KnownEndpointIdGenerator.MakeDocumentId(endpointId)); + await session.SaveChangesAsync(); } + public async Task> GetAllKnownEndpoints() { - using (var session = store.OpenAsyncSession()) - { - var knownEndpoints = await session.Query() - .ToListAsync(); + using var session = store.OpenAsyncSession(); - return knownEndpoints.ToArray(); - } - } + var knownEndpoints = await session.Query() + .ToListAsync(); - public Task Setup() => Task.CompletedTask; + return knownEndpoints.ToArray(); + } - IDocumentStore store; + readonly IDocumentStore store; } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs index acba5555d3..ecae7dbd10 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs @@ -3,7 +3,6 @@ using MessageRedirects; using Microsoft.Extensions.DependencyInjection; using Persistence.Recoverability; - using Raven.Client.Documents; using RavenDb5; using Recoverability; using ServiceControl.CustomChecks; @@ -98,10 +97,10 @@ public IPersistenceInstaller CreateInstaller() ConfigureLifecycle(serviceCollection); var lifecycle = serviceCollection.BuildServiceProvider().GetRequiredService(); - + return new RavenDbInstaller(lifecycle); } readonly RavenDBPersisterSettings settings; } -} +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs b/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs index 0d04ab9904..5d96ce46ce 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs @@ -25,8 +25,10 @@ static class RavenQueryExtensions return source; } - public static IRavenQueryable Paging(this IRavenQueryable source, PagingInfo pagingInfo) - => source.Skip(pagingInfo.Offset).Take(pagingInfo.PageSize); + public static IQueryable Paging(this IOrderedQueryable source, PagingInfo pagingInfo) + => source + .Skip(pagingInfo.Offset) + .Take(pagingInfo.PageSize); public static IRavenQueryable Sort(this IRavenQueryable source, SortInfo sortInfo) where TSource : MessagesViewIndex.SortAndFilterOptions diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveDocumentManager.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveDocumentManager.cs index f8a1522de3..22cd9ba04b 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveDocumentManager.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/ArchiveDocumentManager.cs @@ -99,7 +99,7 @@ public void ArchiveMessageGroupBatch(IAsyncDocumentSession session, ArchiveBatch if (patchCommands != null) { - session.Advanced.Defer(patchCommands.ToArray());; + session.Advanced.Defer(patchCommands.ToArray()); session.Advanced.Defer(new DeleteCommandData(batch.Id, null)); } } @@ -144,7 +144,7 @@ public async Task RemoveArchiveOperation(IDocumentStore store, ArchiveOperation } } - static PatchRequest patchRequest = new PatchRequest {Script = @$"this.Status = {(int)FailedMessageStatus.Archived}"}; + static PatchRequest patchRequest = new PatchRequest { Script = @$"this.Status = {(int)FailedMessageStatus.Archived}" }; public class GroupDetails { diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveDocumentManager.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveDocumentManager.cs index 0c2ae5a09f..3287cc5289 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveDocumentManager.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/UnarchiveDocumentManager.cs @@ -127,21 +127,19 @@ public async Task WaitForIndexUpdateOfUnarchiveOperation(IDocumentStore st } } - public Task UpdateUnarchiveOperation(IAsyncDocumentSession session, UnarchiveOperation unarchiveOperation) + public async Task UpdateUnarchiveOperation(IAsyncDocumentSession session, UnarchiveOperation unarchiveOperation) { - return session.StoreAsync(unarchiveOperation); + await session.StoreAsync(unarchiveOperation); } public async Task RemoveUnarchiveOperation(IDocumentStore store, UnarchiveOperation unarchiveOperation) { - using (var session = store.OpenAsyncSession()) - { - session.Advanced.Defer(new DeleteCommandData(unarchiveOperation.Id, null)); - await session.SaveChangesAsync(); - } + using var session = store.OpenAsyncSession(); + session.Advanced.Defer(new DeleteCommandData(unarchiveOperation.Id, null)); + await session.SaveChangesAsync(); } - static PatchRequest patchRequest = new PatchRequest {Script = $@"this.Status = {(int)FailedMessageStatus.Unresolved}"}; + static PatchRequest patchRequest = new PatchRequest { Script = $@"this.Status = {(int)FailedMessageStatus.Unresolved}" }; public class GroupDetails { diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/RetryHistoryDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/RetryHistoryDataStore.cs index fed4187a33..21fc002aa1 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Recoverability/RetryHistoryDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/RetryHistoryDataStore.cs @@ -14,68 +14,61 @@ public RetryHistoryDataStore(IDocumentStore documentStore) public async Task GetRetryHistory() { - using (var session = documentStore.OpenAsyncSession()) - { - var id = RetryHistory.MakeId(); - var retryHistory = await session.LoadAsync(id); + using var session = documentStore.OpenAsyncSession(); + var id = RetryHistory.MakeId(); + var retryHistory = await session.LoadAsync(id); - retryHistory = retryHistory ?? RetryHistory.CreateNew(); + retryHistory ??= RetryHistory.CreateNew(); - return retryHistory; - } + return retryHistory; } public async Task RecordRetryOperationCompleted(string requestId, RetryType retryType, DateTime startTime, DateTime completionTime, string originator, string classifier, bool messageFailed, int numberOfMessagesProcessed, DateTime lastProcessed, int retryHistoryDepth) { - using (var session = documentStore.OpenAsyncSession()) - { - var retryHistory = await session.LoadAsync(RetryHistory.MakeId()) ?? - RetryHistory.CreateNew(); + using var session = documentStore.OpenAsyncSession(); + var retryHistory = await session.LoadAsync(RetryHistory.MakeId()) ?? RetryHistory.CreateNew(); - retryHistory.AddToUnacknowledged(new UnacknowledgedRetryOperation - { - RequestId = requestId, - RetryType = retryType, - StartTime = startTime, - CompletionTime = completionTime, - Originator = originator, - Classifier = classifier, - Failed = messageFailed, - NumberOfMessagesProcessed = numberOfMessagesProcessed, - Last = lastProcessed - }); + retryHistory.AddToUnacknowledged(new UnacknowledgedRetryOperation + { + RequestId = requestId, + RetryType = retryType, + StartTime = startTime, + CompletionTime = completionTime, + Originator = originator, + Classifier = classifier, + Failed = messageFailed, + NumberOfMessagesProcessed = numberOfMessagesProcessed, + Last = lastProcessed + }); - retryHistory.AddToHistory(new HistoricRetryOperation - { - RequestId = requestId, - RetryType = retryType, - StartTime = startTime, - CompletionTime = completionTime, - Originator = originator, - Failed = messageFailed, - NumberOfMessagesProcessed = numberOfMessagesProcessed - }, retryHistoryDepth); + retryHistory.AddToHistory(new HistoricRetryOperation + { + RequestId = requestId, + RetryType = retryType, + StartTime = startTime, + CompletionTime = completionTime, + Originator = originator, + Failed = messageFailed, + NumberOfMessagesProcessed = numberOfMessagesProcessed + }, retryHistoryDepth); - await session.StoreAsync(retryHistory); - await session.SaveChangesAsync(); - } + await session.StoreAsync(retryHistory); + await session.SaveChangesAsync(); } public async Task AcknowledgeRetryGroup(string groupId) { - using (var session = documentStore.OpenAsyncSession()) + using var session = documentStore.OpenAsyncSession(); + var retryHistory = await session.LoadAsync(RetryHistory.MakeId()); + if (retryHistory != null) { - var retryHistory = await session.LoadAsync(RetryHistory.MakeId()); - if (retryHistory != null) + if (retryHistory.Acknowledge(groupId, RetryType.FailureGroup)) { - if (retryHistory.Acknowledge(groupId, RetryType.FailureGroup)) - { - await session.StoreAsync(retryHistory); - await session.SaveChangesAsync(); + await session.StoreAsync(retryHistory); + await session.SaveChangesAsync(); - return true; - } + return true; } } @@ -84,4 +77,4 @@ public async Task AcknowledgeRetryGroup(string groupId) readonly IDocumentStore documentStore; } -} +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RetryBatchesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RetryBatchesDataStore.cs index f646367a6d..c5dd1b738c 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RetryBatchesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RetryBatchesDataStore.cs @@ -6,7 +6,6 @@ using MessageFailures; using NServiceBus.Logging; using ServiceControl.Recoverability; - using Raven.Client; using Raven.Client.Documents; using Raven.Client.Documents.Commands.Batches; using Raven.Client.Documents.Operations; @@ -16,7 +15,7 @@ class RetryBatchesDataStore : IRetryBatchesDataStore { readonly IDocumentStore documentStore; - static ILog Log = LogManager.GetLogger(typeof(RetryBatchesDataStore)); + static readonly ILog Log = LogManager.GetLogger(typeof(RetryBatchesDataStore)); public RetryBatchesDataStore(IDocumentStore documentStore) { diff --git a/src/ServiceControl.Persistence.RavenDb5/RetryBatchesManager.cs b/src/ServiceControl.Persistence.RavenDb5/RetryBatchesManager.cs index a2e090d5b3..bc3ae2a96b 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RetryBatchesManager.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RetryBatchesManager.cs @@ -1,13 +1,13 @@ namespace ServiceControl.Persistence.RavenDb { - using System; using System.Collections.Generic; + using System.Linq; using System.Threading; using System.Threading.Tasks; using MessageFailures; using Persistence.MessageRedirects; using ServiceControl.Recoverability; - using Raven.Client; + using Raven.Client.Documents; using Raven.Client.Documents.Session; class RetryBatchesManager : AbstractSessionManager, IRetryBatchesManager @@ -20,11 +20,19 @@ public RetryBatchesManager(IAsyncDocumentSession session) : base(session) public void Delete(RetryBatchNowForwarding forwardingBatch) => Session.Delete(forwardingBatch); - public Task GetFailedMessageRetries(IList stagingBatchFailureRetries) => Session.LoadAsync(stagingBatchFailureRetries); + public async Task GetFailedMessageRetries(IList stagingBatchFailureRetries) + { + var result = await Session.LoadAsync(stagingBatchFailureRetries); + return result.Values.ToArray(); + } public void Evict(FailedMessageRetry failedMessageRetry) => Session.Advanced.Evict(failedMessageRetry); - public Task GetFailedMessages(Dictionary.KeyCollection keys) => Session.LoadAsync(keys); + public async Task GetFailedMessages(Dictionary.KeyCollection keys) + { + var result = await Session.LoadAsync(keys); + return result.Values.ToArray(); + } public async Task GetRetryBatchNowForwarding() => await Session.Include(r => r.RetryBatchId) @@ -47,9 +55,8 @@ public async Task GetOrCreateMessageRedirectsCollect if (redirects != null) { - redirects.ETag = Session.Advanced.GetEtagFor(redirects); - redirects.LastModified = Session.Advanced.GetMetadataFor(redirects).Value("@last-modified"); - + redirects.ETag = Session.Advanced.GetChangeVectorFor(redirects); + redirects.LastModified = Session.Advanced.GetLastModifiedFor(redirects)!.Value; return redirects; } diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs index 590d9141d1..0b02f6ff58 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs @@ -55,8 +55,8 @@ ICommandData CreateFailedMessagesPatchCommand(string uniqueMessageId, FailedMess //HINT: RavenDB 3.5 is using Lodash v4.13.1 to provide javascript utility functions // https://ravendb.net/docs/article-page/3.5/csharp/client-api/commands/patches/how-to-use-javascript-to-patch-your-documents#methods-objects-and-variables return new PatchCommandData(documentId, null, new PatchRequest - { - Script = $@"this.{nameof(FailedMessage.Status)} = status; + { + Script = $@"this.{nameof(FailedMessage.Status)} = status; this.{nameof(FailedMessage.FailureGroups)} = failureGroups; var newAttempts = this.{nameof(FailedMessage.ProcessingAttempts)}; @@ -84,13 +84,13 @@ ICommandData CreateFailedMessagesPatchCommand(string uniqueMessageId, FailedMess this.{nameof(FailedMessage.ProcessingAttempts)} = newAttempts; ", - Values = new Dictionary + Values = new Dictionary { {"status", (int)FailedMessageStatus.Unresolved}, {"failureGroups", serializedGroups}, {"attempt", serializedAttempt} }, - }, + }, patchIfMissing: new PatchRequest { Script = $@"this.{nameof(FailedMessage.Status)} = status; From 6880e9ea5ad48f6eb32f06733186cf1478d29eb8 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Tue, 12 Sep 2023 14:34:58 +0200 Subject: [PATCH 52/78] =?UTF-8?q?=F0=9F=A9=B9=20BeforeQueryExecution(index?= =?UTF-8?q?=20=3D>=20index.Cutoff=20=3D=20cutoff)=20can=20no=20longer=20be?= =?UTF-8?q?=20done,=20replaced=20with=20`WaitForNonStaleResults`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../RetryDocumentDataStore.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs index 32c15e560b..757384b3b7 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs @@ -93,8 +93,16 @@ public async Task>> QueryOrphanedBatches(string re { using (var session = store.OpenAsyncSession()) { - var orphanedBatches = await session.Query() - .Customize(c => c.BeforeQueryExecution(index => index.Cutoff = cutoff)) + var orphanedBatches = await session + .Query() + + // TODO: Cutoff no longer exists but guidance isn't clear how to handle this: + // https://ravendb.net/docs/article-page/5.4/Csharp/indexes/stale-indexes + // https://ravendb.net/docs/article-page/5.4/csharp/client-api/session/querying/how-to-customize-query#waitfornonstaleresults + + //.Customize(c => c.BeforeQueryExecuted(index => index.Cutoff = cutoff)) + .Customize(c => c.WaitForNonStaleResults()) // (ramon) I think this is valid as at start orphaned batches should be retrieved based on non-stale results I would assume? + .Where(b => b.Status == RetryBatchStatus.MarkingDocuments && b.RetrySessionId != retrySessionId) .Statistics(out var stats) .ToListAsync(); From 96e0cc24c8f3a92078d777e0f41881d784319c38 Mon Sep 17 00:00:00 2001 From: David Boike Date: Tue, 12 Sep 2023 10:44:31 -0500 Subject: [PATCH 53/78] Simple fixes --- .../ErrorMessagesDataStore.cs | 1 - .../FailedMessageReclassifier.cs | 12 ++--- .../Infrastructure/MigratedTypeAwareBinder.cs | 49 +++++++------------ 3 files changed, 21 insertions(+), 41 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index 45200165ba..6f1eb40459 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -7,7 +7,6 @@ using System.Threading.Tasks; using CompositeViews.Messages; using Editing; - using Newtonsoft.Json.Linq; using NServiceBus.Logging; using Raven.Client.Documents; using Raven.Client.Documents.Linq; diff --git a/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs b/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs index 04a69569fc..a9a8012e71 100644 --- a/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs +++ b/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs @@ -8,14 +8,13 @@ using Microsoft.Extensions.Hosting; using Newtonsoft.Json.Linq; using NServiceBus.Logging; + using Raven.Client.Documents; + using Raven.Client.Documents.Operations; + using Raven.Client.Exceptions; using ServiceControl.MessageFailures; using ServiceControl.MessageFailures.Api; using ServiceControl.Persistence.Infrastructure; using ServiceControl.Recoverability; - using Raven.Client; - using Raven.Client.Documents; - using Raven.Client.Documents.Operations; - using Raven.Client.Exceptions; class FailedMessageReclassifier : IReclassifyFailedMessages { @@ -81,10 +80,7 @@ public async Task ReclassifyFailedMessages(bool force) logger.Info($"Reclassification of failures ended. Reclassified {failedMessagesReclassified} messages"); - if (settings == null) - { - settings = new ReclassifyErrorSettings(); - } + settings ??= new ReclassifyErrorSettings(); settings.ReclassificationDone = true; await session.StoreAsync(settings); diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/MigratedTypeAwareBinder.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/MigratedTypeAwareBinder.cs index 70257f7ffe..e1fd124395 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/MigratedTypeAwareBinder.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/MigratedTypeAwareBinder.cs @@ -17,39 +17,24 @@ class MigratedTypeAwareBinder : DefaultSerializationBinder public override Type BindToType(string assemblyName, string typeName) { var className = GetClassName(typeName); - switch (className) + return className switch { - case nameof(CustomCheck): - return typeof(CustomCheck); - case nameof(CustomCheckDetail): - return typeof(CustomCheckDetail); - case nameof(EndpointDetails): - return typeof(EndpointDetails); - case nameof(ExceptionDetails): - return typeof(ExceptionDetails); - case nameof(FailedMessage): - return typeof(FailedMessage); - case nameof(FailureDetails): - return typeof(FailureDetails); - case nameof(FailureGroup): - return typeof(FailureGroup); - case nameof(GroupComment): - return typeof(GroupComment); - case nameof(KnownEndpoint): - return typeof(KnownEndpoint); - case nameof(ProcessedMessage): - return typeof(ProcessedMessage); - case nameof(ProcessingAttempt): - return typeof(ProcessingAttempt); - case nameof(FailedMessageRetry): - return typeof(FailedMessageRetry); - case nameof(RetryBatch): - return typeof(RetryBatch); - case nameof(RetryBatchNowForwarding): - return typeof(RetryBatchNowForwarding); - default: - return base.BindToType(assemblyName, typeName); - } + nameof(CustomCheck) => typeof(CustomCheck), + nameof(CustomCheckDetail) => typeof(CustomCheckDetail), + nameof(EndpointDetails) => typeof(EndpointDetails), + nameof(ExceptionDetails) => typeof(ExceptionDetails), + nameof(FailedMessage) => typeof(FailedMessage), + nameof(FailureDetails) => typeof(FailureDetails), + nameof(FailureGroup) => typeof(FailureGroup), + nameof(GroupComment) => typeof(GroupComment), + nameof(KnownEndpoint) => typeof(KnownEndpoint), + nameof(ProcessedMessage) => typeof(ProcessedMessage), + nameof(ProcessingAttempt) => typeof(ProcessingAttempt), + nameof(FailedMessageRetry) => typeof(FailedMessageRetry), + nameof(RetryBatch) => typeof(RetryBatch), + nameof(RetryBatchNowForwarding) => typeof(RetryBatchNowForwarding), + _ => base.BindToType(assemblyName, typeName), + }; } string GetClassName(string typeName) From de1e8d96da4ef2f4f58acabc32080911f6f6d481 Mon Sep 17 00:00:00 2001 From: David Boike Date: Tue, 12 Sep 2023 10:45:07 -0500 Subject: [PATCH 54/78] Fix DocumentQuery FilterByStatus, hopefully will still be useful --- .../RavenQueryExtensions.cs | 25 +++---------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs b/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs index 5d96ce46ce..4e9988311e 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenQueryExtensions.cs @@ -6,7 +6,6 @@ namespace ServiceControl.Persistence using System.Linq; using System.Linq.Expressions; using System.Net.Http; - using System.Text; using Raven.Client.Documents.Linq; using Raven.Client.Documents.Session; using ServiceControl.MessageFailures; @@ -179,32 +178,16 @@ public static IAsyncDocumentQuery FilterByStatusWhere(this IAsyncDocumentQ } } - var sb = new StringBuilder(); - - sb.Append("(("); - if (includes.Count == 0) - { - sb.Append("*"); - } - else + if (includes.Any()) { - sb.Append(string.Join(" OR ", includes.ToArray())); + source.WhereIn("Status", includes.Cast()); } - sb.Append(")"); - - if (excludes.Count > 0) + foreach (var exclude in excludes) { - sb.Append(" AND NOT ("); - sb.Append(string.Join(" OR ", excludes.ToArray())); - sb.Append(")"); + source.WhereNotEquals("Status", exclude); } - sb.Append(")"); - - source.AndAlso(); - source.Where($"Status: {sb}"); - return source; } From a54f7542986207d5c43a997a03b37095d9128008 Mon Sep 17 00:00:00 2001 From: David Boike Date: Tue, 12 Sep 2023 11:05:21 -0500 Subject: [PATCH 55/78] RetryDocumentDataStore patch operations --- .../RetryDocumentDataStore.cs | 59 +++++++------------ 1 file changed, 20 insertions(+), 39 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs index 757384b3b7..d387f658e6 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs @@ -5,7 +5,6 @@ using System.Linq; using System.Threading.Tasks; using MessageFailures; - using Newtonsoft.Json.Linq; using NServiceBus.Logging; using Persistence.Infrastructure; using Raven.Client.Documents; @@ -36,24 +35,25 @@ public async Task StageRetryByUniqueMessageIds(string batchDocumentId, string re commands[i] = CreateFailedMessageRetryDocument(batchDocumentId, messageIds[i]); } - await store.AsyncDatabaseCommands.BatchAsync(commands); + using (var session = store.OpenAsyncSession()) + { + var batch = new SingleNodeBatchCommand(store.Conventions, session.Advanced.Context, commands); + await session.Advanced.RequestExecutor.ExecuteAsync(batch, session.Advanced.Context); + } } public async Task MoveBatchToStaging(string batchDocumentId) { try { - await store.AsyncDatabaseCommands.PatchAsync(batchDocumentId, - new[] + await store.Operations.SendAsync(new PatchOperation(batchDocumentId, null, new PatchRequest + { + Script = @"this.Status = args.Status", + Values = { - new PatchRequest - { - Type = PatchCommandType.Set, - Name = "Status", - Value = (int)RetryBatchStatus.Staging, - PrevVal = (int)RetryBatchStatus.MarkingDocuments - } - }); + {"Status", (int)RetryBatchStatus.Staging } + } + })); } catch (ConcurrencyException) { @@ -124,37 +124,18 @@ public async Task> QueryAvailableBatches() static ICommandData CreateFailedMessageRetryDocument(string batchDocumentId, string messageId) { - return new PatchCommandData + return new PatchCommandData(FailedMessageRetry.MakeDocumentId(messageId), null, patch: null, patchIfMissing: new PatchRequest { - Patches = PatchRequestsEmpty, - PatchesIfMissing = new[] + Script = @"this.FailedMessageId = args.MessageId + this.RetryBatchId = args.BatchDocumentId", + Values = { - new PatchRequest - { - Name = "FailedMessageId", - Type = PatchCommandType.Set, - Value = FailedMessageIdGenerator.MakeDocumentId(messageId) - }, - new PatchRequest - { - Name = "RetryBatchId", - Type = PatchCommandType.Set, - Value = batchDocumentId - } - }, - Key = FailedMessageRetry.MakeDocumentId(messageId), - Metadata = DefaultMetadata - }; + { "MessageId", FailedMessageIdGenerator.MakeDocumentId(messageId) }, + { "BatchDocumentId", batchDocumentId } + } + }); } - static JObject DefaultMetadata = JObject.Parse($@" - {{ - ""@collection"": ""{FailedMessageRetry.CollectionName}"", - ""Raven-Clr-Type"": ""{typeof(FailedMessageRetry).AssemblyQualifiedName}"" - }}"); - - static PatchRequest[] PatchRequestsEmpty = Array.Empty(); - static ILog log = LogManager.GetLogger(typeof(RetryDocumentDataStore)); // TODO: Verify Stream queries in this file, which were the result of joining overly-complex IndexBasedBulkRetryRequest From 97a545d268a3b5c2537cddac0e822de41e7dd05d Mon Sep 17 00:00:00 2001 From: David Boike Date: Tue, 12 Sep 2023 11:18:31 -0500 Subject: [PATCH 56/78] Fix weirdly complex include API usage --- .../RetryBatchesManager.cs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/RetryBatchesManager.cs b/src/ServiceControl.Persistence.RavenDb5/RetryBatchesManager.cs index bc3ae2a96b..43da58b7c2 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RetryBatchesManager.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RetryBatchesManager.cs @@ -6,9 +6,9 @@ using System.Threading.Tasks; using MessageFailures; using Persistence.MessageRedirects; - using ServiceControl.Recoverability; using Raven.Client.Documents; using Raven.Client.Documents.Session; + using ServiceControl.Recoverability; class RetryBatchesManager : AbstractSessionManager, IRetryBatchesManager { @@ -41,10 +41,12 @@ await Session.Include(r => r.RetryBatchId) public async Task GetRetryBatch(string retryBatchId, CancellationToken cancellationToken) => await Session.LoadAsync(retryBatchId, cancellationToken); - public async Task GetStagingBatch() => - await Session.Query() - .Customize(q => q.Include(b => b.FailureRetries)) + public async Task GetStagingBatch() + { + return await Session.Query() + .Include(b => b.FailureRetries) .FirstOrDefaultAsync(b => b.Status == RetryBatchStatus.Staging); + } public async Task Store(RetryBatchNowForwarding retryBatchNowForwarding) => await Session.StoreAsync(retryBatchNowForwarding, RetryBatchNowForwarding.Id); From 6b01993326ffbc6db2a1b43acc88fe31216c84ef Mon Sep 17 00:00:00 2001 From: David Boike Date: Tue, 12 Sep 2023 11:18:52 -0500 Subject: [PATCH 57/78] Patches in RetryBatchesDataStore --- .../RetryBatchesDataStore.cs | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/RetryBatchesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RetryBatchesDataStore.cs index c5dd1b738c..d20d0b84da 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RetryBatchesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RetryBatchesDataStore.cs @@ -5,11 +5,12 @@ using System.Threading.Tasks; using MessageFailures; using NServiceBus.Logging; - using ServiceControl.Recoverability; using Raven.Client.Documents; + using Raven.Client.Documents.Commands; using Raven.Client.Documents.Commands.Batches; using Raven.Client.Documents.Operations; using Raven.Client.Exceptions; + using ServiceControl.Recoverability; class RetryBatchesDataStore : IRetryBatchesDataStore { @@ -42,17 +43,14 @@ public async Task RecordFailedStagingAttempt(IReadOnlyCollection Log.Warn($"Attempt {1} of {maxStagingAttempts} to stage a retry message {failedMessage.UniqueMessageId} failed", e); - commands[commandIndex] = new PatchCommandData + commands[commandIndex] = new PatchCommandData(failedMessageRetry.Id, null, new PatchRequest { - Patches = new[] + Script = @"this.StageAttempts = args.Value", + Values = { - new PatchRequest - { - Type = PatchCommandType.Set, Name = "StageAttempts", Value = 1 - } - }, - Key = failedMessageRetry.Id - }; + {"Value", 1 } + } + }); commandIndex++; } @@ -60,7 +58,10 @@ public async Task RecordFailedStagingAttempt(IReadOnlyCollection try { - await documentStore.AsyncDatabaseCommands.BatchAsync(commands); + using var session = documentStore.OpenAsyncSession(); + + var batch = new SingleNodeBatchCommand(documentStore.Conventions, session.Advanced.Context, commands); + await session.Advanced.RequestExecutor.ExecuteAsync(batch, session.Advanced.Context); } catch (ConcurrencyException) { @@ -74,16 +75,10 @@ public async Task IncrementAttemptCounter(FailedMessageRetry message) { try { - await documentStore.AsyncDatabaseCommands.PatchAsync(message.Id, - new[] - { - new PatchRequest - { - Type = PatchCommandType.Set, - Name = "StageAttempts", - Value = message.StageAttempts + 1 - } - }); + await documentStore.Operations.SendAsync(new PatchOperation(message.Id, null, new PatchRequest + { + Script = @"this.StageAttempts += 1" + })); } catch (ConcurrencyException) { @@ -91,6 +86,10 @@ await documentStore.AsyncDatabaseCommands.PatchAsync(message.Id, } } - public Task DeleteFailedMessageRetry(string uniqueMessageId) => documentStore.AsyncDatabaseCommands.DeleteAsync(FailedMessageRetry.MakeDocumentId(uniqueMessageId), null); + public async Task DeleteFailedMessageRetry(string uniqueMessageId) + { + using var session = documentStore.OpenAsyncSession(); + await session.Advanced.RequestExecutor.ExecuteAsync(new DeleteDocumentCommand(FailedMessageRetry.MakeDocumentId(uniqueMessageId), null), session.Advanced.Context); + } } } \ No newline at end of file From 67868ad9a63f81061826effaa76941516366cd45 Mon Sep 17 00:00:00 2001 From: David Boike Date: Tue, 12 Sep 2023 13:16:29 -0500 Subject: [PATCH 58/78] A patch and a delete --- .../FailedErrorImportDataStore.cs | 6 ++++-- .../FailedMessages/FailedMessageReclassifier.cs | 16 +++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/FailedErrorImportDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/FailedErrorImportDataStore.cs index 0668256e27..1a4de2512c 100644 --- a/src/ServiceControl.Persistence.RavenDb5/FailedErrorImportDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/FailedErrorImportDataStore.cs @@ -4,8 +4,9 @@ using System.Threading; using System.Threading.Tasks; using NServiceBus.Logging; - using ServiceControl.Operations; using Raven.Client.Documents; + using Raven.Client.Documents.Commands; + using ServiceControl.Operations; class FailedErrorImportDataStore : IFailedErrorImportDataStore { @@ -33,7 +34,8 @@ public async Task ProcessFailedErrorImports(Func p { await processMessage(transportMessage); - await store.AsyncDatabaseCommands.DeleteAsync(stream.Current.Key, null, cancellationToken); + await session.Advanced.RequestExecutor.ExecuteAsync(new DeleteDocumentCommand(stream.Current.Id, null), session.Advanced.Context, token: cancellationToken); + succeeded++; if (Logger.IsDebugEnabled) diff --git a/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs b/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs index a9a8012e71..ba986ed298 100644 --- a/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs +++ b/src/ServiceControl.Persistence.RavenDb5/FailedMessages/FailedMessageReclassifier.cs @@ -100,16 +100,14 @@ int ReclassifyBatch(IDocumentStore store, IEnumerable Date: Tue, 12 Sep 2023 13:20:04 -0500 Subject: [PATCH 59/78] Comment out the rest of the Transformer stuff --- .../ErrorMessagesDataStore.cs | 15 ++-- .../FailedMessageViewTransformer.cs | 90 +++++++++---------- 2 files changed, 55 insertions(+), 50 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index 6f1eb40459..10d824f09d 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -186,7 +186,8 @@ SortInfo sortInfo .FilterByQueueAddress(queueAddress) .Sort(sortInfo) .Paging(pagingInfo) - .SetResultTransformer(new FailedMessageViewTransformer().TransformerName) + // TODO: Fix SetResultTransformer + //.SetResultTransformer(new FailedMessageViewTransformer().TransformerName) .SelectFields() .ToListAsync(); @@ -232,7 +233,8 @@ SortInfo sortInfo .FilterByLastModifiedRange(modified) .Sort(sortInfo) .Paging(pagingInfo) - .SetResultTransformer(new FailedMessageViewTransformer().TransformerName) + // TODO: Fix SetResultTransformer + //.SetResultTransformer(new FailedMessageViewTransformer().TransformerName) .SelectFields() .ToListAsync(); @@ -405,7 +407,8 @@ PagingInfo pagingInfo .FilterByLastModifiedRange(modified) .Sort(sortInfo) .Paging(pagingInfo) - .SetResultTransformer(FailedMessageViewTransformer.Name) + // TODO: Fix SetResultTransformer + //.SetResultTransformer(FailedMessageViewTransformer.Name) .SelectFields() .ToListAsync(); @@ -482,7 +485,8 @@ public async Task ProcessPendingRetries(DateTime periodFrom, DateTime periodTo, } var query = prequery - .SetResultTransformer(new FailedMessageViewTransformer().TransformerName) + // TODO: Fix SetResultTransformer + //.SetResultTransformer(new FailedMessageViewTransformer().TransformerName) .SelectFields(); using (var ie = await session.Advanced.StreamAsync(query)) @@ -606,7 +610,8 @@ public async Task GetRetryPendingMessages(DateTime from, DateTime to, .WhereBetween(options => options.LastModified, from.Ticks, to.Ticks) .AndAlso() .WhereEquals(o => o.QueueAddress, queueAddress) - .SetResultTransformer(FailedMessageViewTransformer.Name) + // TODO: Fix SetResultTransformer + //.SetResultTransformer(FailedMessageViewTransformer.Name) .SelectFields(new[] { "Id" }); using (var ie = await session.Advanced.StreamAsync(query)) diff --git a/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs b/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs index d4c37a9d61..732bc65c9b 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs @@ -1,49 +1,49 @@ -namespace ServiceControl.MessageFailures.Api -{ - using System; - using System.Linq; +//namespace ServiceControl.MessageFailures.Api +//{ +// using System; +// using System.Linq; - class FailedMessageViewTransformer //: AbstractTransformerCreationTask // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers - { - public FailedMessageViewTransformer() - { - //TransformResults = failures => from failure in failures - // let rec = failure.ProcessingAttempts.Last() - // let edited = rec.Headers["ServiceControl.EditOf"] != null - // select new - // { - // Id = failure.UniqueMessageId, - // MessageType = rec.MessageMetadata["MessageType"], - // IsSystemMessage = (bool)rec.MessageMetadata["IsSystemMessage"], - // SendingEndpoint = rec.MessageMetadata["SendingEndpoint"], - // ReceivingEndpoint = rec.MessageMetadata["ReceivingEndpoint"], - // TimeSent = (DateTime?)rec.MessageMetadata["TimeSent"], - // MessageId = rec.MessageMetadata["MessageId"], - // rec.FailureDetails.Exception, - // QueueAddress = rec.FailureDetails.AddressOfFailingEndpoint, - // NumberOfProcessingAttempts = failure.ProcessingAttempts.Count, - // failure.Status, - // rec.FailureDetails.TimeOfFailure, - // LastModified = MetadataFor(failure)["@last-modified"].Value(), - // Edited = edited, - // EditOf = edited ? rec.Headers["ServiceControl.EditOf"] : "" - // }; - } +// class FailedMessageViewTransformer //: AbstractTransformerCreationTask // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers +// { +// public FailedMessageViewTransformer() +// { +// //TransformResults = failures => from failure in failures +// // let rec = failure.ProcessingAttempts.Last() +// // let edited = rec.Headers["ServiceControl.EditOf"] != null +// // select new +// // { +// // Id = failure.UniqueMessageId, +// // MessageType = rec.MessageMetadata["MessageType"], +// // IsSystemMessage = (bool)rec.MessageMetadata["IsSystemMessage"], +// // SendingEndpoint = rec.MessageMetadata["SendingEndpoint"], +// // ReceivingEndpoint = rec.MessageMetadata["ReceivingEndpoint"], +// // TimeSent = (DateTime?)rec.MessageMetadata["TimeSent"], +// // MessageId = rec.MessageMetadata["MessageId"], +// // rec.FailureDetails.Exception, +// // QueueAddress = rec.FailureDetails.AddressOfFailingEndpoint, +// // NumberOfProcessingAttempts = failure.ProcessingAttempts.Count, +// // failure.Status, +// // rec.FailureDetails.TimeOfFailure, +// // LastModified = MetadataFor(failure)["@last-modified"].Value(), +// // Edited = edited, +// // EditOf = edited ? rec.Headers["ServiceControl.EditOf"] : "" +// // }; +// } - public static string Name - { - get - { - if (transformerName == null) - { - return string.Empty; - //transformerName = new FailedMessageViewTransformer().TransformerName; - } +// public static string Name +// { +// get +// { +// if (transformerName == null) +// { +// return string.Empty; +// //transformerName = new FailedMessageViewTransformer().TransformerName; +// } - return transformerName; - } - } +// return transformerName; +// } +// } - static string transformerName; - } -} \ No newline at end of file +// static string transformerName; +// } +//} \ No newline at end of file From 2fcc3d7af29e4f2b80a1ec702c43550167fef2dc Mon Sep 17 00:00:00 2001 From: David Boike Date: Tue, 12 Sep 2023 13:54:03 -0500 Subject: [PATCH 60/78] Compile is green --- .../ErrorMessagesDataStore.cs | 106 ++++++++++++------ .../UnArchiveMessagesByRangeHandler.cs | 2 - 2 files changed, 71 insertions(+), 37 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index 10d824f09d..7c5202c35b 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -2,14 +2,15 @@ { using System; using System.Collections.Generic; - using System.Globalization; using System.Linq; using System.Threading.Tasks; using CompositeViews.Messages; using Editing; using NServiceBus.Logging; using Raven.Client.Documents; + using Raven.Client.Documents.Commands; using Raven.Client.Documents.Linq; + using Raven.Client.Documents.Operations; using Raven.Client.Documents.Queries; using Raven.Client.Documents.Queries.Facets; using Raven.Client.Documents.Session; @@ -489,7 +490,7 @@ public async Task ProcessPendingRetries(DateTime periodFrom, DateTime periodTo, //.SetResultTransformer(new FailedMessageViewTransformer().TransformerName) .SelectFields(); - using (var ie = await session.Advanced.StreamAsync(query)) + await using (var ie = await session.Advanced.StreamAsync(query)) { while (await ie.MoveNextAsync()) { @@ -506,38 +507,72 @@ class DocumentPatchResult public async Task<(string[] ids, int count)> UnArchiveMessagesByRange(DateTime from, DateTime to, DateTime cutOff) { - var options = new BulkOperationOptions + // TODO: Make sure this new implementation actually works, not going to delete the old implementation (commented below) until then + var patch = new PatchByQueryOperation(new IndexQuery { - AllowStale = true - }; - - var result = await documentStore.AsyncDatabaseCommands.UpdateByIndexAsync( - new FailedMessageViewIndex().IndexName, - new IndexQuery - { - Query = string.Format(CultureInfo.InvariantCulture, "LastModified:[{0} TO {1}] AND Status:{2}", from.Ticks, to.Ticks, (int)FailedMessageStatus.Archived), - Cutoff = cutOff - }, new ScriptedPatchRequest + Query = $@"from index '{new FailedMessageViewIndex().IndexName} as msg + where msg.LastModified >= args.From and msg.LastModified <= args.To + where msg.Status == args.Archived + update + {{ + msg.Status = args.Unresolved + }}", + QueryParameters = { - Script = @" -if(this.Status === archivedStatus) { - this.Status = unresolvedStatus; -} -", - Values = - { - {"archivedStatus", (int)FailedMessageStatus.Archived}, - {"unresolvedStatus", (int)FailedMessageStatus.Unresolved} - } - }, options); - - var patchedDocumentIds = (await result.WaitForCompletionAsync()) - .JsonDeserialization(); - - return ( - patchedDocumentIds.Select(x => FailedMessageIdGenerator.GetMessageIdFromDocumentId(x.Document)).ToArray(), - patchedDocumentIds.Length - ); + { "From", from }, + { "To", to }, + { "Unresolved", (int)FailedMessageStatus.Unresolved }, + { "Archived", (int)FailedMessageStatus.Archived }, + } + }, new QueryOperationOptions + { + AllowStale = true, + RetrieveDetails = true + }); + + var operation = await documentStore.Operations.SendAsync(patch); + + var result = await operation.WaitForCompletionAsync(); + + var ids = result.Details.OfType() + .Select(d => d.Id) + .ToArray(); + + // TODO: Are we *really* returning an array AND the length of the same array? + return (ids, ids.Length); + + // var options = new BulkOperationOptions + // { + // AllowStale = true + // }; + + // var result = await documentStore.AsyncDatabaseCommands.UpdateByIndexAsync( + // new FailedMessageViewIndex().IndexName, + // new IndexQuery + // { + // Query = string.Format(CultureInfo.InvariantCulture, "LastModified:[{0} TO {1}] AND Status:{2}", from.Ticks, to.Ticks, (int)FailedMessageStatus.Archived), + // Cutoff = cutOff + // }, new ScriptedPatchRequest + // { + // Script = @" + //if(this.Status === archivedStatus) { + // this.Status = unresolvedStatus; + //} + //", + // Values = + // { + // {"archivedStatus", (int)FailedMessageStatus.Archived}, + // {"unresolvedStatus", (int)FailedMessageStatus.Unresolved} + // } + // }, options); + + // var patchedDocumentIds = (await result.WaitForCompletionAsync()) + // .JsonDeserialization(); + + // return ( + // patchedDocumentIds.Select(x => FailedMessageIdGenerator.GetMessageIdFromDocumentId(x.Document)).ToArray(), + // patchedDocumentIds.Length + // ); } public async Task<(string[] ids, int count)> UnArchiveMessages(IEnumerable failedMessageIds) @@ -591,9 +626,10 @@ public async Task RevertRetry(string messageUniqueId) } } - public Task RemoveFailedMessageRetryDocument(string uniqueMessageId) + public async Task RemoveFailedMessageRetryDocument(string uniqueMessageId) { - return documentStore.AsyncDatabaseCommands.DeleteAsync(FailedMessageRetry.MakeDocumentId(uniqueMessageId), null); + using var session = documentStore.OpenAsyncSession(); + await session.Advanced.RequestExecutor.ExecuteAsync(new DeleteDocumentCommand(FailedMessageRetry.MakeDocumentId(uniqueMessageId), null), session.Advanced.Context); } // TODO: Once using .NET, consider using IAsyncEnumerable here as this is an unbounded query @@ -614,7 +650,7 @@ public async Task GetRetryPendingMessages(DateTime from, DateTime to, //.SetResultTransformer(FailedMessageViewTransformer.Name) .SelectFields(new[] { "Id" }); - using (var ie = await session.Advanced.StreamAsync(query)) + await using (var ie = await session.Advanced.StreamAsync(query)) { while (await ie.MoveNextAsync()) { diff --git a/src/ServiceControl/MessageFailures/Handlers/UnArchiveMessagesByRangeHandler.cs b/src/ServiceControl/MessageFailures/Handlers/UnArchiveMessagesByRangeHandler.cs index 7c98893fb7..b581e53037 100644 --- a/src/ServiceControl/MessageFailures/Handlers/UnArchiveMessagesByRangeHandler.cs +++ b/src/ServiceControl/MessageFailures/Handlers/UnArchiveMessagesByRangeHandler.cs @@ -1,7 +1,5 @@ namespace ServiceControl.MessageFailures.Handlers { - using System.Collections.Generic; - using System.Linq; using System.Threading.Tasks; using Contracts.MessageFailures; using Infrastructure.DomainEvents; From ad06ef1a15771fbddb03632ac26c5a73a0de7832 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 13 Sep 2023 17:10:09 +0200 Subject: [PATCH 61/78] =?UTF-8?q?=F0=9F=A9=B9Build=20order=20dependencies?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/ServiceControl.sln | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ServiceControl.sln b/src/ServiceControl.sln index b4779e2aa7..f62315c113 100644 --- a/src/ServiceControl.sln +++ b/src/ServiceControl.sln @@ -5,6 +5,9 @@ MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ServiceControl.UnitTests", "ServiceControl.UnitTests\ServiceControl.UnitTests.csproj", "{4F65FAE1-4C51-4BEF-956E-97C6A96807F9}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ServiceControl", "ServiceControl\ServiceControl.csproj", "{4C6B71B2-74EF-4B9E-88FF-C56532727C6D}" + ProjectSection(ProjectDependencies) = postProject + {84627DC0-7B43-480D-80CF-A4DDA514A4EE} = {84627DC0-7B43-480D-80CF-A4DDA514A4EE} + EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Installer", "Installer", "{55C388DD-2B39-4C2F-AEBD-AFD3444815F1}" EndProject From c9a9295c4b638bb38f61b2fd9e3b92b5952185f3 Mon Sep 17 00:00:00 2001 From: David Boike Date: Wed, 13 Sep 2023 14:24:47 -0500 Subject: [PATCH 62/78] *This* is how Raven5 persister gets built before ServiceControl core --- src/Persisters.Primary.Includes.props | 1 + src/ServiceControl.sln | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Persisters.Primary.Includes.props b/src/Persisters.Primary.Includes.props index 8b2eb0241d..ddc16fb034 100644 --- a/src/Persisters.Primary.Includes.props +++ b/src/Persisters.Primary.Includes.props @@ -1,5 +1,6 @@ + \ No newline at end of file diff --git a/src/ServiceControl.sln b/src/ServiceControl.sln index f62315c113..b4779e2aa7 100644 --- a/src/ServiceControl.sln +++ b/src/ServiceControl.sln @@ -5,9 +5,6 @@ MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ServiceControl.UnitTests", "ServiceControl.UnitTests\ServiceControl.UnitTests.csproj", "{4F65FAE1-4C51-4BEF-956E-97C6A96807F9}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ServiceControl", "ServiceControl\ServiceControl.csproj", "{4C6B71B2-74EF-4B9E-88FF-C56532727C6D}" - ProjectSection(ProjectDependencies) = postProject - {84627DC0-7B43-480D-80CF-A4DDA514A4EE} = {84627DC0-7B43-480D-80CF-A4DDA514A4EE} - EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Installer", "Installer", "{55C388DD-2B39-4C2F-AEBD-AFD3444815F1}" EndProject From c1242d0376397c52ebb61a8e7731fef5d586a00d Mon Sep 17 00:00:00 2001 From: David Boike Date: Wed, 13 Sep 2023 15:07:40 -0500 Subject: [PATCH 63/78] Fix the double-executed lifecycle creation --- .../RavenDbEmbeddedPersistenceLifecycle.cs | 1 - src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs | 2 -- 2 files changed, 3 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbEmbeddedPersistenceLifecycle.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbEmbeddedPersistenceLifecycle.cs index 1e40c3a5f3..7df1fa9489 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbEmbeddedPersistenceLifecycle.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbEmbeddedPersistenceLifecycle.cs @@ -3,7 +3,6 @@ using System; using System.Threading; using System.Threading.Tasks; - using Microsoft.Extensions.Hosting; using Raven.Client.Documents; using ServiceControl.Persistence; diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs index ecae7dbd10..55d5fc5527 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs @@ -31,8 +31,6 @@ public void Configure(IServiceCollection serviceCollection) serviceCollection.AddSingleton(settings); - ConfigureLifecycle(serviceCollection); - serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); From 9ec6376b1cace1ff81acbaa68cc0fec1215ad32f Mon Sep 17 00:00:00 2001 From: David Boike Date: Wed, 13 Sep 2023 15:08:02 -0500 Subject: [PATCH 64/78] App.config change to run primary using Raven5 --- src/ServiceControl/App.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ServiceControl/App.config b/src/ServiceControl/App.config index 0f471c0fbc..3305284eca 100644 --- a/src/ServiceControl/App.config +++ b/src/ServiceControl/App.config @@ -23,8 +23,8 @@ These settings are only here so that we can debug ServiceControl while developin --> - - + + From 92baa388a4a40221c561ab1dafb5929f59c9da3a Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 14 Sep 2023 09:56:00 +0200 Subject: [PATCH 65/78] =?UTF-8?q?=F0=9F=A9=B9=20Resolved=20`PersistenceLif?= =?UTF-8?q?ecycleHostedService`=20via=20DI=20to=20ensure=20lifecycle=20is?= =?UTF-8?q?=20resolved=20via=20DI=20only=20once?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Persistence/PersistenceHostBuilderExtensions.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ServiceControl.Audit/Persistence/PersistenceHostBuilderExtensions.cs b/src/ServiceControl.Audit/Persistence/PersistenceHostBuilderExtensions.cs index ca546ea612..a9e468fcac 100644 --- a/src/ServiceControl.Audit/Persistence/PersistenceHostBuilderExtensions.cs +++ b/src/ServiceControl.Audit/Persistence/PersistenceHostBuilderExtensions.cs @@ -15,7 +15,8 @@ public static IHostBuilder SetupPersistence(this IHostBuilder hostBuilder, { var lifecycle = persistence.Configure(serviceCollection); - serviceCollection.AddHostedService(_ => new PersistenceLifecycleHostedService(lifecycle)); + serviceCollection.AddSingleton(new PersistenceLifecycleHostedService(lifecycle)); + serviceCollection.AddHostedService(sp => sp.GetRequiredService()); }); return hostBuilder; From e88f9ae9b26807dae5f962cb82068823f7d4a67c Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 14 Sep 2023 10:01:04 +0200 Subject: [PATCH 66/78] =?UTF-8?q?=F0=9F=A9=B9=20Fix=20RavenDBPersisterSett?= =?UTF-8?q?ings.LogsMode=20init?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs | 1 + .../RavenDBPersisterSettings.cs | 3 ++- .../RavenDbPersistenceConfiguration.cs | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs b/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs index c4f721cae5..ef018fa666 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs @@ -11,5 +11,6 @@ static class RavenBootstrapper public const string ConnectionStringKey = "RavenDB5/ConnectionString"; public const string MinimumStorageLeftRequiredForIngestionKey = "MinimumStorageLeftRequiredForIngestion"; public const string DatabaseNameKey = "RavenDB5/DatabaseName"; + public const string LogsModeKey = "LogMode"; } } diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs index 7a2fbfbf13..e4e771f93b 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs @@ -22,10 +22,11 @@ class RavenDBPersisterSettings : PersistenceSettings public string ConnectionString { get; set; } public bool UseEmbeddedServer => !string.IsNullOrWhiteSpace(ConnectionString); public string LogPath { get; set; } - public string LogsMode { get; set; } + public string LogsMode { get; set; } = LogsModeDefault; public string DatabaseName { get; set; } = DatabaseNameDefault; public const string DatabaseNameDefault = "audit"; public const int DatabaseMaintenancePortDefault = 33334; public const int ExpirationProcessTimerInSecondsDefault = 600; + public const string LogsModeDefault = "Information"; } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs index 10e73dd5d7..2698c551c5 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs @@ -57,6 +57,7 @@ T GetSetting(string key, T defaultValue) AuditRetentionPeriod = GetSetting(AuditRetentionPeriodKey, TimeSpan.Zero), ExternalIntegrationsDispatchingBatchSize = GetSetting(ExternalIntegrationsDispatchingBatchSizeKey, 100), MaintenanceMode = GetSetting(MaintenanceModeKey, false), + LogsMode = GetSetting(RavenBootstrapper.LogsModeKey, RavenDBPersisterSettings.LogsModeDefault) }; CheckFreeDiskSpace.Validate(settings); From 1af53ad5557242ebe4c888edca59a178dab12a6f Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 14 Sep 2023 10:01:22 +0200 Subject: [PATCH 67/78] =?UTF-8?q?=F0=9F=A9=B9=20Fix=20LogPath=20init?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs | 1 + .../RavenDbPersistenceConfiguration.cs | 1 + 2 files changed, 2 insertions(+) diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs b/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs index ef018fa666..20cdd35b8a 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs @@ -11,6 +11,7 @@ static class RavenBootstrapper public const string ConnectionStringKey = "RavenDB5/ConnectionString"; public const string MinimumStorageLeftRequiredForIngestionKey = "MinimumStorageLeftRequiredForIngestion"; public const string DatabaseNameKey = "RavenDB5/DatabaseName"; + public const string LogsPathKey = "LogPath"; public const string LogsModeKey = "LogMode"; } } diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs index 2698c551c5..c7a0f2d8fe 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs @@ -57,6 +57,7 @@ T GetSetting(string key, T defaultValue) AuditRetentionPeriod = GetSetting(AuditRetentionPeriodKey, TimeSpan.Zero), ExternalIntegrationsDispatchingBatchSize = GetSetting(ExternalIntegrationsDispatchingBatchSizeKey, 100), MaintenanceMode = GetSetting(MaintenanceModeKey, false), + LogPath = GetRequiredSetting(RavenBootstrapper.LogsPathKey), LogsMode = GetSetting(RavenBootstrapper.LogsModeKey, RavenDBPersisterSettings.LogsModeDefault) }; From b08ff5f7de2eb95b42c05704cf578135f86fa6b5 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 14 Sep 2023 10:01:55 +0200 Subject: [PATCH 68/78] =?UTF-8?q?=F0=9F=A9=B9=20Fix=20inverted=20UseEmbedd?= =?UTF-8?q?edServer=20init=20issue?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../RavenDBPersisterSettings.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs index e4e771f93b..a23954884b 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs @@ -20,7 +20,7 @@ class RavenDBPersisterSettings : PersistenceSettings //TODO: these are newly added settings, we should remove any duplication public string ServerUrl { get; set; } public string ConnectionString { get; set; } - public bool UseEmbeddedServer => !string.IsNullOrWhiteSpace(ConnectionString); + public bool UseEmbeddedServer => string.IsNullOrWhiteSpace(ConnectionString); public string LogPath { get; set; } public string LogsMode { get; set; } = LogsModeDefault; public string DatabaseName { get; set; } = DatabaseNameDefault; From 01356ad10ff2f61904684fff101563033622b4da Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 14 Sep 2023 10:08:01 +0200 Subject: [PATCH 69/78] =?UTF-8?q?=F0=9F=A9=B9=20Fix=20dubbble=20RavenDbEmb?= =?UTF-8?q?eddedPersistenceLifecycle=20DI=20=20instance=20creation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../RavenDbPersistence.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs index 55d5fc5527..58bf61e6ee 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs @@ -75,8 +75,10 @@ public void ConfigureLifecycle(IServiceCollection serviceCollection) { if (settings.UseEmbeddedServer) { - serviceCollection.AddSingleton(); - serviceCollection.AddSingleton(_ => new RavenDbEmbeddedPersistenceLifecycle(settings).GetDocumentStore()); + var embedded = new RavenDbEmbeddedPersistenceLifecycle(settings); + + serviceCollection.AddSingleton(embedded); + serviceCollection.AddSingleton(_ => embedded.GetDocumentStore()); return; } From 7c4a37e05ac2b175870267a1cbb3d8e6c6a3230f Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 14 Sep 2023 13:55:32 +0200 Subject: [PATCH 70/78] =?UTF-8?q?=F0=9F=A9=B9=20Resolves:=20Cannot=20set?= =?UTF-8?q?=20identity=20value=20'CustomChecks/193-A'=20on=20property=20'I?= =?UTF-8?q?d'=20for=20type=20'ServiceControl.Contracts.CustomChecks.Custom?= =?UTF-8?q?Check'=20because=20property=20type=20is=20not=20a=20string?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../RavenDbCustomCheckDataStore.cs | 2 +- .../RavenDbCustomCheckDataStore.cs | 4 ++-- .../CustomChecksDataStoreTests.cs | 2 +- src/ServiceControl.Persistence/CustomCheck.cs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb/RavenDbCustomCheckDataStore.cs b/src/ServiceControl.Persistence.RavenDb/RavenDbCustomCheckDataStore.cs index 483e3b3f01..b765cd04a0 100644 --- a/src/ServiceControl.Persistence.RavenDb/RavenDbCustomCheckDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb/RavenDbCustomCheckDataStore.cs @@ -20,7 +20,7 @@ public RavenDbCustomCheckDataStore(IDocumentStore store) public async Task UpdateCustomCheckStatus(CustomCheckDetail detail) { var status = CheckStateChange.Unchanged; - var id = detail.GetDeterministicId(); + var id = detail.GetDeterministicId().ToString(); using (var session = store.OpenAsyncSession()) { diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs index f7dc82eb6c..b678f614fa 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs @@ -22,11 +22,11 @@ public RavenDbCustomCheckDataStore(IDocumentStore store) public async Task UpdateCustomCheckStatus(CustomCheckDetail detail) { var status = CheckStateChange.Unchanged; - var id = detail.GetDeterministicId(); + var id = detail.GetDeterministicId().ToString(); using (var session = store.OpenAsyncSession()) { - var customCheck = await session.LoadAsync(id.ToString()); + var customCheck = await session.LoadAsync(id); if (customCheck == null || (customCheck.Status == Status.Fail && !detail.HasFailed) || diff --git a/src/ServiceControl.Persistence.Tests/CustomChecksDataStoreTests.cs b/src/ServiceControl.Persistence.Tests/CustomChecksDataStoreTests.cs index d35058f7e7..589dfa591d 100644 --- a/src/ServiceControl.Persistence.Tests/CustomChecksDataStoreTests.cs +++ b/src/ServiceControl.Persistence.Tests/CustomChecksDataStoreTests.cs @@ -117,7 +117,7 @@ public async Task Should_delete_custom_checks() await CompleteDatabaseOperation(); var storedChecks = await CustomChecks.GetStats(new PagingInfo()); - var check = storedChecks.Results.Where(c => c.Id == checkId).ToList(); + var check = storedChecks.Results.Where(c => c.Id == checkId.ToString()).ToList(); Assert.AreEqual(0, check.Count); } diff --git a/src/ServiceControl.Persistence/CustomCheck.cs b/src/ServiceControl.Persistence/CustomCheck.cs index e0ee96eed3..584598a68f 100644 --- a/src/ServiceControl.Persistence/CustomCheck.cs +++ b/src/ServiceControl.Persistence/CustomCheck.cs @@ -6,7 +6,7 @@ public class CustomCheck { - public Guid Id { get; set; } + public string Id { get; set; } public string CustomCheckId { get; set; } public string Category { get; set; } public Status Status { get; set; } From 1f3dd8a4756d698499599db424a95f2a9aa39619 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 14 Sep 2023 14:16:52 +0200 Subject: [PATCH 71/78] =?UTF-8?q?=F0=9F=A9=B9=20IndexLag=20in=20RavenDB5?= =?UTF-8?q?=20is=20TimeSpan=20which=20resulted=20in=20a=20string=20format?= =?UTF-8?q?=20exception=20as=20format=20`n0`=20can't=20be=20used=20on=20Ti?= =?UTF-8?q?meSpan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../CustomChecks/CheckRavenDBIndexLag.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexLag.cs b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexLag.cs index b45f9d21e6..36f64b99b5 100644 --- a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexLag.cs +++ b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexLag.cs @@ -48,12 +48,12 @@ static int CheckAndReportIndexesWithTooMuchIndexLag(IndexInformation[] indexes) if (indexLag > IndexLagThresholdError) { indexCountWithTooMuchLag++; - Log.Error($"Index [{indexStats.Name}] IndexingLag {indexLag:n0} is above error threshold ({IndexLagThresholdError:n0}). Launch in maintenance mode to let indexes catch up."); + Log.Error($"Index [{indexStats.Name}] IndexingLag {indexLag} is above error threshold ({IndexLagThresholdError}). Launch in maintenance mode to let indexes catch up."); } else if (indexLag > IndexLagThresholdWarning) { indexCountWithTooMuchLag++; - Log.Warn($"Index [{indexStats.Name}] IndexingLag {indexLag:n0} is above warning threshold ({IndexLagThresholdWarning:n0}). Launch in maintenance mode to let indexes catch up."); + Log.Warn($"Index [{indexStats.Name}] IndexingLag {indexLag} is above warning threshold ({IndexLagThresholdWarning}). Launch in maintenance mode to let indexes catch up."); } } } From 6681b0915c1bbf0da9ff29b51d14714b4351414a Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 14 Sep 2023 14:26:58 +0200 Subject: [PATCH 72/78] =?UTF-8?q?=F0=9F=9A=A7=20=20=20TODO:=20Ensure=20aud?= =?UTF-8?q?it=20ravendb5=20persistence=20uses=20the=20same=20index=20lag?= =?UTF-8?q?=20behavior=20based=20on=20time?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../CustomChecks/CheckRavenDBIndexLag.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexLag.cs b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexLag.cs index 36f64b99b5..a6bc6acc4b 100644 --- a/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexLag.cs +++ b/src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexLag.cs @@ -43,7 +43,7 @@ static int CheckAndReportIndexesWithTooMuchIndexLag(IndexInformation[] indexes) { if (indexStats.LastIndexingTime.HasValue) { - var indexLag = DateTime.UtcNow - indexStats.LastIndexingTime.Value; + var indexLag = DateTime.UtcNow - indexStats.LastIndexingTime.Value; // TODO: Ensure audit ravendb5 persistence uses the same index lag behavior based on time if (indexLag > IndexLagThresholdError) { From dabb15fc50ee7791416fd497b388e470a91b6fc0 Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 14 Sep 2023 10:55:58 -0500 Subject: [PATCH 73/78] Restore message queries that must return failed message info to augment Audit instance's audit data --- .../ErrorMessagesDataStore.cs | 85 +++++++++++++++---- 1 file changed, 67 insertions(+), 18 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index 7c5202c35b..9f8534d16c 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -18,7 +18,6 @@ using ServiceControl.MessageFailures; using ServiceControl.MessageFailures.Api; using ServiceControl.Operations; - using ServiceControl.Persistence; using ServiceControl.Persistence.Infrastructure; using ServiceControl.Recoverability; @@ -32,59 +31,109 @@ public ErrorMessagesDataStore(IDocumentStore documentStore) } - static Task>> EmptyQueryResult() => Task.FromResult(new QueryResult>(new List(), new QueryStatsInfo())); - - // TODO: This is for audit messages, should probably not be on IErrorMessageDataStore - public Task>> GetAllMessages( + public async Task>> GetAllMessages( PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages ) { - return EmptyQueryResult(); + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session.Query() + .IncludeSystemMessagesWhere(includeSystemMessages) + .Statistics(out var stats) + .Sort(sortInfo) + .Paging(pagingInfo) + .TransformWith() + .ToListAsync(); + + return new QueryResult>(results, stats.ToQueryStatsInfo()); + } } - // TODO: This is for audit messages, should probably not be on IErrorMessageDataStore - public Task>> GetAllMessagesForEndpoint( + public async Task>> GetAllMessagesForEndpoint( string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages ) { - return EmptyQueryResult(); + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session.Query() + .IncludeSystemMessagesWhere(includeSystemMessages) + .Where(m => m.ReceivingEndpointName == endpointName) + .Statistics(out var stats) + .Sort(sortInfo) + .Paging(pagingInfo) + .TransformWith() + .ToListAsync(); + + return new QueryResult>(results, stats.ToQueryStatsInfo()); + } } - // TODO: This is for audit messages, should probably not be on IErrorMessageDataStore - public Task>> SearchEndpointMessages( + public async Task>> SearchEndpointMessages( string endpointName, string searchKeyword, PagingInfo pagingInfo, SortInfo sortInfo ) { - return EmptyQueryResult(); + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session.Query() + .Statistics(out var stats) + .Search(x => x.Query, searchKeyword) + .Where(m => m.ReceivingEndpointName == endpointName) + .Sort(sortInfo) + .Paging(pagingInfo) + .TransformWith() + .ToListAsync(); + + return new QueryResult>(results, stats.ToQueryStatsInfo()); + } } - // TODO: This is for audit messages, should probably not be on IErrorMessageDataStore - public Task>> GetAllMessagesByConversation( + public async Task>> GetAllMessagesByConversation( string conversationId, PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages ) { - return EmptyQueryResult(); + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session.Query() + .Statistics(out var stats) + .Where(m => m.ConversationId == conversationId) + .Sort(sortInfo) + .Paging(pagingInfo) + .TransformWith() + .ToListAsync(); + + return new QueryResult>(results, stats.ToQueryStatsInfo()); + } } - // TODO: This is for audit messages, should probably not be on IErrorMessageDataStore - public Task>> GetAllMessagesForSearch( + public async Task>> GetAllMessagesForSearch( string searchTerms, PagingInfo pagingInfo, SortInfo sortInfo ) { - return EmptyQueryResult(); + using (var session = documentStore.OpenAsyncSession()) + { + var results = await session.Query() + .Statistics(out var stats) + .Search(x => x.Query, searchTerms) + .Sort(sortInfo) + .Paging(pagingInfo) + .TransformWith() + .ToListAsync(); + + return new QueryResult>(results, stats.ToQueryStatsInfo()); + } } public async Task FailedMessageFetch(string failedMessageId) From 5eb9941fb01370e78445c6b0f435944d9e8b37c0 Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 14 Sep 2023 10:56:19 -0500 Subject: [PATCH 74/78] Revert "MessagesViewTransformer not actually used" This reverts commit dc7e5fcfbbdf54c257c5388c1ece9c2af2f945f8. --- .../Transformers/MessagesViewTransformer.cs | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs diff --git a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs new file mode 100644 index 0000000000..733fca57fa --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs @@ -0,0 +1,75 @@ +namespace ServiceControl.CompositeViews.Messages +{ + using System; + using System.Collections.Generic; + using System.Linq; + using MessageFailures; + using ServiceControl.Persistence; + + class MessagesViewTransformer : AbstractTransformerCreationTask // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers + { + public MessagesViewTransformer() + { + TransformResults = messages => from message in messages + let metadata = + message.ProcessingAttempts != null + ? message.ProcessingAttempts.Last().MessageMetadata + : message.MessageMetadata + let headers = + message.ProcessingAttempts != null ? message.ProcessingAttempts.Last().Headers : message.Headers + let processedAt = + message.ProcessingAttempts != null + ? message.ProcessingAttempts.Last().AttemptedAt + : message.ProcessedAt + let status = + message.ProcessingAttempts == null + ? !(bool)message.MessageMetadata["IsRetried"] + ? MessageStatus.Successful + : MessageStatus.ResolvedSuccessfully + : message.Status == FailedMessageStatus.Resolved + ? MessageStatus.ResolvedSuccessfully + : message.Status == FailedMessageStatus.RetryIssued + ? MessageStatus.RetryIssued + : message.Status == FailedMessageStatus.Archived + ? MessageStatus.ArchivedFailure + : message.ProcessingAttempts.Count == 1 + ? MessageStatus.Failed + : MessageStatus.RepeatedFailure + select new + { + Id = message.UniqueMessageId, + MessageId = metadata["MessageId"], + MessageType = metadata["MessageType"], + SendingEndpoint = metadata["SendingEndpoint"], + ReceivingEndpoint = metadata["ReceivingEndpoint"], + TimeSent = (DateTime?)metadata["TimeSent"], + ProcessedAt = processedAt, + CriticalTime = (TimeSpan)metadata["CriticalTime"], + ProcessingTime = (TimeSpan)metadata["ProcessingTime"], + DeliveryTime = (TimeSpan)metadata["DeliveryTime"], + IsSystemMessage = (bool)metadata["IsSystemMessage"], + ConversationId = metadata["ConversationId"], + //the reason the we need to use a KeyValuePair is that raven seems to interpret the values and convert them + // to real types. In this case it was the NServiceBus.Temporary.DelayDeliveryWith header to was converted to a timespan + Headers = headers.Select(header => new KeyValuePair(header.Key, header.Value)), + Status = status, + MessageIntent = metadata["MessageIntent"], + BodyUrl = metadata["BodyUrl"], + BodySize = (int)metadata["ContentLength"], + InvokedSagas = metadata["InvokedSagas"], + OriginatesFromSaga = metadata["OriginatesFromSaga"] + }; + } + + public class Input + { + public string Id { get; set; } + public string UniqueMessageId { get; set; } + public DateTime ProcessedAt { get; set; } + public Dictionary Headers { get; set; } + public Dictionary MessageMetadata { get; set; } + public List ProcessingAttempts { get; set; } + public FailedMessageStatus Status { get; set; } + } + } +} \ No newline at end of file From 73207910afe3bd3dbe53afd8e8cf73f7f990f7c8 Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 14 Sep 2023 11:16:22 -0500 Subject: [PATCH 75/78] Add enough fake code to compile transformer-related code for now --- .../ErrorMessagesDataStore.cs | 17 ++-- .../Indexes/MessagesViewIndex.cs | 2 +- .../FailedMessageViewTransformer.cs | 88 +++++++++---------- .../Transformers/MessagesViewTransformer.cs | 22 ++++- 4 files changed, 74 insertions(+), 55 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index 9f8534d16c..19db4f2f6b 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -4,7 +4,6 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; - using CompositeViews.Messages; using Editing; using NServiceBus.Logging; using Raven.Client.Documents; @@ -14,6 +13,7 @@ using Raven.Client.Documents.Queries; using Raven.Client.Documents.Queries.Facets; using Raven.Client.Documents.Session; + using ServiceControl.CompositeViews.Messages; using ServiceControl.EventLog; using ServiceControl.MessageFailures; using ServiceControl.MessageFailures.Api; @@ -44,7 +44,8 @@ bool includeSystemMessages .Statistics(out var stats) .Sort(sortInfo) .Paging(pagingInfo) - .TransformWith() + //.TransformWith() + .TransformToMessagesView() .ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); @@ -66,7 +67,8 @@ bool includeSystemMessages .Statistics(out var stats) .Sort(sortInfo) .Paging(pagingInfo) - .TransformWith() + //.TransformWith() + .TransformToMessagesView() .ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); @@ -88,7 +90,8 @@ SortInfo sortInfo .Where(m => m.ReceivingEndpointName == endpointName) .Sort(sortInfo) .Paging(pagingInfo) - .TransformWith() + //.TransformWith() + .TransformToMessagesView() .ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); @@ -109,7 +112,8 @@ bool includeSystemMessages .Where(m => m.ConversationId == conversationId) .Sort(sortInfo) .Paging(pagingInfo) - .TransformWith() + //.TransformWith() + .TransformToMessagesView() .ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); @@ -129,7 +133,8 @@ SortInfo sortInfo .Search(x => x.Query, searchTerms) .Sort(sortInfo) .Paging(pagingInfo) - .TransformWith() + //.TransformWith() + .TransformToMessagesView() .ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/MessagesViewIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/MessagesViewIndex.cs index 9ac54b009d..73f5401e2a 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/MessagesViewIndex.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/MessagesViewIndex.cs @@ -8,7 +8,7 @@ namespace ServiceControl.Persistence using ServiceControl.Operations; // TODO: Consider renaming to FailedMessageIndex - class MessagesViewIndex : AbstractIndexCreationTask + public class MessagesViewIndex : AbstractIndexCreationTask { public MessagesViewIndex() { diff --git a/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs b/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs index 732bc65c9b..77c425d968 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs @@ -1,49 +1,45 @@ -//namespace ServiceControl.MessageFailures.Api -//{ -// using System; -// using System.Linq; +namespace ServiceControl.MessageFailures.Api +{ + using System; + using System.Linq; + using ServiceControl.CompositeViews.Messages; -// class FailedMessageViewTransformer //: AbstractTransformerCreationTask // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers -// { -// public FailedMessageViewTransformer() -// { -// //TransformResults = failures => from failure in failures -// // let rec = failure.ProcessingAttempts.Last() -// // let edited = rec.Headers["ServiceControl.EditOf"] != null -// // select new -// // { -// // Id = failure.UniqueMessageId, -// // MessageType = rec.MessageMetadata["MessageType"], -// // IsSystemMessage = (bool)rec.MessageMetadata["IsSystemMessage"], -// // SendingEndpoint = rec.MessageMetadata["SendingEndpoint"], -// // ReceivingEndpoint = rec.MessageMetadata["ReceivingEndpoint"], -// // TimeSent = (DateTime?)rec.MessageMetadata["TimeSent"], -// // MessageId = rec.MessageMetadata["MessageId"], -// // rec.FailureDetails.Exception, -// // QueueAddress = rec.FailureDetails.AddressOfFailingEndpoint, -// // NumberOfProcessingAttempts = failure.ProcessingAttempts.Count, -// // failure.Status, -// // rec.FailureDetails.TimeOfFailure, -// // LastModified = MetadataFor(failure)["@last-modified"].Value(), -// // Edited = edited, -// // EditOf = edited ? rec.Headers["ServiceControl.EditOf"] : "" -// // }; -// } + class FailedMessageViewTransformer : FakeAbstractTransformerCreationTask // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers + { + public FailedMessageViewTransformer() + { + TransformResults = failures => from failure in failures + let rec = failure.ProcessingAttempts.Last() + let edited = rec.Headers["ServiceControl.EditOf"] != null + select new + { + Id = failure.UniqueMessageId, + MessageType = rec.MessageMetadata["MessageType"], + IsSystemMessage = (bool)rec.MessageMetadata["IsSystemMessage"], + SendingEndpoint = rec.MessageMetadata["SendingEndpoint"], + ReceivingEndpoint = rec.MessageMetadata["ReceivingEndpoint"], + TimeSent = (DateTime?)rec.MessageMetadata["TimeSent"], + MessageId = rec.MessageMetadata["MessageId"], + rec.FailureDetails.Exception, + QueueAddress = rec.FailureDetails.AddressOfFailingEndpoint, + NumberOfProcessingAttempts = failure.ProcessingAttempts.Count, + failure.Status, + rec.FailureDetails.TimeOfFailure, + //LastModified = MetadataFor(failure)["@last-modified"].Value(), + Edited = edited, + EditOf = edited ? rec.Headers["ServiceControl.EditOf"] : "" + }; + } -// public static string Name -// { -// get -// { -// if (transformerName == null) -// { -// return string.Empty; -// //transformerName = new FailedMessageViewTransformer().TransformerName; -// } + public static string Name + { + get + { + transformerName ??= new FailedMessageViewTransformer().TransformerName; + return transformerName; + } + } -// return transformerName; -// } -// } - -// static string transformerName; -// } -//} \ No newline at end of file + static string transformerName; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs index 733fca57fa..4f4f58fc94 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs @@ -1,12 +1,21 @@ namespace ServiceControl.CompositeViews.Messages { using System; + using System.Collections; using System.Collections.Generic; using System.Linq; + using System.Linq.Expressions; using MessageFailures; + using Raven.Client.Documents.Linq; using ServiceControl.Persistence; - class MessagesViewTransformer : AbstractTransformerCreationTask // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers + class FakeAbstractTransformerCreationTask + { + public Expression, IEnumerable>> TransformResults; + public string TransformerName; + } + + class MessagesViewTransformer : FakeAbstractTransformerCreationTask // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers { public MessagesViewTransformer() { @@ -72,4 +81,13 @@ public class Input public FailedMessageStatus Status { get; set; } } } -} \ No newline at end of file + + public static class MessageViewTransformerExtension + { + public static IQueryable TransformToMessagesView(this IQueryable source) + { + return source.Select(m => new MessagesView()); + } + } +} + From 1199961efa723444bc4102ca4e57eee98d112152 Mon Sep 17 00:00:00 2001 From: David Boike Date: Fri, 15 Sep 2023 16:25:37 -0500 Subject: [PATCH 76/78] Fix custom check datastore --- .../RavenDbCustomCheckDataStore.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb/RavenDbCustomCheckDataStore.cs b/src/ServiceControl.Persistence.RavenDb/RavenDbCustomCheckDataStore.cs index b765cd04a0..7cb7c02612 100644 --- a/src/ServiceControl.Persistence.RavenDb/RavenDbCustomCheckDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb/RavenDbCustomCheckDataStore.cs @@ -20,7 +20,7 @@ public RavenDbCustomCheckDataStore(IDocumentStore store) public async Task UpdateCustomCheckStatus(CustomCheckDetail detail) { var status = CheckStateChange.Unchanged; - var id = detail.GetDeterministicId().ToString(); + var id = detail.GetDeterministicId(); using (var session = store.OpenAsyncSession()) { @@ -34,7 +34,7 @@ public async Task UpdateCustomCheckStatus(CustomCheckDetail de { customCheck = new CustomCheck { - Id = id + Id = MakeId(id) }; } @@ -54,6 +54,11 @@ public async Task UpdateCustomCheckStatus(CustomCheckDetail de return status; } + static string MakeId(Guid id) + { + return $"CustomChecks/{id}"; + } + public async Task>> GetStats(PagingInfo paging, string status = null) { using (var session = store.OpenAsyncSession()) @@ -73,7 +78,7 @@ public async Task>> GetStats(PagingInfo paging, s public async Task DeleteCustomCheck(Guid id) { - await store.AsyncDatabaseCommands.DeleteAsync(store.Conventions.DefaultFindFullDocumentKeyFromNonStringIdentifier(id, typeof(CustomCheck), false), null); + await store.AsyncDatabaseCommands.DeleteAsync(MakeId(id), null); } public async Task GetNumberOfFailedChecks() From d7d475477ae132e259cf373669c82585d64488f7 Mon Sep 17 00:00:00 2001 From: David Boike Date: Fri, 15 Sep 2023 16:28:33 -0500 Subject: [PATCH 77/78] Same custom check fix for Raven5 --- .../RavenDbCustomCheckDataStore.cs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs index b678f614fa..2ab04a5ed7 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs @@ -3,14 +3,14 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; - using ServiceControl.Contracts.CustomChecks; - using ServiceControl.Persistence; - using ServiceControl.Persistence.Infrastructure; using Raven.Client.Documents; using Raven.Client.Documents.Commands; using Raven.Client.Documents.Linq; using Raven.Client.Documents.Session; + using ServiceControl.Contracts.CustomChecks; using ServiceControl.Infrastructure.RavenDB; + using ServiceControl.Persistence; + using ServiceControl.Persistence.Infrastructure; class RavenDbCustomCheckDataStore : ICustomChecksDataStore { @@ -22,7 +22,7 @@ public RavenDbCustomCheckDataStore(IDocumentStore store) public async Task UpdateCustomCheckStatus(CustomCheckDetail detail) { var status = CheckStateChange.Unchanged; - var id = detail.GetDeterministicId().ToString(); + var id = detail.GetDeterministicId(); using (var session = store.OpenAsyncSession()) { @@ -32,7 +32,7 @@ public async Task UpdateCustomCheckStatus(CustomCheckDetail de (customCheck.Status == Status.Fail && !detail.HasFailed) || (customCheck.Status == Status.Pass && detail.HasFailed)) { - customCheck ??= new CustomCheck { Id = id }; + customCheck ??= new CustomCheck { Id = MakeId(id) }; status = CheckStateChange.Changed; } @@ -50,6 +50,11 @@ public async Task UpdateCustomCheckStatus(CustomCheckDetail de return status; } + static string MakeId(Guid id) + { + return $"CustomChecks/{id}"; + } + public async Task>> GetStats(PagingInfo paging, string status = null) { using var session = store.OpenAsyncSession(); @@ -67,7 +72,7 @@ public async Task>> GetStats(PagingInfo paging, s public async Task DeleteCustomCheck(Guid id) { - var documentId = store.Conventions.DefaultFindFullDocumentKeyFromNonStringIdentifier(id, typeof(CustomCheck), false); + var documentId = MakeId(id); using var session = store.OpenAsyncSession(new SessionOptions { NoTracking = true, NoCaching = true }); await session.Advanced.RequestExecutor.ExecuteAsync(new DeleteDocumentCommand(documentId, null), session.Advanced.Context); } From fdceb55fe585623adfc1d6d7120af5824654c0dd Mon Sep 17 00:00:00 2001 From: David Boike Date: Fri, 15 Sep 2023 16:53:17 -0500 Subject: [PATCH 78/78] oops --- .../RavenDbCustomCheckDataStore.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs index 2ab04a5ed7..d94e267f04 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbCustomCheckDataStore.cs @@ -8,7 +8,6 @@ using Raven.Client.Documents.Linq; using Raven.Client.Documents.Session; using ServiceControl.Contracts.CustomChecks; - using ServiceControl.Infrastructure.RavenDB; using ServiceControl.Persistence; using ServiceControl.Persistence.Infrastructure; @@ -22,7 +21,7 @@ public RavenDbCustomCheckDataStore(IDocumentStore store) public async Task UpdateCustomCheckStatus(CustomCheckDetail detail) { var status = CheckStateChange.Unchanged; - var id = detail.GetDeterministicId(); + var id = MakeId(detail.GetDeterministicId()); using (var session = store.OpenAsyncSession()) { @@ -32,7 +31,7 @@ public async Task UpdateCustomCheckStatus(CustomCheckDetail de (customCheck.Status == Status.Fail && !detail.HasFailed) || (customCheck.Status == Status.Pass && detail.HasFailed)) { - customCheck ??= new CustomCheck { Id = MakeId(id) }; + customCheck ??= new CustomCheck { Id = id }; status = CheckStateChange.Changed; }