From 8b1967b4470696ec22c99f9bc2fe8c3e099a089c Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 14 Sep 2023 15:34:43 -0500 Subject: [PATCH 01/94] Duplicate persistence tests and add to solution --- .../.editorconfig | 4 + .../API/APIApprovals.cs | 72 +++++ ...IApprovals.CustomCheckDetails.approved.txt | 6 + .../RavenPersistedTypes.Verify.approved.txt | 17 + ...etailsIndexTests.RunMapReduce.approved.txt | 36 +++ ...gaListIndexTests.RunMapReduce.approved.txt | 17 + .../Archiving/ArchiveGroupTests.cs | 115 +++++++ .../RavenAttachmentsBodyStorageTests.cs | 32 ++ .../CompositeViews/FailedMessagesTests.cs | 95 ++++++ .../CompositeViews/MessagesViewTests.cs | 304 ++++++++++++++++++ .../Expiration/ChunkerTests.cs | 86 +++++ .../ProcessedMessageExpirationTests.cs | 256 +++++++++++++++ .../Expiration/RavenLastModifiedScope.cs | 21 ++ .../Expiration/SagaAuditExpirationTests.cs | 205 ++++++++++++ .../Infrastructure/InMemoryStoreBuilder.cs | 25 ++ .../Infrastructure/MessageExtensions.cs | 14 + .../Infrastructure/RavenIndexAwaiter.cs | 18 ++ .../MigrationTests.cs | 28 ++ .../FailedAuditImportCustomCheckTests.cs | 50 +++ .../FailedErrorImportCustomCheckTests.cs | 55 ++++ .../PreventReindexing.cs | 26 ++ .../RavenBootstrapperTests.cs | 14 + .../RavenPersistedTypes.cs | 28 ++ .../Recoverability/FailedMessageRetryTests.cs | 22 ++ .../RetryConfirmationProcessorTests.cs | 127 ++++++++ .../ReturnToSenderDequeuerTests.cs | 224 +++++++++++++ .../SagaAudit/SagaDetailsIndexTests.cs | 80 +++++ .../SagaAudit/SagaListIndexTests.cs | 68 ++++ ...eControl.Persistence.Tests.RavenDb5.csproj | 32 ++ .../ServiceControl.runsettings | 7 + .../SubscriptionPersisterTests.cs | 70 ++++ .../TestPersistenceImpl.cs | 99 ++++++ .../Unarchiving/UnarchiveGroupTests.cs | 115 +++++++ .../app.config | 21 ++ src/ServiceControl.sln | 15 + 35 files changed, 2404 insertions(+) create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/.editorconfig create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/API/APIApprovals.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/APIApprovals.CustomCheckDetails.approved.txt create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/RavenPersistedTypes.Verify.approved.txt create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/SagaDetailsIndexTests.RunMapReduce.approved.txt create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/SagaListIndexTests.RunMapReduce.approved.txt create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/Archiving/ArchiveGroupTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/BodyStorage/RavenAttachmentsBodyStorageTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/FailedMessagesTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/ChunkerTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/ProcessedMessageExpirationTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/RavenLastModifiedScope.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/SagaAuditExpirationTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/InMemoryStoreBuilder.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/MessageExtensions.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/RavenIndexAwaiter.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/MigrationTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedAuditImportCustomCheckTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedErrorImportCustomCheckTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/PreventReindexing.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/RavenBootstrapperTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/RavenPersistedTypes.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/FailedMessageRetryTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/RetryConfirmationProcessorTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/ReturnToSenderDequeuerTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/SagaAudit/SagaDetailsIndexTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/SagaAudit/SagaListIndexTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/ServiceControl.Persistence.Tests.RavenDb5.csproj create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/ServiceControl.runsettings create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/SubscriptionPersisterTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/Unarchiving/UnarchiveGroupTests.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/app.config diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/.editorconfig b/src/ServiceControl.Persistence.Tests.RavenDb5/.editorconfig new file mode 100644 index 0000000000..5f68a610b3 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/.editorconfig @@ -0,0 +1,4 @@ +[*.cs] + +# Justification: Test project +dotnet_diagnostic.CA2007.severity = none diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/API/APIApprovals.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/API/APIApprovals.cs new file mode 100644 index 0000000000..2b8ca6fe9b --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/API/APIApprovals.cs @@ -0,0 +1,72 @@ +namespace ServiceControl.UnitTests.API +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Reflection; + using NServiceBus.CustomChecks; + using NUnit.Framework; + using Particular.Approvals; + using Persistence.RavenDb; + using ServiceBus.Management.Infrastructure.Settings; + + [TestFixture] + class APIApprovals + { + [Test] + public void CustomCheckDetails() + { + // HINT: Custom checks are documented on the docs site and Id and Category are published in integration events + // If any changes have been made to custom checks, this may break customer integration subscribers. + Approver.Verify( + string.Join(Environment.NewLine, + from check in GetCustomChecks() + orderby check.Category, check.Id + select $"{check.Category}: {check.Id}" + ) + ); + } + + static IEnumerable GetCustomChecks() + { + var serviceControlTypes = typeof(RavenDbPersistenceConfiguration).Assembly + .GetTypes() + .Where(t => t.IsAbstract == false); + + var customCheckTypes = serviceControlTypes.Where(t => typeof(ICustomCheck).IsAssignableFrom(t)); + + var supportedConstructorArguments = new List() + { + new Settings(), + new RavenDBPersisterSettings + { + DatabasePath = "%TEMP%" + } + }; + + object MapConstructorParameter(ParameterInfo pi) + { + foreach (var obj in supportedConstructorArguments) + { + if (obj.GetType() == pi.ParameterType) + { + return obj; + } + } + + return null; + } + + + foreach (var customCheckType in customCheckTypes) + { + var constructor = customCheckType.GetConstructors().Single(); + var constructorParameters = constructor.GetParameters() + .Select(MapConstructorParameter) + .ToArray(); + var instance = (ICustomCheck)constructor.Invoke(constructorParameters); + yield return instance; + } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/APIApprovals.CustomCheckDetails.approved.txt b/src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/APIApprovals.CustomCheckDetails.approved.txt new file mode 100644 index 0000000000..3f79bf4f12 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/APIApprovals.CustomCheckDetails.approved.txt @@ -0,0 +1,6 @@ +ServiceControl Health: Audit Message Ingestion +ServiceControl Health: Error Database Index Errors +ServiceControl Health: Error Database Index Lag +ServiceControl Health: Message Ingestion Process +ServiceControl Health: Saga Audit Data Retention +Storage space: ServiceControl database \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/RavenPersistedTypes.Verify.approved.txt b/src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/RavenPersistedTypes.Verify.approved.txt new file mode 100644 index 0000000000..2024580e53 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/RavenPersistedTypes.Verify.approved.txt @@ -0,0 +1,17 @@ +ServiceControl.Contracts.CustomChecks.CustomCheck, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null +ServiceControl.EventLog.EventLogItem, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null +ServiceControl.MessageAuditing.ProcessedMessage, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null +ServiceControl.MessageFailures.FailedMessage, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null +ServiceControl.MessageFailures.GroupComment, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null +ServiceControl.MessageFailures.QueueAddress, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null +ServiceControl.Operations.FailedAuditImport, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null +ServiceControl.Operations.FailedErrorImport, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null +ServiceControl.Persistence.KnownEndpoint, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null +ServiceControl.Persistence.MessagesViewIndex+SortAndFilterOptions, ServiceControl.Persistence.RavenDb, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null +ServiceControl.Persistence.RetryBatch, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null +ServiceControl.Persistence.RetryBatchGroup, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null +ServiceControl.Recoverability.FailedMessageRetry, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null +ServiceControl.Recoverability.FailureGroupMessageView, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null +ServiceControl.Recoverability.FailureGroupView, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null +ServiceControl.SagaAudit.SagaHistory, ServiceControl.Audit.Persistence.SagaAudit, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null +ServiceControl.SagaAudit.SagaListIndex+Result, ServiceControl.Persistence.RavenDb, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/SagaDetailsIndexTests.RunMapReduce.approved.txt b/src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/SagaDetailsIndexTests.RunMapReduce.approved.txt new file mode 100644 index 0000000000..52aaa2cff7 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/SagaDetailsIndexTests.RunMapReduce.approved.txt @@ -0,0 +1,36 @@ +[ + { + "Id": "00000000-0000-0000-0000-000000000000", + "SagaId": "00000000-0000-0000-0000-000000000000", + "SagaType": "MySaga1", + "Changes": [ + { + "StartTime": "2002-01-01T01:01:01Z", + "FinishTime": "2002-01-01T01:01:01Z", + "Status": "Completed", + "StateAfterChange": "Completed", + "InitiatingMessage": null, + "OutgoingMessages": [], + "Endpoint": "MyEndpoint" + }, + { + "StartTime": "2001-01-01T01:01:01Z", + "FinishTime": "2001-01-01T01:01:01Z", + "Status": "Updated", + "StateAfterChange": "Updated", + "InitiatingMessage": null, + "OutgoingMessages": [], + "Endpoint": "MyEndpoint" + }, + { + "StartTime": "2000-01-01T01:01:01Z", + "FinishTime": "2000-01-01T01:01:01Z", + "Status": "Updated", + "StateAfterChange": "Started", + "InitiatingMessage": null, + "OutgoingMessages": [], + "Endpoint": "MyEndpoint" + } + ] + } +] \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/SagaListIndexTests.RunMapReduce.approved.txt b/src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/SagaListIndexTests.RunMapReduce.approved.txt new file mode 100644 index 0000000000..160e2c2aa5 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/SagaListIndexTests.RunMapReduce.approved.txt @@ -0,0 +1,17 @@ +[ + { + "Id": "00000000-0000-0000-0000-000000000003", + "Uri": "api/sagas/00000000-0000-0000-0000-000000000003", + "SagaType": "MySaga3" + }, + { + "Id": "00000000-0000-0000-0000-000000000002", + "Uri": "api/sagas/00000000-0000-0000-0000-000000000002", + "SagaType": "MySaga2" + }, + { + "Id": "00000000-0000-0000-0000-000000000001", + "Uri": "api/sagas/00000000-0000-0000-0000-000000000001", + "SagaType": "MySaga1" + } +] \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Archiving/ArchiveGroupTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Archiving/ArchiveGroupTests.cs new file mode 100644 index 0000000000..41b1cb33c1 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Archiving/ArchiveGroupTests.cs @@ -0,0 +1,115 @@ +namespace ServiceControl.UnitTests.Archiving +{ + using System; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Hosting; + using NServiceBus.Testing; + using NUnit.Framework; + using Raven.Client; + using ServiceControl.PersistenceTests; + using ServiceControl.Recoverability; + + [TestFixture] + class ArchiveGroupTests : PersistenceTestBase + { + IDocumentStore DocumentStore => GetRequiredService(); + + protected override IHostBuilder CreateHostBuilder() => base.CreateHostBuilder().ConfigureServices(services => + { + services.AddSingleton(); + services.AddSingleton(); + }); + + [Test] + public async Task ArchiveGroup_skips_over_empty_batches_but_still_completes() + { + // Arrange + var groupId = "TestGroup"; + var previousArchiveBatchId = ArchiveBatch.MakeId(groupId, ArchiveType.FailureGroup, 1); + + using (var session = DocumentStore.OpenAsyncSession()) + { + var previousArchiveBatch = new ArchiveBatch { Id = previousArchiveBatchId }; + await session.StoreAsync(previousArchiveBatch); + + var previousArchiveOperation = new ArchiveOperation + { + Id = ArchiveOperation.MakeId(groupId, ArchiveType.FailureGroup), + RequestId = groupId, + ArchiveType = ArchiveType.FailureGroup, + TotalNumberOfMessages = 2, + NumberOfMessagesArchived = 0, + Started = DateTime.Now, + GroupName = "Test Group", + NumberOfBatches = 3, + CurrentBatch = 0 + }; + await session.StoreAsync(previousArchiveOperation); + + await session.SaveChangesAsync(); + } + + var handler = GetRequiredService(); // See this.CreateHostBuilder + + var context = new TestableMessageHandlerContext(); + var message = new ArchiveAllInGroup { GroupId = groupId }; + + // Act + await handler.Handle(message, context); + + // Assert + using (var session = DocumentStore.OpenSession()) + { + var loadedBatch = session.Load(previousArchiveBatchId); + Assert.IsNull(loadedBatch); + } + } + + [Test] + public async Task ArchiveGroup_GetGroupDetails_doesnt_fail_with_invalid_groupId() + { + // Arrange + var failureGroupsViewIndex = new FailureGroupsViewIndex(); + await failureGroupsViewIndex.ExecuteAsync(DocumentStore); + + var groupId = "TestGroup"; + var previousArchiveBatchId = ArchiveBatch.MakeId(groupId, ArchiveType.FailureGroup, 1); + + using (var session = DocumentStore.OpenAsyncSession()) + { + var previousArchiveBatch = new ArchiveBatch { Id = previousArchiveBatchId }; + await session.StoreAsync(previousArchiveBatch); + + var previousArchiveOperation = new ArchiveOperation + { + Id = ArchiveOperation.MakeId(groupId, ArchiveType.FailureGroup), + RequestId = groupId, + ArchiveType = ArchiveType.FailureGroup, + TotalNumberOfMessages = 2, + NumberOfMessagesArchived = 0, + Started = DateTime.Now, + GroupName = "Test Group", + NumberOfBatches = 3, + CurrentBatch = 0 + }; + await session.StoreAsync(previousArchiveOperation); + + await session.SaveChangesAsync(); + } + + var handler = GetRequiredService(); // See this.CreateHostBuilder + + var context = new TestableMessageHandlerContext(); + var message = new ArchiveAllInGroup { GroupId = groupId + "Invalid" }; + + // Act + // Assert + Assert.DoesNotThrowAsync(async () => + { + // Act + await handler.Handle(message, context); + }); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/BodyStorage/RavenAttachmentsBodyStorageTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/BodyStorage/RavenAttachmentsBodyStorageTests.cs new file mode 100644 index 0000000000..f477920e26 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/BodyStorage/RavenAttachmentsBodyStorageTests.cs @@ -0,0 +1,32 @@ +namespace ServiceControl.UnitTests.BodyStorage +{ + using System; + using System.IO; + using System.Threading.Tasks; + using NUnit.Framework; + using PersistenceTests; + + [TestFixture] + sealed class RavenAttachmentsBodyStorageTests : PersistenceTestBase + { + [Test] + public async Task Attachments_with_ids_that_contain_backslash_should_be_readable() + { + var messageId = "3f0240a7-9b2e-4e2a-ab39-6114932adad1\\2055783"; + var contentType = "NotImportant"; + var body = BitConverter.GetBytes(0xDEADBEEF); + + await BodyStorage.Store(messageId, contentType, body.Length, new MemoryStream(body)); + + var retrieved = await BodyStorage.TryFetch(messageId); + Assert.IsNotNull(retrieved); + Assert.True(retrieved.HasResult); + Assert.AreEqual(contentType, retrieved.ContentType); + + var buffer = new byte[retrieved.BodySize]; + retrieved.Stream.Read(buffer, 0, retrieved.BodySize); + + Assert.AreEqual(body, buffer); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/FailedMessagesTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/FailedMessagesTests.cs new file mode 100644 index 0000000000..b269299886 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/FailedMessagesTests.cs @@ -0,0 +1,95 @@ +namespace ServiceControl.UnitTests.CompositeViews +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading; + using Contracts.Operations; + using MessageFailures; + using MessageFailures.Api; + using NUnit.Framework; + using Raven.Client; + + [TestFixture] + public class FailedMessagesTests + { + [Test] + public void Should_allow_errors_with_no_metadata() + { + using (var session = documentStore.OpenSession()) + { + var processedMessage = new FailedMessage + { + Id = "1", + UniqueMessageId = "xyz", + Status = FailedMessageStatus.Unresolved, + ProcessingAttempts = new List + { + new FailedMessage.ProcessingAttempt + { + AttemptedAt = DateTime.UtcNow, + MessageMetadata = new Dictionary(), + FailureDetails = new FailureDetails() + } + } + }; + + session.Store(processedMessage); + + session.SaveChanges(); + } + + RavenQueryStatistics stats; + + do + { + using (var session = documentStore.OpenSession()) + { + var results = session.Advanced.DocumentQuery() + .SetResultTransformer(FailedMessageViewTransformer.Name) + .Statistics(out stats) + .SelectFields() + .ToList(); + + + if (!stats.IsStale) + { + Console.Out.WriteLine("Checking result"); + Assert.AreEqual(1, results.Count); + + Assert.AreEqual(null, results.First().TimeSent); + } + } + + + if (stats.IsStale) + { + Thread.Sleep(1000); + } + } + while (stats.IsStale); + } + + + [SetUp] + public void SetUp() + { + documentStore = InMemoryStoreBuilder.GetInMemoryStore(); + + var customIndex = new FailedMessageViewIndex(); + customIndex.Execute(documentStore); + + var transformer = new FailedMessageViewTransformer(); + + transformer.Execute(documentStore); + } + + [TearDown] + public void TearDown() + { + documentStore.Dispose(); + } + + IDocumentStore documentStore; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs new file mode 100644 index 0000000000..65eb6714be --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs @@ -0,0 +1,304 @@ +namespace ServiceControl.UnitTests.CompositeViews +{ + using System; + using System.Collections.Generic; + using System.Linq; + using MessageAuditing; + using MessageFailures; + using NUnit.Framework; + using Raven.Client; + using Raven.Client.Linq; + using ServiceControl.CompositeViews.Messages; + using ServiceControl.Persistence; + + [TestFixture] + public class MessagesViewTests + { + [Test] + public void Filter_out_system_messages() + { + using (var session = documentStore.OpenSession()) + { + var processedMessage = new ProcessedMessage + { + Id = "1" + }; + + processedMessage.MakeSystemMessage(); + session.Store(processedMessage); + + var processedMessage2 = new ProcessedMessage + { + Id = "2" + }; + processedMessage2.MakeSystemMessage(false); + session.Store(processedMessage2); + + session.SaveChanges(); + } + + documentStore.WaitForIndexing(); + + using (var session = documentStore.OpenSession()) + { + var results = session.Query() + .Customize(x => x.WaitForNonStaleResults()) + .Where(x => !x.IsSystemMessage) + .OfType() + .ToList(); + Assert.AreEqual(1, results.Count); + Assert.AreNotEqual("1", results.Single().Id); + } + } + + [Test] + public void Order_by_critical_time() + { + using (var session = documentStore.OpenSession()) + { + session.Store(new ProcessedMessage + { + Id = "1", + MessageMetadata = new Dictionary { { "CriticalTime", TimeSpan.FromSeconds(10) } } + }); + + session.Store(new ProcessedMessage + { + Id = "2", + MessageMetadata = new Dictionary { { "CriticalTime", TimeSpan.FromSeconds(20) } } + }); + + session.Store(new ProcessedMessage + { + Id = "3", + MessageMetadata = new Dictionary { { "CriticalTime", TimeSpan.FromSeconds(15) } } + }); + + session.Store(new FailedMessage + { + Id = "4", + Status = FailedMessageStatus.Unresolved, + ProcessingAttempts = new List + { + new FailedMessage.ProcessingAttempt {MessageMetadata = new Dictionary {{"CriticalTime", TimeSpan.FromSeconds(15)}}} + } + }); + session.SaveChanges(); + } + + documentStore.WaitForIndexing(); + + using (var session = documentStore.OpenSession()) + { + var firstByCriticalTime = session.Query() + .OrderBy(x => x.CriticalTime) + .Where(x => x.CriticalTime.HasValue) + .ProjectFromIndexFieldsInto() + .First(); + + Assert.AreEqual("1", firstByCriticalTime.Id); + + var firstByCriticalTimeDescription = session.Query() + .OrderByDescending(x => x.CriticalTime) + .Where(x => x.CriticalTime.HasValue) + .ProjectFromIndexFieldsInto() + .First(); + Assert.AreEqual("2", firstByCriticalTimeDescription.Id); + } + } + + [Test] + public void Order_by_time_sent() + { + using (var session = documentStore.OpenSession()) + { + session.Store(new ProcessedMessage + { + Id = "1", + MessageMetadata = new Dictionary { { "TimeSent", DateTime.Today.AddSeconds(20) } } + }); + + session.Store(new ProcessedMessage + { + Id = "2", + MessageMetadata = new Dictionary { { "TimeSent", DateTime.Today.AddSeconds(10) } } + }); + session.Store(new ProcessedMessage + { + Id = "3", + MessageMetadata = new Dictionary { { "TimeSent", DateTime.Today.AddDays(-1) } } + }); + session.SaveChanges(); + } + + documentStore.WaitForIndexing(); + + using (var session = documentStore.OpenSession()) + { + var firstByTimeSent = session.Query() + .OrderBy(x => x.TimeSent) + .ProjectFromIndexFieldsInto() + .First(); + Assert.AreEqual("3", firstByTimeSent.Id); + + var firstByTimeSentDescription = session.Query() + .OrderByDescending(x => x.TimeSent) + .ProjectFromIndexFieldsInto() + .First(); + Assert.AreEqual("1", firstByTimeSentDescription.Id); + } + } + + [Test] + public void TimeSent_is_not_cast_to_DateTimeMin_if_null() + { + using (var session = documentStore.OpenSession()) + { + session.Store(new ProcessedMessage + { + MessageMetadata = new Dictionary + { + {"MessageIntent", "1"}, + {"TimeSent", null} + } + }); + session.Store(new FailedMessage + { + ProcessingAttempts = new List + { + new FailedMessage.ProcessingAttempt + { + AttemptedAt = DateTime.Today, + MessageMetadata = new Dictionary + { + {"MessageIntent", "1"}, + {"TimeSent", null} + } + }, + new FailedMessage.ProcessingAttempt + { + AttemptedAt = DateTime.Today, + MessageMetadata = new Dictionary + { + {"MessageIntent", "1"}, + {"TimeSent", null} + } + } + } + }); + + session.SaveChanges(); + } + + documentStore.WaitForIndexing(); + + using (var session = documentStore.OpenSession()) + { + var messageWithNoTimeSent = session.Query() + .TransformWith() + .Customize(x => x.WaitForNonStaleResults()) + .ToArray(); + Assert.AreEqual(null, messageWithNoTimeSent[0].TimeSent); + Assert.AreEqual(null, messageWithNoTimeSent[1].TimeSent); + } + } + + [TestCase(FailedMessageStatus.Archived, MessageStatus.ArchivedFailure)] + [TestCase(FailedMessageStatus.Resolved, MessageStatus.ResolvedSuccessfully)] + [TestCase(FailedMessageStatus.RetryIssued, MessageStatus.RetryIssued)] + [TestCase(FailedMessageStatus.Unresolved, MessageStatus.Failed)] + public void Correct_status_for_failed_messages(FailedMessageStatus failedMessageStatus, MessageStatus expecteMessageStatus) + { + using (var session = documentStore.OpenSession()) + { + session.Store(new FailedMessage + { + Id = "1", + ProcessingAttempts = new List + { + new FailedMessage.ProcessingAttempt + { + AttemptedAt = DateTime.Today, + MessageMetadata = new Dictionary {{"MessageIntent", "1"}} + } + }, + Status = failedMessageStatus + }); + + session.SaveChanges(); + } + + documentStore.WaitForIndexing(); + + using (var session = documentStore.OpenSession()) + { + var message = session.Query() + .TransformWith() + .Customize(x => x.WaitForNonStaleResults()) + .Single(); + + Assert.AreEqual(expecteMessageStatus, message.Status); + } + } + + [Test] + public void Correct_status_for_repeated_errors() + { + using (var session = documentStore.OpenSession()) + { + session.Store(new FailedMessage + { + Id = "1", + ProcessingAttempts = new List + { + new FailedMessage.ProcessingAttempt + { + AttemptedAt = DateTime.Today, + MessageMetadata = new Dictionary {{"MessageIntent", "1"}} + }, + new FailedMessage.ProcessingAttempt + { + AttemptedAt = DateTime.Today, + MessageMetadata = new Dictionary {{"MessageIntent", "1"}} + } + } + }); + + session.SaveChanges(); + } + + documentStore.WaitForIndexing(); + + using (var session = documentStore.OpenSession()) + { + var message = session.Query() + .TransformWith() + .Customize(x => x.WaitForNonStaleResults()) + .Single(); + + Assert.AreEqual(MessageStatus.RepeatedFailure, message.Status); + } + } + + [SetUp] + public void SetUp() + { + documentStore = InMemoryStoreBuilder.GetInMemoryStore(); + + var customIndex = new MessagesViewIndex(); + customIndex.Execute(documentStore); + + var transformer = new MessagesViewTransformer(); + + transformer.Execute(documentStore); + } + + [TearDown] + public void TearDown() + { + documentStore.Dispose(); + } + + IDocumentStore documentStore; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/ChunkerTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/ChunkerTests.cs new file mode 100644 index 0000000000..76d1909291 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/ChunkerTests.cs @@ -0,0 +1,86 @@ +namespace ServiceControl.UnitTests.Expiration +{ + using System.Collections.Generic; + using System.Threading; + using NUnit.Framework; + using ServiceControl.Infrastructure.RavenDB; + + [TestFixture] + public class ChunkerTests + { + [Test] + public void Chunking() + { + var starts = new List(); + var ends = new List(); + + var count = Chunker.ExecuteInChunks(1500, (startList, endList, s, e) => + { + startList.Add(s); + endList.Add(e); + return 1; + }, starts, ends); + + Assert.AreEqual(0, starts[0]); + Assert.AreEqual(499, ends[0]); + + Assert.AreEqual(500, starts[1]); + Assert.AreEqual(999, ends[1]); + + Assert.AreEqual(1000, starts[2]); + Assert.AreEqual(1499, ends[2]); + + Assert.AreEqual(3, starts.Count); + Assert.AreEqual(3, ends.Count); + Assert.AreEqual(3, count); + } + + [Test] + public void LessThenChunkSize() + { + var starts = new List(); + var ends = new List(); + + var count = Chunker.ExecuteInChunks(1, (startList, endList, s, e) => + { + startList.Add(s); + endList.Add(e); + return 1; + }, starts, ends); + + Assert.AreEqual(0, starts[0]); + Assert.AreEqual(0, ends[0]); + + Assert.AreEqual(1, starts.Count); + Assert.AreEqual(1, ends.Count); + Assert.AreEqual(1, count); + } + + [Test] + public void OneExtra() + { + var starts = new List(); + var ends = new List(); + + var count = Chunker.ExecuteInChunks(1001, (startList, endList, s, e) => + { + startList.Add(s); + endList.Add(e); + return 1; + }, starts, ends); + + Assert.AreEqual(0, starts[0]); + Assert.AreEqual(499, ends[0]); + + Assert.AreEqual(500, starts[1]); + Assert.AreEqual(999, ends[1]); + + Assert.AreEqual(1000, starts[2]); + Assert.AreEqual(1000, ends[2]); + + Assert.AreEqual(3, starts.Count); + Assert.AreEqual(3, ends.Count); + Assert.AreEqual(3, count); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/ProcessedMessageExpirationTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/ProcessedMessageExpirationTests.cs new file mode 100644 index 0000000000..1425366ee4 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/ProcessedMessageExpirationTests.cs @@ -0,0 +1,256 @@ +namespace ServiceControl.UnitTests.Expiration +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.IO; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using MessageAuditing; + using MessageFailures; + using NUnit.Framework; + using PersistenceTests; + using Raven.Client; + using Raven.Client.Embedded; + using ServiceControl.Infrastructure.RavenDB.Expiration; + using ServiceControl.Operations.BodyStorage.RavenAttachments; + + sealed class ProcessedMessageExpirationTests : PersistenceTestBase + { + IDocumentStore DocumentStore => GetRequiredService(); + + [Test] + public void Old_documents_are_being_expired() + { + var expiredDate = DateTime.UtcNow.AddDays(-3); + var thresholdDate = DateTime.UtcNow.AddDays(-2); + var processedMessage = new ProcessedMessage + { + Id = "1", + ProcessedAt = expiredDate + }; + + using (var session = DocumentStore.OpenSession()) + { + session.Store(processedMessage); + session.SaveChanges(); + } + + RunExpiry(thresholdDate); + + using (var session = DocumentStore.OpenSession()) + { + Assert.IsEmpty(session.Query()); + } + } + + [Test] + public void Many_documents_are_being_expired() + { + var expiredDate = DateTime.UtcNow.AddDays(-3); + var thresholdDate = DateTime.UtcNow.AddDays(-2); + var recentDate = DateTime.UtcNow.AddDays(-1); + var expiredMessages = BuildExpiredMessaged(expiredDate).ToList(); + using (var session = DocumentStore.OpenSession()) + { + foreach (var message in expiredMessages) + { + session.Store(message); + } + + session.SaveChanges(); + } + + using (var session = DocumentStore.OpenSession()) + { + var recentMessage = new ProcessedMessage + { + Id = "recentMessageId", + ProcessedAt = recentDate + }; + session.Store(recentMessage); + session.SaveChanges(); + } + + RunExpiry(thresholdDate); + foreach (dynamic message in expiredMessages) + { + using (var session = DocumentStore.OpenSession()) + { + Assert.Null(session.Load(message.Id)); + } + } + + using (var session = DocumentStore.OpenSession()) + { + Assert.AreEqual(1, session.Query().Count()); + } + } + + IEnumerable BuildExpiredMessaged(DateTime dateTime) + { + for (var i = 0; i < DocTestRange; i++) + { + yield return new ProcessedMessage + { + Id = Guid.NewGuid().ToString(), + ProcessedAt = dateTime + }; + } + } + + void RunExpiry(DateTime expiryThreshold) + { + var documentStore = DocumentStore; + var embeddableDocumentStore = (EmbeddableDocumentStore)documentStore; + + new ExpiryProcessedMessageIndex().Execute(documentStore); + documentStore.WaitForIndexing(); + AuditMessageCleaner.Clean(DocTestRange, embeddableDocumentStore.DocumentDatabase, expiryThreshold); + documentStore.WaitForIndexing(); + } + + [Test] + public void Only_processed_messages_are_being_expired() + { + var expiredDate = DateTime.UtcNow.AddDays(-3); + var thresholdDate = DateTime.UtcNow.AddDays(-2); + var recentDate = DateTime.UtcNow.AddDays(-1); + var expiredMessage = new ProcessedMessage + { + Id = "1", + ProcessedAt = expiredDate + }; + + using (var session = DocumentStore.OpenSession()) + { + session.Store(expiredMessage); + session.SaveChanges(); + } + + var recentMessage = new ProcessedMessage + { + Id = "2", + ProcessedAt = recentDate + }; + using (var session = DocumentStore.OpenSession()) + { + session.Store(recentMessage); + session.SaveChanges(); + } + + RunExpiry(thresholdDate); + + using (var session = DocumentStore.OpenSession()) + { + Assert.Null(session.Load(expiredMessage.Id)); + Assert.NotNull(session.Load(recentMessage.Id)); + } + } + + [Test] + public async Task Stored_bodies_are_being_removed_when_message_expires() + { + var expiredDate = DateTime.UtcNow.AddDays(-3); + var thresholdDate = DateTime.UtcNow.AddDays(-2); + // Store expired message with associated body + var messageId = "21"; + + var processedMessage = new ProcessedMessage + { + Id = "1", + ProcessedAt = expiredDate, + MessageMetadata = new Dictionary + { + { + "MessageId", messageId + } + } + }; + + using (var session = DocumentStore.OpenSession()) + { + session.Store(processedMessage); + session.SaveChanges(); + } + + var body = new byte[] + { + 1, + 2, + 3, + 4, + 5 + }; + + var bodyStorage = new RavenAttachmentsBodyStorage(DocumentStore); + using (var stream = new MemoryStream(body)) + { + await bodyStorage.Store(messageId, "binary", 5, stream); + } + + RunExpiry(thresholdDate); + + // Verify message expired + using (var session = DocumentStore.OpenSession()) + { + Assert.Null(session.Load(processedMessage.Id)); + } + + var result = await bodyStorage.TryFetch(messageId); + // Verify body expired + Assert.Null(result, "Audit document body should be deleted"); + } + + [Test] + public void Recent_processed_messages_are_not_being_expired() + { + var thresholdDate = DateTime.UtcNow.AddDays(-2); + var recentDate = DateTime.UtcNow.AddDays(-1); + var message = new ProcessedMessage + { + Id = "1", + ProcessedAt = recentDate + }; + using (var session = DocumentStore.OpenSession()) + { + session.Store(message); + session.SaveChanges(); + } + + RunExpiry(thresholdDate); + using (var session = DocumentStore.OpenSession()) + { + Assert.AreEqual(1, session.Query().Count()); + } + } + + [Test] + public void Errors_are_not_being_expired() + { + var failedMsg = new FailedMessage + { + Id = "1" + }; + + using (var session = DocumentStore.OpenSession()) + { + session.Store(failedMsg); + session.SaveChanges(); + + Debug.WriteLine(session.Advanced.GetMetadataFor(failedMsg)["Last-Modified"]); + } + + Thread.Sleep(100); + RunExpiry(DateTime.UtcNow); + + using (var session = DocumentStore.OpenSession()) + { + Assert.NotNull(session.Load(failedMsg.Id)); + } + } + + const int DocTestRange = 999; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/RavenLastModifiedScope.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/RavenLastModifiedScope.cs new file mode 100644 index 0000000000..ae3735f1e0 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/RavenLastModifiedScope.cs @@ -0,0 +1,21 @@ +namespace ServiceControl.UnitTests.Expiration +{ + using System; + using Raven.Abstractions; + + public class RavenLastModifiedScope : IDisposable + { + public RavenLastModifiedScope(DateTime dateTime) + { + previous = SystemTime.UtcDateTime; + SystemTime.UtcDateTime = () => dateTime; + } + + public void Dispose() + { + SystemTime.UtcDateTime = previous; + } + + Func previous; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/SagaAuditExpirationTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/SagaAuditExpirationTests.cs new file mode 100644 index 0000000000..02a17193d5 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/SagaAuditExpirationTests.cs @@ -0,0 +1,205 @@ +namespace ServiceControl.UnitTests.Expiration +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Linq; + using System.Threading; + using MessageFailures; + using NUnit.Framework; + using Raven.Client.Embedded; + using ServiceControl.SagaAudit; + + [TestFixture] + public class SagaAuditExpirationTests + { + [Test] + public void Old_documents_are_being_expired() + { + using (var documentStore = InMemoryStoreBuilder.GetInMemoryStore()) + { + var expiredDate = DateTime.UtcNow.AddDays(-3); + var thresholdDate = DateTime.UtcNow.AddDays(-2); + + var sagaHistoryId = Guid.NewGuid().ToString(); + var sagaHistory = new SagaSnapshot + { + Id = sagaHistoryId + }; + + using (new RavenLastModifiedScope(expiredDate)) + using (var session = documentStore.OpenSession()) + { + session.Store(sagaHistory); + session.SaveChanges(); + } + + RunExpiry(documentStore, thresholdDate); + + using (var session = documentStore.OpenSession()) + { + Assert.IsEmpty(session.Query()); + } + } + } + + [Test] + public void Many_documents_are_being_expired() + { + using (var documentStore = InMemoryStoreBuilder.GetInMemoryStore()) + { + var expiredDate = DateTime.UtcNow.AddDays(-3); + var thresholdDate = DateTime.UtcNow.AddDays(-2); + var recentDate = DateTime.UtcNow.AddDays(-1); + var expiredMessages = BuilExpiredMessaged().ToList(); + using (new RavenLastModifiedScope(expiredDate)) + { + using (var session = documentStore.OpenSession()) + { + foreach (var message in expiredMessages) + { + session.Store(message); + } + + session.SaveChanges(); + } + } + + using (new RavenLastModifiedScope(recentDate)) + using (var session = documentStore.OpenSession()) + { + var recentSagaHistory = new SagaSnapshot + { + Id = Guid.NewGuid().ToString() + }; + session.Store(recentSagaHistory); + session.SaveChanges(); + } + + RunExpiry(documentStore, thresholdDate); + + using (var session = documentStore.OpenSession()) + { + Assert.AreEqual(1, session.Query().Count()); + } + } + } + + IEnumerable BuilExpiredMessaged() + { + for (var i = 0; i < 10; i++) + { + yield return new SagaSnapshot + { + Id = Guid.NewGuid().ToString() + }; + } + } + + static void RunExpiry(EmbeddableDocumentStore documentStore, DateTime expiryThreshold) + { + new ExpirySagaAuditIndex().Execute(documentStore); + documentStore.WaitForIndexing(); + SagaHistoryCleaner.Clean(100, documentStore.DocumentDatabase, expiryThreshold); + documentStore.WaitForIndexing(); + } + + [Test] + public void Only_expired_being_deleted() + { + using (var documentStore = InMemoryStoreBuilder.GetInMemoryStore()) + { + var expiredDate = DateTime.UtcNow.AddDays(-3); + var thresholdDate = DateTime.UtcNow.AddDays(-2); + var recentDate = DateTime.UtcNow.AddDays(-1); + + var expiredSagaHistory = new SagaSnapshot + { + Id = Guid.NewGuid().ToString() + }; + + using (new RavenLastModifiedScope(expiredDate)) + using (var session = documentStore.OpenSession()) + { + session.Store(expiredSagaHistory); + session.SaveChanges(); + } + + var recentSagaHistory = new SagaSnapshot + { + Id = Guid.NewGuid().ToString() + }; + using (new RavenLastModifiedScope(recentDate)) + using (var session = documentStore.OpenSession()) + { + session.Store(recentSagaHistory); + session.SaveChanges(); + } + + RunExpiry(documentStore, thresholdDate); + + using (var session = documentStore.OpenSession()) + { + Assert.Null(session.Load(expiredSagaHistory.Id)); + Assert.NotNull(session.Load(recentSagaHistory.Id)); + } + } + } + + + [Test] + public void Recent_processed_messages_are_not_being_expired() + { + using (var documentStore = InMemoryStoreBuilder.GetInMemoryStore()) + { + var thresholdDate = DateTime.UtcNow.AddDays(-2); + var recentDate = DateTime.UtcNow.AddDays(-1); + var sagaHistory = new SagaSnapshot + { + Id = Guid.NewGuid().ToString() + }; + + using (new RavenLastModifiedScope(recentDate)) + using (var session = documentStore.OpenSession()) + { + session.Store(sagaHistory); + session.SaveChanges(); + } + + RunExpiry(documentStore, thresholdDate); + using (var session = documentStore.OpenSession()) + { + Assert.AreEqual(1, session.Query().Count()); + } + } + } + + [Test] + public void Errors_are_not_being_expired() + { + using (var documentStore = InMemoryStoreBuilder.GetInMemoryStore()) + { + var failedMsg = new FailedMessage + { + Id = "1" + }; + + using (var session = documentStore.OpenSession()) + { + session.Store(failedMsg); + session.SaveChanges(); + + Debug.WriteLine(session.Advanced.GetMetadataFor(failedMsg)["Last-Modified"]); + } + + Thread.Sleep(100); + RunExpiry(documentStore, DateTime.UtcNow); + + using (var session = documentStore.OpenSession()) + { + Assert.NotNull(session.Load(failedMsg.Id)); + } + } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/InMemoryStoreBuilder.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/InMemoryStoreBuilder.cs new file mode 100644 index 0000000000..3421b5bdf4 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/InMemoryStoreBuilder.cs @@ -0,0 +1,25 @@ +using System.IO; +using Raven.Client.Embedded; + +public class InMemoryStoreBuilder +{ + public static EmbeddableDocumentStore GetInMemoryStore() + { + var store = new EmbeddableDocumentStore + { + Configuration = + { + RunInUnreliableYetFastModeThatIsNotSuitableForProduction = true, + RunInMemory = true, + CompiledIndexCacheDirectory = Path.GetTempPath() // RavenDB-2236 + }, + Conventions = + { + SaveEnumsAsIntegers = true + } + }; + store.Initialize(); + + return store; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/MessageExtensions.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/MessageExtensions.cs new file mode 100644 index 0000000000..ee278f421e --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/MessageExtensions.cs @@ -0,0 +1,14 @@ +using ServiceControl.MessageAuditing; + +public static class MessageExtensions +{ + public static void MakeSystemMessage(this ProcessedMessage message, bool isSystem = true) + { + message.MessageMetadata["IsSystemMessage"] = isSystem; + } + + public static void SetMessageId(this ProcessedMessage message, string messageId) + { + message.MessageMetadata["MessageId"] = messageId; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/RavenIndexAwaiter.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/RavenIndexAwaiter.cs new file mode 100644 index 0000000000..26ec4d1a83 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/RavenIndexAwaiter.cs @@ -0,0 +1,18 @@ +using System; +using System.Threading; +using NUnit.Framework; +using Raven.Client; + +public static class RavenIndexAwaiter +{ + public static void WaitForIndexing(this IDocumentStore store) + { + store.WaitForIndexing(10); + } + + static void WaitForIndexing(this IDocumentStore store, int secondsToWait) + { + var databaseCommands = store.DatabaseCommands; + Assert.True(SpinWait.SpinUntil(() => databaseCommands.GetStatistics().StaleIndexes.Length == 0, TimeSpan.FromSeconds(secondsToWait))); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/MigrationTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/MigrationTests.cs new file mode 100644 index 0000000000..3d08231c90 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/MigrationTests.cs @@ -0,0 +1,28 @@ +using System.IO; +using NUnit.Framework; +using Raven.Imports.Newtonsoft.Json; +using ServiceControl.Infrastructure.RavenDB; + +[TestFixture] +class MigrationTests +{ + [Test] + public void VerifyMigration() + { + var serializedForm = @"{ +""$type"":""ServiceControl.SagaAudit.SagaInfo, ServiceControl"", +""ChangeStatus"":null, +""SagaType"":null, +""SagaId"":""00000000-0000-0000-0000-000000000000""}"; + + var serializer = new JsonSerializer + { + TypeNameHandling = TypeNameHandling.All, + Binder = new MigratedTypeAwareBinder() + }; + + serializer.Deserialize(new JsonTextReader(new StringReader(serializedForm))); + + Assert.Pass("Deserialized correctly"); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedAuditImportCustomCheckTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedAuditImportCustomCheckTests.cs new file mode 100644 index 0000000000..e7d75306ea --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedAuditImportCustomCheckTests.cs @@ -0,0 +1,50 @@ +namespace ServiceControl.UnitTests.Operations +{ + using System.Threading.Tasks; + using NServiceBus.CustomChecks; + using NUnit.Framework; + using ServiceControl.Operations; + + [TestFixture] + public class FailedAuditImportCustomCheckTests + { + [Test] + public async Task Pass_if_no_failed_imports() + { + using (var store = InMemoryStoreBuilder.GetInMemoryStore()) + { + store.ExecuteIndex(new FailedAuditImportIndex()); + + var customCheck = new FailedAuditImportCustomCheck(store); + + var result = await customCheck.PerformCheck(); + + Assert.AreEqual(CheckResult.Pass, result); + } + } + + [Test] + public async Task Fail_if_failed_imports() + { + using (var store = InMemoryStoreBuilder.GetInMemoryStore()) + { + store.ExecuteIndex(new FailedAuditImportIndex()); + + using (var session = store.OpenAsyncSession()) + { + await session.StoreAsync(new FailedAuditImport()); + await session.SaveChangesAsync(); + } + + store.WaitForIndexing(); + + var customCheck = new FailedAuditImportCustomCheck(store); + + var result = await customCheck.PerformCheck(); + + Assert.IsTrue(result.HasFailed); + StringAssert.StartsWith("One or more audit messages have failed to import properly into ServiceControl and have been stored in the ServiceControl database.", result.FailureReason); + } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedErrorImportCustomCheckTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedErrorImportCustomCheckTests.cs new file mode 100644 index 0000000000..0cde2d8569 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedErrorImportCustomCheckTests.cs @@ -0,0 +1,55 @@ +namespace ServiceControl.UnitTests.Operations +{ + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Hosting; + using NServiceBus.CustomChecks; + using NUnit.Framework; + using PersistenceTests; + using Raven.Client; + using ServiceControl.Operations; + + [TestFixture] + class FailedErrorImportCustomCheckTests : PersistenceTestBase + { + IDocumentStore DocumentStore => GetRequiredService(); + + protected override IHostBuilder CreateHostBuilder() => base.CreateHostBuilder().ConfigureServices(services => + { + services.AddSingleton(); + }); + + [Test] + public async Task Pass_if_no_failed_imports() + { + await DocumentStore.ExecuteIndexAsync(new FailedErrorImportIndex()); + + var customCheck = GetRequiredService(); + + var result = await customCheck.PerformCheck(); + + Assert.AreEqual(CheckResult.Pass, result); + } + + [Test] + public async Task Fail_if_failed_imports() + { + await DocumentStore.ExecuteIndexAsync(new FailedErrorImportIndex()); + + using (var session = DocumentStore.OpenAsyncSession()) + { + await session.StoreAsync(new FailedErrorImport()); + await session.SaveChangesAsync(); + } + + DocumentStore.WaitForIndexing(); + + var customCheck = GetRequiredService(); + + var result = await customCheck.PerformCheck(); + + Assert.IsTrue(result.HasFailed); + StringAssert.StartsWith("One or more error messages have failed to import properly into ServiceControl and have been stored in the ServiceControl database.", result.FailureReason); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/PreventReindexing.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/PreventReindexing.cs new file mode 100644 index 0000000000..c85384058d --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/PreventReindexing.cs @@ -0,0 +1,26 @@ +namespace ServiceControl.UnitTests.Infrastructure +{ + using System; + using System.Linq; + using MessageFailures.Api; + using NUnit.Framework; + using ServiceControl.Persistence; + + [TestFixture] + public class PreventReindexing + { + [Test] + public void WeApproveThisBecauseChangingThisTypeWillLeadToAMassiveReindexWhichWeWantToAvoid() + { + var msgProperties = typeof(MessagesViewIndex.SortAndFilterOptions).GetProperties(); + var msgTimeSentType = msgProperties.Single(p => p.Name == "TimeSent").PropertyType; + + Assert.AreEqual(typeof(DateTime), msgTimeSentType); + + var failedMsgProperties = typeof(FailedMessageViewIndex.SortAndFilterOptions).GetProperties(); + var failedMsgTimeSentType = failedMsgProperties.Single(p => p.Name == "TimeSent").PropertyType; + + Assert.AreEqual(typeof(DateTime), failedMsgTimeSentType); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/RavenBootstrapperTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/RavenBootstrapperTests.cs new file mode 100644 index 0000000000..cca0b65feb --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/RavenBootstrapperTests.cs @@ -0,0 +1,14 @@ +using NUnit.Framework; +using ServiceControl.Persistence.RavenDb; + +[TestFixture] +class RavenBootstrapperTests +{ + [Test] + public void ReadLicense() + { + var readLicense = RavenBootstrapper.ReadLicense(); + Assert.IsNotNull(readLicense); + Assert.IsNotEmpty(readLicense); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/RavenPersistedTypes.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/RavenPersistedTypes.cs new file mode 100644 index 0000000000..34c7d260a1 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/RavenPersistedTypes.cs @@ -0,0 +1,28 @@ +namespace ServiceControl.UnitTests +{ + using System; + using System.Linq; + using NUnit.Framework; + using Particular.Approvals; + using Particular.ServiceControl; + using Raven.Client.Indexes; + using ServiceControl.Persistence; + + class RavenPersistedTypes + { + [Test] + public void Verify() + { + var allTypes = typeof(Bootstrapper).Assembly.GetTypes().Concat(typeof(RavenQueryExtensions).Assembly.GetTypes()).Concat(typeof(EndpointsView).Assembly.GetTypes()); + + var documentTypes = allTypes + .Where(type => typeof(AbstractIndexCreationTask).IsAssignableFrom(type)) + .SelectMany(indexType => indexType.BaseType?.GenericTypeArguments) + .Distinct(); + + var documentTypeNames = string.Join(Environment.NewLine, documentTypes.Select(t => t.AssemblyQualifiedName).OrderBy(x => x)); + + Approver.Verify(documentTypeNames); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/FailedMessageRetryTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/FailedMessageRetryTests.cs new file mode 100644 index 0000000000..8019c78618 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/FailedMessageRetryTests.cs @@ -0,0 +1,22 @@ +namespace ServiceControl.UnitTests.Operations +{ + using System; + using NUnit.Framework; + using ServiceControl.MessageFailures; + using ServiceControl.Recoverability; + + [TestFixture] + public class FailedMessageRetryTests + { + [Test] + public void Should_Extract_Correct_FailedMessageRetryId_From_FailedMessageId() + { + var messageId = Guid.NewGuid().ToString(); + var failedMessageId = FailedMessageIdGenerator.MakeDocumentId(messageId); + + var extractedFailedMessageRetryId = FailedMessageRetry.MakeDocumentId(FailedMessageIdGenerator.GetMessageIdFromDocumentId(failedMessageId)); + + Assert.AreEqual(FailedMessageRetry.MakeDocumentId(messageId), extractedFailedMessageRetryId); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/RetryConfirmationProcessorTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/RetryConfirmationProcessorTests.cs new file mode 100644 index 0000000000..ee092c8b3f --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/RetryConfirmationProcessorTests.cs @@ -0,0 +1,127 @@ +namespace ServiceControl.UnitTests.Recoverability +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using Contracts.MessageFailures; + using MessageFailures; + using MessageFailures.Handlers; + using NServiceBus.Extensibility; + using NServiceBus.Testing; + using NServiceBus.Transport; + using NUnit.Framework; + using PersistenceTests; + using ServiceControl.Operations; + + class RetryConfirmationProcessorTests : PersistenceTestBase + { + RetryConfirmationProcessor Processor { get; set; } + LegacyMessageFailureResolvedHandler Handler { get; set; } + + [SetUp] + public new async Task Setup() + { + var domainEvents = new FakeDomainEvents(); + Processor = new RetryConfirmationProcessor(domainEvents); + + Handler = new LegacyMessageFailureResolvedHandler(ErrorMessageDataStore, domainEvents); + + await ErrorMessageDataStore.StoreFailedMessagesForTestsOnly( + new FailedMessage + { + Id = MessageId, + Status = FailedMessageStatus.Unresolved + } + ); + + // TODO: Strange... I just commented these lines and the tests are still green.... + + //var retryDocumentCommands = RetryDocumentDataStore.CreateFailedMessageRetryDocument(Guid.NewGuid().ToString(), MessageId); + //await GetRequiredService().AsyncDatabaseCommands.BatchAsync(new[] { retryDocumentCommands }); + } + + [Test] + public async Task Should_handle_multiple_retry_confirmations_in_the_error_ingestion() + { + var messageContexts = new List + { + CreateRetryAcknowledgementMessage(), + CreateRetryAcknowledgementMessage() + }; + + var unitOfWork = await UnitOfWorkFactory.StartNew(); + await Processor.Process(messageContexts, unitOfWork); + + Assert.DoesNotThrowAsync(() => unitOfWork.Complete()); + } + + [Test] + public async Task Should_handle_multiple_legacy_audit_instance_retry_confirmations() + { + await Handler.Handle(CreateLegacyRetryConfirmationCommand(), new TestableMessageHandlerContext()); + + Assert.DoesNotThrowAsync( + () => Handler.Handle(CreateLegacyRetryConfirmationCommand(), new TestableInvokeHandlerContext())); + } + + [Test] + public async Task Should_handle_retry_confirmation_followed_by_legacy_command() + { + var messageContexts = new List + { + CreateRetryAcknowledgementMessage() + }; + + var unitOfWork = await UnitOfWorkFactory.StartNew(); + await Processor.Process(messageContexts, unitOfWork); + await unitOfWork.Complete(); + + Assert.DoesNotThrowAsync( + () => Handler.Handle(CreateLegacyRetryConfirmationCommand(), new TestableInvokeHandlerContext())); + } + + [Test] + public async Task Should_handle_legacy_retry_confirmation_command_followed_by_new_acknowledgement() + { + await Handler.Handle(CreateLegacyRetryConfirmationCommand(), new TestableMessageHandlerContext()); + + var messageContexts = new List + { + CreateRetryAcknowledgementMessage() + }; + + var unitOfWork = await UnitOfWorkFactory.StartNew(); + await Processor.Process(messageContexts, unitOfWork); + Assert.DoesNotThrowAsync(() => unitOfWork.Complete()); + } + + static MarkMessageFailureResolvedByRetry CreateLegacyRetryConfirmationCommand() + { + var retryConfirmation = new MarkMessageFailureResolvedByRetry + { + FailedMessageId = MessageId + }; + return retryConfirmation; + } + + static MessageContext CreateRetryAcknowledgementMessage() + { + var headers = new Dictionary + { + {"ServiceControl.Retry.Successful", string.Empty}, + {"ServiceControl.Retry.UniqueMessageId", MessageId} + }; + var messageContext = new MessageContext( + MessageId, + headers, + Array.Empty(), + new TransportTransaction(), + new CancellationTokenSource(), + new ContextBag()); + return messageContext; + } + + const string MessageId = "83C73A86-A45E-4FDF-8C95-E292526166F5"; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/ReturnToSenderDequeuerTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/ReturnToSenderDequeuerTests.cs new file mode 100644 index 0000000000..0ac1e9d507 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/ReturnToSenderDequeuerTests.cs @@ -0,0 +1,224 @@ +namespace ServiceControl.UnitTests.Operations +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using MessageFailures; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Hosting; + using NServiceBus.Extensibility; + using NServiceBus.Transport; + using NUnit.Framework; + using Raven.Client; + using ServiceControl.CompositeViews.Messages; + using ServiceControl.Operations.BodyStorage; + using ServiceControl.PersistenceTests; + using ServiceControl.Recoverability; + + [TestFixture] + class ReturnToSenderDequeuerTests : PersistenceTestBase + { + MessageContext CreateMessage(string id, Dictionary headers) + { + return new MessageContext( + id, + headers, + new byte[0], + new TransportTransaction(), + new CancellationTokenSource(), + new ContextBag() + ); + } + + protected override IHostBuilder CreateHostBuilder() => base.CreateHostBuilder().ConfigureServices(services => services.AddSingleton()); + + [Test] + public async Task It_removes_staging_id_header() + { + var sender = new FakeSender(); + + var headers = new Dictionary + { + ["ServiceControl.Retry.StagingId"] = "SomeId", + ["ServiceControl.TargetEndpointAddress"] = "TargetEndpoint", + }; + var message = CreateMessage(Guid.NewGuid().ToString(), headers); + + await new ReturnToSender(new FakeBodyStorage(), null).HandleMessage(message, sender, "error"); + + Assert.IsFalse(sender.Message.Headers.ContainsKey("ServiceControl.Retry.StagingId")); + } + + [Test] + public async Task It_fetches_the_body_from_storage_if_provided() + { + var sender = new FakeSender(); + + var headers = new Dictionary + { + ["ServiceControl.Retry.StagingId"] = "SomeId", + ["ServiceControl.TargetEndpointAddress"] = "TargetEndpoint", + ["ServiceControl.Retry.Attempt.MessageId"] = "MessageBodyId", + }; + var message = CreateMessage(Guid.NewGuid().ToString(), headers); + + await new ReturnToSender(new FakeBodyStorage(), null).HandleMessage(message, sender, "error"); + + Assert.AreEqual("MessageBodyId", Encoding.UTF8.GetString(sender.Message.Body)); + } + + [Test] + public async Task It_fetches_the_body_from_index_if_provided() + { + var sender = new FakeSender(); + + var headers = new Dictionary + { + ["ServiceControl.Retry.StagingId"] = "SomeId", + ["ServiceControl.TargetEndpointAddress"] = "TargetEndpoint", + ["ServiceControl.Retry.Attempt.MessageId"] = "MessageBodyId", + ["ServiceControl.Retry.UniqueMessageId"] = "MessageBodyId", + ["ServiceControl.Retry.BodyOnFailedMessage"] = null + }; + var message = CreateMessage(Guid.NewGuid().ToString(), headers); + + var documentStore = GetRequiredService(); + + var failedMessage = new FailedMessage + { + Id = FailedMessageIdGenerator.MakeDocumentId("MessageBodyId"), + ProcessingAttempts = new List + { + new FailedMessage.ProcessingAttempt + { + MessageId = "MessageBodyId", + MessageMetadata = { { "Body", "MessageBodyId" } } + } + } + }; + + await ErrorMessageDataStore.StoreFailedMessagesForTestsOnly(failedMessage); + + var transformer = new MessagesBodyTransformer(); + await transformer.ExecuteAsync(documentStore); + + await CompleteDatabaseOperation(); + + var instance = GetRequiredService(); // See this.CreateHostBuilder + + // Acts + await instance.HandleMessage(message, sender, "error"); + + // Assert + Assert.AreEqual("MessageBodyId", Encoding.UTF8.GetString(sender.Message.Body)); + } + + [Test] + public async Task It_uses_retry_to_if_provided() + { + var sender = new FakeSender(); + + var headers = new Dictionary + { + ["ServiceControl.Retry.StagingId"] = "SomeId", + ["ServiceControl.TargetEndpointAddress"] = "TargetEndpoint", + ["ServiceControl.RetryTo"] = "Proxy", + }; + var message = CreateMessage(Guid.NewGuid().ToString(), headers); + + await new ReturnToSender(new FakeBodyStorage(), null).HandleMessage(message, sender, "error"); + + Assert.AreEqual("Proxy", sender.Destination); + Assert.AreEqual("TargetEndpoint", sender.Message.Headers["ServiceControl.TargetEndpointAddress"]); + } + + [Test] + public async Task It_sends_directly_to_target_if_retry_to_is_not_provided() + { + var sender = new FakeSender(); + + var headers = new Dictionary + { + ["ServiceControl.Retry.StagingId"] = "SomeId", + ["ServiceControl.TargetEndpointAddress"] = "TargetEndpoint", + }; + var message = CreateMessage(Guid.NewGuid().ToString(), headers); + + await new ReturnToSender(new FakeBodyStorage(), null).HandleMessage(message, sender, "error"); + + Assert.AreEqual("TargetEndpoint", sender.Destination); + Assert.IsFalse(sender.Message.Headers.ContainsKey("ServiceControl.TargetEndpointAddress")); + } + + [Test] + public async Task It_restores_body_id_and_target_addres_after_failure() + { + var sender = new FaultySender(); + + var headers = new Dictionary + { + ["ServiceControl.Retry.StagingId"] = "SomeId", + ["ServiceControl.TargetEndpointAddress"] = "TargetEndpoint", + ["ServiceControl.Retry.Attempt.MessageId"] = "MessageBodyId", + }; + var message = CreateMessage(Guid.NewGuid().ToString(), headers); + + try + { + await new ReturnToSender(new FakeBodyStorage(), null).HandleMessage(message, sender, "error"); + } + catch (Exception) + { + //Intentionally empty catch + } + + Assert.IsTrue(message.Headers.ContainsKey("ServiceControl.TargetEndpointAddress")); + Assert.IsTrue(message.Headers.ContainsKey("ServiceControl.Retry.Attempt.MessageId")); + } + + class FaultySender : IDispatchMessages + { + public Task Dispatch(TransportOperations outgoingMessages, TransportTransaction transaction, ContextBag context) + { + throw new Exception("Simulated"); + } + } + + class FakeSender : IDispatchMessages + { + public OutgoingMessage Message { get; private set; } + public string Destination { get; private set; } + + + public Task Dispatch(TransportOperations outgoingMessages, TransportTransaction transaction, ContextBag context) + { + var operation = outgoingMessages.UnicastTransportOperations.Single(); + Message = operation.Message; + Destination = operation.Destination; + return Task.FromResult(0); + } + } + + class FakeBodyStorage : IBodyStorage + { + public Task Store(string bodyId, string contentType, int bodySize, Stream bodyStream) + { + throw new NotImplementedException(); + } + + public Task TryFetch(string bodyId) + { + var stream = new MemoryStream(Encoding.UTF8.GetBytes(bodyId)); //Echo back the body ID. + return Task.FromResult(new MessageBodyStreamResult + { + HasResult = true, + Stream = stream + }); + } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/SagaAudit/SagaDetailsIndexTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/SagaAudit/SagaDetailsIndexTests.cs new file mode 100644 index 0000000000..e241a5c792 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/SagaAudit/SagaDetailsIndexTests.cs @@ -0,0 +1,80 @@ +namespace ServiceControl.UnitTests.SagaAudit +{ + using System; + using System.Collections.Generic; + using System.Linq; + using NUnit.Framework; + using Particular.Approvals; + using ServiceControl.SagaAudit; + + [TestFixture] + class SagaDetailsIndexTests + { + [Test] + public void RunMapReduce() + { + using (var store = InMemoryStoreBuilder.GetInMemoryStore()) + { + store.ExecuteIndex(new SagaDetailsIndex()); + using (var session = store.OpenSession()) + { + foreach (var sagaHistory in GetFakeHistory()) + { + session.Store(sagaHistory); + } + + session.SaveChanges(); + } + + store.WaitForIndexing(); + + using (var session = store.OpenSession()) + { + var mapReduceResults = session.Query() + .ToList(); + Approver.Verify(mapReduceResults); + } + } + } + + static IEnumerable GetFakeHistory() + { + yield return new SagaSnapshot + { + SagaId = Guid.Empty, + SagaType = "MySaga1", + Endpoint = "MyEndpoint", + FinishTime = new DateTime(2001, 1, 1, 1, 1, 1, DateTimeKind.Utc), + Status = SagaStateChangeStatus.Updated, + StartTime = new DateTime(2001, 1, 1, 1, 1, 1, DateTimeKind.Utc), + StateAfterChange = "Updated" + }; + yield return new SagaHistory + { + SagaId = Guid.Empty, + SagaType = "MySaga1", + Changes = new List + { + new SagaStateChange + { + Endpoint = "MyEndpoint", + FinishTime = new DateTime(2000, 1, 1, 1, 1, 1, DateTimeKind.Utc), + Status = SagaStateChangeStatus.Updated, + StartTime = new DateTime(2000, 1, 1, 1, 1, 1, DateTimeKind.Utc), + StateAfterChange = "Started" + } + } + }; + yield return new SagaSnapshot + { + SagaId = Guid.Empty, + SagaType = "MySaga1", + Endpoint = "MyEndpoint", + FinishTime = new DateTime(2002, 1, 1, 1, 1, 1, DateTimeKind.Utc), + Status = SagaStateChangeStatus.Completed, + StartTime = new DateTime(2002, 1, 1, 1, 1, 1, DateTimeKind.Utc), + StateAfterChange = "Completed" + }; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/SagaAudit/SagaListIndexTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/SagaAudit/SagaListIndexTests.cs new file mode 100644 index 0000000000..788a0a08bd --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/SagaAudit/SagaListIndexTests.cs @@ -0,0 +1,68 @@ +namespace ServiceControl.UnitTests.SagaAudit +{ + using System; + using System.Collections.Generic; + using System.Linq; + using NUnit.Framework; + using Particular.Approvals; + using ServiceControl.SagaAudit; + + [TestFixture] + class SagaListIndexTests + { + [Test] + public void RunMapReduce() + { + using (var store = InMemoryStoreBuilder.GetInMemoryStore()) + { + store.ExecuteIndex(new SagaListIndex()); + using (var session = store.OpenSession()) + { + foreach (var sagaHistory in GetFakeHistory()) + { + session.Store(sagaHistory); + } + + session.SaveChanges(); + } + + store.WaitForIndexing(); + + using (var session = store.OpenSession()) + { + var mapReduceResults = session.Query() + .ToList(); + Approver.Verify(mapReduceResults); + } + } + } + + static IEnumerable GetFakeHistory() + { + yield return new SagaSnapshot + { + SagaId = new Guid("00000000-0000-0000-0000-000000000003"), + SagaType = "MySaga3", + FinishTime = new DateTime(2000, 1, 1, 10, 0, 0) + }; + yield return new SagaHistory + { + SagaId = new Guid("00000000-0000-0000-0000-000000000001"), + SagaType = "MySaga1", + Changes = new List + { + new SagaStateChange + { + FinishTime = new DateTime(2000, 1, 1, 1, 1, 1, DateTimeKind.Utc) + } + } + }; + yield return new SagaSnapshot + { + SagaId = new Guid("00000000-0000-0000-0000-000000000002"), + SagaType = "MySaga2", + FinishTime = new DateTime(2000, 1, 1, 15, 0, 0) + }; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/ServiceControl.Persistence.Tests.RavenDb5.csproj b/src/ServiceControl.Persistence.Tests.RavenDb5/ServiceControl.Persistence.Tests.RavenDb5.csproj new file mode 100644 index 0000000000..15bcfda682 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/ServiceControl.Persistence.Tests.RavenDb5.csproj @@ -0,0 +1,32 @@ + + + + net472 + ServiceControl.runsettings + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/ServiceControl.runsettings b/src/ServiceControl.Persistence.Tests.RavenDb5/ServiceControl.runsettings new file mode 100644 index 0000000000..ea2e767f89 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/ServiceControl.runsettings @@ -0,0 +1,7 @@ + + + + + x64 + + diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/SubscriptionPersisterTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/SubscriptionPersisterTests.cs new file mode 100644 index 0000000000..297076bfb6 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/SubscriptionPersisterTests.cs @@ -0,0 +1,70 @@ +namespace ServiceControl.UnitTests.Infrastructure.RavenDB +{ + using System; + using System.Linq; + using System.Threading.Tasks; + using NServiceBus.Extensibility; + using NServiceBus.Unicast.Subscriptions; + using NServiceBus.Unicast.Subscriptions.MessageDrivenSubscriptions; + using NUnit.Framework; + using Raven.Client; + using ServiceControl.Infrastructure.RavenDB.Subscriptions; + + [TestFixture] + public class SubscriptionPersisterTests + { + [Test] + public async Task ShouldReturnSubscriptionsForOlderVersionsOfSameMessageType() + { + var subscriptionPersister = new RavenDbSubscriptionStorage(documentStore, "NServiceBus.Routing.EndpointName", "TestEndpoint", new MessageType[0]); + + var v1MessageType = new MessageType(typeof(SampleMessageType).FullName, new Version(1, 0, 0)); + var v2MessageType = new MessageType(typeof(SampleMessageType).FullName, new Version(2, 0, 0)); + var v1Subscriber = new Subscriber("V1SubscriberAddress", "V1Subscriber"); + + await subscriptionPersister.Subscribe(v1Subscriber, v1MessageType, new ContextBag()); + + var foundSubscriptions = await subscriptionPersister.GetSubscriberAddressesForMessage(new[] { v2MessageType }, new ContextBag()); + + var foundSubscriber = foundSubscriptions.Single(); + Assert.AreEqual(v1Subscriber.Endpoint, foundSubscriber.Endpoint); + Assert.AreEqual(v1Subscriber.TransportAddress, foundSubscriber.TransportAddress); + } + + [Test] + public async Task ShouldReturnSubscriptionsForNewerVersionsOfSameMessageType() + { + var subscriptionPersister = new RavenDbSubscriptionStorage(documentStore, "NServiceBus.Routing.EndpointName", "TestEndpoint", new MessageType[0]); + + var v1MessageType = new MessageType(typeof(SampleMessageType).FullName, new Version(1, 0, 0)); + var v2MessageType = new MessageType(typeof(SampleMessageType).FullName, new Version(2, 0, 0)); + var v2Subscriber = new Subscriber("V2SubscriberAddress", "V2Subscriber"); + + await subscriptionPersister.Subscribe(v2Subscriber, v2MessageType, new ContextBag()); + + var foundSubscriptions = await subscriptionPersister.GetSubscriberAddressesForMessage(new[] { v1MessageType }, new ContextBag()); + + var foundSubscriber = foundSubscriptions.Single(); + Assert.AreEqual(v2Subscriber.Endpoint, foundSubscriber.Endpoint); + Assert.AreEqual(v2Subscriber.TransportAddress, foundSubscriber.TransportAddress); + } + + [SetUp] + public void SetUp() + { + documentStore = InMemoryStoreBuilder.GetInMemoryStore(); + } + + [TearDown] + public void TearDown() + { + documentStore.Dispose(); + } + + IDocumentStore documentStore; + } + + public class SampleMessageType + { + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs new file mode 100644 index 0000000000..20abc166c5 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs @@ -0,0 +1,99 @@ +namespace ServiceControl.PersistenceTests +{ + using System; + using System.Diagnostics; + using System.Runtime.InteropServices; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Hosting; + using NUnit.Framework; + using Persistence; + using Raven.Client; + using ServiceControl.Persistence.RavenDb; + + sealed class TestPersistenceImpl : TestPersistence + { + readonly RavenDBPersisterSettings settings = CreateSettings(); + IDocumentStore documentStore; + + static RavenDBPersisterSettings CreateSettings() + { + var retentionPeriod = TimeSpan.FromMinutes(1); + + var settings = new RavenDBPersisterSettings + { + AuditRetentionPeriod = retentionPeriod, + ErrorRetentionPeriod = retentionPeriod, + EventsRetentionPeriod = retentionPeriod, + RunInMemory = true, + }; + + if (Debugger.IsAttached) + { + Console.WriteLine("If you get 'Access is denied' exception while debugging, comment out this line or create a URLACL reservervation:"); + Console.WriteLine("> netsh http add urlacl http://+:55554/ user=Everyone"); + settings.ExposeRavenDB = true; + } + + return settings; + } + + public override void Configure(IServiceCollection services) + { + var persistence = new RavenDbPersistenceConfiguration().Create(CreateSettings()); + + PersistenceHostBuilderExtensions.CreatePersisterLifecyle(services, persistence); + + services.AddHostedService(p => new Wrapper(this, p.GetRequiredService())); + } + + public override Task CompleteDatabaseOperation() + { + Assert.IsNotNull(documentStore); + documentStore.WaitForIndexing(); + return Task.CompletedTask; + } + + class Wrapper : IHostedService + { + public Wrapper(TestPersistenceImpl instance, IDocumentStore store) + { + instance.documentStore = store; + } + + public Task StartAsync(CancellationToken cancellationToken) => Task.CompletedTask; + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; + } + + public override void BlockToInspectDatabase() + { + if (!Debugger.IsAttached) + { + return; + } + + var url = $"http://localhost:{settings.DatabaseMaintenancePort}/studio/index.html#databases/documents?&database=%3Csystem%3E"; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + url = url.Replace("&", "^&"); + Process.Start(new ProcessStartInfo(url) { UseShellExecute = true }); + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + Process.Start("xdg-open", url); + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + Process.Start("open", url); + } + + while (true) + { + Thread.Sleep(5000); + Trace.Write("Waiting for debugger pause"); + } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Unarchiving/UnarchiveGroupTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Unarchiving/UnarchiveGroupTests.cs new file mode 100644 index 0000000000..34e47ffeab --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Unarchiving/UnarchiveGroupTests.cs @@ -0,0 +1,115 @@ +namespace ServiceControl.UnitTests.Archiving +{ + using System; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Hosting; + using NServiceBus.Testing; + using NUnit.Framework; + using PersistenceTests; + using Raven.Client; + using ServiceControl.Recoverability; + + [TestFixture] + class UnarchiveGroupTests : PersistenceTestBase + { + IDocumentStore DocumentStore => GetRequiredService(); + + protected override IHostBuilder CreateHostBuilder() => base.CreateHostBuilder().ConfigureServices(services => + { + services.AddSingleton(); + services.AddSingleton(); + }); + + [Test] + public async Task UnarchiveGroup_skips_over_empty_batches_but_still_completes() + { + // Arrange + var groupId = "TestGroup"; + var previousUnarchiveBatchId = UnarchiveBatch.MakeId(groupId, ArchiveType.FailureGroup, 1); + + using (var session = DocumentStore.OpenAsyncSession()) + { + var previousUnarchiveBatch = new UnarchiveBatch { Id = previousUnarchiveBatchId }; + await session.StoreAsync(previousUnarchiveBatch); + + var previousUnarchiveOperation = new UnarchiveOperation + { + Id = UnarchiveOperation.MakeId(groupId, ArchiveType.FailureGroup), + RequestId = groupId, + ArchiveType = ArchiveType.FailureGroup, + TotalNumberOfMessages = 2, + NumberOfMessagesUnarchived = 2, + Started = DateTime.Now, + GroupName = "Test Group", + NumberOfBatches = 3, + CurrentBatch = 0 + }; + await session.StoreAsync(previousUnarchiveOperation); + + await session.SaveChangesAsync(); + } + + var handler = GetRequiredService(); // See this.CreateHostBuilder + + var context = new TestableMessageHandlerContext(); + var message = new UnarchiveAllInGroup { GroupId = groupId }; + + // Act + await handler.Handle(message, context); + + // Assert + using (var session = DocumentStore.OpenSession()) + { + var loadedBatch = session.Load(previousUnarchiveBatchId); + Assert.IsNull(loadedBatch); + } + } + + [Test] + public async Task UnarchiveGroup_GetGroupDetails_doesnt_fail_with_invalid_groupId() + { + // Arrange + var failureGroupsViewIndex = new ArchivedGroupsViewIndex(); + await failureGroupsViewIndex.ExecuteAsync(DocumentStore); + + var groupId = "TestGroup"; + var previousUnarchiveBatchId = UnarchiveBatch.MakeId(groupId, ArchiveType.FailureGroup, 1); + + using (var session = DocumentStore.OpenAsyncSession()) + { + var previousUnarchiveBatch = new UnarchiveBatch { Id = previousUnarchiveBatchId }; + await session.StoreAsync(previousUnarchiveBatch); + + var previousUnarchiveOperation = new UnarchiveOperation + { + Id = UnarchiveOperation.MakeId(groupId, ArchiveType.FailureGroup), + RequestId = groupId, + ArchiveType = ArchiveType.FailureGroup, + TotalNumberOfMessages = 2, + NumberOfMessagesUnarchived = 0, + Started = DateTime.Now, + GroupName = "Test Group", + NumberOfBatches = 3, + CurrentBatch = 0 + }; + await session.StoreAsync(previousUnarchiveOperation); + + await session.SaveChangesAsync(); + } + + var handler = GetRequiredService(); // See this.CreateHostBuilder + + var context = new TestableMessageHandlerContext(); + var message = new UnarchiveAllInGroup { GroupId = groupId + "Invalid" }; + + // Act + // Assert + Assert.DoesNotThrowAsync(async () => + { + // Act + await handler.Handle(message, context); + }); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/app.config b/src/ServiceControl.Persistence.Tests.RavenDb5/app.config new file mode 100644 index 0000000000..03d8dcadf7 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/app.config @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/ServiceControl.sln b/src/ServiceControl.sln index b4779e2aa7..8c3009e173 100644 --- a/src/ServiceControl.sln +++ b/src/ServiceControl.sln @@ -183,6 +183,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ServiceControl.AcceptanceTe EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ServiceControl.Persistence.RavenDb5", "ServiceControl.Persistence.RavenDb5\ServiceControl.Persistence.RavenDb5.csproj", "{84627DC0-7B43-480D-80CF-A4DDA514A4EE}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ServiceControl.Persistence.Tests.RavenDb5", "ServiceControl.Persistence.Tests.RavenDb5\ServiceControl.Persistence.Tests.RavenDb5.csproj", "{2EF143A5-E769-42FD-8B7E-7C01C381BC13}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1029,6 +1031,18 @@ Global {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Release|x64.Build.0 = Release|Any CPU {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Release|x86.ActiveCfg = Release|Any CPU {84627DC0-7B43-480D-80CF-A4DDA514A4EE}.Release|x86.Build.0 = Release|Any CPU + {2EF143A5-E769-42FD-8B7E-7C01C381BC13}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2EF143A5-E769-42FD-8B7E-7C01C381BC13}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2EF143A5-E769-42FD-8B7E-7C01C381BC13}.Debug|x64.ActiveCfg = Debug|Any CPU + {2EF143A5-E769-42FD-8B7E-7C01C381BC13}.Debug|x64.Build.0 = Debug|Any CPU + {2EF143A5-E769-42FD-8B7E-7C01C381BC13}.Debug|x86.ActiveCfg = Debug|Any CPU + {2EF143A5-E769-42FD-8B7E-7C01C381BC13}.Debug|x86.Build.0 = Debug|Any CPU + {2EF143A5-E769-42FD-8B7E-7C01C381BC13}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2EF143A5-E769-42FD-8B7E-7C01C381BC13}.Release|Any CPU.Build.0 = Release|Any CPU + {2EF143A5-E769-42FD-8B7E-7C01C381BC13}.Release|x64.ActiveCfg = Release|Any CPU + {2EF143A5-E769-42FD-8B7E-7C01C381BC13}.Release|x64.Build.0 = Release|Any CPU + {2EF143A5-E769-42FD-8B7E-7C01C381BC13}.Release|x86.ActiveCfg = Release|Any CPU + {2EF143A5-E769-42FD-8B7E-7C01C381BC13}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1112,6 +1126,7 @@ Global {F2BD40E4-A077-429A-8A22-1C80AFC240E8} = {350F72AB-142D-4AAD-9EF1-1A83DC991D87} {33D5D084-FABB-4F65-A046-1C6626DF9AFB} = {350F72AB-142D-4AAD-9EF1-1A83DC991D87} {84627DC0-7B43-480D-80CF-A4DDA514A4EE} = {9B52418E-BF18-4D25-BE17-4B56D3FB1154} + {2EF143A5-E769-42FD-8B7E-7C01C381BC13} = {350F72AB-142D-4AAD-9EF1-1A83DC991D87} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {3B9E5B72-F580-465A-A22C-2D2148AF4EB4} From d57736f61257fa6351e95b39aa02b795848d8576 Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 14 Sep 2023 15:39:07 -0500 Subject: [PATCH 02/94] Switch reference to Raven5 persistence --- .../ServiceControl.Persistence.Tests.RavenDb5.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/ServiceControl.Persistence.Tests.RavenDb5.csproj b/src/ServiceControl.Persistence.Tests.RavenDb5/ServiceControl.Persistence.Tests.RavenDb5.csproj index 15bcfda682..f846159034 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/ServiceControl.Persistence.Tests.RavenDb5.csproj +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/ServiceControl.Persistence.Tests.RavenDb5.csproj @@ -6,7 +6,7 @@ - + From cd33a3ccbbb430c41d0779ba3d33fed29ee5345d Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 14 Sep 2023 15:48:42 -0500 Subject: [PATCH 03/94] Namespaces & InternalsVisibleTo --- src/ServiceControl.Persistence.RavenDb5/InternalsVisibleTo.cs | 4 ++-- .../Archiving/ArchiveGroupTests.cs | 2 +- .../CompositeViews/FailedMessagesTests.cs | 2 +- .../CompositeViews/MessagesViewTests.cs | 4 ++-- .../Expiration/ProcessedMessageExpirationTests.cs | 2 +- .../Expiration/RavenLastModifiedScope.cs | 2 +- .../Infrastructure/RavenIndexAwaiter.cs | 2 +- .../MigrationTests.cs | 4 ++-- .../Operations/FailedErrorImportCustomCheckTests.cs | 2 +- .../RavenPersistedTypes.cs | 2 +- .../Recoverability/ReturnToSenderDequeuerTests.cs | 2 +- .../SubscriptionPersisterTests.cs | 2 +- .../TestPersistenceImpl.cs | 2 +- .../Unarchiving/UnarchiveGroupTests.cs | 2 +- src/ServiceControl/InternalsVisibleTo.cs | 2 ++ 15 files changed, 19 insertions(+), 17 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/InternalsVisibleTo.cs b/src/ServiceControl.Persistence.RavenDb5/InternalsVisibleTo.cs index cbd15879aa..4257086e7f 100644 --- a/src/ServiceControl.Persistence.RavenDb5/InternalsVisibleTo.cs +++ b/src/ServiceControl.Persistence.RavenDb5/InternalsVisibleTo.cs @@ -2,6 +2,6 @@ [assembly: InternalsVisibleTo("ServiceControl.UnitTests")] [assembly: InternalsVisibleTo("ServiceControl.PersistenceTests")] -[assembly: InternalsVisibleTo("ServiceControl.Persistence.Tests.RavenDb")] -[assembly: InternalsVisibleTo("ServiceControl.AcceptanceTests.RavenDB")] +[assembly: InternalsVisibleTo("ServiceControl.Persistence.Tests.RavenDb5")] +[assembly: InternalsVisibleTo("ServiceControl.AcceptanceTests.RavenDB5")] [assembly: InternalsVisibleTo("ServiceControl.MultiInstance.AcceptanceTests")] diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Archiving/ArchiveGroupTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Archiving/ArchiveGroupTests.cs index 41b1cb33c1..616bb9cea7 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/Archiving/ArchiveGroupTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Archiving/ArchiveGroupTests.cs @@ -6,7 +6,7 @@ using Microsoft.Extensions.Hosting; using NServiceBus.Testing; using NUnit.Framework; - using Raven.Client; + using Raven.Client.Documents; using ServiceControl.PersistenceTests; using ServiceControl.Recoverability; diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/FailedMessagesTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/FailedMessagesTests.cs index b269299886..1588db7f9f 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/FailedMessagesTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/FailedMessagesTests.cs @@ -8,7 +8,7 @@ using MessageFailures; using MessageFailures.Api; using NUnit.Framework; - using Raven.Client; + using Raven.Client.Documents; [TestFixture] public class FailedMessagesTests diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs index 65eb6714be..783944fad2 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs @@ -6,8 +6,8 @@ using MessageAuditing; using MessageFailures; using NUnit.Framework; - using Raven.Client; - using Raven.Client.Linq; + using Raven.Client.Documents; + using Raven.Client.Documents.Linq; using ServiceControl.CompositeViews.Messages; using ServiceControl.Persistence; diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/ProcessedMessageExpirationTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/ProcessedMessageExpirationTests.cs index 1425366ee4..98f8a8748e 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/ProcessedMessageExpirationTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/ProcessedMessageExpirationTests.cs @@ -11,7 +11,7 @@ using MessageFailures; using NUnit.Framework; using PersistenceTests; - using Raven.Client; + using Raven.Client.Documents; using Raven.Client.Embedded; using ServiceControl.Infrastructure.RavenDB.Expiration; using ServiceControl.Operations.BodyStorage.RavenAttachments; diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/RavenLastModifiedScope.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/RavenLastModifiedScope.cs index ae3735f1e0..1e65ffffda 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/RavenLastModifiedScope.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/RavenLastModifiedScope.cs @@ -1,7 +1,7 @@ namespace ServiceControl.UnitTests.Expiration { using System; - using Raven.Abstractions; + using Raven.Client.Util; public class RavenLastModifiedScope : IDisposable { diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/RavenIndexAwaiter.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/RavenIndexAwaiter.cs index 26ec4d1a83..48b5ce14c7 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/RavenIndexAwaiter.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/RavenIndexAwaiter.cs @@ -1,7 +1,7 @@ using System; using System.Threading; using NUnit.Framework; -using Raven.Client; +using Raven.Client.Documents; public static class RavenIndexAwaiter { diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/MigrationTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/MigrationTests.cs index 3d08231c90..a72931c15b 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/MigrationTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/MigrationTests.cs @@ -1,6 +1,6 @@ using System.IO; +using Newtonsoft.Json; using NUnit.Framework; -using Raven.Imports.Newtonsoft.Json; using ServiceControl.Infrastructure.RavenDB; [TestFixture] @@ -18,7 +18,7 @@ public void VerifyMigration() var serializer = new JsonSerializer { TypeNameHandling = TypeNameHandling.All, - Binder = new MigratedTypeAwareBinder() + SerializationBinder = new MigratedTypeAwareBinder() }; serializer.Deserialize(new JsonTextReader(new StringReader(serializedForm))); diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedErrorImportCustomCheckTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedErrorImportCustomCheckTests.cs index 0cde2d8569..8a1d037a56 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedErrorImportCustomCheckTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedErrorImportCustomCheckTests.cs @@ -6,7 +6,7 @@ using NServiceBus.CustomChecks; using NUnit.Framework; using PersistenceTests; - using Raven.Client; + using Raven.Client.Documents; using ServiceControl.Operations; [TestFixture] diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/RavenPersistedTypes.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/RavenPersistedTypes.cs index 34c7d260a1..cee651e9ae 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/RavenPersistedTypes.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/RavenPersistedTypes.cs @@ -5,7 +5,7 @@ using NUnit.Framework; using Particular.Approvals; using Particular.ServiceControl; - using Raven.Client.Indexes; + using Raven.Client.Documents.Indexes; using ServiceControl.Persistence; class RavenPersistedTypes diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/ReturnToSenderDequeuerTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/ReturnToSenderDequeuerTests.cs index 0ac1e9d507..b977fc04e3 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/ReturnToSenderDequeuerTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/ReturnToSenderDequeuerTests.cs @@ -13,7 +13,7 @@ using NServiceBus.Extensibility; using NServiceBus.Transport; using NUnit.Framework; - using Raven.Client; + using Raven.Client.Documents; using ServiceControl.CompositeViews.Messages; using ServiceControl.Operations.BodyStorage; using ServiceControl.PersistenceTests; diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/SubscriptionPersisterTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/SubscriptionPersisterTests.cs index 297076bfb6..c58d752e22 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/SubscriptionPersisterTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/SubscriptionPersisterTests.cs @@ -7,7 +7,7 @@ using NServiceBus.Unicast.Subscriptions; using NServiceBus.Unicast.Subscriptions.MessageDrivenSubscriptions; using NUnit.Framework; - using Raven.Client; + using Raven.Client.Documents; using ServiceControl.Infrastructure.RavenDB.Subscriptions; [TestFixture] diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs index 20abc166c5..baa90e2a09 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs @@ -9,7 +9,7 @@ using Microsoft.Extensions.Hosting; using NUnit.Framework; using Persistence; - using Raven.Client; + using Raven.Client.Documents; using ServiceControl.Persistence.RavenDb; sealed class TestPersistenceImpl : TestPersistence diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Unarchiving/UnarchiveGroupTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Unarchiving/UnarchiveGroupTests.cs index 34e47ffeab..39485d4e75 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/Unarchiving/UnarchiveGroupTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Unarchiving/UnarchiveGroupTests.cs @@ -7,7 +7,7 @@ using NServiceBus.Testing; using NUnit.Framework; using PersistenceTests; - using Raven.Client; + using Raven.Client.Documents; using ServiceControl.Recoverability; [TestFixture] diff --git a/src/ServiceControl/InternalsVisibleTo.cs b/src/ServiceControl/InternalsVisibleTo.cs index e8c4d6a386..e3c415fb3a 100644 --- a/src/ServiceControl/InternalsVisibleTo.cs +++ b/src/ServiceControl/InternalsVisibleTo.cs @@ -3,8 +3,10 @@ [assembly: InternalsVisibleTo("ServiceControl.UnitTests")] [assembly: InternalsVisibleTo("ServiceControl.AcceptanceTests")] [assembly: InternalsVisibleTo("ServiceControl.AcceptanceTests.RavenDB")] +[assembly: InternalsVisibleTo("ServiceControl.AcceptanceTests.RavenDB5")] [assembly: InternalsVisibleTo("ServiceControl.AcceptanceTesting")] [assembly: InternalsVisibleTo("ServiceControl.PersistenceTests")] [assembly: InternalsVisibleTo("ServiceControl.Persistence.Tests.RavenDB")] +[assembly: InternalsVisibleTo("ServiceControl.Persistence.Tests.RavenDB5")] [assembly: InternalsVisibleTo("ServiceControl.MultiInstance.AcceptanceTests")] [assembly: InternalsVisibleTo("ServiceControl.Persistence.RavenDb")] From a70f6f4449850ccf61359e09e90e9ab714ba704d Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 14 Sep 2023 15:52:36 -0500 Subject: [PATCH 04/94] Don't need tests for expiring audit & saga audit messages --- .../ProcessedMessageExpirationTests.cs | 256 ------------------ .../Expiration/SagaAuditExpirationTests.cs | 205 -------------- 2 files changed, 461 deletions(-) delete mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/ProcessedMessageExpirationTests.cs delete mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/SagaAuditExpirationTests.cs diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/ProcessedMessageExpirationTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/ProcessedMessageExpirationTests.cs deleted file mode 100644 index 98f8a8748e..0000000000 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/ProcessedMessageExpirationTests.cs +++ /dev/null @@ -1,256 +0,0 @@ -namespace ServiceControl.UnitTests.Expiration -{ - using System; - using System.Collections.Generic; - using System.Diagnostics; - using System.IO; - using System.Linq; - using System.Threading; - using System.Threading.Tasks; - using MessageAuditing; - using MessageFailures; - using NUnit.Framework; - using PersistenceTests; - using Raven.Client.Documents; - using Raven.Client.Embedded; - using ServiceControl.Infrastructure.RavenDB.Expiration; - using ServiceControl.Operations.BodyStorage.RavenAttachments; - - sealed class ProcessedMessageExpirationTests : PersistenceTestBase - { - IDocumentStore DocumentStore => GetRequiredService(); - - [Test] - public void Old_documents_are_being_expired() - { - var expiredDate = DateTime.UtcNow.AddDays(-3); - var thresholdDate = DateTime.UtcNow.AddDays(-2); - var processedMessage = new ProcessedMessage - { - Id = "1", - ProcessedAt = expiredDate - }; - - using (var session = DocumentStore.OpenSession()) - { - session.Store(processedMessage); - session.SaveChanges(); - } - - RunExpiry(thresholdDate); - - using (var session = DocumentStore.OpenSession()) - { - Assert.IsEmpty(session.Query()); - } - } - - [Test] - public void Many_documents_are_being_expired() - { - var expiredDate = DateTime.UtcNow.AddDays(-3); - var thresholdDate = DateTime.UtcNow.AddDays(-2); - var recentDate = DateTime.UtcNow.AddDays(-1); - var expiredMessages = BuildExpiredMessaged(expiredDate).ToList(); - using (var session = DocumentStore.OpenSession()) - { - foreach (var message in expiredMessages) - { - session.Store(message); - } - - session.SaveChanges(); - } - - using (var session = DocumentStore.OpenSession()) - { - var recentMessage = new ProcessedMessage - { - Id = "recentMessageId", - ProcessedAt = recentDate - }; - session.Store(recentMessage); - session.SaveChanges(); - } - - RunExpiry(thresholdDate); - foreach (dynamic message in expiredMessages) - { - using (var session = DocumentStore.OpenSession()) - { - Assert.Null(session.Load(message.Id)); - } - } - - using (var session = DocumentStore.OpenSession()) - { - Assert.AreEqual(1, session.Query().Count()); - } - } - - IEnumerable BuildExpiredMessaged(DateTime dateTime) - { - for (var i = 0; i < DocTestRange; i++) - { - yield return new ProcessedMessage - { - Id = Guid.NewGuid().ToString(), - ProcessedAt = dateTime - }; - } - } - - void RunExpiry(DateTime expiryThreshold) - { - var documentStore = DocumentStore; - var embeddableDocumentStore = (EmbeddableDocumentStore)documentStore; - - new ExpiryProcessedMessageIndex().Execute(documentStore); - documentStore.WaitForIndexing(); - AuditMessageCleaner.Clean(DocTestRange, embeddableDocumentStore.DocumentDatabase, expiryThreshold); - documentStore.WaitForIndexing(); - } - - [Test] - public void Only_processed_messages_are_being_expired() - { - var expiredDate = DateTime.UtcNow.AddDays(-3); - var thresholdDate = DateTime.UtcNow.AddDays(-2); - var recentDate = DateTime.UtcNow.AddDays(-1); - var expiredMessage = new ProcessedMessage - { - Id = "1", - ProcessedAt = expiredDate - }; - - using (var session = DocumentStore.OpenSession()) - { - session.Store(expiredMessage); - session.SaveChanges(); - } - - var recentMessage = new ProcessedMessage - { - Id = "2", - ProcessedAt = recentDate - }; - using (var session = DocumentStore.OpenSession()) - { - session.Store(recentMessage); - session.SaveChanges(); - } - - RunExpiry(thresholdDate); - - using (var session = DocumentStore.OpenSession()) - { - Assert.Null(session.Load(expiredMessage.Id)); - Assert.NotNull(session.Load(recentMessage.Id)); - } - } - - [Test] - public async Task Stored_bodies_are_being_removed_when_message_expires() - { - var expiredDate = DateTime.UtcNow.AddDays(-3); - var thresholdDate = DateTime.UtcNow.AddDays(-2); - // Store expired message with associated body - var messageId = "21"; - - var processedMessage = new ProcessedMessage - { - Id = "1", - ProcessedAt = expiredDate, - MessageMetadata = new Dictionary - { - { - "MessageId", messageId - } - } - }; - - using (var session = DocumentStore.OpenSession()) - { - session.Store(processedMessage); - session.SaveChanges(); - } - - var body = new byte[] - { - 1, - 2, - 3, - 4, - 5 - }; - - var bodyStorage = new RavenAttachmentsBodyStorage(DocumentStore); - using (var stream = new MemoryStream(body)) - { - await bodyStorage.Store(messageId, "binary", 5, stream); - } - - RunExpiry(thresholdDate); - - // Verify message expired - using (var session = DocumentStore.OpenSession()) - { - Assert.Null(session.Load(processedMessage.Id)); - } - - var result = await bodyStorage.TryFetch(messageId); - // Verify body expired - Assert.Null(result, "Audit document body should be deleted"); - } - - [Test] - public void Recent_processed_messages_are_not_being_expired() - { - var thresholdDate = DateTime.UtcNow.AddDays(-2); - var recentDate = DateTime.UtcNow.AddDays(-1); - var message = new ProcessedMessage - { - Id = "1", - ProcessedAt = recentDate - }; - using (var session = DocumentStore.OpenSession()) - { - session.Store(message); - session.SaveChanges(); - } - - RunExpiry(thresholdDate); - using (var session = DocumentStore.OpenSession()) - { - Assert.AreEqual(1, session.Query().Count()); - } - } - - [Test] - public void Errors_are_not_being_expired() - { - var failedMsg = new FailedMessage - { - Id = "1" - }; - - using (var session = DocumentStore.OpenSession()) - { - session.Store(failedMsg); - session.SaveChanges(); - - Debug.WriteLine(session.Advanced.GetMetadataFor(failedMsg)["Last-Modified"]); - } - - Thread.Sleep(100); - RunExpiry(DateTime.UtcNow); - - using (var session = DocumentStore.OpenSession()) - { - Assert.NotNull(session.Load(failedMsg.Id)); - } - } - - const int DocTestRange = 999; - } -} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/SagaAuditExpirationTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/SagaAuditExpirationTests.cs deleted file mode 100644 index 02a17193d5..0000000000 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/SagaAuditExpirationTests.cs +++ /dev/null @@ -1,205 +0,0 @@ -namespace ServiceControl.UnitTests.Expiration -{ - using System; - using System.Collections.Generic; - using System.Diagnostics; - using System.Linq; - using System.Threading; - using MessageFailures; - using NUnit.Framework; - using Raven.Client.Embedded; - using ServiceControl.SagaAudit; - - [TestFixture] - public class SagaAuditExpirationTests - { - [Test] - public void Old_documents_are_being_expired() - { - using (var documentStore = InMemoryStoreBuilder.GetInMemoryStore()) - { - var expiredDate = DateTime.UtcNow.AddDays(-3); - var thresholdDate = DateTime.UtcNow.AddDays(-2); - - var sagaHistoryId = Guid.NewGuid().ToString(); - var sagaHistory = new SagaSnapshot - { - Id = sagaHistoryId - }; - - using (new RavenLastModifiedScope(expiredDate)) - using (var session = documentStore.OpenSession()) - { - session.Store(sagaHistory); - session.SaveChanges(); - } - - RunExpiry(documentStore, thresholdDate); - - using (var session = documentStore.OpenSession()) - { - Assert.IsEmpty(session.Query()); - } - } - } - - [Test] - public void Many_documents_are_being_expired() - { - using (var documentStore = InMemoryStoreBuilder.GetInMemoryStore()) - { - var expiredDate = DateTime.UtcNow.AddDays(-3); - var thresholdDate = DateTime.UtcNow.AddDays(-2); - var recentDate = DateTime.UtcNow.AddDays(-1); - var expiredMessages = BuilExpiredMessaged().ToList(); - using (new RavenLastModifiedScope(expiredDate)) - { - using (var session = documentStore.OpenSession()) - { - foreach (var message in expiredMessages) - { - session.Store(message); - } - - session.SaveChanges(); - } - } - - using (new RavenLastModifiedScope(recentDate)) - using (var session = documentStore.OpenSession()) - { - var recentSagaHistory = new SagaSnapshot - { - Id = Guid.NewGuid().ToString() - }; - session.Store(recentSagaHistory); - session.SaveChanges(); - } - - RunExpiry(documentStore, thresholdDate); - - using (var session = documentStore.OpenSession()) - { - Assert.AreEqual(1, session.Query().Count()); - } - } - } - - IEnumerable BuilExpiredMessaged() - { - for (var i = 0; i < 10; i++) - { - yield return new SagaSnapshot - { - Id = Guid.NewGuid().ToString() - }; - } - } - - static void RunExpiry(EmbeddableDocumentStore documentStore, DateTime expiryThreshold) - { - new ExpirySagaAuditIndex().Execute(documentStore); - documentStore.WaitForIndexing(); - SagaHistoryCleaner.Clean(100, documentStore.DocumentDatabase, expiryThreshold); - documentStore.WaitForIndexing(); - } - - [Test] - public void Only_expired_being_deleted() - { - using (var documentStore = InMemoryStoreBuilder.GetInMemoryStore()) - { - var expiredDate = DateTime.UtcNow.AddDays(-3); - var thresholdDate = DateTime.UtcNow.AddDays(-2); - var recentDate = DateTime.UtcNow.AddDays(-1); - - var expiredSagaHistory = new SagaSnapshot - { - Id = Guid.NewGuid().ToString() - }; - - using (new RavenLastModifiedScope(expiredDate)) - using (var session = documentStore.OpenSession()) - { - session.Store(expiredSagaHistory); - session.SaveChanges(); - } - - var recentSagaHistory = new SagaSnapshot - { - Id = Guid.NewGuid().ToString() - }; - using (new RavenLastModifiedScope(recentDate)) - using (var session = documentStore.OpenSession()) - { - session.Store(recentSagaHistory); - session.SaveChanges(); - } - - RunExpiry(documentStore, thresholdDate); - - using (var session = documentStore.OpenSession()) - { - Assert.Null(session.Load(expiredSagaHistory.Id)); - Assert.NotNull(session.Load(recentSagaHistory.Id)); - } - } - } - - - [Test] - public void Recent_processed_messages_are_not_being_expired() - { - using (var documentStore = InMemoryStoreBuilder.GetInMemoryStore()) - { - var thresholdDate = DateTime.UtcNow.AddDays(-2); - var recentDate = DateTime.UtcNow.AddDays(-1); - var sagaHistory = new SagaSnapshot - { - Id = Guid.NewGuid().ToString() - }; - - using (new RavenLastModifiedScope(recentDate)) - using (var session = documentStore.OpenSession()) - { - session.Store(sagaHistory); - session.SaveChanges(); - } - - RunExpiry(documentStore, thresholdDate); - using (var session = documentStore.OpenSession()) - { - Assert.AreEqual(1, session.Query().Count()); - } - } - } - - [Test] - public void Errors_are_not_being_expired() - { - using (var documentStore = InMemoryStoreBuilder.GetInMemoryStore()) - { - var failedMsg = new FailedMessage - { - Id = "1" - }; - - using (var session = documentStore.OpenSession()) - { - session.Store(failedMsg); - session.SaveChanges(); - - Debug.WriteLine(session.Advanced.GetMetadataFor(failedMsg)["Last-Modified"]); - } - - Thread.Sleep(100); - RunExpiry(documentStore, DateTime.UtcNow); - - using (var session = documentStore.OpenSession()) - { - Assert.NotNull(session.Load(failedMsg.Id)); - } - } - } - } -} \ No newline at end of file From 5cc63668c79cc94a0717da9af26d7a640dc678ef Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 14 Sep 2023 15:54:16 -0500 Subject: [PATCH 05/94] RavenLastModifiedScope was only used for expiration tests (and was broken besides) --- .../Expiration/RavenLastModifiedScope.cs | 21 ------------------- 1 file changed, 21 deletions(-) delete mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/RavenLastModifiedScope.cs diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/RavenLastModifiedScope.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/RavenLastModifiedScope.cs deleted file mode 100644 index 1e65ffffda..0000000000 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/RavenLastModifiedScope.cs +++ /dev/null @@ -1,21 +0,0 @@ -namespace ServiceControl.UnitTests.Expiration -{ - using System; - using Raven.Client.Util; - - public class RavenLastModifiedScope : IDisposable - { - public RavenLastModifiedScope(DateTime dateTime) - { - previous = SystemTime.UtcDateTime; - SystemTime.UtcDateTime = () => dateTime; - } - - public void Dispose() - { - SystemTime.UtcDateTime = previous; - } - - Func previous; - } -} \ No newline at end of file From 61f37844e8ea6e647cac41c9f7dd7f602402e2ad Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 14 Sep 2023 16:04:00 -0500 Subject: [PATCH 06/94] Don't need to test saga audits --- .../SagaAudit/SagaDetailsIndexTests.cs | 80 ------------------- .../SagaAudit/SagaListIndexTests.cs | 68 ---------------- 2 files changed, 148 deletions(-) delete mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/SagaAudit/SagaDetailsIndexTests.cs delete mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/SagaAudit/SagaListIndexTests.cs diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/SagaAudit/SagaDetailsIndexTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/SagaAudit/SagaDetailsIndexTests.cs deleted file mode 100644 index e241a5c792..0000000000 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/SagaAudit/SagaDetailsIndexTests.cs +++ /dev/null @@ -1,80 +0,0 @@ -namespace ServiceControl.UnitTests.SagaAudit -{ - using System; - using System.Collections.Generic; - using System.Linq; - using NUnit.Framework; - using Particular.Approvals; - using ServiceControl.SagaAudit; - - [TestFixture] - class SagaDetailsIndexTests - { - [Test] - public void RunMapReduce() - { - using (var store = InMemoryStoreBuilder.GetInMemoryStore()) - { - store.ExecuteIndex(new SagaDetailsIndex()); - using (var session = store.OpenSession()) - { - foreach (var sagaHistory in GetFakeHistory()) - { - session.Store(sagaHistory); - } - - session.SaveChanges(); - } - - store.WaitForIndexing(); - - using (var session = store.OpenSession()) - { - var mapReduceResults = session.Query() - .ToList(); - Approver.Verify(mapReduceResults); - } - } - } - - static IEnumerable GetFakeHistory() - { - yield return new SagaSnapshot - { - SagaId = Guid.Empty, - SagaType = "MySaga1", - Endpoint = "MyEndpoint", - FinishTime = new DateTime(2001, 1, 1, 1, 1, 1, DateTimeKind.Utc), - Status = SagaStateChangeStatus.Updated, - StartTime = new DateTime(2001, 1, 1, 1, 1, 1, DateTimeKind.Utc), - StateAfterChange = "Updated" - }; - yield return new SagaHistory - { - SagaId = Guid.Empty, - SagaType = "MySaga1", - Changes = new List - { - new SagaStateChange - { - Endpoint = "MyEndpoint", - FinishTime = new DateTime(2000, 1, 1, 1, 1, 1, DateTimeKind.Utc), - Status = SagaStateChangeStatus.Updated, - StartTime = new DateTime(2000, 1, 1, 1, 1, 1, DateTimeKind.Utc), - StateAfterChange = "Started" - } - } - }; - yield return new SagaSnapshot - { - SagaId = Guid.Empty, - SagaType = "MySaga1", - Endpoint = "MyEndpoint", - FinishTime = new DateTime(2002, 1, 1, 1, 1, 1, DateTimeKind.Utc), - Status = SagaStateChangeStatus.Completed, - StartTime = new DateTime(2002, 1, 1, 1, 1, 1, DateTimeKind.Utc), - StateAfterChange = "Completed" - }; - } - } -} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/SagaAudit/SagaListIndexTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/SagaAudit/SagaListIndexTests.cs deleted file mode 100644 index 788a0a08bd..0000000000 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/SagaAudit/SagaListIndexTests.cs +++ /dev/null @@ -1,68 +0,0 @@ -namespace ServiceControl.UnitTests.SagaAudit -{ - using System; - using System.Collections.Generic; - using System.Linq; - using NUnit.Framework; - using Particular.Approvals; - using ServiceControl.SagaAudit; - - [TestFixture] - class SagaListIndexTests - { - [Test] - public void RunMapReduce() - { - using (var store = InMemoryStoreBuilder.GetInMemoryStore()) - { - store.ExecuteIndex(new SagaListIndex()); - using (var session = store.OpenSession()) - { - foreach (var sagaHistory in GetFakeHistory()) - { - session.Store(sagaHistory); - } - - session.SaveChanges(); - } - - store.WaitForIndexing(); - - using (var session = store.OpenSession()) - { - var mapReduceResults = session.Query() - .ToList(); - Approver.Verify(mapReduceResults); - } - } - } - - static IEnumerable GetFakeHistory() - { - yield return new SagaSnapshot - { - SagaId = new Guid("00000000-0000-0000-0000-000000000003"), - SagaType = "MySaga3", - FinishTime = new DateTime(2000, 1, 1, 10, 0, 0) - }; - yield return new SagaHistory - { - SagaId = new Guid("00000000-0000-0000-0000-000000000001"), - SagaType = "MySaga1", - Changes = new List - { - new SagaStateChange - { - FinishTime = new DateTime(2000, 1, 1, 1, 1, 1, DateTimeKind.Utc) - } - } - }; - yield return new SagaSnapshot - { - SagaId = new Guid("00000000-0000-0000-0000-000000000002"), - SagaType = "MySaga2", - FinishTime = new DateTime(2000, 1, 1, 15, 0, 0) - }; - } - } -} \ No newline at end of file From 3d8e460d1e10ca48ac5131482dc9a2ddbbb25527 Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 14 Sep 2023 16:05:33 -0500 Subject: [PATCH 07/94] Remove FailedAuditImportCustomChecktests --- .../FailedAuditImportCustomCheckTests.cs | 50 ------------------- 1 file changed, 50 deletions(-) delete mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedAuditImportCustomCheckTests.cs diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedAuditImportCustomCheckTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedAuditImportCustomCheckTests.cs deleted file mode 100644 index e7d75306ea..0000000000 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedAuditImportCustomCheckTests.cs +++ /dev/null @@ -1,50 +0,0 @@ -namespace ServiceControl.UnitTests.Operations -{ - using System.Threading.Tasks; - using NServiceBus.CustomChecks; - using NUnit.Framework; - using ServiceControl.Operations; - - [TestFixture] - public class FailedAuditImportCustomCheckTests - { - [Test] - public async Task Pass_if_no_failed_imports() - { - using (var store = InMemoryStoreBuilder.GetInMemoryStore()) - { - store.ExecuteIndex(new FailedAuditImportIndex()); - - var customCheck = new FailedAuditImportCustomCheck(store); - - var result = await customCheck.PerformCheck(); - - Assert.AreEqual(CheckResult.Pass, result); - } - } - - [Test] - public async Task Fail_if_failed_imports() - { - using (var store = InMemoryStoreBuilder.GetInMemoryStore()) - { - store.ExecuteIndex(new FailedAuditImportIndex()); - - using (var session = store.OpenAsyncSession()) - { - await session.StoreAsync(new FailedAuditImport()); - await session.SaveChangesAsync(); - } - - store.WaitForIndexing(); - - var customCheck = new FailedAuditImportCustomCheck(store); - - var result = await customCheck.PerformCheck(); - - Assert.IsTrue(result.HasFailed); - StringAssert.StartsWith("One or more audit messages have failed to import properly into ServiceControl and have been stored in the ServiceControl database.", result.FailureReason); - } - } - } -} \ No newline at end of file From 049aefec244fd01919390e910932b5b8d6355e4c Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 14 Sep 2023 16:08:28 -0500 Subject: [PATCH 08/94] Magic string tests need to be persister-specific --- .../MagicStringTypeTests.cs | 0 .../MagicStringTypeTests.cs | 21 +++++++++++++++++++ .../Infrastructure/Settings/DataStoreType.cs | 4 +++- 3 files changed, 24 insertions(+), 1 deletion(-) rename src/{ServiceControl.Persistence.Tests => ServiceControl.Persistence.Tests.RavenDb}/MagicStringTypeTests.cs (100%) create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/MagicStringTypeTests.cs diff --git a/src/ServiceControl.Persistence.Tests/MagicStringTypeTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb/MagicStringTypeTests.cs similarity index 100% rename from src/ServiceControl.Persistence.Tests/MagicStringTypeTests.cs rename to src/ServiceControl.Persistence.Tests.RavenDb/MagicStringTypeTests.cs diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/MagicStringTypeTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/MagicStringTypeTests.cs new file mode 100644 index 0000000000..39bd0f3d3d --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/MagicStringTypeTests.cs @@ -0,0 +1,21 @@ +namespace ServiceControl.PersistenceTests +{ + using System.Threading.Tasks; + using NUnit.Framework; + using ServiceBus.Management.Infrastructure.Settings; + using ServiceControl.Persistence.RavenDb; + + class MagicStringTypeTests + { + [Test] + public Task Verify_ravendb_persistence_type_string() + { + var typeNamespace = DataStoreConfig.RavenDB5PersistenceTypeFullyQualifiedName.Split(',')[1].Trim(); + var typeFullName = DataStoreConfig.RavenDB5PersistenceTypeFullyQualifiedName.Split(',')[0].Trim(); + var type = typeof(RavenDbPersistenceConfiguration); + Assert.AreEqual(type.Namespace, typeNamespace); + Assert.AreEqual(type.FullName, typeFullName); + return Task.CompletedTask; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl/Infrastructure/Settings/DataStoreType.cs b/src/ServiceControl/Infrastructure/Settings/DataStoreType.cs index 4ec2d20dd7..0605cba2f4 100644 --- a/src/ServiceControl/Infrastructure/Settings/DataStoreType.cs +++ b/src/ServiceControl/Infrastructure/Settings/DataStoreType.cs @@ -3,12 +3,14 @@ public enum DataStoreType { InMemory = 1, - RavenDB35 = 2 + RavenDB35 = 2, + RavenDB5 = 3, } public static class DataStoreConfig { public static string InMemoryPersistenceTypeFullyQualifiedName = "ServiceControl.Persistence.InMemory.InMemoryPersistenceConfiguration, ServiceControl.Persistence.InMemory"; public static string RavenDB35PersistenceTypeFullyQualifiedName = "ServiceControl.Persistence.RavenDb.RavenDbPersistenceConfiguration, ServiceControl.Persistence.RavenDb"; + public static string RavenDB5PersistenceTypeFullyQualifiedName = "ServiceControl.Persistence.RavenDb.RavenDbPersistenceConfiguration, ServiceControl.Persistence.RavenDb5"; } } \ No newline at end of file From c1ce57607751b7b741b8eeb65eb34b9b9f85afd4 Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Fri, 15 Sep 2023 13:49:49 +0200 Subject: [PATCH 09/94] EmbeddedDatabase test --- .../EmbeddedDatabaseTests.cs | 25 +++++++++++++++++++ .../RavenBootstrapperTests.cs | 14 ----------- 2 files changed, 25 insertions(+), 14 deletions(-) create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/EmbeddedDatabaseTests.cs delete mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/RavenBootstrapperTests.cs diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/EmbeddedDatabaseTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/EmbeddedDatabaseTests.cs new file mode 100644 index 0000000000..836655a338 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/EmbeddedDatabaseTests.cs @@ -0,0 +1,25 @@ +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.Infrastructure; +using ServiceControl.Persistence.RavenDb; +using ServiceControl.Persistence.RavenDb5; + +[TestFixture] +class EmbeddedDatabaseTests +{ + [Test] + public async Task CanStart() + { + using (var embeddedDatabase = EmbeddedDatabase.Start(new RavenDBPersisterSettings { })) + { + + using (var documentStore = await embeddedDatabase.Connect(CancellationToken.None)) + { + var store = new EventLogDataStore(documentStore); + + _ = await store.GetEventLogItems(new PagingInfo()); + } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/RavenBootstrapperTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/RavenBootstrapperTests.cs deleted file mode 100644 index cca0b65feb..0000000000 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/RavenBootstrapperTests.cs +++ /dev/null @@ -1,14 +0,0 @@ -using NUnit.Framework; -using ServiceControl.Persistence.RavenDb; - -[TestFixture] -class RavenBootstrapperTests -{ - [Test] - public void ReadLicense() - { - var readLicense = RavenBootstrapper.ReadLicense(); - Assert.IsNotNull(readLicense); - Assert.IsNotEmpty(readLicense); - } -} \ No newline at end of file From 56c44ebec6b33edd906980762652a9927d494c6a Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Fri, 15 Sep 2023 13:52:00 +0200 Subject: [PATCH 10/94] commenting out transformer tests --- .../CompositeViews/FailedMessagesTests.cs | 19 ++++++++------ .../Infrastructure/InMemoryStoreBuilder.cs | 25 ------------------- 2 files changed, 11 insertions(+), 33 deletions(-) delete mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/InMemoryStoreBuilder.cs diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/FailedMessagesTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/FailedMessagesTests.cs index 1588db7f9f..e490a06b8a 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/FailedMessagesTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/FailedMessagesTests.cs @@ -8,10 +8,12 @@ using MessageFailures; using MessageFailures.Api; using NUnit.Framework; + using PersistenceTests; using Raven.Client.Documents; + using Raven.Client.Documents.Session; [TestFixture] - public class FailedMessagesTests + class FailedMessagesTests : PersistenceTestBase { [Test] public void Should_allow_errors_with_no_metadata() @@ -39,14 +41,14 @@ public void Should_allow_errors_with_no_metadata() session.SaveChanges(); } - RavenQueryStatistics stats; + QueryStatistics stats; do { using (var session = documentStore.OpenSession()) { var results = session.Advanced.DocumentQuery() - .SetResultTransformer(FailedMessageViewTransformer.Name) + //.SetResultTransformer(FailedMessageViewTransformer.Name) .Statistics(out stats) .SelectFields() .ToList(); @@ -72,20 +74,21 @@ public void Should_allow_errors_with_no_metadata() [SetUp] - public void SetUp() + public new void SetUp() { - documentStore = InMemoryStoreBuilder.GetInMemoryStore(); + documentStore = GetRequiredService(); var customIndex = new FailedMessageViewIndex(); customIndex.Execute(documentStore); - var transformer = new FailedMessageViewTransformer(); + //var transformer = new FailedMessageViewTransformer(); - transformer.Execute(documentStore); + //TODO: we need to bring this back + //transformer.Execute(documentStore); } [TearDown] - public void TearDown() + public new void TearDown() { documentStore.Dispose(); } diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/InMemoryStoreBuilder.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/InMemoryStoreBuilder.cs deleted file mode 100644 index 3421b5bdf4..0000000000 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/InMemoryStoreBuilder.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System.IO; -using Raven.Client.Embedded; - -public class InMemoryStoreBuilder -{ - public static EmbeddableDocumentStore GetInMemoryStore() - { - var store = new EmbeddableDocumentStore - { - Configuration = - { - RunInUnreliableYetFastModeThatIsNotSuitableForProduction = true, - RunInMemory = true, - CompiledIndexCacheDirectory = Path.GetTempPath() // RavenDB-2236 - }, - Conventions = - { - SaveEnumsAsIntegers = true - } - }; - store.Initialize(); - - return store; - } -} \ No newline at end of file From 3b082bb7fc46dc3d533d2c3a5c59cae0b5e0ef09 Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Fri, 15 Sep 2023 13:54:00 +0200 Subject: [PATCH 11/94] get stats fixes - not sure this is still needed --- .../Infrastructure/RavenIndexAwaiter.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/RavenIndexAwaiter.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/RavenIndexAwaiter.cs index 48b5ce14c7..0c382f7ab7 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/RavenIndexAwaiter.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/RavenIndexAwaiter.cs @@ -2,6 +2,7 @@ using System.Threading; using NUnit.Framework; using Raven.Client.Documents; +using Raven.Client.Documents.Operations; public static class RavenIndexAwaiter { @@ -12,7 +13,12 @@ public static void WaitForIndexing(this IDocumentStore store) static void WaitForIndexing(this IDocumentStore store, int secondsToWait) { - var databaseCommands = store.DatabaseCommands; - Assert.True(SpinWait.SpinUntil(() => databaseCommands.GetStatistics().StaleIndexes.Length == 0, TimeSpan.FromSeconds(secondsToWait))); + var getStatisticsCommand = new GetStatisticsOperation(); + Assert.True(SpinWait.SpinUntil(() => + { + var stats = store.Maintenance.Send(getStatisticsCommand); + + return stats.StaleIndexes.Length == 0; + }, TimeSpan.FromSeconds(secondsToWait))); } } \ No newline at end of file From 9830b24aa1b500fff6023e5423420013210599ea Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Fri, 15 Sep 2023 13:55:42 +0200 Subject: [PATCH 12/94] subscription tests --- .../SubscriptionPersisterTests.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/SubscriptionPersisterTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/SubscriptionPersisterTests.cs index c58d752e22..a8fc2d33ca 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/SubscriptionPersisterTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/SubscriptionPersisterTests.cs @@ -7,11 +7,12 @@ using NServiceBus.Unicast.Subscriptions; using NServiceBus.Unicast.Subscriptions.MessageDrivenSubscriptions; using NUnit.Framework; + using PersistenceTests; using Raven.Client.Documents; using ServiceControl.Infrastructure.RavenDB.Subscriptions; [TestFixture] - public class SubscriptionPersisterTests + class SubscriptionPersisterTests : PersistenceTestBase { [Test] public async Task ShouldReturnSubscriptionsForOlderVersionsOfSameMessageType() @@ -50,14 +51,16 @@ public async Task ShouldReturnSubscriptionsForNewerVersionsOfSameMessageType() } [SetUp] - public void SetUp() + public new async Task SetUp() { - documentStore = InMemoryStoreBuilder.GetInMemoryStore(); + await base.SetUp(); + documentStore = GetRequiredService(); } [TearDown] - public void TearDown() + public new async Task TearDown() { + await base.TearDown(); documentStore.Dispose(); } From bb13b6a474b8a17bb8a18e29926fb5e5035416cb Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Fri, 15 Sep 2023 13:56:57 +0200 Subject: [PATCH 13/94] we don't need tests for in index body storing (this is never happening with Raven5) --- .../ReturnToSenderDequeuerTests.cs | 46 ------------------- 1 file changed, 46 deletions(-) diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/ReturnToSenderDequeuerTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/ReturnToSenderDequeuerTests.cs index b977fc04e3..934e345d54 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/ReturnToSenderDequeuerTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/ReturnToSenderDequeuerTests.cs @@ -71,52 +71,6 @@ public async Task It_fetches_the_body_from_storage_if_provided() Assert.AreEqual("MessageBodyId", Encoding.UTF8.GetString(sender.Message.Body)); } - [Test] - public async Task It_fetches_the_body_from_index_if_provided() - { - var sender = new FakeSender(); - - var headers = new Dictionary - { - ["ServiceControl.Retry.StagingId"] = "SomeId", - ["ServiceControl.TargetEndpointAddress"] = "TargetEndpoint", - ["ServiceControl.Retry.Attempt.MessageId"] = "MessageBodyId", - ["ServiceControl.Retry.UniqueMessageId"] = "MessageBodyId", - ["ServiceControl.Retry.BodyOnFailedMessage"] = null - }; - var message = CreateMessage(Guid.NewGuid().ToString(), headers); - - var documentStore = GetRequiredService(); - - var failedMessage = new FailedMessage - { - Id = FailedMessageIdGenerator.MakeDocumentId("MessageBodyId"), - ProcessingAttempts = new List - { - new FailedMessage.ProcessingAttempt - { - MessageId = "MessageBodyId", - MessageMetadata = { { "Body", "MessageBodyId" } } - } - } - }; - - await ErrorMessageDataStore.StoreFailedMessagesForTestsOnly(failedMessage); - - var transformer = new MessagesBodyTransformer(); - await transformer.ExecuteAsync(documentStore); - - await CompleteDatabaseOperation(); - - var instance = GetRequiredService(); // See this.CreateHostBuilder - - // Acts - await instance.HandleMessage(message, sender, "error"); - - // Assert - Assert.AreEqual("MessageBodyId", Encoding.UTF8.GetString(sender.Message.Body)); - } - [Test] public async Task It_uses_retry_to_if_provided() { From b4124a059dbe572e5a4637d95f8dfdf4404801e5 Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Fri, 15 Sep 2023 14:00:18 +0200 Subject: [PATCH 14/94] commenting out tansformer tests --- .../CompositeViews/MessagesViewTests.cs | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs index 783944fad2..d884a6b70b 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs @@ -3,16 +3,17 @@ using System; using System.Collections.Generic; using System.Linq; + using System.Threading.Tasks; using MessageAuditing; using MessageFailures; using NUnit.Framework; + using PersistenceTests; using Raven.Client.Documents; using Raven.Client.Documents.Linq; - using ServiceControl.CompositeViews.Messages; using ServiceControl.Persistence; [TestFixture] - public class MessagesViewTests + class MessagesViewTests : PersistenceTestBase { [Test] public void Filter_out_system_messages() @@ -93,7 +94,7 @@ public void Order_by_critical_time() var firstByCriticalTime = session.Query() .OrderBy(x => x.CriticalTime) .Where(x => x.CriticalTime.HasValue) - .ProjectFromIndexFieldsInto() + .ProjectInto() //https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/basics#projectfromindexfieldsinto .First(); Assert.AreEqual("1", firstByCriticalTime.Id); @@ -101,7 +102,7 @@ public void Order_by_critical_time() var firstByCriticalTimeDescription = session.Query() .OrderByDescending(x => x.CriticalTime) .Where(x => x.CriticalTime.HasValue) - .ProjectFromIndexFieldsInto() + .ProjectInto() .First(); Assert.AreEqual("2", firstByCriticalTimeDescription.Id); } @@ -137,13 +138,13 @@ public void Order_by_time_sent() { var firstByTimeSent = session.Query() .OrderBy(x => x.TimeSent) - .ProjectFromIndexFieldsInto() + .ProjectInto() .First(); Assert.AreEqual("3", firstByTimeSent.Id); var firstByTimeSentDescription = session.Query() .OrderByDescending(x => x.TimeSent) - .ProjectFromIndexFieldsInto() + .ProjectInto() .First(); Assert.AreEqual("1", firstByTimeSentDescription.Id); } @@ -195,7 +196,7 @@ public void TimeSent_is_not_cast_to_DateTimeMin_if_null() using (var session = documentStore.OpenSession()) { var messageWithNoTimeSent = session.Query() - .TransformWith() + //.TransformWith() .Customize(x => x.WaitForNonStaleResults()) .ToArray(); Assert.AreEqual(null, messageWithNoTimeSent[0].TimeSent); @@ -233,7 +234,7 @@ public void Correct_status_for_failed_messages(FailedMessageStatus failedMessage using (var session = documentStore.OpenSession()) { var message = session.Query() - .TransformWith() + //.TransformWith() .Customize(x => x.WaitForNonStaleResults()) .Single(); @@ -272,7 +273,7 @@ public void Correct_status_for_repeated_errors() using (var session = documentStore.OpenSession()) { var message = session.Query() - .TransformWith() + //.TransformWith() .Customize(x => x.WaitForNonStaleResults()) .Single(); @@ -281,21 +282,23 @@ public void Correct_status_for_repeated_errors() } [SetUp] - public void SetUp() + public new async Task SetUp() { - documentStore = InMemoryStoreBuilder.GetInMemoryStore(); + await base.SetUp(); + documentStore = GetRequiredService(); var customIndex = new MessagesViewIndex(); customIndex.Execute(documentStore); - var transformer = new MessagesViewTransformer(); + //var transformer = new MessagesViewTransformer(); - transformer.Execute(documentStore); + //transformer.Execute(documentStore); } [TearDown] - public void TearDown() + public new async Task TearDown() { + await base.TearDown(); documentStore.Dispose(); } From da15c0c5190b6dc8a1bb161c1094a4c95bf508c9 Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Fri, 15 Sep 2023 14:13:51 +0200 Subject: [PATCH 15/94] starting lifecycle in the test setup --- .../TestPersistenceImpl.cs | 25 +++++++++++++++++++ .../PersistenceHostBuilderExtensions.cs | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs index baa90e2a09..b0e8f8ed24 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs @@ -2,6 +2,9 @@ { using System; using System.Diagnostics; + using System.IO; + using System.Linq; + using System.Net.NetworkInformation; using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; @@ -27,6 +30,10 @@ static RavenDBPersisterSettings CreateSettings() ErrorRetentionPeriod = retentionPeriod, EventsRetentionPeriod = retentionPeriod, RunInMemory = true, + DatabasePath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Tests", "ErrorData"), + LogPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Logs"), + LogsMode = "Operations", + ServerUrl = $"http://localhost:{FindAvailablePort(33334)}" }; if (Debugger.IsAttached) @@ -55,6 +62,24 @@ public override Task CompleteDatabaseOperation() return Task.CompletedTask; } + //TODO: this method is duplicated at least 3 times in the test project + static int FindAvailablePort(int startPort) + { + var activeTcpListeners = IPGlobalProperties + .GetIPGlobalProperties() + .GetActiveTcpListeners(); + + for (var port = startPort; port < startPort + 1024; port++) + { + var portCopy = port; + if (activeTcpListeners.All(endPoint => endPoint.Port != portCopy)) + { + return port; + } + } + + return startPort; + } class Wrapper : IHostedService { public Wrapper(TestPersistenceImpl instance, IDocumentStore store) diff --git a/src/ServiceControl/Persistence/PersistenceHostBuilderExtensions.cs b/src/ServiceControl/Persistence/PersistenceHostBuilderExtensions.cs index d339c50c93..917ac2a2bf 100644 --- a/src/ServiceControl/Persistence/PersistenceHostBuilderExtensions.cs +++ b/src/ServiceControl/Persistence/PersistenceHostBuilderExtensions.cs @@ -24,4 +24,4 @@ public static void CreatePersisterLifecyle(IServiceCollection serviceCollection, persistence.Configure(serviceCollection); } } -} \ No newline at end of file +} From c62c4e8db877e3fd2b8c87c1959ac61871644a76 Mon Sep 17 00:00:00 2001 From: David Boike Date: Fri, 15 Sep 2023 15:07:34 -0500 Subject: [PATCH 16/94] Duplicate acceptance tests and add to solution --- .../AcceptanceTestStorageConfiguration.cs | 53 +++ .../App.config | 9 + .../Legacy/AuditRetention.cs | 113 ++++++ ...en_processed_message_is_still_available.cs | 114 ++++++ .../MessageFailures/FailedErrorsController.cs | 77 ++++ .../FailedMessageRetriesController.cs | 41 ++ .../When_a_failed_message_is_retried.cs | 352 ++++++++++++++++++ .../When_a_message_fails_to_import.cs | 195 ++++++++++ ...iceControl.AcceptanceTests.RavenDB5.csproj | 31 ++ .../ServiceControl.runsettings | 7 + .../StartupModeTests.cs | 69 ++++ .../TestsFilter.cs | 2 + src/ServiceControl.sln | 15 + 13 files changed, 1078 insertions(+) create mode 100644 src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs create mode 100644 src/ServiceControl.AcceptanceTests.RavenDB5/App.config create mode 100644 src/ServiceControl.AcceptanceTests.RavenDB5/Legacy/AuditRetention.cs create mode 100644 src/ServiceControl.AcceptanceTests.RavenDB5/Legacy/When_processed_message_is_still_available.cs create mode 100644 src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/FailedErrorsController.cs create mode 100644 src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/FailedMessageRetriesController.cs create mode 100644 src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/When_a_failed_message_is_retried.cs create mode 100644 src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/When_a_message_fails_to_import.cs create mode 100644 src/ServiceControl.AcceptanceTests.RavenDB5/ServiceControl.AcceptanceTests.RavenDB5.csproj create mode 100644 src/ServiceControl.AcceptanceTests.RavenDB5/ServiceControl.runsettings create mode 100644 src/ServiceControl.AcceptanceTests.RavenDB5/StartupModeTests.cs create mode 100644 src/ServiceControl.AcceptanceTests.RavenDB5/TestsFilter.cs diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs new file mode 100644 index 0000000000..6c747a5677 --- /dev/null +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs @@ -0,0 +1,53 @@ +namespace ServiceControl.AcceptanceTests +{ + using System; + using System.IO; + using System.Linq; + using System.Net.NetworkInformation; + using System.Threading.Tasks; + using Persistence.RavenDb; + using ServiceBus.Management.Infrastructure.Settings; + + class AcceptanceTestStorageConfiguration + { + public string PersistenceType { get; protected set; } + + public void CustomizeSettings(Settings settings) + { + settings.PersisterSpecificSettings = new RavenDBPersisterSettings + { + RunInMemory = true, + DatabaseMaintenancePort = FindAvailablePort(settings.Port + 1), + DatabasePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()), + ErrorRetentionPeriod = TimeSpan.FromDays(10), + }; + } + + public Task Configure() + { + PersistenceType = typeof(RavenDbPersistenceConfiguration).AssemblyQualifiedName; + + return Task.CompletedTask; + } + + static int FindAvailablePort(int startPort) + { + var activeTcpListeners = IPGlobalProperties + .GetIPGlobalProperties() + .GetActiveTcpListeners(); + + for (var port = startPort; port < startPort + 1024; port++) + { + var portCopy = port; + if (activeTcpListeners.All(endPoint => endPoint.Port != portCopy)) + { + return port; + } + } + + return startPort; + } + + public Task Cleanup() => Task.CompletedTask; + } +} diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/App.config b/src/ServiceControl.AcceptanceTests.RavenDB5/App.config new file mode 100644 index 0000000000..4149eda6a7 --- /dev/null +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/App.config @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/Legacy/AuditRetention.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/Legacy/AuditRetention.cs new file mode 100644 index 0000000000..0297c2ccaa --- /dev/null +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/Legacy/AuditRetention.cs @@ -0,0 +1,113 @@ +namespace ServiceControl.MultiInstance.AcceptanceTests.SagaAudit +{ + using System; + using System.Linq; + using System.Threading.Tasks; + using AcceptanceTesting; + using EventLog; + using Microsoft.Extensions.DependencyInjection; + using NServiceBus; + using NServiceBus.AcceptanceTesting; + using NServiceBus.CustomChecks; + using NUnit.Framework; + using Raven.Client; + using ServiceBus.Management.Infrastructure.Settings; + using ServiceControl.AcceptanceTests; + using ServiceControl.AcceptanceTests.TestSupport.EndpointTemplates; + using ServiceControl.Persistence.RavenDb.SagaAudit; + using ServiceControl.SagaAudit; + + + class AuditRetention : AcceptanceTest + { + [Test] + public async Task + Check_fails_if_no_audit_retention_is_configured() + { + ////Ensure custom checks are enabled + SetSettings = settings => + { + settings.DisableHealthChecks = false; + settings.AuditRetentionPeriod = TimeSpan.FromSeconds(5); + }; + + //Override the configuration of the check in the container in order to make it run more frequently for testing purposes. + CustomizeHostBuilder = hostBuilder => hostBuilder.ConfigureServices((ctx, services) => + services.AddTransient(provider => new AuditRetentionCustomCheck(provider.GetRequiredService(), provider.GetRequiredService(), TimeSpan.FromSeconds(10)))); + + SingleResult customCheckEventEntry = default; + bool sagaAudiDataInMainInstanceIsAvailableForQuery = false; + + await Define() + .WithEndpoint(b => b.When((bus, c) => bus.SendLocal(new MessageInitiatingSaga { Id = "Id" }))) + .Done(async c => + { + if (!c.SagaId.HasValue) + { + return false; + } + + if (sagaAudiDataInMainInstanceIsAvailableForQuery == false) + { + var sagaData = + await this.TryGet($"/api/sagas/{c.SagaId}"); + sagaAudiDataInMainInstanceIsAvailableForQuery = sagaData.HasResult; + return false; + } + + customCheckEventEntry = await this.TryGetSingle("/api/eventlogitems/", + e => e.EventType == "CustomCheckFailed" && e.Description.StartsWith("Saga Audit Data Retention")); + + return customCheckEventEntry; + }) + .Run(); + + Assert.IsTrue(customCheckEventEntry.Item.RelatedTo.Any(item => item == "/customcheck/Saga Audit Data Retention"), "Event log entry should be related to the Saga Audit Data Retention"); + Assert.IsTrue(customCheckEventEntry.Item.RelatedTo.Any(item => item.StartsWith("/endpoint/Particular.ServiceControl")), "Event log entry should be related to the ServiceControl endpoint"); + } + + public class SagaEndpoint : EndpointConfigurationBuilder + { + public SagaEndpoint() + { + EndpointSetup(c => + { + c.AuditSagaStateChanges(Settings.DEFAULT_SERVICE_NAME); + }); + } + + public class MySaga : Saga, IAmStartedByMessages + { + public MyContext TestContext { get; set; } + + public Task Handle(MessageInitiatingSaga message, IMessageHandlerContext context) + { + TestContext.SagaId = Data.Id; + return Task.CompletedTask; + } + + protected override void ConfigureHowToFindSaga(SagaPropertyMapper mapper) + { + mapper.ConfigureMapping(msg => msg.Id).ToSaga(saga => saga.MessageId); + } + } + + public class MySagaData : ContainSagaData + { + public string MessageId { get; set; } + } + } + + + public class MessageInitiatingSaga : ICommand + { + public string Id { get; set; } + } + + + public class MyContext : ScenarioContext + { + public Guid? SagaId { get; set; } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/Legacy/When_processed_message_is_still_available.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/Legacy/When_processed_message_is_still_available.cs new file mode 100644 index 0000000000..51db54d943 --- /dev/null +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/Legacy/When_processed_message_is_still_available.cs @@ -0,0 +1,114 @@ +namespace ServiceControl.AcceptanceTests.RavenDB.Legacy +{ + using System; + using System.Collections.Generic; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using AcceptanceTesting; + using AcceptanceTests; + using CompositeViews.Messages; + using MessageAuditing; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Hosting; + using NServiceBus; + using NServiceBus.AcceptanceTesting; + using NUnit.Framework; + using Operations.BodyStorage; + using Raven.Client; + using ServiceControl.Persistence; + using TestSupport.EndpointTemplates; + + class When_processed_message_is_still_available : AcceptanceTest + { + [Test] + public async Task Should_be_accessible_via_the_rest_api() + { + var messageId = Guid.NewGuid().ToString(); + + CustomizeHostBuilder = hostBuilder + => hostBuilder.ConfigureServices((hostBuilderContext, services) + => services.AddHostedService(sp => new CreateMessageDataMigration(messageId, sp.GetRequiredService()))); + + MessagesView auditedMessage = null; + byte[] body = null; + + var context = await Define() + .WithEndpoint() + .Done(async c => + { + var result = await this.TryGetSingle("/api/messages?include_system_messages=false&sort=id", m => m.MessageId == messageId); + auditedMessage = result; + if (!result) + { + return false; + } + + body = await this.DownloadData(auditedMessage.BodyUrl); + + return true; + }) + .Run(); + + Assert.AreEqual(messageId, auditedMessage.MessageId); + Assert.AreEqual(MessageStatus.Successful, auditedMessage.Status); + + var bodyAsString = Encoding.UTF8.GetString(body); + + Assert.AreEqual("Some Content", bodyAsString); + } + + class CreateMessageDataMigration : IHostedService + { + readonly string messageId; + readonly IDocumentStore store; + + public CreateMessageDataMigration(string messageId, IDocumentStore store) + { + this.messageId = messageId; + this.store = store; + } + + public async Task StartAsync(CancellationToken cancellationToken) + { + using (var documentSession = store.OpenAsyncSession()) + { + var processedMessage = new ProcessedMessage + { + MessageMetadata = new Dictionary + { + {"Body", "Some Content"}, + {"ContentLength", 11}, + {"BodyUrl", string.Format(BodyStorageEnricher.BodyUrlFormatString, messageId)}, + {"BodyNotStored", false}, + {"ContentType", "text/plain" }, + {"MessageIntent", (int)MessageIntentEnum.Send}, + {"MessageId", messageId}, + }, + UniqueMessageId = messageId, + ProcessedAt = DateTime.UtcNow + }; + + await documentSession.StoreAsync(processedMessage, cancellationToken); + await documentSession.SaveChangesAsync(cancellationToken); + } + + SpinWait.SpinUntil(() => store.DatabaseCommands.GetStatistics().StaleIndexes.Length == 0, TimeSpan.FromSeconds(10)); + } + + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; + } + + public class Endpoint : EndpointConfigurationBuilder + { + public Endpoint() + { + EndpointSetup(); + } + } + + public class MyContext : ScenarioContext + { + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/FailedErrorsController.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/FailedErrorsController.cs new file mode 100644 index 0000000000..d266f8d60b --- /dev/null +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/FailedErrorsController.cs @@ -0,0 +1,77 @@ +namespace ServiceControl.AcceptanceTests.RavenDB.Recoverability.MessageFailures +{ + using System; + using System.Net; + using System.Net.Http; + using System.Threading; + using System.Threading.Tasks; + using System.Web.Http; + using Infrastructure.RavenDB.Expiration; + using Infrastructure.WebApi; + using Operations; + using Raven.Client; + using Raven.Client.Embedded; + + public class FailedErrorsCountReponse + { + public int Count { get; set; } + } + + class FailedErrorsController : ApiController + { + public FailedErrorsController(IDocumentStore store, ImportFailedErrors importFailedErrors) + { + this.store = store; + this.importFailedErrors = importFailedErrors; + } + + [Route("failederrors/count")] + [HttpGet] + public async Task GetFailedErrorsCount() + { + using (var session = store.OpenAsyncSession()) + { + var query = + session.Query().Statistics(out var stats); + + var count = await query.CountAsync(); + + return Request.CreateResponse(HttpStatusCode.OK, new FailedErrorsCountReponse + { + Count = count + }) + .WithEtag(stats.IndexEtag); + } + } + + [Route("failederrors/import")] + [HttpPost] + public async Task ImportFailedErrors(CancellationToken cancellationToken = default) + { + var tokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + await importFailedErrors.Run(tokenSource.Token); + return Request.CreateResponse(HttpStatusCode.OK); + } + + [Route("failederrors/forcecleanerrors")] + [HttpPost] + public Task ForceErrorMessageCleanerRun(CancellationToken cancellationToken = default) + { + new ExpiryErrorMessageIndex().Execute(store); + WaitForIndexes(store); + + ErrorMessageCleaner.Clean(1000, ((EmbeddableDocumentStore)store).DocumentDatabase, DateTime.Now, cancellationToken); + WaitForIndexes(store); + + return Task.FromResult(Request.CreateResponse(HttpStatusCode.OK)); + } + + static void WaitForIndexes(IDocumentStore store) + { + SpinWait.SpinUntil(() => store.DatabaseCommands.GetStatistics().StaleIndexes.Length == 0, TimeSpan.FromSeconds(10)); + } + + readonly IDocumentStore store; + readonly ImportFailedErrors importFailedErrors; + } +} \ No newline at end of file diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/FailedMessageRetriesController.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/FailedMessageRetriesController.cs new file mode 100644 index 0000000000..e2ac53e9b2 --- /dev/null +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/FailedMessageRetriesController.cs @@ -0,0 +1,41 @@ +namespace ServiceControl.AcceptanceTests.RavenDB.Recoverability.MessageFailures +{ + using System.Net; + using System.Net.Http; + using System.Threading.Tasks; + using System.Web.Http; + using Infrastructure.WebApi; + using Raven.Client; + using ServiceControl.Recoverability; + + public class FailedMessageRetriesCountReponse + { + public int Count { get; set; } + } + + class FailedMessageRetriesController : ApiController + { + public FailedMessageRetriesController(IDocumentStore store) + { + this.store = store; + } + + [Route("failedmessageretries/count")] + [HttpGet] + public async Task GetFailedMessageRetriesCount() + { + using (var session = store.OpenAsyncSession()) + { + await session.Query().Statistics(out var stats).ToListAsync(); + + return Request.CreateResponse(HttpStatusCode.OK, new FailedMessageRetriesCountReponse + { + Count = stats.TotalResults + }) + .WithEtag(stats.IndexEtag); + } + } + + readonly IDocumentStore store; + } +} \ No newline at end of file diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/When_a_failed_message_is_retried.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/When_a_failed_message_is_retried.cs new file mode 100644 index 0000000000..5a7a9415d5 --- /dev/null +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/When_a_failed_message_is_retried.cs @@ -0,0 +1,352 @@ +namespace ServiceControl.AcceptanceTests.RavenDB.Recoverability.MessageFailures +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading.Tasks; + using AcceptanceTesting; + using Infrastructure; + using Microsoft.Extensions.DependencyInjection; + using NServiceBus; + using NServiceBus.AcceptanceTesting; + using NServiceBus.Features; + using NServiceBus.Settings; + using NUnit.Framework; + using ServiceControl.MessageFailures; + using ServiceControl.Recoverability; + using TestSupport; + using TestSupport.EndpointTemplates; + + class When_a_failed_message_is_retried : AcceptanceTest + { + public When_a_failed_message_is_retried() + { + CustomizeHostBuilder = builder => builder.ConfigureServices((hostContext, services) => + { + services.AddScoped(); + services.AddScoped(); + }); + } + + [Test] + public async Task Should_remove_failedmessageretries_when_retrying_groups() + { + FailedMessageRetriesCountReponse failedMessageRetries = null; + + await Define() + .WithEndpoint(b => b.When(async ctx => + { + if (ctx.UniqueMessageId == null) + { + return false; + } + + FailedMessage failedMessage = await this.TryGet($"/api/errors/{ctx.UniqueMessageId}"); + if (failedMessage == null) + { + return false; + } + + ctx.FailureGroupId = failedMessage.FailureGroups.First().Id; + + return true; + }, async (bus, ctx) => + { + ctx.AboutToSendRetry = true; + await this.Post($"/api/recoverability/groups/{ctx.FailureGroupId}/errors/retry"); + }).DoNotFailOnErrorMessages()) + .Done(async ctx => + { + if (ctx.Retried) + { + failedMessageRetries = await this.TryGet("/api/failedmessageretries/count"); + + return failedMessageRetries.Count == 0; + } + + return false; + }) + .Run(); + + Assert.AreEqual(failedMessageRetries.Count, 0, "FailedMessageRetries not removed"); + } + + [Test] + public async Task Should_remove_failedmessageretries_when_retrying_individual_messages() + { + FailedMessageRetriesCountReponse failedMessageRetries = null; + + await Define() + .WithEndpoint(b => b.When(async ctx => + { + if (ctx.UniqueMessageId == null) + { + return false; + } + + FailedMessage failedMessage = await this.TryGet($"/api/errors/{ctx.UniqueMessageId}"); + if (failedMessage == null) + { + return false; + } + + return true; + }, async (bus, ctx) => + { + ctx.AboutToSendRetry = true; + await this.Post($"/api/errors/{ctx.UniqueMessageId}/retry"); + }).DoNotFailOnErrorMessages()) + .Done(async ctx => + { + if (ctx.Retried) + { + failedMessageRetries = await this.TryGet("/api/failedmessageretries/count"); + + return failedMessageRetries.Count == 0; + } + + return false; + }) + .Run(); + + Assert.AreEqual(failedMessageRetries.Count, 0, "FaileMessageRetries not removed"); + } + + [Test] + public async Task Should_remove_UnacknowledgedOperation_when_retrying_individual_messages() + { + RetryHistory retryHistory = null; + + await Define() + .WithEndpoint(b => b.When(async ctx => + { + if (ctx.UniqueMessageId == null) + { + return false; + } + + FailedMessage failedMessage = await this.TryGet($"/api/errors/{ctx.UniqueMessageId}"); + if (failedMessage == null) + { + return false; + } + + return true; + }, async (bus, ctx) => + { + ctx.AboutToSendRetry = true; + await this.Post("/api/errors/retry", new List { ctx.UniqueMessageId }); + }).DoNotFailOnErrorMessages()) + .Done(async ctx => + { + if (ctx.Retried) + { + retryHistory = await this.TryGet("/api/recoverability/history"); + + return !retryHistory.UnacknowledgedOperations.Any() && retryHistory.HistoricOperations.Any(); + } + + return false; + }) + .Run(); + + Assert.IsEmpty(retryHistory.UnacknowledgedOperations, "Unucknowledged retry operation not removed"); + } + + [Test] + public async Task Should_remove_failedmessageretries_after_expiration_process_passes() + { + FailedMessageRetriesCountReponse failedMessageRetries = null; + + await Define() + .WithEndpoint(b => b.When(async ctx => + { + if (ctx.UniqueMessageId == null) + { + return false; + } + + FailedMessage failedMessage = await this.TryGet($"/api/errors/{ctx.UniqueMessageId}"); + if (failedMessage == null) + { + return false; + } + ctx.FailureGroupId = failedMessage.FailureGroups.First().Id; + + return true; + }, async (bus, ctx) => + { + ctx.AboutToSendRetry = true; + await this.Post($"/api/recoverability/groups/{ctx.FailureGroupId}/errors/retry"); + }) + .DoNotFailOnErrorMessages()) + .Done(async ctx => + { + if (ctx.Retried) + { + // trigger cleanup + await this.Post("/api/failederrors/forcecleanerrors"); + + failedMessageRetries = await this.TryGet("/api/failedmessageretries/count"); + return failedMessageRetries.Count == 0; + } + + return false; + }) + .Run(); + + Assert.AreEqual(failedMessageRetries.Count, 0, "FailedMessageRetries not removed"); + } + + public class FailingEndpoint : EndpointConfigurationBuilder + { + public FailingEndpoint() + { + EndpointSetup(c => + { + c.EnableFeature(); + c.ReportSuccessfulRetriesToServiceControl(); + + var recoverability = c.Recoverability(); + recoverability.Immediate(s => s.NumberOfRetries(0)); + recoverability.Delayed(s => s.NumberOfRetries(0)); + }); + } + + class StartFeature : Feature + { + public StartFeature() + { + EnableByDefault(); + } + + protected override void Setup(FeatureConfigurationContext context) + { + context.RegisterStartupTask(new SendMessageAtStart()); + } + + class SendMessageAtStart : FeatureStartupTask + { + protected override Task OnStart(IMessageSession session) + { + return session.SendLocal(new MyMessage()); + } + + protected override Task OnStop(IMessageSession session) + { + return Task.FromResult(0); + } + } + } + + public class MyMessageHandler : IHandleMessages + { + readonly Context scenarioContext; + readonly ReadOnlySettings settings; + + public MyMessageHandler(Context scenarioContext, ReadOnlySettings settings) + { + this.scenarioContext = scenarioContext; + this.settings = settings; + } + + public Task Handle(MyMessage message, IMessageHandlerContext context) + { + Console.WriteLine("Message Handled"); + if (scenarioContext.AboutToSendRetry) + { + scenarioContext.Retried = true; + } + else + { + scenarioContext.UniqueMessageId = DeterministicGuid.MakeId(context.MessageId, settings.EndpointName()).ToString(); + throw new Exception("Simulated Exception"); + } + + return Task.FromResult(0); + } + } + } + + public class FailingEndpointWithoutAudit : EndpointConfigurationBuilder + { + public FailingEndpointWithoutAudit() + { + EndpointSetup(c => + { + c.EnableFeature(); + + var recoverability = c.Recoverability(); + recoverability.Immediate(s => s.NumberOfRetries(0)); + recoverability.Delayed(s => s.NumberOfRetries(0)); + }); + } + + class StartFeature : Feature + { + public StartFeature() + { + EnableByDefault(); + } + + protected override void Setup(FeatureConfigurationContext context) + { + context.RegisterStartupTask(new SendMessageAtStart()); + } + + class SendMessageAtStart : FeatureStartupTask + { + protected override Task OnStart(IMessageSession session) + { + return session.SendLocal(new MyMessage()); + } + + protected override Task OnStop(IMessageSession session) + { + return Task.FromResult(0); + } + } + } + + public class MyMessageHandler : IHandleMessages + { + readonly Context scenarioContext; + readonly ReadOnlySettings settings; + + public MyMessageHandler(Context scenarioContext, ReadOnlySettings settings) + { + this.scenarioContext = scenarioContext; + this.settings = settings; + } + + public Task Handle(MyMessage message, IMessageHandlerContext context) + { + Console.WriteLine("Message Handled"); + if (scenarioContext.AboutToSendRetry) + { + scenarioContext.Retried = true; + } + else + { + scenarioContext.UniqueMessageId = DeterministicGuid.MakeId(context.MessageId, settings.EndpointName()).ToString(); + throw new Exception("Simulated Exception"); + } + + return Task.FromResult(0); + } + } + } + + public class Context : ScenarioContext + { + public string UniqueMessageId { get; set; } + public string FailureGroupId { get; set; } + public bool Retried { get; set; } + public bool AboutToSendRetry { get; set; } + } + + public class MyMessage : ICommand + { + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/When_a_message_fails_to_import.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/When_a_message_fails_to_import.cs new file mode 100644 index 0000000000..180e9f2813 --- /dev/null +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/When_a_message_fails_to_import.cs @@ -0,0 +1,195 @@ +namespace ServiceControl.AcceptanceTests.RavenDB.Recoverability.MessageFailures +{ + using System; + using System.Threading.Tasks; + using AcceptanceTesting; + using Contracts.MessageFailures; + using Infrastructure; + using Infrastructure.DomainEvents; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Hosting; + using NServiceBus; + using NServiceBus.AcceptanceTesting; + using NServiceBus.AcceptanceTesting.Customization; + using NServiceBus.Settings; + using NUnit.Framework; + using Operations; + using ServiceControl.MessageFailures; + using TestSupport.EndpointTemplates; + using Conventions = NServiceBus.AcceptanceTesting.Customization.Conventions; + + + class When_a_message_fails_to_import : AcceptanceTest + { + [Test] + public async Task It_can_be_reimported() + { + CustomConfiguration = config => + { + config.RegisterComponents(c => + { + //Make sure the error import attempt fails + c.ConfigureComponent(DependencyLifecycle.SingleInstance); + + //Register domain event spy + c.ConfigureComponent(DependencyLifecycle.SingleInstance); + }); + }; + + SetSettings = settings => + { + settings.ForwardErrorMessages = true; + settings.ErrorLogQueue = Conventions.EndpointNamingConvention(typeof(ErrorLogSpy)); + }; + + CustomizeHostBuilder = builder => builder.ConfigureServices(services => services.AddScoped()); + + var runResult = await Define() + .WithEndpoint(b => b.When((bus, c) => bus.Send(new MyMessage()))) + .WithEndpoint(b => b.DoNotFailOnErrorMessages()) + .WithEndpoint() + .Done(async c => + { + if (c.UniqueMessageId == null) + { + return false; + } + + if (!c.WasImportedAgain) + { + var result = await this.TryGet("/api/failederrors/count"); + FailedErrorsCountReponse failedAuditCountsResponse = result; + if (result && failedAuditCountsResponse.Count > 0) + { + c.FailedImport = true; + await this.Post("/api/failederrors/import"); + c.WasImportedAgain = true; + } + + return false; + } + + return await this.TryGet($"/api/errors/{c.UniqueMessageId}") && c.ErrorForwarded; + }) + .Run(); + + Assert.IsTrue(runResult.ErrorForwarded); + Assert.IsTrue(runResult.MessageFailedEventPublished); + } + + class MessageFailedHandler : IDomainHandler + { + public MessageFailedHandler(MyContext scenarioContext) + { + this.scenarioContext = scenarioContext; + } + readonly MyContext scenarioContext; + + public Task Handle(MessageFailed domainEvent) + { + scenarioContext.MessageFailedEventPublished = true; + return Task.CompletedTask; + } + } + + class FailOnceEnricher : IEnrichImportedErrorMessages + { + readonly MyContext scenarioContext; + + public FailOnceEnricher(MyContext scenarioContext) + { + this.scenarioContext = scenarioContext; + } + + public void Enrich(ErrorEnricherContext context) + { + if (!scenarioContext.FailedImport) + { + TestContext.WriteLine("Simulating message processing failure"); + throw new MessageDeserializationException("ID", null); + } + + TestContext.WriteLine("Message processed correctly"); + } + } + + public class ErrorLogSpy : EndpointConfigurationBuilder + { + public ErrorLogSpy() + { + EndpointSetup(); + } + + public class MyMessageHandler : IHandleMessages + { + readonly MyContext scenarioContext; + + public MyMessageHandler(MyContext scenarioContext) + { + this.scenarioContext = scenarioContext; + } + + public Task Handle(MyMessage message, IMessageHandlerContext context) + { + scenarioContext.ErrorForwarded = true; + return Task.CompletedTask; + } + } + } + + public class Sender : EndpointConfigurationBuilder + { + public Sender() + { + EndpointSetup(c => + { + var routing = c.ConfigureTransport().Routing(); + routing.RouteToEndpoint(typeof(MyMessage), typeof(Receiver)); + }); + } + } + + public class Receiver : EndpointConfigurationBuilder + { + public Receiver() + { + EndpointSetup(c => + { + var recoverability = c.Recoverability(); + recoverability.Immediate(x => x.NumberOfRetries(0)); + recoverability.Delayed(x => x.NumberOfRetries(0)); + }); + } + + public class MyMessageHandler : IHandleMessages + { + public MyMessageHandler(MyContext scenarioContext, ReadOnlySettings settings) + { + this.scenarioContext = scenarioContext; + this.settings = settings; + } + readonly MyContext scenarioContext; + readonly ReadOnlySettings settings; + + public Task Handle(MyMessage message, IMessageHandlerContext context) + { + scenarioContext.UniqueMessageId = DeterministicGuid.MakeId(context.MessageId, settings.EndpointName()).ToString(); + throw new Exception("Simulated"); + } + } + } + + public class MyMessage : ICommand + { + } + + public class MyContext : ScenarioContext + { + public bool FailedImport { get; set; } + public string UniqueMessageId { get; set; } + public bool WasImportedAgain { get; set; } + public bool ErrorForwarded { get; set; } + public bool MessageFailedEventPublished { get; set; } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/ServiceControl.AcceptanceTests.RavenDB5.csproj b/src/ServiceControl.AcceptanceTests.RavenDB5/ServiceControl.AcceptanceTests.RavenDB5.csproj new file mode 100644 index 0000000000..be29578e9e --- /dev/null +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/ServiceControl.AcceptanceTests.RavenDB5.csproj @@ -0,0 +1,31 @@ + + + + net472 + ServiceControl.runsettings + 8 + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/ServiceControl.runsettings b/src/ServiceControl.AcceptanceTests.RavenDB5/ServiceControl.runsettings new file mode 100644 index 0000000000..ea2e767f89 --- /dev/null +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/ServiceControl.runsettings @@ -0,0 +1,7 @@ + + + + + x64 + + diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/StartupModeTests.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/StartupModeTests.cs new file mode 100644 index 0000000000..83b9c668da --- /dev/null +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/StartupModeTests.cs @@ -0,0 +1,69 @@ +namespace ServiceControl.AcceptanceTests.RavenDB +{ + using System; + using System.Threading.Tasks; + using Hosting.Commands; + using NServiceBus; + using NUnit.Framework; + using Particular.ServiceControl; + using Particular.ServiceControl.Hosting; + using Persistence.RavenDb; + using ServiceBus.Management.Infrastructure.Settings; + using ServiceControl.AcceptanceTesting.InfrastructureConfig; + + class StartupModeTests : AcceptanceTest + { + Settings settings; + + [SetUp] + public void InitializeSettings() + { + var transportIntegration = new ConfigureEndpointLearningTransport(); + settings = new Settings( + forwardErrorMessages: false, + errorRetentionPeriod: TimeSpan.FromDays(1), + persisterType: typeof(RavenDbPersistenceConfiguration).AssemblyQualifiedName) + { + PersisterSpecificSettings = new RavenDBPersisterSettings + { + ErrorRetentionPeriod = TimeSpan.FromDays(1), + RunInMemory = true + }, + TransportType = transportIntegration.TypeName, + TransportConnectionString = transportIntegration.ConnectionString + }; + } + + [Test] + public async Task CanRunMaintenanceMode() + { + var bootstrapper = new MaintenanceBootstrapper(settings); + + var host = bootstrapper.HostBuilder.Build(); + + await host.StartAsync(); + await host.StopAsync(); + } + + [Test] + public async Task CanRunImportFailedMessagesMode() + { + await new TestableImportFailedErrorsCommand().Execute(new HostArguments(Array.Empty()), settings); + } + + class TestableImportFailedErrorsCommand : ImportFailedErrorsCommand + { + protected override EndpointConfiguration CreateEndpointConfiguration(Settings settings) + { + var configuration = base.CreateEndpointConfiguration(settings); + + //HINT: we want to exclude this assembly to prevent loading features that are part of the acceptance testing framework + var thisAssembly = new[] { typeof(StartupModeTests).Assembly.GetName().Name }; + + configuration.AssemblyScanner().ExcludeAssemblies(thisAssembly); + + return configuration; + } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/TestsFilter.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/TestsFilter.cs new file mode 100644 index 0000000000..b207ccee13 --- /dev/null +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/TestsFilter.cs @@ -0,0 +1,2 @@ +// This project alone tends to take ~13 minutes on GitHub Actions runner +[assembly: IncludeInTestCategory("Raven35Acceptance")] \ No newline at end of file diff --git a/src/ServiceControl.sln b/src/ServiceControl.sln index 8c3009e173..f6cd62926e 100644 --- a/src/ServiceControl.sln +++ b/src/ServiceControl.sln @@ -185,6 +185,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ServiceControl.Persistence. EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ServiceControl.Persistence.Tests.RavenDb5", "ServiceControl.Persistence.Tests.RavenDb5\ServiceControl.Persistence.Tests.RavenDb5.csproj", "{2EF143A5-E769-42FD-8B7E-7C01C381BC13}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ServiceControl.AcceptanceTests.RavenDB5", "ServiceControl.AcceptanceTests.RavenDB5\ServiceControl.AcceptanceTests.RavenDB5.csproj", "{4F2CAFC7-113A-4D31-A365-6669AB4C7787}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1043,6 +1045,18 @@ Global {2EF143A5-E769-42FD-8B7E-7C01C381BC13}.Release|x64.Build.0 = Release|Any CPU {2EF143A5-E769-42FD-8B7E-7C01C381BC13}.Release|x86.ActiveCfg = Release|Any CPU {2EF143A5-E769-42FD-8B7E-7C01C381BC13}.Release|x86.Build.0 = Release|Any CPU + {4F2CAFC7-113A-4D31-A365-6669AB4C7787}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4F2CAFC7-113A-4D31-A365-6669AB4C7787}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4F2CAFC7-113A-4D31-A365-6669AB4C7787}.Debug|x64.ActiveCfg = Debug|Any CPU + {4F2CAFC7-113A-4D31-A365-6669AB4C7787}.Debug|x64.Build.0 = Debug|Any CPU + {4F2CAFC7-113A-4D31-A365-6669AB4C7787}.Debug|x86.ActiveCfg = Debug|Any CPU + {4F2CAFC7-113A-4D31-A365-6669AB4C7787}.Debug|x86.Build.0 = Debug|Any CPU + {4F2CAFC7-113A-4D31-A365-6669AB4C7787}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4F2CAFC7-113A-4D31-A365-6669AB4C7787}.Release|Any CPU.Build.0 = Release|Any CPU + {4F2CAFC7-113A-4D31-A365-6669AB4C7787}.Release|x64.ActiveCfg = Release|Any CPU + {4F2CAFC7-113A-4D31-A365-6669AB4C7787}.Release|x64.Build.0 = Release|Any CPU + {4F2CAFC7-113A-4D31-A365-6669AB4C7787}.Release|x86.ActiveCfg = Release|Any CPU + {4F2CAFC7-113A-4D31-A365-6669AB4C7787}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1127,6 +1141,7 @@ Global {33D5D084-FABB-4F65-A046-1C6626DF9AFB} = {350F72AB-142D-4AAD-9EF1-1A83DC991D87} {84627DC0-7B43-480D-80CF-A4DDA514A4EE} = {9B52418E-BF18-4D25-BE17-4B56D3FB1154} {2EF143A5-E769-42FD-8B7E-7C01C381BC13} = {350F72AB-142D-4AAD-9EF1-1A83DC991D87} + {4F2CAFC7-113A-4D31-A365-6669AB4C7787} = {350F72AB-142D-4AAD-9EF1-1A83DC991D87} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {3B9E5B72-F580-465A-A22C-2D2148AF4EB4} From 818e3902372bd8dc1bf4bed38a10f800d3fd5a7e Mon Sep 17 00:00:00 2001 From: David Boike Date: Fri, 15 Sep 2023 15:08:16 -0500 Subject: [PATCH 17/94] Switch reference to RavenDB 5 --- .../ServiceControl.AcceptanceTests.RavenDB5.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/ServiceControl.AcceptanceTests.RavenDB5.csproj b/src/ServiceControl.AcceptanceTests.RavenDB5/ServiceControl.AcceptanceTests.RavenDB5.csproj index be29578e9e..56ca657cf8 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB5/ServiceControl.AcceptanceTests.RavenDB5.csproj +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/ServiceControl.AcceptanceTests.RavenDB5.csproj @@ -8,7 +8,7 @@ - + From a20464642598797e8db4665609a8a93adf59a520 Mon Sep 17 00:00:00 2001 From: David Boike Date: Fri, 15 Sep 2023 15:15:00 -0500 Subject: [PATCH 18/94] Simple namespace fixes --- .../Legacy/AuditRetention.cs | 2 +- .../Legacy/When_processed_message_is_still_available.cs | 2 +- .../Recoverability/MessageFailures/FailedErrorsController.cs | 2 +- .../MessageFailures/FailedMessageRetriesController.cs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/Legacy/AuditRetention.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/Legacy/AuditRetention.cs index 0297c2ccaa..27d186050c 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB5/Legacy/AuditRetention.cs +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/Legacy/AuditRetention.cs @@ -10,7 +10,7 @@ using NServiceBus.AcceptanceTesting; using NServiceBus.CustomChecks; using NUnit.Framework; - using Raven.Client; + using Raven.Client.Documents; using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.AcceptanceTests; using ServiceControl.AcceptanceTests.TestSupport.EndpointTemplates; diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/Legacy/When_processed_message_is_still_available.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/Legacy/When_processed_message_is_still_available.cs index 51db54d943..d856d836a7 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB5/Legacy/When_processed_message_is_still_available.cs +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/Legacy/When_processed_message_is_still_available.cs @@ -15,7 +15,7 @@ using NServiceBus.AcceptanceTesting; using NUnit.Framework; using Operations.BodyStorage; - using Raven.Client; + using Raven.Client.Documents; using ServiceControl.Persistence; using TestSupport.EndpointTemplates; diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/FailedErrorsController.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/FailedErrorsController.cs index d266f8d60b..8033147c0c 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/FailedErrorsController.cs +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/FailedErrorsController.cs @@ -9,7 +9,7 @@ using Infrastructure.RavenDB.Expiration; using Infrastructure.WebApi; using Operations; - using Raven.Client; + using Raven.Client.Documents; using Raven.Client.Embedded; public class FailedErrorsCountReponse diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/FailedMessageRetriesController.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/FailedMessageRetriesController.cs index e2ac53e9b2..c7acda23a8 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/FailedMessageRetriesController.cs +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/FailedMessageRetriesController.cs @@ -5,7 +5,7 @@ using System.Threading.Tasks; using System.Web.Http; using Infrastructure.WebApi; - using Raven.Client; + using Raven.Client.Documents; using ServiceControl.Recoverability; public class FailedMessageRetriesCountReponse From 7befe4536156aa9a48362a4c6ea61e976af41493 Mon Sep 17 00:00:00 2001 From: David Boike Date: Fri, 15 Sep 2023 15:19:34 -0500 Subject: [PATCH 19/94] Remove audit-related tests --- .../Legacy/AuditRetention.cs | 113 ----------------- ...en_processed_message_is_still_available.cs | 114 ------------------ 2 files changed, 227 deletions(-) delete mode 100644 src/ServiceControl.AcceptanceTests.RavenDB5/Legacy/AuditRetention.cs delete mode 100644 src/ServiceControl.AcceptanceTests.RavenDB5/Legacy/When_processed_message_is_still_available.cs diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/Legacy/AuditRetention.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/Legacy/AuditRetention.cs deleted file mode 100644 index 27d186050c..0000000000 --- a/src/ServiceControl.AcceptanceTests.RavenDB5/Legacy/AuditRetention.cs +++ /dev/null @@ -1,113 +0,0 @@ -namespace ServiceControl.MultiInstance.AcceptanceTests.SagaAudit -{ - using System; - using System.Linq; - using System.Threading.Tasks; - using AcceptanceTesting; - using EventLog; - using Microsoft.Extensions.DependencyInjection; - using NServiceBus; - using NServiceBus.AcceptanceTesting; - using NServiceBus.CustomChecks; - using NUnit.Framework; - using Raven.Client.Documents; - using ServiceBus.Management.Infrastructure.Settings; - using ServiceControl.AcceptanceTests; - using ServiceControl.AcceptanceTests.TestSupport.EndpointTemplates; - using ServiceControl.Persistence.RavenDb.SagaAudit; - using ServiceControl.SagaAudit; - - - class AuditRetention : AcceptanceTest - { - [Test] - public async Task - Check_fails_if_no_audit_retention_is_configured() - { - ////Ensure custom checks are enabled - SetSettings = settings => - { - settings.DisableHealthChecks = false; - settings.AuditRetentionPeriod = TimeSpan.FromSeconds(5); - }; - - //Override the configuration of the check in the container in order to make it run more frequently for testing purposes. - CustomizeHostBuilder = hostBuilder => hostBuilder.ConfigureServices((ctx, services) => - services.AddTransient(provider => new AuditRetentionCustomCheck(provider.GetRequiredService(), provider.GetRequiredService(), TimeSpan.FromSeconds(10)))); - - SingleResult customCheckEventEntry = default; - bool sagaAudiDataInMainInstanceIsAvailableForQuery = false; - - await Define() - .WithEndpoint(b => b.When((bus, c) => bus.SendLocal(new MessageInitiatingSaga { Id = "Id" }))) - .Done(async c => - { - if (!c.SagaId.HasValue) - { - return false; - } - - if (sagaAudiDataInMainInstanceIsAvailableForQuery == false) - { - var sagaData = - await this.TryGet($"/api/sagas/{c.SagaId}"); - sagaAudiDataInMainInstanceIsAvailableForQuery = sagaData.HasResult; - return false; - } - - customCheckEventEntry = await this.TryGetSingle("/api/eventlogitems/", - e => e.EventType == "CustomCheckFailed" && e.Description.StartsWith("Saga Audit Data Retention")); - - return customCheckEventEntry; - }) - .Run(); - - Assert.IsTrue(customCheckEventEntry.Item.RelatedTo.Any(item => item == "/customcheck/Saga Audit Data Retention"), "Event log entry should be related to the Saga Audit Data Retention"); - Assert.IsTrue(customCheckEventEntry.Item.RelatedTo.Any(item => item.StartsWith("/endpoint/Particular.ServiceControl")), "Event log entry should be related to the ServiceControl endpoint"); - } - - public class SagaEndpoint : EndpointConfigurationBuilder - { - public SagaEndpoint() - { - EndpointSetup(c => - { - c.AuditSagaStateChanges(Settings.DEFAULT_SERVICE_NAME); - }); - } - - public class MySaga : Saga, IAmStartedByMessages - { - public MyContext TestContext { get; set; } - - public Task Handle(MessageInitiatingSaga message, IMessageHandlerContext context) - { - TestContext.SagaId = Data.Id; - return Task.CompletedTask; - } - - protected override void ConfigureHowToFindSaga(SagaPropertyMapper mapper) - { - mapper.ConfigureMapping(msg => msg.Id).ToSaga(saga => saga.MessageId); - } - } - - public class MySagaData : ContainSagaData - { - public string MessageId { get; set; } - } - } - - - public class MessageInitiatingSaga : ICommand - { - public string Id { get; set; } - } - - - public class MyContext : ScenarioContext - { - public Guid? SagaId { get; set; } - } - } -} \ No newline at end of file diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/Legacy/When_processed_message_is_still_available.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/Legacy/When_processed_message_is_still_available.cs deleted file mode 100644 index d856d836a7..0000000000 --- a/src/ServiceControl.AcceptanceTests.RavenDB5/Legacy/When_processed_message_is_still_available.cs +++ /dev/null @@ -1,114 +0,0 @@ -namespace ServiceControl.AcceptanceTests.RavenDB.Legacy -{ - using System; - using System.Collections.Generic; - using System.Text; - using System.Threading; - using System.Threading.Tasks; - using AcceptanceTesting; - using AcceptanceTests; - using CompositeViews.Messages; - using MessageAuditing; - using Microsoft.Extensions.DependencyInjection; - using Microsoft.Extensions.Hosting; - using NServiceBus; - using NServiceBus.AcceptanceTesting; - using NUnit.Framework; - using Operations.BodyStorage; - using Raven.Client.Documents; - using ServiceControl.Persistence; - using TestSupport.EndpointTemplates; - - class When_processed_message_is_still_available : AcceptanceTest - { - [Test] - public async Task Should_be_accessible_via_the_rest_api() - { - var messageId = Guid.NewGuid().ToString(); - - CustomizeHostBuilder = hostBuilder - => hostBuilder.ConfigureServices((hostBuilderContext, services) - => services.AddHostedService(sp => new CreateMessageDataMigration(messageId, sp.GetRequiredService()))); - - MessagesView auditedMessage = null; - byte[] body = null; - - var context = await Define() - .WithEndpoint() - .Done(async c => - { - var result = await this.TryGetSingle("/api/messages?include_system_messages=false&sort=id", m => m.MessageId == messageId); - auditedMessage = result; - if (!result) - { - return false; - } - - body = await this.DownloadData(auditedMessage.BodyUrl); - - return true; - }) - .Run(); - - Assert.AreEqual(messageId, auditedMessage.MessageId); - Assert.AreEqual(MessageStatus.Successful, auditedMessage.Status); - - var bodyAsString = Encoding.UTF8.GetString(body); - - Assert.AreEqual("Some Content", bodyAsString); - } - - class CreateMessageDataMigration : IHostedService - { - readonly string messageId; - readonly IDocumentStore store; - - public CreateMessageDataMigration(string messageId, IDocumentStore store) - { - this.messageId = messageId; - this.store = store; - } - - public async Task StartAsync(CancellationToken cancellationToken) - { - using (var documentSession = store.OpenAsyncSession()) - { - var processedMessage = new ProcessedMessage - { - MessageMetadata = new Dictionary - { - {"Body", "Some Content"}, - {"ContentLength", 11}, - {"BodyUrl", string.Format(BodyStorageEnricher.BodyUrlFormatString, messageId)}, - {"BodyNotStored", false}, - {"ContentType", "text/plain" }, - {"MessageIntent", (int)MessageIntentEnum.Send}, - {"MessageId", messageId}, - }, - UniqueMessageId = messageId, - ProcessedAt = DateTime.UtcNow - }; - - await documentSession.StoreAsync(processedMessage, cancellationToken); - await documentSession.SaveChangesAsync(cancellationToken); - } - - SpinWait.SpinUntil(() => store.DatabaseCommands.GetStatistics().StaleIndexes.Length == 0, TimeSpan.FromSeconds(10)); - } - - public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; - } - - public class Endpoint : EndpointConfigurationBuilder - { - public Endpoint() - { - EndpointSetup(); - } - } - - public class MyContext : ScenarioContext - { - } - } -} \ No newline at end of file From c8bbb7f260da8c81e277e8425bc79453c4adff56 Mon Sep 17 00:00:00 2001 From: David Boike Date: Fri, 15 Sep 2023 15:24:02 -0500 Subject: [PATCH 20/94] TestsFilter --- .github/workflows/ci.yml | 2 +- src/ServiceControl.AcceptanceTests.RavenDB5/TestsFilter.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f2424f422..e5ee70b5c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: strategy: matrix: os: [ windows-2022 ] - test-category: [ Default, SqlServer, AzureServiceBus, RabbitMQ, AzureStorageQueues, MSMQ, SQS, Raven35Acceptance ] + test-category: [ Default, SqlServer, AzureServiceBus, RabbitMQ, AzureStorageQueues, MSMQ, SQS, Raven35Acceptance, Raven5Acceptance ] include: - os: windows-2022 os-name: Windows diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/TestsFilter.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/TestsFilter.cs index b207ccee13..9d940ca1b0 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB5/TestsFilter.cs +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/TestsFilter.cs @@ -1,2 +1,2 @@ // This project alone tends to take ~13 minutes on GitHub Actions runner -[assembly: IncludeInTestCategory("Raven35Acceptance")] \ No newline at end of file +[assembly: IncludeInTestCategory("Raven5Acceptance")] \ No newline at end of file From acf23a1a283d60f4f42cdd09504f62a6a994b106 Mon Sep 17 00:00:00 2001 From: David Boike Date: Fri, 15 Sep 2023 15:34:59 -0500 Subject: [PATCH 21/94] couple compile fixes --- .../MessageFailures/FailedErrorsController.cs | 14 ++++++-------- .../FailedMessageRetriesController.cs | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/FailedErrorsController.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/FailedErrorsController.cs index 8033147c0c..668755a855 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/FailedErrorsController.cs +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/FailedErrorsController.cs @@ -6,11 +6,10 @@ using System.Threading; using System.Threading.Tasks; using System.Web.Http; - using Infrastructure.RavenDB.Expiration; using Infrastructure.WebApi; using Operations; using Raven.Client.Documents; - using Raven.Client.Embedded; + using Raven.Client.Documents.Operations; public class FailedErrorsCountReponse { @@ -40,7 +39,7 @@ public async Task GetFailedErrorsCount() { Count = count }) - .WithEtag(stats.IndexEtag); + .WithEtag(stats.ResultEtag.ToString()); } } @@ -55,12 +54,11 @@ public async Task ImportFailedErrors(CancellationToken canc [Route("failederrors/forcecleanerrors")] [HttpPost] - public Task ForceErrorMessageCleanerRun(CancellationToken cancellationToken = default) + public Task ForceErrorMessageCleanerRun() { - new ExpiryErrorMessageIndex().Execute(store); - WaitForIndexes(store); + // TODO: Is there a way to force the Raven5 expiration to happen? Or does it just happen? Won't be able to tell until we redesign that. - ErrorMessageCleaner.Clean(1000, ((EmbeddableDocumentStore)store).DocumentDatabase, DateTime.Now, cancellationToken); + // May not even need WaitForIndexes given Raven5 implementation isn't index-based WaitForIndexes(store); return Task.FromResult(Request.CreateResponse(HttpStatusCode.OK)); @@ -68,7 +66,7 @@ public Task ForceErrorMessageCleanerRun(CancellationToken c static void WaitForIndexes(IDocumentStore store) { - SpinWait.SpinUntil(() => store.DatabaseCommands.GetStatistics().StaleIndexes.Length == 0, TimeSpan.FromSeconds(10)); + SpinWait.SpinUntil(() => store.Maintenance.Send(new GetStatisticsOperation()).StaleIndexes.Length == 0, TimeSpan.FromSeconds(10)); } readonly IDocumentStore store; diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/FailedMessageRetriesController.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/FailedMessageRetriesController.cs index c7acda23a8..d313204a4f 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/FailedMessageRetriesController.cs +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/Recoverability/MessageFailures/FailedMessageRetriesController.cs @@ -32,7 +32,7 @@ public async Task GetFailedMessageRetriesCount() { Count = stats.TotalResults }) - .WithEtag(stats.IndexEtag); + .WithEtag(stats.ResultEtag.ToString()); } } From 9aaf73c24b3cac443b6c83334270b0f0d32b4304 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 20 Sep 2023 16:51:46 +0200 Subject: [PATCH 22/94] =?UTF-8?q?=F0=9F=A9=B9=20RavenDB5=20Cannot=20do=20i?= =?UTF-8?q?n=20memory=20so=20tests=20need=20to=20run=20sequentially?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/ServiceControl.Persistence.Tests/BaseHostTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ServiceControl.Persistence.Tests/BaseHostTest.cs b/src/ServiceControl.Persistence.Tests/BaseHostTest.cs index 366c415eec..1048910c80 100644 --- a/src/ServiceControl.Persistence.Tests/BaseHostTest.cs +++ b/src/ServiceControl.Persistence.Tests/BaseHostTest.cs @@ -5,7 +5,7 @@ using Microsoft.Extensions.DependencyInjection; using ServiceControl.Persistence; -[Parallelizable(ParallelScope.All)] +[Parallelizable(ParallelScope.None)] // RavenDB5 Cannot do in memory so tests need to run sequentially [FixtureLifeCycle(LifeCycle.InstancePerTestCase)] public abstract class BaseHostTest { From 42ed69246e91eb3576964a538ead9789baddbf61 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 20 Sep 2023 16:56:42 +0200 Subject: [PATCH 23/94] =?UTF-8?q?=F0=9F=A9=B9=20Persister=20lifecycle=20wa?= =?UTF-8?q?sn't=20disposed=20by=20host=20as=20it=20was=20injected=20thus?= =?UTF-8?q?=20not=20created=20via=20CI=20and=20then=20DI=20assumes=20life?= =?UTF-8?q?=20time=20is=20controlled=20by=20the=20creator?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../RavenDbEmbeddedPersistenceLifecycle.cs | 7 +++++++ .../RavenDbPersistence.cs | 14 ++++++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbEmbeddedPersistenceLifecycle.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbEmbeddedPersistenceLifecycle.cs index a162178222..cc7863ad02 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbEmbeddedPersistenceLifecycle.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbEmbeddedPersistenceLifecycle.cs @@ -1,6 +1,7 @@ namespace ServiceControl.Persistence.RavenDb5 { using System; + using System.Diagnostics; using System.Threading; using System.Threading.Tasks; using Raven.Client.Documents; @@ -33,11 +34,17 @@ public void Dispose() { documentStore?.Dispose(); database?.Dispose(); + GC.SuppressFinalize(this); } IDocumentStore documentStore; EmbeddedDatabase database; readonly RavenDBPersisterSettings databaseConfiguration; + + ~RavenDbEmbeddedPersistenceLifecycle() + { + Trace.WriteLine("ERROR: RavenDbEmbeddedPersistenceLifecycle isn't properly disposed"); + } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs index 325a54438e..ed99c0916d 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs @@ -75,18 +75,16 @@ public void ConfigureLifecycle(IServiceCollection serviceCollection) { if (settings.UseEmbeddedServer) { - var embedded = new RavenDbEmbeddedPersistenceLifecycle(settings); - - serviceCollection.AddSingleton(embedded); - serviceCollection.AddSingleton(_ => embedded.GetDocumentStore()); - + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(b => b.GetService()); + serviceCollection.AddSingleton(b => b.GetService().GetDocumentStore()); return; } - var external = new RavenDbExternalPersistenceLifecycle(settings); - serviceCollection.AddSingleton(external); - serviceCollection.AddSingleton(_ => external.GetDocumentStore()); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(b => b.GetService()); + serviceCollection.AddSingleton(b => b.GetService().GetDocumentStore()); } public IPersistenceInstaller CreateInstaller() From d9627b2e387083e2c74f8997a8c3601d729a34a7 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 20 Sep 2023 17:15:19 +0200 Subject: [PATCH 24/94] =?UTF-8?q?=F0=9F=A9=B9=20Fixed=20a=20couple=20of=20?= =?UTF-8?q?transformations,=20added=20TODO's=20for=20sort=20that=20is=20br?= =?UTF-8?q?oken?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ErrorMessagesDataStore.cs | 80 ++++++++----- .../FailedMessageViewTransformer.cs | 61 +++++----- .../Transformers/MessagesViewTransformer.cs | 113 ++++++++---------- 3 files changed, 129 insertions(+), 125 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index 19db4f2f6b..4030fca114 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -37,15 +37,19 @@ public async Task>> GetAllMessages( bool includeSystemMessages ) { + await Task.Yield(); + using (var session = documentStore.OpenAsyncSession()) { - var results = await session.Query() - .IncludeSystemMessagesWhere(includeSystemMessages) - .Statistics(out var stats) - .Sort(sortInfo) - .Paging(pagingInfo) - //.TransformWith() - .TransformToMessagesView() + var query = session.Query() + .IncludeSystemMessagesWhere(includeSystemMessages) + .Statistics(out var stats) + .Sort(sortInfo) + .Paging(pagingInfo) + .ProjectInto(); + + var results = await MessagesViewTransformer.Transform(query) + // TODO: Sort not working, seems to work if OrderBy is done here .ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); @@ -61,14 +65,16 @@ bool includeSystemMessages { using (var session = documentStore.OpenAsyncSession()) { - var results = await session.Query() + var query = session.Query() .IncludeSystemMessagesWhere(includeSystemMessages) .Where(m => m.ReceivingEndpointName == endpointName) .Statistics(out var stats) .Sort(sortInfo) .Paging(pagingInfo) - //.TransformWith() - .TransformToMessagesView() + .ProjectInto(); + + var results = await MessagesViewTransformer.Transform(query) + // TODO: Sort not working, seems to work if OrderBy is done here .ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); @@ -84,14 +90,16 @@ SortInfo sortInfo { using (var session = documentStore.OpenAsyncSession()) { - var results = await session.Query() + var query = session.Query() .Statistics(out var stats) .Search(x => x.Query, searchKeyword) .Where(m => m.ReceivingEndpointName == endpointName) .Sort(sortInfo) .Paging(pagingInfo) - //.TransformWith() - .TransformToMessagesView() + .ProjectInto(); + + var results = await MessagesViewTransformer.Transform(query) + // TODO: Sort not working, seems to work if OrderBy is done here .ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); @@ -107,13 +115,15 @@ bool includeSystemMessages { using (var session = documentStore.OpenAsyncSession()) { - var results = await session.Query() + var query = session.Query() .Statistics(out var stats) .Where(m => m.ConversationId == conversationId) .Sort(sortInfo) .Paging(pagingInfo) - //.TransformWith() - .TransformToMessagesView() + .ProjectInto(); + + var results = await MessagesViewTransformer.Transform(query) + // TODO: Sort not working, seems to work if OrderBy is done here .ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); @@ -128,13 +138,15 @@ SortInfo sortInfo { using (var session = documentStore.OpenAsyncSession()) { - var results = await session.Query() + var query = session.Query() .Statistics(out var stats) .Search(x => x.Query, searchTerms) .Sort(sortInfo) .Paging(pagingInfo) - //.TransformWith() - .TransformToMessagesView() + .ProjectInto(); + + var results = await MessagesViewTransformer.Transform(query) + // TODO: Sort not working, seems to work if OrderBy is done here .ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); @@ -233,7 +245,7 @@ SortInfo sortInfo { using (var session = documentStore.OpenAsyncSession()) { - var results = await session.Advanced + var query = session.Advanced .AsyncDocumentQuery() .Statistics(out var stats) .FilterByStatusWhere(status) @@ -241,9 +253,11 @@ SortInfo sortInfo .FilterByQueueAddress(queueAddress) .Sort(sortInfo) .Paging(pagingInfo) - // TODO: Fix SetResultTransformer - //.SetResultTransformer(new FailedMessageViewTransformer().TransformerName) - .SelectFields() + .SelectFields() + .ToQueryable(); + + var results = await FailedMessageViewTransformer.Transform(query) + // TODO: Sort not working, seems to work if OrderBy is done here .ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); @@ -279,7 +293,7 @@ SortInfo sortInfo { using (var session = documentStore.OpenAsyncSession()) { - var results = await session.Advanced + var query = session.Advanced .AsyncDocumentQuery() .Statistics(out var stats) .FilterByStatusWhere(status) @@ -288,9 +302,11 @@ SortInfo sortInfo .FilterByLastModifiedRange(modified) .Sort(sortInfo) .Paging(pagingInfo) - // TODO: Fix SetResultTransformer - //.SetResultTransformer(new FailedMessageViewTransformer().TransformerName) - .SelectFields() + .SelectFields() + .ToQueryable(); + + var results = await FailedMessageViewTransformer.Transform(query) + // TODO: Sort not working, seems to work if OrderBy is done here .ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); @@ -454,7 +470,7 @@ PagingInfo pagingInfo { using (var session = documentStore.OpenAsyncSession()) { - var results = await session.Advanced + var query = session.Advanced .AsyncDocumentQuery() .Statistics(out var stats) .WhereEquals(view => view.FailureGroupId, groupId) @@ -462,9 +478,11 @@ PagingInfo pagingInfo .FilterByLastModifiedRange(modified) .Sort(sortInfo) .Paging(pagingInfo) - // TODO: Fix SetResultTransformer - //.SetResultTransformer(FailedMessageViewTransformer.Name) - .SelectFields() + .SelectFields() + .ToQueryable(); + + var results = await FailedMessageViewTransformer.Transform(query) + // TODO: Sort not working, seems to work if OrderBy is done here .ToListAsync(); return results.ToQueryResult(stats); diff --git a/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs b/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs index 77c425d968..e20884ce05 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs @@ -2,44 +2,37 @@ { using System; using System.Linq; - using ServiceControl.CompositeViews.Messages; + using Raven.Client.Documents.Linq; + using Raven.Client.Documents.Queries; - class FailedMessageViewTransformer : FakeAbstractTransformerCreationTask // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers + class FailedMessageViewTransformer // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers { - public FailedMessageViewTransformer() + public static IQueryable Transform(IRavenQueryable query) { - TransformResults = failures => from failure in failures - let rec = failure.ProcessingAttempts.Last() - let edited = rec.Headers["ServiceControl.EditOf"] != null - select new - { - Id = failure.UniqueMessageId, - MessageType = rec.MessageMetadata["MessageType"], - IsSystemMessage = (bool)rec.MessageMetadata["IsSystemMessage"], - SendingEndpoint = rec.MessageMetadata["SendingEndpoint"], - ReceivingEndpoint = rec.MessageMetadata["ReceivingEndpoint"], - TimeSent = (DateTime?)rec.MessageMetadata["TimeSent"], - MessageId = rec.MessageMetadata["MessageId"], - rec.FailureDetails.Exception, - QueueAddress = rec.FailureDetails.AddressOfFailingEndpoint, - NumberOfProcessingAttempts = failure.ProcessingAttempts.Count, - failure.Status, - rec.FailureDetails.TimeOfFailure, - //LastModified = MetadataFor(failure)["@last-modified"].Value(), - Edited = edited, - EditOf = edited ? rec.Headers["ServiceControl.EditOf"] : "" - }; - } - - public static string Name - { - get + var failures = + from failure in query + let rec = failure.ProcessingAttempts.Last() + let edited = rec.Headers["ServiceControl.EditOf"] != null + select new { - transformerName ??= new FailedMessageViewTransformer().TransformerName; - return transformerName; - } - } + Id = failure.UniqueMessageId, + MessageType = rec.MessageMetadata["MessageType"], + IsSystemMessage = (bool)rec.MessageMetadata["IsSystemMessage"], + SendingEndpoint = rec.MessageMetadata["SendingEndpoint"], + ReceivingEndpoint = rec.MessageMetadata["ReceivingEndpoint"], + TimeSent = (DateTime?)rec.MessageMetadata["TimeSent"], + MessageId = rec.MessageMetadata["MessageId"], + rec.FailureDetails.Exception, + QueueAddress = rec.FailureDetails.AddressOfFailingEndpoint, + NumberOfProcessingAttempts = failure.ProcessingAttempts.Count, + failure.Status, + rec.FailureDetails.TimeOfFailure, + LastModified = RavenQuery.LastModified(failure),// MetadataFor(failure)["@last-modified"].Value(), + Edited = edited, + EditOf = edited ? rec.Headers["ServiceControl.EditOf"] : "" + }; - static string transformerName; + return failures.OfType(); + } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs index 4f4f58fc94..932251cfa7 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs @@ -1,73 +1,67 @@ namespace ServiceControl.CompositeViews.Messages { using System; - using System.Collections; using System.Collections.Generic; using System.Linq; - using System.Linq.Expressions; using MessageFailures; using Raven.Client.Documents.Linq; using ServiceControl.Persistence; - class FakeAbstractTransformerCreationTask + class MessagesViewTransformer //https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers { - public Expression, IEnumerable>> TransformResults; - public string TransformerName; - } - - class MessagesViewTransformer : FakeAbstractTransformerCreationTask // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers - { - public MessagesViewTransformer() + public static IQueryable Transform(IRavenQueryable query) { - TransformResults = messages => from message in messages - let metadata = + var results = + from message in query + let metadata = message.ProcessingAttempts != null + ? message.ProcessingAttempts.Last().MessageMetadata + : message.MessageMetadata + let headers = + message.ProcessingAttempts != null ? message.ProcessingAttempts.Last().Headers : message.Headers + let processedAt = message.ProcessingAttempts != null - ? message.ProcessingAttempts.Last().MessageMetadata - : message.MessageMetadata - let headers = - message.ProcessingAttempts != null ? message.ProcessingAttempts.Last().Headers : message.Headers - let processedAt = - message.ProcessingAttempts != null - ? message.ProcessingAttempts.Last().AttemptedAt - : message.ProcessedAt - let status = - message.ProcessingAttempts == null - ? !(bool)message.MessageMetadata["IsRetried"] - ? MessageStatus.Successful - : MessageStatus.ResolvedSuccessfully - : message.Status == FailedMessageStatus.Resolved - ? MessageStatus.ResolvedSuccessfully - : message.Status == FailedMessageStatus.RetryIssued - ? MessageStatus.RetryIssued - : message.Status == FailedMessageStatus.Archived - ? MessageStatus.ArchivedFailure - : message.ProcessingAttempts.Count == 1 - ? MessageStatus.Failed - : MessageStatus.RepeatedFailure - select new - { - Id = message.UniqueMessageId, - MessageId = metadata["MessageId"], - MessageType = metadata["MessageType"], - SendingEndpoint = metadata["SendingEndpoint"], - ReceivingEndpoint = metadata["ReceivingEndpoint"], - TimeSent = (DateTime?)metadata["TimeSent"], - ProcessedAt = processedAt, - CriticalTime = (TimeSpan)metadata["CriticalTime"], - ProcessingTime = (TimeSpan)metadata["ProcessingTime"], - DeliveryTime = (TimeSpan)metadata["DeliveryTime"], - IsSystemMessage = (bool)metadata["IsSystemMessage"], - ConversationId = metadata["ConversationId"], - //the reason the we need to use a KeyValuePair is that raven seems to interpret the values and convert them - // to real types. In this case it was the NServiceBus.Temporary.DelayDeliveryWith header to was converted to a timespan - Headers = headers.Select(header => new KeyValuePair(header.Key, header.Value)), - Status = status, - MessageIntent = metadata["MessageIntent"], - BodyUrl = metadata["BodyUrl"], - BodySize = (int)metadata["ContentLength"], - InvokedSagas = metadata["InvokedSagas"], - OriginatesFromSaga = metadata["OriginatesFromSaga"] - }; + ? message.ProcessingAttempts.Last().AttemptedAt + : message.ProcessedAt + let status = + message.ProcessingAttempts == null + ? !(bool)message.MessageMetadata["IsRetried"] + ? MessageStatus.Successful + : MessageStatus.ResolvedSuccessfully + : message.Status == FailedMessageStatus.Resolved + ? MessageStatus.ResolvedSuccessfully + : message.Status == FailedMessageStatus.RetryIssued + ? MessageStatus.RetryIssued + : message.Status == FailedMessageStatus.Archived + ? MessageStatus.ArchivedFailure + : message.ProcessingAttempts.Count == 1 + ? MessageStatus.Failed + : MessageStatus.RepeatedFailure + select new // Cannot use type here as this is projected server-side + { + Id = message.UniqueMessageId, + MessageId = metadata["MessageId"], + MessageType = metadata["MessageType"], + SendingEndpoint = metadata["SendingEndpoint"], + ReceivingEndpoint = metadata["ReceivingEndpoint"], + TimeSent = metadata["TimeSent"], + ProcessedAt = processedAt, + CriticalTime = metadata["CriticalTime"], + ProcessingTime = metadata["ProcessingTime"], + DeliveryTime = metadata["DeliveryTime"], + IsSystemMessage = metadata["IsSystemMessage"], + ConversationId = metadata["ConversationId"], + //the reason the we need to use a KeyValuePair is that raven seems to interpret the values and convert them + // to real types. In this case it was the NServiceBus.Temporary.DelayDeliveryWith header to was converted to a timespan + Headers = headers.Select(header => new { header.Key, header.Value }), + Status = status, + MessageIntent = metadata["MessageIntent"], + BodyUrl = metadata["BodyUrl"], + BodySize = metadata["ContentLength"], + InvokedSagas = metadata["InvokedSagas"], + OriginatesFromSaga = metadata["OriginatesFromSaga"] + }; + + return results.OfType(); } public class Input @@ -89,5 +83,4 @@ public static IQueryable TransformToMessagesView(this IQueryable new MessagesView()); } } -} - +} \ No newline at end of file From 2c325a1444b236fed5bec8a9002a61676baeae99 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 20 Sep 2023 17:30:32 +0200 Subject: [PATCH 25/94] =?UTF-8?q?=F0=9F=A9=B9=20Some=20simple=20tests=20to?= =?UTF-8?q?=20prep=20the=20database=20with=20known=20documents=20for=20tes?= =?UTF-8?q?ting=20queries?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ErrorMessageDataStoreTests.cs | 162 ++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/ErrorMessageDataStoreTests.cs diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/ErrorMessageDataStoreTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/ErrorMessageDataStoreTests.cs new file mode 100644 index 0000000000..615d88ef8c --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/ErrorMessageDataStoreTests.cs @@ -0,0 +1,162 @@ +using System; +using System.Threading.Tasks; +using NUnit.Framework; +using Raven.Client.Documents; +using ServiceControl.Contracts.Operations; +using ServiceControl.MessageFailures; +using ServiceControl.MessageFailures.Api; +using ServiceControl.Operations; +using ServiceControl.Persistence; +using ServiceControl.Persistence.Infrastructure; +using ServiceControl.PersistenceTests; +using ServiceControl.SagaAudit; + +[TestFixture] +class ErrorMessageDataStoreTests : PersistenceTestBase +{ + IDocumentStore documentStore; + IErrorMessageDataStore store; + FailedMessage processedMessage1, processedMessage2; + + [Test] + public async Task GetAllMessages() + { + var result = await store.GetAllMessages(new PagingInfo(1, 50), new SortInfo("", ""), false); + Assert.IsNotEmpty(result.Results); + } + + [Test] + [TestCase("", "asc", "a")] + [TestCase("", "dsc", "b")] + [TestCase("message_id", "asc", "a")] + [TestCase("message_id", "dsc", "b")] + [TestCase("critical_time", "asc", "a")] + [TestCase("critical_time", "dsc", "b")] + public async Task GetAllMessagesForEndpoint(string sort, string direction, string id) + { + var result = await store.GetAllMessagesForEndpoint( + "RamonAndTomek", + new PagingInfo(1, 1), + new SortInfo(sort, direction), + false + ); + + Assert.IsNotEmpty(result.Results); + Assert.AreEqual(1, result.Results.Count); + Assert.AreEqual(id, result.Results[0].Id); + } + + [Test] + public async Task ErrorGet() + { + var result = await store.ErrorGet(null, null, null, new PagingInfo(1, 50), new SortInfo("", "")); + Assert.IsNotEmpty(result.Results); + } + + [SetUp] + public async Task GetStore() + { + await Task.Yield(); + await SetupDocumentStore(); + await GenerateAndSaveFailedMessage(); + await CompleteDatabaseOperation(); + + store = GetRequiredService(); + } + + async Task GenerateAndSaveFailedMessage() + { + using (var session = documentStore.OpenAsyncSession()) + { + processedMessage1 = new FailedMessage + { + Id = "1", + UniqueMessageId = "a", + Status = FailedMessageStatus.Unresolved, + ProcessingAttempts = + { + new FailedMessage.ProcessingAttempt + { + Headers = + { + ["Tomek"]="Wizard", + ["Ramon"]="Cool", + }, + AttemptedAt = DateTime.UtcNow, + MessageMetadata = + { + ["TimeSent"]="2023-09-20T12:00:00", + ["MessageId"]="x", + ["MessageType"]="MyType", + ["SendingEndpoint"]=new EndpointDetails{Host="host", HostId = Guid.NewGuid(), Name="RamonAndTomek"}, + ["ReceivingEndpoint"]=new EndpointDetails{Host="host", HostId = Guid.NewGuid(), Name="RamonAndTomek"}, + ["ConversationId"]="abc", + ["MessageIntent"]="Send", + ["BodyUrl"]="https://particular.net", + ["ContentLength"]=11111, + ["InvokedSagas"]=new[]{new SagaInfo{ChangeStatus = "YES!",SagaId = Guid.NewGuid(), SagaType = "XXX.YYY, RamonAndTomek"}}, + ["OriginatesFromSaga"]=new SagaInfo{ChangeStatus = "YES!",SagaId = Guid.NewGuid(), SagaType = "XXX.YYY, RamonAndTomek"}, + ["CriticalTime"]=TimeSpan.FromSeconds(5), + ["ProcessingTime"]=TimeSpan.FromSeconds(5), + ["DeliveryTime"]=TimeSpan.FromSeconds(5), + ["IsSystemMessage"]=false, + }, + FailureDetails = new FailureDetails() + } + } + }; + await session.StoreAsync(processedMessage1); + + processedMessage2 = new FailedMessage + { + Id = "2", + UniqueMessageId = "b", + Status = FailedMessageStatus.Unresolved, + ProcessingAttempts = + { + new FailedMessage.ProcessingAttempt + { + Headers = + { + ["Tomek"]="Wizard", + ["Ramon"]="Cool", + }, + AttemptedAt = DateTime.UtcNow, + MessageMetadata = + { + ["TimeSent"]="2023-09-20T12:00:05", + ["MessageId"]="y", + ["MessageType"]="MyType", + ["SendingEndpoint"]=new EndpointDetails{Host="host", HostId = Guid.NewGuid(), Name="RamonAndTomek"}, + ["ReceivingEndpoint"]=new EndpointDetails{Host="host", HostId = Guid.NewGuid(), Name="RamonAndTomek"}, + ["ConversationId"]="abc", + ["MessageIntent"]="Send", + ["BodyUrl"]="https://particular.net", + ["ContentLength"]=22222, + ["InvokedSagas"]=new[]{new SagaInfo{ChangeStatus = "YES!",SagaId = Guid.NewGuid(), SagaType = "XXX.YYY, RamonAndTomek"}}, + ["OriginatesFromSaga"]=new SagaInfo{ChangeStatus = "YES!",SagaId = Guid.NewGuid(), SagaType = "XXX.YYY, RamonAndTomek"}, + ["CriticalTime"]=TimeSpan.FromSeconds(15), + ["ProcessingTime"]=TimeSpan.FromSeconds(15), + ["DeliveryTime"]=TimeSpan.FromSeconds(15), + ["IsSystemMessage"]=false, + }, + FailureDetails = new FailureDetails() + } + } + }; + await session.StoreAsync(processedMessage2); + + await session.SaveChangesAsync(); + } + } + + async Task SetupDocumentStore() + { + documentStore = GetRequiredService(); + var customIndex = new FailedMessageViewIndex(); + await customIndex.ExecuteAsync(documentStore); + //var transformer = new FailedMessageViewTransformer(); + //TODO: we need to bring this back + //transformer.Execute(documentStore); + } +} \ No newline at end of file From 623a55e361c59fc49f5c94a58ade76034ad26485 Mon Sep 17 00:00:00 2001 From: David Boike Date: Wed, 20 Sep 2023 15:13:48 -0500 Subject: [PATCH 26/94] Provide config for database to start in test --- .github/workflows/ci.yml | 2 +- .../EmbeddedDatabaseTests.cs | 12 ++++++++++-- .../TestsFilter.cs | 1 + 3 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/TestsFilter.cs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5ee70b5c0..aa8362948b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: strategy: matrix: os: [ windows-2022 ] - test-category: [ Default, SqlServer, AzureServiceBus, RabbitMQ, AzureStorageQueues, MSMQ, SQS, Raven35Acceptance, Raven5Acceptance ] + test-category: [ Default, SqlServer, AzureServiceBus, RabbitMQ, AzureStorageQueues, MSMQ, SQS, Raven35Acceptance, Raven5Acceptance, Raven5Persistence ] include: - os: windows-2022 os-name: Windows diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/EmbeddedDatabaseTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/EmbeddedDatabaseTests.cs index 836655a338..e306075183 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/EmbeddedDatabaseTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/EmbeddedDatabaseTests.cs @@ -1,4 +1,6 @@ -using System.Threading; +using System; +using System.IO; +using System.Threading; using System.Threading.Tasks; using NUnit.Framework; using ServiceControl.Persistence.Infrastructure; @@ -11,7 +13,13 @@ class EmbeddedDatabaseTests [Test] public async Task CanStart() { - using (var embeddedDatabase = EmbeddedDatabase.Start(new RavenDBPersisterSettings { })) + var settings = new RavenDBPersisterSettings + { + DatabasePath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Tests", "Embedded"), + LogPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()) + }; + + using (var embeddedDatabase = EmbeddedDatabase.Start(settings)) { using (var documentStore = await embeddedDatabase.Connect(CancellationToken.None)) diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/TestsFilter.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/TestsFilter.cs new file mode 100644 index 0000000000..25445220fe --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/TestsFilter.cs @@ -0,0 +1 @@ +[assembly: IncludeInTestCategory("Raven5Persistence")] \ No newline at end of file From 8257dfb3f04677506e24af386d6d7bbc3c934c2c Mon Sep 17 00:00:00 2001 From: David Boike Date: Wed, 20 Sep 2023 15:14:04 -0500 Subject: [PATCH 27/94] Change migration tests to a type that still exists in primary instance --- .../MigrationTests.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/MigrationTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/MigrationTests.cs index a72931c15b..360a33c3ac 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/MigrationTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/MigrationTests.cs @@ -10,10 +10,10 @@ class MigrationTests public void VerifyMigration() { var serializedForm = @"{ -""$type"":""ServiceControl.SagaAudit.SagaInfo, ServiceControl"", -""ChangeStatus"":null, -""SagaType"":null, -""SagaId"":""00000000-0000-0000-0000-000000000000""}"; +""$type"":""ServiceControl.WhateverNamespace.CustomCheck, ServiceControl"", +""Id"":""SomeId"", +""CustomCheckId"":""CustomCheckId"", +""Category"":""SomeCategory""}"; var serializer = new JsonSerializer { From 369b43200c17ec2a90d68c3a54d2bb9e7f8791be Mon Sep 17 00:00:00 2001 From: David Boike Date: Wed, 20 Sep 2023 15:27:09 -0500 Subject: [PATCH 28/94] Fix controller dependencies test --- .../ControllerDependencies.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/ServiceControl.Persistence.Tests/ControllerDependencies.cs b/src/ServiceControl.Persistence.Tests/ControllerDependencies.cs index c9229bec63..9d9f891438 100644 --- a/src/ServiceControl.Persistence.Tests/ControllerDependencies.cs +++ b/src/ServiceControl.Persistence.Tests/ControllerDependencies.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Net.Http; using System.Reflection; + using System.Threading.Tasks; using System.Web.Http.Controllers; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -24,7 +25,7 @@ class ControllerDependencies /// instantiate each of the WebAPI controllers present in the ServiceControl app. /// [Test] - public void EnsurePersistenceProvidesAllControllerDependencies() + public async Task EnsurePersistenceProvidesAllControllerDependencies() { // Arrange var testPersistence = new TestPersistenceImpl(); @@ -60,6 +61,13 @@ public void EnsurePersistenceProvidesAllControllerDependencies() // Assert Assert.That(host, Is.Not.Null); + // TODO: Kind of a hack, but gets the job done, since Raven35/5 have different startup requirements + // Could come up with a more cohesive way of doing this or just remove the IF when Raven35 goes away + if (testPersistence.GetType().Assembly.FullName.Contains("RavenDb5")) + { + await host.Services.GetRequiredService().Initialize(); + } + // Make sure the list isn't suddenly empty Assert.That(controllerTypes.Length, Is.GreaterThan(10)); foreach (var controllerType in controllerTypes) From b01214f864ad6004c2d0559fa0573cea3b01dcc0 Mon Sep 17 00:00:00 2001 From: David Boike Date: Wed, 20 Sep 2023 15:34:36 -0500 Subject: [PATCH 29/94] That test wasn't even right on Raven35 --- .../MagicStringTypeTests.cs | 4 ++-- .../MagicStringTypeTests.cs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ServiceControl.Persistence.Tests.RavenDb/MagicStringTypeTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb/MagicStringTypeTests.cs index 25ecee97b3..f673516455 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb/MagicStringTypeTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb/MagicStringTypeTests.cs @@ -10,10 +10,10 @@ class MagicStringTypeTests [Test] public Task Verify_ravendb_persistence_type_string() { - var typeNamespace = DataStoreConfig.RavenDB35PersistenceTypeFullyQualifiedName.Split(',')[1].Trim(); + var assemblyName = DataStoreConfig.RavenDB35PersistenceTypeFullyQualifiedName.Split(',')[1].Trim(); var typeFullName = DataStoreConfig.RavenDB35PersistenceTypeFullyQualifiedName.Split(',')[0].Trim(); var type = typeof(RavenDbPersistenceConfiguration); - Assert.AreEqual(type.Namespace, typeNamespace); + Assert.AreEqual(type.Assembly.GetName().Name, assemblyName); Assert.AreEqual(type.FullName, typeFullName); return Task.CompletedTask; } diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/MagicStringTypeTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/MagicStringTypeTests.cs index 39bd0f3d3d..884bde8791 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/MagicStringTypeTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/MagicStringTypeTests.cs @@ -10,10 +10,10 @@ class MagicStringTypeTests [Test] public Task Verify_ravendb_persistence_type_string() { - var typeNamespace = DataStoreConfig.RavenDB5PersistenceTypeFullyQualifiedName.Split(',')[1].Trim(); + var assemblyName = DataStoreConfig.RavenDB5PersistenceTypeFullyQualifiedName.Split(',')[1].Trim(); var typeFullName = DataStoreConfig.RavenDB5PersistenceTypeFullyQualifiedName.Split(',')[0].Trim(); var type = typeof(RavenDbPersistenceConfiguration); - Assert.AreEqual(type.Namespace, typeNamespace); + Assert.AreEqual(type.Assembly.GetName().Name, assemblyName); Assert.AreEqual(type.FullName, typeFullName); return Task.CompletedTask; } From b92d7febc50c198f17e1681de8772757a0291525 Mon Sep 17 00:00:00 2001 From: David Boike Date: Wed, 20 Sep 2023 15:56:39 -0500 Subject: [PATCH 30/94] Make sure to find all the indexes --- .../DatabaseSetup.cs | 34 +++++++------------ 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/DatabaseSetup.cs b/src/ServiceControl.Persistence.RavenDb5/DatabaseSetup.cs index 0b53d5782c..03840a5784 100644 --- a/src/ServiceControl.Persistence.RavenDb5/DatabaseSetup.cs +++ b/src/ServiceControl.Persistence.RavenDb5/DatabaseSetup.cs @@ -1,6 +1,7 @@ namespace ServiceControl.Persistence.RavenDb5 { - using System.Collections.Generic; + using System; + using System.Linq; using System.Threading; using System.Threading.Tasks; using Raven.Client.Documents; @@ -11,10 +12,6 @@ using Raven.Client.Exceptions.Database; using Raven.Client.ServerWide; using Raven.Client.ServerWide.Operations; - using ServiceControl.MessageFailures.Api; - using ServiceControl.Operations; - using ServiceControl.Persistence; - using ServiceControl.Recoverability; class DatabaseSetup { @@ -42,24 +39,11 @@ await documentStore.Maintenance.Server } } - var indexList = new List { - new ArchivedGroupsViewIndex(), - new CustomChecksIndex(), - new FailedErrorImportIndex(), - new FailedMessageFacetsIndex(), - new FailedMessageRetries_ByBatch(), - new FailedMessageViewIndex(), - new FailureGroupsViewIndex(), - new GroupCommentIndex(), - new KnownEndpointIndex(), - new MessagesViewIndex(), - new QueueAddressIndex(), - new RetryBatches_ByStatusAndSession(), - new RetryBatches_ByStatus_ReduceInitialBatchSize() + var indexTypes = typeof(DatabaseSetup).Assembly.GetTypes() + .Where(t => typeof(IAbstractIndexCreationTask).IsAssignableFrom(t)) + .ToList(); - }; - - //TODO: Handle full text search + //TODO: Handle full text search - if necessary add Where clause to query above to remove the two variants //if (settings.EnableFullTextSearch) //{ // indexList.Add(new MessagesViewIndexWithFullTextSearch()); @@ -72,6 +56,12 @@ await documentStore.Maintenance.Server // .SendAsync(new DeleteIndexOperation("MessagesViewIndexWithFullTextSearch"), cancellationToken); //} + var indexList = indexTypes + .Select(t => Activator.CreateInstance(t)) + .OfType(); + + // If no full-text vs not full-text index is required, this can all be simplified using the assembly-based override + // await IndexCreation.CreateIndexesAsync(typeof(DatabaseSetup).Assembly, documentStore, null, null, cancellationToken); await IndexCreation.CreateIndexesAsync(indexList, documentStore, null, null, cancellationToken); var expirationConfig = new ExpirationConfiguration From cfe38d5d58db3d6f0b41b8de7f5405fbae91b528 Mon Sep 17 00:00:00 2001 From: David Boike Date: Wed, 20 Sep 2023 15:56:53 -0500 Subject: [PATCH 31/94] That patch can't be null --- .../RetryDocumentDataStore.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs index d387f658e6..c651ba837e 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RetryDocumentDataStore.cs @@ -124,7 +124,7 @@ public async Task> QueryAvailableBatches() static ICommandData CreateFailedMessageRetryDocument(string batchDocumentId, string messageId) { - return new PatchCommandData(FailedMessageRetry.MakeDocumentId(messageId), null, patch: null, patchIfMissing: new PatchRequest + var patchRequest = new PatchRequest { Script = @"this.FailedMessageId = args.MessageId this.RetryBatchId = args.BatchDocumentId", @@ -133,7 +133,9 @@ static ICommandData CreateFailedMessageRetryDocument(string batchDocumentId, str { "MessageId", FailedMessageIdGenerator.MakeDocumentId(messageId) }, { "BatchDocumentId", batchDocumentId } } - }); + }; + + return new PatchCommandData(FailedMessageRetry.MakeDocumentId(messageId), null, patch: patchRequest, patchIfMissing: patchRequest); } static ILog log = LogManager.GetLogger(typeof(RetryDocumentDataStore)); From 92dac320f49ce26b16184c8da609a8df11b1567f Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 21 Sep 2023 16:11:30 +0200 Subject: [PATCH 32/94] =?UTF-8?q?=E2=9A=9C=EF=B8=8F=20cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Recoverability/FailedMessageRetryTests.cs | 1 - .../Recoverability/EditMessageTests.cs | 2 -- 2 files changed, 3 deletions(-) diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/FailedMessageRetryTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/FailedMessageRetryTests.cs index 8019c78618..dcf4b2a18d 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/FailedMessageRetryTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/FailedMessageRetryTests.cs @@ -2,7 +2,6 @@ { using System; using NUnit.Framework; - using ServiceControl.MessageFailures; using ServiceControl.Recoverability; [TestFixture] diff --git a/src/ServiceControl.Persistence.Tests/Recoverability/EditMessageTests.cs b/src/ServiceControl.Persistence.Tests/Recoverability/EditMessageTests.cs index 3c38181c58..b5bf21a349 100644 --- a/src/ServiceControl.Persistence.Tests/Recoverability/EditMessageTests.cs +++ b/src/ServiceControl.Persistence.Tests/Recoverability/EditMessageTests.cs @@ -88,8 +88,6 @@ public async Task Should_discard_edit_when_different_edit_already_exists() // Act await handler.Handle(message, new TestableMessageHandlerContext()); - BlockToInspectDatabase(); - using (var editFailedMessagesManagerAssert = await ErrorMessageDataStore.CreateEditFailedMessageManager()) { var failedMessage = await editFailedMessagesManagerAssert.GetFailedMessage(failedMessageId); From 9fb11f323418895ace3a04a659dba875b6584613 Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Thu, 21 Sep 2023 13:28:25 +0200 Subject: [PATCH 33/94] Refactored PersistenceTestBase to manage a single EmbeddedDatabase instance for all tests to significantly speed up RavenDB 5 persister tests --- .../MessageRedirectsDataStore.cs | 1 - .../RavenDBPersisterSettings.cs | 2 +- .../Archiving/ArchiveGroupTests.cs | 13 +- .../FailedErrorImportCustomCheckTests.cs | 8 +- .../RetryConfirmationProcessorTests.cs | 2 +- .../ReturnToSenderDequeuerTests.cs | 9 +- ...ceControl.Persistence.Tests.RavenDb.csproj | 4 +- .../TestPersistenceImpl.cs | 11 +- .../Unarchiving/UnarchiveGroupTests.cs | 13 +- .../Archiving/ArchiveGroupTests.cs | 13 +- .../CompositeViews/FailedMessagesTests.cs | 1 - .../CompositeViews/MessagesViewTests.cs | 1 - .../ErrorMessageDataStoreTests.cs | 4 +- .../FailedErrorImportCustomCheckTests.cs | 9 +- .../RetryConfirmationProcessorTests.cs | 2 +- .../ReturnToSenderDequeuerTests.cs | 8 +- ...eControl.Persistence.Tests.RavenDb5.csproj | 4 +- .../SetUpFixture.cs | 12 ++ .../SharedEmbeddedServer.cs | 62 ++++++++++ .../SubscriptionPersisterTests.cs | 1 - .../TestPersistenceImpl.cs | 67 +++------- .../Unarchiving/UnarchiveGroupTests.cs | 12 +- .../BaseHostTest.cs | 37 ------ .../ControllerDependencies.cs | 16 +-- .../CustomChecksDataStoreTests.cs | 10 +- .../MonitoringDataStoreTests.cs | 20 +-- .../PersistenceTestBase.cs | 114 ++++++++++-------- .../Recoverability/EditMessageTests.cs | 14 +-- .../RetryStateTests.cs | 12 +- .../TestPersistence.cs | 6 +- 30 files changed, 257 insertions(+), 231 deletions(-) create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/SetUpFixture.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/SharedEmbeddedServer.cs delete mode 100644 src/ServiceControl.Persistence.Tests/BaseHostTest.cs diff --git a/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs index cc9ac0f7db..6a4bf179b4 100644 --- a/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs @@ -1,6 +1,5 @@ namespace ServiceControl.Persistence.RavenDb.MessageRedirects { - using System; using System.Threading.Tasks; using Raven.Client.Documents; using Raven.Client.Documents.Queries; diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs index a23954884b..f59ebe9fbb 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs @@ -18,7 +18,7 @@ class RavenDBPersisterSettings : PersistenceSettings public int ExternalIntegrationsDispatchingBatchSize { get; set; } = 100; //TODO: these are newly added settings, we should remove any duplication - public string ServerUrl { get; set; } + public string ServerUrl { get; set; } // TODO: This name is SUPER confusing public string ConnectionString { get; set; } public bool UseEmbeddedServer => string.IsNullOrWhiteSpace(ConnectionString); public string LogPath { get; set; } diff --git a/src/ServiceControl.Persistence.Tests.RavenDb/Archiving/ArchiveGroupTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb/Archiving/ArchiveGroupTests.cs index 41b1cb33c1..ab8b7ad705 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb/Archiving/ArchiveGroupTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb/Archiving/ArchiveGroupTests.cs @@ -3,11 +3,9 @@ using System; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; - using Microsoft.Extensions.Hosting; using NServiceBus.Testing; using NUnit.Framework; using Raven.Client; - using ServiceControl.PersistenceTests; using ServiceControl.Recoverability; [TestFixture] @@ -15,11 +13,14 @@ class ArchiveGroupTests : PersistenceTestBase { IDocumentStore DocumentStore => GetRequiredService(); - protected override IHostBuilder CreateHostBuilder() => base.CreateHostBuilder().ConfigureServices(services => + public ArchiveGroupTests() { - services.AddSingleton(); - services.AddSingleton(); - }); + RegisterServices = services => + { + services.AddSingleton(); + services.AddSingleton(); + }; + } [Test] public async Task ArchiveGroup_skips_over_empty_batches_but_still_completes() diff --git a/src/ServiceControl.Persistence.Tests.RavenDb/Operations/FailedErrorImportCustomCheckTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb/Operations/FailedErrorImportCustomCheckTests.cs index 0cde2d8569..746db357b8 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb/Operations/FailedErrorImportCustomCheckTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb/Operations/FailedErrorImportCustomCheckTests.cs @@ -2,10 +2,8 @@ { using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; - using Microsoft.Extensions.Hosting; using NServiceBus.CustomChecks; using NUnit.Framework; - using PersistenceTests; using Raven.Client; using ServiceControl.Operations; @@ -14,10 +12,10 @@ class FailedErrorImportCustomCheckTests : PersistenceTestBase { IDocumentStore DocumentStore => GetRequiredService(); - protected override IHostBuilder CreateHostBuilder() => base.CreateHostBuilder().ConfigureServices(services => + public FailedErrorImportCustomCheckTests() { - services.AddSingleton(); - }); + RegisterServices = services => services.AddSingleton(); + } [Test] public async Task Pass_if_no_failed_imports() diff --git a/src/ServiceControl.Persistence.Tests.RavenDb/Recoverability/RetryConfirmationProcessorTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb/Recoverability/RetryConfirmationProcessorTests.cs index ee092c8b3f..76fe530aac 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb/Recoverability/RetryConfirmationProcessorTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb/Recoverability/RetryConfirmationProcessorTests.cs @@ -20,7 +20,7 @@ class RetryConfirmationProcessorTests : PersistenceTestBase LegacyMessageFailureResolvedHandler Handler { get; set; } [SetUp] - public new async Task Setup() + public async Task Setup() { var domainEvents = new FakeDomainEvents(); Processor = new RetryConfirmationProcessor(domainEvents); diff --git a/src/ServiceControl.Persistence.Tests.RavenDb/Recoverability/ReturnToSenderDequeuerTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb/Recoverability/ReturnToSenderDequeuerTests.cs index 0ac1e9d507..e1f451365a 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb/Recoverability/ReturnToSenderDequeuerTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb/Recoverability/ReturnToSenderDequeuerTests.cs @@ -9,14 +9,12 @@ using System.Threading.Tasks; using MessageFailures; using Microsoft.Extensions.DependencyInjection; - using Microsoft.Extensions.Hosting; using NServiceBus.Extensibility; using NServiceBus.Transport; using NUnit.Framework; using Raven.Client; using ServiceControl.CompositeViews.Messages; using ServiceControl.Operations.BodyStorage; - using ServiceControl.PersistenceTests; using ServiceControl.Recoverability; [TestFixture] @@ -34,7 +32,10 @@ MessageContext CreateMessage(string id, Dictionary headers) ); } - protected override IHostBuilder CreateHostBuilder() => base.CreateHostBuilder().ConfigureServices(services => services.AddSingleton()); + public ReturnToSenderDequeuerTests() + { + RegisterServices = services => services.AddSingleton(); + } [Test] public async Task It_removes_staging_id_header() @@ -106,7 +107,7 @@ public async Task It_fetches_the_body_from_index_if_provided() var transformer = new MessagesBodyTransformer(); await transformer.ExecuteAsync(documentStore); - await CompleteDatabaseOperation(); + CompleteDatabaseOperation(); var instance = GetRequiredService(); // See this.CreateHostBuilder diff --git a/src/ServiceControl.Persistence.Tests.RavenDb/ServiceControl.Persistence.Tests.RavenDb.csproj b/src/ServiceControl.Persistence.Tests.RavenDb/ServiceControl.Persistence.Tests.RavenDb.csproj index 15bcfda682..7ac1cf8c74 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb/ServiceControl.Persistence.Tests.RavenDb.csproj +++ b/src/ServiceControl.Persistence.Tests.RavenDb/ServiceControl.Persistence.Tests.RavenDb.csproj @@ -1,4 +1,4 @@ - + net472 @@ -26,7 +26,7 @@ - + diff --git a/src/ServiceControl.Persistence.Tests.RavenDb/TestPersistenceImpl.cs b/src/ServiceControl.Persistence.Tests.RavenDb/TestPersistenceImpl.cs index 20abc166c5..ad71aa9658 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb/TestPersistenceImpl.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb/TestPersistenceImpl.cs @@ -42,17 +42,14 @@ static RavenDBPersisterSettings CreateSettings() public override void Configure(IServiceCollection services) { var persistence = new RavenDbPersistenceConfiguration().Create(CreateSettings()); - PersistenceHostBuilderExtensions.CreatePersisterLifecyle(services, persistence); - services.AddHostedService(p => new Wrapper(this, p.GetRequiredService())); } - public override Task CompleteDatabaseOperation() + public override void CompleteDatabaseOperation() { Assert.IsNotNull(documentStore); documentStore.WaitForIndexing(); - return Task.CompletedTask; } class Wrapper : IHostedService @@ -95,5 +92,11 @@ public override void BlockToInspectDatabase() Trace.Write("Waiting for debugger pause"); } } + + public override Task TearDown() + { + // Nothing to teardown + return Task.CompletedTask; + } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb/Unarchiving/UnarchiveGroupTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb/Unarchiving/UnarchiveGroupTests.cs index 34e47ffeab..6c1df1edf9 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb/Unarchiving/UnarchiveGroupTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb/Unarchiving/UnarchiveGroupTests.cs @@ -3,10 +3,8 @@ using System; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; - using Microsoft.Extensions.Hosting; using NServiceBus.Testing; using NUnit.Framework; - using PersistenceTests; using Raven.Client; using ServiceControl.Recoverability; @@ -15,11 +13,14 @@ class UnarchiveGroupTests : PersistenceTestBase { IDocumentStore DocumentStore => GetRequiredService(); - protected override IHostBuilder CreateHostBuilder() => base.CreateHostBuilder().ConfigureServices(services => + public UnarchiveGroupTests() { - services.AddSingleton(); - services.AddSingleton(); - }); + RegisterServices = services => + { + services.AddSingleton(); + services.AddSingleton(); + }; + } [Test] public async Task UnarchiveGroup_skips_over_empty_batches_but_still_completes() diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Archiving/ArchiveGroupTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Archiving/ArchiveGroupTests.cs index 616bb9cea7..c084978197 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/Archiving/ArchiveGroupTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Archiving/ArchiveGroupTests.cs @@ -3,11 +3,9 @@ using System; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; - using Microsoft.Extensions.Hosting; using NServiceBus.Testing; using NUnit.Framework; using Raven.Client.Documents; - using ServiceControl.PersistenceTests; using ServiceControl.Recoverability; [TestFixture] @@ -15,11 +13,12 @@ class ArchiveGroupTests : PersistenceTestBase { IDocumentStore DocumentStore => GetRequiredService(); - protected override IHostBuilder CreateHostBuilder() => base.CreateHostBuilder().ConfigureServices(services => - { - services.AddSingleton(); - services.AddSingleton(); - }); + public ArchiveGroupTests() => + RegisterServices = services => + { + services.AddSingleton(); + services.AddSingleton(); + }; [Test] public async Task ArchiveGroup_skips_over_empty_batches_but_still_completes() diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/FailedMessagesTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/FailedMessagesTests.cs index e490a06b8a..31759b5d6a 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/FailedMessagesTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/FailedMessagesTests.cs @@ -8,7 +8,6 @@ using MessageFailures; using MessageFailures.Api; using NUnit.Framework; - using PersistenceTests; using Raven.Client.Documents; using Raven.Client.Documents.Session; diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs index d884a6b70b..6ce2d488ea 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs @@ -7,7 +7,6 @@ using MessageAuditing; using MessageFailures; using NUnit.Framework; - using PersistenceTests; using Raven.Client.Documents; using Raven.Client.Documents.Linq; using ServiceControl.Persistence; diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/ErrorMessageDataStoreTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/ErrorMessageDataStoreTests.cs index 615d88ef8c..49e020ba50 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/ErrorMessageDataStoreTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/ErrorMessageDataStoreTests.cs @@ -8,7 +8,6 @@ using ServiceControl.Operations; using ServiceControl.Persistence; using ServiceControl.Persistence.Infrastructure; -using ServiceControl.PersistenceTests; using ServiceControl.SagaAudit; [TestFixture] @@ -59,7 +58,8 @@ public async Task GetStore() await Task.Yield(); await SetupDocumentStore(); await GenerateAndSaveFailedMessage(); - await CompleteDatabaseOperation(); + + CompleteDatabaseOperation(); store = GetRequiredService(); } diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedErrorImportCustomCheckTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedErrorImportCustomCheckTests.cs index 8a1d037a56..88c977a484 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedErrorImportCustomCheckTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedErrorImportCustomCheckTests.cs @@ -14,10 +14,11 @@ class FailedErrorImportCustomCheckTests : PersistenceTestBase { IDocumentStore DocumentStore => GetRequiredService(); - protected override IHostBuilder CreateHostBuilder() => base.CreateHostBuilder().ConfigureServices(services => - { - services.AddSingleton(); - }); + public FailedErrorImportCustomCheckTests() => + RegisterServices = services => + { + services.AddSingleton(); + }; [Test] public async Task Pass_if_no_failed_imports() diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/RetryConfirmationProcessorTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/RetryConfirmationProcessorTests.cs index ee092c8b3f..76fe530aac 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/RetryConfirmationProcessorTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/RetryConfirmationProcessorTests.cs @@ -20,7 +20,7 @@ class RetryConfirmationProcessorTests : PersistenceTestBase LegacyMessageFailureResolvedHandler Handler { get; set; } [SetUp] - public new async Task Setup() + public async Task Setup() { var domainEvents = new FakeDomainEvents(); Processor = new RetryConfirmationProcessor(domainEvents); diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/ReturnToSenderDequeuerTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/ReturnToSenderDequeuerTests.cs index 934e345d54..074edd62d9 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/ReturnToSenderDequeuerTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Recoverability/ReturnToSenderDequeuerTests.cs @@ -7,16 +7,11 @@ using System.Text; using System.Threading; using System.Threading.Tasks; - using MessageFailures; using Microsoft.Extensions.DependencyInjection; - using Microsoft.Extensions.Hosting; using NServiceBus.Extensibility; using NServiceBus.Transport; using NUnit.Framework; - using Raven.Client.Documents; - using ServiceControl.CompositeViews.Messages; using ServiceControl.Operations.BodyStorage; - using ServiceControl.PersistenceTests; using ServiceControl.Recoverability; [TestFixture] @@ -34,7 +29,8 @@ MessageContext CreateMessage(string id, Dictionary headers) ); } - protected override IHostBuilder CreateHostBuilder() => base.CreateHostBuilder().ConfigureServices(services => services.AddSingleton()); + public ReturnToSenderDequeuerTests() => RegisterServices = services => services.AddSingleton(); + [Test] public async Task It_removes_staging_id_header() diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/ServiceControl.Persistence.Tests.RavenDb5.csproj b/src/ServiceControl.Persistence.Tests.RavenDb5/ServiceControl.Persistence.Tests.RavenDb5.csproj index f846159034..1aa7242079 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/ServiceControl.Persistence.Tests.RavenDb5.csproj +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/ServiceControl.Persistence.Tests.RavenDb5.csproj @@ -1,4 +1,4 @@ - + net472 @@ -26,7 +26,7 @@ - + diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/SetUpFixture.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/SetUpFixture.cs new file mode 100644 index 0000000000..e2035b815c --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/SetUpFixture.cs @@ -0,0 +1,12 @@ +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.RavenDb5; + +[SetUpFixture] +public class SetUpFixture +{ + public static EmbeddedDatabase SharedInstance; + // Needs to be in a SetUpFixture otherwise the OneTimeSetUp is invoked for each inherited test fixture + [OneTimeSetUp] + public static async Task SetupSharedEmbeddedServer() => SharedInstance = await SharedEmbeddedServer.GetInstance(); +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/SharedEmbeddedServer.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/SharedEmbeddedServer.cs new file mode 100644 index 0000000000..abe5f63f82 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/SharedEmbeddedServer.cs @@ -0,0 +1,62 @@ +using System; +using System.IO; +using System.Linq; +using System.Net.NetworkInformation; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.RavenDb5; + +static class SharedEmbeddedServer +{ + public static async Task GetInstance(CancellationToken cancellationToken = default) + { + var settings = new RavenDBPersisterSettings + { + DatabasePath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Tests", "ErrorData"), + LogPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Logs"), + LogsMode = "Operations", + ServerUrl = $"http://localhost:{FindAvailablePort(33334)}" + }; + + Directory.Delete(settings.DatabasePath, true); + + var instance = EmbeddedDatabase.Start(settings); + + //make sure that the database is up + while (true) + { + try + { + using (await instance.Connect(cancellationToken)) + { + //no-op + } + + return instance; + } + catch (Exception) + { + await Task.Delay(500, cancellationToken); + } + } + } + + static int FindAvailablePort(int startPort) + { + var activeTcpListeners = IPGlobalProperties + .GetIPGlobalProperties() + .GetActiveTcpListeners(); + + for (var port = startPort; port < startPort + 1024; port++) + { + var portCopy = port; + if (activeTcpListeners.All(endPoint => endPoint.Port != portCopy)) + { + return port; + } + } + + return startPort; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/SubscriptionPersisterTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/SubscriptionPersisterTests.cs index a8fc2d33ca..d6e0ddeeac 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/SubscriptionPersisterTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/SubscriptionPersisterTests.cs @@ -7,7 +7,6 @@ using NServiceBus.Unicast.Subscriptions; using NServiceBus.Unicast.Subscriptions.MessageDrivenSubscriptions; using NUnit.Framework; - using PersistenceTests; using Raven.Client.Documents; using ServiceControl.Infrastructure.RavenDB.Subscriptions; diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs index b0e8f8ed24..f488400668 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs @@ -2,9 +2,7 @@ { using System; using System.Diagnostics; - using System.IO; using System.Linq; - using System.Net.NetworkInformation; using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; @@ -13,73 +11,44 @@ using NUnit.Framework; using Persistence; using Raven.Client.Documents; + using Raven.Client.ServerWide.Operations; using ServiceControl.Persistence.RavenDb; sealed class TestPersistenceImpl : TestPersistence { - readonly RavenDBPersisterSettings settings = CreateSettings(); + // Deterministic database name make diagnosing easier + readonly string databaseName = new string(TestContext.CurrentContext.Test.FullName.Where(c => char.IsLetterOrDigit(c) || c == '_' || c == '-').ToArray()); + + readonly RavenDBPersisterSettings settings; IDocumentStore documentStore; - static RavenDBPersisterSettings CreateSettings() + public TestPersistenceImpl() { var retentionPeriod = TimeSpan.FromMinutes(1); - var settings = new RavenDBPersisterSettings + settings = new RavenDBPersisterSettings { AuditRetentionPeriod = retentionPeriod, ErrorRetentionPeriod = retentionPeriod, EventsRetentionPeriod = retentionPeriod, - RunInMemory = true, - DatabasePath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Tests", "ErrorData"), - LogPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Logs"), - LogsMode = "Operations", - ServerUrl = $"http://localhost:{FindAvailablePort(33334)}" + ConnectionString = SetUpFixture.SharedInstance.ServerUrl, + DatabaseName = databaseName }; - - if (Debugger.IsAttached) - { - Console.WriteLine("If you get 'Access is denied' exception while debugging, comment out this line or create a URLACL reservervation:"); - Console.WriteLine("> netsh http add urlacl http://+:55554/ user=Everyone"); - settings.ExposeRavenDB = true; - } - - return settings; } public override void Configure(IServiceCollection services) { - var persistence = new RavenDbPersistenceConfiguration().Create(CreateSettings()); - + var persistence = new RavenDbPersistenceConfiguration().Create(settings); PersistenceHostBuilderExtensions.CreatePersisterLifecyle(services, persistence); - services.AddHostedService(p => new Wrapper(this, p.GetRequiredService())); } - public override Task CompleteDatabaseOperation() + public override void CompleteDatabaseOperation() { Assert.IsNotNull(documentStore); documentStore.WaitForIndexing(); - return Task.CompletedTask; } - //TODO: this method is duplicated at least 3 times in the test project - static int FindAvailablePort(int startPort) - { - var activeTcpListeners = IPGlobalProperties - .GetIPGlobalProperties() - .GetActiveTcpListeners(); - - for (var port = startPort; port < startPort + 1024; port++) - { - var portCopy = port; - if (activeTcpListeners.All(endPoint => endPoint.Port != portCopy)) - { - return port; - } - } - - return startPort; - } class Wrapper : IHostedService { public Wrapper(TestPersistenceImpl instance, IDocumentStore store) @@ -98,7 +67,7 @@ public override void BlockToInspectDatabase() return; } - var url = $"http://localhost:{settings.DatabaseMaintenancePort}/studio/index.html#databases/documents?&database=%3Csystem%3E"; + var url = SetUpFixture.SharedInstance.ServerUrl + "/studio/index.html#databases/documents?&database=" + databaseName; if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { @@ -114,11 +83,13 @@ public override void BlockToInspectDatabase() Process.Start("open", url); } - while (true) - { - Thread.Sleep(5000); - Trace.Write("Waiting for debugger pause"); - } + Debugger.Break(); + } + + public override async Task TearDown() + { + var deleteDatabasesOperation = new DeleteDatabasesOperation(new DeleteDatabasesOperation.Parameters { DatabaseNames = new[] { databaseName }, HardDelete = true }); + await documentStore.Maintenance.Server.SendAsync(deleteDatabasesOperation); } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Unarchiving/UnarchiveGroupTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Unarchiving/UnarchiveGroupTests.cs index 39485d4e75..a8a8763a4e 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/Unarchiving/UnarchiveGroupTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Unarchiving/UnarchiveGroupTests.cs @@ -6,7 +6,6 @@ using Microsoft.Extensions.Hosting; using NServiceBus.Testing; using NUnit.Framework; - using PersistenceTests; using Raven.Client.Documents; using ServiceControl.Recoverability; @@ -15,11 +14,12 @@ class UnarchiveGroupTests : PersistenceTestBase { IDocumentStore DocumentStore => GetRequiredService(); - protected override IHostBuilder CreateHostBuilder() => base.CreateHostBuilder().ConfigureServices(services => - { - services.AddSingleton(); - services.AddSingleton(); - }); + public UnarchiveGroupTests() => + RegisterServices = services => + { + services.AddSingleton(); + services.AddSingleton(); + }; [Test] public async Task UnarchiveGroup_skips_over_empty_batches_but_still_completes() diff --git a/src/ServiceControl.Persistence.Tests/BaseHostTest.cs b/src/ServiceControl.Persistence.Tests/BaseHostTest.cs deleted file mode 100644 index 1048910c80..0000000000 --- a/src/ServiceControl.Persistence.Tests/BaseHostTest.cs +++ /dev/null @@ -1,37 +0,0 @@ -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using NUnit.Framework; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using ServiceControl.Persistence; - -[Parallelizable(ParallelScope.None)] // RavenDB5 Cannot do in memory so tests need to run sequentially -[FixtureLifeCycle(LifeCycle.InstancePerTestCase)] -public abstract class BaseHostTest -{ - readonly IHost testHost; - - protected BaseHostTest() => testHost = CreateHostBuilder().Build(); - - [SetUp] - public async Task SetUp() - { - await GetRequiredService().Initialize(); - await testHost.StartAsync(); - } - - [TearDown] - public async Task TearDown() - { - await testHost.StopAsync(); - testHost.Dispose(); - } - - protected T GetRequiredService() => testHost.Services.GetRequiredService(); - - protected virtual IHostBuilder CreateHostBuilder() => Host.CreateDefaultBuilder() - .ConfigureLogging((hostingContext, logging) => - { - logging.AddConsole(); - }); -} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests/ControllerDependencies.cs b/src/ServiceControl.Persistence.Tests/ControllerDependencies.cs index 9d9f891438..e091b14605 100644 --- a/src/ServiceControl.Persistence.Tests/ControllerDependencies.cs +++ b/src/ServiceControl.Persistence.Tests/ControllerDependencies.cs @@ -25,10 +25,10 @@ class ControllerDependencies /// instantiate each of the WebAPI controllers present in the ServiceControl app. /// [Test] - public async Task EnsurePersistenceProvidesAllControllerDependencies() + public Task EnsurePersistenceProvidesAllControllerDependencies() { // Arrange - var testPersistence = new TestPersistenceImpl(); + //var testPersistence = null; //TODO: new TestPersistenceImpl(); var assembly = Assembly.GetAssembly(typeof(WebApiHostBuilderExtensions)); var controllerTypes = assembly.DefinedTypes @@ -43,7 +43,7 @@ public async Task EnsurePersistenceProvidesAllControllerDependencies() serviceCollection.AddSingleton(); serviceCollection.AddSingleton(new LoggingSettings("test")); - testPersistence.Configure(serviceCollection); + //testPersistence.Configure(serviceCollection); }) .UseNServiceBus(_ => { @@ -63,10 +63,10 @@ public async Task EnsurePersistenceProvidesAllControllerDependencies() // TODO: Kind of a hack, but gets the job done, since Raven35/5 have different startup requirements // Could come up with a more cohesive way of doing this or just remove the IF when Raven35 goes away - if (testPersistence.GetType().Assembly.FullName.Contains("RavenDb5")) - { - await host.Services.GetRequiredService().Initialize(); - } + //if (testPersistence.GetType().Assembly.FullName.Contains("RavenDb5")) + //{ + // await host.Services.GetRequiredService().Initialize(); + //} // Make sure the list isn't suddenly empty Assert.That(controllerTypes.Length, Is.GreaterThan(10)); @@ -75,6 +75,8 @@ public async Task EnsurePersistenceProvidesAllControllerDependencies() Console.WriteLine($"Getting service {controllerType.FullName}"); Assert.That(host.Services.GetService(controllerType), Is.Not.Null); } + + return Task.CompletedTask; } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests/CustomChecksDataStoreTests.cs b/src/ServiceControl.Persistence.Tests/CustomChecksDataStoreTests.cs index 589dfa591d..663ac4bf6d 100644 --- a/src/ServiceControl.Persistence.Tests/CustomChecksDataStoreTests.cs +++ b/src/ServiceControl.Persistence.Tests/CustomChecksDataStoreTests.cs @@ -30,7 +30,7 @@ public async Task CustomChecks_load_from_data_store() var status = await CustomChecks.UpdateCustomCheckStatus(checkDetails); - await CompleteDatabaseOperation(); + CompleteDatabaseOperation(); var stats = await CustomChecks.GetStats(new PagingInfo()); Assert.AreEqual(1, stats.Results.Count); @@ -58,7 +58,7 @@ public async Task Storing_failed_custom_checks_returns_unchanged() var statusInitial = await CustomChecks.UpdateCustomCheckStatus(checkDetails); var statusUpdate = await CustomChecks.UpdateCustomCheckStatus(checkDetails); - await CompleteDatabaseOperation(); + CompleteDatabaseOperation(); Assert.AreEqual(CheckStateChange.Changed, statusInitial); Assert.AreEqual(CheckStateChange.Unchanged, statusUpdate); @@ -83,7 +83,7 @@ public async Task Retrieving_custom_checks_by_status() var _ = await CustomChecks.UpdateCustomCheckStatus(checkDetails); - await CompleteDatabaseOperation(); + CompleteDatabaseOperation(); var stats = await CustomChecks.GetStats(new PagingInfo(), "pass"); @@ -110,11 +110,11 @@ public async Task Should_delete_custom_checks() var _ = await CustomChecks.UpdateCustomCheckStatus(checkDetails); - await CompleteDatabaseOperation(); + CompleteDatabaseOperation(); await CustomChecks.DeleteCustomCheck(checkId); - await CompleteDatabaseOperation(); + CompleteDatabaseOperation(); var storedChecks = await CustomChecks.GetStats(new PagingInfo()); var check = storedChecks.Results.Where(c => c.Id == checkId.ToString()).ToList(); diff --git a/src/ServiceControl.Persistence.Tests/MonitoringDataStoreTests.cs b/src/ServiceControl.Persistence.Tests/MonitoringDataStoreTests.cs index df1e3e8c15..ec35e04cf0 100644 --- a/src/ServiceControl.Persistence.Tests/MonitoringDataStoreTests.cs +++ b/src/ServiceControl.Persistence.Tests/MonitoringDataStoreTests.cs @@ -17,7 +17,7 @@ public async Task Endpoints_load_from_dataStore_into_monitor() var endpoint1 = new EndpointDetails() { HostId = Guid.NewGuid(), Host = "Host1", Name = "Name1" }; await MonitoringDataStore.CreateIfNotExists(endpoint1); - await CompleteDatabaseOperation(); + CompleteDatabaseOperation(); await MonitoringDataStore.WarmupMonitoringFromPersistence(endpointInstanceMonitoring); Assert.AreEqual(1, endpointInstanceMonitoring.GetKnownEndpoints().Count(w => w.HostDisplayName == endpoint1.Host)); @@ -31,7 +31,7 @@ public async Task Endpoints_added_more_than_once_are_treated_as_same_endpoint() await MonitoringDataStore.CreateIfNotExists(endpoint1); await MonitoringDataStore.CreateIfNotExists(endpoint1); - await CompleteDatabaseOperation(); + CompleteDatabaseOperation(); await MonitoringDataStore.WarmupMonitoringFromPersistence(endpointInstanceMonitoring); Assert.AreEqual(1, endpointInstanceMonitoring.GetKnownEndpoints().Count(w => w.HostDisplayName == endpoint1.Host)); @@ -45,7 +45,7 @@ public async Task Updating_existing_endpoint_does_not_create_new_ones() await MonitoringDataStore.CreateIfNotExists(endpoint1); await MonitoringDataStore.CreateOrUpdate(endpoint1, endpointInstanceMonitoring); - await CompleteDatabaseOperation(); + CompleteDatabaseOperation(); await MonitoringDataStore.WarmupMonitoringFromPersistence(endpointInstanceMonitoring); Assert.AreEqual(1, endpointInstanceMonitoring.GetKnownEndpoints().Count(w => w.HostDisplayName == endpoint1.Host)); @@ -60,7 +60,7 @@ public async Task Endpoint_is_created_if_doesnt_exist() await MonitoringDataStore.CreateIfNotExists(endpoint1); await MonitoringDataStore.CreateIfNotExists(endpoint2); - await CompleteDatabaseOperation(); + CompleteDatabaseOperation(); await MonitoringDataStore.WarmupMonitoringFromPersistence(endpointInstanceMonitoring); Assert.AreEqual(2, endpointInstanceMonitoring.GetKnownEndpoints().Count(w => w.HostDisplayName == endpoint1.Host || w.HostDisplayName == endpoint2.Host)); @@ -75,7 +75,7 @@ public async Task Endpoint_is_created_if_doesnt_exist_on_update() await MonitoringDataStore.CreateIfNotExists(endpoint1); await MonitoringDataStore.CreateOrUpdate(endpoint2, endpointInstanceMonitoring); - await CompleteDatabaseOperation(); + CompleteDatabaseOperation(); await MonitoringDataStore.WarmupMonitoringFromPersistence(endpointInstanceMonitoring); Assert.AreEqual(2, endpointInstanceMonitoring.GetKnownEndpoints().Count(w => w.HostDisplayName == endpoint1.Host || w.HostDisplayName == endpoint2.Host)); @@ -88,14 +88,14 @@ public async Task Endpoint_is_updated() var endpoint1 = new EndpointDetails() { HostId = Guid.NewGuid(), Host = "Host1", Name = "Name1" }; await MonitoringDataStore.CreateIfNotExists(endpoint1); - await CompleteDatabaseOperation(); + CompleteDatabaseOperation(); await MonitoringDataStore.WarmupMonitoringFromPersistence(endpointInstanceMonitoring); Assert.IsFalse(endpointInstanceMonitoring.IsMonitored(endpointInstanceMonitoring.GetEndpoints()[0].Id)); await MonitoringDataStore.UpdateEndpointMonitoring(endpoint1, true); endpointInstanceMonitoring = new EndpointInstanceMonitoring(new FakeDomainEvents()); - await CompleteDatabaseOperation(); + CompleteDatabaseOperation(); await MonitoringDataStore.WarmupMonitoringFromPersistence(endpointInstanceMonitoring); Assert.IsTrue(endpointInstanceMonitoring.IsMonitored(endpointInstanceMonitoring.GetEndpoints()[0].Id)); } @@ -108,7 +108,7 @@ public async Task Endpoint_is_deleted() var endpoint1 = new EndpointDetails() { HostId = Guid.NewGuid(), Host = "Host1", Name = "Name1" }; await MonitoringDataStore.CreateIfNotExists(endpoint1); - await CompleteDatabaseOperation(); + CompleteDatabaseOperation(); await MonitoringDataStore.WarmupMonitoringFromPersistence(endpointInstanceMonitoring); Assert.AreEqual(1, endpointInstanceMonitoring.GetKnownEndpoints().Count(w => w.HostDisplayName == endpoint1.Host)); @@ -116,7 +116,7 @@ public async Task Endpoint_is_deleted() endpointInstanceMonitoring = new EndpointInstanceMonitoring(new FakeDomainEvents()); - await CompleteDatabaseOperation(); + CompleteDatabaseOperation(); await MonitoringDataStore.WarmupMonitoringFromPersistence(endpointInstanceMonitoring); Assert.AreEqual(0, endpointInstanceMonitoring.GetKnownEndpoints().Count(w => w.HostDisplayName == endpoint1.Host)); } @@ -139,7 +139,7 @@ public async Task Unit_of_work_detects_endpoint() await unitOfWork.Complete(); } - await CompleteDatabaseOperation(); + CompleteDatabaseOperation(); var knownEndpoints = await MonitoringDataStore.GetAllKnownEndpoints(); diff --git a/src/ServiceControl.Persistence.Tests/PersistenceTestBase.cs b/src/ServiceControl.Persistence.Tests/PersistenceTestBase.cs index 57763eda30..8f22230651 100644 --- a/src/ServiceControl.Persistence.Tests/PersistenceTestBase.cs +++ b/src/ServiceControl.Persistence.Tests/PersistenceTestBase.cs @@ -1,56 +1,76 @@ -namespace ServiceControl.PersistenceTests +using System; +using System.Diagnostics; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using NUnit.Framework; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Persistence; +using NServiceBus; +using ServiceControl.Infrastructure.DomainEvents; +using ServiceControl.PersistenceTests; +using ServiceControl.Operations.BodyStorage; +using ServiceControl.Persistence.MessageRedirects; +using ServiceControl.Persistence.Recoverability; +using ServiceControl.Persistence.UnitOfWork; + +//[Parallelizable(ParallelScope.All)] +[FixtureLifeCycle(LifeCycle.InstancePerTestCase)] +public abstract class PersistenceTestBase { - using System.Diagnostics; - using System.Threading.Tasks; - using Infrastructure.DomainEvents; - using Microsoft.Extensions.DependencyInjection; - using Microsoft.Extensions.Hosting; - using NServiceBus; - using NUnit.Framework; - using Persistence; - using Persistence.MessageRedirects; - using ServiceControl.Operations.BodyStorage; - using ServiceControl.Persistence.Recoverability; - using ServiceControl.Persistence.UnitOfWork; - - abstract class PersistenceTestBase : BaseHostTest - { - TestPersistence testPersistence; + IHost host; + readonly TestPersistence testPersistence = new TestPersistenceImpl(); - protected override IHostBuilder CreateHostBuilder() - { - return base.CreateHostBuilder().ConfigureServices(services => + [SetUp] + public async Task SetUp() + { + var hostBuilder = Host.CreateDefaultBuilder() + .ConfigureLogging((hostingContext, logging) => + { + logging.AddConsole(); + }).ConfigureServices(services => { services.AddSingleton(); services.AddSingleton(new CriticalError(null)); - testPersistence = new TestPersistenceImpl(); + testPersistence.Configure(services); + RegisterServices?.Invoke(services); }); - } - - [SetUp] - public virtual Task Setup() - { - return CompleteDatabaseOperation(); - } - - protected Task CompleteDatabaseOperation() - { - return testPersistence.CompleteDatabaseOperation(); - } - - [Conditional("DEBUG")] - protected void BlockToInspectDatabase() => testPersistence.BlockToInspectDatabase(); - - protected IErrorMessageDataStore ErrorStore => GetRequiredService(); - protected IRetryDocumentDataStore RetryStore => GetRequiredService(); - protected IBodyStorage BodyStorage => GetRequiredService(); - protected IRetryBatchesDataStore RetryBatchesStore => GetRequiredService(); - protected IErrorMessageDataStore ErrorMessageDataStore => GetRequiredService(); - protected IMessageRedirectsDataStore MessageRedirectsDataStore => GetRequiredService(); - protected IMonitoringDataStore MonitoringDataStore => GetRequiredService(); - protected IIngestionUnitOfWorkFactory UnitOfWorkFactory => GetRequiredService(); - protected ICustomChecksDataStore CustomChecks => GetRequiredService(); - protected IArchiveMessages ArchiveMessages => GetRequiredService(); + + host = hostBuilder.Build(); + + var persistenceLifecycle = GetRequiredService(); + await persistenceLifecycle.Initialize(); + await host.StartAsync(); + + CompleteDatabaseOperation(); } + + [TearDown] + public async Task TearDown() + { + await testPersistence.TearDown(); + await host.StopAsync(); + host.Dispose(); + } + + protected T GetRequiredService() => host.Services.GetRequiredService(); + + protected Action RegisterServices { get; set; } + + protected void CompleteDatabaseOperation() => testPersistence.CompleteDatabaseOperation(); + + [Conditional("DEBUG")] + protected void BlockToInspectDatabase() => testPersistence.BlockToInspectDatabase(); + + protected IErrorMessageDataStore ErrorStore => GetRequiredService(); + protected IRetryDocumentDataStore RetryStore => GetRequiredService(); + protected IBodyStorage BodyStorage => GetRequiredService(); + protected IRetryBatchesDataStore RetryBatchesStore => GetRequiredService(); + protected IErrorMessageDataStore ErrorMessageDataStore => GetRequiredService(); + protected IMessageRedirectsDataStore MessageRedirectsDataStore => GetRequiredService(); + protected IMonitoringDataStore MonitoringDataStore => GetRequiredService(); + protected IIngestionUnitOfWorkFactory UnitOfWorkFactory => GetRequiredService(); + protected ICustomChecksDataStore CustomChecks => GetRequiredService(); + protected IArchiveMessages ArchiveMessages => GetRequiredService(); } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests/Recoverability/EditMessageTests.cs b/src/ServiceControl.Persistence.Tests/Recoverability/EditMessageTests.cs index b5bf21a349..ab70fcdf1c 100644 --- a/src/ServiceControl.Persistence.Tests/Recoverability/EditMessageTests.cs +++ b/src/ServiceControl.Persistence.Tests/Recoverability/EditMessageTests.cs @@ -8,13 +8,11 @@ using Contracts.Operations; using MessageFailures; using Microsoft.Extensions.DependencyInjection; - using Microsoft.Extensions.Hosting; using NServiceBus.Extensibility; using NServiceBus.Testing; using NServiceBus.Transport; using NUnit.Framework; using ServiceControl.Persistence.MessageRedirects; - using ServiceControl.PersistenceTests; using ServiceControl.Recoverability; using ServiceControl.Recoverability.Editing; @@ -23,15 +21,15 @@ sealed class EditMessageTests : PersistenceTestBase EditHandler handler; readonly TestableUnicastDispatcher dispatcher = new TestableUnicastDispatcher(); - protected override IHostBuilder CreateHostBuilder() => base - .CreateHostBuilder() - .ConfigureServices(services => services + public EditMessageTests() + { + RegisterServices = services => services .AddSingleton(dispatcher) - .AddTransient() - ); + .AddTransient(); + } [SetUp] - public new void Setup() + public void Setup() { handler = GetRequiredService(); } diff --git a/src/ServiceControl.Persistence.Tests/RetryStateTests.cs b/src/ServiceControl.Persistence.Tests/RetryStateTests.cs index 20742f525b..593bb12f37 100644 --- a/src/ServiceControl.Persistence.Tests/RetryStateTests.cs +++ b/src/ServiceControl.Persistence.Tests/RetryStateTests.cs @@ -44,7 +44,7 @@ public async Task When_a_group_is_prepared_and_SC_is_started_the_group_is_marked var orphanage = new RecoverabilityComponent.AdoptOrphanBatchesFromPreviousSessionHostedService(documentManager, new AsyncTimer()); await orphanage.AdoptOrphanedBatchesAsync(); - await CompleteDatabaseOperation(); + CompleteDatabaseOperation(); var status = retryManager.GetStatusForRetryOperation("Test-group", RetryType.FailureGroup); Assert.True(status.Failed); @@ -62,7 +62,7 @@ public async Task When_a_group_is_prepared_with_three_batches_and_SC_is_restarte var processor = new RetryProcessor(RetryBatchesStore, domainEvents, new TestReturnToSenderDequeuer(new ReturnToSender(BodyStorage, ErrorStore), ErrorStore, domainEvents, "TestEndpoint"), retryManager); // Needs index RetryBatches_ByStatus_ReduceInitialBatchSize - await CompleteDatabaseOperation(); + CompleteDatabaseOperation(); await processor.ProcessBatches(sender); // mark ready @@ -161,7 +161,7 @@ public async Task When_a_group_has_one_batch_out_of_two_forwarded_the_status_is_ var processor = new RetryProcessor(RetryBatchesStore, domainEvents, new TestReturnToSenderDequeuer(returnToSender, ErrorStore, domainEvents, "TestEndpoint"), retryManager); - await CompleteDatabaseOperation(); + CompleteDatabaseOperation(); await processor.ProcessBatches(sender); // mark ready await processor.ProcessBatches(sender); @@ -207,19 +207,19 @@ async Task CreateAFailedMessageAndMarkAsPartOfRetryBatch(RetryingManager retryMa // Needs index FailedMessages_ByGroup // Needs index FailedMessages_UniqueMessageIdAndTimeOfFailures - await CompleteDatabaseOperation(); + CompleteDatabaseOperation(); var documentManager = new CustomRetryDocumentManager(progressToStaged, RetryStore, retryManager); var gateway = new CustomRetriesGateway(progressToStaged, RetryStore, retryManager); gateway.EnqueueRetryForFailureGroup(new RetriesGateway.RetryForFailureGroup(groupId, "Test-Context", groupType: null, DateTime.UtcNow)); - await CompleteDatabaseOperation(); + CompleteDatabaseOperation(); await gateway.ProcessNextBulkRetry(); // Wait for indexes to catch up - await CompleteDatabaseOperation(); + CompleteDatabaseOperation(); } class CustomRetriesGateway : RetriesGateway diff --git a/src/ServiceControl.Persistence.Tests/TestPersistence.cs b/src/ServiceControl.Persistence.Tests/TestPersistence.cs index 7bd9e6694b..85208e3799 100644 --- a/src/ServiceControl.Persistence.Tests/TestPersistence.cs +++ b/src/ServiceControl.Persistence.Tests/TestPersistence.cs @@ -8,11 +8,13 @@ abstract class TestPersistence { public abstract void Configure(IServiceCollection services); - public abstract Task CompleteDatabaseOperation(); + public abstract void CompleteDatabaseOperation(); public override string ToString() => GetType().Name; [Conditional("DEBUG")] - public virtual void BlockToInspectDatabase() { } + public abstract void BlockToInspectDatabase(); + + public abstract Task TearDown(); } } \ No newline at end of file From d52976d6fb9d92590d4629fa5054a12751c15ce0 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 21 Sep 2023 15:54:12 +0200 Subject: [PATCH 34/94] =?UTF-8?q?=F0=9F=A9=B9=20Removed=20obsolete=20setti?= =?UTF-8?q?ngs=20for=20RavenDB=205?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../AcceptanceTestStorageConfiguration.cs | 1 - .../StartupModeTests.cs | 1 - src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs | 4 +--- .../RavenDBPersisterSettings.cs | 2 -- .../RavenDbPersistenceConfiguration.cs | 2 -- 5 files changed, 1 insertion(+), 9 deletions(-) diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs index 6c747a5677..f3484d3b1b 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs @@ -16,7 +16,6 @@ public void CustomizeSettings(Settings settings) { settings.PersisterSpecificSettings = new RavenDBPersisterSettings { - RunInMemory = true, DatabaseMaintenancePort = FindAvailablePort(settings.Port + 1), DatabasePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()), ErrorRetentionPeriod = TimeSpan.FromDays(10), diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/StartupModeTests.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/StartupModeTests.cs index 83b9c668da..4aa678d36b 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB5/StartupModeTests.cs +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/StartupModeTests.cs @@ -27,7 +27,6 @@ public void InitializeSettings() PersisterSpecificSettings = new RavenDBPersisterSettings { ErrorRetentionPeriod = TimeSpan.FromDays(1), - RunInMemory = true }, TransportType = transportIntegration.TypeName, TransportConnectionString = transportIntegration.ConnectionString diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs b/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs index 20cdd35b8a..81271c1b14 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenBootstrapper.cs @@ -5,13 +5,11 @@ static class RavenBootstrapper public const string DatabasePathKey = "DbPath"; public const string HostNameKey = "HostName"; public const string DatabaseMaintenancePortKey = "DatabaseMaintenancePort"; - public const string ExposeRavenDBKey = "ExposeRavenDB"; public const string ExpirationProcessTimerInSecondsKey = "ExpirationProcessTimerInSeconds"; - public const string RunInMemoryKey = "RavenDB35/RunInMemory"; public const string ConnectionStringKey = "RavenDB5/ConnectionString"; public const string MinimumStorageLeftRequiredForIngestionKey = "MinimumStorageLeftRequiredForIngestion"; public const string DatabaseNameKey = "RavenDB5/DatabaseName"; public const string LogsPathKey = "LogPath"; public const string LogsModeKey = "LogMode"; } -} +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs index f59ebe9fbb..cfdae61273 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs @@ -7,9 +7,7 @@ class RavenDBPersisterSettings : PersistenceSettings public string HostName { get; set; } = "localhost"; public int DatabaseMaintenancePort { get; set; } = DatabaseMaintenancePortDefault; public string DatabaseMaintenanceUrl => $"http://{HostName}:{DatabaseMaintenancePort}"; - public bool ExposeRavenDB { get; set; } public int ExpirationProcessTimerInSeconds { get; set; } = ExpirationProcessTimerInSecondsDefault; - public bool RunInMemory { get; set; } public int MinimumStorageLeftRequiredForIngestion { get; set; } = CheckMinimumStorageRequiredForIngestion.MinimumStorageLeftRequiredForIngestionDefault; public int DataSpaceRemainingThreshold { get; set; } = CheckFreeDiskSpace.DataSpaceRemainingThresholdDefault; public TimeSpan ErrorRetentionPeriod { get; set; } diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs index c7a0f2d8fe..98c20c7627 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs @@ -47,9 +47,7 @@ T GetSetting(string key, T defaultValue) DatabasePath = GetSetting(RavenBootstrapper.DatabasePathKey, default), HostName = GetSetting(RavenBootstrapper.HostNameKey, "localhost"), DatabaseMaintenancePort = GetSetting(RavenBootstrapper.DatabaseMaintenancePortKey, RavenDBPersisterSettings.DatabaseMaintenancePortDefault), - ExposeRavenDB = GetSetting(RavenBootstrapper.ExposeRavenDBKey, false), ExpirationProcessTimerInSeconds = GetSetting(RavenBootstrapper.ExpirationProcessTimerInSecondsKey, 600), - RunInMemory = GetSetting(RavenBootstrapper.RunInMemoryKey, false), MinimumStorageLeftRequiredForIngestion = GetSetting(RavenBootstrapper.MinimumStorageLeftRequiredForIngestionKey, CheckMinimumStorageRequiredForIngestion.MinimumStorageLeftRequiredForIngestionDefault), DataSpaceRemainingThreshold = GetSetting(DataSpaceRemainingThresholdKey, CheckFreeDiskSpace.DataSpaceRemainingThresholdDefault), ErrorRetentionPeriod = GetRequiredSetting(ErrorRetentionPeriodKey), From 91befc4ec592caf2d018952572a4f876674dec41 Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 21 Sep 2023 12:11:00 -0500 Subject: [PATCH 35/94] Fix where the test filter attributes live --- .../TestsFilter.cs | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/{ServiceControl.Persistence.Tests => ServiceControl.Persistence.Tests.RavenDb}/TestsFilter.cs (100%) diff --git a/src/ServiceControl.Persistence.Tests/TestsFilter.cs b/src/ServiceControl.Persistence.Tests.RavenDb/TestsFilter.cs similarity index 100% rename from src/ServiceControl.Persistence.Tests/TestsFilter.cs rename to src/ServiceControl.Persistence.Tests.RavenDb/TestsFilter.cs From 68cc75050f8e350d6fd27b8f6c024b08eacfd2e5 Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 21 Sep 2023 12:33:34 -0500 Subject: [PATCH 36/94] Don't want to delete the whole database server directory --- .../SharedEmbeddedServer.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/SharedEmbeddedServer.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/SharedEmbeddedServer.cs index abe5f63f82..5b67f406b8 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/SharedEmbeddedServer.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/SharedEmbeddedServer.cs @@ -19,8 +19,6 @@ public static async Task GetInstance(CancellationToken cancell ServerUrl = $"http://localhost:{FindAvailablePort(33334)}" }; - Directory.Delete(settings.DatabasePath, true); - var instance = EmbeddedDatabase.Start(settings); //make sure that the database is up From a12bc7083d9bb7b06cda1cbbc7525c537cf739a8 Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 21 Sep 2023 12:59:08 -0500 Subject: [PATCH 37/94] Make acceptance tests create DB properly --- .../AcceptanceTestStorageConfiguration.cs | 2 ++ src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs | 1 + 2 files changed, 3 insertions(+) diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs index f3484d3b1b..de2f8b20f0 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Net.NetworkInformation; using System.Threading.Tasks; + using NUnit.Framework; using Persistence.RavenDb; using ServiceBus.Management.Infrastructure.Settings; @@ -18,6 +19,7 @@ public void CustomizeSettings(Settings settings) { DatabaseMaintenancePort = FindAvailablePort(settings.Port + 1), DatabasePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()), + LogPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Logs"), ErrorRetentionPeriod = TimeSpan.FromDays(10), }; } diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs index ed99c0916d..7bbe5b8db2 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs @@ -90,6 +90,7 @@ public void ConfigureLifecycle(IServiceCollection serviceCollection) public IPersistenceInstaller CreateInstaller() { var serviceCollection = new ServiceCollection(); + ConfigureLifecycle(serviceCollection); serviceCollection.AddSingleton(settings); return new RavenDbInstaller(serviceCollection); From cd0f37ee5ec12859f4e064045414bb58c56b191f Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 21 Sep 2023 14:58:10 -0500 Subject: [PATCH 38/94] Should also dispose the SharedInstance --- .../SetUpFixture.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/SetUpFixture.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/SetUpFixture.cs index e2035b815c..a4ceade541 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/SetUpFixture.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/SetUpFixture.cs @@ -9,4 +9,10 @@ public class SetUpFixture // Needs to be in a SetUpFixture otherwise the OneTimeSetUp is invoked for each inherited test fixture [OneTimeSetUp] public static async Task SetupSharedEmbeddedServer() => SharedInstance = await SharedEmbeddedServer.GetInstance(); + + [OneTimeTearDown] + public static async Task TearDown() + { + SharedInstance.Dispose(); + } } From cd7b1073634b46cbb915c58dc183fce813e8d6be Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 21 Sep 2023 14:58:28 -0500 Subject: [PATCH 39/94] Try to fix KnownEndpoints related errors from Guid vs String Ids --- ...mporaryIdsThatAreDuplicateDataMigration.cs | 8 ++++++- .../RavenDbMonitoringDataStore.cs | 19 ++++++++++------- .../RavenDbMonitoringIngestionUnitOfWork.cs | 2 +- .../KnownEndpointIdGenerator.cs | 7 ------- ...mporaryIdsThatAreDuplicateDataMigration.cs | 3 ++- .../RavenDbMonitoringDataStore.cs | 21 +++++++++++-------- .../RavenDbMonitoringIngestionUnitOfWork.cs | 6 ++++-- .../MonitoringDataStoreTests.cs | 3 --- .../KnownEndpoint.cs | 2 -- .../Operations/ErrorProcessor.cs | 1 - 10 files changed, 37 insertions(+), 35 deletions(-) delete mode 100644 src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/KnownEndpointIdGenerator.cs diff --git a/src/ServiceControl.Persistence.RavenDb/Infrastructure/Migrations/PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration.cs b/src/ServiceControl.Persistence.RavenDb/Infrastructure/Migrations/PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration.cs index 3980de5820..a5105005bc 100644 --- a/src/ServiceControl.Persistence.RavenDb/Infrastructure/Migrations/PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration.cs +++ b/src/ServiceControl.Persistence.RavenDb/Infrastructure/Migrations/PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration.cs @@ -5,6 +5,7 @@ using Raven.Abstractions.Extensions; using Raven.Client; using ServiceControl.Persistence; + using ServiceControl.Persistence.RavenDb; class PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration : IDataMigration { @@ -21,7 +22,12 @@ public Task Migrate(IDocumentStore store) //If we have knowEndpoints with non temp ids, we should delete all temp ids ones. if (fixedIdsCount > 0) { - knownEndpoints.Where(e => e.HasTemporaryId).ForEach(k => { store.DatabaseCommands.Delete(store.Conventions.DefaultFindFullDocumentKeyFromNonStringIdentifier(k.Id, typeof(KnownEndpoint), false), null); }); + knownEndpoints.Where(e => e.HasTemporaryId) + .ForEach(k => + { + string documentId = RavenDbMonitoringDataStore.MakeDocumentId(k.EndpointDetails.GetDeterministicId()); + store.DatabaseCommands.Delete(documentId, null); + }); } } } diff --git a/src/ServiceControl.Persistence.RavenDb/RavenDbMonitoringDataStore.cs b/src/ServiceControl.Persistence.RavenDb/RavenDbMonitoringDataStore.cs index f04a89718b..689fe7ed4e 100644 --- a/src/ServiceControl.Persistence.RavenDb/RavenDbMonitoringDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb/RavenDbMonitoringDataStore.cs @@ -15,13 +15,16 @@ public RavenDbMonitoringDataStore(IDocumentStore store) this.store = store; } + public static string MakeDocumentId(Guid id) => $"{KnownEndpoint.CollectionName}/{id}"; + public async Task CreateIfNotExists(EndpointDetails endpoint) { var id = endpoint.GetDeterministicId(); + var docId = MakeDocumentId(id); using (var session = store.OpenAsyncSession()) { - var knownEndpoint = await session.LoadAsync(id); + var knownEndpoint = await session.LoadAsync(docId); if (knownEndpoint != null) { @@ -30,13 +33,12 @@ public async Task CreateIfNotExists(EndpointDetails endpoint) knownEndpoint = new KnownEndpoint { - Id = id, EndpointDetails = endpoint, HostDisplayName = endpoint.Host, Monitored = false }; - await session.StoreAsync(knownEndpoint); + await session.StoreAsync(knownEndpoint, docId); await session.SaveChangesAsync(); } @@ -45,22 +47,22 @@ public async Task CreateIfNotExists(EndpointDetails endpoint) public async Task CreateOrUpdate(EndpointDetails endpoint, IEndpointInstanceMonitoring endpointInstanceMonitoring) { var id = endpoint.GetDeterministicId(); + var docId = MakeDocumentId(id); using (var session = store.OpenAsyncSession()) { - var knownEndpoint = await session.LoadAsync(id); + var knownEndpoint = await session.LoadAsync(docId); if (knownEndpoint == null) { knownEndpoint = new KnownEndpoint { - Id = id, EndpointDetails = endpoint, HostDisplayName = endpoint.Host, Monitored = true }; - await session.StoreAsync(knownEndpoint); + await session.StoreAsync(knownEndpoint, docId); } else { @@ -74,10 +76,11 @@ public async Task CreateOrUpdate(EndpointDetails endpoint, IEndpointInstanceMoni public async Task UpdateEndpointMonitoring(EndpointDetails endpoint, bool isMonitored) { var id = endpoint.GetDeterministicId(); + var docId = MakeDocumentId(id); using (var session = store.OpenAsyncSession()) { - var knownEndpoint = await session.LoadAsync(id); + var knownEndpoint = await session.LoadAsync(docId); if (knownEndpoint != null) { @@ -109,7 +112,7 @@ public async Task Delete(Guid endpointId) { using (var session = store.OpenAsyncSession()) { - session.Delete(endpointId); + session.Delete(MakeDocumentId(endpointId)); await session.SaveChangesAsync(); } } diff --git a/src/ServiceControl.Persistence.RavenDb/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs index f3ad877561..753d1f7529 100644 --- a/src/ServiceControl.Persistence.RavenDb/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs @@ -24,7 +24,7 @@ public Task RecordKnownEndpoint(KnownEndpoint knownEndpoint) { Document = RavenJObject.FromObject(endpoint), Etag = null, - Key = endpoint.Id.ToString(), + Key = RavenDbMonitoringDataStore.MakeDocumentId(endpoint.EndpointDetails.GetDeterministicId()), Metadata = KnownEndpointMetadata }; diff --git a/src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/KnownEndpointIdGenerator.cs b/src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/KnownEndpointIdGenerator.cs deleted file mode 100644 index ea368580af..0000000000 --- a/src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/KnownEndpointIdGenerator.cs +++ /dev/null @@ -1,7 +0,0 @@ -using System; - -static class KnownEndpointIdGenerator -{ - const string CollectionName = "KnownEndpoint"; - public static string MakeDocumentId(Guid endpointId) => $"{CollectionName}/{endpointId}"; -} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration.cs index 0b3f0253d4..cec8ef0598 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Migrations/PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration.cs @@ -5,6 +5,7 @@ using Raven.Client.Documents; using Raven.Client.Documents.Commands; using ServiceControl.Persistence; + using ServiceControl.Persistence.RavenDb; // TODO: I don't know if we can delete this because no prior Raven5 database will exist, or if it's an ongoing need to purge these things on every startup class PurgeKnownEndpointsWithTemporaryIdsThatAreDuplicateDataMigration : IDataMigration @@ -24,7 +25,7 @@ public Task Migrate(IDocumentStore store) { foreach (var key in knownEndpoints.Where(e => e.HasTemporaryId)) { - var documentId = store.Conventions.DefaultFindFullDocumentKeyFromNonStringIdentifier(key.Id, typeof(KnownEndpoint), false); + string documentId = RavenDbMonitoringDataStore.MakeDocumentId(key.EndpointDetails.GetDeterministicId()); session.Advanced.RequestExecutor.Execute(new DeleteDocumentCommand(documentId, null), session.Advanced.Context); } } diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbMonitoringDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbMonitoringDataStore.cs index 3d6613bda2..bce8248e12 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbMonitoringDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbMonitoringDataStore.cs @@ -3,9 +3,9 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; + using Raven.Client.Documents; using ServiceControl.Operations; using ServiceControl.Persistence; - using Raven.Client.Documents; class RavenDbMonitoringDataStore : IMonitoringDataStore { @@ -14,13 +14,16 @@ public RavenDbMonitoringDataStore(IDocumentStore store) this.store = store; } + public static string MakeDocumentId(Guid id) => $"{KnownEndpoint.CollectionName}/{id}"; + public async Task CreateIfNotExists(EndpointDetails endpoint) { var id = endpoint.GetDeterministicId(); + var docId = MakeDocumentId(id); using var session = store.OpenAsyncSession(); - var knownEndpoint = await session.LoadAsync(id.ToString()); + var knownEndpoint = await session.LoadAsync(docId); if (knownEndpoint != null) { @@ -29,13 +32,12 @@ public async Task CreateIfNotExists(EndpointDetails endpoint) knownEndpoint = new KnownEndpoint { - Id = id, EndpointDetails = endpoint, HostDisplayName = endpoint.Host, Monitored = false }; - await session.StoreAsync(knownEndpoint); + await session.StoreAsync(knownEndpoint, docId); await session.SaveChangesAsync(); } @@ -43,22 +45,22 @@ public async Task CreateIfNotExists(EndpointDetails endpoint) public async Task CreateOrUpdate(EndpointDetails endpoint, IEndpointInstanceMonitoring endpointInstanceMonitoring) { var id = endpoint.GetDeterministicId(); + var docId = MakeDocumentId(id); using var session = store.OpenAsyncSession(); - var knownEndpoint = await session.LoadAsync(id.ToString()); + var knownEndpoint = await session.LoadAsync(docId); if (knownEndpoint == null) { knownEndpoint = new KnownEndpoint { - Id = id, EndpointDetails = endpoint, HostDisplayName = endpoint.Host, Monitored = true }; - await session.StoreAsync(knownEndpoint); + await session.StoreAsync(knownEndpoint, docId); } else { @@ -71,10 +73,11 @@ public async Task CreateOrUpdate(EndpointDetails endpoint, IEndpointInstanceMoni public async Task UpdateEndpointMonitoring(EndpointDetails endpoint, bool isMonitored) { var id = endpoint.GetDeterministicId(); + var docId = MakeDocumentId(id); using var session = store.OpenAsyncSession(); - var knownEndpoint = await session.LoadAsync(id.ToString()); + var knownEndpoint = await session.LoadAsync(docId); if (knownEndpoint != null) { @@ -100,7 +103,7 @@ public async Task WarmupMonitoringFromPersistence(IEndpointInstanceMonitoring en public async Task Delete(Guid endpointId) { using var session = store.OpenAsyncSession(); - session.Delete(KnownEndpointIdGenerator.MakeDocumentId(endpointId)); + session.Delete(MakeDocumentId(endpointId)); await session.SaveChangesAsync(); } diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs index e3a3114b93..63820e622c 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs @@ -2,9 +2,9 @@ { using System.Threading.Tasks; using Newtonsoft.Json.Linq; - using ServiceControl.Persistence.UnitOfWork; using Raven.Client.Documents.Commands.Batches; using Raven.Client.Documents.Operations; + using ServiceControl.Persistence.UnitOfWork; class RavenDbMonitoringIngestionUnitOfWork : IMonitoringIngestionUnitOfWork { @@ -26,7 +26,9 @@ static PatchCommandData CreateKnownEndpointsPutCommand(KnownEndpoint endpoint) var document = JObject.FromObject(endpoint); document["@metadata"] = KnownEndpointMetadata; - return new PatchCommandData(endpoint.Id.ToString(), null, new PatchRequest + var docId = RavenDbMonitoringDataStore.MakeDocumentId(endpoint.EndpointDetails.GetDeterministicId()); + + return new PatchCommandData(docId, null, new PatchRequest { //TODO: check if this works Script = $"put('{KnownEndpoint.CollectionName}/', {document}" diff --git a/src/ServiceControl.Persistence.Tests/MonitoringDataStoreTests.cs b/src/ServiceControl.Persistence.Tests/MonitoringDataStoreTests.cs index ec35e04cf0..03db261ffc 100644 --- a/src/ServiceControl.Persistence.Tests/MonitoringDataStoreTests.cs +++ b/src/ServiceControl.Persistence.Tests/MonitoringDataStoreTests.cs @@ -130,8 +130,6 @@ public async Task Unit_of_work_detects_endpoint() EndpointDetails = new EndpointDetails { Host = "Host1", HostId = Guid.NewGuid(), Name = "Endpoint" } }; - knownEndpoint.Id = knownEndpoint.EndpointDetails.GetDeterministicId(); - using (var unitOfWork = await UnitOfWorkFactory.StartNew()) { await unitOfWork.Monitoring.RecordKnownEndpoint(knownEndpoint); @@ -146,7 +144,6 @@ public async Task Unit_of_work_detects_endpoint() Assert.AreEqual(1, knownEndpoints.Count); var fromStorage = knownEndpoints[0]; - Assert.AreEqual(knownEndpoint.Id, fromStorage.Id, "Id should match"); Assert.AreEqual(knownEndpoint.EndpointDetails.Host, fromStorage.EndpointDetails.Host, "EndpointDetails.Host should match"); Assert.AreEqual(knownEndpoint.EndpointDetails.HostId, fromStorage.EndpointDetails.HostId, "EndpointDetails.HostId should match"); Assert.AreEqual(knownEndpoint.EndpointDetails.Name, fromStorage.EndpointDetails.Name, "EndpointDetails.Name should match"); diff --git a/src/ServiceControl.Persistence/KnownEndpoint.cs b/src/ServiceControl.Persistence/KnownEndpoint.cs index 2b9570cc3f..15227f7ea2 100644 --- a/src/ServiceControl.Persistence/KnownEndpoint.cs +++ b/src/ServiceControl.Persistence/KnownEndpoint.cs @@ -1,11 +1,9 @@ namespace ServiceControl.Persistence { - using System; using ServiceControl.Operations; public class KnownEndpoint { - public Guid Id { get; set; } public string HostDisplayName { get; set; } public bool Monitored { get; set; } public EndpointDetails EndpointDetails { get; set; } diff --git a/src/ServiceControl/Operations/ErrorProcessor.cs b/src/ServiceControl/Operations/ErrorProcessor.cs index 2edc47ada4..68597e8829 100644 --- a/src/ServiceControl/Operations/ErrorProcessor.cs +++ b/src/ServiceControl/Operations/ErrorProcessor.cs @@ -163,7 +163,6 @@ static void RecordKnownEndpoints(EndpointDetails observedEndpoint, Dictionary Date: Thu, 21 Sep 2023 15:42:11 -0500 Subject: [PATCH 40/94] compile plz --- src/ServiceControl.Persistence.Tests.RavenDb5/SetUpFixture.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/SetUpFixture.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/SetUpFixture.cs index a4ceade541..ca727bd856 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/SetUpFixture.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/SetUpFixture.cs @@ -11,7 +11,7 @@ public class SetUpFixture public static async Task SetupSharedEmbeddedServer() => SharedInstance = await SharedEmbeddedServer.GetInstance(); [OneTimeTearDown] - public static async Task TearDown() + public static void TearDown() { SharedInstance.Dispose(); } From da066cedd55a6aa02eafb969fd66c18092c7283c Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 21 Sep 2023 16:03:50 -0500 Subject: [PATCH 41/94] Approval changed because Raven5 does not have audit/saga data --- .../ApprovalFiles/APIApprovals.CustomCheckDetails.approved.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/APIApprovals.CustomCheckDetails.approved.txt b/src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/APIApprovals.CustomCheckDetails.approved.txt index 3f79bf4f12..8c907ba40d 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/APIApprovals.CustomCheckDetails.approved.txt +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/APIApprovals.CustomCheckDetails.approved.txt @@ -1,6 +1,4 @@ -ServiceControl Health: Audit Message Ingestion ServiceControl Health: Error Database Index Errors ServiceControl Health: Error Database Index Lag ServiceControl Health: Message Ingestion Process -ServiceControl Health: Saga Audit Data Retention Storage space: ServiceControl database \ No newline at end of file From 226b2e1a1e1223784de65f2fba4d4f721a128feb Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 21 Sep 2023 16:14:17 -0500 Subject: [PATCH 42/94] EmbeddedDatabaseTests.CanStart() serves no purpose now --- .../EmbeddedDatabaseTests.cs | 33 ------------------- 1 file changed, 33 deletions(-) delete mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/EmbeddedDatabaseTests.cs diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/EmbeddedDatabaseTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/EmbeddedDatabaseTests.cs deleted file mode 100644 index e306075183..0000000000 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/EmbeddedDatabaseTests.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System; -using System.IO; -using System.Threading; -using System.Threading.Tasks; -using NUnit.Framework; -using ServiceControl.Persistence.Infrastructure; -using ServiceControl.Persistence.RavenDb; -using ServiceControl.Persistence.RavenDb5; - -[TestFixture] -class EmbeddedDatabaseTests -{ - [Test] - public async Task CanStart() - { - var settings = new RavenDBPersisterSettings - { - DatabasePath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Tests", "Embedded"), - LogPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()) - }; - - using (var embeddedDatabase = EmbeddedDatabase.Start(settings)) - { - - using (var documentStore = await embeddedDatabase.Connect(CancellationToken.None)) - { - var store = new EventLogDataStore(documentStore); - - _ = await store.GetEventLogItems(new PagingInfo()); - } - } - } -} \ No newline at end of file From 16a1fe7e2ba59f03d1e0af5da5779ef2c8fe505d Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 21 Sep 2023 16:17:55 -0500 Subject: [PATCH 43/94] Database named by guid so they aren't too long --- .../TestPersistenceImpl.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs index f488400668..1cd9092152 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs @@ -2,7 +2,6 @@ { using System; using System.Diagnostics; - using System.Linq; using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; @@ -16,16 +15,18 @@ sealed class TestPersistenceImpl : TestPersistence { - // Deterministic database name make diagnosing easier - readonly string databaseName = new string(TestContext.CurrentContext.Test.FullName.Where(c => char.IsLetterOrDigit(c) || c == '_' || c == '-').ToArray()); + readonly string databaseName; readonly RavenDBPersisterSettings settings; IDocumentStore documentStore; public TestPersistenceImpl() { + databaseName = Guid.NewGuid().ToString("n"); var retentionPeriod = TimeSpan.FromMinutes(1); + TestContext.Out.WriteLine($"Test Database Name: {databaseName}"); + settings = new RavenDBPersisterSettings { AuditRetentionPeriod = retentionPeriod, @@ -49,6 +50,7 @@ public override void CompleteDatabaseOperation() documentStore.WaitForIndexing(); } + // TODO: Doesn't appear to be doing anything in the Start/Stop, do we need this? class Wrapper : IHostedService { public Wrapper(TestPersistenceImpl instance, IDocumentStore store) @@ -88,6 +90,7 @@ public override void BlockToInspectDatabase() public override async Task TearDown() { + // Comment this out temporarily to be able to inspect a database after the test has completed var deleteDatabasesOperation = new DeleteDatabasesOperation(new DeleteDatabasesOperation.Parameters { DatabaseNames = new[] { databaseName }, HardDelete = true }); await documentStore.Maintenance.Server.SendAsync(deleteDatabasesOperation); } From 6f824b0c9ead4225d182eb25ed94ee49177f9403 Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 21 Sep 2023 16:24:30 -0500 Subject: [PATCH 44/94] Minor --- .../ControllerDependencies.cs | 1 - .../PersistenceTestBase.cs | 12 +++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/ServiceControl.Persistence.Tests/ControllerDependencies.cs b/src/ServiceControl.Persistence.Tests/ControllerDependencies.cs index e091b14605..d0dce8ebee 100644 --- a/src/ServiceControl.Persistence.Tests/ControllerDependencies.cs +++ b/src/ServiceControl.Persistence.Tests/ControllerDependencies.cs @@ -15,7 +15,6 @@ using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.Infrastructure.DomainEvents; using ServiceControl.Infrastructure.WebApi; - using ServiceControl.PersistenceTests; [TestFixture] class ControllerDependencies diff --git a/src/ServiceControl.Persistence.Tests/PersistenceTestBase.cs b/src/ServiceControl.Persistence.Tests/PersistenceTestBase.cs index 8f22230651..cf247993f0 100644 --- a/src/ServiceControl.Persistence.Tests/PersistenceTestBase.cs +++ b/src/ServiceControl.Persistence.Tests/PersistenceTestBase.cs @@ -1,18 +1,18 @@ using System; using System.Diagnostics; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using NUnit.Framework; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; -using ServiceControl.Persistence; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; using NServiceBus; +using NUnit.Framework; using ServiceControl.Infrastructure.DomainEvents; -using ServiceControl.PersistenceTests; using ServiceControl.Operations.BodyStorage; +using ServiceControl.Persistence; using ServiceControl.Persistence.MessageRedirects; using ServiceControl.Persistence.Recoverability; using ServiceControl.Persistence.UnitOfWork; +using ServiceControl.PersistenceTests; //[Parallelizable(ParallelScope.All)] [FixtureLifeCycle(LifeCycle.InstancePerTestCase)] @@ -49,7 +49,9 @@ public async Task SetUp() [TearDown] public async Task TearDown() { + // Needs to go first or database will be disposed await testPersistence.TearDown(); + await host.StopAsync(); host.Dispose(); } From 66316200ff535a86057effd217ab60eb447972be Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 21 Sep 2023 16:38:25 -0500 Subject: [PATCH 45/94] Database name shouldn't be "audit" --- .../RavenDbPersistenceConfiguration.cs | 2 +- .../RavenDBPersisterSettings.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs b/src/ServiceControl.Audit.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs index 816eedfc70..f9bded18f3 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs @@ -39,7 +39,7 @@ internal static DatabaseConfiguration GetDatabaseConfiguration(PersistenceSettin { if (!settings.PersisterSpecificSettings.TryGetValue(DatabaseNameKey, out var databaseName)) { - databaseName = "audit"; + databaseName = "primary"; } ServerConfiguration serverConfiguration; diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs index cfdae61273..5f4e33a76c 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDBPersisterSettings.cs @@ -23,7 +23,7 @@ class RavenDBPersisterSettings : PersistenceSettings public string LogsMode { get; set; } = LogsModeDefault; public string DatabaseName { get; set; } = DatabaseNameDefault; - public const string DatabaseNameDefault = "audit"; + public const string DatabaseNameDefault = "primary"; public const int DatabaseMaintenancePortDefault = 33334; public const int ExpirationProcessTimerInSecondsDefault = 600; public const string LogsModeDefault = "Information"; From 56e7f8c03844cddc5730e785b7ac0112b8f0a94a Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 21 Sep 2023 16:52:49 -0500 Subject: [PATCH 46/94] Be sure to run RavenDB out of a non-virtual drive --- .../SharedEmbeddedServer.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/SharedEmbeddedServer.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/SharedEmbeddedServer.cs index 5b67f406b8..08870f140d 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/SharedEmbeddedServer.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/SharedEmbeddedServer.cs @@ -4,17 +4,18 @@ using System.Net.NetworkInformation; using System.Threading; using System.Threading.Tasks; -using NUnit.Framework; using ServiceControl.Persistence.RavenDb5; static class SharedEmbeddedServer { public static async Task GetInstance(CancellationToken cancellationToken = default) { + var basePath = Path.Combine(Path.GetTempPath(), "ServiceControlTests", "Primary"); + var settings = new RavenDBPersisterSettings { - DatabasePath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Tests", "ErrorData"), - LogPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Logs"), + DatabasePath = Path.Combine(basePath, "DB"), + LogPath = Path.Combine(basePath, "Logs"), LogsMode = "Operations", ServerUrl = $"http://localhost:{FindAvailablePort(33334)}" }; From b30f940dc44f6f43f1d906bd7bb53866ff87d5d2 Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Fri, 22 Sep 2023 10:34:18 +0200 Subject: [PATCH 47/94] body storage fixes --- .../RavenAttachmentsBodyStorage.cs | 2 +- .../BodyStorage/RavenAttachmentsBodyStorageTests.cs | 9 ++++++++- src/ServiceControl.Persistence/IBodyStorage.cs | 2 +- .../Retrying/Infrastructure/ReturnToSender.cs | 13 ++++++++++--- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs b/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs index 5a65756a95..a95adde166 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs @@ -50,7 +50,7 @@ public async Task TryFetch(string messageId) return new MessageBodyStreamResult { HasResult = true, - Stream = (MemoryStream)result.Stream, + Stream = result.Stream, ContentType = result.Details.ContentType, BodySize = (int)result.Details.Size, Etag = result.Details.ChangeVector diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/BodyStorage/RavenAttachmentsBodyStorageTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/BodyStorage/RavenAttachmentsBodyStorageTests.cs index f477920e26..b1584e1553 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/BodyStorage/RavenAttachmentsBodyStorageTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/BodyStorage/RavenAttachmentsBodyStorageTests.cs @@ -3,8 +3,9 @@ using System; using System.IO; using System.Threading.Tasks; + using MessageFailures; using NUnit.Framework; - using PersistenceTests; + using Raven.Client.Documents; [TestFixture] sealed class RavenAttachmentsBodyStorageTests : PersistenceTestBase @@ -16,6 +17,12 @@ public async Task Attachments_with_ids_that_contain_backslash_should_be_readable var contentType = "NotImportant"; var body = BitConverter.GetBytes(0xDEADBEEF); + using (var session = GetRequiredService().OpenAsyncSession()) + { + await session.StoreAsync(new FailedMessage { Id = messageId }); + await session.SaveChangesAsync(); + } + await BodyStorage.Store(messageId, contentType, body.Length, new MemoryStream(body)); var retrieved = await BodyStorage.TryFetch(messageId); diff --git a/src/ServiceControl.Persistence/IBodyStorage.cs b/src/ServiceControl.Persistence/IBodyStorage.cs index 9c688e1c88..df5c3a0eb1 100644 --- a/src/ServiceControl.Persistence/IBodyStorage.cs +++ b/src/ServiceControl.Persistence/IBodyStorage.cs @@ -12,7 +12,7 @@ public interface IBodyStorage public class MessageBodyStreamResult { public bool HasResult; - public MemoryStream Stream; // Intentional, other streams could require a context + public Stream Stream; // Intentional, other streams could require a context public string ContentType; public int BodySize; public string Etag; diff --git a/src/ServiceControl/Recoverability/Retrying/Infrastructure/ReturnToSender.cs b/src/ServiceControl/Recoverability/Retrying/Infrastructure/ReturnToSender.cs index f133ee9961..666326fbe6 100644 --- a/src/ServiceControl/Recoverability/Retrying/Infrastructure/ReturnToSender.cs +++ b/src/ServiceControl/Recoverability/Retrying/Infrastructure/ReturnToSender.cs @@ -2,6 +2,8 @@ namespace ServiceControl.Recoverability { using System; using System.Collections.Generic; + using System.IO; + using System.Text; using System.Threading.Tasks; using NServiceBus.Logging; using NServiceBus.Routing; @@ -114,10 +116,15 @@ async Task FetchFromBodyStore(string attemptMessageId, string messageId) { // Unfortunately we can't use the buffer manager here yet because core doesn't allow to set the length property so usage of GetBuffer is not possible // furthermore call ToArray would neglect many of the benefits of the recyclable stream - // RavenDB always returns a memory stream so there is no need to pretend we need to do buffered reads since the memory is anyway fully allocated already - // this assumption might change when the database is upgraded but right now this is the most memory efficient way to do things + // RavenDB always returns a memory stream in ver. 3.5 so there is no need to pretend we need to do buffered reads since the memory is anyway fully allocated already + // this assumption might change when we stop supporting RavenDB 3.5 but right now this is the most memory efficient way to do things // https://github.com/microsoft/Microsoft.IO.RecyclableMemoryStream#getbuffer-and-toarray - body = result.Stream.ToArray(); + using (var memoryStream = new MemoryStream()) + { + await result.Stream.CopyToAsync(memoryStream); + + body = memoryStream.ToArray(); + } } if (Log.IsDebugEnabled) From b014748138b68aa5f2775d807c95f70ff25bdab5 Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Fri, 22 Sep 2023 11:07:45 +0200 Subject: [PATCH 48/94] failed message builder --- .../CompositeViews/MessagesViewTests.cs | 23 +++-- .../ErrorMessageDataStoreTests.cs | 91 ++++--------------- .../FailedMessageBuilder.cs | 64 +++++++++++++ .../Retrying/Infrastructure/ReturnToSender.cs | 1 - 4 files changed, 97 insertions(+), 82 deletions(-) create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/FailedMessageBuilder.cs diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs index 6ce2d488ea..492ec25661 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs @@ -9,6 +9,7 @@ using NUnit.Framework; using Raven.Client.Documents; using Raven.Client.Documents.Linq; + using ServiceControl.CompositeViews.Messages; using ServiceControl.Persistence; [TestFixture] @@ -207,7 +208,7 @@ public void TimeSent_is_not_cast_to_DateTimeMin_if_null() [TestCase(FailedMessageStatus.Resolved, MessageStatus.ResolvedSuccessfully)] [TestCase(FailedMessageStatus.RetryIssued, MessageStatus.RetryIssued)] [TestCase(FailedMessageStatus.Unresolved, MessageStatus.Failed)] - public void Correct_status_for_failed_messages(FailedMessageStatus failedMessageStatus, MessageStatus expecteMessageStatus) + public async Task Correct_status_for_failed_messages(FailedMessageStatus failedMessageStatus, MessageStatus expecteMessageStatus) { using (var session = documentStore.OpenSession()) { @@ -219,7 +220,11 @@ public void Correct_status_for_failed_messages(FailedMessageStatus failedMessage new FailedMessage.ProcessingAttempt { AttemptedAt = DateTime.Today, - MessageMetadata = new Dictionary {{"MessageIntent", "1"}} + MessageMetadata = + { + ["MessageIntent"] = "Send", + ["CriticalTime"] = TimeSpan.FromSeconds(1) + } } }, Status = failedMessageStatus @@ -230,12 +235,16 @@ public void Correct_status_for_failed_messages(FailedMessageStatus failedMessage documentStore.WaitForIndexing(); - using (var session = documentStore.OpenSession()) + using (var session = documentStore.OpenAsyncSession()) { - var message = session.Query() - //.TransformWith() - .Customize(x => x.WaitForNonStaleResults()) - .Single(); + var query = session.Query() + .ProjectInto() + .Customize(x => x.WaitForNonStaleResults()); + + var result = await MessagesViewTransformer.Transform(query) + .ToListAsync(); + + var message = result.Single(); Assert.AreEqual(expecteMessageStatus, message.Status); } diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/ErrorMessageDataStoreTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/ErrorMessageDataStoreTests.cs index 49e020ba50..64fd5b3636 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/ErrorMessageDataStoreTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/ErrorMessageDataStoreTests.cs @@ -1,14 +1,15 @@ using System; +using System.Linq; using System.Threading.Tasks; using NUnit.Framework; using Raven.Client.Documents; -using ServiceControl.Contracts.Operations; using ServiceControl.MessageFailures; using ServiceControl.MessageFailures.Api; using ServiceControl.Operations; using ServiceControl.Persistence; using ServiceControl.Persistence.Infrastructure; -using ServiceControl.SagaAudit; +using ServiceControl.Persistence.Tests.RavenDb5; +using ServiceControl.Persistence.Tests.RavenDb5.ObjectExtensions; [TestFixture] class ErrorMessageDataStoreTests : PersistenceTestBase @@ -68,82 +69,24 @@ async Task GenerateAndSaveFailedMessage() { using (var session = documentStore.OpenAsyncSession()) { - processedMessage1 = new FailedMessage + processedMessage1 = FailedMessageBuilder.Build(m => { - Id = "1", - UniqueMessageId = "a", - Status = FailedMessageStatus.Unresolved, - ProcessingAttempts = - { - new FailedMessage.ProcessingAttempt - { - Headers = - { - ["Tomek"]="Wizard", - ["Ramon"]="Cool", - }, - AttemptedAt = DateTime.UtcNow, - MessageMetadata = - { - ["TimeSent"]="2023-09-20T12:00:00", - ["MessageId"]="x", - ["MessageType"]="MyType", - ["SendingEndpoint"]=new EndpointDetails{Host="host", HostId = Guid.NewGuid(), Name="RamonAndTomek"}, - ["ReceivingEndpoint"]=new EndpointDetails{Host="host", HostId = Guid.NewGuid(), Name="RamonAndTomek"}, - ["ConversationId"]="abc", - ["MessageIntent"]="Send", - ["BodyUrl"]="https://particular.net", - ["ContentLength"]=11111, - ["InvokedSagas"]=new[]{new SagaInfo{ChangeStatus = "YES!",SagaId = Guid.NewGuid(), SagaType = "XXX.YYY, RamonAndTomek"}}, - ["OriginatesFromSaga"]=new SagaInfo{ChangeStatus = "YES!",SagaId = Guid.NewGuid(), SagaType = "XXX.YYY, RamonAndTomek"}, - ["CriticalTime"]=TimeSpan.FromSeconds(5), - ["ProcessingTime"]=TimeSpan.FromSeconds(5), - ["DeliveryTime"]=TimeSpan.FromSeconds(5), - ["IsSystemMessage"]=false, - }, - FailureDetails = new FailureDetails() - } - } - }; + m.Id = "1"; + m.UniqueMessageId = "a"; + m.ProcessingAttempts.First().MessageMetadata["ReceivingEndpoint"].CastTo().Name = "RamonAndTomek"; + m.ProcessingAttempts.First().MessageMetadata["CriticalTime"] = TimeSpan.FromSeconds(5); + }); + await session.StoreAsync(processedMessage1); - processedMessage2 = new FailedMessage + processedMessage2 = FailedMessageBuilder.Build(m => { - Id = "2", - UniqueMessageId = "b", - Status = FailedMessageStatus.Unresolved, - ProcessingAttempts = - { - new FailedMessage.ProcessingAttempt - { - Headers = - { - ["Tomek"]="Wizard", - ["Ramon"]="Cool", - }, - AttemptedAt = DateTime.UtcNow, - MessageMetadata = - { - ["TimeSent"]="2023-09-20T12:00:05", - ["MessageId"]="y", - ["MessageType"]="MyType", - ["SendingEndpoint"]=new EndpointDetails{Host="host", HostId = Guid.NewGuid(), Name="RamonAndTomek"}, - ["ReceivingEndpoint"]=new EndpointDetails{Host="host", HostId = Guid.NewGuid(), Name="RamonAndTomek"}, - ["ConversationId"]="abc", - ["MessageIntent"]="Send", - ["BodyUrl"]="https://particular.net", - ["ContentLength"]=22222, - ["InvokedSagas"]=new[]{new SagaInfo{ChangeStatus = "YES!",SagaId = Guid.NewGuid(), SagaType = "XXX.YYY, RamonAndTomek"}}, - ["OriginatesFromSaga"]=new SagaInfo{ChangeStatus = "YES!",SagaId = Guid.NewGuid(), SagaType = "XXX.YYY, RamonAndTomek"}, - ["CriticalTime"]=TimeSpan.FromSeconds(15), - ["ProcessingTime"]=TimeSpan.FromSeconds(15), - ["DeliveryTime"]=TimeSpan.FromSeconds(15), - ["IsSystemMessage"]=false, - }, - FailureDetails = new FailureDetails() - } - } - }; + m.Id = "2"; + m.UniqueMessageId = "b"; + m.ProcessingAttempts.First().MessageMetadata["ReceivingEndpoint"].CastTo().Name = "RamonAndTomek"; + m.ProcessingAttempts.First().MessageMetadata["CriticalTime"] = TimeSpan.FromSeconds(15); + }); + await session.StoreAsync(processedMessage2); await session.SaveChangesAsync(); diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/FailedMessageBuilder.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/FailedMessageBuilder.cs new file mode 100644 index 0000000000..9ad1e0b752 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/FailedMessageBuilder.cs @@ -0,0 +1,64 @@ +namespace ServiceControl.Persistence.Tests.RavenDb5 +{ + using System; + using Contracts.Operations; + using MessageFailures; + using Operations; + using SagaAudit; + + static class FailedMessageBuilder + { + public static FailedMessage Build(Action customize) + { + var result = new FailedMessage + { + Id = "1", + UniqueMessageId = "a", + Status = FailedMessageStatus.Unresolved, + ProcessingAttempts = + { + new FailedMessage.ProcessingAttempt + { + Headers = + { + ["HeaderA"]="Wizard", + ["HeaderB"]="Cool", + }, + AttemptedAt = DateTime.UtcNow, + MessageMetadata = + { + ["TimeSent"]="2023-09-20T12:00:00", + ["MessageId"]="x", + ["MessageType"]="MyType", + ["SendingEndpoint"]=new EndpointDetails{Host="host", HostId = Guid.NewGuid(), Name="ASender"}, + ["ReceivingEndpoint"]=new EndpointDetails{Host="host", HostId = Guid.NewGuid(), Name="AReceiver"}, + ["ConversationId"]="abc", + ["MessageIntent"]="Send", + ["BodyUrl"]="https://particular.net", + ["ContentLength"]=11111, + ["InvokedSagas"]=new[]{new SagaInfo{ChangeStatus = "YES!",SagaId = Guid.NewGuid(), SagaType = "XXX.YYY, ASagaType"}}, + ["OriginatesFromSaga"]=new SagaInfo{ChangeStatus = "YES!",SagaId = Guid.NewGuid(), SagaType = "XXX.YYY, SomeOtherSagaType"}, + ["CriticalTime"]=TimeSpan.FromSeconds(5), + ["ProcessingTime"]=TimeSpan.FromSeconds(5), + ["DeliveryTime"]=TimeSpan.FromSeconds(5), + ["IsSystemMessage"]=false, + }, + FailureDetails = new FailureDetails() + } + } + }; + + customize(result); + + return result; + } + } + + namespace ObjectExtensions + { + static class ExtensionMethods + { + public static T CastTo(this object o) => (T)o; + } + } +} \ No newline at end of file diff --git a/src/ServiceControl/Recoverability/Retrying/Infrastructure/ReturnToSender.cs b/src/ServiceControl/Recoverability/Retrying/Infrastructure/ReturnToSender.cs index 666326fbe6..97f195b191 100644 --- a/src/ServiceControl/Recoverability/Retrying/Infrastructure/ReturnToSender.cs +++ b/src/ServiceControl/Recoverability/Retrying/Infrastructure/ReturnToSender.cs @@ -3,7 +3,6 @@ namespace ServiceControl.Recoverability using System; using System.Collections.Generic; using System.IO; - using System.Text; using System.Threading.Tasks; using NServiceBus.Logging; using NServiceBus.Routing; From 57788df286a732de075948959b1923aba113311f Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Fri, 22 Sep 2023 13:00:10 +0200 Subject: [PATCH 49/94] MessageViewTransformer defaults --- .../Transformers/MessagesViewTransformer.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs index 932251cfa7..76a9ac6968 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs @@ -45,10 +45,10 @@ from message in query ReceivingEndpoint = metadata["ReceivingEndpoint"], TimeSent = metadata["TimeSent"], ProcessedAt = processedAt, - CriticalTime = metadata["CriticalTime"], - ProcessingTime = metadata["ProcessingTime"], - DeliveryTime = metadata["DeliveryTime"], - IsSystemMessage = metadata["IsSystemMessage"], + CriticalTime = metadata["CriticalTime"] ?? "00:00:00", + ProcessingTime = metadata["ProcessingTime"] ?? "00:00:00", + DeliveryTime = metadata["DeliveryTime"] ?? "00:00:00", + IsSystemMessage = metadata["IsSystemMessage"] ?? false, ConversationId = metadata["ConversationId"], //the reason the we need to use a KeyValuePair is that raven seems to interpret the values and convert them // to real types. In this case it was the NServiceBus.Temporary.DelayDeliveryWith header to was converted to a timespan @@ -56,7 +56,7 @@ from message in query Status = status, MessageIntent = metadata["MessageIntent"], BodyUrl = metadata["BodyUrl"], - BodySize = metadata["ContentLength"], + BodySize = metadata["ContentLength"] ?? 0, InvokedSagas = metadata["InvokedSagas"], OriginatesFromSaga = metadata["OriginatesFromSaga"] }; From e0a7a279225bd50e5b05850f135917d8afd5768a Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Fri, 22 Sep 2023 13:00:23 +0200 Subject: [PATCH 50/94] MessageViewTest part 1 --- .../CompositeViews/MessagesViewTests.cs | 76 +++++++------------ 1 file changed, 29 insertions(+), 47 deletions(-) diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs index 492ec25661..64749640ac 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs @@ -18,7 +18,7 @@ class MessagesViewTests : PersistenceTestBase [Test] public void Filter_out_system_messages() { - using (var session = documentStore.OpenSession()) + using (var session = DocumentStore.OpenSession()) { var processedMessage = new ProcessedMessage { @@ -38,9 +38,9 @@ public void Filter_out_system_messages() session.SaveChanges(); } - documentStore.WaitForIndexing(); + DocumentStore.WaitForIndexing(); - using (var session = documentStore.OpenSession()) + using (var session = DocumentStore.OpenSession()) { var results = session.Query() .Customize(x => x.WaitForNonStaleResults()) @@ -55,7 +55,7 @@ public void Filter_out_system_messages() [Test] public void Order_by_critical_time() { - using (var session = documentStore.OpenSession()) + using (var session = DocumentStore.OpenSession()) { session.Store(new ProcessedMessage { @@ -87,9 +87,9 @@ public void Order_by_critical_time() session.SaveChanges(); } - documentStore.WaitForIndexing(); + DocumentStore.WaitForIndexing(); - using (var session = documentStore.OpenSession()) + using (var session = DocumentStore.OpenSession()) { var firstByCriticalTime = session.Query() .OrderBy(x => x.CriticalTime) @@ -111,7 +111,7 @@ public void Order_by_critical_time() [Test] public void Order_by_time_sent() { - using (var session = documentStore.OpenSession()) + using (var session = DocumentStore.OpenSession()) { session.Store(new ProcessedMessage { @@ -132,9 +132,9 @@ public void Order_by_time_sent() session.SaveChanges(); } - documentStore.WaitForIndexing(); + DocumentStore.WaitForIndexing(); - using (var session = documentStore.OpenSession()) + using (var session = DocumentStore.OpenSession()) { var firstByTimeSent = session.Query() .OrderBy(x => x.TimeSent) @@ -153,7 +153,7 @@ public void Order_by_time_sent() [Test] public void TimeSent_is_not_cast_to_DateTimeMin_if_null() { - using (var session = documentStore.OpenSession()) + using (var session = DocumentStore.OpenSession()) { session.Store(new ProcessedMessage { @@ -191,9 +191,9 @@ public void TimeSent_is_not_cast_to_DateTimeMin_if_null() session.SaveChanges(); } - documentStore.WaitForIndexing(); + DocumentStore.WaitForIndexing(); - using (var session = documentStore.OpenSession()) + using (var session = DocumentStore.OpenSession()) { var messageWithNoTimeSent = session.Query() //.TransformWith() @@ -210,7 +210,7 @@ public void TimeSent_is_not_cast_to_DateTimeMin_if_null() [TestCase(FailedMessageStatus.Unresolved, MessageStatus.Failed)] public async Task Correct_status_for_failed_messages(FailedMessageStatus failedMessageStatus, MessageStatus expecteMessageStatus) { - using (var session = documentStore.OpenSession()) + using (var session = DocumentStore.OpenSession()) { session.Store(new FailedMessage { @@ -233,16 +233,15 @@ public async Task Correct_status_for_failed_messages(FailedMessageStatus failedM session.SaveChanges(); } - documentStore.WaitForIndexing(); + DocumentStore.WaitForIndexing(); - using (var session = documentStore.OpenAsyncSession()) + using (var session = DocumentStore.OpenAsyncSession()) { var query = session.Query() .ProjectInto() .Customize(x => x.WaitForNonStaleResults()); - var result = await MessagesViewTransformer.Transform(query) - .ToListAsync(); + var result = await MessagesViewTransformer.Transform(query).ToListAsync(); var message = result.Single(); @@ -251,9 +250,9 @@ public async Task Correct_status_for_failed_messages(FailedMessageStatus failedM } [Test] - public void Correct_status_for_repeated_errors() + public async Task Correct_status_for_repeated_errors() { - using (var session = documentStore.OpenSession()) + using (var session = DocumentStore.OpenSession()) { session.Store(new FailedMessage { @@ -276,40 +275,23 @@ public void Correct_status_for_repeated_errors() session.SaveChanges(); } - documentStore.WaitForIndexing(); + DocumentStore.WaitForIndexing(); - using (var session = documentStore.OpenSession()) + using (var session = DocumentStore.OpenAsyncSession()) { - var message = session.Query() - //.TransformWith() - .Customize(x => x.WaitForNonStaleResults()) - .Single(); - - Assert.AreEqual(MessageStatus.RepeatedFailure, message.Status); - } - } - - [SetUp] - public new async Task SetUp() - { - await base.SetUp(); - documentStore = GetRequiredService(); - - var customIndex = new MessagesViewIndex(); - customIndex.Execute(documentStore); + var query = session + .Query() + .ProjectInto() + .Customize(x => x.WaitForNonStaleResults()); - //var transformer = new MessagesViewTransformer(); + var result = await MessagesViewTransformer.Transform(query).ToListAsync(); - //transformer.Execute(documentStore); - } + var message = result.Single(); - [TearDown] - public new async Task TearDown() - { - await base.TearDown(); - documentStore.Dispose(); + Assert.AreEqual(MessageStatus.RepeatedFailure, message.Status); + } } - IDocumentStore documentStore; + IDocumentStore DocumentStore => GetRequiredService(); } } \ No newline at end of file From 6e20f480cfbe169647e74fdb0dfddb4bd3382d50 Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Fri, 22 Sep 2023 13:30:19 +0200 Subject: [PATCH 51/94] CompositeViewTests --- .../Indexes/MessagesViewIndex.cs | 6 +- .../CompositeViews/FailedMessagesTests.cs | 27 +--- .../CompositeViews/MessagesViewTests.cs | 145 +++++++++--------- .../FailedMessageBuilder.cs | 27 +++- .../Infrastructure/MessageExtensions.cs | 14 -- 5 files changed, 102 insertions(+), 117 deletions(-) delete mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/MessageExtensions.cs diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/MessagesViewIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/MessagesViewIndex.cs index 73f5401e2a..db018d9502 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/MessagesViewIndex.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/MessagesViewIndex.cs @@ -29,9 +29,9 @@ public MessagesViewIndex() TimeSent = (DateTime)last.MessageMetadata["TimeSent"], ProcessedAt = last.AttemptedAt, ReceivingEndpointName = ((EndpointDetails)last.MessageMetadata["ReceivingEndpoint"]).Name, - CriticalTime = null, - ProcessingTime = null, - DeliveryTime = null, + CriticalTime = (TimeSpan?)last.MessageMetadata["CriticalTime"], + ProcessingTime = (TimeSpan?)last.MessageMetadata["ProcessingTime"], + DeliveryTime = (TimeSpan?)last.MessageMetadata["DeliveryTime"], Query = last.MessageMetadata.Select(_ => _.Value.ToString()).Union(new[] { string.Join(" ", last.Headers.Select(x => x.Value)) }).ToArray(), ConversationId = (string)last.MessageMetadata["ConversationId"] }; diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/FailedMessagesTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/FailedMessagesTests.cs index 31759b5d6a..70937b83e2 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/FailedMessagesTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/FailedMessagesTests.cs @@ -17,7 +17,7 @@ class FailedMessagesTests : PersistenceTestBase [Test] public void Should_allow_errors_with_no_metadata() { - using (var session = documentStore.OpenSession()) + using (var session = DocumentStore.OpenSession()) { var processedMessage = new FailedMessage { @@ -44,7 +44,7 @@ public void Should_allow_errors_with_no_metadata() do { - using (var session = documentStore.OpenSession()) + using (var session = DocumentStore.OpenSession()) { var results = session.Advanced.DocumentQuery() //.SetResultTransformer(FailedMessageViewTransformer.Name) @@ -71,27 +71,6 @@ public void Should_allow_errors_with_no_metadata() while (stats.IsStale); } - - [SetUp] - public new void SetUp() - { - documentStore = GetRequiredService(); - - var customIndex = new FailedMessageViewIndex(); - customIndex.Execute(documentStore); - - //var transformer = new FailedMessageViewTransformer(); - - //TODO: we need to bring this back - //transformer.Execute(documentStore); - } - - [TearDown] - public new void TearDown() - { - documentStore.Dispose(); - } - - IDocumentStore documentStore; + IDocumentStore DocumentStore => GetRequiredService(); } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs index 64749640ac..cb1b9f3cca 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs @@ -7,6 +7,7 @@ using MessageAuditing; using MessageFailures; using NUnit.Framework; + using Persistence.Tests.RavenDb5; using Raven.Client.Documents; using Raven.Client.Documents.Linq; using ServiceControl.CompositeViews.Messages; @@ -20,19 +21,20 @@ public void Filter_out_system_messages() { using (var session = DocumentStore.OpenSession()) { - var processedMessage = new ProcessedMessage + var processedMessage = FailedMessageBuilder.Minimal(m => { - Id = "1" - }; + m.Id = "1"; + m.ProcessingAttempts.First().MessageMetadata["IsSystemMessage"] = true; + }); - processedMessage.MakeSystemMessage(); session.Store(processedMessage); - var processedMessage2 = new ProcessedMessage + var processedMessage2 = FailedMessageBuilder.Minimal(m => { - Id = "2" - }; - processedMessage2.MakeSystemMessage(false); + m.Id = "2"; + m.ProcessingAttempts.First().MessageMetadata["IsSystemMessage"] = false; + }); + session.Store(processedMessage2); session.SaveChanges(); @@ -57,33 +59,31 @@ public void Order_by_critical_time() { using (var session = DocumentStore.OpenSession()) { - session.Store(new ProcessedMessage + session.Store(FailedMessageBuilder.Minimal(m => { - Id = "1", - MessageMetadata = new Dictionary { { "CriticalTime", TimeSpan.FromSeconds(10) } } - }); + m.Id = "1"; + m.ProcessingAttempts.First().MessageMetadata["CriticalTime"] = TimeSpan.FromSeconds(10); + })); - session.Store(new ProcessedMessage + session.Store(FailedMessageBuilder.Minimal(m => { - Id = "2", - MessageMetadata = new Dictionary { { "CriticalTime", TimeSpan.FromSeconds(20) } } - }); + m.Id = "2"; + m.ProcessingAttempts.First().MessageMetadata["CriticalTime"] = TimeSpan.FromSeconds(20); + })); - session.Store(new ProcessedMessage + session.Store(FailedMessageBuilder.Minimal(m => { - Id = "3", - MessageMetadata = new Dictionary { { "CriticalTime", TimeSpan.FromSeconds(15) } } - }); + m.Id = "3"; + m.ProcessingAttempts.First().MessageMetadata["CriticalTime"] = TimeSpan.FromSeconds(15); + })); - session.Store(new FailedMessage + session.Store(FailedMessageBuilder.Minimal(m => { - Id = "4", - Status = FailedMessageStatus.Unresolved, - ProcessingAttempts = new List - { - new FailedMessage.ProcessingAttempt {MessageMetadata = new Dictionary {{"CriticalTime", TimeSpan.FromSeconds(15)}}} - } - }); + m.Id = "4"; + m.Status = FailedMessageStatus.Unresolved; + m.ProcessingAttempts.First().MessageMetadata["CriticalTime"] = TimeSpan.FromSeconds(15); + })); + session.SaveChanges(); } @@ -113,22 +113,24 @@ public void Order_by_time_sent() { using (var session = DocumentStore.OpenSession()) { - session.Store(new ProcessedMessage + session.Store(FailedMessageBuilder.Minimal(m => { - Id = "1", - MessageMetadata = new Dictionary { { "TimeSent", DateTime.Today.AddSeconds(20) } } - }); + m.Id = "1"; + m.ProcessingAttempts.First().MessageMetadata["TimeSent"] = DateTime.Today.AddSeconds(20); + })); - session.Store(new ProcessedMessage + session.Store(FailedMessageBuilder.Minimal(m => { - Id = "2", - MessageMetadata = new Dictionary { { "TimeSent", DateTime.Today.AddSeconds(10) } } - }); - session.Store(new ProcessedMessage + m.Id = "2"; + m.ProcessingAttempts.First().MessageMetadata["TimeSent"] = DateTime.Today.AddSeconds(10); + })); + + session.Store(FailedMessageBuilder.Minimal(m => { - Id = "3", - MessageMetadata = new Dictionary { { "TimeSent", DateTime.Today.AddDays(-1) } } - }); + m.Id = "3"; + m.ProcessingAttempts.First().MessageMetadata["TimeSent"] = DateTime.Today.AddDays(-1); + })); + session.SaveChanges(); } @@ -151,56 +153,49 @@ public void Order_by_time_sent() } [Test] - public void TimeSent_is_not_cast_to_DateTimeMin_if_null() + public async Task TimeSent_is_not_cast_to_DateTimeMin_if_null() { using (var session = DocumentStore.OpenSession()) { - session.Store(new ProcessedMessage + session.Store(FailedMessageBuilder.Minimal(m => { - MessageMetadata = new Dictionary - { - {"MessageIntent", "1"}, - {"TimeSent", null} - } - }); - session.Store(new FailedMessage + m.Id = "1"; + m.ProcessingAttempts.First().MessageMetadata["TimeSent"] = null; + })); + + session.Store(FailedMessageBuilder.Minimal(m => { - ProcessingAttempts = new List + m.Id = "2"; + + m.ProcessingAttempts.First().AttemptedAt = DateTime.Today; + m.ProcessingAttempts.First().MessageMetadata["TimeSent"] = null; + + m.ProcessingAttempts.Add(new FailedMessage.ProcessingAttempt { - new FailedMessage.ProcessingAttempt - { - AttemptedAt = DateTime.Today, - MessageMetadata = new Dictionary - { - {"MessageIntent", "1"}, - {"TimeSent", null} - } - }, - new FailedMessage.ProcessingAttempt + AttemptedAt = DateTime.Today, + MessageMetadata = { - AttemptedAt = DateTime.Today, - MessageMetadata = new Dictionary - { - {"MessageIntent", "1"}, - {"TimeSent", null} - } + ["MessageIntent"] = "Send", + ["TimeSent"] = null } - } - }); + }); + })); session.SaveChanges(); } DocumentStore.WaitForIndexing(); - using (var session = DocumentStore.OpenSession()) + using (var session = DocumentStore.OpenAsyncSession()) { - var messageWithNoTimeSent = session.Query() - //.TransformWith() - .Customize(x => x.WaitForNonStaleResults()) - .ToArray(); - Assert.AreEqual(null, messageWithNoTimeSent[0].TimeSent); - Assert.AreEqual(null, messageWithNoTimeSent[1].TimeSent); + var query = session.Query() + .ProjectInto() + .Customize(x => x.WaitForNonStaleResults()); + + var messagesWithNoTimestamp = await MessagesViewTransformer.Transform(query).ToArrayAsync(); + + Assert.AreEqual(null, messagesWithNoTimestamp[0].TimeSent); + Assert.AreEqual(null, messagesWithNoTimestamp[1].TimeSent); } } diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/FailedMessageBuilder.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/FailedMessageBuilder.cs index 9ad1e0b752..72ff5f4b0f 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/FailedMessageBuilder.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/FailedMessageBuilder.cs @@ -1,6 +1,7 @@ namespace ServiceControl.Persistence.Tests.RavenDb5 { using System; + using System.Collections.Generic; using Contracts.Operations; using MessageFailures; using Operations; @@ -8,7 +9,31 @@ static class FailedMessageBuilder { - public static FailedMessage Build(Action customize) + public static FailedMessage Minimal(Action customize) + { + var message = new FailedMessage + { + Id = "1", + ProcessingAttempts = new List + { + new FailedMessage.ProcessingAttempt + { + AttemptedAt = DateTime.Today, + MessageMetadata = + { + ["MessageIntent"] = "Send", ["CriticalTime"] = TimeSpan.FromSeconds(1) + } + } + }, + Status = FailedMessageStatus.Unresolved + }; + + customize(message); + + return message; + } + + public static FailedMessage Default(Action customize) { var result = new FailedMessage { diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/MessageExtensions.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/MessageExtensions.cs deleted file mode 100644 index ee278f421e..0000000000 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/Infrastructure/MessageExtensions.cs +++ /dev/null @@ -1,14 +0,0 @@ -using ServiceControl.MessageAuditing; - -public static class MessageExtensions -{ - public static void MakeSystemMessage(this ProcessedMessage message, bool isSystem = true) - { - message.MessageMetadata["IsSystemMessage"] = isSystem; - } - - public static void SetMessageId(this ProcessedMessage message, string messageId) - { - message.MessageMetadata["MessageId"] = messageId; - } -} \ No newline at end of file From 5e2ba8bdca96a5dfb3c68e9d7dd5b952a6bb7ca6 Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Fri, 22 Sep 2023 13:31:32 +0200 Subject: [PATCH 52/94] minor clean-up --- .../ErrorMessageDataStoreTests.cs | 20 ++++--------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/ErrorMessageDataStoreTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/ErrorMessageDataStoreTests.cs index 64fd5b3636..825552ffb6 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/ErrorMessageDataStoreTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/ErrorMessageDataStoreTests.cs @@ -4,7 +4,6 @@ using NUnit.Framework; using Raven.Client.Documents; using ServiceControl.MessageFailures; -using ServiceControl.MessageFailures.Api; using ServiceControl.Operations; using ServiceControl.Persistence; using ServiceControl.Persistence.Infrastructure; @@ -14,9 +13,9 @@ [TestFixture] class ErrorMessageDataStoreTests : PersistenceTestBase { - IDocumentStore documentStore; IErrorMessageDataStore store; FailedMessage processedMessage1, processedMessage2; + IDocumentStore DocumentStore => GetRequiredService(); [Test] public async Task GetAllMessages() @@ -57,7 +56,6 @@ public async Task ErrorGet() public async Task GetStore() { await Task.Yield(); - await SetupDocumentStore(); await GenerateAndSaveFailedMessage(); CompleteDatabaseOperation(); @@ -67,9 +65,9 @@ public async Task GetStore() async Task GenerateAndSaveFailedMessage() { - using (var session = documentStore.OpenAsyncSession()) + using (var session = DocumentStore.OpenAsyncSession()) { - processedMessage1 = FailedMessageBuilder.Build(m => + processedMessage1 = FailedMessageBuilder.Default(m => { m.Id = "1"; m.UniqueMessageId = "a"; @@ -79,7 +77,7 @@ async Task GenerateAndSaveFailedMessage() await session.StoreAsync(processedMessage1); - processedMessage2 = FailedMessageBuilder.Build(m => + processedMessage2 = FailedMessageBuilder.Default(m => { m.Id = "2"; m.UniqueMessageId = "b"; @@ -92,14 +90,4 @@ async Task GenerateAndSaveFailedMessage() await session.SaveChangesAsync(); } } - - async Task SetupDocumentStore() - { - documentStore = GetRequiredService(); - var customIndex = new FailedMessageViewIndex(); - await customIndex.ExecuteAsync(documentStore); - //var transformer = new FailedMessageViewTransformer(); - //TODO: we need to bring this back - //transformer.Execute(documentStore); - } } \ No newline at end of file From 2eb123eb8967ca7c126c6717c8435669bc4bc90c Mon Sep 17 00:00:00 2001 From: David Boike Date: Fri, 22 Sep 2023 09:54:44 -0500 Subject: [PATCH 53/94] Solution-wide remove unused usings --- .../CustomChecks/When_email_notifications_are_enabled.cs | 1 - .../When_a_successful_retry_at_old_endpoint_is_detected.cs | 1 - .../When_a_successful_retry_from_old_SC_is_detected.cs | 1 - .../Transformers/MessagesViewTransformer.cs | 1 - .../PersistenceManifestLibraryTests.cs | 1 - src/ServiceControl.Audit/Auditing/ImportFailedAudits.cs | 2 -- .../Modules/EventAggregationAutoSubscriptionModule.cs | 4 ---- src/ServiceControl.Config/Xaml/Controls/FormComboBox.cs | 1 - src/ServiceControl.Config/Xaml/Controls/FormPasswordBox.cs | 1 - src/ServiceControl.Config/Xaml/Controls/FormPathTextBox.cs | 1 - src/ServiceControl.Config/Xaml/Controls/FormTextBox.cs | 1 - .../Infrastructure/EndpointMessageType.cs | 2 -- .../Expiration/ErrorMessageCleaner.cs | 1 - src/ServiceControl.Persistence.RavenDb5/Extensions.cs | 3 --- .../FailedMessageViewIndexNotifications.cs | 1 - .../Subscriptions/RavenDbSubscriptionStorage.cs | 1 - src/ServiceControl.Persistence.RavenDb5/RavenStartup.cs | 1 - .../Recoverability/Archiving/MessageArchiver.cs | 1 - .../Transactions/RavenTransactionalDataStore.cs | 1 - .../UnitOfWork/RavenDbIngestionUnitOfWorkFactory.cs | 1 - .../BodyStorage/RavenAttachmentsBodyStorageTests.cs | 1 - .../Expiration/ChunkerTests.cs | 1 - .../Expiration/ProcessedMessageExpirationTests.cs | 1 - .../Recoverability/FailedMessageRetryTests.cs | 1 - .../Expiration/ChunkerTests.cs | 1 - .../Operations/FailedErrorImportCustomCheckTests.cs | 2 -- .../Unarchiving/UnarchiveGroupTests.cs | 1 - ...abbitMQQuorumConventionalRoutingTransportCustomization.cs | 5 ----- .../Infrastructure/Settings/SettingsTests.cs | 1 - .../CustomChecks/CustomCheckResultProcessor.cs | 1 - .../MessageFailures/Handlers/UnArchiveMessagesHandler.cs | 1 - src/ServiceControl/Operations/ErrorEnricherContext.cs | 1 - .../Retrying/Infrastructure/FailedMessageEqualityComparer.cs | 1 - .../Api/LUID_AND_ATTRIBUTES.cs | 1 - src/ServiceControlInstaller.Engine/Api/TokenPrivileges.cs | 1 - .../Configuration/ServiceControl/AppConfig.cs | 1 - 36 files changed, 48 deletions(-) diff --git a/src/ServiceControl.AcceptanceTests/Monitoring/CustomChecks/When_email_notifications_are_enabled.cs b/src/ServiceControl.AcceptanceTests/Monitoring/CustomChecks/When_email_notifications_are_enabled.cs index 2b673fc322..48517d3164 100644 --- a/src/ServiceControl.AcceptanceTests/Monitoring/CustomChecks/When_email_notifications_are_enabled.cs +++ b/src/ServiceControl.AcceptanceTests/Monitoring/CustomChecks/When_email_notifications_are_enabled.cs @@ -9,7 +9,6 @@ using NUnit.Framework; using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.Persistence; - using ServiceControl.Persistence.RavenDb.Editing; using System; using System.IO; using System.Linq; diff --git a/src/ServiceControl.Audit.AcceptanceTests/Recoverability/When_a_successful_retry_at_old_endpoint_is_detected.cs b/src/ServiceControl.Audit.AcceptanceTests/Recoverability/When_a_successful_retry_at_old_endpoint_is_detected.cs index b8b76d6bab..51c4cf6d07 100644 --- a/src/ServiceControl.Audit.AcceptanceTests/Recoverability/When_a_successful_retry_at_old_endpoint_is_detected.cs +++ b/src/ServiceControl.Audit.AcceptanceTests/Recoverability/When_a_successful_retry_at_old_endpoint_is_detected.cs @@ -2,7 +2,6 @@ { using System; using System.Threading.Tasks; - using AcceptanceTesting; using NServiceBus; using NServiceBus.AcceptanceTesting; using NServiceBus.Features; diff --git a/src/ServiceControl.Audit.AcceptanceTests/Recoverability/When_a_successful_retry_from_old_SC_is_detected.cs b/src/ServiceControl.Audit.AcceptanceTests/Recoverability/When_a_successful_retry_from_old_SC_is_detected.cs index d8a93b527e..65406538bc 100644 --- a/src/ServiceControl.Audit.AcceptanceTests/Recoverability/When_a_successful_retry_from_old_SC_is_detected.cs +++ b/src/ServiceControl.Audit.AcceptanceTests/Recoverability/When_a_successful_retry_from_old_SC_is_detected.cs @@ -3,7 +3,6 @@ using System; using System.Linq; using System.Threading.Tasks; - using AcceptanceTesting; using NServiceBus; using NServiceBus.AcceptanceTesting; using NUnit.Framework; diff --git a/src/ServiceControl.Audit.Persistence.RavenDb/Transformers/MessagesViewTransformer.cs b/src/ServiceControl.Audit.Persistence.RavenDb/Transformers/MessagesViewTransformer.cs index 8d9597cd2f..59a64fa13e 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDb/Transformers/MessagesViewTransformer.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDb/Transformers/MessagesViewTransformer.cs @@ -3,7 +3,6 @@ namespace ServiceControl.Audit.Persistence.RavenDb.Transformers using System; using System.Collections.Generic; using System.Linq; - using Monitoring; using Raven.Client.Indexes; using ServiceControl.Audit.Monitoring; diff --git a/src/ServiceControl.Audit.Persistence.Tests/PersistenceManifestLibraryTests.cs b/src/ServiceControl.Audit.Persistence.Tests/PersistenceManifestLibraryTests.cs index 7488d7f003..b1fd3c39d8 100644 --- a/src/ServiceControl.Audit.Persistence.Tests/PersistenceManifestLibraryTests.cs +++ b/src/ServiceControl.Audit.Persistence.Tests/PersistenceManifestLibraryTests.cs @@ -1,6 +1,5 @@ namespace ServiceControl.Audit.Persistence.Tests { - using System; using System.IO; using System.Linq; using System.Reflection; diff --git a/src/ServiceControl.Audit/Auditing/ImportFailedAudits.cs b/src/ServiceControl.Audit/Auditing/ImportFailedAudits.cs index 57489204bc..d761c0c30e 100644 --- a/src/ServiceControl.Audit/Auditing/ImportFailedAudits.cs +++ b/src/ServiceControl.Audit/Auditing/ImportFailedAudits.cs @@ -4,10 +4,8 @@ namespace ServiceControl.Audit.Auditing using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; - using Infrastructure; using NServiceBus.Extensibility; using NServiceBus.Logging; - using NServiceBus.Raw; using NServiceBus.Transport; using ServiceControl.Audit.Persistence; using ServiceControl.Transports; diff --git a/src/ServiceControl.Config/Framework/Modules/EventAggregationAutoSubscriptionModule.cs b/src/ServiceControl.Config/Framework/Modules/EventAggregationAutoSubscriptionModule.cs index 408ae54d0c..4c39c67f57 100644 --- a/src/ServiceControl.Config/Framework/Modules/EventAggregationAutoSubscriptionModule.cs +++ b/src/ServiceControl.Config/Framework/Modules/EventAggregationAutoSubscriptionModule.cs @@ -1,7 +1,5 @@ namespace ServiceControl.Config.Framework.Modules { - using System; - using System.Diagnostics; using System.Linq; using Autofac; using Autofac.Core; @@ -9,8 +7,6 @@ using Autofac.Core.Resolving.Pipeline; using Caliburn.Micro; using Rx; - using UI.InstanceDetails; - using Action = Caliburn.Micro.Action; public class EventAggregationAutoSubscriptionModule : Module { diff --git a/src/ServiceControl.Config/Xaml/Controls/FormComboBox.cs b/src/ServiceControl.Config/Xaml/Controls/FormComboBox.cs index 852414a300..0e946b9d03 100644 --- a/src/ServiceControl.Config/Xaml/Controls/FormComboBox.cs +++ b/src/ServiceControl.Config/Xaml/Controls/FormComboBox.cs @@ -1,6 +1,5 @@ namespace ServiceControl.Config.Xaml.Controls { - using System; using System.Windows; using System.Windows.Controls; diff --git a/src/ServiceControl.Config/Xaml/Controls/FormPasswordBox.cs b/src/ServiceControl.Config/Xaml/Controls/FormPasswordBox.cs index 4f1578d002..77a6360e8c 100644 --- a/src/ServiceControl.Config/Xaml/Controls/FormPasswordBox.cs +++ b/src/ServiceControl.Config/Xaml/Controls/FormPasswordBox.cs @@ -1,6 +1,5 @@ namespace ServiceControl.Config.Xaml.Controls { - using System; using System.Windows; using System.Windows.Controls; diff --git a/src/ServiceControl.Config/Xaml/Controls/FormPathTextBox.cs b/src/ServiceControl.Config/Xaml/Controls/FormPathTextBox.cs index db6421802a..dda1904d9b 100644 --- a/src/ServiceControl.Config/Xaml/Controls/FormPathTextBox.cs +++ b/src/ServiceControl.Config/Xaml/Controls/FormPathTextBox.cs @@ -1,6 +1,5 @@ namespace ServiceControl.Config.Xaml.Controls { - using System; using System.Windows; using System.Windows.Controls; using System.Windows.Data; diff --git a/src/ServiceControl.Config/Xaml/Controls/FormTextBox.cs b/src/ServiceControl.Config/Xaml/Controls/FormTextBox.cs index 59ec52a894..32ca35013b 100644 --- a/src/ServiceControl.Config/Xaml/Controls/FormTextBox.cs +++ b/src/ServiceControl.Config/Xaml/Controls/FormTextBox.cs @@ -1,6 +1,5 @@ namespace ServiceControl.Config.Xaml.Controls { - using System; using System.Windows; using System.Windows.Controls; using System.Windows.Data; diff --git a/src/ServiceControl.Monitoring/Infrastructure/EndpointMessageType.cs b/src/ServiceControl.Monitoring/Infrastructure/EndpointMessageType.cs index ded1c670cd..847647f2e0 100644 --- a/src/ServiceControl.Monitoring/Infrastructure/EndpointMessageType.cs +++ b/src/ServiceControl.Monitoring/Infrastructure/EndpointMessageType.cs @@ -1,7 +1,5 @@ namespace ServiceControl.Monitoring.Infrastructure { - using System; - public readonly struct EndpointMessageType { public EndpointMessageType(string endpointName, string enclosedMessageTypes) diff --git a/src/ServiceControl.Persistence.RavenDb/Expiration/ErrorMessageCleaner.cs b/src/ServiceControl.Persistence.RavenDb/Expiration/ErrorMessageCleaner.cs index 8e91475081..5056c8866a 100644 --- a/src/ServiceControl.Persistence.RavenDb/Expiration/ErrorMessageCleaner.cs +++ b/src/ServiceControl.Persistence.RavenDb/Expiration/ErrorMessageCleaner.cs @@ -11,7 +11,6 @@ using Raven.Abstractions.Data; using Raven.Abstractions.Exceptions; using Raven.Database; - using ServiceControl.MessageFailures; using ServiceControl.Recoverability; static class ErrorMessageCleaner diff --git a/src/ServiceControl.Persistence.RavenDb5/Extensions.cs b/src/ServiceControl.Persistence.RavenDb5/Extensions.cs index 55f2936ace..39734a5337 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Extensions.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Extensions.cs @@ -1,10 +1,7 @@ namespace ServiceControl.Infrastructure.RavenDB { using System; - using System.Threading; - using Newtonsoft.Json.Linq; using Raven.Client.Documents.Conventions; - using Raven.Client.Documents.Queries; static class Extensions { diff --git a/src/ServiceControl.Persistence.RavenDb5/FailedMessageViewIndexNotifications.cs b/src/ServiceControl.Persistence.RavenDb5/FailedMessageViewIndexNotifications.cs index 92a5945fdc..f9bb350a13 100644 --- a/src/ServiceControl.Persistence.RavenDb5/FailedMessageViewIndexNotifications.cs +++ b/src/ServiceControl.Persistence.RavenDb5/FailedMessageViewIndexNotifications.cs @@ -7,7 +7,6 @@ using Microsoft.Extensions.Hosting; using NServiceBus.Logging; using Persistence; - using Raven.Client; using Raven.Client.Documents; class FailedMessageViewIndexNotifications diff --git a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/RavenDbSubscriptionStorage.cs b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/RavenDbSubscriptionStorage.cs index b56b86263c..804a38c386 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/RavenDbSubscriptionStorage.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Infrastructure/Subscriptions/RavenDbSubscriptionStorage.cs @@ -13,7 +13,6 @@ using NServiceBus.Settings; using NServiceBus.Unicast.Subscriptions; using NServiceBus.Unicast.Subscriptions.MessageDrivenSubscriptions; - using Raven.Client; using Raven.Client.Documents; using Raven.Client.Documents.Commands; using Raven.Client.Documents.Session; diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenStartup.cs b/src/ServiceControl.Persistence.RavenDb5/RavenStartup.cs index 3a535665ec..5e2fcafa14 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenStartup.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenStartup.cs @@ -2,7 +2,6 @@ { using System.Threading.Tasks; using NServiceBus.Logging; - using Raven.Client; using Raven.Client.Documents; using Raven.Client.Documents.Indexes; diff --git a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/MessageArchiver.cs b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/MessageArchiver.cs index bdeba84603..53b83bf7df 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/MessageArchiver.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Recoverability/Archiving/MessageArchiver.cs @@ -5,7 +5,6 @@ using System.Linq; using System.Threading.Tasks; using NServiceBus.Logging; - using Raven.Client; using Raven.Client.Documents; using ServiceControl.Infrastructure.DomainEvents; using ServiceControl.Persistence.Recoverability; diff --git a/src/ServiceControl.Persistence.RavenDb5/Transactions/RavenTransactionalDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/Transactions/RavenTransactionalDataStore.cs index f67b84c65c..1f6442a227 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Transactions/RavenTransactionalDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Transactions/RavenTransactionalDataStore.cs @@ -1,7 +1,6 @@ namespace ServiceControl.Persistence.RavenDb { using System.Threading.Tasks; - using Raven.Client; using Raven.Client.Documents.Session; abstract class AbstractSessionManager : IDataSessionManager diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWorkFactory.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWorkFactory.cs index 60d64a8e2c..079ff263cd 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWorkFactory.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWorkFactory.cs @@ -2,7 +2,6 @@ { using System.Threading.Tasks; using ServiceControl.Persistence.UnitOfWork; - using Raven.Client; using Raven.Client.Documents; class RavenDbIngestionUnitOfWorkFactory : IIngestionUnitOfWorkFactory diff --git a/src/ServiceControl.Persistence.Tests.RavenDb/BodyStorage/RavenAttachmentsBodyStorageTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb/BodyStorage/RavenAttachmentsBodyStorageTests.cs index f477920e26..a087ef29ef 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb/BodyStorage/RavenAttachmentsBodyStorageTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb/BodyStorage/RavenAttachmentsBodyStorageTests.cs @@ -4,7 +4,6 @@ using System.IO; using System.Threading.Tasks; using NUnit.Framework; - using PersistenceTests; [TestFixture] sealed class RavenAttachmentsBodyStorageTests : PersistenceTestBase diff --git a/src/ServiceControl.Persistence.Tests.RavenDb/Expiration/ChunkerTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb/Expiration/ChunkerTests.cs index 76d1909291..dcc61637ed 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb/Expiration/ChunkerTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb/Expiration/ChunkerTests.cs @@ -1,7 +1,6 @@ namespace ServiceControl.UnitTests.Expiration { using System.Collections.Generic; - using System.Threading; using NUnit.Framework; using ServiceControl.Infrastructure.RavenDB; diff --git a/src/ServiceControl.Persistence.Tests.RavenDb/Expiration/ProcessedMessageExpirationTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb/Expiration/ProcessedMessageExpirationTests.cs index 1425366ee4..07fa633bd2 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb/Expiration/ProcessedMessageExpirationTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb/Expiration/ProcessedMessageExpirationTests.cs @@ -10,7 +10,6 @@ using MessageAuditing; using MessageFailures; using NUnit.Framework; - using PersistenceTests; using Raven.Client; using Raven.Client.Embedded; using ServiceControl.Infrastructure.RavenDB.Expiration; diff --git a/src/ServiceControl.Persistence.Tests.RavenDb/Recoverability/FailedMessageRetryTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb/Recoverability/FailedMessageRetryTests.cs index 8019c78618..dcf4b2a18d 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb/Recoverability/FailedMessageRetryTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb/Recoverability/FailedMessageRetryTests.cs @@ -2,7 +2,6 @@ { using System; using NUnit.Framework; - using ServiceControl.MessageFailures; using ServiceControl.Recoverability; [TestFixture] diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/ChunkerTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/ChunkerTests.cs index 76d1909291..dcc61637ed 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/ChunkerTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Expiration/ChunkerTests.cs @@ -1,7 +1,6 @@ namespace ServiceControl.UnitTests.Expiration { using System.Collections.Generic; - using System.Threading; using NUnit.Framework; using ServiceControl.Infrastructure.RavenDB; diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedErrorImportCustomCheckTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedErrorImportCustomCheckTests.cs index 88c977a484..6ebc170ab9 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedErrorImportCustomCheckTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedErrorImportCustomCheckTests.cs @@ -2,10 +2,8 @@ { using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; - using Microsoft.Extensions.Hosting; using NServiceBus.CustomChecks; using NUnit.Framework; - using PersistenceTests; using Raven.Client.Documents; using ServiceControl.Operations; diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Unarchiving/UnarchiveGroupTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Unarchiving/UnarchiveGroupTests.cs index a8a8763a4e..c0bc19816c 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/Unarchiving/UnarchiveGroupTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Unarchiving/UnarchiveGroupTests.cs @@ -3,7 +3,6 @@ using System; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; - using Microsoft.Extensions.Hosting; using NServiceBus.Testing; using NUnit.Framework; using Raven.Client.Documents; diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuorumConventionalRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuorumConventionalRoutingTransportCustomization.cs index 8101cfd0aa..61a52d2de0 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuorumConventionalRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuorumConventionalRoutingTransportCustomization.cs @@ -1,11 +1,6 @@ namespace ServiceControl.Transports.RabbitMQ { - using System; - using System.Threading.Tasks; using NServiceBus; - using NServiceBus.Raw; - using NServiceBus.Transport; - using ServiceControl.Transports; public class RabbitMQQuorumConventionalRoutingTransportCustomization : RabbitMQConventionalRoutingTransportCustomization { diff --git a/src/ServiceControl.UnitTests/Infrastructure/Settings/SettingsTests.cs b/src/ServiceControl.UnitTests/Infrastructure/Settings/SettingsTests.cs index 935bffc15c..a601425559 100644 --- a/src/ServiceControl.UnitTests/Infrastructure/Settings/SettingsTests.cs +++ b/src/ServiceControl.UnitTests/Infrastructure/Settings/SettingsTests.cs @@ -1,7 +1,6 @@ namespace ServiceControl.UnitTests.Infrastructure.Settings { using System.Collections.Generic; - using System.Configuration; using NUnit.Framework; using ServiceBus.Management.Infrastructure.Settings; diff --git a/src/ServiceControl/CustomChecks/CustomCheckResultProcessor.cs b/src/ServiceControl/CustomChecks/CustomCheckResultProcessor.cs index 3f3f8c9217..7cff349e67 100644 --- a/src/ServiceControl/CustomChecks/CustomCheckResultProcessor.cs +++ b/src/ServiceControl/CustomChecks/CustomCheckResultProcessor.cs @@ -3,7 +3,6 @@ namespace ServiceControl.CustomChecks using System; using System.Threading.Tasks; using Contracts.CustomChecks; - using Infrastructure; using Infrastructure.DomainEvents; using NServiceBus.Logging; using ServiceControl.Persistence; diff --git a/src/ServiceControl/MessageFailures/Handlers/UnArchiveMessagesHandler.cs b/src/ServiceControl/MessageFailures/Handlers/UnArchiveMessagesHandler.cs index f805940d40..a37ed23b18 100644 --- a/src/ServiceControl/MessageFailures/Handlers/UnArchiveMessagesHandler.cs +++ b/src/ServiceControl/MessageFailures/Handlers/UnArchiveMessagesHandler.cs @@ -1,6 +1,5 @@ namespace ServiceControl.MessageFailures.Handlers { - using System.Linq; using System.Threading.Tasks; using Contracts.MessageFailures; using Infrastructure.DomainEvents; diff --git a/src/ServiceControl/Operations/ErrorEnricherContext.cs b/src/ServiceControl/Operations/ErrorEnricherContext.cs index 40062919a1..29bf7568b2 100644 --- a/src/ServiceControl/Operations/ErrorEnricherContext.cs +++ b/src/ServiceControl/Operations/ErrorEnricherContext.cs @@ -2,7 +2,6 @@ { using System.Collections.Generic; using System.Linq; - using ServiceControl.Persistence; class ErrorEnricherContext { diff --git a/src/ServiceControl/Recoverability/Retrying/Infrastructure/FailedMessageEqualityComparer.cs b/src/ServiceControl/Recoverability/Retrying/Infrastructure/FailedMessageEqualityComparer.cs index a8fa80bf17..282c88955e 100644 --- a/src/ServiceControl/Recoverability/Retrying/Infrastructure/FailedMessageEqualityComparer.cs +++ b/src/ServiceControl/Recoverability/Retrying/Infrastructure/FailedMessageEqualityComparer.cs @@ -1,7 +1,6 @@ namespace ServiceControl.Recoverability { using System.Collections.Generic; - using ServiceControl.Persistence; class FailedMessageEqualityComparer : IEqualityComparer { diff --git a/src/ServiceControlInstaller.Engine/Api/LUID_AND_ATTRIBUTES.cs b/src/ServiceControlInstaller.Engine/Api/LUID_AND_ATTRIBUTES.cs index 318aad09da..4253fba10e 100644 --- a/src/ServiceControlInstaller.Engine/Api/LUID_AND_ATTRIBUTES.cs +++ b/src/ServiceControlInstaller.Engine/Api/LUID_AND_ATTRIBUTES.cs @@ -1,6 +1,5 @@ namespace ServiceControlInstaller.Engine.Api { - using System; using System.Runtime.InteropServices; [StructLayout(LayoutKind.Sequential)] diff --git a/src/ServiceControlInstaller.Engine/Api/TokenPrivileges.cs b/src/ServiceControlInstaller.Engine/Api/TokenPrivileges.cs index 125de7971d..07713f375e 100644 --- a/src/ServiceControlInstaller.Engine/Api/TokenPrivileges.cs +++ b/src/ServiceControlInstaller.Engine/Api/TokenPrivileges.cs @@ -1,6 +1,5 @@ namespace ServiceControlInstaller.Engine.Api { - using System; using System.Runtime.InteropServices; struct TokenPrivileges diff --git a/src/ServiceControlInstaller.Engine/Configuration/ServiceControl/AppConfig.cs b/src/ServiceControlInstaller.Engine/Configuration/ServiceControl/AppConfig.cs index aa1edd44e8..50e4552de1 100644 --- a/src/ServiceControlInstaller.Engine/Configuration/ServiceControl/AppConfig.cs +++ b/src/ServiceControlInstaller.Engine/Configuration/ServiceControl/AppConfig.cs @@ -1,6 +1,5 @@ namespace ServiceControlInstaller.Engine.Configuration.ServiceControl { - using System.Collections.Generic; using System.Linq; using System.Xml.Linq; From 582a57072c182f2c8d8ba56007cad7cab2e8a8ae Mon Sep 17 00:00:00 2001 From: David Boike Date: Fri, 22 Sep 2023 10:29:53 -0500 Subject: [PATCH 54/94] Shared database setup for Raven5 acceptance tests --- .../AcceptanceTestStorageConfiguration.cs | 23 +--------- .../AcceptanceTestStorageConfiguration.cs | 33 +++++---------- .../SharedDatabaseSetup.cs | 23 ++++++++++ .../SharedEmbeddedServer.cs | 42 +++++++++++++++++++ .../ServiceControlComponentRunner.cs | 24 +---------- .../AcceptanceTestStorageConfiguration.cs | 23 +--------- .../ServiceControlComponentRunner.cs | 23 +--------- .../PersistenceTestsConfiguration.cs | 23 +--------- .../EmbeddedLifecycleTests.cs | 23 +--------- .../SharedEmbeddedServer.cs | 23 +--------- .../ServiceControlComponentRunner.cs | 23 +--------- .../ServiceControlComponentRunner.cs | 27 ++---------- .../EmbeddedDatabase.cs | 2 +- .../SetUpFixture.cs | 18 -------- .../SharedDatabaseSetup.cs | 23 ++++++++++ .../SharedEmbeddedServer.cs | 25 ++--------- .../TestPersistenceImpl.cs | 11 +++-- src/TestHelper/PortUtility.cs | 26 ++++++++++++ 18 files changed, 153 insertions(+), 262 deletions(-) create mode 100644 src/ServiceControl.AcceptanceTests.RavenDB5/SharedDatabaseSetup.cs create mode 100644 src/ServiceControl.AcceptanceTests.RavenDB5/SharedEmbeddedServer.cs delete mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/SetUpFixture.cs create mode 100644 src/ServiceControl.Persistence.Tests.RavenDb5/SharedDatabaseSetup.cs create mode 100644 src/TestHelper/PortUtility.cs diff --git a/src/ServiceControl.AcceptanceTests.RavenDB/AcceptanceTestStorageConfiguration.cs b/src/ServiceControl.AcceptanceTests.RavenDB/AcceptanceTestStorageConfiguration.cs index 6c747a5677..6c68056ac2 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB/AcceptanceTestStorageConfiguration.cs +++ b/src/ServiceControl.AcceptanceTests.RavenDB/AcceptanceTestStorageConfiguration.cs @@ -2,11 +2,10 @@ { using System; using System.IO; - using System.Linq; - using System.Net.NetworkInformation; using System.Threading.Tasks; using Persistence.RavenDb; using ServiceBus.Management.Infrastructure.Settings; + using TestHelper; class AcceptanceTestStorageConfiguration { @@ -17,7 +16,7 @@ public void CustomizeSettings(Settings settings) settings.PersisterSpecificSettings = new RavenDBPersisterSettings { RunInMemory = true, - DatabaseMaintenancePort = FindAvailablePort(settings.Port + 1), + DatabaseMaintenancePort = PortUtility.FindAvailablePort(settings.Port + 1), DatabasePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()), ErrorRetentionPeriod = TimeSpan.FromDays(10), }; @@ -30,24 +29,6 @@ public Task Configure() return Task.CompletedTask; } - static int FindAvailablePort(int startPort) - { - var activeTcpListeners = IPGlobalProperties - .GetIPGlobalProperties() - .GetActiveTcpListeners(); - - for (var port = startPort; port < startPort + 1024; port++) - { - var portCopy = port; - if (activeTcpListeners.All(endPoint => endPoint.Port != portCopy)) - { - return port; - } - } - - return startPort; - } - public Task Cleanup() => Task.CompletedTask; } } diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs index de2f8b20f0..f06c6a29bb 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs @@ -1,26 +1,24 @@ namespace ServiceControl.AcceptanceTests { using System; - using System.IO; - using System.Linq; - using System.Net.NetworkInformation; using System.Threading.Tasks; - using NUnit.Framework; using Persistence.RavenDb; + using Raven.Client.ServerWide.Operations; using ServiceBus.Management.Infrastructure.Settings; class AcceptanceTestStorageConfiguration { + readonly string databaseName = Guid.NewGuid().ToString("n"); + public string PersistenceType { get; protected set; } public void CustomizeSettings(Settings settings) { settings.PersisterSpecificSettings = new RavenDBPersisterSettings { - DatabaseMaintenancePort = FindAvailablePort(settings.Port + 1), - DatabasePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()), - LogPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Logs"), ErrorRetentionPeriod = TimeSpan.FromDays(10), + ConnectionString = SharedDatabaseSetup.SharedInstance.ServerUrl, + DatabaseName = databaseName }; } @@ -31,24 +29,13 @@ public Task Configure() return Task.CompletedTask; } - static int FindAvailablePort(int startPort) + public async Task Cleanup() { - var activeTcpListeners = IPGlobalProperties - .GetIPGlobalProperties() - .GetActiveTcpListeners(); - - for (var port = startPort; port < startPort + 1024; port++) - { - var portCopy = port; - if (activeTcpListeners.All(endPoint => endPoint.Port != portCopy)) - { - return port; - } - } + var documentStore = await SharedDatabaseSetup.SharedInstance.Connect(); - return startPort; + // Comment this out temporarily to be able to inspect a database after the test has completed + var deleteDatabasesOperation = new DeleteDatabasesOperation(new DeleteDatabasesOperation.Parameters { DatabaseNames = new[] { databaseName }, HardDelete = true }); + await documentStore.Maintenance.Server.SendAsync(deleteDatabasesOperation); } - - public Task Cleanup() => Task.CompletedTask; } } diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/SharedDatabaseSetup.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/SharedDatabaseSetup.cs new file mode 100644 index 0000000000..35b48a243c --- /dev/null +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/SharedDatabaseSetup.cs @@ -0,0 +1,23 @@ +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.RavenDb5; + +[SetUpFixture] +public class SharedDatabaseSetup +{ + public static EmbeddedDatabase SharedInstance { get; private set; } + + // Needs to be in a SetUpFixture otherwise the OneTimeSetUp is invoked for each inherited test fixture + [OneTimeSetUp] + public static async Task SetupSharedEmbeddedServer() + { + using (var cancellationSource = new CancellationTokenSource(10_000)) + { + SharedInstance = await SharedEmbeddedServer.GetInstance(); + } + } + + [OneTimeTearDown] + public static void TearDown() => SharedInstance.Dispose(); +} diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/SharedEmbeddedServer.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/SharedEmbeddedServer.cs new file mode 100644 index 0000000000..e3d08f5fac --- /dev/null +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/SharedEmbeddedServer.cs @@ -0,0 +1,42 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Persistence.RavenDb5; +using TestHelper; + +static class SharedEmbeddedServer +{ + public static async Task GetInstance(CancellationToken cancellationToken = default) + { + var basePath = Path.Combine(Path.GetTempPath(), "ServiceControlTests", "Primary.Raven5Acceptance"); + + var settings = new RavenDBPersisterSettings + { + DatabasePath = Path.Combine(basePath, "DB"), + LogPath = Path.Combine(basePath, "Logs"), + LogsMode = "Operations", + ServerUrl = $"http://localhost:{PortUtility.FindAvailablePort(33334)}" + }; + + var instance = EmbeddedDatabase.Start(settings); + + //make sure that the database is up + while (true) + { + try + { + using (await instance.Connect(cancellationToken)) + { + //no-op + } + + return instance; + } + catch (Exception) + { + await Task.Delay(500, cancellationToken); + } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs b/src/ServiceControl.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs index c3ed830287..0220a725ca 100644 --- a/src/ServiceControl.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs +++ b/src/ServiceControl.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs @@ -3,10 +3,8 @@ using System; using System.Configuration; using System.IO; - using System.Linq; using System.Net.Http; using System.Net.Http.Headers; - using System.Net.NetworkInformation; using System.Threading.Tasks; using AcceptanceTesting; using Infrastructure.DomainEvents; @@ -23,6 +21,7 @@ using Particular.ServiceControl; using ServiceBus.Management.Infrastructure.OWIN; using ServiceBus.Management.Infrastructure.Settings; + using TestHelper; class ServiceControlComponentRunner : ComponentRunner, IAcceptanceTestInfrastructureProvider { @@ -48,28 +47,9 @@ public Task Initialize(RunDescriptor run) return InitializeServiceControl(run.ScenarioContext); } - static int FindAvailablePort(int startPort) - { - var activeTcpListeners = IPGlobalProperties - .GetIPGlobalProperties() - .GetActiveTcpListeners(); - - for (var port = startPort; port < startPort + 1024; port++) - { - var portCopy = port; - if (activeTcpListeners.All(endPoint => endPoint.Port != portCopy)) - { - return port; - } - } - - return startPort; - } - - async Task InitializeServiceControl(ScenarioContext context) { - var instancePort = FindAvailablePort(33333); + var instancePort = PortUtility.FindAvailablePort(33333); ConfigurationManager.AppSettings.Set("ServiceControl/TransportType", transportToUse.TypeName); diff --git a/src/ServiceControl.Audit.AcceptanceTests.RavenDB/AcceptanceTestStorageConfiguration.cs b/src/ServiceControl.Audit.AcceptanceTests.RavenDB/AcceptanceTestStorageConfiguration.cs index 0423bbf4ed..6819e92004 100644 --- a/src/ServiceControl.Audit.AcceptanceTests.RavenDB/AcceptanceTestStorageConfiguration.cs +++ b/src/ServiceControl.Audit.AcceptanceTests.RavenDB/AcceptanceTestStorageConfiguration.cs @@ -1,10 +1,9 @@ namespace ServiceControl.Audit.AcceptanceTests { using System.Collections.Generic; - using System.Linq; - using System.Net.NetworkInformation; using System.Threading.Tasks; using ServiceControl.Audit.Persistence.RavenDb; + using TestHelper; class AcceptanceTestStorageConfiguration { @@ -15,7 +14,7 @@ public Task> CustomizeSettings() return Task.FromResult>(new Dictionary { { "RavenDB35/RunInMemory", bool.TrueString}, - { "DatabaseMaintenancePort", FindAvailablePort(33334).ToString()}, + { "DatabaseMaintenancePort", PortUtility.FindAvailablePort(33334).ToString()}, { "HostName", "localhost" } }); } @@ -27,24 +26,6 @@ public Task Configure() return Task.CompletedTask; } - static int FindAvailablePort(int startPort) - { - var activeTcpListeners = IPGlobalProperties - .GetIPGlobalProperties() - .GetActiveTcpListeners(); - - for (var port = startPort; port < startPort + 1024; port++) - { - var portCopy = port; - if (activeTcpListeners.All(endPoint => endPoint.Port != portCopy)) - { - return port; - } - } - - return startPort; - } - public Task Cleanup() => Task.CompletedTask; } } diff --git a/src/ServiceControl.Audit.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs b/src/ServiceControl.Audit.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs index 17287cc301..009b4bf38c 100644 --- a/src/ServiceControl.Audit.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs +++ b/src/ServiceControl.Audit.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs @@ -4,10 +4,8 @@ namespace ServiceControl.Audit.AcceptanceTests.TestSupport using System.Collections.Generic; using System.Configuration; using System.IO; - using System.Linq; using System.Net.Http; using System.Net.Http.Headers; - using System.Net.NetworkInformation; using System.Threading.Tasks; using AcceptanceTesting; using Auditing; @@ -24,6 +22,7 @@ namespace ServiceControl.Audit.AcceptanceTests.TestSupport using NServiceBus.AcceptanceTesting.Support; using NServiceBus.Configuration.AdvancedExtensibility; using NServiceBus.Logging; + using TestHelper; using LogLevel = NServiceBus.Logging.LogLevel; class ServiceControlComponentRunner : ComponentRunner, IAcceptanceTestInfrastructureProvider @@ -51,27 +50,9 @@ public Task Initialize(RunDescriptor run) return InitializeServiceControl(run.ScenarioContext); } - static int FindAvailablePort(int startPort) - { - var activeTcpListeners = IPGlobalProperties - .GetIPGlobalProperties() - .GetActiveTcpListeners(); - - for (var port = startPort; port < startPort + 1024; port++) - { - var portCopy = port; - if (activeTcpListeners.All(endPoint => endPoint.Port != portCopy)) - { - return port; - } - } - - return startPort; - } - async Task InitializeServiceControl(ScenarioContext context) { - var instancePort = FindAvailablePort(33333); + var instancePort = PortUtility.FindAvailablePort(33333); settings = new Settings(instanceName, transportToUse.TypeName, persistenceToUse.PersistenceType) { diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/PersistenceTestsConfiguration.cs b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/PersistenceTestsConfiguration.cs index a13c7871f4..2b28a576ca 100644 --- a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/PersistenceTestsConfiguration.cs +++ b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/PersistenceTestsConfiguration.cs @@ -1,13 +1,12 @@ namespace ServiceControl.Audit.Persistence.Tests { using System; - using System.Linq; - using System.Net.NetworkInformation; using System.Threading.Tasks; using global::Raven.Client; using Microsoft.Extensions.DependencyInjection; using ServiceControl.Audit.Auditing.BodyStorage; using ServiceControl.Audit.Persistence.RavenDb; + using TestHelper; using UnitOfWork; partial class PersistenceTestsConfiguration @@ -26,7 +25,7 @@ public async Task Configure(Action setSettings) var settings = new PersistenceSettings(TimeSpan.FromHours(1), true, 100000); settings.PersisterSpecificSettings["RavenDB35/RunInMemory"] = bool.TrueString; - settings.PersisterSpecificSettings["DatabaseMaintenancePort"] = FindAvailablePort(33334).ToString(); + settings.PersisterSpecificSettings["DatabaseMaintenancePort"] = PortUtility.FindAvailablePort(33334).ToString(); settings.PersisterSpecificSettings["HostName"] = "localhost"; setSettings(settings); @@ -63,23 +62,5 @@ public async Task Cleanup() public IDocumentStore DocumentStore { get; private set; } public string Name => "RavenDB"; - - static int FindAvailablePort(int startPort) - { - var activeTcpListeners = IPGlobalProperties - .GetIPGlobalProperties() - .GetActiveTcpListeners(); - - for (var port = startPort; port < startPort + 1024; port++) - { - var portCopy = port; - if (activeTcpListeners.All(endPoint => endPoint.Port != portCopy)) - { - return port; - } - } - - return startPort; - } } } \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDb5/EmbeddedLifecycleTests.cs b/src/ServiceControl.Audit.Persistence.Tests.RavenDb5/EmbeddedLifecycleTests.cs index 03535d5bd9..c645b0663c 100644 --- a/src/ServiceControl.Audit.Persistence.Tests.RavenDb5/EmbeddedLifecycleTests.cs +++ b/src/ServiceControl.Audit.Persistence.Tests.RavenDb5/EmbeddedLifecycleTests.cs @@ -2,11 +2,10 @@ { using System; using System.IO; - using System.Linq; - using System.Net.NetworkInformation; using System.Threading.Tasks; using NUnit.Framework; using ServiceControl.Audit.Persistence.RavenDb; + using TestHelper; [TestFixture] class EmbeddedLifecycleTests : PersistenceTestFixture @@ -20,7 +19,7 @@ public override async Task Setup() { dbPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Tests", "Embedded"); logPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); - var databaseMaintenancePort = FindAvailablePort(33335); + var databaseMaintenancePort = PortUtility.FindAvailablePort(33335); s.PersisterSpecificSettings[RavenDbPersistenceConfiguration.DatabasePathKey] = dbPath; s.PersisterSpecificSettings[RavenDbPersistenceConfiguration.LogPathKey] = logPath; @@ -41,23 +40,5 @@ public async Task Verify_embedded_database() DirectoryAssert.Exists(dbPath); DirectoryAssert.Exists(logPath); } - - static int FindAvailablePort(int startPort) - { - var activeTcpListeners = IPGlobalProperties - .GetIPGlobalProperties() - .GetActiveTcpListeners(); - - for (var port = startPort; port < startPort + 1024; port++) - { - var portCopy = port; - if (activeTcpListeners.All(endPoint => endPoint.Port != portCopy)) - { - return port; - } - } - - return startPort; - } } } \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDb5/SharedEmbeddedServer.cs b/src/ServiceControl.Audit.Persistence.Tests.RavenDb5/SharedEmbeddedServer.cs index 3a0b52ca0a..cdc5e61b63 100644 --- a/src/ServiceControl.Audit.Persistence.Tests.RavenDb5/SharedEmbeddedServer.cs +++ b/src/ServiceControl.Audit.Persistence.Tests.RavenDb5/SharedEmbeddedServer.cs @@ -2,12 +2,11 @@ { using System; using System.IO; - using System.Linq; - using System.Net.NetworkInformation; using System.Threading; using System.Threading.Tasks; using NUnit.Framework; using ServiceControl.Audit.Persistence.RavenDb; + using TestHelper; class SharedEmbeddedServer { @@ -25,7 +24,7 @@ public static async Task GetInstance(CancellationToken cancell var dbPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Tests", "AuditData"); var logPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Logs"); var logsMode = "Operations"; - var serverUrl = $"http://localhost:{FindAvailablePort(33334)}"; + var serverUrl = $"http://localhost:{PortUtility.FindAvailablePort(33334)}"; embeddedDatabase = EmbeddedDatabase.Start(new DatabaseConfiguration("audit", 60, true, TimeSpan.FromMinutes(5), 120000, 5, new ServerConfiguration(dbPath, serverUrl, logPath, logsMode))); @@ -67,24 +66,6 @@ public static async Task Stop(CancellationToken cancellationToken = default) } } - static int FindAvailablePort(int startPort) - { - var activeTcpListeners = IPGlobalProperties - .GetIPGlobalProperties() - .GetActiveTcpListeners(); - - for (var port = startPort; port < startPort + 1024; port++) - { - var portCopy = port; - if (activeTcpListeners.All(endPoint => endPoint.Port != portCopy)) - { - return port; - } - } - - return startPort; - } - static EmbeddedDatabase embeddedDatabase; static SemaphoreSlim semaphoreSlim = new SemaphoreSlim(1, 1); } diff --git a/src/ServiceControl.Monitoring.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs b/src/ServiceControl.Monitoring.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs index 684acb5d5e..5a26496443 100644 --- a/src/ServiceControl.Monitoring.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs +++ b/src/ServiceControl.Monitoring.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs @@ -3,10 +3,8 @@ namespace ServiceControl.Monitoring.AcceptanceTests.TestSupport using System; using System.Configuration; using System.IO; - using System.Linq; using System.Net.Http; using System.Net.Http.Headers; - using System.Net.NetworkInformation; using System.Threading.Tasks; using AcceptanceTesting; using Infrastructure; @@ -21,6 +19,7 @@ namespace ServiceControl.Monitoring.AcceptanceTests.TestSupport using NServiceBus.Configuration.AdvancedExtensibility; using NServiceBus.Logging; using ServiceBus.Management.Infrastructure.OWIN; + using TestHelper; class ServiceControlComponentRunner : ComponentRunner, IAcceptanceTestInfrastructureProvider { @@ -43,27 +42,9 @@ public Task Initialize(RunDescriptor run) return InitializeServiceControl(run.ScenarioContext); } - static int FindAvailablePort(int startPort) - { - var activeTcpListeners = IPGlobalProperties - .GetIPGlobalProperties() - .GetActiveTcpListeners(); - - for (var port = startPort; port < startPort + 1024; port++) - { - var portCopy = port; - if (activeTcpListeners.All(endPoint => endPoint.Port != portCopy)) - { - return port; - } - } - - return startPort; - } - async Task InitializeServiceControl(ScenarioContext context) { - var instancePort = FindAvailablePort(33333); + var instancePort = PortUtility.FindAvailablePort(33333); ConfigurationManager.AppSettings.Set("Monitoring/TransportType", transportToUse.TypeName); diff --git a/src/ServiceControl.MultiInstance.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs b/src/ServiceControl.MultiInstance.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs index 81730bb0fc..236df6d894 100644 --- a/src/ServiceControl.MultiInstance.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs +++ b/src/ServiceControl.MultiInstance.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs @@ -4,10 +4,8 @@ namespace ServiceControl.MultiInstance.AcceptanceTests.TestSupport using System.Collections.Generic; using System.Configuration; using System.IO; - using System.Linq; using System.Net.Http; using System.Net.Http.Headers; - using System.Net.NetworkInformation; using System.Reflection; using System.Threading; using System.Threading.Tasks; @@ -25,6 +23,7 @@ namespace ServiceControl.MultiInstance.AcceptanceTests.TestSupport using Particular.ServiceControl; using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.Infrastructure.WebApi; + using TestHelper; class ServiceControlComponentRunner : ComponentRunner, IAcceptanceTestInfrastructureProviderMultiInstance { @@ -49,32 +48,14 @@ public async Task Initialize(RunDescriptor run) var startPort = 33334; - var mainInstancePort = FindAvailablePort(startPort); - var mainInstanceDbPort = FindAvailablePort(mainInstancePort + 1); - var auditInstancePort = FindAvailablePort(mainInstanceDbPort + 1); + var mainInstancePort = PortUtility.FindAvailablePort(startPort); + var mainInstanceDbPort = PortUtility.FindAvailablePort(mainInstancePort + 1); + var auditInstancePort = PortUtility.FindAvailablePort(mainInstanceDbPort + 1); await InitializeServiceControlAudit(run.ScenarioContext, auditInstancePort); await InitializeServiceControl(run.ScenarioContext, mainInstancePort, mainInstanceDbPort, auditInstancePort); } - static int FindAvailablePort(int startPort) - { - var activeTcpListeners = IPGlobalProperties - .GetIPGlobalProperties() - .GetActiveTcpListeners(); - - for (var port = startPort; port < startPort + 1024; port++) - { - var portCopy = port; - if (activeTcpListeners.All(endPoint => endPoint.Port != portCopy)) - { - return port; - } - } - - return startPort; - } - async Task InitializeServiceControl(ScenarioContext context, int instancePort, int maintenancePort, int auditInstanceApiPort) { var instanceName = Settings.DEFAULT_SERVICE_NAME; diff --git a/src/ServiceControl.Persistence.RavenDb5/EmbeddedDatabase.cs b/src/ServiceControl.Persistence.RavenDb5/EmbeddedDatabase.cs index 536d5247fc..b9bd385a3f 100644 --- a/src/ServiceControl.Persistence.RavenDb5/EmbeddedDatabase.cs +++ b/src/ServiceControl.Persistence.RavenDb5/EmbeddedDatabase.cs @@ -128,7 +128,7 @@ void Start(ServerOptions serverOptions) }); } - public async Task Connect(CancellationToken cancellationToken) + public async Task Connect(CancellationToken cancellationToken = default) { var dbOptions = new DatabaseOptions(configuration.DatabaseName) { diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/SetUpFixture.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/SetUpFixture.cs deleted file mode 100644 index ca727bd856..0000000000 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/SetUpFixture.cs +++ /dev/null @@ -1,18 +0,0 @@ -using System.Threading.Tasks; -using NUnit.Framework; -using ServiceControl.Persistence.RavenDb5; - -[SetUpFixture] -public class SetUpFixture -{ - public static EmbeddedDatabase SharedInstance; - // Needs to be in a SetUpFixture otherwise the OneTimeSetUp is invoked for each inherited test fixture - [OneTimeSetUp] - public static async Task SetupSharedEmbeddedServer() => SharedInstance = await SharedEmbeddedServer.GetInstance(); - - [OneTimeTearDown] - public static void TearDown() - { - SharedInstance.Dispose(); - } -} diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/SharedDatabaseSetup.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/SharedDatabaseSetup.cs new file mode 100644 index 0000000000..8e9296cfe6 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/SharedDatabaseSetup.cs @@ -0,0 +1,23 @@ +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.RavenDb5; + +[SetUpFixture] +public class SharedDatabaseSetup +{ + public static EmbeddedDatabase SharedInstance { get; private set; } + + // Needs to be in a SetUpFixture otherwise the OneTimeSetUp is invoked for each inherited test fixture + [OneTimeSetUp] + public static async Task SetupSharedEmbeddedServer() + { + using (var cancellation = new CancellationTokenSource(10_000)) + { + SharedInstance = await SharedEmbeddedServer.GetInstance(cancellation.Token); + } + } + + [OneTimeTearDown] + public static void TearDown() => SharedInstance.Dispose(); +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/SharedEmbeddedServer.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/SharedEmbeddedServer.cs index 08870f140d..e965d01653 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/SharedEmbeddedServer.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/SharedEmbeddedServer.cs @@ -1,23 +1,22 @@ using System; using System.IO; -using System.Linq; -using System.Net.NetworkInformation; using System.Threading; using System.Threading.Tasks; using ServiceControl.Persistence.RavenDb5; +using TestHelper; static class SharedEmbeddedServer { public static async Task GetInstance(CancellationToken cancellationToken = default) { - var basePath = Path.Combine(Path.GetTempPath(), "ServiceControlTests", "Primary"); + var basePath = Path.Combine(Path.GetTempPath(), "ServiceControlTests", "Primary.Raven5Persistence"); var settings = new RavenDBPersisterSettings { DatabasePath = Path.Combine(basePath, "DB"), LogPath = Path.Combine(basePath, "Logs"), LogsMode = "Operations", - ServerUrl = $"http://localhost:{FindAvailablePort(33334)}" + ServerUrl = $"http://localhost:{PortUtility.FindAvailablePort(33334)}" }; var instance = EmbeddedDatabase.Start(settings); @@ -40,22 +39,4 @@ public static async Task GetInstance(CancellationToken cancell } } } - - static int FindAvailablePort(int startPort) - { - var activeTcpListeners = IPGlobalProperties - .GetIPGlobalProperties() - .GetActiveTcpListeners(); - - for (var port = startPort; port < startPort + 1024; port++) - { - var portCopy = port; - if (activeTcpListeners.All(endPoint => endPoint.Port != portCopy)) - { - return port; - } - } - - return startPort; - } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs index 1cd9092152..644e4bb160 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs @@ -32,7 +32,7 @@ public TestPersistenceImpl() AuditRetentionPeriod = retentionPeriod, ErrorRetentionPeriod = retentionPeriod, EventsRetentionPeriod = retentionPeriod, - ConnectionString = SetUpFixture.SharedInstance.ServerUrl, + ConnectionString = SharedDatabaseSetup.SharedInstance.ServerUrl, DatabaseName = databaseName }; } @@ -41,7 +41,7 @@ public override void Configure(IServiceCollection services) { var persistence = new RavenDbPersistenceConfiguration().Create(settings); PersistenceHostBuilderExtensions.CreatePersisterLifecyle(services, persistence); - services.AddHostedService(p => new Wrapper(this, p.GetRequiredService())); + services.AddHostedService(p => new FakeServiceToExtractDocumentStore(this, p.GetRequiredService())); } public override void CompleteDatabaseOperation() @@ -50,10 +50,9 @@ public override void CompleteDatabaseOperation() documentStore.WaitForIndexing(); } - // TODO: Doesn't appear to be doing anything in the Start/Stop, do we need this? - class Wrapper : IHostedService + class FakeServiceToExtractDocumentStore : IHostedService { - public Wrapper(TestPersistenceImpl instance, IDocumentStore store) + public FakeServiceToExtractDocumentStore(TestPersistenceImpl instance, IDocumentStore store) { instance.documentStore = store; } @@ -69,7 +68,7 @@ public override void BlockToInspectDatabase() return; } - var url = SetUpFixture.SharedInstance.ServerUrl + "/studio/index.html#databases/documents?&database=" + databaseName; + var url = SharedDatabaseSetup.SharedInstance.ServerUrl + "/studio/index.html#databases/documents?&database=" + databaseName; if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { diff --git a/src/TestHelper/PortUtility.cs b/src/TestHelper/PortUtility.cs new file mode 100644 index 0000000000..7fc4648031 --- /dev/null +++ b/src/TestHelper/PortUtility.cs @@ -0,0 +1,26 @@ +namespace TestHelper +{ + using System.Linq; + using System.Net.NetworkInformation; + + public static class PortUtility + { + public static int FindAvailablePort(int startPort) + { + var activeTcpListeners = IPGlobalProperties + .GetIPGlobalProperties() + .GetActiveTcpListeners(); + + for (var port = startPort; port < startPort + 1024; port++) + { + var portCopy = port; + if (activeTcpListeners.All(endPoint => endPoint.Port != portCopy)) + { + return port; + } + } + + return startPort; + } + } +} From 24dca4178b7fcccddf3bac1b6ecb9dd51c5ae0d2 Mon Sep 17 00:00:00 2001 From: David Boike Date: Fri, 22 Sep 2023 12:43:10 -0500 Subject: [PATCH 55/94] Better DB management and cleanup --- .../SharedDatabaseSetup.cs | 2 +- .../SharedEmbeddedServer.cs | 34 +++++++++++-------- .../SharedDatabaseSetup.cs | 2 +- .../SharedEmbeddedServer.cs | 33 ++++++++++-------- 4 files changed, 39 insertions(+), 32 deletions(-) diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/SharedDatabaseSetup.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/SharedDatabaseSetup.cs index 35b48a243c..32426c3e2e 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB5/SharedDatabaseSetup.cs +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/SharedDatabaseSetup.cs @@ -12,7 +12,7 @@ public class SharedDatabaseSetup [OneTimeSetUp] public static async Task SetupSharedEmbeddedServer() { - using (var cancellationSource = new CancellationTokenSource(10_000)) + using (var cancellationSource = new CancellationTokenSource(30_000)) { SharedInstance = await SharedEmbeddedServer.GetInstance(); } diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/SharedEmbeddedServer.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/SharedEmbeddedServer.cs index e3d08f5fac..b4f7881774 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB5/SharedEmbeddedServer.cs +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/SharedEmbeddedServer.cs @@ -1,7 +1,9 @@ -using System; -using System.IO; +using System.IO; +using System.Linq; using System.Threading; using System.Threading.Tasks; +using Raven.Client.Documents; +using Raven.Client.ServerWide.Operations; using ServiceControl.Persistence.RavenDb5; using TestHelper; @@ -10,10 +12,12 @@ static class SharedEmbeddedServer public static async Task GetInstance(CancellationToken cancellationToken = default) { var basePath = Path.Combine(Path.GetTempPath(), "ServiceControlTests", "Primary.Raven5Acceptance"); + var dbPath = Path.Combine(basePath, "DB"); + var databasesPath = Path.Combine(dbPath, "Databases"); var settings = new RavenDBPersisterSettings { - DatabasePath = Path.Combine(basePath, "DB"), + DatabasePath = dbPath, LogPath = Path.Combine(basePath, "Logs"), LogsMode = "Operations", ServerUrl = $"http://localhost:{PortUtility.FindAvailablePort(33334)}" @@ -21,22 +25,22 @@ public static async Task GetInstance(CancellationToken cancell var instance = EmbeddedDatabase.Start(settings); - //make sure that the database is up - while (true) + // Make sure that the database is up - this blocks until the cancellation token times out + using (var docStore = await instance.Connect(cancellationToken)) { - try - { - using (await instance.Connect(cancellationToken)) - { - //no-op - } + var cleanupDatabases = new DirectoryInfo(databasesPath) + .GetDirectories() + .Select(di => di.Name) + .Where(name => name.Length == 32) + .ToArray(); - return instance; - } - catch (Exception) + if (cleanupDatabases.Any()) { - await Task.Delay(500, cancellationToken); + var cleanupOperation = new DeleteDatabasesOperation(new DeleteDatabasesOperation.Parameters { DatabaseNames = cleanupDatabases, HardDelete = true }); + await docStore.Maintenance.Server.SendAsync(cleanupOperation, CancellationToken.None); } } + + return instance; } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/SharedDatabaseSetup.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/SharedDatabaseSetup.cs index 8e9296cfe6..38d9cc0c43 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/SharedDatabaseSetup.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/SharedDatabaseSetup.cs @@ -12,7 +12,7 @@ public class SharedDatabaseSetup [OneTimeSetUp] public static async Task SetupSharedEmbeddedServer() { - using (var cancellation = new CancellationTokenSource(10_000)) + using (var cancellation = new CancellationTokenSource(30_000)) { SharedInstance = await SharedEmbeddedServer.GetInstance(cancellation.Token); } diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/SharedEmbeddedServer.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/SharedEmbeddedServer.cs index e965d01653..c8850164ff 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/SharedEmbeddedServer.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/SharedEmbeddedServer.cs @@ -1,7 +1,8 @@ -using System; -using System.IO; +using System.IO; +using System.Linq; using System.Threading; using System.Threading.Tasks; +using Raven.Client.ServerWide.Operations; using ServiceControl.Persistence.RavenDb5; using TestHelper; @@ -10,10 +11,12 @@ static class SharedEmbeddedServer public static async Task GetInstance(CancellationToken cancellationToken = default) { var basePath = Path.Combine(Path.GetTempPath(), "ServiceControlTests", "Primary.Raven5Persistence"); + var dbPath = Path.Combine(basePath, "DB"); + var databasesPath = Path.Combine(dbPath, "Databases"); var settings = new RavenDBPersisterSettings { - DatabasePath = Path.Combine(basePath, "DB"), + DatabasePath = dbPath, LogPath = Path.Combine(basePath, "Logs"), LogsMode = "Operations", ServerUrl = $"http://localhost:{PortUtility.FindAvailablePort(33334)}" @@ -21,22 +24,22 @@ public static async Task GetInstance(CancellationToken cancell var instance = EmbeddedDatabase.Start(settings); - //make sure that the database is up - while (true) + // Make sure that the database is up - this blocks until the cancellation token times out + using (var docStore = await instance.Connect(cancellationToken)) { - try - { - using (await instance.Connect(cancellationToken)) - { - //no-op - } + var cleanupDatabases = new DirectoryInfo(databasesPath) + .GetDirectories() + .Select(di => di.Name) + .Where(name => name.Length == 32) + .ToArray(); - return instance; - } - catch (Exception) + if (cleanupDatabases.Any()) { - await Task.Delay(500, cancellationToken); + var cleanupOperation = new DeleteDatabasesOperation(new DeleteDatabasesOperation.Parameters { DatabaseNames = cleanupDatabases, HardDelete = true }); + await docStore.Maintenance.Server.SendAsync(cleanupOperation, CancellationToken.None); } } + + return instance; } } \ No newline at end of file From 5fc3908c515eecc01ea5f629b69e03a76f118b3a Mon Sep 17 00:00:00 2001 From: David Boike Date: Fri, 22 Sep 2023 14:27:47 -0500 Subject: [PATCH 56/94] Improvements to sequence testing so it doesn't query in such tight loops --- .../ScenarioWithEndpointBehaviorExtensions.cs | 19 ++++++++++++++++++- .../Sequence.cs | 4 ++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/ServiceControl.AcceptanceTesting/ScenarioWithEndpointBehaviorExtensions.cs b/src/ServiceControl.AcceptanceTesting/ScenarioWithEndpointBehaviorExtensions.cs index b14ecad05f..270a0014c7 100644 --- a/src/ServiceControl.AcceptanceTesting/ScenarioWithEndpointBehaviorExtensions.cs +++ b/src/ServiceControl.AcceptanceTesting/ScenarioWithEndpointBehaviorExtensions.cs @@ -53,7 +53,24 @@ public SequenceBuilder Do(string step, Func handler) public IScenarioWithEndpointBehavior Done(Func doneCriteria = null) { var behavior = new ServiceControlClient(context => sequence.Continue(context)); - return endpointBehavior.WithComponent(behavior).Done(ctx => sequence.IsFinished(ctx) && (doneCriteria == null || doneCriteria(ctx))); + return endpointBehavior.WithComponent(behavior).Done(async ctx => + { + if (sequence.IsFinished(ctx)) + { + if (doneCriteria == null || doneCriteria(ctx)) + { + return true; + } + else + { + // If sequence is done but test is not finished, small delay to avoid tight loop check + await Task.Delay(250); + } + } + + // If sequence is not finished immediately return false, since each step will enforce delays + return false; + }); } IScenarioWithEndpointBehavior endpointBehavior; diff --git a/src/ServiceControl.AcceptanceTesting/Sequence.cs b/src/ServiceControl.AcceptanceTesting/Sequence.cs index 37c366fea6..63f60e79ab 100644 --- a/src/ServiceControl.AcceptanceTesting/Sequence.cs +++ b/src/ServiceControl.AcceptanceTesting/Sequence.cs @@ -56,6 +56,10 @@ public async Task Continue(TContext context) context.Step = nextStep; return finished; } + else + { + await Task.Delay(250); + } return false; } From 73e37a38e54667053d65499e6df6e7b625497e12 Mon Sep 17 00:00:00 2001 From: David Boike Date: Fri, 22 Sep 2023 14:30:12 -0500 Subject: [PATCH 57/94] Fixing a bunch of bugs with error ingestion and recoverability --- .../.gitignore | 1 + .../When_a_failed_message_is_archived.cs | 6 +-- ...n_a_failed_message_is_resolved_by_retry.cs | 6 +-- .../ErrorMessagesDataStore.cs | 12 +++-- ...avenDbRecoverabilityIngestionUnitOfWork.cs | 53 +++++++------------ 5 files changed, 32 insertions(+), 46 deletions(-) create mode 100644 src/ServiceControl.AcceptanceTests.RavenDB5/.gitignore diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/.gitignore b/src/ServiceControl.AcceptanceTests.RavenDB5/.gitignore new file mode 100644 index 0000000000..34b2948279 --- /dev/null +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/.gitignore @@ -0,0 +1 @@ +.transport \ No newline at end of file diff --git a/src/ServiceControl.AcceptanceTests/Recoverability/ExternalIntegration/When_a_failed_message_is_archived.cs b/src/ServiceControl.AcceptanceTests/Recoverability/ExternalIntegration/When_a_failed_message_is_archived.cs index 45bb4fa687..0776de4275 100644 --- a/src/ServiceControl.AcceptanceTests/Recoverability/ExternalIntegration/When_a_failed_message_is_archived.cs +++ b/src/ServiceControl.AcceptanceTests/Recoverability/ExternalIntegration/When_a_failed_message_is_archived.cs @@ -2,14 +2,14 @@ { using System.Threading.Tasks; using AcceptanceTesting; + using Contracts; + using Newtonsoft.Json; using NServiceBus; using NServiceBus.AcceptanceTesting; using NUnit.Framework; - using Contracts; + using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.MessageFailures; - using Newtonsoft.Json; using TestSupport.EndpointTemplates; - using ServiceBus.Management.Infrastructure.Settings; class When_a_failed_message_is_archived : ExternalIntegrationAcceptanceTest { diff --git a/src/ServiceControl.AcceptanceTests/Recoverability/ExternalIntegration/When_a_failed_message_is_resolved_by_retry.cs b/src/ServiceControl.AcceptanceTests/Recoverability/ExternalIntegration/When_a_failed_message_is_resolved_by_retry.cs index af2fcead75..8a9d064372 100644 --- a/src/ServiceControl.AcceptanceTests/Recoverability/ExternalIntegration/When_a_failed_message_is_resolved_by_retry.cs +++ b/src/ServiceControl.AcceptanceTests/Recoverability/ExternalIntegration/When_a_failed_message_is_resolved_by_retry.cs @@ -3,14 +3,14 @@ using System; using System.Threading.Tasks; using AcceptanceTesting; + using Contracts; + using Newtonsoft.Json; using NServiceBus; using NServiceBus.AcceptanceTesting; using NUnit.Framework; - using Contracts; + using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.MessageFailures; - using Newtonsoft.Json; using TestSupport.EndpointTemplates; - using ServiceBus.Management.Infrastructure.Settings; class When_a_failed_message_is_resolved_by_retry : ExternalIntegrationAcceptanceTest { diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index 4030fca114..7be0c88a02 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -153,11 +153,12 @@ SortInfo sortInfo } } + // TODO: There seem to be several copies of this operation in here public async Task FailedMessageFetch(string failedMessageId) { using (var session = documentStore.OpenAsyncSession()) { - return await session.LoadAsync(failedMessageId); + return await session.LoadAsync(FailedMessageIdGenerator.MakeDocumentId(failedMessageId)); } } @@ -165,7 +166,7 @@ public async Task FailedMessageMarkAsArchived(string failedMessageId) { using (var session = documentStore.OpenAsyncSession()) { - var failedMessage = await session.LoadAsync(failedMessageId); + var failedMessage = await session.LoadAsync(FailedMessageIdGenerator.MakeDocumentId(failedMessageId)); if (failedMessage.Status != FailedMessageStatus.Archived) { @@ -180,7 +181,8 @@ public async Task FailedMessagesFetch(Guid[] ids) { using (var session = documentStore.OpenAsyncSession()) { - var results = await session.LoadAsync(ids.Select(g => g.ToString())); + var docIds = ids.Select(g => FailedMessageIdGenerator.MakeDocumentId(g.ToString())); + var results = await session.LoadAsync(docIds); return results.Values.Where(x => x != null).ToArray(); } } @@ -351,7 +353,7 @@ public async Task ErrorBy(Guid failedMessageId) { using (var session = documentStore.OpenAsyncSession()) { - var message = await session.LoadAsync(failedMessageId.ToString()); + var message = await session.LoadAsync(FailedMessageIdGenerator.MakeDocumentId(failedMessageId.ToString())); return message; } } @@ -377,7 +379,7 @@ public async Task ErrorLastBy(Guid failedMessageId) { using (var session = documentStore.OpenAsyncSession()) { - var message = await session.LoadAsync(failedMessageId.ToString()); + var message = await session.LoadAsync(FailedMessageIdGenerator.MakeDocumentId(failedMessageId.ToString())); if (message == null) { return null; diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs index 0b02f6ff58..38923ea86c 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs @@ -2,13 +2,11 @@ { using System.Collections.Generic; using System.Threading.Tasks; - using Newtonsoft.Json; - using Newtonsoft.Json.Linq; + using Raven.Client.Documents.Commands.Batches; + using Raven.Client.Documents.Operations; using ServiceControl.MessageFailures; using ServiceControl.Persistence.UnitOfWork; using ServiceControl.Recoverability; - using Raven.Client.Documents.Commands.Batches; - using Raven.Client.Documents.Operations; class RavenDbRecoverabilityIngestionUnitOfWork : IRecoverabilityIngestionUnitOfWork { @@ -37,7 +35,7 @@ public Task RecordSuccessfulRetry(string retriedMessageUniqueId) parentUnitOfWork.AddCommand(new PatchCommandData(failedMessageDocumentId, null, new PatchRequest { - Script = $@"this.nameof(FailedMessage.Status) = {(int)FailedMessageStatus.Resolved};" + Script = $@"this.{nameof(FailedMessage.Status)} = {(int)FailedMessageStatus.Resolved};" })); parentUnitOfWork.AddCommand(new DeleteCommandData(failedMessageRetryDocumentId, null)); @@ -49,15 +47,12 @@ ICommandData CreateFailedMessagesPatchCommand(string uniqueMessageId, FailedMess { var documentId = FailedMessageIdGenerator.MakeDocumentId(uniqueMessageId); - var serializedGroups = JToken.FromObject(groups); - var serializedAttempt = JToken.FromObject(processingAttempt, Serializer); - //HINT: RavenDB 3.5 is using Lodash v4.13.1 to provide javascript utility functions // https://ravendb.net/docs/article-page/3.5/csharp/client-api/commands/patches/how-to-use-javascript-to-patch-your-documents#methods-objects-and-variables return new PatchCommandData(documentId, null, new PatchRequest { - Script = $@"this.{nameof(FailedMessage.Status)} = status; - this.{nameof(FailedMessage.FailureGroups)} = failureGroups; + Script = $@"this.{nameof(FailedMessage.Status)} = args.status; + this.{nameof(FailedMessage.FailureGroups)} = args.failureGroups; var newAttempts = this.{nameof(FailedMessage.ProcessingAttempts)}; @@ -68,7 +63,7 @@ ICommandData CreateFailedMessagesPatchCommand(string uniqueMessageId, FailedMess }}); if(duplicateIndex === -1){{ - newAttempts = _.union(newAttempts, [attempt]); + newAttempts = _.union(newAttempts, [args.attempt]); }} //Trim to the latest MaxProcessingAttempts @@ -87,43 +82,31 @@ ICommandData CreateFailedMessagesPatchCommand(string uniqueMessageId, FailedMess Values = new Dictionary { {"status", (int)FailedMessageStatus.Unresolved}, - {"failureGroups", serializedGroups}, - {"attempt", serializedAttempt} + {"failureGroups", groups}, + {"attempt", processingAttempt} }, }, patchIfMissing: new PatchRequest { - Script = $@"this.{nameof(FailedMessage.Status)} = status; - this.{nameof(FailedMessage.FailureGroups)} = failureGroups; - this.{nameof(FailedMessage.ProcessingAttempts)} = [attempt]; - this.{nameof(FailedMessage.UniqueMessageId)} = uniqueMessageId; - this.@metadata = {FailedMessageMetadata} + Script = $@"this.{nameof(FailedMessage.Status)} = args.status; + this.{nameof(FailedMessage.FailureGroups)} = args.failureGroups; + this.{nameof(FailedMessage.ProcessingAttempts)} = [args.attempt]; + this.{nameof(FailedMessage.UniqueMessageId)} = args.uniqueMessageId; + this['@metadata'] = {{ + '@collection': '{FailedMessageIdGenerator.CollectionName}', + 'Raven-Clr-Type': '{typeof(FailedMessage).AssemblyQualifiedName}' + }} ", Values = new Dictionary { {"status", (int)FailedMessageStatus.Unresolved}, - {"failureGroups", serializedGroups}, - {"attempt", serializedAttempt}, + {"failureGroups", groups}, + {"attempt", processingAttempt}, {"uniqueMessageId", uniqueMessageId} } }); } - static RavenDbRecoverabilityIngestionUnitOfWork() - { - //TODO: check if this actually works - Serializer = JsonSerializer.CreateDefault(); - Serializer.TypeNameHandling = TypeNameHandling.Auto; - - FailedMessageMetadata = JObject.Parse($@" - {{ - ""@collection"": ""{FailedMessageIdGenerator.CollectionName}"", - ""Raven-Clr-Type"": ""{typeof(FailedMessage).AssemblyQualifiedName}"" - }}"); - } - static int MaxProcessingAttempts = 10; - static readonly JObject FailedMessageMetadata; - static readonly JsonSerializer Serializer; } } \ No newline at end of file From dd9cc9eb4dec9407ffb9bd15aa286e09101d6ca2 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Mon, 25 Sep 2023 12:07:14 +0200 Subject: [PATCH 58/94] =?UTF-8?q?=F0=9F=A9=B9=20Removed=20Yield=20that=20w?= =?UTF-8?q?as=20there=20to=20please=20=20the=20editor=20config=20demon=20d?= =?UTF-8?q?uring=20testing...?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ErrorMessagesDataStore.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index 7be0c88a02..e5bd7901a9 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -37,8 +37,6 @@ public async Task>> GetAllMessages( bool includeSystemMessages ) { - await Task.Yield(); - using (var session = documentStore.OpenAsyncSession()) { var query = session.Query() From f7ae06ad3a0b7dcffdb29cf2ded59680f6f9c02e Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Mon, 25 Sep 2023 13:26:44 +0200 Subject: [PATCH 59/94] =?UTF-8?q?=F0=9F=A9=B9=20Transformer=20was=20incorr?= =?UTF-8?q?ectly=20using=20MessageId=20from=20METADATA=20from=20sorting=20?= =?UTF-8?q?while=20Index=20is=20based=20on=20document=20property=20resulti?= =?UTF-8?q?ng=20in=20RavenDB=20to=20ignore=20the=20ordering.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ErrorMessagesDataStore.cs | 9 ----- .../Transformers/MessagesViewTransformer.cs | 34 +++++-------------- .../ErrorMessageDataStoreTests.cs | 4 +++ 3 files changed, 13 insertions(+), 34 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index e5bd7901a9..82ae252310 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -28,7 +28,6 @@ class ErrorMessagesDataStore : IErrorMessageDataStore public ErrorMessagesDataStore(IDocumentStore documentStore) { this.documentStore = documentStore; - } public async Task>> GetAllMessages( @@ -47,7 +46,6 @@ bool includeSystemMessages .ProjectInto(); var results = await MessagesViewTransformer.Transform(query) - // TODO: Sort not working, seems to work if OrderBy is done here .ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); @@ -72,7 +70,6 @@ bool includeSystemMessages .ProjectInto(); var results = await MessagesViewTransformer.Transform(query) - // TODO: Sort not working, seems to work if OrderBy is done here .ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); @@ -97,7 +94,6 @@ SortInfo sortInfo .ProjectInto(); var results = await MessagesViewTransformer.Transform(query) - // TODO: Sort not working, seems to work if OrderBy is done here .ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); @@ -121,7 +117,6 @@ bool includeSystemMessages .ProjectInto(); var results = await MessagesViewTransformer.Transform(query) - // TODO: Sort not working, seems to work if OrderBy is done here .ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); @@ -144,7 +139,6 @@ SortInfo sortInfo .ProjectInto(); var results = await MessagesViewTransformer.Transform(query) - // TODO: Sort not working, seems to work if OrderBy is done here .ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); @@ -257,7 +251,6 @@ SortInfo sortInfo .ToQueryable(); var results = await FailedMessageViewTransformer.Transform(query) - // TODO: Sort not working, seems to work if OrderBy is done here .ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); @@ -306,7 +299,6 @@ SortInfo sortInfo .ToQueryable(); var results = await FailedMessageViewTransformer.Transform(query) - // TODO: Sort not working, seems to work if OrderBy is done here .ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); @@ -482,7 +474,6 @@ PagingInfo pagingInfo .ToQueryable(); var results = await FailedMessageViewTransformer.Transform(query) - // TODO: Sort not working, seems to work if OrderBy is done here .ToListAsync(); return results.ToQueryResult(stats); diff --git a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs index 76a9ac6968..6c3286d62d 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs @@ -1,6 +1,5 @@ namespace ServiceControl.CompositeViews.Messages { - using System; using System.Collections.Generic; using System.Linq; using MessageFailures; @@ -13,21 +12,10 @@ public static IQueryable Transform(IRavenQueryable query) { var results = from message in query - let metadata = message.ProcessingAttempts != null - ? message.ProcessingAttempts.Last().MessageMetadata - : message.MessageMetadata - let headers = - message.ProcessingAttempts != null ? message.ProcessingAttempts.Last().Headers : message.Headers - let processedAt = - message.ProcessingAttempts != null - ? message.ProcessingAttempts.Last().AttemptedAt - : message.ProcessedAt - let status = - message.ProcessingAttempts == null - ? !(bool)message.MessageMetadata["IsRetried"] - ? MessageStatus.Successful - : MessageStatus.ResolvedSuccessfully - : message.Status == FailedMessageStatus.Resolved + let metadata = message.ProcessingAttempts.Last().MessageMetadata + let headers = message.ProcessingAttempts.Last().Headers + let processedAt = message.ProcessingAttempts.Last().AttemptedAt + let status = message.Status == FailedMessageStatus.Resolved ? MessageStatus.ResolvedSuccessfully : message.Status == FailedMessageStatus.RetryIssued ? MessageStatus.RetryIssued @@ -39,7 +27,7 @@ from message in query select new // Cannot use type here as this is projected server-side { Id = message.UniqueMessageId, - MessageId = metadata["MessageId"], + message.MessageId, MessageType = metadata["MessageType"], SendingEndpoint = metadata["SendingEndpoint"], ReceivingEndpoint = metadata["ReceivingEndpoint"], @@ -64,15 +52,11 @@ from message in query return results.OfType(); } - public class Input + public class Input : MessagesViewIndex.SortAndFilterOptions { - public string Id { get; set; } - public string UniqueMessageId { get; set; } - public DateTime ProcessedAt { get; set; } - public Dictionary Headers { get; set; } - public Dictionary MessageMetadata { get; set; } - public List ProcessingAttempts { get; set; } - public FailedMessageStatus Status { get; set; } + public new FailedMessageStatus Status { get; } + public string UniqueMessageId { get; } + public List ProcessingAttempts { get; } } } diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/ErrorMessageDataStoreTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/ErrorMessageDataStoreTests.cs index 825552ffb6..4dc94ba90a 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/ErrorMessageDataStoreTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/ErrorMessageDataStoreTests.cs @@ -73,6 +73,8 @@ async Task GenerateAndSaveFailedMessage() m.UniqueMessageId = "a"; m.ProcessingAttempts.First().MessageMetadata["ReceivingEndpoint"].CastTo().Name = "RamonAndTomek"; m.ProcessingAttempts.First().MessageMetadata["CriticalTime"] = TimeSpan.FromSeconds(5); + m.ProcessingAttempts.First().MessageMetadata["TimeSent"] = DateTime.Parse("2023-09-23 00:00:00"); + m.ProcessingAttempts.First().MessageId = "MessageId-1"; }); await session.StoreAsync(processedMessage1); @@ -83,6 +85,8 @@ async Task GenerateAndSaveFailedMessage() m.UniqueMessageId = "b"; m.ProcessingAttempts.First().MessageMetadata["ReceivingEndpoint"].CastTo().Name = "RamonAndTomek"; m.ProcessingAttempts.First().MessageMetadata["CriticalTime"] = TimeSpan.FromSeconds(15); + m.ProcessingAttempts.First().MessageMetadata["TimeSent"] = DateTime.Parse("2023-09-23 00:01:00"); + m.ProcessingAttempts.First().MessageId = "MessageId-2"; }); await session.StoreAsync(processedMessage2); From 6d1470562fd70350862a1a115682b9f1e59932a6 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Mon, 25 Sep 2023 13:27:20 +0200 Subject: [PATCH 60/94] =?UTF-8?q?=F0=9F=A9=B9=20Apply=20previous=20commit?= =?UTF-8?q?=20change=20also=20on=20transformer=20of=20RavenDB=203.5=20impl?= =?UTF-8?q?ementation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Transformers/FailedMessageViewTransformer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ServiceControl.Persistence.RavenDb/Transformers/FailedMessageViewTransformer.cs b/src/ServiceControl.Persistence.RavenDb/Transformers/FailedMessageViewTransformer.cs index 157a7ce15e..3eafe8c7b0 100644 --- a/src/ServiceControl.Persistence.RavenDb/Transformers/FailedMessageViewTransformer.cs +++ b/src/ServiceControl.Persistence.RavenDb/Transformers/FailedMessageViewTransformer.cs @@ -19,7 +19,7 @@ public FailedMessageViewTransformer() SendingEndpoint = rec.MessageMetadata["SendingEndpoint"], ReceivingEndpoint = rec.MessageMetadata["ReceivingEndpoint"], TimeSent = (DateTime?)rec.MessageMetadata["TimeSent"], - MessageId = rec.MessageMetadata["MessageId"], + rec.MessageId, rec.FailureDetails.Exception, QueueAddress = rec.FailureDetails.AddressOfFailingEndpoint, NumberOfProcessingAttempts = failure.ProcessingAttempts.Count, From b0dabe073beff1463ccba196f7516a39c0ee286c Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Mon, 25 Sep 2023 13:51:01 +0200 Subject: [PATCH 61/94] =?UTF-8?q?=F0=9F=A9=B9=20RavenQuery.LastModified=20?= =?UTF-8?q?only=20works=20in=20query=20expressions,=20replaced=20with=20se?= =?UTF-8?q?ssion.Advanced.GetLastModifiedFor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../MessageRedirects/MessageRedirectsDataStore.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs index 6a4bf179b4..532c83324c 100644 --- a/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/MessageRedirects/MessageRedirectsDataStore.cs @@ -2,7 +2,6 @@ { using System.Threading.Tasks; using Raven.Client.Documents; - using Raven.Client.Documents.Queries; using ServiceControl.Persistence.MessageRedirects; class MessageRedirectsDataStore : IMessageRedirectsDataStore @@ -23,7 +22,7 @@ public async Task GetOrCreate() if (redirects != null) { redirects.ETag = session.Advanced.GetChangeVectorFor(redirects); - redirects.LastModified = RavenQuery.LastModified(redirects); + redirects.LastModified = session.Advanced.GetLastModifiedFor(redirects).Value; return redirects; } From e3249e4beb014315404e07ea7a297cd4d167911d Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Mon, 25 Sep 2023 14:21:53 +0200 Subject: [PATCH 62/94] controller dep --- .../ControllerDependencies.cs | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/src/ServiceControl.Persistence.Tests/ControllerDependencies.cs b/src/ServiceControl.Persistence.Tests/ControllerDependencies.cs index d0dce8ebee..fb03610195 100644 --- a/src/ServiceControl.Persistence.Tests/ControllerDependencies.cs +++ b/src/ServiceControl.Persistence.Tests/ControllerDependencies.cs @@ -2,6 +2,7 @@ { using System; using System.Collections.Generic; + using System.IO; using System.Linq; using System.Net.Http; using System.Reflection; @@ -12,6 +13,7 @@ using NServiceBus; using NUnit.Framework; using Particular.ServiceControl; + using PersistenceTests; using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.Infrastructure.DomainEvents; using ServiceControl.Infrastructure.WebApi; @@ -24,10 +26,10 @@ class ControllerDependencies /// instantiate each of the WebAPI controllers present in the ServiceControl app. /// [Test] - public Task EnsurePersistenceProvidesAllControllerDependencies() + public async Task EnsurePersistenceProvidesAllControllerDependencies() { // Arrange - //var testPersistence = null; //TODO: new TestPersistenceImpl(); + var testPersistence = new TestPersistenceImpl(); var assembly = Assembly.GetAssembly(typeof(WebApiHostBuilderExtensions)); var controllerTypes = assembly.DefinedTypes @@ -42,12 +44,12 @@ public Task EnsurePersistenceProvidesAllControllerDependencies() serviceCollection.AddSingleton(); serviceCollection.AddSingleton(new LoggingSettings("test")); - //testPersistence.Configure(serviceCollection); + testPersistence.Configure(serviceCollection); }) .UseNServiceBus(_ => { var config = new EndpointConfiguration("test"); - config.UseTransport(); + config.UseTransport().StorageDirectory(Path.Combine(TestContext.CurrentContext.WorkDirectory, "DependencyTest")); return config; }) .UseServiceControlComponents(new Settings(), ServiceControlMainInstance.Components); @@ -57,16 +59,11 @@ public Task EnsurePersistenceProvidesAllControllerDependencies() .UseWebApi(new List { assembly }, string.Empty, false) .Build(); + await host.Services.GetRequiredService().Initialize(); + // Assert Assert.That(host, Is.Not.Null); - // TODO: Kind of a hack, but gets the job done, since Raven35/5 have different startup requirements - // Could come up with a more cohesive way of doing this or just remove the IF when Raven35 goes away - //if (testPersistence.GetType().Assembly.FullName.Contains("RavenDb5")) - //{ - // await host.Services.GetRequiredService().Initialize(); - //} - // Make sure the list isn't suddenly empty Assert.That(controllerTypes.Length, Is.GreaterThan(10)); foreach (var controllerType in controllerTypes) @@ -74,8 +71,6 @@ public Task EnsurePersistenceProvidesAllControllerDependencies() Console.WriteLine($"Getting service {controllerType.FullName}"); Assert.That(host.Services.GetService(controllerType), Is.Not.Null); } - - return Task.CompletedTask; } } } \ No newline at end of file From 5da77a2a1184aa0ffe7f7d690cf2720138c3280a Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Mon, 25 Sep 2023 14:36:01 +0200 Subject: [PATCH 63/94] failed error import test --- .../Operations/FailedErrorImportCustomCheckTests.cs | 8 +++++++- src/ServiceControl.Persistence/FailedErrorImport.cs | 4 +++- .../Operations/ErrorIngestionFaultPolicy.cs | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedErrorImportCustomCheckTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedErrorImportCustomCheckTests.cs index 6ebc170ab9..e55969b1fc 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedErrorImportCustomCheckTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/Operations/FailedErrorImportCustomCheckTests.cs @@ -1,5 +1,6 @@ namespace ServiceControl.UnitTests.Operations { + using System; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using NServiceBus.CustomChecks; @@ -37,7 +38,12 @@ public async Task Fail_if_failed_imports() using (var session = DocumentStore.OpenAsyncSession()) { - await session.StoreAsync(new FailedErrorImport()); + await session.StoreAsync(new FailedErrorImport + { + Id = FailedErrorImport.MakeDocumentId(Guid.NewGuid()) + }); + + BlockToInspectDatabase(); await session.SaveChangesAsync(); } diff --git a/src/ServiceControl.Persistence/FailedErrorImport.cs b/src/ServiceControl.Persistence/FailedErrorImport.cs index 4790fb7341..ccae65dd17 100644 --- a/src/ServiceControl.Persistence/FailedErrorImport.cs +++ b/src/ServiceControl.Persistence/FailedErrorImport.cs @@ -4,7 +4,9 @@ namespace ServiceControl.Operations public class FailedErrorImport { - public Guid Id { get; set; } + public string Id { get; set; } public FailedTransportMessage Message { get; set; } + + public static string MakeDocumentId(Guid id) => $"FailedErrorImports/{id}"; } } \ No newline at end of file diff --git a/src/ServiceControl/Operations/ErrorIngestionFaultPolicy.cs b/src/ServiceControl/Operations/ErrorIngestionFaultPolicy.cs index 42b3d89ac9..1b0520243e 100644 --- a/src/ServiceControl/Operations/ErrorIngestionFaultPolicy.cs +++ b/src/ServiceControl/Operations/ErrorIngestionFaultPolicy.cs @@ -67,7 +67,7 @@ async Task Handle(Exception exception, FailedErrorImport failure) async Task DoLogging(Exception exception, FailedErrorImport failure) { - failure.Id = Guid.NewGuid(); + failure.Id = FailedErrorImport.MakeDocumentId(Guid.NewGuid()); // Write to data store await store.StoreFailedErrorImport(failure); From 2366def1f44611914f4335ae2e65744bf5b5c43e Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Mon, 25 Sep 2023 17:13:39 +0200 Subject: [PATCH 64/94] =?UTF-8?q?=F0=9F=A9=B9=20Forgot=20to=20pass=20the?= =?UTF-8?q?=20cancellation=20token=20source=20token=20and=20made=20both=20?= =?UTF-8?q?SharedDatabaseSetup=20files=20identical?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../SharedDatabaseSetup.cs | 6 +++--- .../SharedDatabaseSetup.cs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/SharedDatabaseSetup.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/SharedDatabaseSetup.cs index 32426c3e2e..6723857f8f 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB5/SharedDatabaseSetup.cs +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/SharedDatabaseSetup.cs @@ -12,12 +12,12 @@ public class SharedDatabaseSetup [OneTimeSetUp] public static async Task SetupSharedEmbeddedServer() { - using (var cancellationSource = new CancellationTokenSource(30_000)) + using (var cancellation = new CancellationTokenSource(30_000)) { - SharedInstance = await SharedEmbeddedServer.GetInstance(); + SharedInstance = await SharedEmbeddedServer.GetInstance(cancellation.Token); } } [OneTimeTearDown] public static void TearDown() => SharedInstance.Dispose(); -} +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/SharedDatabaseSetup.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/SharedDatabaseSetup.cs index 38d9cc0c43..6723857f8f 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/SharedDatabaseSetup.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/SharedDatabaseSetup.cs @@ -20,4 +20,4 @@ public static async Task SetupSharedEmbeddedServer() [OneTimeTearDown] public static void TearDown() => SharedInstance.Dispose(); -} +} \ No newline at end of file From 0cbca56d92f25bde99970bc7f0be073842c43e6a Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Mon, 25 Sep 2023 17:36:14 +0200 Subject: [PATCH 65/94] =?UTF-8?q?=F0=9F=A9=B9=20DocumentStore=20was=20disp?= =?UTF-8?q?osed=20in=20test=20it=20is=20resolved=20via=20host=20container?= =?UTF-8?q?=20(previously=20this=20was=20needed=20because=20the=20host=20w?= =?UTF-8?q?as=20not=20disposed)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../SubscriptionPersisterTests.cs | 7 ------- .../TestPersistenceImpl.cs | 13 ++++++++++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/SubscriptionPersisterTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/SubscriptionPersisterTests.cs index d6e0ddeeac..42ef67f41a 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/SubscriptionPersisterTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/SubscriptionPersisterTests.cs @@ -56,13 +56,6 @@ public async Task ShouldReturnSubscriptionsForNewerVersionsOfSameMessageType() documentStore = GetRequiredService(); } - [TearDown] - public new async Task TearDown() - { - await base.TearDown(); - documentStore.Dispose(); - } - IDocumentStore documentStore; } diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs index 644e4bb160..86d9fce004 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/TestPersistenceImpl.cs @@ -89,9 +89,16 @@ public override void BlockToInspectDatabase() public override async Task TearDown() { - // Comment this out temporarily to be able to inspect a database after the test has completed - var deleteDatabasesOperation = new DeleteDatabasesOperation(new DeleteDatabasesOperation.Parameters { DatabaseNames = new[] { databaseName }, HardDelete = true }); - await documentStore.Maintenance.Server.SendAsync(deleteDatabasesOperation); + try + { + // Comment this out temporarily to be able to inspect a database after the test has completed + var deleteDatabasesOperation = new DeleteDatabasesOperation(new DeleteDatabasesOperation.Parameters { DatabaseNames = new[] { databaseName }, HardDelete = true }); + await documentStore.Maintenance.Server.SendAsync(deleteDatabasesOperation); + } + catch (OperationCanceledException ex) + { + throw new Exception("OCE during database deletion, review if the document store is accidentally disposed too early in the test", ex); + } } } } \ No newline at end of file From 7f766cf39d58461f9b54b5b1868441bba58bdb74 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Mon, 25 Sep 2023 17:44:47 +0200 Subject: [PATCH 66/94] =?UTF-8?q?=F0=9F=A9=B9=20Removed=20the=20audit=20ty?= =?UTF-8?q?pes=20but=20keeping=20`ServiceControl.EventLog.EventLogItem`=20?= =?UTF-8?q?as=20likely=20that=20should=20be=20returned?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ApprovalFiles/RavenPersistedTypes.Verify.approved.txt | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/RavenPersistedTypes.Verify.approved.txt b/src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/RavenPersistedTypes.Verify.approved.txt index 2024580e53..67ba6c313a 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/RavenPersistedTypes.Verify.approved.txt +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/RavenPersistedTypes.Verify.approved.txt @@ -1,17 +1,13 @@ ServiceControl.Contracts.CustomChecks.CustomCheck, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null ServiceControl.EventLog.EventLogItem, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null -ServiceControl.MessageAuditing.ProcessedMessage, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null ServiceControl.MessageFailures.FailedMessage, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null ServiceControl.MessageFailures.GroupComment, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null ServiceControl.MessageFailures.QueueAddress, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null -ServiceControl.Operations.FailedAuditImport, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null ServiceControl.Operations.FailedErrorImport, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null ServiceControl.Persistence.KnownEndpoint, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null -ServiceControl.Persistence.MessagesViewIndex+SortAndFilterOptions, ServiceControl.Persistence.RavenDb, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null +ServiceControl.Persistence.MessagesViewIndex+SortAndFilterOptions, ServiceControl.Persistence.RavenDb5, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null ServiceControl.Persistence.RetryBatch, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null ServiceControl.Persistence.RetryBatchGroup, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null ServiceControl.Recoverability.FailedMessageRetry, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null ServiceControl.Recoverability.FailureGroupMessageView, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null -ServiceControl.Recoverability.FailureGroupView, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null -ServiceControl.SagaAudit.SagaHistory, ServiceControl.Audit.Persistence.SagaAudit, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null -ServiceControl.SagaAudit.SagaListIndex+Result, ServiceControl.Persistence.RavenDb, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null \ No newline at end of file +ServiceControl.Recoverability.FailureGroupView, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null \ No newline at end of file From 2fb2be51e71811b15b388cc96d22027897304f9c Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Mon, 25 Sep 2023 18:19:27 +0200 Subject: [PATCH 67/94] =?UTF-8?q?=F0=9F=A9=B9=20Missing=20closing=20bracke?= =?UTF-8?q?t,=20test=20is=20still=20failing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../UnitOfWork/RavenDbIngestionUnitOfWork.cs | 3 ++- .../UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs index f1f67516c3..2a3ca06465 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs @@ -26,7 +26,8 @@ public override async Task Complete() using (var session = store.OpenAsyncSession()) { // not really interested in the batch results since a batch is atomic - session.Advanced.Defer(commands.ToArray()); + var commands = this.commands.ToArray(); + session.Advanced.Defer(commands); await session.SaveChangesAsync(); } } diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs index 63820e622c..69bb23e6b4 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs @@ -31,7 +31,7 @@ static PatchCommandData CreateKnownEndpointsPutCommand(KnownEndpoint endpoint) return new PatchCommandData(docId, null, new PatchRequest { //TODO: check if this works - Script = $"put('{KnownEndpoint.CollectionName}/', {document}" + Script = $"put('{KnownEndpoint.CollectionName}/', {document})" }); } From dbc6865b04f284654230838f109e84b932608f51 Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Tue, 26 Sep 2023 08:48:15 +0200 Subject: [PATCH 68/94] copying over properties to insert know endpoints --- .../RavenDbMonitoringIngestionUnitOfWork.cs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs index 69bb23e6b4..624002dfd1 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbMonitoringIngestionUnitOfWork.cs @@ -28,11 +28,19 @@ static PatchCommandData CreateKnownEndpointsPutCommand(KnownEndpoint endpoint) var docId = RavenDbMonitoringDataStore.MakeDocumentId(endpoint.EndpointDetails.GetDeterministicId()); - return new PatchCommandData(docId, null, new PatchRequest + var request = new PatchRequest { - //TODO: check if this works - Script = $"put('{KnownEndpoint.CollectionName}/', {document})" - }); + Script = @$" + var insert = {document}; + + for(var key in insert) {{ + if(insert.hasOwnProperty(key)) {{ + this[key] = insert[key]; + }} + }}" + }; + + return new PatchCommandData(docId, null, request, request); } static RavenDbMonitoringIngestionUnitOfWork() From b9fe1dc905f58e19ee7f14d8e496daa5a236bc03 Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Tue, 26 Sep 2023 12:57:18 +0200 Subject: [PATCH 69/94] eventlogitem cleanup --- .../ErrorMessagesDataStore.cs | 10 +++++++++- .../RavenPersistedTypes.Verify.approved.txt | 1 - .../EventLog/EventLogItem.cs | 5 ----- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index 82ae252310..ef170248a6 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Editing; using NServiceBus.Logging; + using Raven.Client; using Raven.Client.Documents; using Raven.Client.Documents.Commands; using Raven.Client.Documents.Linq; @@ -24,10 +25,12 @@ class ErrorMessagesDataStore : IErrorMessageDataStore { readonly IDocumentStore documentStore; + readonly TimeSpan eventsRetentionPeriod; - public ErrorMessagesDataStore(IDocumentStore documentStore) + public ErrorMessagesDataStore(IDocumentStore documentStore, RavenDBPersisterSettings settings) { this.documentStore = documentStore; + eventsRetentionPeriod = settings.EventsRetentionPeriod; } public async Task>> GetAllMessages( @@ -732,9 +735,14 @@ public Task FetchFromFailedMessage(string uniqueMessageId) public async Task StoreEventLogItem(EventLogItem logItem) { + var expiration = DateTime.UtcNow + eventsRetentionPeriod; + using (var session = documentStore.OpenAsyncSession()) { await session.StoreAsync(logItem); + + session.Advanced.GetMetadataFor(logItem)[Constants.Documents.Metadata.Expires] = expiration; + await session.SaveChangesAsync(); } } diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/RavenPersistedTypes.Verify.approved.txt b/src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/RavenPersistedTypes.Verify.approved.txt index 67ba6c313a..a38084fb1e 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/RavenPersistedTypes.Verify.approved.txt +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/ApprovalFiles/RavenPersistedTypes.Verify.approved.txt @@ -1,5 +1,4 @@ ServiceControl.Contracts.CustomChecks.CustomCheck, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null -ServiceControl.EventLog.EventLogItem, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null ServiceControl.MessageFailures.FailedMessage, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null ServiceControl.MessageFailures.GroupComment, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null ServiceControl.MessageFailures.QueueAddress, ServiceControl.Persistence, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null diff --git a/src/ServiceControl.Persistence/EventLog/EventLogItem.cs b/src/ServiceControl.Persistence/EventLog/EventLogItem.cs index 22648ae28e..cf75a8da22 100644 --- a/src/ServiceControl.Persistence/EventLog/EventLogItem.cs +++ b/src/ServiceControl.Persistence/EventLog/EventLogItem.cs @@ -5,11 +5,6 @@ public class EventLogItem { - public EventLogItem() - { - RelatedTo = new List(); - } - public string Id { get; set; } public string Description { get; set; } public Severity Severity { get; set; } From 90ae7a899f86d09a95d848110dc1144a8739b5dc Mon Sep 17 00:00:00 2001 From: David Boike Date: Mon, 25 Sep 2023 13:29:58 -0500 Subject: [PATCH 70/94] Consolidate use of "Memory" into Infrastructure project --- .../Infrastructure/Memory.cs | 9 --------- .../UnitOfWork/RavenDbAuditIngestionUnitOfWork.cs | 3 +-- .../ServiceControl.Audit.Persistence.csproj | 2 +- src/ServiceControl.Audit/Auditing/AuditPersister.cs | 1 + src/ServiceControl.Audit/Infrastructure/Memory.cs | 9 --------- .../Memory.cs | 2 +- .../ServiceControl.Infrastructure.csproj | 1 + 7 files changed, 5 insertions(+), 22 deletions(-) delete mode 100644 src/ServiceControl.Audit.Persistence.RavenDb5/Infrastructure/Memory.cs delete mode 100644 src/ServiceControl.Audit/Infrastructure/Memory.cs rename src/{ServiceControl/Infrastructure => ServiceControl.Infrastructure}/Memory.cs (86%) diff --git a/src/ServiceControl.Audit.Persistence.RavenDb5/Infrastructure/Memory.cs b/src/ServiceControl.Audit.Persistence.RavenDb5/Infrastructure/Memory.cs deleted file mode 100644 index 7c5ca82b1f..0000000000 --- a/src/ServiceControl.Audit.Persistence.RavenDb5/Infrastructure/Memory.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace ServiceControl.Audit.Persistence.RavenDb.Infrastructure -{ - using Microsoft.IO; - - static class Memory - { - public static readonly RecyclableMemoryStreamManager Manager = new RecyclableMemoryStreamManager(); - } -} \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence.RavenDb5/UnitOfWork/RavenDbAuditIngestionUnitOfWork.cs b/src/ServiceControl.Audit.Persistence.RavenDb5/UnitOfWork/RavenDbAuditIngestionUnitOfWork.cs index 7e2c625224..14fca8533f 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDb5/UnitOfWork/RavenDbAuditIngestionUnitOfWork.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDb5/UnitOfWork/RavenDbAuditIngestionUnitOfWork.cs @@ -10,7 +10,6 @@ using Raven.Client; using Raven.Client.Documents.BulkInsert; using Raven.Client.Json; - using ServiceControl.Audit.Persistence.RavenDb.Infrastructure; using ServiceControl.SagaAudit; class RavenDbAuditIngestionUnitOfWork : IAuditIngestionUnitOfWork @@ -43,7 +42,7 @@ public async Task RecordProcessedMessage(ProcessedMessage processedMessage, byte if (body != null) { - using (var stream = Memory.Manager.GetStream(Guid.NewGuid(), processedMessage.Id, body, 0, body.Length)) + using (var stream = ServiceControl.Infrastructure.Memory.Manager.GetStream(Guid.NewGuid(), processedMessage.Id, body, 0, body.Length)) { if (!processedMessage.Headers.TryGetValue(Headers.ContentType, out var contentType)) { diff --git a/src/ServiceControl.Audit.Persistence/ServiceControl.Audit.Persistence.csproj b/src/ServiceControl.Audit.Persistence/ServiceControl.Audit.Persistence.csproj index 26707ed30d..616f8e0286 100644 --- a/src/ServiceControl.Audit.Persistence/ServiceControl.Audit.Persistence.csproj +++ b/src/ServiceControl.Audit.Persistence/ServiceControl.Audit.Persistence.csproj @@ -6,12 +6,12 @@ + - diff --git a/src/ServiceControl.Audit/Auditing/AuditPersister.cs b/src/ServiceControl.Audit/Auditing/AuditPersister.cs index 98a9f74e6c..1ef7a350c8 100644 --- a/src/ServiceControl.Audit/Auditing/AuditPersister.cs +++ b/src/ServiceControl.Audit/Auditing/AuditPersister.cs @@ -16,6 +16,7 @@ using ServiceControl.Audit.Persistence.Infrastructure; using ServiceControl.Audit.Persistence.Monitoring; using ServiceControl.EndpointPlugin.Messages.SagaState; + using ServiceControl.Infrastructure; using ServiceControl.Infrastructure.Metrics; using ServiceControl.SagaAudit; using JsonSerializer = Newtonsoft.Json.JsonSerializer; diff --git a/src/ServiceControl.Audit/Infrastructure/Memory.cs b/src/ServiceControl.Audit/Infrastructure/Memory.cs deleted file mode 100644 index 4fde1b7bfc..0000000000 --- a/src/ServiceControl.Audit/Infrastructure/Memory.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace ServiceControl.Audit.Infrastructure -{ - using Microsoft.IO; - - static class Memory - { - public static readonly RecyclableMemoryStreamManager Manager = new RecyclableMemoryStreamManager(); - } -} \ No newline at end of file diff --git a/src/ServiceControl/Infrastructure/Memory.cs b/src/ServiceControl.Infrastructure/Memory.cs similarity index 86% rename from src/ServiceControl/Infrastructure/Memory.cs rename to src/ServiceControl.Infrastructure/Memory.cs index c9fb71debf..da0012aa35 100644 --- a/src/ServiceControl/Infrastructure/Memory.cs +++ b/src/ServiceControl.Infrastructure/Memory.cs @@ -2,7 +2,7 @@ { using Microsoft.IO; - static class Memory + public static class Memory { public static readonly RecyclableMemoryStreamManager Manager = new RecyclableMemoryStreamManager(); } diff --git a/src/ServiceControl.Infrastructure/ServiceControl.Infrastructure.csproj b/src/ServiceControl.Infrastructure/ServiceControl.Infrastructure.csproj index c81f250251..643937d08c 100644 --- a/src/ServiceControl.Infrastructure/ServiceControl.Infrastructure.csproj +++ b/src/ServiceControl.Infrastructure/ServiceControl.Infrastructure.csproj @@ -6,6 +6,7 @@ + From 58ca2cb696f086d0c41943407c2be1be621b36c6 Mon Sep 17 00:00:00 2001 From: David Boike Date: Mon, 25 Sep 2023 13:05:03 -0500 Subject: [PATCH 71/94] Move Raven35-specific body storage enricher details into Raven35 implementation --- .../RavenDbPersistence.cs | 2 ++ .../UnitOfWork}/BodyStorageEnricher.cs | 10 ++++---- .../UnitOfWork/RavenDbIngestionUnitOfWork.cs | 5 ++-- .../RavenDbIngestionUnitOfWorkFactory.cs | 7 ++++-- ...avenDbRecoverabilityIngestionUnitOfWork.cs | 24 ++++++++++++------- .../RavenAttachmentsBodyStorage.cs | 3 --- ...avenDbRecoverabilityIngestionUnitOfWork.cs | 10 ++++---- .../BodyStorage/BodyStorageEnricherTests.cs | 11 ++++----- .../TransportMessageExtensions.cs | 2 +- .../PersistenceSettings.cs | 4 ++++ .../IRecoverabilityIngestionUnitOfWork.cs | 4 +++- .../Operations/ErrorIngestor.cs | 5 +--- .../Operations/ErrorProcessor.cs | 9 ++----- .../Recoverability/RecoverabilityComponent.cs | 4 ---- 14 files changed, 54 insertions(+), 46 deletions(-) rename src/{ServiceControl/Operations/BodyStorage => ServiceControl.Persistence.RavenDb/UnitOfWork}/BodyStorageEnricher.cs (90%) rename src/{ServiceControl.UnitTests => ServiceControl.Persistence.Tests.RavenDb}/BodyStorage/BodyStorageEnricherTests.cs (95%) diff --git a/src/ServiceControl.Persistence.RavenDb/RavenDbPersistence.cs b/src/ServiceControl.Persistence.RavenDb/RavenDbPersistence.cs index c80cb622a5..fe5e51bf71 100644 --- a/src/ServiceControl.Persistence.RavenDb/RavenDbPersistence.cs +++ b/src/ServiceControl.Persistence.RavenDb/RavenDbPersistence.cs @@ -43,6 +43,8 @@ public void Configure(IServiceCollection serviceCollection) serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); serviceCollection.AddSingleton(p => p.GetRequiredService()); serviceCollection.AddHostedService(p => p.GetRequiredService()); diff --git a/src/ServiceControl/Operations/BodyStorage/BodyStorageEnricher.cs b/src/ServiceControl.Persistence.RavenDb/UnitOfWork/BodyStorageEnricher.cs similarity index 90% rename from src/ServiceControl/Operations/BodyStorage/BodyStorageEnricher.cs rename to src/ServiceControl.Persistence.RavenDb/UnitOfWork/BodyStorageEnricher.cs index 1bef32efe0..df7744bc71 100644 --- a/src/ServiceControl/Operations/BodyStorage/BodyStorageEnricher.cs +++ b/src/ServiceControl.Persistence.RavenDb/UnitOfWork/BodyStorageEnricher.cs @@ -1,17 +1,18 @@ namespace ServiceControl.Operations.BodyStorage { - using Infrastructure; using System.Collections.Generic; using System.Text; using System.Threading.Tasks; + using Infrastructure; using NServiceBus; using NServiceBus.Logging; - using ServiceBus.Management.Infrastructure.Settings; + using ServiceControl.Persistence; + using ServiceControl.Persistence.Infrastructure; using ProcessingAttempt = MessageFailures.FailedMessage.ProcessingAttempt; class BodyStorageEnricher { - public BodyStorageEnricher(IBodyStorage bodyStorage, Settings settings) + public BodyStorageEnricher(IBodyStorage bodyStorage, PersistenceSettings settings) { this.settings = settings; this.bodyStorage = bodyStorage; @@ -58,6 +59,7 @@ async ValueTask StoreBody(byte[] body, ProcessingAttempt processingAttempt, int { try { + // TODO: Make sure this is set, I just added the property to PersistenceSettings because I didn't have access to the root Settings if (settings.EnableFullTextSearchOnBodies) { processingAttempt.MessageMetadata.Add("Body", enc.GetString(body)); @@ -93,7 +95,7 @@ async Task StoreBodyInBodyStorage(byte[] body, string bodyId, string contentType static readonly Encoding enc = new UTF8Encoding(true, true); static readonly ILog log = LogManager.GetLogger(); readonly IBodyStorage bodyStorage; - readonly Settings settings; + readonly PersistenceSettings settings; // large object heap starts above 85000 bytes and not above 85 KB! internal const int LargeObjectHeapThreshold = 85 * 1000; diff --git a/src/ServiceControl.Persistence.RavenDb/UnitOfWork/RavenDbIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb/UnitOfWork/RavenDbIngestionUnitOfWork.cs index d6b4d9bcaa..66d646352c 100644 --- a/src/ServiceControl.Persistence.RavenDb/UnitOfWork/RavenDbIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb/UnitOfWork/RavenDbIngestionUnitOfWork.cs @@ -4,6 +4,7 @@ using System.Threading.Tasks; using Raven.Abstractions.Commands; using Raven.Client; + using ServiceControl.Operations.BodyStorage; using ServiceControl.Persistence.UnitOfWork; class RavenDbIngestionUnitOfWork : IngestionUnitOfWorkBase @@ -11,12 +12,12 @@ class RavenDbIngestionUnitOfWork : IngestionUnitOfWorkBase readonly IDocumentStore store; readonly ConcurrentBag commands; - public RavenDbIngestionUnitOfWork(IDocumentStore store) + public RavenDbIngestionUnitOfWork(IDocumentStore store, BodyStorageEnricher bodyStorageEnricher) { this.store = store; commands = new ConcurrentBag(); Monitoring = new RavenDbMonitoringIngestionUnitOfWork(this); - Recoverability = new RavenDbRecoverabilityIngestionUnitOfWork(this); + Recoverability = new RavenDbRecoverabilityIngestionUnitOfWork(this, bodyStorageEnricher); } internal void AddCommand(ICommandData command) => commands.Add(command); diff --git a/src/ServiceControl.Persistence.RavenDb/UnitOfWork/RavenDbIngestionUnitOfWorkFactory.cs b/src/ServiceControl.Persistence.RavenDb/UnitOfWork/RavenDbIngestionUnitOfWorkFactory.cs index 3fe030ec17..8ef4f9604e 100644 --- a/src/ServiceControl.Persistence.RavenDb/UnitOfWork/RavenDbIngestionUnitOfWorkFactory.cs +++ b/src/ServiceControl.Persistence.RavenDb/UnitOfWork/RavenDbIngestionUnitOfWorkFactory.cs @@ -2,21 +2,24 @@ { using System.Threading.Tasks; using Raven.Client; + using ServiceControl.Operations.BodyStorage; using ServiceControl.Persistence.UnitOfWork; class RavenDbIngestionUnitOfWorkFactory : IIngestionUnitOfWorkFactory { readonly IDocumentStore store; readonly MinimumRequiredStorageState customCheckState; + readonly BodyStorageEnricher bodyStorageEnricher; - public RavenDbIngestionUnitOfWorkFactory(IDocumentStore store, MinimumRequiredStorageState customCheckState) + public RavenDbIngestionUnitOfWorkFactory(IDocumentStore store, MinimumRequiredStorageState customCheckState, BodyStorageEnricher bodyStorageEnricher) { this.store = store; this.customCheckState = customCheckState; + this.bodyStorageEnricher = bodyStorageEnricher; } public ValueTask StartNew() - => new ValueTask(new RavenDbIngestionUnitOfWork(store)); + => new ValueTask(new RavenDbIngestionUnitOfWork(store, bodyStorageEnricher)); public bool CanIngestMore() { diff --git a/src/ServiceControl.Persistence.RavenDb/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs index 6ec853c166..c736f7eb89 100644 --- a/src/ServiceControl.Persistence.RavenDb/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs @@ -2,33 +2,41 @@ { using System.Collections.Generic; using System.Threading.Tasks; + using NServiceBus.Transport; using Raven.Abstractions.Commands; using Raven.Abstractions.Data; using Raven.Abstractions.Extensions; using Raven.Imports.Newtonsoft.Json; using Raven.Json.Linq; using ServiceControl.MessageFailures; + using ServiceControl.Operations.BodyStorage; + using ServiceControl.Persistence.Infrastructure; using ServiceControl.Persistence.UnitOfWork; using ServiceControl.Recoverability; class RavenDbRecoverabilityIngestionUnitOfWork : IRecoverabilityIngestionUnitOfWork { - RavenDbIngestionUnitOfWork parentUnitOfWork; + readonly RavenDbIngestionUnitOfWork parentUnitOfWork; + readonly BodyStorageEnricher bodyStorageEnricher; - public RavenDbRecoverabilityIngestionUnitOfWork(RavenDbIngestionUnitOfWork parentUnitOfWork) + public RavenDbRecoverabilityIngestionUnitOfWork(RavenDbIngestionUnitOfWork parentUnitOfWork, BodyStorageEnricher bodyStorageEnricher) { this.parentUnitOfWork = parentUnitOfWork; + this.bodyStorageEnricher = bodyStorageEnricher; } - public Task RecordFailedProcessingAttempt( - string uniqueMessageId, + public async Task RecordFailedProcessingAttempt( + MessageContext context, FailedMessage.ProcessingAttempt processingAttempt, List groups) { - parentUnitOfWork.AddCommand( - CreateFailedMessagesPatchCommand(uniqueMessageId, processingAttempt, groups) - ); - return Task.CompletedTask; + // Store body - out of band of the Unit of Work in Raven 3.5 + await bodyStorageEnricher.StoreErrorMessageBody(context.Body, processingAttempt); + + // Add command to unit of work to store metadata + var uniqueMessageId = context.Headers.UniqueId(); + var storeMessageCmd = CreateFailedMessagesPatchCommand(uniqueMessageId, processingAttempt, groups); + parentUnitOfWork.AddCommand(storeMessageCmd); } public Task RecordSuccessfulRetry(string retriedMessageUniqueId) diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs b/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs index a95adde166..de3569a160 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs @@ -5,9 +5,6 @@ using Raven.Client.Documents; using Raven.Client.Documents.Operations.Attachments; - // TODO: For Raven5, look at how the Audit instance is implementing this, as Attachments won't exist - // and there will be no need for a fallback method on a new persistence - // Ramon: Don't understand the comment, audit RavenDB 5 is using attachments.... class RavenAttachmentsBodyStorage : IBodyStorage { const string AttachmentName = "body"; diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs index 38923ea86c..9d2c95a9ae 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs @@ -2,9 +2,11 @@ { using System.Collections.Generic; using System.Threading.Tasks; + using NServiceBus.Transport; using Raven.Client.Documents.Commands.Batches; using Raven.Client.Documents.Operations; using ServiceControl.MessageFailures; + using ServiceControl.Persistence.Infrastructure; using ServiceControl.Persistence.UnitOfWork; using ServiceControl.Recoverability; @@ -18,13 +20,13 @@ public RavenDbRecoverabilityIngestionUnitOfWork(RavenDbIngestionUnitOfWork paren } public Task RecordFailedProcessingAttempt( - string uniqueMessageId, + MessageContext context, FailedMessage.ProcessingAttempt processingAttempt, List groups) { - parentUnitOfWork.AddCommand( - CreateFailedMessagesPatchCommand(uniqueMessageId, processingAttempt, groups) - ); + var uniqueMessageId = context.Headers.UniqueId(); + var storeMessageCmd = CreateFailedMessagesPatchCommand(uniqueMessageId, processingAttempt, groups); + parentUnitOfWork.AddCommand(storeMessageCmd); return Task.CompletedTask; } diff --git a/src/ServiceControl.UnitTests/BodyStorage/BodyStorageEnricherTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb/BodyStorage/BodyStorageEnricherTests.cs similarity index 95% rename from src/ServiceControl.UnitTests/BodyStorage/BodyStorageEnricherTests.cs rename to src/ServiceControl.Persistence.Tests.RavenDb/BodyStorage/BodyStorageEnricherTests.cs index 884913d329..67bac82193 100644 --- a/src/ServiceControl.UnitTests/BodyStorage/BodyStorageEnricherTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb/BodyStorage/BodyStorageEnricherTests.cs @@ -8,7 +8,6 @@ namespace ServiceControl.UnitTests.BodyStorage using MessageFailures; using NServiceBus; using NUnit.Framework; - using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.Operations.BodyStorage; [TestFixture] @@ -21,7 +20,7 @@ public class BodyStorageEnricherTests public async Task Should_store_body_in_storage_when_binary_and_below_LOH_threshold() { var fakeStorage = new FakeBodyStorage(); - var settings = new Settings(); + var settings = new RavenDBPersisterSettings(); var enricher = new BodyStorageEnricher(fakeStorage, settings); var expectedBodySize = MinBodySizeAboveLOHThreshold; @@ -42,7 +41,7 @@ public async Task Should_store_body_in_storage_when_binary_and_below_LOH_thresho public async Task Should_store_body_in_metadata_when_not_binary_and_below_LOH_threshold() { var fakeStorage = new FakeBodyStorage(); - var settings = new Settings(); + var settings = new RavenDBPersisterSettings(); var enricher = new BodyStorageEnricher(fakeStorage, settings); var expectedBodySize = MaxBodySizeBelowLOHThreshold; @@ -63,7 +62,7 @@ public async Task Should_store_body_in_metadata_when_not_binary_and_below_LOH_th public async Task Should_store_body_in_non_indexed_metadata_when_full_text_disabled_and_not_binary_and_below_LOH_threshold() { var fakeStorage = new FakeBodyStorage(); - var settings = new Settings + var settings = new RavenDBPersisterSettings { EnableFullTextSearchOnBodies = false, }; @@ -86,7 +85,7 @@ public async Task Should_store_body_in_non_indexed_metadata_when_full_text_disab public async Task Should_store_body_in_storage_when_not_binary_and_above_LOH_threshold() { var fakeStorage = new FakeBodyStorage(); - var settings = new Settings(); + var settings = new RavenDBPersisterSettings(); var enricher = new BodyStorageEnricher(fakeStorage, settings); var expectedBodySize = MinBodySizeAboveLOHThreshold; @@ -106,7 +105,7 @@ public async Task Should_store_body_in_storage_when_not_binary_and_above_LOH_thr public async Task Should_store_body_in_storage_when_encoding_fails() { var fakeStorage = new FakeBodyStorage(); - var settings = new Settings(); + var settings = new RavenDBPersisterSettings(); var enricher = new BodyStorageEnricher(fakeStorage, settings); var body = new byte[] { 0x00, 0xDE }; diff --git a/src/ServiceControl.Persistence/Infrastructure/TransportMessageExtensions.cs b/src/ServiceControl.Persistence/Infrastructure/TransportMessageExtensions.cs index 6fb0ef6d45..1e194d1bcf 100644 --- a/src/ServiceControl.Persistence/Infrastructure/TransportMessageExtensions.cs +++ b/src/ServiceControl.Persistence/Infrastructure/TransportMessageExtensions.cs @@ -5,7 +5,7 @@ using NServiceBus; using NServiceBus.Faults; - static class HeaderExtensions + public static class HeaderExtensions { public static string ProcessingEndpointName(this IReadOnlyDictionary headers) { diff --git a/src/ServiceControl.Persistence/PersistenceSettings.cs b/src/ServiceControl.Persistence/PersistenceSettings.cs index 6ee4345f73..d3d39b8fbd 100644 --- a/src/ServiceControl.Persistence/PersistenceSettings.cs +++ b/src/ServiceControl.Persistence/PersistenceSettings.cs @@ -8,5 +8,9 @@ public abstract class PersistenceSettings public bool MaintenanceMode { get; set; } //HINT: This needs to be here so that ServerControl instance can add an instance specific metadata to tweak the DatabasePath value public string DatabasePath { get; set; } + + + // TODO: This isn't set yet, but body storage, index creation will all need to know this + public bool EnableFullTextSearchOnBodies { get; set; } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence/UnitOfWork/IRecoverabilityIngestionUnitOfWork.cs b/src/ServiceControl.Persistence/UnitOfWork/IRecoverabilityIngestionUnitOfWork.cs index 139524c472..8d7b275421 100644 --- a/src/ServiceControl.Persistence/UnitOfWork/IRecoverabilityIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence/UnitOfWork/IRecoverabilityIngestionUnitOfWork.cs @@ -2,13 +2,15 @@ { using System.Collections.Generic; using System.Threading.Tasks; + using NServiceBus.Transport; using ServiceControl.MessageFailures; public interface IRecoverabilityIngestionUnitOfWork { - Task RecordFailedProcessingAttempt(string uniqueMessageId, + Task RecordFailedProcessingAttempt(MessageContext context, FailedMessage.ProcessingAttempt processingAttempt, List groups); + Task RecordSuccessfulRetry(string retriedMessageUniqueId); } } \ No newline at end of file diff --git a/src/ServiceControl/Operations/ErrorIngestor.cs b/src/ServiceControl/Operations/ErrorIngestor.cs index 8222e081f4..e26e9a4592 100644 --- a/src/ServiceControl/Operations/ErrorIngestor.cs +++ b/src/ServiceControl/Operations/ErrorIngestor.cs @@ -5,7 +5,6 @@ using System.Diagnostics; using System.Linq; using System.Threading.Tasks; - using BodyStorage; using Contracts.Operations; using Infrastructure.DomainEvents; using Infrastructure.Metrics; @@ -25,7 +24,6 @@ public ErrorIngestor(Metrics metrics, IEnumerable errorEnrichers, IEnumerable failedMessageEnrichers, IDomainEvents domainEvents, - IBodyStorage bodyStorage, IIngestionUnitOfWorkFactory unitOfWorkFactory, Settings settings) { this.unitOfWorkFactory = unitOfWorkFactory; @@ -42,8 +40,7 @@ public ErrorIngestor(Metrics metrics, }.Concat(errorEnrichers).ToArray(); - var bodyStorageEnricher = new BodyStorageEnricher(bodyStorage, settings); - errorProcessor = new ErrorProcessor(bodyStorageEnricher, enrichers, failedMessageEnrichers.ToArray(), domainEvents, ingestedMeter); + errorProcessor = new ErrorProcessor(enrichers, failedMessageEnrichers.ToArray(), domainEvents, ingestedMeter); retryConfirmationProcessor = new RetryConfirmationProcessor(domainEvents); } diff --git a/src/ServiceControl/Operations/ErrorProcessor.cs b/src/ServiceControl/Operations/ErrorProcessor.cs index 68597e8829..457eddf130 100644 --- a/src/ServiceControl/Operations/ErrorProcessor.cs +++ b/src/ServiceControl/Operations/ErrorProcessor.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; - using BodyStorage; using Contracts.MessageFailures; using Contracts.Operations; using Infrastructure; @@ -18,10 +17,9 @@ class ErrorProcessor { - public ErrorProcessor(BodyStorageEnricher bodyStorageEnricher, IEnrichImportedErrorMessages[] enrichers, IFailedMessageEnricher[] failedMessageEnrichers, IDomainEvents domainEvents, + public ErrorProcessor(IEnrichImportedErrorMessages[] enrichers, IFailedMessageEnricher[] failedMessageEnrichers, IDomainEvents domainEvents, Counter ingestedCounter) { - this.bodyStorageEnricher = bodyStorageEnricher; this.enrichers = enrichers; this.domainEvents = domainEvents; this.ingestedCounter = ingestedCounter; @@ -120,11 +118,9 @@ async Task ProcessMessage(MessageContext context, IIngestionUnitOfWork unitOfWor new Dictionary(metadata), failureDetails); - await bodyStorageEnricher.StoreErrorMessageBody(context.Body, processingAttempt); - var groups = failedMessageFactory.GetGroups((string)metadata["MessageType"], failureDetails, processingAttempt); - await unitOfWork.Recoverability.RecordFailedProcessingAttempt(context.Headers.UniqueId(), processingAttempt, groups); + await unitOfWork.Recoverability.RecordFailedProcessingAttempt(context, processingAttempt, groups); context.Extensions.Set(failureDetails); context.Extensions.Set(enricherContext.NewEndpoints); @@ -173,7 +169,6 @@ static void RecordKnownEndpoints(EndpointDetails observedEndpoint, Dictionary(); } diff --git a/src/ServiceControl/Recoverability/RecoverabilityComponent.cs b/src/ServiceControl/Recoverability/RecoverabilityComponent.cs index bdc92a0e82..96f91c7667 100644 --- a/src/ServiceControl/Recoverability/RecoverabilityComponent.cs +++ b/src/ServiceControl/Recoverability/RecoverabilityComponent.cs @@ -15,7 +15,6 @@ using NServiceBus.Logging; using NServiceBus.Transport; using Operations; - using Operations.BodyStorage; using Particular.ServiceControl; using Retrying; using ServiceBus.Management.Infrastructure.Settings; @@ -76,9 +75,6 @@ public override void Configure(Settings settings, IHostBuilder hostBuilder) //Failed messages collection.AddHostedService(); - //Body storage - collection.AddSingleton(); - //Health checks collection.AddCustomCheck(); collection.AddCustomCheck(); From 427d4606e023533deb19bde3d3af1beaf631d211 Mon Sep 17 00:00:00 2001 From: David Boike Date: Tue, 26 Sep 2023 12:36:37 -0500 Subject: [PATCH 72/94] Fix error ingestion for Raven5 to use empty MessageBodies/X documents with attachment (without cleanup) --- .../RavenAttachmentsBodyStorage.cs | 2 +- .../UnitOfWork/BodyStorageEnricher.cs | 2 +- .../ErrorMessagesDataStore.cs | 4 ++- .../RavenAttachmentsBodyStorage.cs | 4 +-- .../UnitOfWork/RavenDbIngestionUnitOfWork.cs | 9 ++--- ...avenDbRecoverabilityIngestionUnitOfWork.cs | 36 +++++++++++++++++++ 6 files changed, 48 insertions(+), 9 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs b/src/ServiceControl.Audit.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs index c5b98e63cb..b8d2f15f58 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs @@ -29,7 +29,7 @@ public async Task TryFetch(string bodyId) { using (var session = sessionProvider.OpenSession()) { - var result = await session.Advanced.Attachments.GetAsync($"message/{bodyId}", "body"); + var result = await session.Advanced.Attachments.GetAsync($"MessageBodies/{bodyId}", "body"); if (result == null) { diff --git a/src/ServiceControl.Persistence.RavenDb/UnitOfWork/BodyStorageEnricher.cs b/src/ServiceControl.Persistence.RavenDb/UnitOfWork/BodyStorageEnricher.cs index df7744bc71..8ebcdb0cec 100644 --- a/src/ServiceControl.Persistence.RavenDb/UnitOfWork/BodyStorageEnricher.cs +++ b/src/ServiceControl.Persistence.RavenDb/UnitOfWork/BodyStorageEnricher.cs @@ -12,7 +12,7 @@ class BodyStorageEnricher { - public BodyStorageEnricher(IBodyStorage bodyStorage, PersistenceSettings settings) + public BodyStorageEnricher(IBodyStorage bodyStorage, RavenDBPersisterSettings settings) { this.settings = settings; this.bodyStorage = bodyStorage; diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index ef170248a6..5434831614 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -516,11 +516,13 @@ public async Task>> GetGroup(string groupId, public async Task MarkMessageAsResolved(string failedMessageId) { + var documentId = FailedMessageIdGenerator.MakeDocumentId(failedMessageId); + using (var session = documentStore.OpenAsyncSession()) { session.Advanced.UseOptimisticConcurrency = true; - var failedMessage = await session.LoadAsync(failedMessageId); + var failedMessage = await session.LoadAsync(documentId); if (failedMessage == null) { diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs b/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs index de3569a160..18355e127b 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs @@ -33,11 +33,11 @@ public async Task Store(string messageId, string contentType, int bodySize, Stre public async Task TryFetch(string messageId) { - //var messageId = MessageBodyIdGenerator.MakeDocumentId(bodyId); // TODO: Not needed? Not used by audit + var documentId = MessageBodyIdGenerator.MakeDocumentId(messageId); using var session = documentStore.OpenAsyncSession(); - var result = await session.Advanced.Attachments.GetAsync(messageId, AttachmentName); + var result = await session.Advanced.Attachments.GetAsync(documentId, AttachmentName); if (result == null) { diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs index 2a3ca06465..c583807b60 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs @@ -2,24 +2,25 @@ { using System.Collections.Concurrent; using System.Threading.Tasks; - using ServiceControl.Persistence.UnitOfWork; using Raven.Client.Documents; using Raven.Client.Documents.Commands.Batches; + using ServiceControl.Persistence.UnitOfWork; class RavenDbIngestionUnitOfWork : IngestionUnitOfWorkBase { readonly IDocumentStore store; - readonly ConcurrentBag commands; + // Must be ordered - can't put attachments until after document exists + readonly ConcurrentQueue commands; public RavenDbIngestionUnitOfWork(IDocumentStore store) { this.store = store; - commands = new ConcurrentBag(); + commands = new ConcurrentQueue(); Monitoring = new RavenDbMonitoringIngestionUnitOfWork(this); Recoverability = new RavenDbRecoverabilityIngestionUnitOfWork(this); } - internal void AddCommand(ICommandData command) => commands.Add(command); + internal void AddCommand(ICommandData command) => commands.Enqueue(command); public override async Task Complete() { diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs index 9d2c95a9ae..c0cb85428a 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs @@ -2,13 +2,16 @@ { using System.Collections.Generic; using System.Threading.Tasks; + using NServiceBus; using NServiceBus.Transport; using Raven.Client.Documents.Commands.Batches; using Raven.Client.Documents.Operations; + using ServiceControl.Infrastructure; using ServiceControl.MessageFailures; using ServiceControl.Persistence.Infrastructure; using ServiceControl.Persistence.UnitOfWork; using ServiceControl.Recoverability; + using Sparrow.Json.Parsing; class RavenDbRecoverabilityIngestionUnitOfWork : IRecoverabilityIngestionUnitOfWork { @@ -25,8 +28,16 @@ public Task RecordFailedProcessingAttempt( List groups) { var uniqueMessageId = context.Headers.UniqueId(); + var bodyId = processingAttempt.Headers.MessageId(); + var contentType = GetContentType(context.Headers, "text/xml"); + processingAttempt.MessageMetadata.Add("ContentType", contentType); + processingAttempt.MessageMetadata.Add("BodyUrl", $"/messages/{bodyId}/body"); + var storeMessageCmd = CreateFailedMessagesPatchCommand(uniqueMessageId, processingAttempt, groups); parentUnitOfWork.AddCommand(storeMessageCmd); + + AddStoreBodyCommands(context, contentType); + return Task.CompletedTask; } @@ -109,6 +120,31 @@ ICommandData CreateFailedMessagesPatchCommand(string uniqueMessageId, FailedMess }); } + void AddStoreBodyCommands(MessageContext context, string contentType) + { + var messageId = context.Headers.MessageId(); + var documentId = $"MessageBodies/{messageId}"; + + var emptyDoc = new DynamicJsonValue(); + var putOwnerDocumentCmd = new PutCommandData(documentId, null, emptyDoc); + + var stream = Memory.Manager.GetStream(context.Body); + var putAttachmentCmd = new PutAttachmentCommandData(documentId, "body", stream, contentType, changeVector: null); + + parentUnitOfWork.AddCommand(putOwnerDocumentCmd); + parentUnitOfWork.AddCommand(putAttachmentCmd); + } + + static string GetContentType(IReadOnlyDictionary headers, string defaultContentType) + { + if (!headers.TryGetValue(Headers.ContentType, out var contentType)) + { + contentType = defaultContentType; + } + + return contentType; + } + static int MaxProcessingAttempts = 10; } } \ No newline at end of file From e46caf6b9674a5994441f138690d41f02cec9981 Mon Sep 17 00:00:00 2001 From: David Boike Date: Tue, 26 Sep 2023 14:18:30 -0500 Subject: [PATCH 73/94] Allow StartupModeTests to use shared DB --- .../AcceptanceTestStorageConfiguration.cs | 23 ++---------- .../SharedDatabaseSetup.cs | 36 +++++++++++++++++-- .../StartupModeTests.cs | 16 ++++++++- .../TestSupport/AcceptanceTest.cs | 4 +-- 4 files changed, 54 insertions(+), 25 deletions(-) diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs index f06c6a29bb..84e58cc999 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs @@ -1,26 +1,16 @@ namespace ServiceControl.AcceptanceTests { - using System; using System.Threading.Tasks; using Persistence.RavenDb; - using Raven.Client.ServerWide.Operations; using ServiceBus.Management.Infrastructure.Settings; class AcceptanceTestStorageConfiguration { - readonly string databaseName = Guid.NewGuid().ToString("n"); + readonly DatabaseLease databaseLease = SharedDatabaseSetup.LeaseDatabase(); public string PersistenceType { get; protected set; } - public void CustomizeSettings(Settings settings) - { - settings.PersisterSpecificSettings = new RavenDBPersisterSettings - { - ErrorRetentionPeriod = TimeSpan.FromDays(10), - ConnectionString = SharedDatabaseSetup.SharedInstance.ServerUrl, - DatabaseName = databaseName - }; - } + public void CustomizeSettings(Settings settings) => databaseLease.CustomizeSettings(settings); public Task Configure() { @@ -29,13 +19,6 @@ public Task Configure() return Task.CompletedTask; } - public async Task Cleanup() - { - var documentStore = await SharedDatabaseSetup.SharedInstance.Connect(); - - // Comment this out temporarily to be able to inspect a database after the test has completed - var deleteDatabasesOperation = new DeleteDatabasesOperation(new DeleteDatabasesOperation.Parameters { DatabaseNames = new[] { databaseName }, HardDelete = true }); - await documentStore.Maintenance.Server.SendAsync(deleteDatabasesOperation); - } + public ValueTask Cleanup() => databaseLease.DisposeAsync(); } } diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/SharedDatabaseSetup.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/SharedDatabaseSetup.cs index 6723857f8f..ba89c89e29 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB5/SharedDatabaseSetup.cs +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/SharedDatabaseSetup.cs @@ -1,10 +1,13 @@ -using System.Threading; +using System; +using System.Threading; using System.Threading.Tasks; using NUnit.Framework; +using Raven.Client.ServerWide.Operations; +using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.Persistence.RavenDb5; [SetUpFixture] -public class SharedDatabaseSetup +public static class SharedDatabaseSetup { public static EmbeddedDatabase SharedInstance { get; private set; } @@ -20,4 +23,33 @@ public static async Task SetupSharedEmbeddedServer() [OneTimeTearDown] public static void TearDown() => SharedInstance.Dispose(); + + public static DatabaseLease LeaseDatabase() + { + return new DatabaseLease(); + } +} + +public class DatabaseLease : IAsyncDisposable +{ + public string DatabaseName { get; } = Guid.NewGuid().ToString("n"); + + public void CustomizeSettings(Settings settings) + { + settings.PersisterSpecificSettings = new RavenDBPersisterSettings + { + ErrorRetentionPeriod = TimeSpan.FromDays(10), + ConnectionString = SharedDatabaseSetup.SharedInstance.ServerUrl, + DatabaseName = DatabaseName + }; + } + + public async ValueTask DisposeAsync() + { + var documentStore = await SharedDatabaseSetup.SharedInstance.Connect(); + + // Comment this out temporarily to be able to inspect a database after the test has completed + var deleteDatabasesOperation = new DeleteDatabasesOperation(new DeleteDatabasesOperation.Parameters { DatabaseNames = new[] { DatabaseName }, HardDelete = true }); + await documentStore.Maintenance.Server.SendAsync(deleteDatabasesOperation); + } } \ No newline at end of file diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/StartupModeTests.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/StartupModeTests.cs index 4aa678d36b..46551a478d 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB5/StartupModeTests.cs +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/StartupModeTests.cs @@ -11,14 +11,19 @@ using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.AcceptanceTesting.InfrastructureConfig; + class StartupModeTests : AcceptanceTest { Settings settings; + DatabaseLease database; [SetUp] public void InitializeSettings() { var transportIntegration = new ConfigureEndpointLearningTransport(); + + database = SharedDatabaseSetup.LeaseDatabase(); + settings = new Settings( forwardErrorMessages: false, errorRetentionPeriod: TimeSpan.FromDays(1), @@ -27,12 +32,21 @@ public void InitializeSettings() PersisterSpecificSettings = new RavenDBPersisterSettings { ErrorRetentionPeriod = TimeSpan.FromDays(1), + ConnectionString = SharedDatabaseSetup.SharedInstance.ServerUrl, + DatabaseName = database.DatabaseName }, TransportType = transportIntegration.TypeName, - TransportConnectionString = transportIntegration.ConnectionString + TransportConnectionString = transportIntegration.ConnectionString, + }; } + [TearDown] + public async Task Cleanup() + { + await database.DisposeAsync(); + } + [Test] public async Task CanRunMaintenanceMode() { diff --git a/src/ServiceControl.AcceptanceTests/TestSupport/AcceptanceTest.cs b/src/ServiceControl.AcceptanceTests/TestSupport/AcceptanceTest.cs index 2410c9ab75..78e7887eb5 100644 --- a/src/ServiceControl.AcceptanceTests/TestSupport/AcceptanceTest.cs +++ b/src/ServiceControl.AcceptanceTests/TestSupport/AcceptanceTest.cs @@ -85,14 +85,14 @@ public async Task Setup() } [TearDown] - public Task Teardown() + public async Task Teardown() { TransportIntegration = null; Trace.Flush(); Trace.Close(); Trace.Listeners.Remove(textWriterTraceListener); - return StorageConfiguration.Cleanup(); + await StorageConfiguration.Cleanup(); } #pragma warning disable IDE0060 // Remove unused parameter From d9de3efe116588b22ea0bd8f61230e04e734db15 Mon Sep 17 00:00:00 2001 From: David Boike Date: Tue, 26 Sep 2023 15:01:28 -0500 Subject: [PATCH 74/94] Enable acceptance tests to run in parallel --- .../ConfigureEndpointLearningTransport.cs | 3 +-- .../AcceptanceTestStorageConfiguration.cs | 6 ++++- .../SharedDatabaseSetup.cs | 26 +++++++++++++++++++ .../TestSupport/AcceptanceTest.cs | 1 + 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/ServiceControl.AcceptanceTesting/InfrastructureConfig/ConfigureEndpointLearningTransport.cs b/src/ServiceControl.AcceptanceTesting/InfrastructureConfig/ConfigureEndpointLearningTransport.cs index 9c174981d0..943ec5f5e5 100644 --- a/src/ServiceControl.AcceptanceTesting/InfrastructureConfig/ConfigureEndpointLearningTransport.cs +++ b/src/ServiceControl.AcceptanceTesting/InfrastructureConfig/ConfigureEndpointLearningTransport.cs @@ -11,8 +11,7 @@ public class ConfigureEndpointLearningTransport : ITransportIntegration { public ConfigureEndpointLearningTransport() { - var relativePath = Path.Combine(TestContext.CurrentContext.TestDirectory, @"..", "..", "..", ".transport"); - ConnectionString = Path.GetFullPath(relativePath); + ConnectionString = Path.Combine(Path.GetTempPath(), "ServiceControlTests", "TestTransport", TestContext.CurrentContext.Test.ID); } public string ConnectionString { get; set; } diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs index 84e58cc999..9dc90a5edd 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs @@ -10,7 +10,11 @@ class AcceptanceTestStorageConfiguration public string PersistenceType { get; protected set; } - public void CustomizeSettings(Settings settings) => databaseLease.CustomizeSettings(settings); + public void CustomizeSettings(Settings settings) + { + databaseLease.CustomizeSettings(settings); + settings.Port = databaseLease.LeasePort(); + } public Task Configure() { diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/SharedDatabaseSetup.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/SharedDatabaseSetup.cs index ba89c89e29..5604ad03e1 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB5/SharedDatabaseSetup.cs +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/SharedDatabaseSetup.cs @@ -1,10 +1,13 @@ using System; +using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; using NUnit.Framework; using Raven.Client.ServerWide.Operations; using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.Persistence.RavenDb5; +using TestHelper; [SetUpFixture] public static class SharedDatabaseSetup @@ -33,6 +36,8 @@ public static DatabaseLease LeaseDatabase() public class DatabaseLease : IAsyncDisposable { public string DatabaseName { get; } = Guid.NewGuid().ToString("n"); + static HashSet allPortsInUse = new HashSet(); + List usedPorts = new List(); public void CustomizeSettings(Settings settings) { @@ -44,8 +49,29 @@ public void CustomizeSettings(Settings settings) }; } + public int LeasePort() + { + lock (allPortsInUse) + { + var start = allPortsInUse.Any() ? allPortsInUse.Max() + 1 : 33334; + var port = PortUtility.FindAvailablePort(start); + allPortsInUse.Add(port); + usedPorts.Add(port); + TestContext.Out.WriteLine($"Port leased: {port}"); + return port; + } + } + public async ValueTask DisposeAsync() { + lock (allPortsInUse) + { + foreach (var port in usedPorts) + { + allPortsInUse.Remove(port); + } + } + var documentStore = await SharedDatabaseSetup.SharedInstance.Connect(); // Comment this out temporarily to be able to inspect a database after the test has completed diff --git a/src/ServiceControl.AcceptanceTests/TestSupport/AcceptanceTest.cs b/src/ServiceControl.AcceptanceTests/TestSupport/AcceptanceTest.cs index 78e7887eb5..c57e235d56 100644 --- a/src/ServiceControl.AcceptanceTests/TestSupport/AcceptanceTest.cs +++ b/src/ServiceControl.AcceptanceTests/TestSupport/AcceptanceTest.cs @@ -21,6 +21,7 @@ namespace ServiceControl.AcceptanceTests using TestSupport; [TestFixture] + [Parallelizable(ParallelScope.All)] abstract class AcceptanceTest : NServiceBusAcceptanceTest, IAcceptanceTestInfrastructureProvider { protected AcceptanceTest() From f1a4485d168833f59f44ac0d24e9120344c1db6f Mon Sep 17 00:00:00 2001 From: David Boike Date: Tue, 26 Sep 2023 15:07:37 -0500 Subject: [PATCH 75/94] Set FullText searching related temp setting to true to get LOH-related test to pass --- src/ServiceControl.Persistence/PersistenceSettings.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ServiceControl.Persistence/PersistenceSettings.cs b/src/ServiceControl.Persistence/PersistenceSettings.cs index d3d39b8fbd..5e5cfb999e 100644 --- a/src/ServiceControl.Persistence/PersistenceSettings.cs +++ b/src/ServiceControl.Persistence/PersistenceSettings.cs @@ -11,6 +11,6 @@ public abstract class PersistenceSettings // TODO: This isn't set yet, but body storage, index creation will all need to know this - public bool EnableFullTextSearchOnBodies { get; set; } + public bool EnableFullTextSearchOnBodies { get; set; } = true; } } \ No newline at end of file From 2b237c389550ee126a0e7914068f495becefef85 Mon Sep 17 00:00:00 2001 From: David Boike Date: Tue, 26 Sep 2023 17:18:44 -0500 Subject: [PATCH 76/94] Fix failed message patching --- ...avenDbRecoverabilityIngestionUnitOfWork.cs | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs index c0cb85428a..6aa91e5c54 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs @@ -60,8 +60,10 @@ ICommandData CreateFailedMessagesPatchCommand(string uniqueMessageId, FailedMess { var documentId = FailedMessageIdGenerator.MakeDocumentId(uniqueMessageId); - //HINT: RavenDB 3.5 is using Lodash v4.13.1 to provide javascript utility functions - // https://ravendb.net/docs/article-page/3.5/csharp/client-api/commands/patches/how-to-use-javascript-to-patch-your-documents#methods-objects-and-variables + const string ProcessingAttempts = nameof(FailedMessage.ProcessingAttempts); + const string AttemptedAt = nameof(FailedMessage.ProcessingAttempt.AttemptedAt); + + //HINT: RavenDB 4.2 removed Lodash utility functions, but supports ECMAScript 5.1 and some 6.0 features like arrow functions and array primitive functions return new PatchCommandData(documentId, null, new PatchRequest { Script = $@"this.{nameof(FailedMessage.Status)} = args.status; @@ -70,27 +72,21 @@ ICommandData CreateFailedMessagesPatchCommand(string uniqueMessageId, FailedMess var newAttempts = this.{nameof(FailedMessage.ProcessingAttempts)}; //De-duplicate attempts by AttemptedAt value + var duplicateIndex = this.{ProcessingAttempts}.findIndex(a => a.{AttemptedAt} === attempt.{AttemptedAt}); - var duplicateIndex = _.findIndex(this.{nameof(FailedMessage.ProcessingAttempts)}, function(a){{ - return a.{nameof(FailedMessage.ProcessingAttempt.AttemptedAt)} === attempt.{nameof(FailedMessage.ProcessingAttempt.AttemptedAt)}; - }}); - - if(duplicateIndex === -1){{ - newAttempts = _.union(newAttempts, [args.attempt]); + if(duplicateIndex < 0){{ + newAttempts.push(args.attempt); }} - //Trim to the latest MaxProcessingAttempts - - newAttempts = _.sortBy(newAttempts, function(a) {{ - return a.{nameof(FailedMessage.ProcessingAttempt.AttemptedAt)}; - }}); + //Trim to the latest MaxProcessingAttempts + newAttempts.sort((a, b) => a.{AttemptedAt} > b.{AttemptedAt} ? 1 : -1); if(newAttempts.length > {MaxProcessingAttempts}) {{ - newAttempts = _.slice(newAttempts, newAttempts.length - {MaxProcessingAttempts}, newAttempts.length); + newAttempts = newAttempts.slice(newAttempts.length - {MaxProcessingAttempts}, newAttempts.length); }} - this.{nameof(FailedMessage.ProcessingAttempts)} = newAttempts; + this.{ProcessingAttempts} = newAttempts; ", Values = new Dictionary { From e622cdd7307b86a1aa526c7689071c4eb1c1892c Mon Sep 17 00:00:00 2001 From: David Boike Date: Wed, 27 Sep 2023 09:59:01 -0500 Subject: [PATCH 77/94] Handle full-text indexing of failed messages --- ...failed_message_searched_by_body_content.cs | 4 +-- .../When_a_message_is_retried.cs | 2 +- .../RavenDbPersistenceConfiguration.cs | 1 + .../UnitOfWork/BodyStorageEnricher.cs | 1 - .../RavenDbPersistenceConfiguration.cs | 3 +- .../UnitOfWork/RavenDbIngestionUnitOfWork.cs | 4 +-- .../RavenDbIngestionUnitOfWorkFactory.cs | 8 +++-- ...avenDbRecoverabilityIngestionUnitOfWork.cs | 31 +++++++++++++++++-- .../PersistenceSettings.cs | 2 -- src/ServiceControl/Bootstrapper.cs | 2 +- .../Infrastructure/Settings/Settings.cs | 3 -- 11 files changed, 43 insertions(+), 18 deletions(-) diff --git a/src/ServiceControl.AcceptanceTests/Recoverability/MessageFailures/When_failed_message_searched_by_body_content.cs b/src/ServiceControl.AcceptanceTests/Recoverability/MessageFailures/When_failed_message_searched_by_body_content.cs index 1931feb419..476e27911c 100644 --- a/src/ServiceControl.AcceptanceTests/Recoverability/MessageFailures/When_failed_message_searched_by_body_content.cs +++ b/src/ServiceControl.AcceptanceTests/Recoverability/MessageFailures/When_failed_message_searched_by_body_content.cs @@ -16,7 +16,7 @@ class When_failed_message_searched_by_body_content : AcceptanceTest public async Task Should_be_found_when_fulltext_search_enabled() { // setting it even if it is the default - SetSettings = settings => settings.EnableFullTextSearchOnBodies = true; + SetSettings = settings => settings.PersisterSpecificSettings.EnableFullTextSearchOnBodies = true; var searchString = "forty-two"; @@ -48,7 +48,7 @@ public async Task Should_be_found_when_fulltext_search_enabled() [Test] public async Task Should_not_be_found_when_fulltext_search_disabled() { - SetSettings = settings => settings.EnableFullTextSearchOnBodies = false; + SetSettings = settings => settings.PersisterSpecificSettings.EnableFullTextSearchOnBodies = false; var searchString = "forty-two"; diff --git a/src/ServiceControl.AcceptanceTests/Recoverability/When_a_message_is_retried.cs b/src/ServiceControl.AcceptanceTests/Recoverability/When_a_message_is_retried.cs index 1ce1e9e767..49ef1139d0 100644 --- a/src/ServiceControl.AcceptanceTests/Recoverability/When_a_message_is_retried.cs +++ b/src/ServiceControl.AcceptanceTests/Recoverability/When_a_message_is_retried.cs @@ -46,7 +46,7 @@ public async Task Should_work_with_various_body_size(bool largeMessageBodies, bo { SetSettings = settings => { - settings.EnableFullTextSearchOnBodies = enableFullTextSearch; + settings.PersisterSpecificSettings.EnableFullTextSearchOnBodies = enableFullTextSearch; }; string content = $"{{\"Content\":\"{(largeMessageBodies ? new string('a', 86 * 1024) : "Small")}\"}}"; diff --git a/src/ServiceControl.Persistence.RavenDb/RavenDbPersistenceConfiguration.cs b/src/ServiceControl.Persistence.RavenDb/RavenDbPersistenceConfiguration.cs index 640e732654..896c534f81 100644 --- a/src/ServiceControl.Persistence.RavenDb/RavenDbPersistenceConfiguration.cs +++ b/src/ServiceControl.Persistence.RavenDb/RavenDbPersistenceConfiguration.cs @@ -59,6 +59,7 @@ T GetSetting(string key, T defaultValue) AuditRetentionPeriod = GetSetting(AuditRetentionPeriodKey, TimeSpan.Zero), ExternalIntegrationsDispatchingBatchSize = GetSetting(ExternalIntegrationsDispatchingBatchSizeKey, 100), MaintenanceMode = GetSetting(MaintenanceModeKey, false), + EnableFullTextSearchOnBodies = GetSetting("EnableFullTextSearchOnBodies", true) }; CheckFreeDiskSpace.Validate(settings); diff --git a/src/ServiceControl.Persistence.RavenDb/UnitOfWork/BodyStorageEnricher.cs b/src/ServiceControl.Persistence.RavenDb/UnitOfWork/BodyStorageEnricher.cs index 8ebcdb0cec..3480857f35 100644 --- a/src/ServiceControl.Persistence.RavenDb/UnitOfWork/BodyStorageEnricher.cs +++ b/src/ServiceControl.Persistence.RavenDb/UnitOfWork/BodyStorageEnricher.cs @@ -59,7 +59,6 @@ async ValueTask StoreBody(byte[] body, ProcessingAttempt processingAttempt, int { try { - // TODO: Make sure this is set, I just added the property to PersistenceSettings because I didn't have access to the root Settings if (settings.EnableFullTextSearchOnBodies) { processingAttempt.MessageMetadata.Add("Body", enc.GetString(body)); diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs index 98c20c7627..9e414f9299 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenDbPersistenceConfiguration.cs @@ -56,7 +56,8 @@ T GetSetting(string key, T defaultValue) ExternalIntegrationsDispatchingBatchSize = GetSetting(ExternalIntegrationsDispatchingBatchSizeKey, 100), MaintenanceMode = GetSetting(MaintenanceModeKey, false), LogPath = GetRequiredSetting(RavenBootstrapper.LogsPathKey), - LogsMode = GetSetting(RavenBootstrapper.LogsModeKey, RavenDBPersisterSettings.LogsModeDefault) + LogsMode = GetSetting(RavenBootstrapper.LogsModeKey, RavenDBPersisterSettings.LogsModeDefault), + EnableFullTextSearchOnBodies = GetSetting("EnableFullTextSearchOnBodies", true) }; CheckFreeDiskSpace.Validate(settings); diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs index c583807b60..7c93305548 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWork.cs @@ -12,12 +12,12 @@ class RavenDbIngestionUnitOfWork : IngestionUnitOfWorkBase // Must be ordered - can't put attachments until after document exists readonly ConcurrentQueue commands; - public RavenDbIngestionUnitOfWork(IDocumentStore store) + public RavenDbIngestionUnitOfWork(IDocumentStore store, RavenDBPersisterSettings settings) { this.store = store; commands = new ConcurrentQueue(); Monitoring = new RavenDbMonitoringIngestionUnitOfWork(this); - Recoverability = new RavenDbRecoverabilityIngestionUnitOfWork(this); + Recoverability = new RavenDbRecoverabilityIngestionUnitOfWork(this, settings.EnableFullTextSearchOnBodies); } internal void AddCommand(ICommandData command) => commands.Enqueue(command); diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWorkFactory.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWorkFactory.cs index 079ff263cd..1f5b6fceba 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWorkFactory.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbIngestionUnitOfWorkFactory.cs @@ -1,22 +1,24 @@ namespace ServiceControl.Persistence.RavenDb { using System.Threading.Tasks; - using ServiceControl.Persistence.UnitOfWork; using Raven.Client.Documents; + using ServiceControl.Persistence.UnitOfWork; class RavenDbIngestionUnitOfWorkFactory : IIngestionUnitOfWorkFactory { readonly IDocumentStore store; readonly MinimumRequiredStorageState customCheckState; + readonly RavenDBPersisterSettings settings; - public RavenDbIngestionUnitOfWorkFactory(IDocumentStore store, MinimumRequiredStorageState customCheckState) + public RavenDbIngestionUnitOfWorkFactory(IDocumentStore store, MinimumRequiredStorageState customCheckState, RavenDBPersisterSettings settings) { this.store = store; this.customCheckState = customCheckState; + this.settings = settings; } public ValueTask StartNew() - => new ValueTask(new RavenDbIngestionUnitOfWork(store)); + => new ValueTask(new RavenDbIngestionUnitOfWork(store, settings)); public bool CanIngestMore() { diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs index 6aa91e5c54..7b7935f7c8 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs @@ -1,6 +1,8 @@ namespace ServiceControl.Persistence.RavenDb { + using System; using System.Collections.Generic; + using System.Text; using System.Threading.Tasks; using NServiceBus; using NServiceBus.Transport; @@ -15,11 +17,13 @@ class RavenDbRecoverabilityIngestionUnitOfWork : IRecoverabilityIngestionUnitOfWork { - RavenDbIngestionUnitOfWork parentUnitOfWork; + readonly RavenDbIngestionUnitOfWork parentUnitOfWork; + readonly bool doFullTextIndexing; - public RavenDbRecoverabilityIngestionUnitOfWork(RavenDbIngestionUnitOfWork parentUnitOfWork) + public RavenDbRecoverabilityIngestionUnitOfWork(RavenDbIngestionUnitOfWork parentUnitOfWork, bool doFullTextIndexing) { this.parentUnitOfWork = parentUnitOfWork; + this.doFullTextIndexing = doFullTextIndexing; } public Task RecordFailedProcessingAttempt( @@ -33,6 +37,25 @@ public Task RecordFailedProcessingAttempt( processingAttempt.MessageMetadata.Add("ContentType", contentType); processingAttempt.MessageMetadata.Add("BodyUrl", $"/messages/{bodyId}/body"); + if (doFullTextIndexing) + { + var bodySize = context.Body?.Length ?? 0; + var avoidsLargeObjectHeap = bodySize < LargeObjectHeapThreshold; + var isBinary = processingAttempt.Headers.IsBinary(); + if (avoidsLargeObjectHeap && !isBinary) + { + try + { + var bodyString = utf8.GetString(context.Body); + processingAttempt.MessageMetadata.Add("Body", bodyString); + } + catch (ArgumentException) + { + // If it won't decode to text, don't index it + } + } + } + var storeMessageCmd = CreateFailedMessagesPatchCommand(uniqueMessageId, processingAttempt, groups); parentUnitOfWork.AddCommand(storeMessageCmd); @@ -142,5 +165,9 @@ static string GetContentType(IReadOnlyDictionary headers, string } static int MaxProcessingAttempts = 10; + // large object heap starts above 85000 bytes and not above 85 KB! + internal const int LargeObjectHeapThreshold = 85_000; + static readonly Encoding utf8 = new UTF8Encoding(true, true); + } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence/PersistenceSettings.cs b/src/ServiceControl.Persistence/PersistenceSettings.cs index 5e5cfb999e..94014308b0 100644 --- a/src/ServiceControl.Persistence/PersistenceSettings.cs +++ b/src/ServiceControl.Persistence/PersistenceSettings.cs @@ -9,8 +9,6 @@ public abstract class PersistenceSettings //HINT: This needs to be here so that ServerControl instance can add an instance specific metadata to tweak the DatabasePath value public string DatabasePath { get; set; } - - // TODO: This isn't set yet, but body storage, index creation will all need to know this public bool EnableFullTextSearchOnBodies { get; set; } = true; } } \ No newline at end of file diff --git a/src/ServiceControl/Bootstrapper.cs b/src/ServiceControl/Bootstrapper.cs index ef32019750..7296120b12 100644 --- a/src/ServiceControl/Bootstrapper.cs +++ b/src/ServiceControl/Bootstrapper.cs @@ -175,7 +175,7 @@ Audit Retention Period (optional): {settings.AuditRetentionPeriod} settings.RetryHistoryDepth, settings.PersisterSpecificSettings, settings.SkipQueueCreation, - settings.EnableFullTextSearchOnBodies, + settings.PersisterSpecificSettings.EnableFullTextSearchOnBodies, settings.TransportType, settings.AllowMessageEditing, }, diff --git a/src/ServiceControl/Infrastructure/Settings/Settings.cs b/src/ServiceControl/Infrastructure/Settings/Settings.cs index b2557ebe03..22368d557c 100644 --- a/src/ServiceControl/Infrastructure/Settings/Settings.cs +++ b/src/ServiceControl/Infrastructure/Settings/Settings.cs @@ -56,7 +56,6 @@ public Settings( DataSpaceRemainingThreshold = GetDataSpaceRemainingThreshold(); TimeToRestartErrorIngestionAfterFailure = GetTimeToRestartErrorIngestionAfterFailure(); DisableExternalIntegrationsPublishing = SettingsReader.Read("DisableExternalIntegrationsPublishing", false); - EnableFullTextSearchOnBodies = SettingsReader.Read("EnableFullTextSearchOnBodies", true); } public string NotificationsFilter { get; set; } @@ -155,8 +154,6 @@ public TimeSpan HeartbeatGracePeriod public int DataSpaceRemainingThreshold { get; set; } - public bool EnableFullTextSearchOnBodies { get; set; } - public bool DisableHealthChecks { get; set; } public bool ExposeApi { get; set; } = true; From a4b5d6e4fcdbfc0460b203464739033e041608b9 Mon Sep 17 00:00:00 2001 From: David Boike Date: Wed, 27 Sep 2023 12:08:10 -0500 Subject: [PATCH 78/94] Clean learning transport directory once before all acceptance tests --- .../CleanTransportDirectoryOnStartup.cs | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 src/ServiceControl.AcceptanceTests/TestSupport/CleanTransportDirectoryOnStartup.cs diff --git a/src/ServiceControl.AcceptanceTests/TestSupport/CleanTransportDirectoryOnStartup.cs b/src/ServiceControl.AcceptanceTests/TestSupport/CleanTransportDirectoryOnStartup.cs new file mode 100644 index 0000000000..e4cef4dc3b --- /dev/null +++ b/src/ServiceControl.AcceptanceTests/TestSupport/CleanTransportDirectoryOnStartup.cs @@ -0,0 +1,26 @@ +using System.IO; +using NUnit.Framework; + +[SetUpFixture] +public class CleanTransportDirectoryOnStartup +{ + [OneTimeSetUp] + public void RemoveTransportDirectories() + { + var path = Path.Combine(Path.GetTempPath(), "ServiceControlTests", "TestTransport"); + + if (Directory.Exists(path)) + { + foreach (var subdirectory in Directory.GetDirectories(path)) + { + try + { + Directory.Delete(subdirectory, true); + } + catch (DirectoryNotFoundException) + { + } + } + } + } +} \ No newline at end of file From 874f1da0885b8a8c0f463db3e743f9b84489a647 Mon Sep 17 00:00:00 2001 From: David Boike Date: Wed, 27 Sep 2023 12:23:06 -0500 Subject: [PATCH 79/94] Standardize use of MessageBodyIdGenerator --- .../MessageBodyIdGenerator.cs | 2 +- .../RavenAttachmentsBodyStorage.cs | 25 +++++++++++-------- ...avenDbRecoverabilityIngestionUnitOfWork.cs | 2 +- .../RavenAttachmentsBodyStorageTests.cs | 8 ------ 4 files changed, 17 insertions(+), 20 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/MessageBodyIdGenerator.cs b/src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/MessageBodyIdGenerator.cs index 9891ffdc14..1fc5da71b1 100644 --- a/src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/MessageBodyIdGenerator.cs +++ b/src/ServiceControl.Persistence.RavenDb5/DocumentIdGenerators/MessageBodyIdGenerator.cs @@ -1,6 +1,6 @@ static class MessageBodyIdGenerator { - const string CollectionName = "messagebodies"; + const string CollectionName = "MessageBodies"; public static string MakeDocumentId(string messageUniqueId) { diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs b/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs index 18355e127b..6b35cf1272 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs @@ -1,9 +1,13 @@ namespace ServiceControl.Operations.BodyStorage.RavenAttachments { using System.IO; + using System.Runtime.Remoting.Contexts; using System.Threading.Tasks; using Raven.Client.Documents; + using Raven.Client.Documents.Commands.Batches; using Raven.Client.Documents.Operations.Attachments; + using ServiceControl.Infrastructure; + using Sparrow.Json.Parsing; class RavenAttachmentsBodyStorage : IBodyStorage { @@ -15,20 +19,21 @@ public RavenAttachmentsBodyStorage(IDocumentStore documentStore) this.documentStore = documentStore; } + // TODO: This method is only used in tests and not by ServiceControl itself! But in the Raven3.5 persister, it IS used! + // It should probably be removed and tests should use the RavenDbRecoverabilityIngestionUnitOfWork public async Task Store(string messageId, string contentType, int bodySize, Stream bodyStream) { - // var id = MessageBodyIdGenerator.MakeDocumentId(messageId); // TODO: Not needed? Not used by audit + var documentId = MessageBodyIdGenerator.MakeDocumentId(messageId); - using var session = documentStore.OpenAsyncSession(); + var emptyDoc = new DynamicJsonValue(); + var putOwnerDocumentCmd = new PutCommandData(documentId, null, emptyDoc); - // Following is possible to but not documented in the Raven docs. - //session.Advanced.Attachments.Store(messageId,"body",bodyStream,contentType); - // https://ravendb.net/docs/article-page/5.4/csharp/client-api/operations/attachments/get-attachment - _ = await documentStore.Operations.SendAsync( - new PutAttachmentOperation(messageId, - AttachmentName, - bodyStream, - contentType)); + var stream = bodyStream; + var putAttachmentCmd = new PutAttachmentCommandData(documentId, "body", stream, contentType, changeVector: null); + + using var session = documentStore.OpenAsyncSession(); + session.Advanced.Defer(new ICommandData[] { putOwnerDocumentCmd, putAttachmentCmd }); + await session.SaveChangesAsync(); } public async Task TryFetch(string messageId) diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs index 7b7935f7c8..96e02d0073 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs @@ -142,7 +142,7 @@ ICommandData CreateFailedMessagesPatchCommand(string uniqueMessageId, FailedMess void AddStoreBodyCommands(MessageContext context, string contentType) { var messageId = context.Headers.MessageId(); - var documentId = $"MessageBodies/{messageId}"; + var documentId = MessageBodyIdGenerator.MakeDocumentId(messageId); var emptyDoc = new DynamicJsonValue(); var putOwnerDocumentCmd = new PutCommandData(documentId, null, emptyDoc); diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/BodyStorage/RavenAttachmentsBodyStorageTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/BodyStorage/RavenAttachmentsBodyStorageTests.cs index b1584e1553..a087ef29ef 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/BodyStorage/RavenAttachmentsBodyStorageTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/BodyStorage/RavenAttachmentsBodyStorageTests.cs @@ -3,9 +3,7 @@ using System; using System.IO; using System.Threading.Tasks; - using MessageFailures; using NUnit.Framework; - using Raven.Client.Documents; [TestFixture] sealed class RavenAttachmentsBodyStorageTests : PersistenceTestBase @@ -17,12 +15,6 @@ public async Task Attachments_with_ids_that_contain_backslash_should_be_readable var contentType = "NotImportant"; var body = BitConverter.GetBytes(0xDEADBEEF); - using (var session = GetRequiredService().OpenAsyncSession()) - { - await session.StoreAsync(new FailedMessage { Id = messageId }); - await session.SaveChangesAsync(); - } - await BodyStorage.Store(messageId, contentType, body.Length, new MemoryStream(body)); var retrieved = await BodyStorage.TryFetch(messageId); From 1f629b7b7ff51be9667f00fd0b1fff89aeec672f Mon Sep 17 00:00:00 2001 From: David Boike Date: Wed, 27 Sep 2023 12:23:44 -0500 Subject: [PATCH 80/94] Non-parallel acceptance tests for now --- src/ServiceControl.AcceptanceTests/TestSupport/AcceptanceTest.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ServiceControl.AcceptanceTests/TestSupport/AcceptanceTest.cs b/src/ServiceControl.AcceptanceTests/TestSupport/AcceptanceTest.cs index c57e235d56..78e7887eb5 100644 --- a/src/ServiceControl.AcceptanceTests/TestSupport/AcceptanceTest.cs +++ b/src/ServiceControl.AcceptanceTests/TestSupport/AcceptanceTest.cs @@ -21,7 +21,6 @@ namespace ServiceControl.AcceptanceTests using TestSupport; [TestFixture] - [Parallelizable(ParallelScope.All)] abstract class AcceptanceTest : NServiceBusAcceptanceTest, IAcceptanceTestInfrastructureProvider { protected AcceptanceTest() From c7c6260ef93af13581f71db70757d6a94e6be33a Mon Sep 17 00:00:00 2001 From: David Boike Date: Wed, 27 Sep 2023 12:58:59 -0500 Subject: [PATCH 81/94] Moving the full-text setting to persister won't affect platform sample --- .../APIApprovals.PlatformSampleSettings.approved.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt index 918b30ef20..0edf138ed9 100644 --- a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt +++ b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt @@ -37,7 +37,6 @@ "RetryHistoryDepth": 10, "RemoteInstances": [], "DataSpaceRemainingThreshold": 20, - "EnableFullTextSearchOnBodies": true, "DisableHealthChecks": false, "ExposeApi": true } \ No newline at end of file From 6bd1a9bebfe04eb698f79b472ac44ea4facd82a3 Mon Sep 17 00:00:00 2001 From: David Boike Date: Wed, 27 Sep 2023 13:53:56 -0500 Subject: [PATCH 82/94] Make Raven35 tests support parallel execution as well --- .../ConfigureEndpointLearningTransport.cs | 2 + .../PortPool.cs | 69 +++++++++++++++++++ .../ServiceControl.AcceptanceTesting.csproj | 1 + .../AcceptanceTestStorageConfiguration.cs | 15 +++- .../AcceptanceTestStorageConfiguration.cs | 12 +++- .../DatabaseLease.cs | 28 ++++++++ .../SharedDatabaseSetup.cs | 55 +-------------- .../TestSupport/AcceptanceTest.cs | 1 + .../RavenAttachmentsBodyStorage.cs | 3 - 9 files changed, 124 insertions(+), 62 deletions(-) create mode 100644 src/ServiceControl.AcceptanceTesting/PortPool.cs create mode 100644 src/ServiceControl.AcceptanceTests.RavenDB5/DatabaseLease.cs diff --git a/src/ServiceControl.AcceptanceTesting/InfrastructureConfig/ConfigureEndpointLearningTransport.cs b/src/ServiceControl.AcceptanceTesting/InfrastructureConfig/ConfigureEndpointLearningTransport.cs index 943ec5f5e5..3de681f62a 100644 --- a/src/ServiceControl.AcceptanceTesting/InfrastructureConfig/ConfigureEndpointLearningTransport.cs +++ b/src/ServiceControl.AcceptanceTesting/InfrastructureConfig/ConfigureEndpointLearningTransport.cs @@ -1,5 +1,6 @@ namespace ServiceControl.AcceptanceTesting.InfrastructureConfig { + using System; using System.IO; using System.Threading.Tasks; using NServiceBus; @@ -18,6 +19,7 @@ public ConfigureEndpointLearningTransport() public Task Configure(string endpointName, EndpointConfiguration configuration, RunSettings settings, PublisherMetadata publisherMetadata) { + Console.WriteLine($"Transport Directory: {ConnectionString}"); Directory.CreateDirectory(ConnectionString); var transportConfig = configuration.UseTransport(); diff --git a/src/ServiceControl.AcceptanceTesting/PortPool.cs b/src/ServiceControl.AcceptanceTesting/PortPool.cs new file mode 100644 index 0000000000..b64680beb1 --- /dev/null +++ b/src/ServiceControl.AcceptanceTesting/PortPool.cs @@ -0,0 +1,69 @@ +namespace ServiceControl.AcceptanceTesting +{ + using System; + using System.Collections.Generic; + using System.Linq; + using NUnit.Framework; + using TestHelper; + + public class PortPool + { + readonly int startingPort; + readonly HashSet inUse; + + public PortPool(int startingPort) + { + this.startingPort = startingPort; + inUse = new HashSet(); + } + + public PortLease GetLease() => new PortLease(this); + + internal int LeasePort() + { + lock (inUse) + { + var start = inUse.Any() ? inUse.Max() + 1 : startingPort; + var port = PortUtility.FindAvailablePort(start); + inUse.Add(port); + TestContext.Out.WriteLine($"Port leased: {port}"); + return port; + } + } + + internal void Return(int port) + { + lock (inUse) + { + inUse.Remove(port); + } + } + } + + public class PortLease : IDisposable + { + readonly PortPool owner; + readonly List leasedPorts; + + internal PortLease(PortPool owner) + { + this.owner = owner; + leasedPorts = new List(); + } + + public int GetPort() + { + var port = owner.LeasePort(); + leasedPorts.Add(port); + return port; + } + + public void Dispose() + { + foreach (var port in leasedPorts) + { + owner.Return(port); + } + } + } +} diff --git a/src/ServiceControl.AcceptanceTesting/ServiceControl.AcceptanceTesting.csproj b/src/ServiceControl.AcceptanceTesting/ServiceControl.AcceptanceTesting.csproj index 46fdddf56e..3bb44a0479 100644 --- a/src/ServiceControl.AcceptanceTesting/ServiceControl.AcceptanceTesting.csproj +++ b/src/ServiceControl.AcceptanceTesting/ServiceControl.AcceptanceTesting.csproj @@ -7,6 +7,7 @@ + diff --git a/src/ServiceControl.AcceptanceTests.RavenDB/AcceptanceTestStorageConfiguration.cs b/src/ServiceControl.AcceptanceTests.RavenDB/AcceptanceTestStorageConfiguration.cs index 6c68056ac2..af5f8e46f3 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB/AcceptanceTestStorageConfiguration.cs +++ b/src/ServiceControl.AcceptanceTests.RavenDB/AcceptanceTestStorageConfiguration.cs @@ -5,18 +5,23 @@ using System.Threading.Tasks; using Persistence.RavenDb; using ServiceBus.Management.Infrastructure.Settings; - using TestHelper; + using ServiceControl.AcceptanceTesting; class AcceptanceTestStorageConfiguration { + static readonly PortPool portPool = new PortPool(33334); + readonly PortLease portLease = portPool.GetLease(); + public string PersistenceType { get; protected set; } public void CustomizeSettings(Settings settings) { + settings.Port = portLease.GetPort(); + settings.PersisterSpecificSettings = new RavenDBPersisterSettings { RunInMemory = true, - DatabaseMaintenancePort = PortUtility.FindAvailablePort(settings.Port + 1), + DatabaseMaintenancePort = portLease.GetPort(), DatabasePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()), ErrorRetentionPeriod = TimeSpan.FromDays(10), }; @@ -29,6 +34,10 @@ public Task Configure() return Task.CompletedTask; } - public Task Cleanup() => Task.CompletedTask; + public Task Cleanup() + { + portLease.Dispose(); + return Task.CompletedTask; + } } } diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs index 9dc90a5edd..686d3d4ba9 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/AcceptanceTestStorageConfiguration.cs @@ -3,17 +3,21 @@ using System.Threading.Tasks; using Persistence.RavenDb; using ServiceBus.Management.Infrastructure.Settings; + using ServiceControl.AcceptanceTesting; class AcceptanceTestStorageConfiguration { + static readonly PortPool portPool = new PortPool(33334); + readonly DatabaseLease databaseLease = SharedDatabaseSetup.LeaseDatabase(); + readonly PortLease portLease = portPool.GetLease(); public string PersistenceType { get; protected set; } public void CustomizeSettings(Settings settings) { databaseLease.CustomizeSettings(settings); - settings.Port = databaseLease.LeasePort(); + settings.Port = portLease.GetPort(); } public Task Configure() @@ -23,6 +27,10 @@ public Task Configure() return Task.CompletedTask; } - public ValueTask Cleanup() => databaseLease.DisposeAsync(); + public async ValueTask Cleanup() + { + portLease.Dispose(); + await databaseLease.DisposeAsync(); + } } } diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/DatabaseLease.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/DatabaseLease.cs new file mode 100644 index 0000000000..b9931e209c --- /dev/null +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/DatabaseLease.cs @@ -0,0 +1,28 @@ +using System; +using System.Threading.Tasks; +using Raven.Client.ServerWide.Operations; +using ServiceBus.Management.Infrastructure.Settings; + +public class DatabaseLease : IAsyncDisposable +{ + public string DatabaseName { get; } = Guid.NewGuid().ToString("n"); + + public void CustomizeSettings(Settings settings) + { + settings.PersisterSpecificSettings = new RavenDBPersisterSettings + { + ErrorRetentionPeriod = TimeSpan.FromDays(10), + ConnectionString = SharedDatabaseSetup.SharedInstance.ServerUrl, + DatabaseName = DatabaseName + }; + } + + public async ValueTask DisposeAsync() + { + var documentStore = await SharedDatabaseSetup.SharedInstance.Connect(); + + // Comment this out temporarily to be able to inspect a database after the test has completed + var deleteDatabasesOperation = new DeleteDatabasesOperation(new DeleteDatabasesOperation.Parameters { DatabaseNames = new[] { DatabaseName }, HardDelete = true }); + await documentStore.Maintenance.Server.SendAsync(deleteDatabasesOperation); + } +} \ No newline at end of file diff --git a/src/ServiceControl.AcceptanceTests.RavenDB5/SharedDatabaseSetup.cs b/src/ServiceControl.AcceptanceTests.RavenDB5/SharedDatabaseSetup.cs index 5604ad03e1..a3399d71e8 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB5/SharedDatabaseSetup.cs +++ b/src/ServiceControl.AcceptanceTests.RavenDB5/SharedDatabaseSetup.cs @@ -1,13 +1,7 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; +using System.Threading; using System.Threading.Tasks; using NUnit.Framework; -using Raven.Client.ServerWide.Operations; -using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.Persistence.RavenDb5; -using TestHelper; [SetUpFixture] public static class SharedDatabaseSetup @@ -32,50 +26,3 @@ public static DatabaseLease LeaseDatabase() return new DatabaseLease(); } } - -public class DatabaseLease : IAsyncDisposable -{ - public string DatabaseName { get; } = Guid.NewGuid().ToString("n"); - static HashSet allPortsInUse = new HashSet(); - List usedPorts = new List(); - - public void CustomizeSettings(Settings settings) - { - settings.PersisterSpecificSettings = new RavenDBPersisterSettings - { - ErrorRetentionPeriod = TimeSpan.FromDays(10), - ConnectionString = SharedDatabaseSetup.SharedInstance.ServerUrl, - DatabaseName = DatabaseName - }; - } - - public int LeasePort() - { - lock (allPortsInUse) - { - var start = allPortsInUse.Any() ? allPortsInUse.Max() + 1 : 33334; - var port = PortUtility.FindAvailablePort(start); - allPortsInUse.Add(port); - usedPorts.Add(port); - TestContext.Out.WriteLine($"Port leased: {port}"); - return port; - } - } - - public async ValueTask DisposeAsync() - { - lock (allPortsInUse) - { - foreach (var port in usedPorts) - { - allPortsInUse.Remove(port); - } - } - - var documentStore = await SharedDatabaseSetup.SharedInstance.Connect(); - - // Comment this out temporarily to be able to inspect a database after the test has completed - var deleteDatabasesOperation = new DeleteDatabasesOperation(new DeleteDatabasesOperation.Parameters { DatabaseNames = new[] { DatabaseName }, HardDelete = true }); - await documentStore.Maintenance.Server.SendAsync(deleteDatabasesOperation); - } -} \ No newline at end of file diff --git a/src/ServiceControl.AcceptanceTests/TestSupport/AcceptanceTest.cs b/src/ServiceControl.AcceptanceTests/TestSupport/AcceptanceTest.cs index 78e7887eb5..c57e235d56 100644 --- a/src/ServiceControl.AcceptanceTests/TestSupport/AcceptanceTest.cs +++ b/src/ServiceControl.AcceptanceTests/TestSupport/AcceptanceTest.cs @@ -21,6 +21,7 @@ namespace ServiceControl.AcceptanceTests using TestSupport; [TestFixture] + [Parallelizable(ParallelScope.All)] abstract class AcceptanceTest : NServiceBusAcceptanceTest, IAcceptanceTestInfrastructureProvider { protected AcceptanceTest() diff --git a/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs b/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs index 6b35cf1272..94bb31a7cd 100644 --- a/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs +++ b/src/ServiceControl.Persistence.RavenDb5/RavenAttachmentsBodyStorage.cs @@ -1,12 +1,9 @@ namespace ServiceControl.Operations.BodyStorage.RavenAttachments { using System.IO; - using System.Runtime.Remoting.Contexts; using System.Threading.Tasks; using Raven.Client.Documents; using Raven.Client.Documents.Commands.Batches; - using Raven.Client.Documents.Operations.Attachments; - using ServiceControl.Infrastructure; using Sparrow.Json.Parsing; class RavenAttachmentsBodyStorage : IBodyStorage From 1146818b97de1ba0ac3bf9eb9b78f766f4389298 Mon Sep 17 00:00:00 2001 From: David Boike Date: Wed, 27 Sep 2023 15:30:48 -0500 Subject: [PATCH 83/94] Fix neat little hidden assumption that message metadata containing "Body" (used for FT search) means body not in body storage --- .../UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs index 96e02d0073..fbe421e69b 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs @@ -47,7 +47,7 @@ public Task RecordFailedProcessingAttempt( try { var bodyString = utf8.GetString(context.Body); - processingAttempt.MessageMetadata.Add("Body", bodyString); + processingAttempt.MessageMetadata.Add("MsgFullText", bodyString); } catch (ArgumentException) { From dad2f38c8bd8b91fe6c782214d4983fd22d4752e Mon Sep 17 00:00:00 2001 From: David Boike Date: Wed, 27 Sep 2023 15:45:48 -0500 Subject: [PATCH 84/94] This is set declaratively, no need to touch global ConfigurationManager --- .../TestSupport/ServiceControlComponentRunner.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/ServiceControl.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs b/src/ServiceControl.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs index 0220a725ca..e661b70b1d 100644 --- a/src/ServiceControl.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs +++ b/src/ServiceControl.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs @@ -51,8 +51,6 @@ async Task InitializeServiceControl(ScenarioContext context) { var instancePort = PortUtility.FindAvailablePort(33333); - ConfigurationManager.AppSettings.Set("ServiceControl/TransportType", transportToUse.TypeName); - var settings = new Settings(instanceName, transportToUse.TypeName, persistenceToUse.PersistenceType, forwardErrorMessages: false, errorRetentionPeriod: TimeSpan.FromDays(10)) { AllowMessageEditing = true, From 61e657e8b0cd077a8dcac067474992373d62e01f Mon Sep 17 00:00:00 2001 From: David Boike Date: Wed, 27 Sep 2023 18:25:12 -0500 Subject: [PATCH 85/94] ignore SagaAudit acceptance tests in Raven5 --- .../SagaAudit/When_a_saga_instance_is_being_created.cs | 5 +++++ .../When_multiple_messages_are_emitted_by_a_saga.cs | 5 +++++ .../SagaAudit/When_requesting_timeout_from_a_saga.cs | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/src/ServiceControl.AcceptanceTests/SagaAudit/When_a_saga_instance_is_being_created.cs b/src/ServiceControl.AcceptanceTests/SagaAudit/When_a_saga_instance_is_being_created.cs index bb7c5b2cdc..5b39006306 100644 --- a/src/ServiceControl.AcceptanceTests/SagaAudit/When_a_saga_instance_is_being_created.cs +++ b/src/ServiceControl.AcceptanceTests/SagaAudit/When_a_saga_instance_is_being_created.cs @@ -16,6 +16,11 @@ class When_a_saga_instance_is_being_created : AcceptanceTest [Test] public async Task Saga_audit_trail_should_contain_the_state_change() { + if (typeof(AcceptanceTest).Assembly.FullName.Contains("RavenDB5")) + { + Assert.Ignore("SagaAudit data is not stored in RavenDB 5"); + } + SagaHistory sagaHistory = null; var context = await Define() diff --git a/src/ServiceControl.AcceptanceTests/SagaAudit/When_multiple_messages_are_emitted_by_a_saga.cs b/src/ServiceControl.AcceptanceTests/SagaAudit/When_multiple_messages_are_emitted_by_a_saga.cs index 904767f5d7..728d182b4e 100644 --- a/src/ServiceControl.AcceptanceTests/SagaAudit/When_multiple_messages_are_emitted_by_a_saga.cs +++ b/src/ServiceControl.AcceptanceTests/SagaAudit/When_multiple_messages_are_emitted_by_a_saga.cs @@ -18,6 +18,11 @@ class When_multiple_messages_are_emitted_by_a_saga : AcceptanceTest [Test] public async Task Should_capture_all_outgoing_message_intents() { + if (typeof(AcceptanceTest).Assembly.FullName.Contains("RavenDB5")) + { + Assert.Ignore("SagaAudit data is not stored in RavenDB 5"); + } + SagaHistory sagaHistory = null; var context = await Define() diff --git a/src/ServiceControl.AcceptanceTests/SagaAudit/When_requesting_timeout_from_a_saga.cs b/src/ServiceControl.AcceptanceTests/SagaAudit/When_requesting_timeout_from_a_saga.cs index 3a7e58af28..7050b3b110 100644 --- a/src/ServiceControl.AcceptanceTests/SagaAudit/When_requesting_timeout_from_a_saga.cs +++ b/src/ServiceControl.AcceptanceTests/SagaAudit/When_requesting_timeout_from_a_saga.cs @@ -16,6 +16,11 @@ class When_requesting_timeout_from_a_saga : AcceptanceTest [Test] public async Task Saga_audit_trail_should_contain_the_state_change() { + if (typeof(AcceptanceTest).Assembly.FullName.Contains("RavenDB5")) + { + Assert.Ignore("SagaAudit data is not stored in RavenDB 5"); + } + SagaHistory sagaHistory = null; var context = await Define() From 5c7af94157e42b1f8fb9f95db5326f31b81d8542 Mon Sep 17 00:00:00 2001 From: David Boike Date: Wed, 27 Sep 2023 18:25:43 -0500 Subject: [PATCH 86/94] And back to sequential --- .../TestSupport/AcceptanceTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ServiceControl.AcceptanceTests/TestSupport/AcceptanceTest.cs b/src/ServiceControl.AcceptanceTests/TestSupport/AcceptanceTest.cs index c57e235d56..67dc29385e 100644 --- a/src/ServiceControl.AcceptanceTests/TestSupport/AcceptanceTest.cs +++ b/src/ServiceControl.AcceptanceTests/TestSupport/AcceptanceTest.cs @@ -21,7 +21,7 @@ namespace ServiceControl.AcceptanceTests using TestSupport; [TestFixture] - [Parallelizable(ParallelScope.All)] + //[Parallelizable(ParallelScope.All)] abstract class AcceptanceTest : NServiceBusAcceptanceTest, IAcceptanceTestInfrastructureProvider { protected AcceptanceTest() From 07fcd65988416f7de9b67184c3f46c5ae2c1ea1c Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 28 Sep 2023 14:27:37 +0200 Subject: [PATCH 87/94] =?UTF-8?q?=F0=9F=A9=B9=20Fixed=20the=20MessageViewT?= =?UTF-8?q?ransformer,=20in=20its=20mapping=20it=20used=20the=20`Input.Mes?= =?UTF-8?q?sageId,=20while=20this=20is=20in=20the=20RavenDB=20index=20reco?= =?UTF-8?q?rd=20this=20cannot=20be=20used=20in=20the=20mapping=20will=20re?= =?UTF-8?q?turn=20the=20default=20value=20as=20the=20query=20will=20only?= =?UTF-8?q?=20have=20access=20to=20the=20FailedMessage=20document.=20Raven?= =?UTF-8?q?DB=20internally=20will=20optimize=20to=20only=20use=20the=20ind?= =?UTF-8?q?ex=20record=20if=20it=20can.=0B=0BRemoved=20`MessagesViewTransf?= =?UTF-8?q?ormer.Input`=20class=20as=20it=20not=20needed=20for=20querying?= =?UTF-8?q?=20which=20makes=20the=20code=20way=20more=20readable=20and=20u?= =?UTF-8?q?nderstandable.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ErrorMessagesDataStore.cs | 39 ++++++++-------- .../Indexes/FailedMessageViewIndex.cs | 27 +++++------ .../Transformers/MessagesViewTransformer.cs | 46 +++++++------------ .../CompositeViews/MessagesViewTests.cs | 12 ++--- 4 files changed, 56 insertions(+), 68 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index 5434831614..d2bc977b17 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -42,14 +42,14 @@ bool includeSystemMessages using (var session = documentStore.OpenAsyncSession()) { var query = session.Query() - .IncludeSystemMessagesWhere(includeSystemMessages) - .Statistics(out var stats) - .Sort(sortInfo) - .Paging(pagingInfo) - .ProjectInto(); + .IncludeSystemMessagesWhere(includeSystemMessages) + .Statistics(out var stats) + .Sort(sortInfo) + .Paging(pagingInfo) + .ProjectInto() + .TransformToMessageView(); - var results = await MessagesViewTransformer.Transform(query) - .ToListAsync(); + var results = await query.ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); } @@ -70,10 +70,11 @@ bool includeSystemMessages .Statistics(out var stats) .Sort(sortInfo) .Paging(pagingInfo) - .ProjectInto(); + .ProjectInto() + .TransformToMessageView(); + + var results = await query.ToListAsync(); - var results = await MessagesViewTransformer.Transform(query) - .ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); } @@ -94,10 +95,10 @@ SortInfo sortInfo .Where(m => m.ReceivingEndpointName == endpointName) .Sort(sortInfo) .Paging(pagingInfo) - .ProjectInto(); + .ProjectInto() + .TransformToMessageView(); - var results = await MessagesViewTransformer.Transform(query) - .ToListAsync(); + var results = await query.ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); } @@ -117,10 +118,10 @@ bool includeSystemMessages .Where(m => m.ConversationId == conversationId) .Sort(sortInfo) .Paging(pagingInfo) - .ProjectInto(); + .ProjectInto() + .TransformToMessageView(); - var results = await MessagesViewTransformer.Transform(query) - .ToListAsync(); + var results = await query.ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); } @@ -139,10 +140,10 @@ SortInfo sortInfo .Search(x => x.Query, searchTerms) .Sort(sortInfo) .Paging(pagingInfo) - .ProjectInto(); + .ProjectInto() + .TransformToMessageView(); - var results = await MessagesViewTransformer.Transform(query) - .ToListAsync(); + var results = await query.ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); } diff --git a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageViewIndex.cs b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageViewIndex.cs index b390b2c5cb..11fa44ed64 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageViewIndex.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Indexes/FailedMessageViewIndex.cs @@ -9,19 +9,20 @@ class FailedMessageViewIndex : AbstractIndexCreationTask { public FailedMessageViewIndex() { - Map = messages => from message in messages - let processingAttemptsLast = message.ProcessingAttempts.Last() - select new - { - MessageId = processingAttemptsLast.MessageMetadata["MessageId"], - MessageType = processingAttemptsLast.MessageMetadata["MessageType"], - message.Status, - TimeSent = (DateTime)processingAttemptsLast.MessageMetadata["TimeSent"], - ReceivingEndpointName = ((EndpointDetails)processingAttemptsLast.MessageMetadata["ReceivingEndpoint"]).Name, - QueueAddress = processingAttemptsLast.FailureDetails.AddressOfFailingEndpoint, - processingAttemptsLast.FailureDetails.TimeOfFailure, - LastModified = MetadataFor(message).Value("@last-modified").Ticks - }; + Map = messages => + from message in messages + let processingAttemptsLast = message.ProcessingAttempts.Last() + select new + { + MessageId = processingAttemptsLast.MessageMetadata["MessageId"], + MessageType = processingAttemptsLast.MessageMetadata["MessageType"], + message.Status, + TimeSent = (DateTime)processingAttemptsLast.MessageMetadata["TimeSent"], + ReceivingEndpointName = ((EndpointDetails)processingAttemptsLast.MessageMetadata["ReceivingEndpoint"]).Name, + QueueAddress = processingAttemptsLast.FailureDetails.AddressOfFailingEndpoint, + processingAttemptsLast.FailureDetails.TimeOfFailure, + LastModified = MetadataFor(message).Value("@last-modified").Ticks + }; } public class SortAndFilterOptions : IHaveStatus diff --git a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs index 6c3286d62d..daed587531 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Transformers/MessagesViewTransformer.cs @@ -1,33 +1,34 @@ namespace ServiceControl.CompositeViews.Messages { - using System.Collections.Generic; using System.Linq; using MessageFailures; using Raven.Client.Documents.Linq; using ServiceControl.Persistence; - class MessagesViewTransformer //https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers + static class MessagesViewTransformer //https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers { - public static IQueryable Transform(IRavenQueryable query) + public static IQueryable TransformToMessageView(this IRavenQueryable query) { var results = from message in query - let metadata = message.ProcessingAttempts.Last().MessageMetadata - let headers = message.ProcessingAttempts.Last().Headers - let processedAt = message.ProcessingAttempts.Last().AttemptedAt let status = message.Status == FailedMessageStatus.Resolved - ? MessageStatus.ResolvedSuccessfully - : message.Status == FailedMessageStatus.RetryIssued - ? MessageStatus.RetryIssued - : message.Status == FailedMessageStatus.Archived - ? MessageStatus.ArchivedFailure - : message.ProcessingAttempts.Count == 1 - ? MessageStatus.Failed - : MessageStatus.RepeatedFailure + ? MessageStatus.ResolvedSuccessfully + : message.Status == FailedMessageStatus.RetryIssued + ? MessageStatus.RetryIssued + : message.Status == FailedMessageStatus.Archived + ? MessageStatus.ArchivedFailure + : message.ProcessingAttempts.Count == 1 + ? MessageStatus.Failed + : MessageStatus.RepeatedFailure + let last = message.ProcessingAttempts.Last() + let metadata = last.MessageMetadata + let headers = last.Headers + let processedAt = last.AttemptedAt + select new // Cannot use type here as this is projected server-side { Id = message.UniqueMessageId, - message.MessageId, + MessageId = metadata["MessageId"], MessageType = metadata["MessageType"], SendingEndpoint = metadata["SendingEndpoint"], ReceivingEndpoint = metadata["ReceivingEndpoint"], @@ -51,20 +52,5 @@ from message in query return results.OfType(); } - - public class Input : MessagesViewIndex.SortAndFilterOptions - { - public new FailedMessageStatus Status { get; } - public string UniqueMessageId { get; } - public List ProcessingAttempts { get; } - } - } - - public static class MessageViewTransformerExtension - { - public static IQueryable TransformToMessagesView(this IQueryable source) - { - return source.Select(m => new MessagesView()); - } } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs index cb1b9f3cca..e753e208de 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDb5/CompositeViews/MessagesViewTests.cs @@ -189,10 +189,10 @@ public async Task TimeSent_is_not_cast_to_DateTimeMin_if_null() using (var session = DocumentStore.OpenAsyncSession()) { var query = session.Query() - .ProjectInto() + .ProjectInto() .Customize(x => x.WaitForNonStaleResults()); - var messagesWithNoTimestamp = await MessagesViewTransformer.Transform(query).ToArrayAsync(); + var messagesWithNoTimestamp = await MessagesViewTransformer.TransformToMessageView(query).ToArrayAsync(); Assert.AreEqual(null, messagesWithNoTimestamp[0].TimeSent); Assert.AreEqual(null, messagesWithNoTimestamp[1].TimeSent); @@ -233,10 +233,10 @@ public async Task Correct_status_for_failed_messages(FailedMessageStatus failedM using (var session = DocumentStore.OpenAsyncSession()) { var query = session.Query() - .ProjectInto() + .ProjectInto() .Customize(x => x.WaitForNonStaleResults()); - var result = await MessagesViewTransformer.Transform(query).ToListAsync(); + var result = await MessagesViewTransformer.TransformToMessageView(query).ToListAsync(); var message = result.Single(); @@ -276,10 +276,10 @@ public async Task Correct_status_for_repeated_errors() { var query = session .Query() - .ProjectInto() + .ProjectInto() .Customize(x => x.WaitForNonStaleResults()); - var result = await MessagesViewTransformer.Transform(query).ToListAsync(); + var result = await MessagesViewTransformer.TransformToMessageView(query).ToListAsync(); var message = result.Single(); From e3b8e260261ea59f097fae85f365f273f520ad40 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 28 Sep 2023 14:31:34 +0200 Subject: [PATCH 88/94] =?UTF-8?q?=F0=9F=94=A8=20=20Applied=20same=20refact?= =?UTF-8?q?oring=20on=20FailedMessageViewTransformer=20as=20on=20Transform?= =?UTF-8?q?ToMessageView?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ErrorMessagesDataStore.cs | 15 +++++++++------ .../Transformers/FailedMessageViewTransformer.cs | 4 ++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index d2bc977b17..fceada7ba4 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -252,9 +252,10 @@ SortInfo sortInfo .Sort(sortInfo) .Paging(pagingInfo) .SelectFields() - .ToQueryable(); + .ToQueryable() + .TransformToFailedMessageView(); - var results = await FailedMessageViewTransformer.Transform(query) + var results = await query .ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); @@ -300,9 +301,10 @@ SortInfo sortInfo .Sort(sortInfo) .Paging(pagingInfo) .SelectFields() - .ToQueryable(); + .ToQueryable() + .TransformToFailedMessageView(); - var results = await FailedMessageViewTransformer.Transform(query) + var results = await query .ToListAsync(); return new QueryResult>(results, stats.ToQueryStatsInfo()); @@ -475,9 +477,10 @@ PagingInfo pagingInfo .Sort(sortInfo) .Paging(pagingInfo) .SelectFields() - .ToQueryable(); + .ToQueryable() + .TransformToFailedMessageView(); - var results = await FailedMessageViewTransformer.Transform(query) + var results = await query .ToListAsync(); return results.ToQueryResult(stats); diff --git a/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs b/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs index e20884ce05..96b8b40006 100644 --- a/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs +++ b/src/ServiceControl.Persistence.RavenDb5/Transformers/FailedMessageViewTransformer.cs @@ -5,9 +5,9 @@ using Raven.Client.Documents.Linq; using Raven.Client.Documents.Queries; - class FailedMessageViewTransformer // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers + static class FailedMessageViewTransformer // https://ravendb.net/docs/article-page/4.2/csharp/migration/client-api/session/querying/transformers { - public static IQueryable Transform(IRavenQueryable query) + public static IQueryable TransformToFailedMessageView(this IRavenQueryable query) { var failures = from failure in query From 0a44822d57ec079797c63f331945bb062aad87d8 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 28 Sep 2023 14:32:23 +0200 Subject: [PATCH 89/94] Added DebuggerDisplay for easier debugging --- src/ServiceControl.Persistence/Infrastructure/PagingInfo.cs | 3 +++ src/ServiceControl.Persistence/Infrastructure/SortInfo.cs | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/ServiceControl.Persistence/Infrastructure/PagingInfo.cs b/src/ServiceControl.Persistence/Infrastructure/PagingInfo.cs index 5f6fe7746e..269828f8a6 100644 --- a/src/ServiceControl.Persistence/Infrastructure/PagingInfo.cs +++ b/src/ServiceControl.Persistence/Infrastructure/PagingInfo.cs @@ -1,5 +1,8 @@ namespace ServiceControl.Persistence.Infrastructure { + using System.Diagnostics; + + [DebuggerDisplay("{Page}/{PageSize}")] public class PagingInfo { public const int DefaultPageSize = 50; diff --git a/src/ServiceControl.Persistence/Infrastructure/SortInfo.cs b/src/ServiceControl.Persistence/Infrastructure/SortInfo.cs index b2ef7e6287..42ec4df00b 100644 --- a/src/ServiceControl.Persistence/Infrastructure/SortInfo.cs +++ b/src/ServiceControl.Persistence/Infrastructure/SortInfo.cs @@ -1,5 +1,8 @@ namespace ServiceControl.Persistence.Infrastructure { + using System.Diagnostics; + + [DebuggerDisplay("{Sort} {Direction}")] public class SortInfo { public string Direction { get; } From ae46e2f76b92cba95a410164e07b785ecad2b121 Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Thu, 28 Sep 2023 14:51:42 +0200 Subject: [PATCH 90/94] when_a_pending_retry test fix --- .../ErrorMessagesDataStore.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index fceada7ba4..9d0d14538c 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -547,9 +547,9 @@ public async Task ProcessPendingRetries(DateTime periodFrom, DateTime periodTo, { var prequery = session.Advanced .AsyncDocumentQuery() - .WhereEquals("Status", (int)FailedMessageStatus.RetryIssued) - .AndAlso() - .WhereBetween("LastModified", periodFrom.Ticks, periodTo.Ticks); + .WhereEquals("Status", (int)FailedMessageStatus.RetryIssued) + .AndAlso() + .WhereBetween("LastModified", periodFrom.Ticks, periodTo.Ticks); if (!string.IsNullOrWhiteSpace(queueAddress)) { @@ -558,9 +558,9 @@ public async Task ProcessPendingRetries(DateTime periodFrom, DateTime periodTo, } var query = prequery - // TODO: Fix SetResultTransformer - //.SetResultTransformer(new FailedMessageViewTransformer().TransformerName) - .SelectFields(); + .SelectFields() + .ToQueryable() + .TransformToFailedMessageView(); await using (var ie = await session.Advanced.StreamAsync(query)) { From 3be1657a820ccec27a597d358a436607b74e9cb7 Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Thu, 28 Sep 2023 14:57:43 +0200 Subject: [PATCH 91/94] yet another todo fixed --- .../ErrorMessagesDataStore.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs index 9d0d14538c..ecc19fb28f 100644 --- a/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDb5/ErrorMessagesDataStore.cs @@ -718,9 +718,9 @@ public async Task GetRetryPendingMessages(DateTime from, DateTime to, .WhereBetween(options => options.LastModified, from.Ticks, to.Ticks) .AndAlso() .WhereEquals(o => o.QueueAddress, queueAddress) - // TODO: Fix SetResultTransformer - //.SetResultTransformer(FailedMessageViewTransformer.Name) - .SelectFields(new[] { "Id" }); + .SelectFields() + .ToQueryable() + .TransformToFailedMessageView(); await using (var ie = await session.Advanced.StreamAsync(query)) { From 2a1de98d8b285a2256756844f8f370ee04700c4f Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Thu, 28 Sep 2023 15:09:06 +0200 Subject: [PATCH 92/94] moar fixes --- .../UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs index fbe421e69b..2591ea5852 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs @@ -95,7 +95,7 @@ ICommandData CreateFailedMessagesPatchCommand(string uniqueMessageId, FailedMess var newAttempts = this.{nameof(FailedMessage.ProcessingAttempts)}; //De-duplicate attempts by AttemptedAt value - var duplicateIndex = this.{ProcessingAttempts}.findIndex(a => a.{AttemptedAt} === attempt.{AttemptedAt}); + var duplicateIndex = this.{ProcessingAttempts}.findIndex(a => a.{AttemptedAt} === args.attempt.{AttemptedAt}); if(duplicateIndex < 0){{ newAttempts.push(args.attempt); From 5a1f0bcc9bcc2348f70c38c7200e3a90b75b4a4c Mon Sep 17 00:00:00 2001 From: Tomek Masternak Date: Thu, 28 Sep 2023 15:27:41 +0200 Subject: [PATCH 93/94] tests are green? --- .../UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs index 2591ea5852..726efc7eb1 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs @@ -34,12 +34,14 @@ public Task RecordFailedProcessingAttempt( var uniqueMessageId = context.Headers.UniqueId(); var bodyId = processingAttempt.Headers.MessageId(); var contentType = GetContentType(context.Headers, "text/xml"); + var bodySize = context.Body?.Length ?? 0; + processingAttempt.MessageMetadata.Add("ContentType", contentType); + processingAttempt.MessageMetadata.Add("ContentLength", bodySize); processingAttempt.MessageMetadata.Add("BodyUrl", $"/messages/{bodyId}/body"); if (doFullTextIndexing) { - var bodySize = context.Body?.Length ?? 0; var avoidsLargeObjectHeap = bodySize < LargeObjectHeapThreshold; var isBinary = processingAttempt.Headers.IsBinary(); if (avoidsLargeObjectHeap && !isBinary) From 4fe47bd131cbb6c988b8e363c832e04f885d8e39 Mon Sep 17 00:00:00 2001 From: David Boike Date: Thu, 28 Sep 2023 16:01:27 -0500 Subject: [PATCH 94/94] The message body docs should have a collection name --- .../RavenDbRecoverabilityIngestionUnitOfWork.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs index 726efc7eb1..63152ec2fd 100644 --- a/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.RavenDb5/UnitOfWork/RavenDbRecoverabilityIngestionUnitOfWork.cs @@ -146,7 +146,13 @@ void AddStoreBodyCommands(MessageContext context, string contentType) var messageId = context.Headers.MessageId(); var documentId = MessageBodyIdGenerator.MakeDocumentId(messageId); - var emptyDoc = new DynamicJsonValue(); + var emptyDoc = new DynamicJsonValue + { + ["@metadata"] = new DynamicJsonValue + { + ["@collection"] = "MessageBodies" + } + }; var putOwnerDocumentCmd = new PutCommandData(documentId, null, emptyDoc); var stream = Memory.Manager.GetStream(context.Body);