Move MultiInstance tests to RavenDB 5

src/ServiceControl.Audit/Infrastructure/NServiceBusFactory.cs

@@ -7,6 +7,7 @@ namespace ServiceControl.Audit.Infrastructure
     using Contracts.MessageFailures;
     using NServiceBus;
     using NServiceBus.Configuration.AdvancedExtensibility;
+    using ServiceControl.Infrastructure;
     using Settings;
     using Transports;
 
@@ -63,6 +64,8 @@ public static void Configure(Settings.Settings settings, TransportCustomization
             {
                 configuration.EnableInstallers();
             }
+
+            configuration.Recoverability().AddUnrecoverableException<UnrecoverableException>();
         }
 
         static bool IsExternalContract(Type t)

src/ServiceControl.Infrastructure/UnrecoverableException.cs

@@ -0,0 +1,15 @@
+namespace ServiceControl.Infrastructure
+{
+    using System;
+
+    public class UnrecoverableException : Exception
+    {
+        public UnrecoverableException(string message) : base(message)
+        {
+        }
+
+        public UnrecoverableException(string message, Exception innerException) : base(message, innerException)
+        {
+        }
+    }
+}

src/ServiceControl.MultiInstance.AcceptanceTests/AcceptanceTest.cs

@@ -68,7 +68,7 @@ public void Setup()
 
             DataStoreConfiguration = new DataStoreConfiguration
             {
-                DataStoreTypeName = "RavenDB35"
+                DataStoreTypeName = "RavenDB5"
             };
 
             serviceControlRunnerBehavior = new ServiceControlComponentBehavior(

src/ServiceControl.MultiInstance.AcceptanceTests/DatabaseLease.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Threading.Tasks;
+using Raven.Client.ServerWide.Operations;
+using ServiceBus.Management.Infrastructure.Settings;
+
+public class DatabaseLease : IAsyncDisposable
+{
+    public string DatabaseName { get; } = Guid.NewGuid().ToString("n");
+
+    public void CustomizeSettings(Settings settings)
+    {
+        settings.PersisterSpecificSettings = new RavenDBPersisterSettings
+        {
+            ErrorRetentionPeriod = TimeSpan.FromDays(10),
+            ConnectionString = SharedDatabaseSetup.SharedInstance.ServerUrl,
+            DatabaseName = DatabaseName
+        };
+    }
+
+    public async ValueTask DisposeAsync()
+    {
+        var documentStore = await SharedDatabaseSetup.SharedInstance.Connect();
+
+        // Comment this out temporarily to be able to inspect a database after the test has completed
+        var deleteDatabasesOperation = new DeleteDatabasesOperation(new DeleteDatabasesOperation.Parameters { DatabaseNames = new[] { DatabaseName }, HardDelete = true });
+        await documentStore.Maintenance.Server.SendAsync(deleteDatabasesOperation);
+    }
+}

src/ServiceControl.MultiInstance.AcceptanceTests/SagaAudit/When_sending_saga_audit_to_main_instance.cs

@@ -8,6 +8,7 @@
     using NServiceBus;
     using NServiceBus.AcceptanceTesting;
     using NUnit.Framework;
+    using ServiceControl.EventLog;
     using ServiceControl.SagaAudit;
     using TestSupport;
     using TestSupport.EndpointTemplates;
@@ -55,10 +56,17 @@ public void ConfigTeardown()
         public async Task Saga_history_can_be_fetched_from_main_instance()
         {
             SagaHistory sagaHistory = null;
+            EventLogItem eventLog = null;
+
+            CustomServiceControlSettings = settings =>
+            {
+                settings.DisableHealthChecks = false;
+                settings.PersisterSpecificSettings.OverrideCustomCheckRepeatTime = TimeSpan.FromSeconds(2);
+            };
 
             var context = await Define<MyContext>()
                 .WithEndpoint<SagaEndpoint>(b => b.When((bus, c) => bus.SendLocal(new MessageInitiatingSaga { Id = "Id" })))
-                .Done(async c =>
+                .Do("GetSagaHistory", async c =>
                 {
                     if (!c.SagaId.HasValue)
                     {
@@ -69,9 +77,17 @@ public async Task Saga_history_can_be_fetched_from_main_instance()
                     sagaHistory = result;
                     return result;
                 })
+                .Do("GetEventLog", async c =>
+                {
+                    var result = await this.TryGetMany<EventLogItem>("/api/eventlogitems/");
+                    eventLog = result.Items.FirstOrDefault(e => e.Description.Contains("Saga Audit Destination") && e.Description.Contains("endpoints have reported saga audit data to the ServiceControl Primary instance"));
+                    return eventLog != null;
+                })
+                .Done(c => eventLog != null)
                 .Run();
 
             Assert.NotNull(sagaHistory);
+            Assert.NotNull(eventLog);
 
             Assert.AreEqual(context.SagaId, sagaHistory.SagaId);
             Assert.AreEqual(typeof(SagaEndpoint.MySaga).FullName, sagaHistory.SagaType);
@@ -119,9 +135,10 @@ public class MessageInitiatingSaga : ICommand
         }
 
 
-        public class MyContext : ScenarioContext
+        public class MyContext : ScenarioContext, ISequenceContext
         {
             public Guid? SagaId { get; set; }
+            public int Step { get; set; }
         }
     }
 }

src/ServiceControl.MultiInstance.AcceptanceTests/ServiceControl.MultiInstance.AcceptanceTests.csproj

@@ -10,7 +10,7 @@
     <ProjectReference Include="..\ServiceControl.AcceptanceTesting\ServiceControl.AcceptanceTesting.csproj" />
     <ProjectReference Include="..\ServiceControl.Audit.Persistence.InMemory\ServiceControl.Audit.Persistence.InMemory.csproj" />
     <ProjectReference Include="..\ServiceControl.Audit\ServiceControl.Audit.csproj" />
-    <ProjectReference Include="..\ServiceControl.Persistence.RavenDb\ServiceControl.Persistence.RavenDb.csproj" />
+    <ProjectReference Include="..\ServiceControl.Persistence.RavenDb5\ServiceControl.Persistence.RavenDb5.csproj" />
     <ProjectReference Include="..\ServiceControl.Transports.Learning\ServiceControl.Transports.Learning.csproj" />
     <ProjectReference Include="..\ServiceControlInstaller.Engine\ServiceControlInstaller.Engine.csproj" />
     <ProjectReference Include="..\TestHelper\TestHelper.csproj" />

src/ServiceControl.MultiInstance.AcceptanceTests/SharedDatabaseSetup.cs

@@ -0,0 +1,28 @@
+using System.Threading;
+using System.Threading.Tasks;
+using NUnit.Framework;
+using ServiceControl.Persistence.RavenDb5;
+
+[SetUpFixture]
+public static class SharedDatabaseSetup
+{
+    public static EmbeddedDatabase SharedInstance { get; private set; }
+
+    // Needs to be in a SetUpFixture otherwise the OneTimeSetUp is invoked for each inherited test fixture
+    [OneTimeSetUp]
+    public static async Task SetupSharedEmbeddedServer()
+    {
+        using (var cancellation = new CancellationTokenSource(30_000))
+        {
+            SharedInstance = await SharedEmbeddedServer.GetInstance(cancellation.Token);
+        }
+    }
+
+    [OneTimeTearDown]
+    public static void TearDown() => SharedInstance.Dispose();
+
+    public static DatabaseLease LeaseDatabase()
+    {
+        return new DatabaseLease();
+    }
+}

src/ServiceControl.MultiInstance.AcceptanceTests/SharedEmbeddedServer.cs

@@ -0,0 +1,46 @@
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Raven.Client.Documents;
+using Raven.Client.ServerWide.Operations;
+using ServiceControl.Persistence.RavenDb5;
+using TestHelper;
+
+static class SharedEmbeddedServer
+{
+    public static async Task<EmbeddedDatabase> GetInstance(CancellationToken cancellationToken = default)
+    {
+        var basePath = Path.Combine(Path.GetTempPath(), "ServiceControlTests", "Primary.Raven5MultiInstance");
+        var dbPath = Path.Combine(basePath, "DB");
+        var databasesPath = Path.Combine(dbPath, "Databases");
+
+        var settings = new RavenDBPersisterSettings
+        {
+            DatabasePath = dbPath,
+            LogPath = Path.Combine(basePath, "Logs"),
+            LogsMode = "Operations",
+            ServerUrl = $"http://localhost:{PortUtility.FindAvailablePort(33334)}"
+        };
+
+        var instance = EmbeddedDatabase.Start(settings);
+
+        // Make sure that the database is up - this blocks until the cancellation token times out
+        using (var docStore = await instance.Connect(cancellationToken))
+        {
+            var cleanupDatabases = new DirectoryInfo(databasesPath)
+                .GetDirectories()
+                .Select(di => di.Name)
+                .Where(name => name.Length == 32)
+                .ToArray();
+
+            if (cleanupDatabases.Any())
+            {
+                var cleanupOperation = new DeleteDatabasesOperation(new DeleteDatabasesOperation.Parameters { DatabaseNames = cleanupDatabases, HardDelete = true });
+                await docStore.Maintenance.Server.SendAsync(cleanupOperation, CancellationToken.None);
+            }
+        }
+
+        return instance;
+    }
+}

src/ServiceControl.MultiInstance.AcceptanceTests/TestSupport/ServiceControlComponentRunner.cs

@@ -23,7 +23,6 @@ namespace ServiceControl.MultiInstance.AcceptanceTests.TestSupport
     using Particular.ServiceControl;
     using ServiceBus.Management.Infrastructure.Settings;
     using ServiceControl.Infrastructure.WebApi;
-    using TestHelper;
 
     class ServiceControlComponentRunner : ComponentRunner, IAcceptanceTestInfrastructureProviderMultiInstance
     {
@@ -46,17 +45,14 @@ public async Task Initialize(RunDescriptor run)
         {
             SettingsPerInstance.Clear();
 
-            var startPort = 33334;
-
-            var mainInstancePort = PortUtility.FindAvailablePort(startPort);
-            var mainInstanceDbPort = PortUtility.FindAvailablePort(mainInstancePort + 1);
-            var auditInstancePort = PortUtility.FindAvailablePort(mainInstanceDbPort + 1);
+            var mainInstancePort = portLeases.GetPort();
+            var auditInstancePort = portLeases.GetPort();
 
             await InitializeServiceControlAudit(run.ScenarioContext, auditInstancePort);
-            await InitializeServiceControl(run.ScenarioContext, mainInstancePort, mainInstanceDbPort, auditInstancePort);
+            await InitializeServiceControl(run.ScenarioContext, mainInstancePort, auditInstancePort);
         }
 
-        async Task InitializeServiceControl(ScenarioContext context, int instancePort, int maintenancePort, int auditInstanceApiPort)
+        async Task InitializeServiceControl(ScenarioContext context, int instancePort, int auditInstanceApiPort)
         {
             var instanceName = Settings.DEFAULT_SERVICE_NAME;
             typeof(ScenarioContext).GetProperty("CurrentEndpoint", BindingFlags.Static | BindingFlags.NonPublic)?.SetValue(context, instanceName);
@@ -66,11 +62,6 @@ async Task InitializeServiceControl(ScenarioContext context, int instancePort, i
             var settings = new Settings(instanceName, transportToUse.TypeName, dataStoreConfiguration.DataStoreTypeName)
             {
                 Port = instancePort,
-                PersisterSpecificSettings = new RavenDBPersisterSettings
-                {
-                    RunInMemory = true,
-                    DatabaseMaintenancePort = maintenancePort
-                },
                 ForwardErrorMessages = false,
                 TransportType = transportToUse.TypeName,
                 TransportConnectionString = transportToUse.ConnectionString,
@@ -120,6 +111,8 @@ async Task InitializeServiceControl(ScenarioContext context, int instancePort, i
                 }
             };
 
+            databaseLease.CustomizeSettings(settings);
+
             customServiceControlSettings(settings);
             SettingsPerInstance[instanceName] = settings;
 
@@ -250,7 +243,7 @@ async Task InitializeServiceControlAudit(ScenarioContext context, int instancePo
             var excludedAssemblies = new[]
             {
                 Path.GetFileName(typeof(Settings).Assembly.CodeBase), // ServiceControl.exe
-                "ServiceControl.Persistence.RavenDb.dll",
+                "ServiceControl.Persistence.RavenDb5.dll",
                 typeof(ServiceControlComponentRunner).Assembly.GetName().Name // This project
             };
 
@@ -366,6 +359,9 @@ public override async Task Stop()
                 }
             }
 
+            await databaseLease.DisposeAsync();
+            portLeases?.Dispose();
+
             hosts.Clear();
             HttpClients.Clear();
             portToHandler.Clear();
@@ -390,6 +386,10 @@ HttpClient HttpClientFactory()
         Action<Audit.Infrastructure.Settings.Settings> customServiceControlAuditSettings;
         Action<Settings> customServiceControlSettings;
 
+        static readonly PortPool portPool = new PortPool(33335);
+        DatabaseLease databaseLease = SharedDatabaseSetup.LeaseDatabase();
+        PortLease portLeases = portPool.GetLease();
+
         class ForwardingHandler : DelegatingHandler
         {
             public ForwardingHandler(Dictionary<int, HttpMessageHandler> portsToHttpMessageHandlers)

src/ServiceControl.Persistence.RavenDb/SagaAudit/SagaAuditDataStore.cs

@@ -13,12 +13,13 @@ public SagaAuditDataStore(IDocumentStore store)
             this.store = store;
         }
 
-        public async Task StoreSnapshot(SagaSnapshot sagaSnapshot)
+        public async Task<bool> StoreSnapshot(SagaSnapshot sagaSnapshot)
         {
             using (var session = store.OpenAsyncSession())
             {
                 await session.StoreAsync(sagaSnapshot);
                 await session.SaveChangesAsync();
+                return true;
             }
         }
 

src/ServiceControl.Persistence.RavenDb5/RavenDbPersistence.cs

@@ -14,6 +14,7 @@
     using ServiceControl.Persistence.MessageRedirects;
     using ServiceControl.Persistence.UnitOfWork;
     using ServiceControl.Recoverability;
+    using ServiceControl.SagaAudit;
 
     class RavenDbPersistence : IPersistence
     {
@@ -67,8 +68,12 @@ public void Configure(IServiceCollection serviceCollection)
             serviceCollection.AddSingleton<IRetryBatchesDataStore, RetryBatchesDataStore>();
             serviceCollection.AddSingleton<IRetryDocumentDataStore, RetryDocumentDataStore>();
             serviceCollection.AddSingleton<IRetryHistoryDataStore, RetryHistoryDataStore>();
-            serviceCollection.AddSingleton<ISagaAuditDataStore, NoImplementationSagaAuditDataStore>();
             serviceCollection.AddSingleton<IServiceControlSubscriptionStorage, RavenDbSubscriptionStorage>();
+
+            // Forward saga audit messages and warn in ServiceControl 5, remove in 6
+            serviceCollection.AddSingleton<ISagaAuditDataStore, SagaAuditDeprecationDataStore>();
+            serviceCollection.AddCustomCheck<SagaAuditDestinationCustomCheck>();
+            serviceCollection.AddSingleton<SagaAuditDestinationCustomCheck.State>();
         }
 
         public void ConfigureLifecycle(IServiceCollection serviceCollection)

src/ServiceControl.Persistence.RavenDb5/SagaAudit/SagaAuditDeprecationDataStore.cs

@@ -0,0 +1,25 @@
+namespace ServiceControl.Persistence.RavenDb
+{
+    using System;
+    using System.Threading.Tasks;
+    using ServiceControl.Persistence.Infrastructure;
+    using ServiceControl.SagaAudit;
+
+    class SagaAuditDeprecationDataStore : ISagaAuditDataStore
+    {
+        public SagaAuditDeprecationDataStore(SagaAuditDestinationCustomCheck.State customCheckState)
+        {
+            this.customCheckState = customCheckState;
+        }
+
+        public Task<bool> StoreSnapshot(SagaSnapshot sagaSnapshot)
+        {
+            customCheckState.Fail(sagaSnapshot.Endpoint);
+            return Task.FromResult(false);
+        }
+
+        public Task<QueryResult<SagaHistory>> GetSagaById(Guid sagaId) => Task.FromResult(QueryResult<SagaHistory>.Empty());
+
+        readonly SagaAuditDestinationCustomCheck.State customCheckState;
+    }
+}