RavenDB 5 persister code but invisible to the app and no tests

src/Persisters.Primary.Includes.props

@@ -1,5 +1,6 @@
 <Project>
 	<ItemGroup Label="Persisters">
 		<ProjectReference Include="..\ServiceControl.Persistence.RavenDb\ServiceControl.Persistence.RavenDb.csproj" ReferenceOutputAssembly="false" Private="false" />
+		<ProjectReference Include="..\ServiceControl.Persistence.RavenDb5\ServiceControl.Persistence.RavenDb5.csproj" ReferenceOutputAssembly="false" Private="false" />
 	</ItemGroup>
 </Project>

src/ServiceControl.Audit/Persistence/PersistenceHostBuilderExtensions.cs

@@ -15,7 +15,8 @@ public static IHostBuilder SetupPersistence(this IHostBuilder hostBuilder,
             {
                 var lifecycle = persistence.Configure(serviceCollection);
 
-                serviceCollection.AddHostedService(_ => new PersistenceLifecycleHostedService(lifecycle));
+                serviceCollection.AddSingleton(new PersistenceLifecycleHostedService(lifecycle));
+                serviceCollection.AddHostedService(sp => sp.GetRequiredService<PersistenceLifecycleHostedService>());
             });
 
             return hostBuilder;

src/ServiceControl.Persistence.RavenDb/CustomChecks/CheckRavenDBIndexErrors.cs

@@ -41,8 +41,8 @@ public override Task<CheckResult> PerformCheck()
             return CheckResult.Failed(message);
         }
 
-        static ILog Logger = LogManager.GetLogger<CheckRavenDBIndexLag>();
+        static readonly ILog Logger = LogManager.GetLogger<CheckRavenDBIndexLag>();
 
-        IDocumentStore store;
+        readonly IDocumentStore store;
     }
 }

src/ServiceControl.Persistence.RavenDb/ErrorMessagesDataStore.cs

@@ -136,28 +136,6 @@ SortInfo sortInfo
             }
         }
 
-        public async Task<QueryResult<IList<MessagesView>>> GetAllMessagesForEndpoint(
-            string searchTerms,
-            string receivingEndpointName,
-            PagingInfo pagingInfo,
-            SortInfo sortInfo
-            )
-        {
-            using (var session = documentStore.OpenAsyncSession())
-            {
-                var results = await session.Query<MessagesViewIndex.SortAndFilterOptions, MessagesViewIndex>()
-                    .Statistics(out var stats)
-                    .Search(x => x.Query, searchTerms)
-                    .Where(m => m.ReceivingEndpointName == receivingEndpointName)
-                    .Sort(sortInfo)
-                    .Paging(pagingInfo)
-                    .TransformWith<MessagesViewTransformer, MessagesView>()
-                    .ToListAsync();
-
-                return new QueryResult<IList<MessagesView>>(results, stats.ToQueryStatsInfo());
-            }
-        }
-
         public async Task<FailedMessage> FailedMessageFetch(string failedMessageId)
         {
             using (var session = documentStore.OpenAsyncSession())

src/ServiceControl.Persistence.RavenDb/EventLogDataStore.cs

@@ -29,7 +29,9 @@ public async Task Add(EventLogItem logItem)
         {
             using (var session = documentStore.OpenAsyncSession())
             {
-                var results = await session.Query<EventLogItem>().Statistics(out var stats)
+                var results = await session
+                    .Query<EventLogItem>()
+                    .Statistics(out var stats)
                     .OrderByDescending(p => p.RaisedAt)
                     .Paging(pagingInfo)
                     .ToListAsync();

src/ServiceControl.Persistence.RavenDb/RavenDbCustomCheckDataStore.cs

@@ -34,7 +34,7 @@ public async Task<CheckStateChange> UpdateCustomCheckStatus(CustomCheckDetail de
                     {
                         customCheck = new CustomCheck
                         {
-                            Id = id
+                            Id = MakeId(id)
                         };
                     }
 
@@ -54,6 +54,11 @@ public async Task<CheckStateChange> UpdateCustomCheckStatus(CustomCheckDetail de
             return status;
         }
 
+        static string MakeId(Guid id)
+        {
+            return $"CustomChecks/{id}";
+        }
+
         public async Task<QueryResult<IList<CustomCheck>>> GetStats(PagingInfo paging, string status = null)
         {
             using (var session = store.OpenAsyncSession())
@@ -73,7 +78,7 @@ public async Task<QueryResult<IList<CustomCheck>>> GetStats(PagingInfo paging, s
 
         public async Task DeleteCustomCheck(Guid id)
         {
-            await store.AsyncDatabaseCommands.DeleteAsync(store.Conventions.DefaultFindFullDocumentKeyFromNonStringIdentifier(id, typeof(CustomCheck), false), null);
+            await store.AsyncDatabaseCommands.DeleteAsync(MakeId(id), null);
         }
 
         public async Task<int> GetNumberOfFailedChecks()

src/ServiceControl.Persistence.RavenDb/RavenDbPersistence.cs

@@ -78,10 +78,7 @@ public void Configure(IServiceCollection serviceCollection)
             serviceCollection.AddSingleton<IServiceControlSubscriptionStorage, RavenDbSubscriptionStorage>();
         }
 
-        public IPersistenceLifecycle CreateLifecycle()
-        {
-            return new RavenDbPersistenceLifecycle(ravenStartup, documentStore);
-        }
+        public void ConfigureLifecycle(IServiceCollection serviceCollection) => serviceCollection.AddSingleton<IPersistenceLifecycle>(new RavenDbPersistenceLifecycle(ravenStartup, documentStore));
 
         public IPersistenceInstaller CreateInstaller()
         {

src/ServiceControl.Persistence.RavenDb5/.editorconfig

@@ -0,0 +1,4 @@
+[*.cs]
+
+# Justification: ServiceControl app has no synchronization context
+dotnet_diagnostic.CA2007.severity = none

src/ServiceControl.Persistence.RavenDb5/Chunker.cs

@@ -0,0 +1,40 @@
+namespace ServiceControl.Infrastructure.RavenDB
+{
+    using System;
+    using System.Threading;
+
+    static class Chunker
+    {
+        public static int ExecuteInChunks<T1, T2>(int total, Func<T1, T2, int, int, int> action, T1 t1, T2 t2, CancellationToken cancellationToken = default)
+        {
+            if (total == 0)
+            {
+                return 0;
+            }
+
+            if (total < CHUNK_SIZE)
+            {
+                return action(t1, t2, 0, total - 1);
+            }
+
+            int start = 0, end = CHUNK_SIZE - 1;
+            var chunkCount = 0;
+            do
+            {
+                chunkCount += action(t1, t2, start, end);
+
+                start = end + 1;
+                end += CHUNK_SIZE;
+                if (end >= total)
+                {
+                    end = total - 1;
+                }
+            }
+            while (start < total && !cancellationToken.IsCancellationRequested);
+
+            return chunkCount;
+        }
+
+        const int CHUNK_SIZE = 500;
+    }
+}

src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckFreeDiskSpace.cs

@@ -0,0 +1,72 @@
+namespace ServiceControl.Operations
+{
+    using System;
+    using System.IO;
+    using System.Threading.Tasks;
+    using NServiceBus.CustomChecks;
+    using NServiceBus.Logging;
+    using Persistence.RavenDb;
+
+    class CheckFreeDiskSpace : CustomCheck
+    {
+        public CheckFreeDiskSpace(RavenDBPersisterSettings settings) : base("ServiceControl database", "Storage space", TimeSpan.FromMinutes(5))
+        {
+            dataPath = settings.DatabasePath;
+            percentageThreshold = settings.DataSpaceRemainingThreshold;
+
+            Logger.Debug($"Check ServiceControl data drive space remaining custom check starting. Threshold {percentageThreshold:P0}");
+        }
+
+        public override Task<CheckResult> PerformCheck()
+        {
+            var dataPathRoot = Path.GetPathRoot(dataPath);
+
+            if (dataPathRoot == null)
+            {
+                throw new Exception($"Unable to find the root of the data path {dataPath}");
+            }
+
+            var dataDriveInfo = new DriveInfo(dataPathRoot);
+            var availableFreeSpace = (decimal)dataDriveInfo.AvailableFreeSpace;
+            var totalSpace = (decimal)dataDriveInfo.TotalSize;
+
+            var percentRemaining = (decimal)dataDriveInfo.AvailableFreeSpace / dataDriveInfo.TotalSize;
+
+            if (Logger.IsDebugEnabled)
+            {
+                Logger.Debug($"Free space: {availableFreeSpace:N0}B | Total: {totalSpace:N0}B | Percent remaining {percentRemaining:P1}");
+            }
+
+            return percentRemaining > percentageThreshold
+                ? CheckResult.Pass
+                : CheckResult.Failed($"{percentRemaining:P0} disk space remaining on data drive '{dataDriveInfo.VolumeLabel} ({dataDriveInfo.RootDirectory})' on '{Environment.MachineName}'.");
+        }
+
+        public static void Validate(RavenDBPersisterSettings settings)
+        {
+            var threshold = settings.DataSpaceRemainingThreshold;
+
+            string message;
+
+            if (threshold < 0)
+            {
+                message = $"{RavenDbPersistenceConfiguration.DataSpaceRemainingThresholdKey} is invalid, minimum value is 0.";
+                Logger.Fatal(message);
+                throw new Exception(message);
+            }
+
+            if (threshold > 100)
+            {
+                message = $"{RavenDbPersistenceConfiguration.DataSpaceRemainingThresholdKey} is invalid, maximum value is 100.";
+                Logger.Fatal(message);
+                throw new Exception(message);
+            }
+        }
+
+        readonly string dataPath;
+        readonly decimal percentageThreshold;
+
+        public const int DataSpaceRemainingThresholdDefault = 20;
+        static readonly ILog Logger = LogManager.GetLogger(typeof(CheckFreeDiskSpace));
+    }
+}

src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckMinimumStorageRequiredForIngestion.cs

@@ -0,0 +1,90 @@
+namespace ServiceControl.Operations
+{
+    using System;
+    using System.IO;
+    using System.Threading.Tasks;
+    using NServiceBus.CustomChecks;
+    using NServiceBus.Logging;
+    using Persistence.RavenDb;
+    using ServiceControl.Persistence;
+
+    class CheckMinimumStorageRequiredForIngestion : CustomCheck
+    {
+        public CheckMinimumStorageRequiredForIngestion(
+            MinimumRequiredStorageState stateHolder,
+            RavenDBPersisterSettings settings)
+            : base("Message Ingestion Process", "ServiceControl Health", TimeSpan.FromSeconds(5))
+        {
+            this.stateHolder = stateHolder;
+            this.settings = settings;
+
+            dataPathRoot = Path.GetPathRoot(settings.DatabasePath);
+        }
+
+        public override Task<CheckResult> PerformCheck()
+        {
+            percentageThreshold = settings.MinimumStorageLeftRequiredForIngestion / 100m;
+
+            if (dataPathRoot == null)
+            {
+                stateHolder.CanIngestMore = true;
+                return SuccessResult;
+            }
+
+            Logger.Debug($"Check ServiceControl data drive space starting. Threshold {percentageThreshold:P0}");
+
+            var dataDriveInfo = new DriveInfo(dataPathRoot);
+            var availableFreeSpace = (decimal)dataDriveInfo.AvailableFreeSpace;
+            var totalSpace = (decimal)dataDriveInfo.TotalSize;
+
+            var percentRemaining = (decimal)dataDriveInfo.AvailableFreeSpace / dataDriveInfo.TotalSize;
+
+            if (Logger.IsDebugEnabled)
+            {
+                Logger.Debug($"Free space: {availableFreeSpace:N0}B | Total: {totalSpace:N0}B | Percent remaining {percentRemaining:P1}");
+            }
+
+            if (percentRemaining > percentageThreshold)
+            {
+                stateHolder.CanIngestMore = true;
+                return SuccessResult;
+            }
+
+            var message = $"Error message ingestion stopped! {percentRemaining:P0} disk space remaining on data drive '{dataDriveInfo.VolumeLabel} ({dataDriveInfo.RootDirectory})' on '{Environment.MachineName}'. This is less than {percentageThreshold}% - the minimal required space configured. The threshold can be set using the {RavenBootstrapper.MinimumStorageLeftRequiredForIngestionKey} configuration setting.";
+            Logger.Warn(message);
+            stateHolder.CanIngestMore = false;
+            return CheckResult.Failed(message);
+        }
+
+        public static void Validate(RavenDBPersisterSettings settings)
+        {
+            int threshold = settings.MinimumStorageLeftRequiredForIngestion;
+
+            string message;
+            if (threshold < 0)
+            {
+                message = $"{RavenBootstrapper.MinimumStorageLeftRequiredForIngestionKey} is invalid, minimum value is 0.";
+                Logger.Fatal(message);
+                throw new Exception(message);
+            }
+
+            if (threshold > 100)
+            {
+                message = $"{RavenBootstrapper.MinimumStorageLeftRequiredForIngestionKey} is invalid, maximum value is 100.";
+                Logger.Fatal(message);
+                throw new Exception(message);
+            }
+        }
+
+        public const int MinimumStorageLeftRequiredForIngestionDefault = 5;
+
+        readonly MinimumRequiredStorageState stateHolder;
+        readonly RavenDBPersisterSettings settings;
+        readonly string dataPathRoot;
+
+        decimal percentageThreshold;
+
+        static readonly Task<CheckResult> SuccessResult = Task.FromResult(CheckResult.Pass);
+        static readonly ILog Logger = LogManager.GetLogger(typeof(CheckMinimumStorageRequiredForIngestion));
+    }
+}

src/ServiceControl.Persistence.RavenDb5/CustomChecks/CheckRavenDBIndexErrors.cs

@@ -0,0 +1,50 @@
+namespace ServiceControl
+{
+    using System;
+    using System.Text;
+    using System.Threading.Tasks;
+    using NServiceBus.CustomChecks;
+    using NServiceBus.Logging;
+    using Raven.Client.Documents;
+    using Raven.Client.Documents.Operations.Indexes;
+
+    class CheckRavenDBIndexErrors : CustomCheck
+    {
+        public CheckRavenDBIndexErrors(IDocumentStore store)
+            : base("Error Database Index Errors", "ServiceControl Health", TimeSpan.FromMinutes(5))
+        {
+            this.store = store;
+        }
+
+        public override Task<CheckResult> PerformCheck()
+        {
+            var indexErrors = store.Maintenance.Send(new GetIndexErrorsOperation());
+
+            if (indexErrors.Length == 0)
+            {
+                return CheckResult.Pass;
+            }
+
+            var text = new StringBuilder();
+            text.AppendLine("Detected RavenDB index errors, please start maintenance mode and resolve the following issues:");
+
+            foreach (var indexError in indexErrors)
+            {
+                foreach (var indexingError in indexError.Errors)
+                {
+                    text.AppendLine($"- Index [{indexError.Name}] error: {indexError.Name} (Action: {indexingError.Action},  Doc: {indexingError.Document}, At: {indexingError.Timestamp})");
+                }
+            }
+
+            text.AppendLine().AppendLine("See: https://docs.particular.net/search?q=servicecontrol+troubleshooting");
+
+            var message = text.ToString();
+            Logger.Error(message);
+            return CheckResult.Failed(message);
+        }
+
+        static readonly ILog Logger = LogManager.GetLogger<CheckRavenDBIndexLag>();
+
+        readonly IDocumentStore store;
+    }
+}