Update to RavenDB 6.2

.github/workflows/build-db-container.yml

@@ -26,6 +26,8 @@ jobs:
           version: ${{ env.MinVerVersion }}
       - name: Log in to GitHub container registry
         run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+      - name: Install Docker arm64 emulation
+        run: docker run --privileged --rm tonistiigi/binfmt --install arm64
       - name: Build images
         env:
           TAG_NAME: ${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.number) || env.MinVerVersion }}

.github/workflows/container-integration-test.yml

@@ -57,6 +57,11 @@ jobs:
         uses: Particular/run-minver-action@v1.0.0
       - name: Log in to GitHub container registry
         run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3.3.0
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Azure Login
         if: ${{ matrix.name == 'asb' || matrix.name == 'asq' }}
         uses: azure/login@v2.2.0
@@ -101,13 +106,20 @@ jobs:
           }
           Write-Output "healthy-containers=$numHealthy" | Out-File -FilePath $Env:GITHUB_OUTPUT -Encoding utf-8 -Append
       - name: Dump Docker container list
+        if: ${{ always() }}
         run: docker ps --all
       - name: Dump Primary instance logs
+        if: ${{ always() }}
         run: docker logs service-platform-servicecontrol-1
       - name: Dump Audit instance logs
+        if: ${{ always() }}
         run: docker logs service-platform-servicecontrol-audit-1
       - name: Dump Monitoring instance logs
+        if: ${{ always() }}
         run: docker logs service-platform-servicecontrol-monitoring-1
+      - name: Dump Database logs
+        if: ${{ always() }}
+        run: docker logs service-platform-servicecontrol-db-1
       - name: Return status
         shell: pwsh
         run: |

src/Directory.Packages.props

@@ -60,7 +60,7 @@
     <PackageVersion Include="PropertyChanged.Fody" Version="4.1.0" />
     <PackageVersion Include="PropertyChanging.Fody" Version="1.30.3" />
     <PackageVersion Include="PublicApiGenerator" Version="11.1.0" />
-    <PackageVersion Include="RavenDB.Embedded" Version="5.4.201" />
+    <PackageVersion Include="RavenDB.Embedded" Version="6.2.0" />
     <PackageVersion Include="ReactiveUI.WPF" Version="20.1.1" />
     <PackageVersion Include="ServiceControl.Contracts" Version="5.0.0" />
     <PackageVersion Include="System.Configuration.ConfigurationManager" Version="8.0.1" />

src/ServiceControl.AcceptanceTests.RavenDB/Recoverability/MessageFailures/FailedMessageRetriesController.cs

@@ -10,7 +10,7 @@
 
     public class FailedMessageRetriesCountReponse
     {
-        public int Count { get; set; }
+        public long Count { get; set; }
     }
 
     [ApiController]

src/ServiceControl.Audit.Persistence.RavenDB/DatabaseSetup.cs

@@ -1,69 +1,73 @@
 namespace ServiceControl.Audit.Persistence.RavenDB
 {
+    using System;
     using System.Collections.Generic;
     using System.Threading;
     using System.Threading.Tasks;
     using Raven.Client.Documents;
     using Raven.Client.Documents.Indexes;
-    using Raven.Client.Documents.Operations;
     using Raven.Client.Documents.Operations.Expiration;
     using Raven.Client.Documents.Operations.Indexes;
     using Raven.Client.Exceptions;
-    using Raven.Client.Exceptions.Database;
     using Raven.Client.ServerWide;
     using Raven.Client.ServerWide.Operations;
+    using Raven.Client.ServerWide.Operations.Configuration;
     using ServiceControl.Audit.Persistence.RavenDB.Indexes;
     using ServiceControl.SagaAudit;
 
     class DatabaseSetup(DatabaseConfiguration configuration)
     {
         public async Task Execute(IDocumentStore documentStore, CancellationToken cancellationToken)
         {
-            try
-            {
-                await documentStore.Maintenance.ForDatabase(configuration.Name).SendAsync(new GetStatisticsOperation(), cancellationToken);
-            }
-            catch (DatabaseDoesNotExistException)
+            await CreateDatabase(documentStore, configuration.Name, cancellationToken);
+            await UpdateDatabaseSettings(documentStore, configuration.Name, cancellationToken);
+
+            await CreateIndexes(documentStore, cancellationToken);
+
+            await ConfigureExpiration(documentStore, cancellationToken);
+        }
+
+        async Task CreateDatabase(IDocumentStore documentStore, string databaseName, CancellationToken cancellationToken)
+        {
+            var dbRecord = await documentStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName), cancellationToken);
+
+            if (dbRecord is null)
             {
                 try
                 {
-                    await documentStore.Maintenance.Server
-                        .SendAsync(new CreateDatabaseOperation(new DatabaseRecord(configuration.Name)), cancellationToken);
+                    var databaseRecord = new DatabaseRecord(databaseName);
+                    databaseRecord.Settings.Add("Indexing.Auto.SearchEngineType", "Corax");
+                    databaseRecord.Settings.Add("Indexing.Static.SearchEngineType", "Corax");
+
+                    await documentStore.Maintenance.Server.SendAsync(new CreateDatabaseOperation(databaseRecord), cancellationToken);
                 }
                 catch (ConcurrencyException)
                 {
                     // The database was already created before calling CreateDatabaseOperation
                 }
             }
+        }
 
-            var indexList = new List<AbstractIndexCreationTask> {
-                new FailedAuditImportIndex(),
-                new SagaDetailsIndex()
-            };
-
-            await DeleteLegacySagaDetailsIndex(documentStore, cancellationToken);
+        async Task UpdateDatabaseSettings(IDocumentStore documentStore, string databaseName, CancellationToken cancellationToken)
+        {
+            var dbRecord = await documentStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName), cancellationToken);
 
-            if (configuration.EnableFullTextSearch)
+            if (dbRecord is null)
             {
-                indexList.Add(new MessagesViewIndexWithFullTextSearch());
-                await documentStore.Maintenance.SendAsync(new DeleteIndexOperation("MessagesViewIndex"), cancellationToken);
-            }
-            else
-            {
-                indexList.Add(new MessagesViewIndex());
-                await documentStore.Maintenance
-                    .SendAsync(new DeleteIndexOperation("MessagesViewIndexWithFullTextSearch"), cancellationToken);
+                throw new InvalidOperationException($"Database '{databaseName}' does not exist.");
             }
 
-            await IndexCreation.CreateIndexesAsync(indexList, documentStore, null, null, cancellationToken);
+            var updated = false;
 
-            var expirationConfig = new ExpirationConfiguration
-            {
-                Disabled = false,
-                DeleteFrequencyInSec = configuration.ExpirationProcessTimerInSeconds
-            };
+            updated |= dbRecord.Settings.TryAdd("Indexing.Auto.SearchEngineType", "Corax");
+            updated |= dbRecord.Settings.TryAdd("Indexing.Static.SearchEngineType", "Corax");
 
-            await documentStore.Maintenance.SendAsync(new ConfigureExpirationOperation(expirationConfig), cancellationToken);
+            if (updated)
+            {
+                await documentStore.Maintenance.ForDatabase(databaseName).SendAsync(new PutDatabaseSettingsOperation(databaseName, dbRecord.Settings), cancellationToken);
+                await documentStore.Maintenance.Server.SendAsync(new ToggleDatabasesStateOperation(databaseName, true), cancellationToken);
+                await documentStore.Maintenance.Server.SendAsync(new ToggleDatabasesStateOperation(databaseName, false), cancellationToken);
+            }
         }
 
         public static async Task DeleteLegacySagaDetailsIndex(IDocumentStore documentStore, CancellationToken cancellationToken)
@@ -81,5 +85,36 @@ public static async Task DeleteLegacySagaDetailsIndex(IDocumentStore documentSto
                 await documentStore.Maintenance.SendAsync(new DeleteIndexOperation("SagaDetailsIndex"), cancellationToken);
             }
         }
+
+        async Task CreateIndexes(IDocumentStore documentStore, CancellationToken cancellationToken)
+        {
+            await DeleteLegacySagaDetailsIndex(documentStore, cancellationToken);
+
+            List<AbstractIndexCreationTask> indexList = [new FailedAuditImportIndex(), new SagaDetailsIndex()];
+
+            if (configuration.EnableFullTextSearch)
+            {
+                indexList.Add(new MessagesViewIndexWithFullTextSearch());
+                await documentStore.Maintenance.SendAsync(new DeleteIndexOperation("MessagesViewIndex"), cancellationToken);
+            }
+            else
+            {
+                indexList.Add(new MessagesViewIndex());
+                await documentStore.Maintenance.SendAsync(new DeleteIndexOperation("MessagesViewIndexWithFullTextSearch"), cancellationToken);
+            }
+
+            await IndexCreation.CreateIndexesAsync(indexList, documentStore, null, null, cancellationToken);
+        }
+
+        async Task ConfigureExpiration(IDocumentStore documentStore, CancellationToken cancellationToken)
+        {
+            var expirationConfig = new ExpirationConfiguration
+            {
+                Disabled = false,
+                DeleteFrequencyInSec = configuration.ExpirationProcessTimerInSeconds
+            };
+
+            await documentStore.Maintenance.SendAsync(new ConfigureExpirationOperation(expirationConfig), cancellationToken);
+        }
     }
 }

src/ServiceControl.Audit.Persistence.RavenDB/RavenAuditDataStore.cs

@@ -175,9 +175,9 @@ public async Task<QueryResult<IList<AuditCount>>> QueryAuditCounts(string endpoi
                         .Take(0)
                         .ToArrayAsync();
 
-                    if (stats.LongTotalResults > 0)
+                    if (stats.TotalResults > 0)
                     {
-                        results.Add(new AuditCount { UtcDate = date, Count = stats.LongTotalResults });
+                        results.Add(new AuditCount { UtcDate = date, Count = stats.TotalResults });
                     }
                 }
             }

src/ServiceControl.Audit.Persistence.RavenDB/RavenExternalPersistenceLifecycle.cs

@@ -70,9 +70,6 @@ public async Task Initialize(CancellationToken cancellationToken = default)
 
                 var databaseSetup = new DatabaseSetup(configuration);
                 await databaseSetup.Execute(store, cancellationToken);
-
-                // Must go after the database setup, as database must exist
-                StartupChecks.EnsureSingleNodeTopology(store);
             }
             finally
             {

src/ServiceControl.Audit.Persistence/Infrastructure/QueryStatsInfo.cs

@@ -3,10 +3,10 @@ namespace ServiceControl.Audit.Auditing.MessagesView
     public struct QueryStatsInfo
     {
         public readonly string ETag;
-        public readonly int TotalCount;
-        public readonly int HighestTotalCountOfAllTheInstances;
+        public readonly long TotalCount;
+        public readonly long HighestTotalCountOfAllTheInstances;
 
-        public QueryStatsInfo(string eTag, int totalCount, int? highestTotalCountOfAllTheInstances = null)
+        public QueryStatsInfo(string eTag, long totalCount, long? highestTotalCountOfAllTheInstances = null)
         {
             ETag = eTag;
             TotalCount = totalCount;

src/ServiceControl.Audit/Infrastructure/WebApi/HttpResponseExtensions.cs

@@ -12,7 +12,7 @@ namespace ServiceControl.Audit.Infrastructure.WebApi
 
     static class HttpResponseExtensions
     {
-        public static void WithTotalCount(this HttpResponse response, int totalCount) => response.WithHeader("Total-Count", totalCount.ToString(CultureInfo.InvariantCulture));
+        public static void WithTotalCount(this HttpResponse response, long totalCount) => response.WithHeader("Total-Count", totalCount.ToString(CultureInfo.InvariantCulture));
 
         public static void WithEtag(this HttpResponse response, StringValues value) => response.Headers.ETag = value;
 
@@ -35,7 +35,7 @@ public static void WithDeterministicEtag(this HttpResponse response, string data
 
         static void WithHeader(this HttpResponse response, string header, StringValues value) => response.Headers.Append(header, value);
 
-        public static void WithPagingLinks(this HttpResponse response, PagingInfo pageInfo, int highestTotalCountOfAllInstances, int totalResults)
+        public static void WithPagingLinks(this HttpResponse response, PagingInfo pageInfo, long highestTotalCountOfAllInstances, long totalResults)
         {
             if (totalResults <= PagingInfo.DefaultPageSize)
             {
@@ -96,7 +96,7 @@ public static void WithQueryStatsAndPagingInfo(this HttpResponse response, Query
         }
 
         public static void WithPagingLinksAndTotalCount(this HttpResponse response,
-            PagingInfo pagingInfo, int totalCount, int highestTotalCountOfAllInstances = 1)
+            PagingInfo pagingInfo, long totalCount, long highestTotalCountOfAllInstances = 1)
         {
             response.WithTotalCount(totalCount);
             response.WithPagingLinks(pagingInfo, highestTotalCountOfAllInstances, totalCount);