From 209f2e37565c489d662a461241b53e87c16ade70 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 19 Feb 2026 14:22:48 -0500 Subject: [PATCH] DuckDB checkpoint optimization and timing accuracy fix - Add checkpoint_threshold=1GB to connection string to prevent auto-checkpoint stalls - Add manual CHECKPOINT after each collection cycle during idle time - Change collector execution from parallel Task.WhenAll to sequential per-server - Fix using var timing bug across all 16 collector files: change to explicit using blocks so appender Dispose (flush + connection close) is captured inside the DuckDB stopwatch, giving accurate timing in collection_log Co-Authored-By: Claude Opus 4.6 --- Lite/Database/DuckDbInitializer.cs | 24 ++- Lite/Services/CollectionBackgroundService.cs | 4 + ...teCollectorService.BlockedProcessReport.cs | 123 ++++++++------- Lite/Services/RemoteCollectorService.Cpu.cs | 48 +++--- .../RemoteCollectorService.Deadlocks.cs | 39 ++--- .../Services/RemoteCollectorService.FileIo.cs | 80 +++++----- .../Services/RemoteCollectorService.Memory.cs | 79 +++++----- .../RemoteCollectorService.MemoryGrants.cs | 53 ++++--- .../RemoteCollectorService.Perfmon.cs | 60 ++++---- .../RemoteCollectorService.ProcedureStats.cs | 124 +++++++-------- .../RemoteCollectorService.QuerySnapshots.cs | 68 +++++---- .../RemoteCollectorService.QueryStats.cs | 144 +++++++++--------- .../RemoteCollectorService.QueryStore.cs | 102 +++++++------ .../RemoteCollectorService.RunningJobs.cs | 45 +++--- .../Services/RemoteCollectorService.TempDb.cs | 41 ++--- .../RemoteCollectorService.WaitStats.cs | 54 ++++--- .../RemoteCollectorService.WaitingTasks.cs | 54 ++++--- Lite/Services/RemoteCollectorService.cs | 27 +++- 18 files changed, 640 insertions(+), 529 deletions(-) diff --git a/Lite/Database/DuckDbInitializer.cs b/Lite/Database/DuckDbInitializer.cs index 13e7874b..77a441fd 100644 --- a/Lite/Database/DuckDbInitializer.cs +++ b/Lite/Database/DuckDbInitializer.cs @@ -29,8 +29,10 @@ public DuckDbInitializer(string databasePath, ILogger? logger /// /// Gets the connection string for the DuckDB database. + /// Disables automatic WAL checkpoints to prevent 2-3s stop-the-world stalls + /// during collector writes. Manual CHECKPOINT runs between collection cycles instead. /// - public string ConnectionString => $"Data Source={_databasePath}"; + public string ConnectionString => $"Data Source={_databasePath};checkpoint_threshold=1GB"; /// /// Ensures the database exists and all tables are created. @@ -408,6 +410,26 @@ public DuckDBConnection CreateConnection() return new DuckDBConnection(ConnectionString); } + /// + /// Runs a manual WAL checkpoint. Call this between collection cycles + /// to flush the WAL during idle time instead of during collector writes. + /// + public async Task CheckpointAsync() + { + try + { + using var connection = CreateConnection(); + await connection.OpenAsync(); + using var command = connection.CreateCommand(); + command.CommandText = "CHECKPOINT"; + await command.ExecuteNonQueryAsync(); + } + catch (Exception ex) + { + _logger?.LogDebug(ex, "Manual checkpoint failed (non-critical)"); + } + } + /// /// Executes a non-query SQL statement. /// diff --git a/Lite/Services/CollectionBackgroundService.cs b/Lite/Services/CollectionBackgroundService.cs index dd15d340..d52e1429 100644 --- a/Lite/Services/CollectionBackgroundService.cs +++ b/Lite/Services/CollectionBackgroundService.cs @@ -85,6 +85,10 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) IsCollecting = true; await _collectorService.RunDueCollectorsAsync(stoppingToken); LastCollectionTime = DateTime.UtcNow; + + /* Flush WAL during idle time instead of letting auto-checkpoint + stall collectors mid-write with 2-3s stop-the-world pauses */ + await _collectorService.CheckpointAsync(); } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { diff --git a/Lite/Services/RemoteCollectorService.BlockedProcessReport.cs b/Lite/Services/RemoteCollectorService.BlockedProcessReport.cs index 7c6da3d3..f81df834 100644 --- a/Lite/Services/RemoteCollectorService.BlockedProcessReport.cs +++ b/Lite/Services/RemoteCollectorService.BlockedProcessReport.cs @@ -390,71 +390,76 @@ as it lingers in the ring buffer across collection cycles. */ _lastSqlMs = sqlSw.ElapsedMilliseconds; var duckSw = Stopwatch.StartNew(); - using var duckConnection = _duckDb.CreateConnection(); - await duckConnection.OpenAsync(cancellationToken); - using var appender = duckConnection.CreateAppender("blocked_process_reports"); - - while (await reader.ReadAsync(cancellationToken)) + using (var duckConnection = _duckDb.CreateConnection()) { - var eventTime = reader.IsDBNull(0) ? (DateTime?)null : reader.GetDateTime(0); - var reportXml = reader.IsDBNull(1) ? null : reader.GetString(1); - - if (string.IsNullOrEmpty(reportXml)) - { - continue; - } + await duckConnection.OpenAsync(cancellationToken); - /* Parse the blocked process report XML in C# */ - var parsed = ParseBlockedProcessReportXml(reportXml, eventTime); - if (parsed == null) + using (var appender = duckConnection.CreateAppender("blocked_process_reports")) { - continue; + while (await reader.ReadAsync(cancellationToken)) + { + var eventTime = reader.IsDBNull(0) ? (DateTime?)null : reader.GetDateTime(0); + var reportXml = reader.IsDBNull(1) ? null : reader.GetString(1); + + if (string.IsNullOrEmpty(reportXml)) + { + continue; + } + + /* Parse the blocked process report XML in C# */ + var parsed = ParseBlockedProcessReportXml(reportXml, eventTime); + if (parsed == null) + { + continue; + } + + var row = appender.CreateRow(); + row.AppendValue(GenerateCollectionId()) + .AppendValue(collectionTime) + .AppendValue(serverId) + .AppendValue(server.ServerName) + .AppendValue(parsed.EventTime) + .AppendValue(parsed.DatabaseName) + .AppendValue(parsed.BlockedSpid) + .AppendValue(parsed.BlockedEcid) + .AppendValue(parsed.BlockingSpid) + .AppendValue(parsed.BlockingEcid) + .AppendValue(parsed.WaitTimeMs) + .AppendValue(parsed.WaitResource) + .AppendValue(parsed.LockMode) + .AppendValue(parsed.BlockedStatus) + .AppendValue(parsed.BlockedIsolationLevel) + .AppendValue(parsed.BlockedLogUsed) + .AppendValue(parsed.BlockedTransactionCount) + .AppendValue(parsed.BlockedClientApp) + .AppendValue(parsed.BlockedHostName) + .AppendValue(parsed.BlockedLoginName) + .AppendValue(parsed.BlockedSqlText) + .AppendValue(parsed.BlockingStatus) + .AppendValue(parsed.BlockingIsolationLevel) + .AppendValue(parsed.BlockingClientApp) + .AppendValue(parsed.BlockingHostName) + .AppendValue(parsed.BlockingLoginName) + .AppendValue(parsed.BlockingSqlText) + .AppendValue(parsed.BlockedTransactionName) + .AppendValue(parsed.BlockingTransactionName) + .AppendValue(parsed.BlockedLastTranStarted) + .AppendValue(parsed.BlockingLastTranStarted) + .AppendValue(parsed.BlockedLastBatchStarted) + .AppendValue(parsed.BlockingLastBatchStarted) + .AppendValue(parsed.BlockedLastBatchCompleted) + .AppendValue(parsed.BlockingLastBatchCompleted) + .AppendValue(parsed.BlockedPriority) + .AppendValue(parsed.BlockingPriority) + .AppendValue(reportXml) + .EndRow(); + + rowsCollected++; + } } - - var row = appender.CreateRow(); - row.AppendValue(GenerateCollectionId()) - .AppendValue(collectionTime) - .AppendValue(serverId) - .AppendValue(server.ServerName) - .AppendValue(parsed.EventTime) - .AppendValue(parsed.DatabaseName) - .AppendValue(parsed.BlockedSpid) - .AppendValue(parsed.BlockedEcid) - .AppendValue(parsed.BlockingSpid) - .AppendValue(parsed.BlockingEcid) - .AppendValue(parsed.WaitTimeMs) - .AppendValue(parsed.WaitResource) - .AppendValue(parsed.LockMode) - .AppendValue(parsed.BlockedStatus) - .AppendValue(parsed.BlockedIsolationLevel) - .AppendValue(parsed.BlockedLogUsed) - .AppendValue(parsed.BlockedTransactionCount) - .AppendValue(parsed.BlockedClientApp) - .AppendValue(parsed.BlockedHostName) - .AppendValue(parsed.BlockedLoginName) - .AppendValue(parsed.BlockedSqlText) - .AppendValue(parsed.BlockingStatus) - .AppendValue(parsed.BlockingIsolationLevel) - .AppendValue(parsed.BlockingClientApp) - .AppendValue(parsed.BlockingHostName) - .AppendValue(parsed.BlockingLoginName) - .AppendValue(parsed.BlockingSqlText) - .AppendValue(parsed.BlockedTransactionName) - .AppendValue(parsed.BlockingTransactionName) - .AppendValue(parsed.BlockedLastTranStarted) - .AppendValue(parsed.BlockingLastTranStarted) - .AppendValue(parsed.BlockedLastBatchStarted) - .AppendValue(parsed.BlockingLastBatchStarted) - .AppendValue(parsed.BlockedLastBatchCompleted) - .AppendValue(parsed.BlockingLastBatchCompleted) - .AppendValue(parsed.BlockedPriority) - .AppendValue(parsed.BlockingPriority) - .AppendValue(reportXml) - .EndRow(); - - rowsCollected++; } + duckSw.Stop(); _lastDuckDbMs = duckSw.ElapsedMilliseconds; } diff --git a/Lite/Services/RemoteCollectorService.Cpu.cs b/Lite/Services/RemoteCollectorService.Cpu.cs index 09fd6df5..800fb01c 100644 --- a/Lite/Services/RemoteCollectorService.Cpu.cs +++ b/Lite/Services/RemoteCollectorService.Cpu.cs @@ -128,30 +128,34 @@ drs.end_time DESC /* Insert into DuckDB using Appender for bulk performance */ var duckSw = Stopwatch.StartNew(); - using var duckConnection = _duckDb.CreateConnection(); - await duckConnection.OpenAsync(cancellationToken); - using var appender = duckConnection.CreateAppender("cpu_utilization_stats"); - - while (await reader.ReadAsync(cancellationToken)) + using (var duckConnection = _duckDb.CreateConnection()) { - var sampleTime = reader.GetDateTime(0); - - /* Client-side dedup for ring buffer (computed sample_time can't be filtered in SQL) */ - if (!isAzureSqlDb && lastSampleTime.HasValue && sampleTime <= lastSampleTime.Value) - continue; - - var row = appender.CreateRow(); - row.AppendValue(GenerateCollectionId()) - .AppendValue(collectionTime) - .AppendValue(serverId) - .AppendValue(server.ServerName) - .AppendValue(sampleTime) - .AppendValue(reader.IsDBNull(1) ? 0 : reader.GetInt32(1)) - .AppendValue(reader.IsDBNull(2) ? 0 : reader.GetInt32(2)) - .EndRow(); - - rowsCollected++; + await duckConnection.OpenAsync(cancellationToken); + + using (var appender = duckConnection.CreateAppender("cpu_utilization_stats")) + { + while (await reader.ReadAsync(cancellationToken)) + { + var sampleTime = reader.GetDateTime(0); + + /* Client-side dedup for ring buffer (computed sample_time can't be filtered in SQL) */ + if (!isAzureSqlDb && lastSampleTime.HasValue && sampleTime <= lastSampleTime.Value) + continue; + + var row = appender.CreateRow(); + row.AppendValue(GenerateCollectionId()) + .AppendValue(collectionTime) + .AppendValue(serverId) + .AppendValue(server.ServerName) + .AppendValue(sampleTime) + .AppendValue(reader.IsDBNull(1) ? 0 : reader.GetInt32(1)) + .AppendValue(reader.IsDBNull(2) ? 0 : reader.GetInt32(2)) + .EndRow(); + + rowsCollected++; + } + } } duckSw.Stop(); diff --git a/Lite/Services/RemoteCollectorService.Deadlocks.cs b/Lite/Services/RemoteCollectorService.Deadlocks.cs index 2ff5e9f9..5fc79330 100644 --- a/Lite/Services/RemoteCollectorService.Deadlocks.cs +++ b/Lite/Services/RemoteCollectorService.Deadlocks.cs @@ -388,26 +388,31 @@ and was previously misattributed as DuckDB time. */ } var duckSw = Stopwatch.StartNew(); - using var duckConnection = _duckDb.CreateConnection(); - await duckConnection.OpenAsync(cancellationToken); - using var appender = duckConnection.CreateAppender("deadlocks"); - - foreach (var (deadlockTime, victimProcessId, victimSqlText, graphXml) in deadlockRows) + using (var duckConnection = _duckDb.CreateConnection()) { - var row = appender.CreateRow(); - row.AppendValue(GenerateCollectionId()) - .AppendValue(collectionTime) - .AppendValue(serverId) - .AppendValue(server.ServerName) - .AppendValue(deadlockTime) - .AppendValue(victimProcessId) - .AppendValue(victimSqlText) - .AppendValue(graphXml) - .EndRow(); - - rowsCollected++; + await duckConnection.OpenAsync(cancellationToken); + + using (var appender = duckConnection.CreateAppender("deadlocks")) + { + foreach (var (deadlockTime, victimProcessId, victimSqlText, graphXml) in deadlockRows) + { + var row = appender.CreateRow(); + row.AppendValue(GenerateCollectionId()) + .AppendValue(collectionTime) + .AppendValue(serverId) + .AppendValue(server.ServerName) + .AppendValue(deadlockTime) + .AppendValue(victimProcessId) + .AppendValue(victimSqlText) + .AppendValue(graphXml) + .EndRow(); + + rowsCollected++; + } + } } + duckSw.Stop(); _lastDuckDbMs = duckSw.ElapsedMilliseconds; } diff --git a/Lite/Services/RemoteCollectorService.FileIo.cs b/Lite/Services/RemoteCollectorService.FileIo.cs index e829e143..1681624a 100644 --- a/Lite/Services/RemoteCollectorService.FileIo.cs +++ b/Lite/Services/RemoteCollectorService.FileIo.cs @@ -120,46 +120,50 @@ AND vfs.database_id < 32761 /* Insert into DuckDB with delta calculations */ var duckSw = Stopwatch.StartNew(); - using var duckConnection = _duckDb.CreateConnection(); - await duckConnection.OpenAsync(cancellationToken); - using var appender = duckConnection.CreateAppender("file_io_stats"); - - foreach (var stat in fileStats) + using (var duckConnection = _duckDb.CreateConnection()) { - var deltaKey = $"{stat.DatabaseId}_{stat.FileId}"; - var deltaReads = _deltaCalculator.CalculateDelta(serverId, "file_io_reads", deltaKey, stat.NumOfReads); - var deltaWrites = _deltaCalculator.CalculateDelta(serverId, "file_io_writes", deltaKey, stat.NumOfWrites); - var deltaReadBytes = _deltaCalculator.CalculateDelta(serverId, "file_io_read_bytes", deltaKey, stat.ReadBytes); - var deltaWriteBytes = _deltaCalculator.CalculateDelta(serverId, "file_io_write_bytes", deltaKey, stat.WriteBytes); - var deltaStallReadMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_read", deltaKey, stat.IoStallReadMs); - var deltaStallWriteMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_write", deltaKey, stat.IoStallWriteMs); - - var row = appender.CreateRow(); - row.AppendValue(GenerateCollectionId()) - .AppendValue(collectionTime) - .AppendValue(serverId) - .AppendValue(server.ServerName) - .AppendValue(stat.DatabaseName) - .AppendValue(stat.FileName) - .AppendValue(stat.FileType) - .AppendValue(stat.PhysicalName) - .AppendValue(stat.SizeMb) - .AppendValue(stat.NumOfReads) - .AppendValue(stat.NumOfWrites) - .AppendValue(stat.ReadBytes) - .AppendValue(stat.WriteBytes) - .AppendValue(stat.IoStallReadMs) - .AppendValue(stat.IoStallWriteMs) - .AppendValue(deltaReads) - .AppendValue(deltaWrites) - .AppendValue(deltaReadBytes) - .AppendValue(deltaWriteBytes) - .AppendValue(deltaStallReadMs) - .AppendValue(deltaStallWriteMs) - .EndRow(); - - rowsCollected++; + await duckConnection.OpenAsync(cancellationToken); + + using (var appender = duckConnection.CreateAppender("file_io_stats")) + { + foreach (var stat in fileStats) + { + var deltaKey = $"{stat.DatabaseId}_{stat.FileId}"; + var deltaReads = _deltaCalculator.CalculateDelta(serverId, "file_io_reads", deltaKey, stat.NumOfReads); + var deltaWrites = _deltaCalculator.CalculateDelta(serverId, "file_io_writes", deltaKey, stat.NumOfWrites); + var deltaReadBytes = _deltaCalculator.CalculateDelta(serverId, "file_io_read_bytes", deltaKey, stat.ReadBytes); + var deltaWriteBytes = _deltaCalculator.CalculateDelta(serverId, "file_io_write_bytes", deltaKey, stat.WriteBytes); + var deltaStallReadMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_read", deltaKey, stat.IoStallReadMs); + var deltaStallWriteMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_write", deltaKey, stat.IoStallWriteMs); + + var row = appender.CreateRow(); + row.AppendValue(GenerateCollectionId()) + .AppendValue(collectionTime) + .AppendValue(serverId) + .AppendValue(server.ServerName) + .AppendValue(stat.DatabaseName) + .AppendValue(stat.FileName) + .AppendValue(stat.FileType) + .AppendValue(stat.PhysicalName) + .AppendValue(stat.SizeMb) + .AppendValue(stat.NumOfReads) + .AppendValue(stat.NumOfWrites) + .AppendValue(stat.ReadBytes) + .AppendValue(stat.WriteBytes) + .AppendValue(stat.IoStallReadMs) + .AppendValue(stat.IoStallWriteMs) + .AppendValue(deltaReads) + .AppendValue(deltaWrites) + .AppendValue(deltaReadBytes) + .AppendValue(deltaWriteBytes) + .AppendValue(deltaStallReadMs) + .AppendValue(deltaStallWriteMs) + .EndRow(); + + rowsCollected++; + } + } } duckSw.Stop(); diff --git a/Lite/Services/RemoteCollectorService.Memory.cs b/Lite/Services/RemoteCollectorService.Memory.cs index 4531413b..e9ac7e90 100644 --- a/Lite/Services/RemoteCollectorService.Memory.cs +++ b/Lite/Services/RemoteCollectorService.Memory.cs @@ -146,26 +146,31 @@ FROM sys.dm_os_performance_counters /* Insert into DuckDB using Appender */ var duckSw = Stopwatch.StartNew(); - using var duckConnection = _duckDb.CreateConnection(); - await duckConnection.OpenAsync(cancellationToken); - - using var appender = duckConnection.CreateAppender("memory_stats"); - var row = appender.CreateRow(); - row.AppendValue(GenerateCollectionId()) - .AppendValue(collectionTime) - .AppendValue(serverId) - .AppendValue(server.ServerName) - .AppendValue(totalPhysicalMb) - .AppendValue(availablePhysicalMb) - .AppendValue(totalPageFileMb) - .AppendValue(availablePageFileMb) - .AppendValue(systemMemoryState) - .AppendValue(sqlMemoryModel) - .AppendValue(targetServerMemoryMb) - .AppendValue(totalServerMemoryMb) - .AppendValue(bufferPoolMb) - .AppendValue(planCacheMb) - .EndRow(); + + using (var duckConnection = _duckDb.CreateConnection()) + { + await duckConnection.OpenAsync(cancellationToken); + + using (var appender = duckConnection.CreateAppender("memory_stats")) + { + var row = appender.CreateRow(); + row.AppendValue(GenerateCollectionId()) + .AppendValue(collectionTime) + .AppendValue(serverId) + .AppendValue(server.ServerName) + .AppendValue(totalPhysicalMb) + .AppendValue(availablePhysicalMb) + .AppendValue(totalPageFileMb) + .AppendValue(availablePageFileMb) + .AppendValue(systemMemoryState) + .AppendValue(sqlMemoryModel) + .AppendValue(targetServerMemoryMb) + .AppendValue(totalServerMemoryMb) + .AppendValue(bufferPoolMb) + .AppendValue(planCacheMb) + .EndRow(); + } + } duckSw.Stop(); _lastSqlMs = sqlSw.ElapsedMilliseconds; @@ -211,23 +216,27 @@ ORDER BY /* Insert into DuckDB */ var duckSw = Stopwatch.StartNew(); - using var duckConnection = _duckDb.CreateConnection(); - await duckConnection.OpenAsync(cancellationToken); - - using var appender = duckConnection.CreateAppender("memory_clerks"); - while (await reader.ReadAsync(cancellationToken)) + using (var duckConnection = _duckDb.CreateConnection()) { - var row = appender.CreateRow(); - row.AppendValue(GenerateCollectionId()) - .AppendValue(collectionTime) - .AppendValue(serverId) - .AppendValue(server.ServerName) - .AppendValue(reader.GetString(0)) - .AppendValue(reader.GetDecimal(1)) - .EndRow(); - - rowsCollected++; + await duckConnection.OpenAsync(cancellationToken); + + using (var appender = duckConnection.CreateAppender("memory_clerks")) + { + while (await reader.ReadAsync(cancellationToken)) + { + var row = appender.CreateRow(); + row.AppendValue(GenerateCollectionId()) + .AppendValue(collectionTime) + .AppendValue(serverId) + .AppendValue(server.ServerName) + .AppendValue(reader.GetString(0)) + .AppendValue(reader.GetDecimal(1)) + .EndRow(); + + rowsCollected++; + } + } } duckSw.Stop(); diff --git a/Lite/Services/RemoteCollectorService.MemoryGrants.cs b/Lite/Services/RemoteCollectorService.MemoryGrants.cs index fd455f18..dfaeab5e 100644 --- a/Lite/Services/RemoteCollectorService.MemoryGrants.cs +++ b/Lite/Services/RemoteCollectorService.MemoryGrants.cs @@ -84,32 +84,37 @@ WHERE mg.session_id <> @@SPID sqlSw.Stop(); var duckSw = Stopwatch.StartNew(); - using var duckConnection = _duckDb.CreateConnection(); - await duckConnection.OpenAsync(cancellationToken); - using var appender = duckConnection.CreateAppender("memory_grant_stats"); - foreach (var r in rows) + using (var duckConnection = _duckDb.CreateConnection()) { - var row = appender.CreateRow(); - row.AppendValue(GenerateCollectionId()) - .AppendValue(collectionTime) - .AppendValue(serverId) - .AppendValue(server.ServerName) - .AppendValue(r.SessionId) - .AppendValue(r.DatabaseName) - .AppendValue(r.QueryText) - .AppendValue(r.RequestedMb) - .AppendValue(r.GrantedMb) - .AppendValue(r.UsedMb) - .AppendValue(r.MaxUsedMb) - .AppendValue(r.IdealMb) - .AppendValue(r.RequiredMb) - .AppendValue(r.WaitTimeMs) - .AppendValue(r.IsSmall) - .AppendValue(r.Dop) - .AppendValue(r.QueryCost) - .EndRow(); - rowsCollected++; + await duckConnection.OpenAsync(cancellationToken); + + using (var appender = duckConnection.CreateAppender("memory_grant_stats")) + { + foreach (var r in rows) + { + var row = appender.CreateRow(); + row.AppendValue(GenerateCollectionId()) + .AppendValue(collectionTime) + .AppendValue(serverId) + .AppendValue(server.ServerName) + .AppendValue(r.SessionId) + .AppendValue(r.DatabaseName) + .AppendValue(r.QueryText) + .AppendValue(r.RequestedMb) + .AppendValue(r.GrantedMb) + .AppendValue(r.UsedMb) + .AppendValue(r.MaxUsedMb) + .AppendValue(r.IdealMb) + .AppendValue(r.RequiredMb) + .AppendValue(r.WaitTimeMs) + .AppendValue(r.IsSmall) + .AppendValue(r.Dop) + .AppendValue(r.QueryCost) + .EndRow(); + rowsCollected++; + } + } } duckSw.Stop(); diff --git a/Lite/Services/RemoteCollectorService.Perfmon.cs b/Lite/Services/RemoteCollectorService.Perfmon.cs index 9d48c48a..867f878a 100644 --- a/Lite/Services/RemoteCollectorService.Perfmon.cs +++ b/Lite/Services/RemoteCollectorService.Perfmon.cs @@ -149,36 +149,40 @@ WHERE pc.counter_name IN ( _lastSqlMs = sqlSw.ElapsedMilliseconds; var duckSw = Stopwatch.StartNew(); - using var duckConnection = _duckDb.CreateConnection(); - await duckConnection.OpenAsync(cancellationToken); - using var appender = duckConnection.CreateAppender("perfmon_stats"); - - while (await reader.ReadAsync(cancellationToken)) + using (var duckConnection = _duckDb.CreateConnection()) { - var objectName = reader.IsDBNull(0) ? "" : reader.GetString(0); - var counterName = reader.IsDBNull(1) ? "" : reader.GetString(1); - var instanceName = reader.IsDBNull(2) ? "" : reader.GetString(2); - var cntrValue = reader.GetInt64(3); - - /* Delta for per-second counters */ - var deltaKey = $"{objectName}|{counterName}|{instanceName}"; - var deltaCntrValue = _deltaCalculator.CalculateDelta(serverId, "perfmon", deltaKey, cntrValue); - - var row = appender.CreateRow(); - row.AppendValue(GenerateCollectionId()) - .AppendValue(collectionTime) - .AppendValue(serverId) - .AppendValue(server.ServerName) - .AppendValue(objectName) - .AppendValue(counterName) - .AppendValue(instanceName) - .AppendValue(cntrValue) - .AppendValue(deltaCntrValue) - .AppendValue(600) /* 10-minute interval */ - .EndRow(); - - rowsCollected++; + await duckConnection.OpenAsync(cancellationToken); + + using (var appender = duckConnection.CreateAppender("perfmon_stats")) + { + while (await reader.ReadAsync(cancellationToken)) + { + var objectName = reader.IsDBNull(0) ? "" : reader.GetString(0); + var counterName = reader.IsDBNull(1) ? "" : reader.GetString(1); + var instanceName = reader.IsDBNull(2) ? "" : reader.GetString(2); + var cntrValue = reader.GetInt64(3); + + /* Delta for per-second counters */ + var deltaKey = $"{objectName}|{counterName}|{instanceName}"; + var deltaCntrValue = _deltaCalculator.CalculateDelta(serverId, "perfmon", deltaKey, cntrValue); + + var row = appender.CreateRow(); + row.AppendValue(GenerateCollectionId()) + .AppendValue(collectionTime) + .AppendValue(serverId) + .AppendValue(server.ServerName) + .AppendValue(objectName) + .AppendValue(counterName) + .AppendValue(instanceName) + .AppendValue(cntrValue) + .AppendValue(deltaCntrValue) + .AppendValue(600) /* 10-minute interval */ + .EndRow(); + + rowsCollected++; + } + } } duckSw.Stop(); diff --git a/Lite/Services/RemoteCollectorService.ProcedureStats.cs b/Lite/Services/RemoteCollectorService.ProcedureStats.cs index a0c1eb52..acc6d653 100644 --- a/Lite/Services/RemoteCollectorService.ProcedureStats.cs +++ b/Lite/Services/RemoteCollectorService.ProcedureStats.cs @@ -199,71 +199,75 @@ ORDER BY s.total_elapsed_time DESC sqlSw.Stop(); var duckSw = Stopwatch.StartNew(); - using var duckConnection = _duckDb.CreateConnection(); - await duckConnection.OpenAsync(cancellationToken); - using var appender = duckConnection.CreateAppender("procedure_stats"); - - while (await reader.ReadAsync(cancellationToken)) + using (var duckConnection = _duckDb.CreateConnection()) { - var dbName = reader.IsDBNull(0) ? "" : reader.GetString(0); - var schemaName = reader.IsDBNull(1) ? "" : reader.GetString(1); - var objectName = reader.IsDBNull(2) ? "" : reader.GetString(2); - var objectType = reader.IsDBNull(3) ? "" : reader.GetString(3); - var execCount = reader.GetInt64(4); - var workerTime = reader.GetInt64(5); - var elapsedTime = reader.GetInt64(6); - var logicalReads = reader.GetInt64(7); - var physicalReads = reader.GetInt64(8); - var logicalWrites = reader.GetInt64(9); - var minWorkerTime = reader.GetInt64(10); - var maxWorkerTime = reader.GetInt64(11); - var minElapsedTime = reader.GetInt64(12); - var maxElapsedTime = reader.GetInt64(13); - var totalSpills = reader.GetInt64(14); - var sqlHandle = reader.IsDBNull(15) ? (string?)null : reader.GetString(15); - var planHandle = reader.IsDBNull(16) ? (string?)null : reader.GetString(16); + await duckConnection.OpenAsync(cancellationToken); + + using (var appender = duckConnection.CreateAppender("procedure_stats")) + { + while (await reader.ReadAsync(cancellationToken)) + { + var dbName = reader.IsDBNull(0) ? "" : reader.GetString(0); + var schemaName = reader.IsDBNull(1) ? "" : reader.GetString(1); + var objectName = reader.IsDBNull(2) ? "" : reader.GetString(2); + var objectType = reader.IsDBNull(3) ? "" : reader.GetString(3); + var execCount = reader.GetInt64(4); + var workerTime = reader.GetInt64(5); + var elapsedTime = reader.GetInt64(6); + var logicalReads = reader.GetInt64(7); + var physicalReads = reader.GetInt64(8); + var logicalWrites = reader.GetInt64(9); + var minWorkerTime = reader.GetInt64(10); + var maxWorkerTime = reader.GetInt64(11); + var minElapsedTime = reader.GetInt64(12); + var maxElapsedTime = reader.GetInt64(13); + var totalSpills = reader.GetInt64(14); + var sqlHandle = reader.IsDBNull(15) ? (string?)null : reader.GetString(15); + var planHandle = reader.IsDBNull(16) ? (string?)null : reader.GetString(16); - /* Delta key: database.schema.object */ - var deltaKey = $"{dbName}.{schemaName}.{objectName}"; - var deltaExec = _deltaCalculator.CalculateDelta(serverId, "proc_stats_exec", deltaKey, execCount); - var deltaWorker = _deltaCalculator.CalculateDelta(serverId, "proc_stats_worker", deltaKey, workerTime); - var deltaElapsed = _deltaCalculator.CalculateDelta(serverId, "proc_stats_elapsed", deltaKey, elapsedTime); - var deltaReads = _deltaCalculator.CalculateDelta(serverId, "proc_stats_reads", deltaKey, logicalReads); - var deltaWrites = _deltaCalculator.CalculateDelta(serverId, "proc_stats_writes", deltaKey, logicalWrites); - var deltaPhysReads = _deltaCalculator.CalculateDelta(serverId, "proc_stats_phys_reads", deltaKey, physicalReads); + /* Delta key: database.schema.object */ + var deltaKey = $"{dbName}.{schemaName}.{objectName}"; + var deltaExec = _deltaCalculator.CalculateDelta(serverId, "proc_stats_exec", deltaKey, execCount); + var deltaWorker = _deltaCalculator.CalculateDelta(serverId, "proc_stats_worker", deltaKey, workerTime); + var deltaElapsed = _deltaCalculator.CalculateDelta(serverId, "proc_stats_elapsed", deltaKey, elapsedTime); + var deltaReads = _deltaCalculator.CalculateDelta(serverId, "proc_stats_reads", deltaKey, logicalReads); + var deltaWrites = _deltaCalculator.CalculateDelta(serverId, "proc_stats_writes", deltaKey, logicalWrites); + var deltaPhysReads = _deltaCalculator.CalculateDelta(serverId, "proc_stats_phys_reads", deltaKey, physicalReads); - var row = appender.CreateRow(); - row.AppendValue(GenerateCollectionId()) - .AppendValue(collectionTime) - .AppendValue(serverId) - .AppendValue(server.ServerName) - .AppendValue(dbName) - .AppendValue(schemaName) - .AppendValue(objectName) - .AppendValue(objectType) - .AppendValue(execCount) - .AppendValue(workerTime) - .AppendValue(elapsedTime) - .AppendValue(logicalReads) - .AppendValue(physicalReads) - .AppendValue(logicalWrites) - .AppendValue(minWorkerTime) - .AppendValue(maxWorkerTime) - .AppendValue(minElapsedTime) - .AppendValue(maxElapsedTime) - .AppendValue(totalSpills) - .AppendValue(sqlHandle) - .AppendValue(planHandle) - .AppendValue(deltaExec) - .AppendValue(deltaWorker) - .AppendValue(deltaElapsed) - .AppendValue(deltaReads) - .AppendValue(deltaWrites) - .AppendValue(deltaPhysReads) - .EndRow(); + var row = appender.CreateRow(); + row.AppendValue(GenerateCollectionId()) + .AppendValue(collectionTime) + .AppendValue(serverId) + .AppendValue(server.ServerName) + .AppendValue(dbName) + .AppendValue(schemaName) + .AppendValue(objectName) + .AppendValue(objectType) + .AppendValue(execCount) + .AppendValue(workerTime) + .AppendValue(elapsedTime) + .AppendValue(logicalReads) + .AppendValue(physicalReads) + .AppendValue(logicalWrites) + .AppendValue(minWorkerTime) + .AppendValue(maxWorkerTime) + .AppendValue(minElapsedTime) + .AppendValue(maxElapsedTime) + .AppendValue(totalSpills) + .AppendValue(sqlHandle) + .AppendValue(planHandle) + .AppendValue(deltaExec) + .AppendValue(deltaWorker) + .AppendValue(deltaElapsed) + .AppendValue(deltaReads) + .AppendValue(deltaWrites) + .AppendValue(deltaPhysReads) + .EndRow(); - rowsCollected++; + rowsCollected++; + } + } } duckSw.Stop(); diff --git a/Lite/Services/RemoteCollectorService.QuerySnapshots.cs b/Lite/Services/RemoteCollectorService.QuerySnapshots.cs index ee791fc9..894a8221 100644 --- a/Lite/Services/RemoteCollectorService.QuerySnapshots.cs +++ b/Lite/Services/RemoteCollectorService.QuerySnapshots.cs @@ -103,41 +103,45 @@ AND dest.text IS NOT NULL _lastSqlMs = sqlSw.ElapsedMilliseconds; var duckSw = Stopwatch.StartNew(); - using var duckConnection = _duckDb.CreateConnection(); - await duckConnection.OpenAsync(cancellationToken); - using var appender = duckConnection.CreateAppender("query_snapshots"); - - while (await reader.ReadAsync(cancellationToken)) + using (var duckConnection = _duckDb.CreateConnection()) { - var row = appender.CreateRow(); - row.AppendValue(GenerateCollectionId()) - .AppendValue(collectionTime) - .AppendValue(serverId) - .AppendValue(server.ServerName) - .AppendValue(Convert.ToInt32(reader.GetValue(0))) /* session_id */ - .AppendValue(reader.IsDBNull(1) ? (string?)null : reader.GetString(1)) /* database_name */ - .AppendValue(reader.IsDBNull(2) ? (string?)null : reader.GetString(2)) /* elapsed_time_formatted */ - .AppendValue(reader.IsDBNull(3) ? (string?)null : reader.GetString(3)) /* query_text */ - .AppendValue(reader.IsDBNull(4) ? (string?)null : reader.GetString(4)) /* query_plan */ - .AppendValue(reader.IsDBNull(5) ? (string?)null : reader.GetValue(5)?.ToString()) /* live_query_plan (xml) */ - .AppendValue(reader.IsDBNull(6) ? (string?)null : reader.GetString(6)) /* status */ - .AppendValue(reader.IsDBNull(7) ? 0 : Convert.ToInt32(reader.GetValue(7))) /* blocking_session_id */ - .AppendValue(reader.IsDBNull(8) ? (string?)null : reader.GetString(8)) /* wait_type */ - .AppendValue(reader.IsDBNull(9) ? 0L : Convert.ToInt64(reader.GetValue(9))) /* wait_time_ms */ - .AppendValue(reader.IsDBNull(10) ? (string?)null : reader.GetString(10)) /* wait_resource */ - .AppendValue(reader.IsDBNull(11) ? 0L : Convert.ToInt64(reader.GetValue(11))) /* cpu_time_ms */ - .AppendValue(reader.IsDBNull(12) ? 0L : Convert.ToInt64(reader.GetValue(12))) /* total_elapsed_time_ms */ - .AppendValue(reader.IsDBNull(13) ? 0L : Convert.ToInt64(reader.GetValue(13))) /* reads */ - .AppendValue(reader.IsDBNull(14) ? 0L : Convert.ToInt64(reader.GetValue(14))) /* writes */ - .AppendValue(reader.IsDBNull(15) ? 0L : Convert.ToInt64(reader.GetValue(15))) /* logical_reads */ - .AppendValue(reader.IsDBNull(16) ? 0m : reader.GetDecimal(16)) /* granted_query_memory_gb */ - .AppendValue(reader.IsDBNull(17) ? (string?)null : reader.GetString(17)) /* transaction_isolation_level */ - .AppendValue(reader.IsDBNull(18) ? 0 : Convert.ToInt32(reader.GetValue(18))) /* dop */ - .AppendValue(reader.IsDBNull(19) ? 0 : Convert.ToInt32(reader.GetValue(19))) /* parallel_worker_count */ - .EndRow(); + await duckConnection.OpenAsync(cancellationToken); + + using (var appender = duckConnection.CreateAppender("query_snapshots")) + { + while (await reader.ReadAsync(cancellationToken)) + { + var row = appender.CreateRow(); + row.AppendValue(GenerateCollectionId()) + .AppendValue(collectionTime) + .AppendValue(serverId) + .AppendValue(server.ServerName) + .AppendValue(Convert.ToInt32(reader.GetValue(0))) /* session_id */ + .AppendValue(reader.IsDBNull(1) ? (string?)null : reader.GetString(1)) /* database_name */ + .AppendValue(reader.IsDBNull(2) ? (string?)null : reader.GetString(2)) /* elapsed_time_formatted */ + .AppendValue(reader.IsDBNull(3) ? (string?)null : reader.GetString(3)) /* query_text */ + .AppendValue(reader.IsDBNull(4) ? (string?)null : reader.GetString(4)) /* query_plan */ + .AppendValue(reader.IsDBNull(5) ? (string?)null : reader.GetValue(5)?.ToString()) /* live_query_plan (xml) */ + .AppendValue(reader.IsDBNull(6) ? (string?)null : reader.GetString(6)) /* status */ + .AppendValue(reader.IsDBNull(7) ? 0 : Convert.ToInt32(reader.GetValue(7))) /* blocking_session_id */ + .AppendValue(reader.IsDBNull(8) ? (string?)null : reader.GetString(8)) /* wait_type */ + .AppendValue(reader.IsDBNull(9) ? 0L : Convert.ToInt64(reader.GetValue(9))) /* wait_time_ms */ + .AppendValue(reader.IsDBNull(10) ? (string?)null : reader.GetString(10)) /* wait_resource */ + .AppendValue(reader.IsDBNull(11) ? 0L : Convert.ToInt64(reader.GetValue(11))) /* cpu_time_ms */ + .AppendValue(reader.IsDBNull(12) ? 0L : Convert.ToInt64(reader.GetValue(12))) /* total_elapsed_time_ms */ + .AppendValue(reader.IsDBNull(13) ? 0L : Convert.ToInt64(reader.GetValue(13))) /* reads */ + .AppendValue(reader.IsDBNull(14) ? 0L : Convert.ToInt64(reader.GetValue(14))) /* writes */ + .AppendValue(reader.IsDBNull(15) ? 0L : Convert.ToInt64(reader.GetValue(15))) /* logical_reads */ + .AppendValue(reader.IsDBNull(16) ? 0m : reader.GetDecimal(16)) /* granted_query_memory_gb */ + .AppendValue(reader.IsDBNull(17) ? (string?)null : reader.GetString(17)) /* transaction_isolation_level */ + .AppendValue(reader.IsDBNull(18) ? 0 : Convert.ToInt32(reader.GetValue(18))) /* dop */ + .AppendValue(reader.IsDBNull(19) ? 0 : Convert.ToInt32(reader.GetValue(19))) /* parallel_worker_count */ + .EndRow(); - rowsCollected++; + rowsCollected++; + } + } } duckSw.Stop(); diff --git a/Lite/Services/RemoteCollectorService.QueryStats.cs b/Lite/Services/RemoteCollectorService.QueryStats.cs index 497c4e7f..16774801 100644 --- a/Lite/Services/RemoteCollectorService.QueryStats.cs +++ b/Lite/Services/RemoteCollectorService.QueryStats.cs @@ -156,78 +156,82 @@ qs.total_elapsed_time DESC sqlSw.Stop(); var duckSw = Stopwatch.StartNew(); - using var duckConnection = _duckDb.CreateConnection(); - await duckConnection.OpenAsync(cancellationToken); - using var appender = duckConnection.CreateAppender("query_stats"); - - while (await reader.ReadAsync(cancellationToken)) + using (var duckConnection = _duckDb.CreateConnection()) { - var queryHash = reader.IsDBNull(1) ? "" : reader.GetString(1); - var executionCount = reader.IsDBNull(3) ? 0L : reader.GetInt64(3); - var totalWorkerTime = reader.IsDBNull(4) ? 0L : reader.GetInt64(4); - var totalElapsedTime = reader.IsDBNull(5) ? 0L : reader.GetInt64(5); - var totalLogicalReads = reader.IsDBNull(6) ? 0L : reader.GetInt64(6); - var totalLogicalWrites = reader.IsDBNull(7) ? 0L : reader.GetInt64(7); - var totalPhysicalReads = reader.IsDBNull(8) ? 0L : reader.GetInt64(8); - var totalRows = reader.IsDBNull(9) ? 0L : reader.GetInt64(9); - var totalSpills = reader.IsDBNull(10) ? 0L : reader.GetInt64(10); - var minWorkerTime = reader.IsDBNull(11) ? 0L : reader.GetInt64(11); - var maxWorkerTime = reader.IsDBNull(12) ? 0L : reader.GetInt64(12); - var minElapsedTime = reader.IsDBNull(13) ? 0L : reader.GetInt64(13); - var maxElapsedTime = reader.IsDBNull(14) ? 0L : reader.GetInt64(14); - var minDop = reader.IsDBNull(15) ? 0 : Convert.ToInt32(reader.GetValue(15)); - var maxDop = reader.IsDBNull(16) ? 0 : Convert.ToInt32(reader.GetValue(16)); - var sqlHandle = reader.IsDBNull(17) ? (string?)null : reader.GetString(17); - var planHandle = reader.IsDBNull(18) ? (string?)null : reader.GetString(18); - - /* Delta calculations based on query_hash */ - var deltaExecCount = _deltaCalculator.CalculateDelta(serverId, "query_stats_exec", queryHash, executionCount); - var deltaWorkerTime = _deltaCalculator.CalculateDelta(serverId, "query_stats_worker", queryHash, totalWorkerTime); - var deltaElapsedTime = _deltaCalculator.CalculateDelta(serverId, "query_stats_elapsed", queryHash, totalElapsedTime); - var deltaLogicalReads = _deltaCalculator.CalculateDelta(serverId, "query_stats_reads", queryHash, totalLogicalReads); - var deltaLogicalWrites = _deltaCalculator.CalculateDelta(serverId, "query_stats_writes", queryHash, totalLogicalWrites); - var deltaPhysicalReads = _deltaCalculator.CalculateDelta(serverId, "query_stats_phys_reads", queryHash, totalPhysicalReads); - var deltaRows = _deltaCalculator.CalculateDelta(serverId, "query_stats_rows", queryHash, totalRows); - var deltaSpills = _deltaCalculator.CalculateDelta(serverId, "query_stats_spills", queryHash, totalSpills); - - var row = appender.CreateRow(); - row.AppendValue(GenerateCollectionId()) - .AppendValue(collectionTime) - .AppendValue(serverId) - .AppendValue(server.ServerName) - .AppendValue(reader.IsDBNull(0) ? (string?)null : reader.GetString(0)) - .AppendValue(queryHash) - .AppendValue(reader.IsDBNull(2) ? (string?)null : reader.GetString(2)) - .AppendValue(executionCount) - .AppendValue(totalWorkerTime) - .AppendValue(totalElapsedTime) - .AppendValue(totalLogicalReads) - .AppendValue(totalLogicalWrites) - .AppendValue(totalPhysicalReads) - .AppendValue(totalRows) - .AppendValue(totalSpills) - .AppendValue(minWorkerTime) - .AppendValue(maxWorkerTime) - .AppendValue(minElapsedTime) - .AppendValue(maxElapsedTime) - .AppendValue(minDop) - .AppendValue(maxDop) - .AppendValue(reader.IsDBNull(19) ? (string?)null : reader.GetString(19)) - .AppendValue((string?)null) /* query plans retrieved on-demand */ - .AppendValue(sqlHandle) - .AppendValue(planHandle) - .AppendValue(deltaExecCount) - .AppendValue(deltaWorkerTime) - .AppendValue(deltaElapsedTime) - .AppendValue(deltaLogicalReads) - .AppendValue(deltaLogicalWrites) - .AppendValue(deltaPhysicalReads) - .AppendValue(deltaRows) - .AppendValue(deltaSpills) - .EndRow(); - - rowsCollected++; + await duckConnection.OpenAsync(cancellationToken); + + using (var appender = duckConnection.CreateAppender("query_stats")) + { + while (await reader.ReadAsync(cancellationToken)) + { + var queryHash = reader.IsDBNull(1) ? "" : reader.GetString(1); + var executionCount = reader.IsDBNull(3) ? 0L : reader.GetInt64(3); + var totalWorkerTime = reader.IsDBNull(4) ? 0L : reader.GetInt64(4); + var totalElapsedTime = reader.IsDBNull(5) ? 0L : reader.GetInt64(5); + var totalLogicalReads = reader.IsDBNull(6) ? 0L : reader.GetInt64(6); + var totalLogicalWrites = reader.IsDBNull(7) ? 0L : reader.GetInt64(7); + var totalPhysicalReads = reader.IsDBNull(8) ? 0L : reader.GetInt64(8); + var totalRows = reader.IsDBNull(9) ? 0L : reader.GetInt64(9); + var totalSpills = reader.IsDBNull(10) ? 0L : reader.GetInt64(10); + var minWorkerTime = reader.IsDBNull(11) ? 0L : reader.GetInt64(11); + var maxWorkerTime = reader.IsDBNull(12) ? 0L : reader.GetInt64(12); + var minElapsedTime = reader.IsDBNull(13) ? 0L : reader.GetInt64(13); + var maxElapsedTime = reader.IsDBNull(14) ? 0L : reader.GetInt64(14); + var minDop = reader.IsDBNull(15) ? 0 : Convert.ToInt32(reader.GetValue(15)); + var maxDop = reader.IsDBNull(16) ? 0 : Convert.ToInt32(reader.GetValue(16)); + var sqlHandle = reader.IsDBNull(17) ? (string?)null : reader.GetString(17); + var planHandle = reader.IsDBNull(18) ? (string?)null : reader.GetString(18); + + /* Delta calculations based on query_hash */ + var deltaExecCount = _deltaCalculator.CalculateDelta(serverId, "query_stats_exec", queryHash, executionCount); + var deltaWorkerTime = _deltaCalculator.CalculateDelta(serverId, "query_stats_worker", queryHash, totalWorkerTime); + var deltaElapsedTime = _deltaCalculator.CalculateDelta(serverId, "query_stats_elapsed", queryHash, totalElapsedTime); + var deltaLogicalReads = _deltaCalculator.CalculateDelta(serverId, "query_stats_reads", queryHash, totalLogicalReads); + var deltaLogicalWrites = _deltaCalculator.CalculateDelta(serverId, "query_stats_writes", queryHash, totalLogicalWrites); + var deltaPhysicalReads = _deltaCalculator.CalculateDelta(serverId, "query_stats_phys_reads", queryHash, totalPhysicalReads); + var deltaRows = _deltaCalculator.CalculateDelta(serverId, "query_stats_rows", queryHash, totalRows); + var deltaSpills = _deltaCalculator.CalculateDelta(serverId, "query_stats_spills", queryHash, totalSpills); + + var row = appender.CreateRow(); + row.AppendValue(GenerateCollectionId()) + .AppendValue(collectionTime) + .AppendValue(serverId) + .AppendValue(server.ServerName) + .AppendValue(reader.IsDBNull(0) ? (string?)null : reader.GetString(0)) + .AppendValue(queryHash) + .AppendValue(reader.IsDBNull(2) ? (string?)null : reader.GetString(2)) + .AppendValue(executionCount) + .AppendValue(totalWorkerTime) + .AppendValue(totalElapsedTime) + .AppendValue(totalLogicalReads) + .AppendValue(totalLogicalWrites) + .AppendValue(totalPhysicalReads) + .AppendValue(totalRows) + .AppendValue(totalSpills) + .AppendValue(minWorkerTime) + .AppendValue(maxWorkerTime) + .AppendValue(minElapsedTime) + .AppendValue(maxElapsedTime) + .AppendValue(minDop) + .AppendValue(maxDop) + .AppendValue(reader.IsDBNull(19) ? (string?)null : reader.GetString(19)) + .AppendValue((string?)null) /* query plans retrieved on-demand */ + .AppendValue(sqlHandle) + .AppendValue(planHandle) + .AppendValue(deltaExecCount) + .AppendValue(deltaWorkerTime) + .AppendValue(deltaElapsedTime) + .AppendValue(deltaLogicalReads) + .AppendValue(deltaLogicalWrites) + .AppendValue(deltaPhysicalReads) + .AppendValue(deltaRows) + .AppendValue(deltaSpills) + .EndRow(); + + rowsCollected++; + } + } } duckSw.Stop(); diff --git a/Lite/Services/RemoteCollectorService.QueryStore.cs b/Lite/Services/RemoteCollectorService.QueryStore.cs index 127a3878..6f69db5e 100644 --- a/Lite/Services/RemoteCollectorService.QueryStore.cs +++ b/Lite/Services/RemoteCollectorService.QueryStore.cs @@ -74,15 +74,17 @@ ORDER BY d.name } var duckSw = new Stopwatch(); - using var duckConnection = _duckDb.CreateConnection(); - await duckConnection.OpenAsync(cancellationToken); - /* For each database, collect new query store intervals since last collection */ - foreach (var dbName in databases) + using (var duckConnection = _duckDb.CreateConnection()) { - try + await duckConnection.OpenAsync(cancellationToken); + + /* For each database, collect new query store intervals since last collection */ + foreach (var dbName in databases) { - var qsQuery = $@" + try + { + var qsQuery = $@" EXECUTE [{dbName.Replace("]", "]]")}].sys.sp_executesql N'SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; @@ -112,50 +114,54 @@ WHERE qsrs.last_execution_time > @cutoff_time N'@cutoff_time datetime2(7)', @cutoff_time;"; - sqlSw.Start(); - using var qsCommand = new SqlCommand(qsQuery, sqlConnection); - qsCommand.CommandTimeout = CommandTimeoutSeconds; - qsCommand.Parameters.Add(new SqlParameter("@cutoff_time", System.Data.SqlDbType.DateTime2) { Value = cutoffTime }); - - using var reader = await qsCommand.ExecuteReaderAsync(cancellationToken); - sqlSw.Stop(); - - duckSw.Start(); - using var appender = duckConnection.CreateAppender("query_store_stats"); - - while (await reader.ReadAsync(cancellationToken)) + sqlSw.Start(); + using var qsCommand = new SqlCommand(qsQuery, sqlConnection); + qsCommand.CommandTimeout = CommandTimeoutSeconds; + qsCommand.Parameters.Add(new SqlParameter("@cutoff_time", System.Data.SqlDbType.DateTime2) { Value = cutoffTime }); + + using var reader = await qsCommand.ExecuteReaderAsync(cancellationToken); + sqlSw.Stop(); + + duckSw.Start(); + + using (var appender = duckConnection.CreateAppender("query_store_stats")) + { + while (await reader.ReadAsync(cancellationToken)) + { + var row = appender.CreateRow(); + row.AppendValue(GenerateCollectionId()) + .AppendValue(collectionTime) + .AppendValue(serverId) + .AppendValue(server.ServerName) + .AppendValue(dbName) + .AppendValue(reader.GetInt64(0)) /* query_id */ + .AppendValue(reader.GetInt64(1)) /* plan_id */ + .AppendValue(reader.IsDBNull(2) ? (string?)null : reader.GetString(2)) /* query_text */ + .AppendValue(reader.IsDBNull(3) ? (string?)null : reader.GetString(3)) /* query_hash */ + .AppendValue(reader.GetInt64(4)) /* execution_count */ + .AppendValue(reader.IsDBNull(5) ? 0m : reader.GetDecimal(5)) /* avg_duration_ms */ + .AppendValue(reader.IsDBNull(6) ? 0m : reader.GetDecimal(6)) /* avg_cpu_time_ms */ + .AppendValue(reader.IsDBNull(7) ? 0m : reader.GetDecimal(7)) /* avg_logical_reads */ + .AppendValue(reader.IsDBNull(8) ? 0m : reader.GetDecimal(8)) /* avg_logical_writes */ + .AppendValue(reader.IsDBNull(9) ? 0m : reader.GetDecimal(9)) /* avg_physical_reads */ + .AppendValue(reader.IsDBNull(10) ? 0m : reader.GetDecimal(10)) /* avg_rowcount */ + .AppendValue(reader.IsDBNull(11) ? (DateTime?)null : ((DateTimeOffset)reader.GetValue(11)).UtcDateTime) + .AppendValue(reader.IsDBNull(12) ? (string?)null : reader.GetString(12)) + .EndRow(); + + totalRows++; + } + } + + duckSw.Stop(); + } + catch (SqlException ex) { - var row = appender.CreateRow(); - row.AppendValue(GenerateCollectionId()) - .AppendValue(collectionTime) - .AppendValue(serverId) - .AppendValue(server.ServerName) - .AppendValue(dbName) - .AppendValue(reader.GetInt64(0)) /* query_id */ - .AppendValue(reader.GetInt64(1)) /* plan_id */ - .AppendValue(reader.IsDBNull(2) ? (string?)null : reader.GetString(2)) /* query_text */ - .AppendValue(reader.IsDBNull(3) ? (string?)null : reader.GetString(3)) /* query_hash */ - .AppendValue(reader.GetInt64(4)) /* execution_count */ - .AppendValue(reader.IsDBNull(5) ? 0m : reader.GetDecimal(5)) /* avg_duration_ms */ - .AppendValue(reader.IsDBNull(6) ? 0m : reader.GetDecimal(6)) /* avg_cpu_time_ms */ - .AppendValue(reader.IsDBNull(7) ? 0m : reader.GetDecimal(7)) /* avg_logical_reads */ - .AppendValue(reader.IsDBNull(8) ? 0m : reader.GetDecimal(8)) /* avg_logical_writes */ - .AppendValue(reader.IsDBNull(9) ? 0m : reader.GetDecimal(9)) /* avg_physical_reads */ - .AppendValue(reader.IsDBNull(10) ? 0m : reader.GetDecimal(10)) /* avg_rowcount */ - .AppendValue(reader.IsDBNull(11) ? (DateTime?)null : ((DateTimeOffset)reader.GetValue(11)).UtcDateTime) - .AppendValue(reader.IsDBNull(12) ? (string?)null : reader.GetString(12)) - .EndRow(); - - totalRows++; + sqlSw.Stop(); + duckSw.Stop(); + _logger?.LogWarning("Failed to collect Query Store data from [{Database}] on '{Server}': {Message}", + dbName, server.DisplayName, ex.Message); } - duckSw.Stop(); - } - catch (SqlException ex) - { - sqlSw.Stop(); - duckSw.Stop(); - _logger?.LogWarning("Failed to collect Query Store data from [{Database}] on '{Server}': {Message}", - dbName, server.DisplayName, ex.Message); } } diff --git a/Lite/Services/RemoteCollectorService.RunningJobs.cs b/Lite/Services/RemoteCollectorService.RunningJobs.cs index 63c29de2..00d26b87 100644 --- a/Lite/Services/RemoteCollectorService.RunningJobs.cs +++ b/Lite/Services/RemoteCollectorService.RunningJobs.cs @@ -153,28 +153,33 @@ rj.current_duration_seconds DESC sqlSw.Stop(); var duckSw = Stopwatch.StartNew(); - using var duckConnection = _duckDb.CreateConnection(); - await duckConnection.OpenAsync(cancellationToken); - using var appender = duckConnection.CreateAppender("running_jobs"); - foreach (var r in rows) + using (var duckConnection = _duckDb.CreateConnection()) { - var row = appender.CreateRow(); - row.AppendValue(collectionTime) - .AppendValue(serverId) - .AppendValue(server.ServerName) - .AppendValue(r.JobName) - .AppendValue(r.JobId) - .AppendValue(r.JobEnabled) - .AppendValue(r.StartTime) - .AppendValue(r.CurrentDuration) - .AppendValue(r.AvgDuration) - .AppendValue(r.P95Duration) - .AppendValue(r.SuccessfulRunCount) - .AppendValue(r.IsRunningLong) - .AppendValue(r.PercentOfAverage) - .EndRow(); - rowsCollected++; + await duckConnection.OpenAsync(cancellationToken); + + using (var appender = duckConnection.CreateAppender("running_jobs")) + { + foreach (var r in rows) + { + var row = appender.CreateRow(); + row.AppendValue(collectionTime) + .AppendValue(serverId) + .AppendValue(server.ServerName) + .AppendValue(r.JobName) + .AppendValue(r.JobId) + .AppendValue(r.JobEnabled) + .AppendValue(r.StartTime) + .AppendValue(r.CurrentDuration) + .AppendValue(r.AvgDuration) + .AppendValue(r.P95Duration) + .AppendValue(r.SuccessfulRunCount) + .AppendValue(r.IsRunningLong) + .AppendValue(r.PercentOfAverage) + .EndRow(); + rowsCollected++; + } + } } duckSw.Stop(); diff --git a/Lite/Services/RemoteCollectorService.TempDb.cs b/Lite/Services/RemoteCollectorService.TempDb.cs index 4cafdcef..ef0a300d 100644 --- a/Lite/Services/RemoteCollectorService.TempDb.cs +++ b/Lite/Services/RemoteCollectorService.TempDb.cs @@ -77,24 +77,29 @@ FROM sys.dm_db_session_space_usage AS ssu /* Insert into DuckDB using Appender */ var duckSw = Stopwatch.StartNew(); - using var duckConnection = _duckDb.CreateConnection(); - await duckConnection.OpenAsync(cancellationToken); - - using var appender = duckConnection.CreateAppender("tempdb_stats"); - var row = appender.CreateRow(); - row.AppendValue(GenerateCollectionId()) - .AppendValue(collectionTime) - .AppendValue(serverId) - .AppendValue(server.ServerName) - .AppendValue(userObjMb) - .AppendValue(internalObjMb) - .AppendValue(versionStoreMb) - .AppendValue(totalReservedMb) - .AppendValue(unallocatedMb) - .AppendValue(totalSessions) - .AppendValue(topSessionId) - .AppendValue(topSessionMb) - .EndRow(); + + using (var duckConnection = _duckDb.CreateConnection()) + { + await duckConnection.OpenAsync(cancellationToken); + + using (var appender = duckConnection.CreateAppender("tempdb_stats")) + { + var row = appender.CreateRow(); + row.AppendValue(GenerateCollectionId()) + .AppendValue(collectionTime) + .AppendValue(serverId) + .AppendValue(server.ServerName) + .AppendValue(userObjMb) + .AppendValue(internalObjMb) + .AppendValue(versionStoreMb) + .AppendValue(totalReservedMb) + .AppendValue(unallocatedMb) + .AppendValue(totalSessions) + .AppendValue(topSessionId) + .AppendValue(topSessionMb) + .EndRow(); + } + } duckSw.Stop(); _lastSqlMs = sqlSw.ElapsedMilliseconds; diff --git a/Lite/Services/RemoteCollectorService.WaitStats.cs b/Lite/Services/RemoteCollectorService.WaitStats.cs index 26ad9cb1..52767e7f 100644 --- a/Lite/Services/RemoteCollectorService.WaitStats.cs +++ b/Lite/Services/RemoteCollectorService.WaitStats.cs @@ -121,33 +121,37 @@ WHERE ws.wait_time_ms > 0 /* Insert into DuckDB with delta calculations using Appender for bulk performance */ var duckSw = Stopwatch.StartNew(); - using var duckConnection = _duckDb.CreateConnection(); - await duckConnection.OpenAsync(cancellationToken); - using var appender = duckConnection.CreateAppender("wait_stats"); - - foreach (var stat in waitStats) + using (var duckConnection = _duckDb.CreateConnection()) { - var deltaKey = stat.WaitType; - var deltaWaitingTasks = _deltaCalculator.CalculateDelta(serverId, "wait_stats_tasks", deltaKey, stat.WaitingTasks); - var deltaWaitTimeMs = _deltaCalculator.CalculateDelta(serverId, "wait_stats_time", deltaKey, stat.WaitTimeMs); - var deltaSignalWaitTimeMs = _deltaCalculator.CalculateDelta(serverId, "wait_stats_signal", deltaKey, stat.SignalWaitTimeMs); - - var row = appender.CreateRow(); - row.AppendValue(GenerateCollectionId()) /* collection_id BIGINT */ - .AppendValue(collectionTime) /* collection_time TIMESTAMP */ - .AppendValue(serverId) /* server_id INTEGER */ - .AppendValue(server.ServerName) /* server_name VARCHAR */ - .AppendValue(stat.WaitType) /* wait_type VARCHAR */ - .AppendValue(stat.WaitingTasks) /* waiting_tasks_count BIGINT */ - .AppendValue(stat.WaitTimeMs) /* wait_time_ms BIGINT */ - .AppendValue(stat.SignalWaitTimeMs) /* signal_wait_time_ms BIGINT */ - .AppendValue(deltaWaitingTasks) /* delta_waiting_tasks BIGINT */ - .AppendValue(deltaWaitTimeMs) /* delta_wait_time_ms BIGINT */ - .AppendValue(deltaSignalWaitTimeMs) /* delta_signal_wait_time_ms BIGINT */ - .EndRow(); - - rowsCollected++; + await duckConnection.OpenAsync(cancellationToken); + + using (var appender = duckConnection.CreateAppender("wait_stats")) + { + foreach (var stat in waitStats) + { + var deltaKey = stat.WaitType; + var deltaWaitingTasks = _deltaCalculator.CalculateDelta(serverId, "wait_stats_tasks", deltaKey, stat.WaitingTasks); + var deltaWaitTimeMs = _deltaCalculator.CalculateDelta(serverId, "wait_stats_time", deltaKey, stat.WaitTimeMs); + var deltaSignalWaitTimeMs = _deltaCalculator.CalculateDelta(serverId, "wait_stats_signal", deltaKey, stat.SignalWaitTimeMs); + + var row = appender.CreateRow(); + row.AppendValue(GenerateCollectionId()) /* collection_id BIGINT */ + .AppendValue(collectionTime) /* collection_time TIMESTAMP */ + .AppendValue(serverId) /* server_id INTEGER */ + .AppendValue(server.ServerName) /* server_name VARCHAR */ + .AppendValue(stat.WaitType) /* wait_type VARCHAR */ + .AppendValue(stat.WaitingTasks) /* waiting_tasks_count BIGINT */ + .AppendValue(stat.WaitTimeMs) /* wait_time_ms BIGINT */ + .AppendValue(stat.SignalWaitTimeMs) /* signal_wait_time_ms BIGINT */ + .AppendValue(deltaWaitingTasks) /* delta_waiting_tasks BIGINT */ + .AppendValue(deltaWaitTimeMs) /* delta_wait_time_ms BIGINT */ + .AppendValue(deltaSignalWaitTimeMs) /* delta_signal_wait_time_ms BIGINT */ + .EndRow(); + + rowsCollected++; + } + } } duckSw.Stop(); diff --git a/Lite/Services/RemoteCollectorService.WaitingTasks.cs b/Lite/Services/RemoteCollectorService.WaitingTasks.cs index 99a21022..888811e0 100644 --- a/Lite/Services/RemoteCollectorService.WaitingTasks.cs +++ b/Lite/Services/RemoteCollectorService.WaitingTasks.cs @@ -59,35 +59,39 @@ AND wt.wait_type IS NOT NULL _lastSqlMs = sqlSw.ElapsedMilliseconds; var duckSw = Stopwatch.StartNew(); - using var duckConnection = _duckDb.CreateConnection(); - await duckConnection.OpenAsync(cancellationToken); - using var appender = duckConnection.CreateAppender("waiting_tasks"); - - while (await reader.ReadAsync(cancellationToken)) + using (var duckConnection = _duckDb.CreateConnection()) { - /* session_id and blocking_session_id are smallint in sys.dm_os_waiting_tasks */ - var sessionId = reader.IsDBNull(0) ? 0 : reader.GetInt16(0); - var waitType = reader.IsDBNull(1) ? null : reader.GetString(1); - var waitDurationMs = reader.IsDBNull(2) ? 0L : reader.GetInt64(2); - var blockingSessionId = reader.IsDBNull(3) ? (short?)null : reader.GetInt16(3); - var resourceDescription = reader.IsDBNull(4) ? null : reader.GetString(4); - var databaseName = reader.IsDBNull(5) ? null : reader.GetString(5); + await duckConnection.OpenAsync(cancellationToken); + + using (var appender = duckConnection.CreateAppender("waiting_tasks")) + { + while (await reader.ReadAsync(cancellationToken)) + { + /* session_id and blocking_session_id are smallint in sys.dm_os_waiting_tasks */ + var sessionId = reader.IsDBNull(0) ? 0 : reader.GetInt16(0); + var waitType = reader.IsDBNull(1) ? null : reader.GetString(1); + var waitDurationMs = reader.IsDBNull(2) ? 0L : reader.GetInt64(2); + var blockingSessionId = reader.IsDBNull(3) ? (short?)null : reader.GetInt16(3); + var resourceDescription = reader.IsDBNull(4) ? null : reader.GetString(4); + var databaseName = reader.IsDBNull(5) ? null : reader.GetString(5); - var row = appender.CreateRow(); - row.AppendValue(GenerateCollectionId()) - .AppendValue(collectionTime) - .AppendValue(serverId) - .AppendValue(server.ServerName) - .AppendValue((int)sessionId) - .AppendValue(waitType) - .AppendValue(waitDurationMs) - .AppendValue(blockingSessionId.HasValue ? (int?)blockingSessionId.Value : null) - .AppendValue(resourceDescription) - .AppendValue(databaseName) - .EndRow(); + var row = appender.CreateRow(); + row.AppendValue(GenerateCollectionId()) + .AppendValue(collectionTime) + .AppendValue(serverId) + .AppendValue(server.ServerName) + .AppendValue((int)sessionId) + .AppendValue(waitType) + .AppendValue(waitDurationMs) + .AppendValue(blockingSessionId.HasValue ? (int?)blockingSessionId.Value : null) + .AppendValue(resourceDescription) + .AppendValue(databaseName) + .EndRow(); - rowsCollected++; + rowsCollected++; + } + } } duckSw.Stop(); diff --git a/Lite/Services/RemoteCollectorService.cs b/Lite/Services/RemoteCollectorService.cs index bda30b09..9b2e1b3e 100644 --- a/Lite/Services/RemoteCollectorService.cs +++ b/Lite/Services/RemoteCollectorService.cs @@ -120,6 +120,11 @@ public RemoteCollectorService( /// public Task SeedDeltaCacheAsync() => _deltaCalculator.SeedFromDatabaseAsync(_duckDb); + /// + /// Runs a manual DuckDB WAL checkpoint during idle time between collection cycles. + /// + public Task CheckpointAsync() => _duckDb.CheckpointAsync(); + /// /// Gets a summary of collector health. When serverId is provided, filters to that server only. /// @@ -201,8 +206,8 @@ public async Task RunDueCollectorsAsync(CancellationToken cancellationToken = de return; } - var tasks = new List(); int skippedOffline = 0; + var onlineServers = new List(); foreach (var server in enabledServers) { @@ -213,17 +218,25 @@ public async Task RunDueCollectorsAsync(CancellationToken cancellationToken = de _logger?.LogDebug("Skipping offline server '{Server}'", server.DisplayName); continue; } + onlineServers.Add(server); + } + + _logger?.LogInformation("Running {CollectorCount} collectors for {OnlineCount}/{TotalCount} servers ({SkippedCount} offline, skipped)", + dueCollectors.Count, onlineServers.Count, enabledServers.Count, skippedOffline); + /* Run servers in parallel, but collectors within each server sequentially. + DuckDB is single-writer; running all collectors in parallel causes spin-wait + contention (50%+ CPU, multi-second stalls). Sequential per-server eliminates + this while still allowing multi-server parallelism. */ + var serverTasks = onlineServers.Select(server => Task.Run(async () => + { foreach (var collector in dueCollectors) { - tasks.Add(RunCollectorAsync(server, collector.Name, cancellationToken)); + await RunCollectorAsync(server, collector.Name, cancellationToken); } - } - - _logger?.LogInformation("Running {CollectorCount} collectors for {OnlineCount}/{TotalCount} servers ({SkippedCount} offline, skipped)", - dueCollectors.Count, enabledServers.Count - skippedOffline, enabledServers.Count, skippedOffline); + }, cancellationToken)); - await Task.WhenAll(tasks); + await Task.WhenAll(serverTasks); } ///