From a657c7061047ba7e109ce28715ea23889a1d15bb Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 19 Mar 2026 08:23:48 -0400 Subject: [PATCH] Add gap detection to all delta collectors (#633) The maxGapSeconds + collectionTime gap detection was only on the perfmon collector. All other cumulative-counter collectors (file I/O, wait stats, query stats, procedure stats, memory grants) produced inflated deltas after app restart because the cached baseline could be hours old. Now all CalculateDelta calls pass collectionTime and maxGapSeconds=300 (5 minutes). If the gap since the last cached value exceeds 5 minutes, the delta is treated as a new baseline (returns 0) instead of computing against the stale value. Co-Authored-By: Claude Opus 4.6 (1M context) --- Lite/Services/RemoteCollectorService.FileIo.cs | 16 ++++++++-------- .../RemoteCollectorService.MemoryGrants.cs | 4 ++-- .../RemoteCollectorService.ProcedureStats.cs | 12 ++++++------ .../RemoteCollectorService.QueryStats.cs | 16 ++++++++-------- .../Services/RemoteCollectorService.WaitStats.cs | 6 +++--- 5 files changed, 27 insertions(+), 27 deletions(-) diff --git a/Lite/Services/RemoteCollectorService.FileIo.cs b/Lite/Services/RemoteCollectorService.FileIo.cs index 595bb9d0..f7114105 100644 --- a/Lite/Services/RemoteCollectorService.FileIo.cs +++ b/Lite/Services/RemoteCollectorService.FileIo.cs @@ -143,14 +143,14 @@ AND vfs.database_id < 32761 foreach (var stat in fileStats) { var deltaKey = $"{stat.DatabaseName}|{stat.FileName}"; - var deltaReads = _deltaCalculator.CalculateDelta(serverId, "file_io_reads", deltaKey, stat.NumOfReads, baselineOnly: true); - var deltaWrites = _deltaCalculator.CalculateDelta(serverId, "file_io_writes", deltaKey, stat.NumOfWrites, baselineOnly: true); - var deltaReadBytes = _deltaCalculator.CalculateDelta(serverId, "file_io_read_bytes", deltaKey, stat.ReadBytes, baselineOnly: true); - var deltaWriteBytes = _deltaCalculator.CalculateDelta(serverId, "file_io_write_bytes", deltaKey, stat.WriteBytes, baselineOnly: true); - var deltaStallReadMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_read", deltaKey, stat.IoStallReadMs, baselineOnly: true); - var deltaStallWriteMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_write", deltaKey, stat.IoStallWriteMs, baselineOnly: true); - var deltaStallQueuedReadMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_queued_read", deltaKey, stat.IoStallQueuedReadMs, baselineOnly: true); - var deltaStallQueuedWriteMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_queued_write", deltaKey, stat.IoStallQueuedWriteMs, baselineOnly: true); + var deltaReads = _deltaCalculator.CalculateDelta(serverId, "file_io_reads", deltaKey, stat.NumOfReads, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); + var deltaWrites = _deltaCalculator.CalculateDelta(serverId, "file_io_writes", deltaKey, stat.NumOfWrites, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); + var deltaReadBytes = _deltaCalculator.CalculateDelta(serverId, "file_io_read_bytes", deltaKey, stat.ReadBytes, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); + var deltaWriteBytes = _deltaCalculator.CalculateDelta(serverId, "file_io_write_bytes", deltaKey, stat.WriteBytes, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); + var deltaStallReadMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_read", deltaKey, stat.IoStallReadMs, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); + var deltaStallWriteMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_write", deltaKey, stat.IoStallWriteMs, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); + var deltaStallQueuedReadMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_queued_read", deltaKey, stat.IoStallQueuedReadMs, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); + var deltaStallQueuedWriteMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_queued_write", deltaKey, stat.IoStallQueuedWriteMs, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); var row = appender.CreateRow(); row.AppendValue(GenerateCollectionId()) diff --git a/Lite/Services/RemoteCollectorService.MemoryGrants.cs b/Lite/Services/RemoteCollectorService.MemoryGrants.cs index 244e08b2..e2bf31d4 100644 --- a/Lite/Services/RemoteCollectorService.MemoryGrants.cs +++ b/Lite/Services/RemoteCollectorService.MemoryGrants.cs @@ -92,8 +92,8 @@ WHERE deqrs.max_target_memory_kb IS NOT NULL foreach (var r in rows) { var deltaKey = $"{r.PoolId}_{r.ResourceSemaphoreId}"; - var deltaTimeouts = _deltaCalculator.CalculateDelta(serverId, "memory_grants_timeouts", deltaKey, r.TimeoutErrorCount, baselineOnly: true); - var deltaForced = _deltaCalculator.CalculateDelta(serverId, "memory_grants_forced", deltaKey, r.ForcedGrantCount, baselineOnly: true); + var deltaTimeouts = _deltaCalculator.CalculateDelta(serverId, "memory_grants_timeouts", deltaKey, r.TimeoutErrorCount, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); + var deltaForced = _deltaCalculator.CalculateDelta(serverId, "memory_grants_forced", deltaKey, r.ForcedGrantCount, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); var row = appender.CreateRow(); row.AppendValue(GenerateCollectionId()) diff --git a/Lite/Services/RemoteCollectorService.ProcedureStats.cs b/Lite/Services/RemoteCollectorService.ProcedureStats.cs index e46b71ab..2b541c50 100644 --- a/Lite/Services/RemoteCollectorService.ProcedureStats.cs +++ b/Lite/Services/RemoteCollectorService.ProcedureStats.cs @@ -278,12 +278,12 @@ ORDER BY s.total_elapsed_time DESC /* Delta key: plan_handle to prevent cross-contamination when multiple plans exist for the same object */ var deltaKey = planHandle ?? $"{dbName}.{schemaName}.{objectName}"; - var deltaExec = _deltaCalculator.CalculateDelta(serverId, "proc_stats_exec", deltaKey, execCount, baselineOnly: true); - var deltaWorker = _deltaCalculator.CalculateDelta(serverId, "proc_stats_worker", deltaKey, workerTime, baselineOnly: true); - var deltaElapsed = _deltaCalculator.CalculateDelta(serverId, "proc_stats_elapsed", deltaKey, elapsedTime, baselineOnly: true); - var deltaReads = _deltaCalculator.CalculateDelta(serverId, "proc_stats_reads", deltaKey, logicalReads, baselineOnly: true); - var deltaWrites = _deltaCalculator.CalculateDelta(serverId, "proc_stats_writes", deltaKey, logicalWrites, baselineOnly: true); - var deltaPhysReads = _deltaCalculator.CalculateDelta(serverId, "proc_stats_phys_reads", deltaKey, physicalReads, baselineOnly: true); + var deltaExec = _deltaCalculator.CalculateDelta(serverId, "proc_stats_exec", deltaKey, execCount, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); + var deltaWorker = _deltaCalculator.CalculateDelta(serverId, "proc_stats_worker", deltaKey, workerTime, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); + var deltaElapsed = _deltaCalculator.CalculateDelta(serverId, "proc_stats_elapsed", deltaKey, elapsedTime, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); + var deltaReads = _deltaCalculator.CalculateDelta(serverId, "proc_stats_reads", deltaKey, logicalReads, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); + var deltaWrites = _deltaCalculator.CalculateDelta(serverId, "proc_stats_writes", deltaKey, logicalWrites, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); + var deltaPhysReads = _deltaCalculator.CalculateDelta(serverId, "proc_stats_phys_reads", deltaKey, physicalReads, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); /* Appender column order must match DuckDB table definition exactly */ var row = appender.CreateRow(); diff --git a/Lite/Services/RemoteCollectorService.QueryStats.cs b/Lite/Services/RemoteCollectorService.QueryStats.cs index f1aee244..b47705a6 100644 --- a/Lite/Services/RemoteCollectorService.QueryStats.cs +++ b/Lite/Services/RemoteCollectorService.QueryStats.cs @@ -266,14 +266,14 @@ qs.total_elapsed_time DESC /* Delta calculations keyed by plan_handle to prevent cross-contamination when multiple plans exist for the same query_hash */ var deltaKey = planHandle ?? queryHash; - var deltaExecCount = _deltaCalculator.CalculateDelta(serverId, "query_stats_exec", deltaKey, executionCount, baselineOnly: true); - var deltaWorkerTime = _deltaCalculator.CalculateDelta(serverId, "query_stats_worker", deltaKey, totalWorkerTime, baselineOnly: true); - var deltaElapsedTime = _deltaCalculator.CalculateDelta(serverId, "query_stats_elapsed", deltaKey, totalElapsedTime, baselineOnly: true); - var deltaLogicalReads = _deltaCalculator.CalculateDelta(serverId, "query_stats_reads", deltaKey, totalLogicalReads, baselineOnly: true); - var deltaLogicalWrites = _deltaCalculator.CalculateDelta(serverId, "query_stats_writes", deltaKey, totalLogicalWrites, baselineOnly: true); - var deltaPhysicalReads = _deltaCalculator.CalculateDelta(serverId, "query_stats_phys_reads", deltaKey, totalPhysicalReads, baselineOnly: true); - var deltaRows = _deltaCalculator.CalculateDelta(serverId, "query_stats_rows", deltaKey, totalRows, baselineOnly: true); - var deltaSpills = _deltaCalculator.CalculateDelta(serverId, "query_stats_spills", deltaKey, totalSpills, baselineOnly: true); + var deltaExecCount = _deltaCalculator.CalculateDelta(serverId, "query_stats_exec", deltaKey, executionCount, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); + var deltaWorkerTime = _deltaCalculator.CalculateDelta(serverId, "query_stats_worker", deltaKey, totalWorkerTime, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); + var deltaElapsedTime = _deltaCalculator.CalculateDelta(serverId, "query_stats_elapsed", deltaKey, totalElapsedTime, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); + var deltaLogicalReads = _deltaCalculator.CalculateDelta(serverId, "query_stats_reads", deltaKey, totalLogicalReads, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); + var deltaLogicalWrites = _deltaCalculator.CalculateDelta(serverId, "query_stats_writes", deltaKey, totalLogicalWrites, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); + var deltaPhysicalReads = _deltaCalculator.CalculateDelta(serverId, "query_stats_phys_reads", deltaKey, totalPhysicalReads, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); + var deltaRows = _deltaCalculator.CalculateDelta(serverId, "query_stats_rows", deltaKey, totalRows, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); + var deltaSpills = _deltaCalculator.CalculateDelta(serverId, "query_stats_spills", deltaKey, totalSpills, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); /* Appender column order must match DuckDB table definition exactly */ var row = appender.CreateRow(); diff --git a/Lite/Services/RemoteCollectorService.WaitStats.cs b/Lite/Services/RemoteCollectorService.WaitStats.cs index 35ade9ab..632e2845 100644 --- a/Lite/Services/RemoteCollectorService.WaitStats.cs +++ b/Lite/Services/RemoteCollectorService.WaitStats.cs @@ -127,9 +127,9 @@ WHERE ws.wait_time_ms > 0 foreach (var stat in waitStats) { var deltaKey = stat.WaitType; - var deltaWaitingTasks = _deltaCalculator.CalculateDelta(serverId, "wait_stats_tasks", deltaKey, stat.WaitingTasks, baselineOnly: true); - var deltaWaitTimeMs = _deltaCalculator.CalculateDelta(serverId, "wait_stats_time", deltaKey, stat.WaitTimeMs, baselineOnly: true); - var deltaSignalWaitTimeMs = _deltaCalculator.CalculateDelta(serverId, "wait_stats_signal", deltaKey, stat.SignalWaitTimeMs, baselineOnly: true); + var deltaWaitingTasks = _deltaCalculator.CalculateDelta(serverId, "wait_stats_tasks", deltaKey, stat.WaitingTasks, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); + var deltaWaitTimeMs = _deltaCalculator.CalculateDelta(serverId, "wait_stats_time", deltaKey, stat.WaitTimeMs, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); + var deltaSignalWaitTimeMs = _deltaCalculator.CalculateDelta(serverId, "wait_stats_signal", deltaKey, stat.SignalWaitTimeMs, baselineOnly: true, collectionTime: collectionTime, maxGapSeconds: 300); var row = appender.CreateRow(); row.AppendValue(GenerateCollectionId()) /* collection_id BIGINT */