From ef22ac76778e4d6fe3886b8e0d0729d6274fdbf9 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 9 Mar 2026 09:14:40 -0400 Subject: [PATCH] Fix first-collection spike for PerfMon and other cumulative counters (#482) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first collection with no baseline stored the raw cumulative DMV value as the delta, creating a massive spike that skewed the chart Y-axis. Added baselineOnly parameter to DeltaCalculator.CalculateDelta — when true, the first sighting returns 0 (baseline only) instead of the raw cumulative value. Applied to perfmon, wait stats, file I/O, and memory grant collectors. Query/proc stats keep the existing behavior so single-execution queries still surface in top-N views. Co-Authored-By: Claude Opus 4.6 --- Lite/Services/DeltaCalculator.cs | 9 +++++---- Lite/Services/RemoteCollectorService.FileIo.cs | 16 ++++++++-------- .../RemoteCollectorService.MemoryGrants.cs | 4 ++-- Lite/Services/RemoteCollectorService.Perfmon.cs | 2 +- .../Services/RemoteCollectorService.WaitStats.cs | 6 +++--- 5 files changed, 19 insertions(+), 18 deletions(-) diff --git a/Lite/Services/DeltaCalculator.cs b/Lite/Services/DeltaCalculator.cs index ec90821b..377130d2 100644 --- a/Lite/Services/DeltaCalculator.cs +++ b/Lite/Services/DeltaCalculator.cs @@ -65,7 +65,7 @@ public async Task SeedFromDatabaseAsync(DuckDbInitializer duckDb) /// Counter reset (value decreased): returns 0 to avoid inflated deltas from plan cache churn. /// Thread-safe via atomic AddOrUpdate. /// - public long CalculateDelta(int serverId, string collectorName, string key, long currentValue) + public long CalculateDelta(int serverId, string collectorName, string key, long currentValue, bool baselineOnly = false) { var serverCache = _cache.GetOrAdd(serverId, _ => new ConcurrentDictionary>()); var collectorCache = serverCache.GetOrAdd(collectorName, _ => new ConcurrentDictionary()); @@ -74,11 +74,12 @@ public long CalculateDelta(int serverId, string collectorName, string key, long collectorCache.AddOrUpdate( key, - /* Add: first time seeing this key — use current value as delta - so queries that execute once still surface in top-N views */ + /* Add: first time seeing this key. + baselineOnly = true: store baseline only, return 0 (for cumulative counters like perfmon). + baselineOnly = false: use current value as delta so single-execution queries surface. */ _ => { - delta = currentValue; + delta = baselineOnly ? 0 : currentValue; return currentValue; }, /* Update: compute delta atomically */ diff --git a/Lite/Services/RemoteCollectorService.FileIo.cs b/Lite/Services/RemoteCollectorService.FileIo.cs index 0f9ab5d3..b9d280aa 100644 --- a/Lite/Services/RemoteCollectorService.FileIo.cs +++ b/Lite/Services/RemoteCollectorService.FileIo.cs @@ -139,14 +139,14 @@ AND vfs.database_id < 32761 foreach (var stat in fileStats) { var deltaKey = $"{stat.DatabaseName}|{stat.FileName}"; - var deltaReads = _deltaCalculator.CalculateDelta(serverId, "file_io_reads", deltaKey, stat.NumOfReads); - var deltaWrites = _deltaCalculator.CalculateDelta(serverId, "file_io_writes", deltaKey, stat.NumOfWrites); - var deltaReadBytes = _deltaCalculator.CalculateDelta(serverId, "file_io_read_bytes", deltaKey, stat.ReadBytes); - var deltaWriteBytes = _deltaCalculator.CalculateDelta(serverId, "file_io_write_bytes", deltaKey, stat.WriteBytes); - var deltaStallReadMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_read", deltaKey, stat.IoStallReadMs); - var deltaStallWriteMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_write", deltaKey, stat.IoStallWriteMs); - var deltaStallQueuedReadMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_queued_read", deltaKey, stat.IoStallQueuedReadMs); - var deltaStallQueuedWriteMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_queued_write", deltaKey, stat.IoStallQueuedWriteMs); + var deltaReads = _deltaCalculator.CalculateDelta(serverId, "file_io_reads", deltaKey, stat.NumOfReads, baselineOnly: true); + var deltaWrites = _deltaCalculator.CalculateDelta(serverId, "file_io_writes", deltaKey, stat.NumOfWrites, baselineOnly: true); + var deltaReadBytes = _deltaCalculator.CalculateDelta(serverId, "file_io_read_bytes", deltaKey, stat.ReadBytes, baselineOnly: true); + var deltaWriteBytes = _deltaCalculator.CalculateDelta(serverId, "file_io_write_bytes", deltaKey, stat.WriteBytes, baselineOnly: true); + var deltaStallReadMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_read", deltaKey, stat.IoStallReadMs, baselineOnly: true); + var deltaStallWriteMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_write", deltaKey, stat.IoStallWriteMs, baselineOnly: true); + var deltaStallQueuedReadMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_queued_read", deltaKey, stat.IoStallQueuedReadMs, baselineOnly: true); + var deltaStallQueuedWriteMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_queued_write", deltaKey, stat.IoStallQueuedWriteMs, baselineOnly: true); var row = appender.CreateRow(); row.AppendValue(GenerateCollectionId()) diff --git a/Lite/Services/RemoteCollectorService.MemoryGrants.cs b/Lite/Services/RemoteCollectorService.MemoryGrants.cs index 53b3438a..a8db4a6e 100644 --- a/Lite/Services/RemoteCollectorService.MemoryGrants.cs +++ b/Lite/Services/RemoteCollectorService.MemoryGrants.cs @@ -92,8 +92,8 @@ WHERE deqrs.max_target_memory_kb IS NOT NULL foreach (var r in rows) { var deltaKey = $"{r.PoolId}_{r.ResourceSemaphoreId}"; - var deltaTimeouts = _deltaCalculator.CalculateDelta(serverId, "memory_grants_timeouts", deltaKey, r.TimeoutErrorCount); - var deltaForced = _deltaCalculator.CalculateDelta(serverId, "memory_grants_forced", deltaKey, r.ForcedGrantCount); + var deltaTimeouts = _deltaCalculator.CalculateDelta(serverId, "memory_grants_timeouts", deltaKey, r.TimeoutErrorCount, baselineOnly: true); + var deltaForced = _deltaCalculator.CalculateDelta(serverId, "memory_grants_forced", deltaKey, r.ForcedGrantCount, baselineOnly: true); var row = appender.CreateRow(); row.AppendValue(GenerateCollectionId()) diff --git a/Lite/Services/RemoteCollectorService.Perfmon.cs b/Lite/Services/RemoteCollectorService.Perfmon.cs index 9c39a86f..9bfe972f 100644 --- a/Lite/Services/RemoteCollectorService.Perfmon.cs +++ b/Lite/Services/RemoteCollectorService.Perfmon.cs @@ -180,7 +180,7 @@ WHERE pc.counter_name IN ( /* Delta for per-second counters */ var deltaKey = $"{objectName}|{counterName}|{instanceName}"; - var deltaCntrValue = _deltaCalculator.CalculateDelta(serverId, "perfmon", deltaKey, cntrValue); + var deltaCntrValue = _deltaCalculator.CalculateDelta(serverId, "perfmon", deltaKey, cntrValue, baselineOnly: true); var row = appender.CreateRow(); row.AppendValue(GenerateCollectionId()) diff --git a/Lite/Services/RemoteCollectorService.WaitStats.cs b/Lite/Services/RemoteCollectorService.WaitStats.cs index 2ba17854..0b5dcfbe 100644 --- a/Lite/Services/RemoteCollectorService.WaitStats.cs +++ b/Lite/Services/RemoteCollectorService.WaitStats.cs @@ -127,9 +127,9 @@ WHERE ws.wait_time_ms > 0 foreach (var stat in waitStats) { var deltaKey = stat.WaitType; - var deltaWaitingTasks = _deltaCalculator.CalculateDelta(serverId, "wait_stats_tasks", deltaKey, stat.WaitingTasks); - var deltaWaitTimeMs = _deltaCalculator.CalculateDelta(serverId, "wait_stats_time", deltaKey, stat.WaitTimeMs); - var deltaSignalWaitTimeMs = _deltaCalculator.CalculateDelta(serverId, "wait_stats_signal", deltaKey, stat.SignalWaitTimeMs); + var deltaWaitingTasks = _deltaCalculator.CalculateDelta(serverId, "wait_stats_tasks", deltaKey, stat.WaitingTasks, baselineOnly: true); + var deltaWaitTimeMs = _deltaCalculator.CalculateDelta(serverId, "wait_stats_time", deltaKey, stat.WaitTimeMs, baselineOnly: true); + var deltaSignalWaitTimeMs = _deltaCalculator.CalculateDelta(serverId, "wait_stats_signal", deltaKey, stat.SignalWaitTimeMs, baselineOnly: true); var row = appender.CreateRow(); row.AppendValue(GenerateCollectionId()) /* collection_id BIGINT */