Skip to content

Fix PerfMon first-value spike that skews graphs #483

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
erikdarlingdata merged 1 commit into from
Mar 9, 2026
Merged

Fix PerfMon first-value spike that skews graphs #483

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions Lite/Services/DeltaCalculator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ public async Task SeedFromDatabaseAsync(DuckDbInitializer duckDb)
/// Counter reset (value decreased): returns 0 to avoid inflated deltas from plan cache churn.
/// Thread-safe via atomic AddOrUpdate.
/// </summary>
public long CalculateDelta(int serverId, string collectorName, string key, long currentValue)
public long CalculateDelta(int serverId, string collectorName, string key, long currentValue, bool baselineOnly = false)
{
var serverCache = _cache.GetOrAdd(serverId, _ => new ConcurrentDictionary<string, ConcurrentDictionary<string, long>>());
var collectorCache = serverCache.GetOrAdd(collectorName, _ => new ConcurrentDictionary<string, long>());
Expand All @@ -74,11 +74,12 @@ public long CalculateDelta(int serverId, string collectorName, string key, long

collectorCache.AddOrUpdate(
key,
/* Add: first time seeing this key — use current value as delta
so queries that execute once still surface in top-N views */
/* Add: first time seeing this key.
baselineOnly = true: store baseline only, return 0 (for cumulative counters like perfmon).
baselineOnly = false: use current value as delta so single-execution queries surface. */
_ =>
{
delta = currentValue;
delta = baselineOnly ? 0 : currentValue;
return currentValue;
},
/* Update: compute delta atomically */
Expand Down
16 changes: 8 additions & 8 deletions Lite/Services/RemoteCollectorService.FileIo.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,14 +139,14 @@ AND vfs.database_id < 32761
foreach (var stat in fileStats)
{
var deltaKey = $"{stat.DatabaseName}|{stat.FileName}";
var deltaReads = _deltaCalculator.CalculateDelta(serverId, "file_io_reads", deltaKey, stat.NumOfReads);
var deltaWrites = _deltaCalculator.CalculateDelta(serverId, "file_io_writes", deltaKey, stat.NumOfWrites);
var deltaReadBytes = _deltaCalculator.CalculateDelta(serverId, "file_io_read_bytes", deltaKey, stat.ReadBytes);
var deltaWriteBytes = _deltaCalculator.CalculateDelta(serverId, "file_io_write_bytes", deltaKey, stat.WriteBytes);
var deltaStallReadMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_read", deltaKey, stat.IoStallReadMs);
var deltaStallWriteMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_write", deltaKey, stat.IoStallWriteMs);
var deltaStallQueuedReadMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_queued_read", deltaKey, stat.IoStallQueuedReadMs);
var deltaStallQueuedWriteMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_queued_write", deltaKey, stat.IoStallQueuedWriteMs);
var deltaReads = _deltaCalculator.CalculateDelta(serverId, "file_io_reads", deltaKey, stat.NumOfReads, baselineOnly: true);
var deltaWrites = _deltaCalculator.CalculateDelta(serverId, "file_io_writes", deltaKey, stat.NumOfWrites, baselineOnly: true);
var deltaReadBytes = _deltaCalculator.CalculateDelta(serverId, "file_io_read_bytes", deltaKey, stat.ReadBytes, baselineOnly: true);
var deltaWriteBytes = _deltaCalculator.CalculateDelta(serverId, "file_io_write_bytes", deltaKey, stat.WriteBytes, baselineOnly: true);
var deltaStallReadMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_read", deltaKey, stat.IoStallReadMs, baselineOnly: true);
var deltaStallWriteMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_write", deltaKey, stat.IoStallWriteMs, baselineOnly: true);
var deltaStallQueuedReadMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_queued_read", deltaKey, stat.IoStallQueuedReadMs, baselineOnly: true);
var deltaStallQueuedWriteMs = _deltaCalculator.CalculateDelta(serverId, "file_io_stall_queued_write", deltaKey, stat.IoStallQueuedWriteMs, baselineOnly: true);

var row = appender.CreateRow();
row.AppendValue(GenerateCollectionId())
Expand Down
4 changes: 2 additions & 2 deletions Lite/Services/RemoteCollectorService.MemoryGrants.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,8 @@ WHERE deqrs.max_target_memory_kb IS NOT NULL
foreach (var r in rows)
{
var deltaKey = $"{r.PoolId}_{r.ResourceSemaphoreId}";
var deltaTimeouts = _deltaCalculator.CalculateDelta(serverId, "memory_grants_timeouts", deltaKey, r.TimeoutErrorCount);
var deltaForced = _deltaCalculator.CalculateDelta(serverId, "memory_grants_forced", deltaKey, r.ForcedGrantCount);
var deltaTimeouts = _deltaCalculator.CalculateDelta(serverId, "memory_grants_timeouts", deltaKey, r.TimeoutErrorCount, baselineOnly: true);
var deltaForced = _deltaCalculator.CalculateDelta(serverId, "memory_grants_forced", deltaKey, r.ForcedGrantCount, baselineOnly: true);

var row = appender.CreateRow();
row.AppendValue(GenerateCollectionId())
Expand Down
2 changes: 1 addition & 1 deletion Lite/Services/RemoteCollectorService.Perfmon.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ WHERE pc.counter_name IN (

/* Delta for per-second counters */
var deltaKey = $"{objectName}|{counterName}|{instanceName}";
var deltaCntrValue = _deltaCalculator.CalculateDelta(serverId, "perfmon", deltaKey, cntrValue);
var deltaCntrValue = _deltaCalculator.CalculateDelta(serverId, "perfmon", deltaKey, cntrValue, baselineOnly: true);

var row = appender.CreateRow();
row.AppendValue(GenerateCollectionId())
Expand Down
6 changes: 3 additions & 3 deletions Lite/Services/RemoteCollectorService.WaitStats.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,9 +127,9 @@ WHERE ws.wait_time_ms > 0
foreach (var stat in waitStats)
{
var deltaKey = stat.WaitType;
var deltaWaitingTasks = _deltaCalculator.CalculateDelta(serverId, "wait_stats_tasks", deltaKey, stat.WaitingTasks);
var deltaWaitTimeMs = _deltaCalculator.CalculateDelta(serverId, "wait_stats_time", deltaKey, stat.WaitTimeMs);
var deltaSignalWaitTimeMs = _deltaCalculator.CalculateDelta(serverId, "wait_stats_signal", deltaKey, stat.SignalWaitTimeMs);
var deltaWaitingTasks = _deltaCalculator.CalculateDelta(serverId, "wait_stats_tasks", deltaKey, stat.WaitingTasks, baselineOnly: true);
var deltaWaitTimeMs = _deltaCalculator.CalculateDelta(serverId, "wait_stats_time", deltaKey, stat.WaitTimeMs, baselineOnly: true);
var deltaSignalWaitTimeMs = _deltaCalculator.CalculateDelta(serverId, "wait_stats_signal", deltaKey, stat.SignalWaitTimeMs, baselineOnly: true);

var row = appender.CreateRow();
row.AppendValue(GenerateCollectionId()) /* collection_id BIGINT */
Expand Down
Loading