Skip to content

Add plan-regression and parameter-sensitivity triage detectors (Lite) #975

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
erikdarlingdata wants to merge 1 commit into
base: dev
Choose a base branch
from
Open

Add plan-regression and parameter-sensitivity triage detectors (Lite) #975

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 66 additions & 0 deletions Lite.Tests/ScenarioTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,72 @@ public async Task EverythingOnFire_BlockingAndDeadlocksPresent()
Assert.True(facts["DEADLOCKS"].Severity > 0, "Deadlocks severity should be non-zero");
}

/* ── Parameter Sensitivity ── */

[Fact]
public async Task ParameterSensitive_FactCollectedAtHighSeverity()
{
var (stories, facts) = await RunFullPipelineAsync(s => s.SeedParameterSensitiveServerAsync());
PrintStories("PARAMETER SENSITIVITY", stories);

Assert.True(facts.ContainsKey("PARAMETER_SENSITIVITY"), "PARAMETER_SENSITIVITY should be collected");
// Worst ratio ~1000x → base severity 1.0; grant/spill/systemic amplifiers push it higher.
Assert.True(facts["PARAMETER_SENSITIVITY"].Severity >= 1.0,
$"Expected high severity, got {facts["PARAMETER_SENSITIVITY"].Severity:F2}");
}

[Fact]
public async Task ParameterSensitive_AppearsInStories()
{
var (stories, _) = await RunFullPipelineAsync(s => s.SeedParameterSensitiveServerAsync());
Assert.Contains(stories, s => s.Path.Contains("PARAMETER_SENSITIVITY"));
}

[Fact]
public async Task ParameterSensitive_ThreeOffendersWithDivergenceFlags()
{
var (_, facts) = await RunFullPipelineAsync(s => s.SeedParameterSensitiveServerAsync());

var fact = facts["PARAMETER_SENSITIVITY"];
Assert.Equal(3.0, fact.Metadata["offender_count"]);
// Worst offender: grant ratio ~1024x and spills on some parameter values only.
Assert.Equal(1.0, fact.Metadata["grant_divergence"]);
Assert.Equal(1.0, fact.Metadata["spill_divergence"]);
}

/* ── Plan Regression ── */

[Fact]
public async Task PlanRegression_FactCollectedAtHighSeverity()
{
var (stories, facts) = await RunFullPipelineAsync(s => s.SeedPlanRegressionServerAsync());
PrintStories("PLAN REGRESSION", stories);

Assert.True(facts.ContainsKey("PLAN_REGRESSION"), "PLAN_REGRESSION should be collected");
// Worst factor ~12x is past the critical threshold (10x) → base severity 1.0.
Assert.True(facts["PLAN_REGRESSION"].Severity >= 1.0,
$"Expected high severity, got {facts["PLAN_REGRESSION"].Severity:F2}");
}

[Fact]
public async Task PlanRegression_AppearsInStories()
{
var (stories, _) = await RunFullPipelineAsync(s => s.SeedPlanRegressionServerAsync());
Assert.Contains(stories, s => s.Path.Contains("PLAN_REGRESSION"));
}

[Fact]
public async Task PlanRegression_WorstFactorIsCpuDriven()
{
var (_, facts) = await RunFullPipelineAsync(s => s.SeedPlanRegressionServerAsync());

var fact = facts["PLAN_REGRESSION"];
// 1.2s vs 100ms CPU = 12x; 1.35s vs 120ms duration = 11.25x → CPU dimension wins.
Assert.Equal(12.0, fact.Metadata["worst_regression_factor"], precision: 1);
Assert.Equal(1.0, fact.Metadata["regressed_dimension"]); // 1 = cpu
Assert.Equal(1.0, fact.Metadata["offender_count"]);
}


/* ── Helper ── */

Expand Down
234 changes: 234 additions & 0 deletions Lite/Analysis/DrillDownCollector.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,12 @@ public async Task EnrichFindingsAsync(List<AnalysisFinding> findings, AnalysisCo
if (pathKeys.Any(k => k.StartsWith("BAD_ACTOR_", StringComparison.OrdinalIgnoreCase)))
await CollectBadActorDetail(finding, context);

if (pathKeys.Contains("PARAMETER_SENSITIVITY"))
await CollectParameterSensitiveQueries(finding, context);

if (pathKeys.Contains("PLAN_REGRESSION"))
await CollectRegressedQueries(finding, context);

// Plan analysis: for findings with top queries, analyze their cached plans
await CollectPlanAnalysis(finding, context);

Expand Down Expand Up @@ -171,6 +177,234 @@ ORDER BY wait_time_ms DESC
finding.DrillDown!["top_blocking_chains"] = items;
}

/// <summary>
/// Top parameter-sensitive plans behind a PARAMETER_SENSITIVITY finding.
/// Re-runs Detector A's detection (standard analysis window) for the top 5 offenders.
/// </summary>
private async Task CollectParameterSensitiveQueries(AnalysisFinding finding, AnalysisContext context)
{
using var readLock = _duckDb.AcquireReadLock();
using var connection = _duckDb.CreateConnection();
await connection.OpenAsync();

using var cmd = connection.CreateCommand();
cmd.CommandText = @"
WITH latest AS
(
SELECT
database_name,
query_hash,
query_plan_hash,
execution_count,
creation_time,
min_worker_time,
max_worker_time,
min_grant_kb,
max_grant_kb,
min_spills,
max_spills,
query_text,
ROW_NUMBER() OVER
(
PARTITION BY database_name, query_hash, query_plan_hash
ORDER BY collection_time DESC
) AS rn
FROM v_query_stats
WHERE server_id = $1
AND collection_time >= $2
AND collection_time <= $3
AND delta_execution_count > 0
)
SELECT
database_name,
query_hash,
query_plan_hash,
execution_count,
min_worker_time,
max_worker_time,
max_worker_time::DOUBLE / NULLIF(min_worker_time, 0) AS worker_ratio,
max_grant_kb::DOUBLE / NULLIF(min_grant_kb, 0) AS grant_ratio,
CASE WHEN max_spills > 0 AND min_spills = 0 THEN 1 ELSE 0 END AS spill_divergence,
LEFT(query_text, 500) AS query_text
FROM latest
WHERE rn = 1
AND min_worker_time >= 10000
AND max_worker_time >= 250000
AND execution_count >= 20
AND creation_time <= $2
AND max_worker_time::DOUBLE / NULLIF(min_worker_time, 0) >= 10
ORDER BY worker_ratio DESC
LIMIT 5";

cmd.Parameters.Add(new DuckDBParameter { Value = context.ServerId });
cmd.Parameters.Add(new DuckDBParameter { Value = context.TimeRangeStart });
cmd.Parameters.Add(new DuckDBParameter { Value = context.TimeRangeEnd });

var items = new List<object>();
using var reader = await cmd.ExecuteReaderAsync();
while (await reader.ReadAsync())
{
items.Add(new
{
database = reader.IsDBNull(0) ? "" : reader.GetString(0),
query_hash = reader.IsDBNull(1) ? "" : reader.GetString(1),
query_plan_hash = reader.IsDBNull(2) ? "" : reader.GetString(2),
execution_count = reader.IsDBNull(3) ? 0L : Convert.ToInt64(reader.GetValue(3)),
min_worker_time_us = reader.IsDBNull(4) ? 0L : Convert.ToInt64(reader.GetValue(4)),
max_worker_time_us = reader.IsDBNull(5) ? 0L : Convert.ToInt64(reader.GetValue(5)),
worker_ratio = reader.IsDBNull(6) ? 0.0 : Convert.ToDouble(reader.GetValue(6)),
grant_ratio = reader.IsDBNull(7) ? 0.0 : Convert.ToDouble(reader.GetValue(7)),
spills_on_some_inputs = !reader.IsDBNull(8) && Convert.ToInt32(reader.GetValue(8)) == 1,
query_text = reader.IsDBNull(9) ? "" : reader.GetString(9)
});
}

if (items.Count > 0)
finding.DrillDown!["parameter_sensitive_queries"] = items;
}

/// <summary>
/// Top regressed queries behind a PLAN_REGRESSION finding.
/// Re-runs Detector B's detection for the top 5 offenders. Uses the same 14-day
/// last_execution_time comparison window as the detector — NOT the standard analysis
/// window — so the days-old "best plan" baseline is present.
/// </summary>
private async Task CollectRegressedQueries(AnalysisFinding finding, AnalysisContext context)
{
using var readLock = _duckDb.AcquireReadLock();
using var connection = _duckDb.CreateConnection();
await connection.OpenAsync();

using var cmd = connection.CreateCommand();
cmd.CommandText = @"
WITH deduped AS
(
SELECT
database_name,
query_id,
plan_id,
query_plan_hash,
execution_count,
avg_cpu_time_us,
avg_duration_us,
last_execution_time,
query_text,
ROW_NUMBER() OVER
(
PARTITION BY database_name, query_id, plan_id, first_execution_time
ORDER BY collection_time DESC
) AS rn
FROM v_query_store_stats
WHERE server_id = $1
AND execution_type_desc = 'Regular'
AND last_execution_time >= $2
),
plan_agg AS
(
SELECT
database_name,
query_id,
plan_id,
any_value(query_plan_hash) AS query_plan_hash,
any_value(query_text) AS query_text,
SUM(execution_count) AS execs,
SUM(avg_cpu_time_us * execution_count) / NULLIF(SUM(execution_count), 0) AS cpu_per_exec,
SUM(avg_duration_us * execution_count) / NULLIF(SUM(execution_count), 0) AS dur_per_exec,
MAX(last_execution_time) AS last_exec
FROM deduped
WHERE rn = 1
GROUP BY database_name, query_id, plan_id
),
plan_dedup AS
(
SELECT
database_name,
query_id,
query_plan_hash,
any_value(query_text) AS query_text,
SUM(execs) AS execs,
SUM(cpu_per_exec * execs) / NULLIF(SUM(execs), 0) AS cpu_per_exec,
SUM(dur_per_exec * execs) / NULLIF(SUM(execs), 0) AS dur_per_exec,
MAX(last_exec) AS last_exec
FROM plan_agg
GROUP BY database_name, query_id, query_plan_hash
HAVING SUM(execs) >= 25
),
ranked AS
(
SELECT
*,
ROW_NUMBER() OVER (PARTITION BY database_name, query_id ORDER BY last_exec DESC) AS recency,
ROW_NUMBER() OVER (PARTITION BY database_name, query_id ORDER BY cpu_per_exec ASC) AS cheapness
FROM plan_dedup
),
compared AS
(
SELECT
l.database_name,
l.query_id,
l.query_plan_hash AS latest_plan_hash,
l.cpu_per_exec AS latest_cpu,
l.dur_per_exec AS latest_dur,
b.query_plan_hash AS best_plan_hash,
b.cpu_per_exec AS best_cpu,
b.dur_per_exec AS best_dur,
l.query_text,
GREATEST
(
l.cpu_per_exec / NULLIF(b.cpu_per_exec, 0),
l.dur_per_exec / NULLIF(b.dur_per_exec, 0)
) AS regression_factor
FROM ranked AS l
JOIN ranked AS b
ON b.database_name = l.database_name
AND b.query_id = l.query_id
AND b.cheapness = 1
WHERE l.recency = 1
AND l.query_plan_hash <> b.query_plan_hash
)
SELECT
database_name,
query_id,
latest_plan_hash,
latest_cpu,
latest_dur,
best_plan_hash,
best_cpu,
best_dur,
regression_factor,
LEFT(query_text, 500) AS query_text
FROM compared
WHERE regression_factor >= 2
ORDER BY regression_factor DESC
LIMIT 5";

cmd.Parameters.Add(new DuckDBParameter { Value = context.ServerId });
cmd.Parameters.Add(new DuckDBParameter { Value = context.TimeRangeStart.AddDays(-14) });

var items = new List<object>();
using var reader = await cmd.ExecuteReaderAsync();
while (await reader.ReadAsync())
{
items.Add(new
{
database = reader.IsDBNull(0) ? "" : reader.GetString(0),
query_id = reader.IsDBNull(1) ? 0L : Convert.ToInt64(reader.GetValue(1)),
latest_plan_hash = reader.IsDBNull(2) ? "" : reader.GetString(2),
latest_cpu_per_exec_us = reader.IsDBNull(3) ? 0.0 : Convert.ToDouble(reader.GetValue(3)),
latest_duration_per_exec_us = reader.IsDBNull(4) ? 0.0 : Convert.ToDouble(reader.GetValue(4)),
best_plan_hash = reader.IsDBNull(5) ? "" : reader.GetString(5),
best_cpu_per_exec_us = reader.IsDBNull(6) ? 0.0 : Convert.ToDouble(reader.GetValue(6)),
best_duration_per_exec_us = reader.IsDBNull(7) ? 0.0 : Convert.ToDouble(reader.GetValue(7)),
regression_factor = reader.IsDBNull(8) ? 0.0 : Convert.ToDouble(reader.GetValue(8)),
query_text = reader.IsDBNull(9) ? "" : reader.GetString(9)
});
}

if (items.Count > 0)
finding.DrillDown!["regressed_queries"] = items;
}

private async Task CollectQueriesAtSpike(AnalysisFinding finding, AnalysisContext context)
{
// Find the peak CPU time, then get queries active within 2 minutes of it
Expand Down
Loading
Loading