diff --git a/src/sentry/tasks/seer/night_shift.py b/src/sentry/tasks/seer/night_shift.py index 74407c0b4ba2fa..fad1463296697f 100644 --- a/src/sentry/tasks/seer/night_shift.py +++ b/src/sentry/tasks/seer/night_shift.py @@ -2,14 +2,23 @@ import logging from collections.abc import Sequence +from dataclasses import dataclass from datetime import timedelta import sentry_sdk +from django.db.models import F from sentry import features, options +from sentry.constants import ObjectStatus +from sentry.models.group import Group, GroupStatus from sentry.models.organization import Organization, OrganizationStatus +from sentry.models.project import Project +from sentry.seer.autofix.constants import AutofixAutomationTuningSettings +from sentry.seer.autofix.utils import is_issue_category_eligible +from sentry.seer.models.project_repository import SeerProjectRepository from sentry.tasks.base import instrumented_task from sentry.taskworker.namespaces import seer_tasks +from sentry.types.group import PriorityLevel from sentry.utils.iterators import chunked from sentry.utils.query import RangeQuerySetWrapper @@ -17,6 +26,13 @@ NIGHT_SHIFT_DISPATCH_STEP_SECONDS = 37 NIGHT_SHIFT_SPREAD_DURATION = timedelta(hours=4) +NIGHT_SHIFT_MAX_CANDIDATES = 10 +NIGHT_SHIFT_ISSUE_FETCH_LIMIT = 100 + +# Weights for candidate scoring. Set to 0 to disable a signal. +WEIGHT_FIXABILITY = 1.0 +WEIGHT_SEVERITY = 0.0 +WEIGHT_TIMES_SEEN = 0.0 FEATURE_NAMES = [ "organizations:seer-night-shift", @@ -65,6 +81,28 @@ def schedule_night_shift() -> None: ) +@dataclass +class _ScoredCandidate: + """A candidate issue with raw signals for ranking.""" + + group_id: int + project_id: int + fixability: float + times_seen: int + severity: float + + @property + def score(self) -> float: + return ( + WEIGHT_FIXABILITY * self.fixability + + WEIGHT_SEVERITY * self.severity + + WEIGHT_TIMES_SEEN * min(self.times_seen / 1000.0, 1.0) + ) + + def __lt__(self, other: _ScoredCandidate) -> bool: + return self.score < other.score + + @instrumented_task( name="sentry.tasks.seer.night_shift.run_night_shift_for_org", namespace=seer_tasks, @@ -85,11 +123,37 @@ def run_night_shift_for_org(organization_id: int) -> None: } ) + eligible_projects = _get_eligible_projects(organization) + if not eligible_projects: + logger.info( + "night_shift.no_eligible_projects", + extra={ + "organization_id": organization_id, + "organization_slug": organization.slug, + }, + ) + return + + top_candidates = _fixability_score_strategy(eligible_projects) + logger.info( - "night_shift.org_dispatched", + "night_shift.candidates_selected", extra={ "organization_id": organization_id, "organization_slug": organization.slug, + "num_eligible_projects": len(eligible_projects), + "num_candidates": len(top_candidates), + "candidates": [ + { + "group_id": c.group_id, + "project_id": c.project_id, + "score": c.score, + "fixability": c.fixability, + "severity": c.severity, + "times_seen": c.times_seen, + } + for c in top_candidates + ], }, ) @@ -114,3 +178,60 @@ def _get_eligible_orgs_from_batch( return [] return eligible + + +def _get_eligible_projects(organization: Organization) -> list[Project]: + """Return active projects that have automation enabled and connected repos.""" + projects_with_repos = set( + SeerProjectRepository.objects.filter( + project__organization=organization, + project__status=ObjectStatus.ACTIVE, + ).values_list("project_id", flat=True) + ) + if not projects_with_repos: + return [] + + projects = Project.objects.filter(id__in=projects_with_repos) + return [ + p + for p in projects + if p.get_option("sentry:autofix_automation_tuning") != AutofixAutomationTuningSettings.OFF + ] + + +def _fixability_score_strategy( + projects: Sequence[Project], +) -> list[_ScoredCandidate]: + """ + Rank issues by existing fixability score with times_seen as tiebreaker. + Simple baseline — doesn't require any additional LLM calls. + """ + all_candidates: list[_ScoredCandidate] = [] + + for project_id_batch in chunked(projects, 100): + groups = Group.objects.filter( + project_id__in=[p.id for p in project_id_batch], + status=GroupStatus.UNRESOLVED, + seer_autofix_last_triggered__isnull=True, + seer_explorer_autofix_last_triggered__isnull=True, + ).order_by( + F("seer_fixability_score").desc(nulls_last=True), + F("times_seen").desc(), + )[:NIGHT_SHIFT_ISSUE_FETCH_LIMIT] + + for group in groups: + if not is_issue_category_eligible(group): + continue + + all_candidates.append( + _ScoredCandidate( + group_id=group.id, + project_id=group.project_id, + fixability=group.seer_fixability_score or 0.0, + times_seen=group.times_seen, + severity=(group.priority or 0) / PriorityLevel.HIGH, + ) + ) + + all_candidates.sort(reverse=True) + return all_candidates[:NIGHT_SHIFT_MAX_CANDIDATES] diff --git a/tests/sentry/tasks/seer/test_night_shift.py b/tests/sentry/tasks/seer/test_night_shift.py index b379f605b1f55a..d014767612e653 100644 --- a/tests/sentry/tasks/seer/test_night_shift.py +++ b/tests/sentry/tasks/seer/test_night_shift.py @@ -1,6 +1,13 @@ from unittest.mock import patch +from django.utils import timezone + +from sentry.models.group import GroupStatus +from sentry.seer.autofix.constants import AutofixAutomationTuningSettings +from sentry.seer.models.project_repository import SeerProjectRepository from sentry.tasks.seer.night_shift import ( + _fixability_score_strategy, + _get_eligible_projects, run_night_shift_for_org, schedule_night_shift, ) @@ -63,17 +70,148 @@ def test_skips_orgs_with_hidden_ai(self) -> None: mock_worker.apply_async.assert_not_called() +@django_db_all +class TestGetEligibleProjects(TestCase): + def test_filters_by_automation_and_repos(self) -> None: + org = self.create_organization() + + # Eligible: automation on + connected repo + eligible = self.create_project(organization=org) + eligible.update_option( + "sentry:autofix_automation_tuning", AutofixAutomationTuningSettings.MEDIUM + ) + repo = self.create_repo(project=eligible, provider="github") + SeerProjectRepository.objects.create(project=eligible, repository=repo) + + # Automation off (even with repo) + off = self.create_project(organization=org) + off.update_option("sentry:autofix_automation_tuning", AutofixAutomationTuningSettings.OFF) + repo2 = self.create_repo(project=off, provider="github") + SeerProjectRepository.objects.create(project=off, repository=repo2) + + # No connected repo + self.create_project(organization=org) + + assert _get_eligible_projects(org) == [eligible] + + @django_db_all class TestRunNightShiftForOrg(TestCase): - def test_logs_for_valid_org(self) -> None: + def _make_eligible(self, project): + project.update_option( + "sentry:autofix_automation_tuning", AutofixAutomationTuningSettings.MEDIUM + ) + repo = self.create_repo(project=project, provider="github") + SeerProjectRepository.objects.create(project=project, repository=repo) + + def test_nonexistent_org(self) -> None: + with patch("sentry.tasks.seer.night_shift.logger") as mock_logger: + run_night_shift_for_org(999999999) + mock_logger.info.assert_not_called() + + def test_no_eligible_projects(self) -> None: org = self.create_organization() + self.create_project(organization=org) with patch("sentry.tasks.seer.night_shift.logger") as mock_logger: run_night_shift_for_org(org.id) mock_logger.info.assert_called_once() - assert mock_logger.info.call_args.args[0] == "night_shift.org_dispatched" + assert mock_logger.info.call_args.args[0] == "night_shift.no_eligible_projects" + + def test_selects_candidates_and_skips_triggered(self) -> None: + org = self.create_organization() + project = self.create_project(organization=org) + self._make_eligible(project) + + high_fix = self.create_group( + project=project, + status=GroupStatus.UNRESOLVED, + seer_fixability_score=0.9, + times_seen=5, + ) + low_fix = self.create_group( + project=project, + status=GroupStatus.UNRESOLVED, + seer_fixability_score=0.2, + times_seen=100, + ) + # Already triggered — should be excluded + self.create_group( + project=project, + status=GroupStatus.UNRESOLVED, + seer_fixability_score=0.95, + seer_autofix_last_triggered=timezone.now(), + ) - def test_nonexistent_org(self) -> None: with patch("sentry.tasks.seer.night_shift.logger") as mock_logger: - run_night_shift_for_org(999999999) - mock_logger.info.assert_not_called() + run_night_shift_for_org(org.id) + + call_extra = mock_logger.info.call_args.kwargs["extra"] + assert call_extra["num_candidates"] == 2 + candidates = call_extra["candidates"] + assert candidates[0]["group_id"] == high_fix.id + assert candidates[1]["group_id"] == low_fix.id + + def test_global_ranking_across_projects(self) -> None: + org = self.create_organization() + project_a = self.create_project(organization=org) + project_b = self.create_project(organization=org) + self._make_eligible(project_a) + self._make_eligible(project_b) + + low_group = self.create_group( + project=project_a, + status=GroupStatus.UNRESOLVED, + seer_fixability_score=0.3, + ) + high_group = self.create_group( + project=project_b, + status=GroupStatus.UNRESOLVED, + seer_fixability_score=0.95, + ) + + with patch("sentry.tasks.seer.night_shift.logger") as mock_logger: + run_night_shift_for_org(org.id) + + candidates = mock_logger.info.call_args.kwargs["extra"]["candidates"] + assert candidates[0]["group_id"] == high_group.id + assert candidates[0]["project_id"] == project_b.id + assert candidates[1]["group_id"] == low_group.id + assert candidates[1]["project_id"] == project_a.id + + +@django_db_all +class TestFixabilityScoreStrategy(TestCase): + @patch("sentry.tasks.seer.night_shift.NIGHT_SHIFT_ISSUE_FETCH_LIMIT", 3) + def test_ranks_and_captures_signals(self) -> None: + project = self.create_project() + high = self.create_group( + project=project, + status=GroupStatus.UNRESOLVED, + seer_fixability_score=0.9, + times_seen=5, + priority=75, + ) + low = self.create_group( + project=project, + status=GroupStatus.UNRESOLVED, + seer_fixability_score=0.2, + times_seen=500, + ) + # NULL-scored issues should sort after scored ones even with a tight DB limit. + # Without nulls_last these would fill the limit and exclude scored issues. + for _ in range(3): + self.create_group( + project=project, + status=GroupStatus.UNRESOLVED, + seer_fixability_score=None, + times_seen=100, + ) + + result = _fixability_score_strategy([project]) + + assert result[0].group_id == high.id + assert result[0].fixability == 0.9 + assert result[0].times_seen == 5 + assert result[0].severity == 1.0 + assert result[1].group_id == low.id