feat(seer): Add candidate issue selection to night shift

src/sentry/tasks/seer/night_shift.py

@@ -2,21 +2,37 @@
 
 import logging
 from collections.abc import Sequence
+from dataclasses import dataclass
 from datetime import timedelta
 
 import sentry_sdk
+from django.db.models import F
 
 from sentry import features, options
+from sentry.constants import ObjectStatus
+from sentry.models.group import Group, GroupStatus
 from sentry.models.organization import Organization, OrganizationStatus
+from sentry.models.project import Project
+from sentry.seer.autofix.constants import AutofixAutomationTuningSettings
+from sentry.seer.autofix.utils import is_issue_category_eligible
+from sentry.seer.models.project_repository import SeerProjectRepository
 from sentry.tasks.base import instrumented_task
 from sentry.taskworker.namespaces import seer_tasks
+from sentry.types.group import PriorityLevel
 from sentry.utils.iterators import chunked
 from sentry.utils.query import RangeQuerySetWrapper
 
 logger = logging.getLogger("sentry.tasks.seer.night_shift")
 
 NIGHT_SHIFT_DISPATCH_STEP_SECONDS = 37
 NIGHT_SHIFT_SPREAD_DURATION = timedelta(hours=4)
+NIGHT_SHIFT_MAX_CANDIDATES = 10
+NIGHT_SHIFT_ISSUE_FETCH_LIMIT = 100
+
+# Weights for candidate scoring. Set to 0 to disable a signal.
+WEIGHT_FIXABILITY = 1.0
+WEIGHT_SEVERITY = 0.0
+WEIGHT_TIMES_SEEN = 0.0
 
 FEATURE_NAMES = [
     "organizations:seer-night-shift",
@@ -65,6 +81,28 @@ def schedule_night_shift() -> None:
     )
 
 
+@dataclass
+class _ScoredCandidate:
+    """A candidate issue with raw signals for ranking."""
+
+    group_id: int
+    project_id: int
+    fixability: float
+    times_seen: int
+    severity: float
+
+    @property
+    def score(self) -> float:
+        return (
+            WEIGHT_FIXABILITY * self.fixability
+            + WEIGHT_SEVERITY * self.severity
+            + WEIGHT_TIMES_SEEN * min(self.times_seen / 1000.0, 1.0)
+        )
+
+    def __lt__(self, other: _ScoredCandidate) -> bool:
+        return self.score < other.score
+
+
 @instrumented_task(
     name="sentry.tasks.seer.night_shift.run_night_shift_for_org",
     namespace=seer_tasks,
@@ -85,11 +123,37 @@ def run_night_shift_for_org(organization_id: int) -> None:
         }
     )
 
+    eligible_projects = _get_eligible_projects(organization)
+    if not eligible_projects:
+        logger.info(
+            "night_shift.no_eligible_projects",
+            extra={
+                "organization_id": organization_id,
+                "organization_slug": organization.slug,
+            },
+        )
+        return
+
+    top_candidates = _fixability_score_strategy(eligible_projects)
+
     logger.info(
-        "night_shift.org_dispatched",
+        "night_shift.candidates_selected",
         extra={
             "organization_id": organization_id,
             "organization_slug": organization.slug,
+            "num_eligible_projects": len(eligible_projects),
+            "num_candidates": len(top_candidates),
+            "candidates": [
+                {
+                    "group_id": c.group_id,
+                    "project_id": c.project_id,
+                    "score": c.score,
+                    "fixability": c.fixability,
+                    "severity": c.severity,
+                    "times_seen": c.times_seen,
+                }
+                for c in top_candidates
+            ],
         },
     )
 
@@ -114,3 +178,60 @@ def _get_eligible_orgs_from_batch(
             return []
 
     return eligible
+
+
+def _get_eligible_projects(organization: Organization) -> list[Project]:
+    """Return active projects that have automation enabled and connected repos."""
+    projects_with_repos = set(
+        SeerProjectRepository.objects.filter(
+            project__organization=organization,
+            project__status=ObjectStatus.ACTIVE,
+        ).values_list("project_id", flat=True)
+    )
+    if not projects_with_repos:
+        return []
+
+    projects = Project.objects.filter(id__in=projects_with_repos)
+    return [
+        p
+        for p in projects
+        if p.get_option("sentry:autofix_automation_tuning") != AutofixAutomationTuningSettings.OFF
+    ]
+
+
+def _fixability_score_strategy(
+    projects: Sequence[Project],
+) -> list[_ScoredCandidate]:
+    """
+    Rank issues by existing fixability score with times_seen as tiebreaker.
+    Simple baseline — doesn't require any additional LLM calls.
+    """
+    all_candidates: list[_ScoredCandidate] = []
+
+    for project_id_batch in chunked(projects, 100):
+        groups = Group.objects.filter(
+            project_id__in=[p.id for p in project_id_batch],
+            status=GroupStatus.UNRESOLVED,
+            seer_autofix_last_triggered__isnull=True,
+            seer_explorer_autofix_last_triggered__isnull=True,
+        ).order_by(
+            F("seer_fixability_score").desc(nulls_last=True),
+            F("times_seen").desc(),
+        )[:NIGHT_SHIFT_ISSUE_FETCH_LIMIT]
+
+        for group in groups:
+            if not is_issue_category_eligible(group):
+                continue
+
+            all_candidates.append(
+                _ScoredCandidate(
+                    group_id=group.id,
+                    project_id=group.project_id,
+                    fixability=group.seer_fixability_score or 0.0,
+                    times_seen=group.times_seen,
+                    severity=(group.priority or 0) / PriorityLevel.HIGH,
+                )
+            )
+
+    all_candidates.sort(reverse=True)
+    return all_candidates[:NIGHT_SHIFT_MAX_CANDIDATES]

tests/sentry/tasks/seer/test_night_shift.py

@@ -1,6 +1,13 @@
 from unittest.mock import patch
 
+from django.utils import timezone
+
+from sentry.models.group import GroupStatus
+from sentry.seer.autofix.constants import AutofixAutomationTuningSettings
+from sentry.seer.models.project_repository import SeerProjectRepository
 from sentry.tasks.seer.night_shift import (
+    _fixability_score_strategy,
+    _get_eligible_projects,
     run_night_shift_for_org,
     schedule_night_shift,
 )
@@ -63,17 +70,148 @@ def test_skips_orgs_with_hidden_ai(self) -> None:
             mock_worker.apply_async.assert_not_called()
 
 
+@django_db_all
+class TestGetEligibleProjects(TestCase):
+    def test_filters_by_automation_and_repos(self) -> None:
+        org = self.create_organization()
+
+        # Eligible: automation on + connected repo
+        eligible = self.create_project(organization=org)
+        eligible.update_option(
+            "sentry:autofix_automation_tuning", AutofixAutomationTuningSettings.MEDIUM
+        )
+        repo = self.create_repo(project=eligible, provider="github")
+        SeerProjectRepository.objects.create(project=eligible, repository=repo)
+
+        # Automation off (even with repo)
+        off = self.create_project(organization=org)
+        off.update_option("sentry:autofix_automation_tuning", AutofixAutomationTuningSettings.OFF)
+        repo2 = self.create_repo(project=off, provider="github")
+        SeerProjectRepository.objects.create(project=off, repository=repo2)
+
+        # No connected repo
+        self.create_project(organization=org)
+
+        assert _get_eligible_projects(org) == [eligible]
+
+
 @django_db_all
 class TestRunNightShiftForOrg(TestCase):
-    def test_logs_for_valid_org(self) -> None:
+    def _make_eligible(self, project):
+        project.update_option(
+            "sentry:autofix_automation_tuning", AutofixAutomationTuningSettings.MEDIUM
+        )
+        repo = self.create_repo(project=project, provider="github")
+        SeerProjectRepository.objects.create(project=project, repository=repo)
+
+    def test_nonexistent_org(self) -> None:
+        with patch("sentry.tasks.seer.night_shift.logger") as mock_logger:
+            run_night_shift_for_org(999999999)
+            mock_logger.info.assert_not_called()
+
+    def test_no_eligible_projects(self) -> None:
         org = self.create_organization()
+        self.create_project(organization=org)
 
         with patch("sentry.tasks.seer.night_shift.logger") as mock_logger:
             run_night_shift_for_org(org.id)
             mock_logger.info.assert_called_once()
-            assert mock_logger.info.call_args.args[0] == "night_shift.org_dispatched"
+            assert mock_logger.info.call_args.args[0] == "night_shift.no_eligible_projects"
+
+    def test_selects_candidates_and_skips_triggered(self) -> None:
+        org = self.create_organization()
+        project = self.create_project(organization=org)
+        self._make_eligible(project)
+
+        high_fix = self.create_group(
+            project=project,
+            status=GroupStatus.UNRESOLVED,
+            seer_fixability_score=0.9,
+            times_seen=5,
+        )
+        low_fix = self.create_group(
+            project=project,
+            status=GroupStatus.UNRESOLVED,
+            seer_fixability_score=0.2,
+            times_seen=100,
+        )
+        # Already triggered — should be excluded
+        self.create_group(
+            project=project,
+            status=GroupStatus.UNRESOLVED,
+            seer_fixability_score=0.95,
+            seer_autofix_last_triggered=timezone.now(),
+        )
 
-    def test_nonexistent_org(self) -> None:
         with patch("sentry.tasks.seer.night_shift.logger") as mock_logger:
-            run_night_shift_for_org(999999999)
-            mock_logger.info.assert_not_called()
+            run_night_shift_for_org(org.id)
+
+            call_extra = mock_logger.info.call_args.kwargs["extra"]
+            assert call_extra["num_candidates"] == 2
+            candidates = call_extra["candidates"]
+            assert candidates[0]["group_id"] == high_fix.id
+            assert candidates[1]["group_id"] == low_fix.id
+
+    def test_global_ranking_across_projects(self) -> None:
+        org = self.create_organization()
+        project_a = self.create_project(organization=org)
+        project_b = self.create_project(organization=org)
+        self._make_eligible(project_a)
+        self._make_eligible(project_b)
+
+        low_group = self.create_group(
+            project=project_a,
+            status=GroupStatus.UNRESOLVED,
+            seer_fixability_score=0.3,
+        )
+        high_group = self.create_group(
+            project=project_b,
+            status=GroupStatus.UNRESOLVED,
+            seer_fixability_score=0.95,
+        )
+
+        with patch("sentry.tasks.seer.night_shift.logger") as mock_logger:
+            run_night_shift_for_org(org.id)
+
+            candidates = mock_logger.info.call_args.kwargs["extra"]["candidates"]
+            assert candidates[0]["group_id"] == high_group.id
+            assert candidates[0]["project_id"] == project_b.id
+            assert candidates[1]["group_id"] == low_group.id
+            assert candidates[1]["project_id"] == project_a.id
+
+
+@django_db_all
+class TestFixabilityScoreStrategy(TestCase):
+    @patch("sentry.tasks.seer.night_shift.NIGHT_SHIFT_ISSUE_FETCH_LIMIT", 3)
+    def test_ranks_and_captures_signals(self) -> None:
+        project = self.create_project()
+        high = self.create_group(
+            project=project,
+            status=GroupStatus.UNRESOLVED,
+            seer_fixability_score=0.9,
+            times_seen=5,
+            priority=75,
+        )
+        low = self.create_group(
+            project=project,
+            status=GroupStatus.UNRESOLVED,
+            seer_fixability_score=0.2,
+            times_seen=500,
+        )
+        # NULL-scored issues should sort after scored ones even with a tight DB limit.
+        # Without nulls_last these would fill the limit and exclude scored issues.
+        for _ in range(3):
+            self.create_group(
+                project=project,
+                status=GroupStatus.UNRESOLVED,
+                seer_fixability_score=None,
+                times_seen=100,
+            )
+
+        result = _fixability_score_strategy([project])
+
+        assert result[0].group_id == high.id
+        assert result[0].fixability == 0.9
+        assert result[0].times_seen == 5
+        assert result[0].severity == 1.0
+        assert result[1].group_id == low.id