From bb200fbb8e1a0eb0260a05f204a3003685d33c5f Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Sat, 28 Feb 2026 14:53:16 +0200 Subject: [PATCH] feat: support per-repo AI prompt via TESTS_ORACLE.md If TESTS_ORACLE.md exists in the repository root, its path is passed to the oracle server as prompt_file for custom AI analysis. Closes #1003 --- CLAUDE.md | 2 + README.md | 2 +- examples/.github-webhook-server.yaml | 1 + examples/config.yaml | 2 + webhook_server/libs/test_oracle.py | 7 ++++ webhook_server/tests/test_test_oracle.py | 53 ++++++++++++++++++++++++ 6 files changed, 66 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 8ae14469..aed9bb7f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -663,6 +663,8 @@ External AI service integration for test recommendations via [pr-test-oracle](ht **Trigger events:** `approved`, `pr-opened`, `pr-synchronized` +**Prompt customization:** Place `TESTS_ORACLE.md` in the repository root to customize the AI analysis prompt. + **Comment command:** `/test-oracle` (always works when configured, no trigger needed) **Module:** `webhook_server/libs/test_oracle.py` - `call_test_oracle()` shared helper diff --git a/README.md b/README.md index 70e9415d..8283e71f 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ GitHub Events → Webhook Server → Repository Management - **PyPI package publishing** for Python projects - **Tox testing integration** with configurable test environments - **Pre-commit hook validation** for code quality assurance -- **PR Test Oracle** - AI-powered test recommendations based on PR diff analysis +- **PR Test Oracle** - AI-powered test recommendations based on PR diff analysis (supports per-repo `TESTS_ORACLE.md` prompt customization) ### 👥 User Commands diff --git a/examples/.github-webhook-server.yaml b/examples/.github-webhook-server.yaml index 5843d0e0..0a5f7238 100644 --- a/examples/.github-webhook-server.yaml +++ b/examples/.github-webhook-server.yaml @@ -146,6 +146,7 @@ pr-size-thresholds: # PR Test Oracle integration (overrides global config) # Analyzes PR diffs with AI and recommends which tests to run +# Place a TESTS_ORACLE.md file in your repository root to customize the AI prompt # See: https://github.com/myk-org/pr-test-oracle test-oracle: server-url: "http://localhost:8000" diff --git a/examples/config.yaml b/examples/config.yaml index 53b011c7..f71ab7c9 100644 --- a/examples/config.yaml +++ b/examples/config.yaml @@ -109,6 +109,7 @@ branch-protection: # PR Test Oracle integration # Analyzes PR diffs with AI and recommends which tests to run +# Place a TESTS_ORACLE.md file in your repository root to customize the AI prompt # See: https://github.com/myk-org/pr-test-oracle test-oracle: server-url: "http://localhost:8000" @@ -238,6 +239,7 @@ repositories: - main # PR Test Oracle (overrides global) + # Place a TESTS_ORACLE.md file in your repository root to customize the AI prompt # See: https://github.com/myk-org/pr-test-oracle test-oracle: server-url: "http://localhost:8000" diff --git a/webhook_server/libs/test_oracle.py b/webhook_server/libs/test_oracle.py index be88c8d9..23a8b904 100644 --- a/webhook_server/libs/test_oracle.py +++ b/webhook_server/libs/test_oracle.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import os from typing import TYPE_CHECKING, Any import httpx @@ -76,6 +77,12 @@ async def call_test_oracle( if "test-patterns" in config: payload["test_patterns"] = config["test-patterns"] + # If TESTS_ORACLE.md exists in the repo, pass it as the prompt file + oracle_prompt_path = os.path.join(github_webhook.clone_repo_dir, "TESTS_ORACLE.md") + if os.path.isfile(oracle_prompt_path): + payload["prompt_file"] = oracle_prompt_path + github_webhook.logger.debug(f"{log_prefix} Using TESTS_ORACLE.md prompt from {oracle_prompt_path}") + # Call analyze try: github_webhook.logger.info(f"{log_prefix} Calling Test Oracle for {pr_url}") diff --git a/webhook_server/tests/test_test_oracle.py b/webhook_server/tests/test_test_oracle.py index b4d9599a..be130668 100644 --- a/webhook_server/tests/test_test_oracle.py +++ b/webhook_server/tests/test_test_oracle.py @@ -22,6 +22,7 @@ def mock_github_webhook(self) -> Mock: mock_webhook.logger = Mock() mock_webhook.log_prefix = "[TEST]" mock_webhook.token = TEST_GITHUB_TOKEN + mock_webhook.clone_repo_dir = "/tmp/test-repo" mock_webhook.config = Mock() mock_webhook.config.get_value = Mock( return_value={ @@ -334,3 +335,55 @@ async def test_analyze_invalid_json_logged(self, mock_github_webhook: Mock, mock mock_github_webhook.logger.error.assert_called_once() error_msg = mock_github_webhook.logger.error.call_args[0][0] assert "invalid JSON" in error_msg + + @pytest.mark.asyncio + async def test_oracle_md_included_in_payload(self, mock_github_webhook: Mock, mock_pull_request: Mock) -> None: + """Test that TESTS_ORACLE.md path is included in payload when file exists.""" + mock_github_webhook.clone_repo_dir = "/tmp/test-repo" + + with patch("webhook_server.libs.test_oracle.httpx.AsyncClient") as mock_client_cls: + mock_client = AsyncMock() + mock_client_cls.return_value.__aenter__ = AsyncMock(return_value=mock_client) + mock_client_cls.return_value.__aexit__ = AsyncMock(return_value=False) + mock_client.get.return_value = Mock(status_code=200, raise_for_status=Mock()) + mock_client.post.return_value = Mock( + status_code=200, + raise_for_status=Mock(), + json=Mock(return_value={"summary": "ok", "review_posted": True}), + ) + + with patch("webhook_server.libs.test_oracle.os.path.isfile", return_value=True): + with patch( + "asyncio.to_thread", new_callable=AsyncMock, return_value="https://github.com/test/repo/pull/1" + ): + await call_test_oracle(github_webhook=mock_github_webhook, pull_request=mock_pull_request) + + payload = mock_client.post.call_args[1]["json"] + assert payload["prompt_file"] == "/tmp/test-repo/TESTS_ORACLE.md" + + @pytest.mark.asyncio + async def test_oracle_md_not_included_when_missing( + self, mock_github_webhook: Mock, mock_pull_request: Mock + ) -> None: + """Test that prompt_file is not in payload when TESTS_ORACLE.md doesn't exist.""" + mock_github_webhook.clone_repo_dir = "/tmp/test-repo" + + with patch("webhook_server.libs.test_oracle.httpx.AsyncClient") as mock_client_cls: + mock_client = AsyncMock() + mock_client_cls.return_value.__aenter__ = AsyncMock(return_value=mock_client) + mock_client_cls.return_value.__aexit__ = AsyncMock(return_value=False) + mock_client.get.return_value = Mock(status_code=200, raise_for_status=Mock()) + mock_client.post.return_value = Mock( + status_code=200, + raise_for_status=Mock(), + json=Mock(return_value={"summary": "ok", "review_posted": True}), + ) + + with patch("webhook_server.libs.test_oracle.os.path.isfile", return_value=False): + with patch( + "asyncio.to_thread", new_callable=AsyncMock, return_value="https://github.com/test/repo/pull/1" + ): + await call_test_oracle(github_webhook=mock_github_webhook, pull_request=mock_pull_request) + + payload = mock_client.post.call_args[1]["json"] + assert "prompt_file" not in payload