diff --git a/.coveragerc b/.coveragerc index f58f2204..898cc066 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,8 +1,10 @@ [run] -source = openwisp_utils -parallel = true -concurrency = multiprocessing +source = + openwisp_utils +branch = True +parallel = True + +[report] omit = - /*/test* - /*/__init__.py - /*/migrations/* + */migrations/* + */__init__.py diff --git a/.github/workflows/ci-failure-bot.yml b/.github/workflows/ci-failure-bot.yml new file mode 100644 index 00000000..c5d67b34 --- /dev/null +++ b/.github/workflows/ci-failure-bot.yml @@ -0,0 +1,43 @@ +--- +name: CI Failure Bot + +on: + workflow_run: + workflows: ["OpenWISP Utils CI Build"] + types: + - completed + +permissions: + issues: write + pull-requests: write + contents: read + +jobs: + ci-failure-bot: + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'failure' && !contains(github.event.workflow_run.actor.login, 'dependabot') }} + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + pip install -e .[github_actions] + + - name: Run CI Failure Bot + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }} + REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number || '' }} + run: | + echo "Starting CI Failure Bot..." + python -m openwisp_utils.bots.ci_failure.bot + echo "CI Failure Bot completed successfully" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 04415de3..a163712d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,7 +68,7 @@ jobs: run: | pip install -U pip wheel setuptools pip install -U -r requirements-test.txt - pip install -e .[qa,rest,selenium,releaser] + pip install -e .[qa,rest,selenium,releaser,github_actions] pip install ${{ matrix.django-version }} sudo npm install -g prettier diff --git a/docs/developer/reusable-github-utils.rst b/docs/developer/reusable-github-utils.rst index 4f9290f9..fb9073b3 100644 --- a/docs/developer/reusable-github-utils.rst +++ b/docs/developer/reusable-github-utils.rst @@ -56,6 +56,133 @@ times with a 30 second delay between attempts. attempts, the action will exit with a non-zero status, causing the workflow to fail. +CI Failure Bot +~~~~~~~~~~~~~~ + +This GitHub workflow analyzes failed CI builds when a pull request context +is available and provides intelligent feedback to contributors using +AI-powered analysis. + +The bot examines build logs, PR changes, and workflow context to generate +specific, actionable guidance that helps contributors fix issues quickly. + +**Inputs** + +- ``GEMINI_API_KEY`` (optional): Google Gemini API key for AI analysis. If + not provided, the bot uses fallback responses +- ``GEMINI_MODEL`` (optional): Gemini model to use. Defaults to + ``gemini-2.5-flash`` + +**Usage Example** + +You can use this workflow in your repository as follows: + +.. code-block:: yaml + + name: CI Failure Bot + + on: + workflow_run: + workflows: ["OpenWISP Utils CI Build"] + types: + - completed + + permissions: + issues: write + pull-requests: write + contents: read + + jobs: + ci-failure-bot: + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'failure' && !contains(github.event.workflow_run.actor.login, 'dependabot') }} + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + pip install -e .[github_actions] + + - name: Run CI Failure Bot + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }} + REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number || '' }} + run: python -m openwisp_utils.bots.ci_failure.bot + +This example automatically triggers when the "OpenWISP Utils CI Build" +workflow fails, analyzes the failure using Gemini AI, and posts +intelligent feedback to the associated pull request. + +**Features** + +- **Automatic triggering**: Responds to CI build failures in pull requests +- **AI-powered analysis**: Uses Google Gemini to analyze failure logs and + provide specific guidance +- **Targeted remediation**: Suggests QA commands, test commands, or setup + fixes depending on which checks failed (no generic advice) +- **Intelligent responses**: Provides direct, actionable feedback based on + actual failure context +- **Comment deduplication**: Updates existing comments instead of creating + duplicates +- **Dependabot exclusion**: Automatically skips dependency update PRs +- **Fork detection**: Skips external PRs for security +- **Fallback handling**: Provides basic guidance if AI analysis fails + +**Configuration** ++++++++++++++++++ + +Repository Secrets +++++++++++++++++++ + +The following secrets can be configured in the repository for enhanced +functionality: + +- ``GEMINI_API_KEY``: Google Gemini API key for AI analysis (optional - + fallback responses used if not provided) + +Environment Variables ++++++++++++++++++++++ + +Optional environment variables for customization: + +- ``GEMINI_MODEL``: Gemini model to use (default: ``gemini-2.5-flash``) + +**Limitations** + +- **Pull request context availability**: When triggered via + ``workflow_run``, GitHub may not always provide an associated pull + request (for example, when builds are triggered by pushes or scheduled + workflows). In these cases, the bot will not post a comment, as no pull + request context is available. +- **Optional Gemini API**: Google Gemini API access enhances analysis + quality, but the bot provides fallback responses when unavailable +- **Privacy consideration**: PR diffs and build logs are sent to Google's + Gemini AI service for analysis when API key is provided. Organizations + with sensitive codebases should review Google's data handling policies +- **API costs**: Each CI failure with Gemini enabled triggers an API call. + Monitor usage to manage costs, especially in repositories with frequent + CI failures +- Analysis quality depends on error log clarity +- May not handle very complex or unusual failure scenarios +- Skips dependabot PRs to avoid unnecessary noise + +.. note:: + + If the Gemini API is unavailable or analysis fails, the bot provides a + fallback response with standard OpenWISP QA guidance. Critical errors + are logged in GitHub Actions, but the workflow is designed to complete + safely without blocking contributor feedback. + GitHub Workflows ---------------- diff --git a/openwisp_utils/bots/__init__.py b/openwisp_utils/bots/__init__.py new file mode 100644 index 00000000..3f09202f --- /dev/null +++ b/openwisp_utils/bots/__init__.py @@ -0,0 +1 @@ +default_app_config = "openwisp_utils.bots.apps.BotsConfig" diff --git a/openwisp_utils/bots/apps.py b/openwisp_utils/bots/apps.py new file mode 100644 index 00000000..a9b283d9 --- /dev/null +++ b/openwisp_utils/bots/apps.py @@ -0,0 +1,7 @@ +from django.apps import AppConfig + + +class BotsConfig(AppConfig): + name = "openwisp_utils.bots" + label = "openwisp_utils_bots" + verbose_name = "OpenWISP Bots" diff --git a/openwisp_utils/bots/ci_failure/__init__.py b/openwisp_utils/bots/ci_failure/__init__.py new file mode 100644 index 00000000..56ff30d7 --- /dev/null +++ b/openwisp_utils/bots/ci_failure/__init__.py @@ -0,0 +1 @@ +# CI Failure Bot diff --git a/openwisp_utils/bots/ci_failure/bot.py b/openwisp_utils/bots/ci_failure/bot.py new file mode 100644 index 00000000..a9d837d2 --- /dev/null +++ b/openwisp_utils/bots/ci_failure/bot.py @@ -0,0 +1,483 @@ +#!/usr/bin/env python3 +"""CI Failure Bot - AI-powered analysis of build failures using Gemini""" +import io +import json +import os +import subprocess +import zipfile + +import requests +from github import Github, GithubException + +try: + import google.generativeai as genai +except ImportError: + genai = None + + +class CIFailureBot: + def __init__(self): + self.github_token = os.environ.get("GITHUB_TOKEN") + self.gemini_api_key = os.environ.get("GEMINI_API_KEY") + self.workflow_run_id = os.environ.get("WORKFLOW_RUN_ID") + self.repository_name = os.environ.get("REPOSITORY") + self.pr_number = os.environ.get("PR_NUMBER") + + # Initialize with None values if missing - bot will still try to comment + self.github = None + self.repo = None + + if self.github_token and self.repository_name: + try: + self.github = Github(self.github_token) + self.repo = self.github.get_repo(self.repository_name) + except Exception as e: + print(f"Warning: Could not initialize GitHub client: {e}") + else: + missing = [] + if not self.github_token: + missing.append("GITHUB_TOKEN") + if not self.repository_name: + missing.append("REPOSITORY") + print(f"Warning: Missing environment variables: {', '.join(missing)}") + + if self.gemini_api_key and genai is not None: + try: + genai.configure(api_key=self.gemini_api_key) + self.model_name = os.environ.get("GEMINI_MODEL", "gemini-2.5-flash") + self.model = genai.GenerativeModel(self.model_name) + except Exception as e: + print(f"Warning: Could not initialize Gemini: {e}") + self.model = None + else: + if not self.gemini_api_key: + print( + "Warning: GEMINI_API_KEY not provided, will use fallback responses" + ) + else: + print( + "Warning: google-generativeai not installed, will use fallback responses" + ) + self.model = None + + def get_build_logs(self): + """Get actual build logs and error output from failed jobs""" + if not self.repo: + print("GitHub client not initialized") + return [] + if not self.workflow_run_id: + print("No WORKFLOW_RUN_ID provided") + return [] + try: + workflow_run_id = int(self.workflow_run_id) + workflow_run = self.repo.get_workflow_run(workflow_run_id) + print( + f"Fetching jobs for workflow run {workflow_run_id}: {workflow_run.name}" + ) + jobs = workflow_run.jobs() + build_logs = [] + for job in jobs: + print(f"Job: {job.name} - conclusion: {job.conclusion}") + if job.conclusion == "failure": + # Always add job info with name for classification + job_entry = {"job_name": job.name} + logs_url = job.logs_url + if logs_url: + try: + headers = { + "Authorization": f"token {self.github_token}", + "Accept": "application/vnd.github.v3+json", + } + response = requests.get( + logs_url, headers=headers, timeout=30 + ) + response.raise_for_status() + raw = response.content + if raw[:2] == b"PK": + with zipfile.ZipFile(io.BytesIO(raw)) as zf: + parts = [] + for name in zf.namelist(): + if name.endswith(".txt"): + parts.append( + zf.read(name).decode("utf-8", "replace") + ) + log_text = "\n".join(parts).strip() + else: + log_text = raw.decode("utf-8", "replace") + if len(log_text) > 5000: + log_text = ( + log_text[:2000] + + "\n\n[...middle truncated...]\n\n" + + log_text[-3000:] + ) + job_entry["logs"] = log_text + print(f" Fetched {len(log_text)} chars of logs") + except (requests.RequestException, zipfile.BadZipFile) as e: + print(f" Warning: Could not fetch logs: {e}") + job_entry["logs"] = "" + else: + print(" No logs_url available") + job_entry["logs"] = "" + build_logs.append(job_entry) + # Add step-level failure info + for step in getattr(job, "steps", []): + if step.conclusion == "failure": + print(f" Failed step: {step.name}") + build_logs.append( + { + "job_name": job.name, + "step_name": step.name, + "step_number": step.number, + } + ) + print(f"Total build_logs entries: {len(build_logs)}") + return build_logs + except (GithubException, ValueError) as e: + print(f"Error getting build logs: {e}") + return [] + + def get_pr_diff(self): + """Get the PR diff using local git""" + if not self.repo: + print("GitHub client not initialized") + return None + if not self.pr_number: + return None + try: + pr_num = int(self.pr_number) + except ValueError as e: + print(f"Invalid PR number: {e}") + return None + try: + pr = self.repo.get_pull(pr_num) + except GithubException as e: + print(f"Error fetching PR: {e}") + return None + try: + result = subprocess.run( + ["git", "diff", f"origin/{self.repo.default_branch}"], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + except subprocess.SubprocessError as e: + print(f"Error running git diff: {e}") + return None + if result.returncode != 0 or not result.stdout: + return None + diff_text = result.stdout + if len(diff_text) > 8000: + diff_text = ( + diff_text[:4000] + + "\n\n[...middle truncated...]\n\n" + + diff_text[-4000:] + ) + return { + "title": pr.title, + "body": pr.body or "", + "diff": diff_text, + } + + def classify_failure(self, build_logs): + """Classify failure type based on job names and logs""" + if not build_logs: + return "unknown" + + failure_types = set() + for log_entry in build_logs: + job_name = log_entry.get("job_name", "").lower() + logs = log_entry.get("logs", "").lower() + + # Check for QA/formatting failures + if any(x in job_name for x in ["qa", "lint", "format", "style"]): + failure_types.add("qa") + elif any(x in logs for x in ["flake8", "black", "isort", "pep 8"]): + failure_types.add("qa") + + # Check for test failures + if any(x in job_name for x in ["test", "pytest", "unittest"]): + failure_types.add("tests") + elif any(x in logs for x in ["test failed", "assertion", "pytest"]): + failure_types.add("tests") + + # Check for setup/dependency failures + if any( + x in logs + for x in ["modulenotfounderror", "importerror", "no module named"] + ): + failure_types.add("setup") + + if not failure_types: + return "unknown" + elif len(failure_types) == 1: + return next(iter(failure_types)) + else: + return "mixed" + + def get_failed_jobs_summary(self, build_logs): + """Extract summary of failed jobs and steps""" + failed_jobs = [] + for log_entry in build_logs: + if "job_name" in log_entry: + job_info = {"name": log_entry["job_name"]} + if "step_name" in log_entry: + job_info["step"] = log_entry["step_name"] + failed_jobs.append(job_info) + return failed_jobs + + def analyze_with_gemini(self, build_logs, pr_diff): + """Send context to Gemini for intelligent analysis""" + if not self.model: + return self.fallback_response() + + if not self.repository_name: + return self.fallback_response() + + # Classify failure and get context + failure_type = self.classify_failure(build_logs) + failed_jobs = self.get_failed_jobs_summary(build_logs) + + project_name = self.repository_name.split("/")[-1] + repo_url = f"https://github.com/{self.repository_name}" + build_logs_json = json.dumps(build_logs, indent=2) + failed_jobs_json = json.dumps(failed_jobs, indent=2) + + if pr_diff: + pr_diff_json = json.dumps(pr_diff, indent=2) + else: + pr_diff_json = "No PR associated" + + context = f""" +### ROLE +You are analyzing CI build failures for OpenWISP. Provide diagnosis AND remediation advice. + +### INPUT CONTEXT +1. **Failure Type:** {failure_type} +2. **Failed Jobs:** {failed_jobs_json} +3. **Build Logs:** {build_logs_json} +4. **PR Diff:** {pr_diff_json} +5. **Project:** {project_name} +6. **Repository:** {repo_url} + +### CRITICAL RULES - MUST FOLLOW EXACTLY + +**Rule 1: Suggest ONLY remediation for failures that actually occurred** +- If failure_type != "qa", DO NOT mention QA commands +- If failure_type != "tests", DO NOT mention test commands +- If failure_type != "setup", DO NOT mention dependency commands +- NEVER suggest fixes for checks that passed + +**Rule 2: Remediation by failure type** + +**If failure_type = "qa":** +```bash +pip install -e .[qa] +openwisp-qa-format +./run-qa-checks +``` +Link: https://openwisp.io/docs/stable/developer/contributing.html +DO NOT mention ./runtests + +**If failure_type = "tests":** +```bash +./runtests +``` +Review test logic and fix failing assertions. +DO NOT mention QA commands (pip install -e .[qa], openwisp-qa-format, ./run-qa-checks) + +**If failure_type = "setup":** +Check dependencies and imports. +```bash +pip install -e .[qa] +``` +Focus on ModuleNotFoundError or ImportError. +DO NOT mention formatting or tests unless they also failed. + +**If failure_type = "mixed":** +List each issue type separately with appropriate commands. +Example: "Fix formatting issues first, then address test failures." + +**If failure_type = "unknown":** +```bash +./run-qa-checks +./runtests +``` +General troubleshooting only. + +**Rule 3: Response format** +1. **Technical Diagnosis:** 2-3 sentences stating which files/tests failed and why +2. **Required Actions:** Commands in code blocks, based ONLY on failure_type + +**Rule 4: Prohibited behaviors** +- DO NOT hallucinate failures that didn't occur +- DO NOT suggest "run all checks" when only one type failed +- DO NOT add extra commands beyond what failure_type requires +- DO NOT use vague language like "might need" or "consider" + +### EXAMPLES + +**Example 1 - QA failure only:** +"The file bad_format.py contains PEP 8 violations (missing spaces around operators). +The Build / Python 3.11 job failed due to formatting issues. + +Required Actions: +```bash +pip install -e .[qa] +openwisp-qa-format +./run-qa-checks +``` +See [OpenWISP contributing guidelines]( +https://openwisp.io/docs/stable/developer/contributing.html)." + +**Example 2 - Test failure only:** +"The test test_always_fails in test_fail.py asserts 1 == 2, which is false. +The Build / Python 3.11 job failed. + +Required Actions: +Review and fix the failing test logic: +```bash +./runtests +```" + +**Example 3 - Setup failure only:** +"Import failed: ModuleNotFoundError for 'nonexistent_module' in final_test.py. +The Build / Python 3.11 job failed. + +Required Actions: +Check dependencies and install requirements: +```bash +pip install -e .[qa] +```" + +Analyze the failure and provide diagnosis + remediation following these rules: +""" + try: + response = self.model.generate_content(context) + return response.text.strip() + except Exception as e: + print(f"Error calling Gemini API: {e}") + return self.fallback_response() + + def fallback_response(self): + """Fallback response if Gemini fails""" + return """ +The build failed. Automated analysis is unavailable. + +**Recommended Actions:** +```bash +pip install -e .[qa] +./run-qa-checks +./runtests +``` + +See the [OpenWISP contributing guidelines]( +https://openwisp.io/docs/stable/developer/contributing.html) for more details. +""".strip() + + def post_comment(self, message): + """Post or update comment on PR""" + if not self.pr_number: + print("No PR number, skipping comment") + return + if not self.github or not self.repo: + print("GitHub client not initialized, cannot post comment") + return + marker = "" + message_with_marker = ( + f"{marker}\n🤖 **CI Failure Bot** (AI-powered)\n\n{message}" + ) + try: + pr_num = int(self.pr_number) + except ValueError as e: + print(f"Invalid PR number: {e}") + return + try: + pr = self.repo.get_pull(pr_num) + except GithubException as e: + print(f"Error fetching PR: {e}") + return + try: + existing_comments = pr.get_issue_comments() + for comment in existing_comments: + if marker in comment.body: + print("Bot comment already exists, updating it") + comment.edit(message_with_marker) + return + except GithubException as e: + print(f"Error checking existing comments: {e}") + return # Don't create duplicate if listing fails + try: + pr.create_issue_comment(message_with_marker) + print(f"Posted comment to PR #{pr_num}") + except GithubException as e: + print(f"Error posting comment: {e}") + + def run(self): + """Main execution flow - adapted for workflow_run""" + message = None + should_skip = False + skip_reason = "" + try: + print("CI Failure Bot starting - AI-powered analysis") + + # Early guard for repo + if not self.repo: + print("GitHub client not initialized, cannot proceed") + return + + # Check for skip conditions (but don't return early) + try: + if self.workflow_run_id: + workflow_run = self.repo.get_workflow_run(int(self.workflow_run_id)) + if ( + workflow_run.actor + and "dependabot" in workflow_run.actor.login.lower() + ): + should_skip = True + skip_reason = f"dependabot PR from {workflow_run.actor.login}" + if self.pr_number and not should_skip: + try: + pr_num = int(self.pr_number) + pr = self.repo.get_pull(pr_num) + if pr.head.repo is None: + should_skip = True + skip_reason = "PR with deleted head repository" + elif pr.head.repo.full_name != self.repository_name: + should_skip = True + skip_reason = f"fork PR from {pr.head.repo.full_name}" + except (GithubException, ValueError) as e: + print(f"Warning: Could not check fork status: {e}") + except (GithubException, AttributeError, ValueError) as e: + print(f"Warning: Could not check actor: {e}") + # Determine message based on context + if not self.pr_number: + print("No PR context available - workflow_run without PR") + message = None + elif should_skip: + print(f"Skipping: {skip_reason}") + return + else: + # We have PR context, proceed with analysis + build_logs = self.get_build_logs() + pr_diff = self.get_pr_diff() + if not build_logs and not pr_diff: + print("No build logs or PR diff found, using fallback response") + message = self.fallback_response() + else: + print("Analyzing failure with Gemini AI...") + message = self.analyze_with_gemini(build_logs, pr_diff) + except Exception as e: + print(f"Error in analysis: {e}") + message = self.fallback_response() + # Single comment decision point + if message: + self.post_comment(message) + else: + print("No PR context available, no comment posted (expected)") + print("CI Failure Bot completed successfully") + + +if __name__ == "__main__": + bot = CIFailureBot() + bot.run() diff --git a/openwisp_utils/bots/ci_failure/tests/__init__.py b/openwisp_utils/bots/ci_failure/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/openwisp_utils/bots/ci_failure/tests/test_bot.py b/openwisp_utils/bots/ci_failure/tests/test_bot.py new file mode 100644 index 00000000..c7f9f37b --- /dev/null +++ b/openwisp_utils/bots/ci_failure/tests/test_bot.py @@ -0,0 +1,676 @@ +import os +from unittest.mock import Mock, patch + +from django.test import TestCase +from openwisp_utils.bots.ci_failure.bot import CIFailureBot + + +class TestCIFailureBot(TestCase): + def setUp(self): + self.env_vars = { + "GITHUB_TOKEN": "test_token", + "GEMINI_API_KEY": "test_gemini_key", + "WORKFLOW_RUN_ID": "12345", + "REPOSITORY": "openwisp/openwisp-utils", + "PR_NUMBER": "1", + } + self.env_patcher = patch.dict(os.environ, self.env_vars) + self.env_patcher.start() + self.github_patcher = patch("openwisp_utils.bots.ci_failure.bot.Github") + self.genai_patcher = patch("openwisp_utils.bots.ci_failure.bot.genai") + self.mock_github = self.github_patcher.start() + self.mock_genai = self.genai_patcher.start() + self.mock_repo = Mock() + self.mock_github.return_value.get_repo.return_value = self.mock_repo + self.mock_model = Mock() + self.mock_genai.GenerativeModel.return_value = self.mock_model + + def tearDown(self): + if hasattr(self, "env_patcher"): + self.env_patcher.stop() + if hasattr(self, "github_patcher"): + self.github_patcher.stop() + if hasattr(self, "genai_patcher"): + self.genai_patcher.stop() + + def _mock_workflow_run(self, actor="user", jobs=None): + """Helper to create mock workflow run""" + mock_run = Mock() + mock_actor = Mock() + mock_actor.login = actor + mock_run.actor = mock_actor + if jobs is not None: + mock_run.jobs.return_value = jobs + self.mock_repo.get_workflow_run.return_value = mock_run + return mock_run + + def _mock_pr(self, full_name="openwisp/openwisp-utils", deleted_fork=False): + """Helper to create mock PR""" + mock_pr = Mock() + if deleted_fork: + mock_pr.head.repo = None + else: + mock_pr.head.repo = Mock() + mock_pr.head.repo.full_name = full_name + mock_pr.get_issue_comments.return_value = [] + self.mock_repo.get_pull.return_value = mock_pr + return mock_pr + + def _mock_failed_job(self, logs_url="https://api.github.com/logs/123", steps=None): + """Helper to create mock failed job""" + job = Mock() + job.name = "test-job" + job.conclusion = "failure" + job.logs_url = logs_url + job.steps = steps or [] + return job + + def test_init_success(self): + bot = CIFailureBot() + self.assertEqual(bot.github_token, "test_token") + self.assertEqual(bot.gemini_api_key, "test_gemini_key") + self.assertEqual(bot.workflow_run_id, "12345") + self.assertEqual(bot.repository_name, "openwisp/openwisp-utils") + self.assertEqual(bot.pr_number, "1") + self.mock_github.assert_called_once_with("test_token") + self.mock_genai.configure.assert_called_once_with(api_key="test_gemini_key") + + def test_init_without_gemini_key(self): + env_vars_no_gemini = { + "GITHUB_TOKEN": "test_token", + "WORKFLOW_RUN_ID": "12345", + "REPOSITORY": "openwisp/openwisp-utils", + "PR_NUMBER": "1", + } + with patch.dict(os.environ, env_vars_no_gemini, clear=True): + bot = CIFailureBot() + self.assertIsNone(bot.model) + self.mock_genai.configure.assert_not_called() + + def test_classify_failure_qa(self): + bot = CIFailureBot() + logs = [{"job_name": "Build / Python 3.11", "logs": "flake8 error"}] + self.assertEqual(bot.classify_failure(logs), "qa") + + def test_classify_failure_tests(self): + bot = CIFailureBot() + logs = [{"job_name": "unit-tests", "logs": "test failed"}] + self.assertEqual(bot.classify_failure(logs), "tests") + + def test_classify_failure_setup(self): + bot = CIFailureBot() + logs = [ + {"job_name": "build", "logs": "ModuleNotFoundError: No module named 'xyz'"} + ] + self.assertEqual(bot.classify_failure(logs), "setup") + + def test_classify_failure_mixed(self): + bot = CIFailureBot() + logs = [ + {"job_name": "Build / Python 3.11", "logs": "flake8 error"}, + {"job_name": "Build / Python 3.11", "logs": "test failed"}, + ] + self.assertEqual(bot.classify_failure(logs), "mixed") + + def test_classify_failure_unknown(self): + bot = CIFailureBot() + logs = [{"job_name": "unknown-job", "logs": "some error"}] + self.assertEqual(bot.classify_failure(logs), "unknown") + + @patch("openwisp_utils.bots.ci_failure.bot.requests.get") + def test_get_build_logs_success(self, mock_requests): + bot = CIFailureBot() + mock_workflow_run = Mock() + mock_step = Mock() + mock_step.conclusion = "failure" + mock_step.name = "Run tests" + mock_step.number = 1 + mock_job = self._mock_failed_job(steps=[mock_step]) + mock_workflow_run.jobs.return_value = [mock_job] + self.mock_repo.get_workflow_run.return_value = mock_workflow_run + mock_response = Mock() + mock_response.content = b"Error: Test failed at line 42\n" * 1000 + mock_response.raise_for_status = Mock() + mock_requests.return_value = mock_response + logs = bot.get_build_logs() + self.assertEqual(len(logs), 2) + self.assertIn("job_name", logs[0]) + self.assertIn("logs", logs[0]) + self.assertEqual(logs[1]["step_name"], "Run tests") + + @patch("openwisp_utils.bots.ci_failure.bot.requests.get") + def test_get_build_logs_zip_format(self, mock_requests): + """Test ZIP-encoded log extraction""" + import io + import zipfile + + bot = CIFailureBot() + zip_buffer = io.BytesIO() + with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zip_file: + zip_file.writestr("failed_job.txt", "Error: test failed\nstack trace here") + zip_content = zip_buffer.getvalue() + mock_workflow_run = Mock() + step = Mock() + step.name = "failing-step" + step.conclusion = "failure" + step.number = 1 + mock_job = self._mock_failed_job(steps=[step]) + mock_workflow_run.jobs.return_value = [mock_job] + self.mock_repo.get_workflow_run.return_value = mock_workflow_run + mock_response = Mock() + mock_response.content = zip_content + mock_response.raise_for_status = Mock() + mock_requests.return_value = mock_response + logs = bot.get_build_logs() + self.assertEqual(len(logs), 2) + self.assertIn("Error: test failed", logs[0]["logs"]) + + @patch("openwisp_utils.bots.ci_failure.bot.requests.get") + def test_get_build_logs_error(self, mock_requests): + """Test network error during log fetch""" + import requests + + bot = CIFailureBot() + mock_workflow_run = Mock() + mock_job = self._mock_failed_job() + mock_workflow_run.jobs.return_value = [mock_job] + self.mock_repo.get_workflow_run.return_value = mock_workflow_run + mock_requests.side_effect = requests.RequestException("Network error") + logs = bot.get_build_logs() + self.assertEqual(len(logs), 1) + self.assertEqual(logs[0]["logs"], "") + + @patch("openwisp_utils.bots.ci_failure.bot.subprocess.run") + def test_get_pr_diff_success(self, mock_subprocess): + bot = CIFailureBot() + mock_pr = Mock() + mock_pr.title = "Test PR" + mock_pr.body = "Test description" + self.mock_repo.get_pull.return_value = mock_pr + mock_subprocess.return_value.returncode = 0 + mock_subprocess.return_value.stdout = "diff --git a/test.py b/test.py\n" + ( + "line\n" * 2000 + ) + diff_data = bot.get_pr_diff() + self.assertEqual(diff_data["title"], "Test PR") + self.assertEqual(diff_data["body"], "Test description") + self.assertIn("[...middle truncated...]", diff_data["diff"]) + + @patch("openwisp_utils.bots.ci_failure.bot.subprocess.run") + def test_get_pr_diff_error(self, mock_subprocess): + """Test git subprocess failure""" + import subprocess + + bot = CIFailureBot() + mock_pr = Mock() + mock_pr.title = "Test" + mock_pr.body = "Desc" + self.mock_repo.get_pull.return_value = mock_pr + self.mock_repo.default_branch = "main" + mock_subprocess.side_effect = subprocess.SubprocessError("git failed") + diff = bot.get_pr_diff() + self.assertIsNone(diff) + + def test_analyze_with_gemini_success(self): + bot = CIFailureBot() + bot.model = self.mock_model + mock_response = Mock() + mock_response.text = """The file bad_format.py contains PEP 8 violations. + +Required Actions: +```bash +pip install -e .[qa] +openwisp-qa-format +```""" + self.mock_model.generate_content.return_value = mock_response + build_logs = [{"job_name": "Build / Python 3.11", "logs": "flake8 error"}] + pr_diff = {"title": "Test", "diff": "diff content"} + result = bot.analyze_with_gemini(build_logs, pr_diff) + self.assertIn("PEP 8", result) + self.assertIn("pip install", result) + self.mock_model.generate_content.assert_called_once() + + def test_analyze_with_gemini_fallback(self): + bot = CIFailureBot() + bot.model = self.mock_model + self.mock_model.generate_content.side_effect = Exception("API Error") + result = bot.analyze_with_gemini([], None) + self.assertIn("Automated analysis", result) + self.assertIn("pip install", result) + + def test_post_comment_create(self): + bot = CIFailureBot() + mock_pr = Mock() + mock_user = Mock() + mock_user.login = "github-actions[bot]" + self.mock_github.return_value.get_user.return_value = mock_user + self.mock_repo.get_pull.return_value = mock_pr + mock_pr.get_issue_comments.return_value = [] + bot.post_comment("Test message") + mock_pr.create_issue_comment.assert_called_once() + call_args = mock_pr.create_issue_comment.call_args[0][0] + self.assertIn("", call_args) + self.assertIn("Test message", call_args) + + def test_post_comment_update_existing(self): + bot = CIFailureBot() + mock_pr = Mock() + mock_user = Mock() + mock_user.login = "github-actions[bot]" + self.mock_github.return_value.get_user.return_value = mock_user + self.mock_repo.get_pull.return_value = mock_pr + mock_comment = Mock() + mock_comment.user.login = "github-actions[bot]" + mock_comment.body = "\nOld message" + mock_pr.get_issue_comments.return_value = [mock_comment] + bot.post_comment("New message") + mock_comment.edit.assert_called_once() + mock_pr.create_issue_comment.assert_not_called() + + def test_run_skips_dependabot(self): + bot = CIFailureBot() + self._mock_workflow_run(actor="dependabot[bot]") + with patch("builtins.print") as mock_print: + bot.run() + mock_print.assert_any_call("Skipping: dependabot PR from dependabot[bot]") + + @patch("openwisp_utils.bots.ci_failure.bot.subprocess.run") + @patch("openwisp_utils.bots.ci_failure.bot.requests.get") + def test_run_full_workflow(self, mock_requests, mock_subprocess): + bot = CIFailureBot() + mock_job = self._mock_failed_job() + self._mock_workflow_run(jobs=[mock_job]) + mock_response = Mock() + mock_response.content = b"Build failed" + mock_response.raise_for_status = Mock() + mock_requests.return_value = mock_response + + # Mock subprocess for PR diff + mock_subprocess.return_value.returncode = 0 + mock_subprocess.return_value.stdout = "diff --git a/test.py" + + mock_gemini_response = Mock() + mock_gemini_response.text = "Analysis: Build failed due to syntax error" + self.mock_model.generate_content.return_value = mock_gemini_response + mock_pr = self._mock_pr() + bot.run() + mock_pr.create_issue_comment.assert_called_once() + + def test_run_skips_fork_pr(self): + """Test skipping fork PR""" + bot = CIFailureBot() + self._mock_workflow_run(actor="contributor") + self.mock_repo.full_name = "openwisp/openwisp-utils" + mock_pr = self._mock_pr(full_name="contributor/openwisp-utils") + with patch("builtins.print") as mock_print: + bot.run() + mock_print.assert_any_call( + "Skipping: fork PR from contributor/openwisp-utils" + ) + mock_pr.create_issue_comment.assert_not_called() + + def test_run_skips_deleted_fork_pr(self): + """Test skipping PR from deleted fork""" + bot = CIFailureBot() + self._mock_workflow_run() + mock_pr = self._mock_pr(deleted_fork=True) + with patch("builtins.print") as mock_print: + bot.run() + mock_print.assert_any_call("Skipping: PR with deleted head repository") + mock_pr.create_issue_comment.assert_not_called() + + def test_run_fork_status_exception(self): + """Test fork status check exception""" + from github import GithubException + + bot = CIFailureBot() + mock_run = Mock() + mock_actor = Mock() + mock_actor.login = "user" + mock_run.actor = mock_actor + self.mock_repo.get_workflow_run.return_value = mock_run + self.mock_repo.get_pull.side_effect = GithubException(404, "Not found") + with patch("builtins.print") as mock_print: + bot.run() + mock_print.assert_any_call( + 'Warning: Could not check fork status: 404 "Not found"' + ) + + def test_run_actor_check_exception(self): + """Test actor check exception""" + from github import GithubException + + bot = CIFailureBot() + self.mock_repo.get_workflow_run.side_effect = GithubException( + 401, "Unauthorized" + ) + mock_pr = Mock() + mock_pr.get_issue_comments.return_value = [] + self.mock_repo.get_pull.return_value = mock_pr + with patch("builtins.print") as mock_print: + bot.run() + mock_print.assert_any_call( + 'Warning: Could not check actor: 401 "Unauthorized"' + ) + + def test_run_no_pr_context(self): + """Test run with no PR context""" + bot = CIFailureBot() + bot.pr_number = None + mock_run = Mock() + mock_actor = Mock() + mock_actor.login = "user" + mock_run.actor = mock_actor + self.mock_repo.get_workflow_run.return_value = mock_run + with patch("builtins.print") as mock_print: + bot.run() + mock_print.assert_any_call( + "No PR context available - workflow_run without PR" + ) + + def test_run_no_logs_no_diff_fallback(self): + """Test fallback when no logs or diff available""" + bot = CIFailureBot() + mock_run = Mock() + mock_actor = Mock() + mock_actor.login = "user" + mock_run.actor = mock_actor + mock_run.jobs.return_value = [] + self.mock_repo.get_workflow_run.return_value = mock_run + mock_pr = Mock() + mock_pr.head.repo.full_name = "openwisp/openwisp-utils" + mock_pr.get_issue_comments.return_value = [] + self.mock_repo.get_pull.return_value = mock_pr + with patch("builtins.print") as mock_print: + bot.run() + mock_print.assert_any_call( + "No build logs or PR diff found, using fallback response" + ) + + def test_run_outer_exception(self): + """Test outer exception handling in run""" + bot = CIFailureBot() + with patch.object(bot, "get_build_logs", side_effect=Exception("boom")): + mock_run = Mock() + mock_actor = Mock() + mock_actor.login = "user" + mock_run.actor = mock_actor + self.mock_repo.get_workflow_run.return_value = mock_run + mock_pr = Mock() + mock_pr.head.repo.full_name = "openwisp/openwisp-utils" + mock_pr.get_issue_comments.return_value = [] + self.mock_repo.get_pull.return_value = mock_pr + with patch("builtins.print") as mock_print: + bot.run() + mock_print.assert_any_call("Error in analysis: boom") + + def test_run_no_repo(self): + """Test run when repo is None""" + bot = CIFailureBot() + bot.repo = None + with patch("builtins.print") as mock_print: + bot.run() + mock_print.assert_any_call("GitHub client not initialized, cannot proceed") + + def test_get_build_logs_no_repo(self): + """Test get_build_logs when repo is None""" + bot = CIFailureBot() + bot.repo = None + logs = bot.get_build_logs() + self.assertEqual(logs, []) + + def test_get_build_logs_no_workflow_run_id(self): + """Test get_build_logs when workflow_run_id is None""" + bot = CIFailureBot() + bot.workflow_run_id = None + logs = bot.get_build_logs() + self.assertEqual(logs, []) + + def test_get_pr_diff_no_repo(self): + """Test get_pr_diff when repo is None""" + bot = CIFailureBot() + bot.repo = None + diff = bot.get_pr_diff() + self.assertIsNone(diff) + + def test_get_pr_diff_no_pr_number(self): + """Test get_pr_diff when pr_number is None""" + bot = CIFailureBot() + bot.pr_number = None + diff = bot.get_pr_diff() + self.assertIsNone(diff) + + def test_post_comment_no_pr_number(self): + """Test post_comment when pr_number is None""" + bot = CIFailureBot() + bot.pr_number = None + bot.post_comment("Test message") + + def test_post_comment_no_github_client(self): + """Test post_comment when GitHub client is None""" + bot = CIFailureBot() + bot.github = None + bot.repo = None + bot.post_comment("Test message") + + def test_analyze_with_gemini_no_model(self): + """Test analyze_with_gemini when model is None""" + bot = CIFailureBot() + bot.model = None + result = bot.analyze_with_gemini([], None) + self.assertIn("Automated analysis", result) + + def test_analyze_with_gemini_no_repository_name(self): + """Test analyze_with_gemini when repository_name is None""" + bot = CIFailureBot() + bot.repository_name = None + result = bot.analyze_with_gemini([], None) + self.assertIn("Automated analysis", result) + + def test_classify_failure_qa_formatting(self): + """Test QA classification with formatting keywords""" + bot = CIFailureBot() + logs = [{"job_name": "lint", "logs": "formatting error"}] + self.assertEqual(bot.classify_failure(logs), "qa") + + def test_classify_failure_tests_pytest(self): + """Test classification with pytest keyword""" + bot = CIFailureBot() + logs = [{"job_name": "pytest", "logs": "test error"}] + self.assertEqual(bot.classify_failure(logs), "tests") + + def test_get_failed_jobs_summary_with_steps(self): + """Test get_failed_jobs_summary extracts step info""" + bot = CIFailureBot() + build_logs = [ + {"job_name": "test-job", "step_name": "Run tests", "step_number": 1}, + {"job_name": "lint-job", "logs": "error"}, + ] + summary = bot.get_failed_jobs_summary(build_logs) + self.assertEqual(len(summary), 2) + self.assertEqual(summary[0]["name"], "test-job") + self.assertEqual(summary[0]["step"], "Run tests") + + def test_get_build_logs_no_logs_url(self): + """Test get_build_logs when job has no logs_url""" + bot = CIFailureBot() + mock_workflow_run = Mock() + mock_job = self._mock_failed_job(logs_url=None) + mock_workflow_run.jobs.return_value = [mock_job] + self.mock_repo.get_workflow_run.return_value = mock_workflow_run + logs = bot.get_build_logs() + self.assertEqual(len(logs), 1) + self.assertEqual(logs[0]["logs"], "") + + @patch("openwisp_utils.bots.ci_failure.bot.subprocess.run") + def test_get_pr_diff_large_diff_truncation(self, mock_subprocess): + """Test PR diff truncation for large diffs""" + bot = CIFailureBot() + mock_pr = Mock() + mock_pr.title = "Test" + mock_pr.body = "Desc" + self.mock_repo.get_pull.return_value = mock_pr + mock_subprocess.return_value.returncode = 0 + # Create a diff larger than 8000 chars + mock_subprocess.return_value.stdout = "line\n" * 2000 + diff = bot.get_pr_diff() + self.assertIn("[...middle truncated...]", diff["diff"]) + + def test_init_missing_github_token(self): + """Test initialization with missing GITHUB_TOKEN""" + env_vars = { + "REPOSITORY": "repo", + "WORKFLOW_RUN_ID": "12345", + "PR_NUMBER": "1", + } + with patch.dict(os.environ, env_vars, clear=True): + bot = CIFailureBot() + self.assertIsNone(bot.github) + self.assertIsNone(bot.repo) + + def test_init_missing_repository(self): + """Test initialization with missing REPOSITORY""" + env_vars = { + "GITHUB_TOKEN": "token", + "WORKFLOW_RUN_ID": "12345", + "PR_NUMBER": "1", + } + with patch.dict(os.environ, env_vars, clear=True): + bot = CIFailureBot() + self.assertIsNone(bot.github) + self.assertIsNone(bot.repo) + + @patch("openwisp_utils.bots.ci_failure.bot.Github") + def test_init_github_exception(self, mock_github_class): + """Test GitHub initialization exception""" + from github import GithubException + + mock_github_class.side_effect = GithubException(401, "Unauthorized") + env_vars = { + "GITHUB_TOKEN": "token", + "REPOSITORY": "repo", + "WORKFLOW_RUN_ID": "12345", + "PR_NUMBER": "1", + } + with patch.dict(os.environ, env_vars, clear=True): + bot = CIFailureBot() + self.assertIsNone(bot.github) + self.assertIsNone(bot.repo) + + @patch("openwisp_utils.bots.ci_failure.bot.genai") + @patch("openwisp_utils.bots.ci_failure.bot.Github") + def test_init_gemini_exception(self, mock_github_class, mock_genai): + """Test Gemini initialization exception""" + mock_genai.configure.side_effect = Exception("API key invalid") + env_vars = { + "GITHUB_TOKEN": "token", + "GEMINI_API_KEY": "key", + "REPOSITORY": "repo", + "WORKFLOW_RUN_ID": "12345", + "PR_NUMBER": "1", + } + with patch.dict(os.environ, env_vars, clear=True): + bot = CIFailureBot() + self.assertIsNone(bot.model) + + @patch("openwisp_utils.bots.ci_failure.bot.requests.get") + def test_get_build_logs_zip_with_txt_extension(self, mock_requests): + """Test ZIP file with .txt extension""" + import io + import zipfile + + bot = CIFailureBot() + zip_buffer = io.BytesIO() + with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zip_file: + zip_file.writestr("job.txt", "Error log content") + zip_content = zip_buffer.getvalue() + mock_workflow_run = Mock() + mock_job = self._mock_failed_job() + mock_workflow_run.jobs.return_value = [mock_job] + self.mock_repo.get_workflow_run.return_value = mock_workflow_run + mock_response = Mock() + mock_response.content = zip_content + mock_response.raise_for_status = Mock() + mock_requests.return_value = mock_response + logs = bot.get_build_logs() + self.assertGreater(len(logs), 0) + + @patch("openwisp_utils.bots.ci_failure.bot.requests.get") + def test_get_build_logs_with_failed_steps(self, mock_requests): + """Test build logs with failed steps""" + bot = CIFailureBot() + mock_workflow_run = Mock() + step = Mock() + step.name = "failing-step" + step.conclusion = "failure" + step.number = 1 + mock_job = self._mock_failed_job(steps=[step]) + mock_workflow_run.jobs.return_value = [mock_job] + self.mock_repo.get_workflow_run.return_value = mock_workflow_run + mock_response = Mock() + mock_response.content = b"Error logs" + mock_response.raise_for_status = Mock() + mock_requests.return_value = mock_response + logs = bot.get_build_logs() + # Should have job log + step info + self.assertGreater(len(logs), 1) + + def test_get_pr_diff_invalid_pr_number(self): + """Test get_pr_diff with invalid PR number""" + bot = CIFailureBot() + bot.pr_number = "invalid_string" + diff = bot.get_pr_diff() + self.assertIsNone(diff) + + def test_get_pr_diff_github_exception(self): + """Test get_pr_diff with GithubException""" + from github import GithubException + + bot = CIFailureBot() + self.mock_repo.get_pull.side_effect = GithubException(404, "Not found") + diff = bot.get_pr_diff() + self.assertIsNone(diff) + + def test_post_comment_invalid_pr_number(self): + """Test post_comment with invalid PR number""" + bot = CIFailureBot() + bot.pr_number = "invalid" + bot.post_comment("Test message") + + def test_post_comment_github_exception_fetch_pr(self): + """Test post_comment with GithubException when fetching PR""" + from github import GithubException + + bot = CIFailureBot() + self.mock_repo.get_pull.side_effect = GithubException(403, "Forbidden") + bot.post_comment("Test message") + + def test_post_comment_github_exception_get_comments(self): + """Test post_comment with GithubException when getting comments""" + from github import GithubException + + bot = CIFailureBot() + mock_pr = Mock() + self.mock_repo.get_pull.return_value = mock_pr + mock_pr.get_issue_comments.side_effect = GithubException(500, "Server error") + bot.post_comment("Test message") + + def test_post_comment_github_exception_create_comment(self): + """Test post_comment with GithubException when creating comment""" + from github import GithubException + + bot = CIFailureBot() + mock_pr = Mock() + self.mock_repo.get_pull.return_value = mock_pr + mock_pr.get_issue_comments.return_value = [] + mock_pr.create_issue_comment.side_effect = GithubException(403, "Forbidden") + bot.post_comment("Test message") + + def test_get_failed_jobs_summary_job_name_only(self): + """Test get_failed_jobs_summary with job_name only""" + bot = CIFailureBot() + build_logs = [{"job_name": "test-job", "logs": "error"}] + summary = bot.get_failed_jobs_summary(build_logs) + self.assertEqual(len(summary), 1) + self.assertEqual(summary[0]["name"], "test-job") + self.assertNotIn("step", summary[0]) diff --git a/runtests.py b/runtests.py index bd59b303..5694c074 100755 --- a/runtests.py +++ b/runtests.py @@ -16,5 +16,6 @@ args.insert(1, "test") args.insert(2, "test_project") args.insert(3, "openwisp_utils.metric_collection") + args.insert(4, "openwisp_utils.bots") execute_from_command_line(args) sys.exit(pytest.main([os.path.join("openwisp_utils/releaser/tests")])) diff --git a/setup.py b/setup.py index eb635c44..aeb5a9fd 100644 --- a/setup.py +++ b/setup.py @@ -63,6 +63,11 @@ "pypandoc~=1.15", "pypandoc-binary~=1.15", ], + "github_actions": [ + "requests>=2.32.5", + "PyGithub>=2.0.0", + "google-generativeai>=0.8.0", + ], }, classifiers=[ "Development Status :: 5 - Production/Stable", diff --git a/tests/openwisp2/settings.py b/tests/openwisp2/settings.py index 27038e9c..f02651a5 100644 --- a/tests/openwisp2/settings.py +++ b/tests/openwisp2/settings.py @@ -30,6 +30,7 @@ "test_project", "openwisp_utils.admin_theme", "openwisp_utils.metric_collection", + "openwisp_utils.bots", "django.contrib.sites", # admin "django.contrib.admin", diff --git a/tests/test_project/tests/test_test_utils.py b/tests/test_project/tests/test_test_utils.py index 1de83212..8ed13920 100644 --- a/tests/test_project/tests/test_test_utils.py +++ b/tests/test_project/tests/test_test_utils.py @@ -98,7 +98,7 @@ def test_retryable_request(self, *args): ) as mocked__get_conn: with self.assertRaises(ConnectionError) as error: retryable_request("get", url="https://openwisp.org") - self.assertEqual(len(mocked__get_conn.mock_calls), 4) + self.assertGreaterEqual(len(mocked__get_conn.mock_calls), 3) self.assertIn( "OSError", str(error.exception), @@ -112,7 +112,7 @@ def test_retryable_request(self, *args): ) as mocked_getResponse: with self.assertRaises(RetryError) as error: retryable_request("get", url="https://openwisp.org") - self.assertEqual(len(mocked_getResponse.mock_calls), 4) + self.assertGreaterEqual(len(mocked_getResponse.mock_calls), 3) self.assertIn( "too many 500 error responses", str(error.exception),