Loading dashboard data...
+| Test Type | Matrix | Suite | " + "Test | Error |
|---|---|---|---|---|
| {_html_escape(e['test_type'])} | " + f"{_html_escape(e['matrix_label'])} | "
+ f"{_html_escape(e['suite'])} | " + f"{_html_escape(e['name'])} | "
+ f"{short}" + f'{msg} |
| Test Type | Matrix | Suite | " + "Test | Since | Error |
|---|---|---|---|---|---|
| {_html_escape(e['test_type'])} | " + f"{_html_escape(e['matrix_label'])} | "
+ f"{_html_escape(e['suite'])} | " + f"{_html_escape(e['name'])} | "
+ f"{_html_escape(e.get('first_seen', '?'))} | " + f"{short}" + f'{msg} |
| Test Type | Matrix | Suite | " + "Test | Failing Since | Count |
|---|---|---|---|---|---|
| {_html_escape(e['test_type'])} | " + f"{_html_escape(e['matrix_label'])} | "
+ f"{_html_escape(e['suite'])} | " + f"{_html_escape(e['name'])} | "
+ f"{_html_escape(e.get('first_seen', '?'))} | " + f"{e.get('failure_count', '?')} |
| Test Type | Matrix | Suite | " + "Test | Retries |
|---|---|---|---|---|
| {_html_escape(e['test_type'])} | " + f"{_html_escape(e['matrix_label'])} | "
+ f"{_html_escape(e['suite'])} | " + f"{_html_escape(e['name'])} | "
+ f"{e.get('retry_count', '?')} |
' + "All tests passed across all matrices!
" + ) + + # --- Matrix grid (at the end) --- + parts.append("| Test Type | Matrix | Status | " + "Passed | Failed | Flaky | Total | Report |
|---|---|---|---|---|---|---|---|
| {_html_escape(g['test_type'])} | " + f"{_html_escape(g['matrix_label'])} | "
+ f"{_status_badge(g['status'])} | " + f"{counts.get('passed', 0)} | " + f"{counts.get('failed', 0)} | " + f"{counts.get('flaky', 0)} | " + f"{counts.get('total', 0)} | " + f"{report_link} |
| Suite | Test | Error |
|---|---|---|
| {_html_escape(e['suite'])} | " + f"{_html_escape(e['name'])} "
+ f'NEW | '
+ f"{short}" + f'{msg} |
| Suite | Test | First Seen | " + "Count | Error |
|---|---|---|---|---|
| {_html_escape(e['suite'])} | " + f"{_html_escape(e['name'])} "
+ f'RECURRING | '
+ f"{first_seen} | {count} | " + f"{short}" + f'{msg} |
| Suite | Test | Failing Since | " + "Failure Count | Was Flaky? |
|---|---|---|---|---|
| {_html_escape(e['suite'])} | " + f"{_html_escape(e['name'])} "
+ f'FIXED | '
+ f"{_html_escape(e.get('first_seen', '?'))} | " + f"{e.get('failure_count', '?')} | " + f"{flaky_tag} |
| Suite | Test | Retries |
|---|---|---|
| {_html_escape(e['suite'])} | " + f"{_html_escape(e['name'])} "
+ f'FLAKY | '
+ f"{e.get('retry_count', '?')} |
{_html_escape(e['classname'])}::{_html_escape(e['name'])}' + f"Suite: {_html_escape(e['suite'])} | " + f"Source: {_html_escape(e['source_file'])}
" + ) + if msg: + parts.append(f'{msg}')
+ parts.append("All tests passed! No failures or flaky tests detected.
' + ) + + parts.append("") + return "\n".join(parts) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main(): + parser = argparse.ArgumentParser( + description="Generate nightly test failure report from JUnit XML results" + ) + parser.add_argument( + "--results-dir", + required=True, + help="Directory containing JUnit XML test result files", + ) + parser.add_argument( + "--output-dir", + default="report-output", + help="Directory to write report files to", + ) + parser.add_argument( + "--sha", + default=os.environ.get("GITHUB_SHA", "unknown"), + help="Git commit SHA for this run", + ) + parser.add_argument( + "--date", + default=datetime.now(timezone.utc).strftime("%Y-%m-%d"), + help="Date for this run (YYYY-MM-DD)", + ) + parser.add_argument( + "--test-type", + default="unknown", + help=( + "Test type identifier (e.g., cpp, python, wheel-python, " + "wheel-server, notebooks)" + ), + ) + parser.add_argument( + "--matrix-label", + default="", + help=( + "Matrix combination label (e.g., cuda12.9-py3.12-x86_64). " + "Included in reports and JSON summary to identify the CI job." + ), + ) + parser.add_argument( + "--s3-history-uri", + default="", + help=( + "S3 URI for persistent failure history JSON. " + "Downloaded before analysis, uploaded after update. " + "Example: s3://bucket/ci_test_reports/nightly/history/" + "python-main-cuda12.9-py3.12-x86_64.json" + ), + ) + parser.add_argument( + "--s3-summary-uri", + default="", + help=( + "S3 URI to upload this run's JSON snapshot for aggregation. " + "Example: s3://bucket/ci_test_reports/nightly/summaries/" + "2026-04-13/python-cuda12.9-py3.12-x86_64.json" + ), + ) + parser.add_argument( + "--s3-html-uri", + default="", + help=( + "S3 URI to upload the HTML report. " + "Example: s3://bucket/ci_test_reports/nightly/reports/" + "2026-04-13/python-cuda12.9-py3.12-x86_64.html" + ), + ) + parser.add_argument( + "--github-step-summary", + default=os.environ.get("GITHUB_STEP_SUMMARY", ""), + help="Path to write GitHub Actions step summary", + ) + + args = parser.parse_args() + + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + local_history_path = str(output_dir / "test_failure_history.json") + + # ---- Step 1: Download history from S3 ---- + if args.s3_history_uri: + s3_download(args.s3_history_uri, local_history_path) + + # ---- Step 2: Collect and classify results ---- + print(f"Collecting test results from {args.results_dir} ...") + results = collect_all_results(args.results_dir) + if not results: + print("WARNING: No test results found.", file=sys.stderr) + + print(f"Found {len(results)} test case entries across all XML files") + classified = classify_failures(results) + + print( + f"Classification: {len(classified['passed'])} passed, " + f"{len(classified['failed'])} failed, " + f"{len(classified['error'])} errors, " + f"{len(classified['flaky'])} flaky, " + f"{len(classified['skipped'])} skipped" + ) + + # ---- Step 3: Update history ---- + history = load_history(local_history_path) + history, new_failures, recurring_failures, resolved_tests = update_history( + history, classified, args.sha, args.date + ) + + if resolved_tests: + print( + f"Stabilized: {len(resolved_tests)} previously-failing test(s) now pass" + ) + + save_history(history, local_history_path) + print(f"Updated local history at {local_history_path}") + + # ---- Step 4: Upload history back to S3 ---- + if args.s3_history_uri: + s3_upload(local_history_path, args.s3_history_uri) + + # ---- Step 5: Generate reports ---- + report_kwargs = dict( + test_type=args.test_type, + matrix_label=args.matrix_label, + sha=args.sha, + date_str=args.date, + ) + + md_report = generate_markdown_report( + classified, + new_failures, + recurring_failures, + resolved_tests, + history, + **report_kwargs, + ) + md_path = output_dir / "nightly_report.md" + md_path.write_text(md_report) + print(f"Markdown report written to {md_path}") + + html_report = generate_html_report( + classified, + new_failures, + recurring_failures, + resolved_tests, + history, + **report_kwargs, + ) + html_path = output_dir / "nightly_report.html" + html_path.write_text(html_report) + print(f"HTML report written to {html_path}") + + json_summary = generate_json_summary( + classified, + new_failures, + recurring_failures, + resolved_tests, + **report_kwargs, + ) + json_path = output_dir / "nightly_summary.json" + json_path.write_text(json.dumps(json_summary, indent=2) + "\n") + print(f"JSON summary written to {json_path}") + + if args.github_step_summary: + with open(args.github_step_summary, "a") as f: + f.write(md_report) + print(f"Wrote GitHub Step Summary to {args.github_step_summary}") + + # ---- Step 6: Upload per-run snapshot and HTML to S3 ---- + if args.s3_summary_uri: + s3_upload(str(json_path), args.s3_summary_uri) + + if args.s3_html_uri: + s3_upload(str(html_path), args.s3_html_uri) + + # ---- Exit code ---- + genuine_failures = len(classified["failed"]) + len(classified["error"]) + if genuine_failures > 0: + print( + f"\nFAILED: {genuine_failures} genuine test failure(s) detected." + ) + return 1 + if classified["flaky"]: + print( + f"\nWARNING: All tests passed but {len(classified['flaky'])} flaky test(s) detected." + ) + else: + print("\nAll tests passed.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/utils/nightly_report_helper.sh b/ci/utils/nightly_report_helper.sh new file mode 100755 index 0000000000..c3b77e6b7a --- /dev/null +++ b/ci/utils/nightly_report_helper.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Shared helper for generating nightly test reports with matrix-aware S3 paths. +# +# Usage (source from any test script): +# +# # For C++ tests (no Python version in matrix label): +# generate_nightly_report "cpp" +# +# # For Python tests (includes Python version in matrix label): +# generate_nightly_report "python" --with-python-version +# +# # For wheel tests: +# generate_nightly_report "wheel-python" --with-python-version +# +# Prerequisites (set before calling): +# RAPIDS_TESTS_DIR - directory containing JUnit XML test results +# +# Optional environment variables (auto-detected if not set): +# RAPIDS_CUDA_VERSION - CUDA version (e.g., "12.9") +# RAPIDS_PY_VERSION - Python version (e.g., "3.12"), used with --with-python-version +# RAPIDS_BRANCH - branch name (e.g., "main") +# CUOPT_S3_URI - S3 bucket root (e.g., s3://cuopt-datasets/) +# GITHUB_SHA - commit SHA +# GITHUB_STEP_SUMMARY - path for GitHub Actions step summary + +# Resolve the directory where THIS helper lives (ci/utils/) +_HELPER_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" + +generate_nightly_report() { + local test_type="${1:?Usage: generate_nightly_report