From d0fbb979ce019c6155be27e56964774e575a92f1 Mon Sep 17 00:00:00 2001 From: santhoshct Date: Mon, 16 Dec 2024 18:37:28 +0530 Subject: [PATCH 1/6] added test cases to cleared tests report and added table view to all the reports --- .github/scripts/develocity_reports.py | 209 ++++++++++++++++++-------- 1 file changed, 144 insertions(+), 65 deletions(-) diff --git a/.github/scripts/develocity_reports.py b/.github/scripts/develocity_reports.py index 43285a336fd16..81c68e7d182bd 100644 --- a/.github/scripts/develocity_reports.py +++ b/.github/scripts/develocity_reports.py @@ -698,6 +698,7 @@ def get_cleared_tests(self, project: str, results: List[TestResult], """ cleared_tests = {} current_time = datetime.now(pytz.UTC) + chunk_start = current_time - timedelta(days=7) # Last 7 days for test cases for result in results: # Only consider tests with sufficient recent executions @@ -705,25 +706,65 @@ def get_cleared_tests(self, project: str, results: List[TestResult], if len(recent_executions) < min_executions: continue - # Calculate success rate + # Calculate success rate at class level successful_runs = sum(1 for t in recent_executions if t.outcome == 'passed') success_rate = successful_runs / len(recent_executions) - # Check if the test meets clearing criteria + # Check if the test meets clearing criteria at class level if success_rate >= success_threshold: # Verify no recent failures or flaky behavior has_recent_issues = any(t.outcome in ['failed', 'flaky'] for t in recent_executions[-min_executions:]) if not has_recent_issues: - cleared_tests[result.name] = { - 'result': result, - 'success_rate': success_rate, - 'total_executions': len(recent_executions), - 'successful_runs': successful_runs, - 'recent_executions': recent_executions[-min_executions:] - } + try: + # Get test case details + test_cases = self.get_test_case_details( + result.name, + project, + chunk_start, + current_time, + test_type="quarantinedTest" + ) + + # Only include if all test cases are also passing consistently + all_cases_passing = True + passing_test_cases = [] + + for test_case in test_cases: + case_total = test_case.outcome_distribution.total + if case_total >= min_executions: + case_success_rate = test_case.outcome_distribution.passed / case_total + + # Check recent executions for the test case + recent_case_issues = any(t.outcome in ['failed', 'flaky'] + for t in test_case.timeline[-min_executions:]) + + if case_success_rate >= success_threshold and not recent_case_issues: + passing_test_cases.append({ + 'name': test_case.name, + 'success_rate': case_success_rate, + 'total_executions': case_total, + 'recent_executions': sorted(test_case.timeline, + key=lambda x: x.timestamp)[-min_executions:] + }) + else: + all_cases_passing = False + break + + if all_cases_passing and passing_test_cases: + cleared_tests[result.name] = { + 'result': result, + 'success_rate': success_rate, + 'total_executions': len(recent_executions), + 'successful_runs': successful_runs, + 'recent_executions': recent_executions[-min_executions:], + 'test_cases': passing_test_cases + } + + except Exception as e: + logger.error(f"Error getting test case details for {result.name}: {str(e)}") return cleared_tests @@ -855,34 +896,63 @@ def main(): if not flaky_regressions: print("No flaky test regressions found.") else: + print(f"Found {len(flaky_regressions)} tests that have started showing increased flaky behavior recently.") + print("\n") + print("") + for test_name, details in flaky_regressions.items(): - print(f"\n{test_name}") - print(f"Recent Flaky Rate: {details['recent_flaky_rate']:.2%}") - print(f"Historical Flaky Rate: {details['historical_flaky_rate']:.2%}") - print(f"\nRecent Executions (last {len(details['recent_executions'])} runs):") + print(f"") + print(f"" + f"" + f"") + + # Add recent execution details in sub-rows + print("") for entry in sorted(details['recent_executions'], key=lambda x: x.timestamp)[-5:]: - print(f" {entry.timestamp.strftime('%Y-%m-%d %H:%M')} - {entry.outcome}") + date_str = entry.timestamp.strftime('%Y-%m-%d %H:%M') + print(f"") + print("
Test ClassRecent Flaky RateHistorical RateRecent Executions
{test_name}
{details['recent_flaky_rate']:.2%}{details['historical_flaky_rate']:.2%}{len(details['recent_executions'])}
Recent Executions:
{date_str} - {entry.outcome}
") # Print Cleared Tests print("\n## Cleared Tests (Ready for Unquarantine)") if not cleared_tests: print("No tests ready to be cleared from quarantine.") else: - # Print summary - print("") + print("\n
ClassTest CaseSuccess RateBuild Scans
") + print("") + for test_name, details in cleared_tests.items(): - print(f"") + # Print class-level information + print(f"") + print(f"" + f"" + f"" + f"") + + # Print method-level information + for test_case in details['test_cases']: + method_name = test_case['name'].split('.')[-1] + + # Get most recent status + recent_status = "N/A" + if test_case['recent_executions']: + recent_status = test_case['recent_executions'][-1].outcome + + print(f"" + f"" + f"" + f"") + + # Print recent executions for the method + print("") + for entry in test_case['recent_executions']: + date_str = entry.timestamp.strftime('%Y-%m-%d %H:%M') + print(f"") + + print("") # Add spacing between classes print("
Test ClassTest MethodSuccess RateTotal RunsRecent Status
{test_name}{details['success_rate']:.2%}{details['total_executions']}
{test_name}
Class Overall{details['success_rate']:.2%}{details['total_executions']}{details['successful_runs']} passed
{method_name}{test_case['success_rate']:.2%}{test_case['total_executions']}{recent_status}
Recent Executions:
{date_str} - {entry.outcome}
 
") - - for test_name, details in cleared_tests.items(): - print(f"\n{test_name}") - print(f"Success Rate: {details['success_rate']:.2%}") - print(f"Total Executions: {details['total_executions']}") - print(f"\nRecent Executions (last {len(details['recent_executions'])} runs):") - for entry in sorted(details['recent_executions'], key=lambda x: x.timestamp): - print(f" {entry.timestamp.strftime('%Y-%m-%d %H:%M')} - {entry.outcome}") - - # Print Defective Tests + + # Print High-Priority Quarantined Tests print("\n## High-Priority Quarantined Tests") if not problematic_tests: print("No high-priority quarantined tests found.") @@ -895,51 +965,57 @@ def main(): ) print(f"\nFound {len(sorted_tests)} high-priority quarantined test classes:") + print("\n") + print("") + for full_class_name, details in sorted_tests: class_result = details['container_result'] - class_name = full_class_name.split(".")[-1] - print(f"### {class_name}") - print(f"{full_class_name} has been quarantined for {details['days_quarantined']} days") - print(f"Overall class failure: {details['failure_rate']:.2%}") - print(f"Recent class failure: {details['recent_failure_rate']:.2%}") - print("\nOverall Build Outcomes:") - print(f" Total Runs: {class_result.outcome_distribution.total}") - print(f" Failed: {class_result.outcome_distribution.failed}") - print(f" Flaky: {class_result.outcome_distribution.flaky}") - print(f" Passed: {class_result.outcome_distribution.passed}") - - print("\nQuarantined Methods (Last 7 Days):") - - # Sort test methods by failure rate - sorted_methods = sorted( - details['test_cases'], - key=lambda x: (x.outcome_distribution.failed + x.outcome_distribution.flaky) / x.outcome_distribution.total if x.outcome_distribution.total > 0 else 0, - reverse=True - ) - for test_method in sorted_methods: + # Print class-level information + print(f"") + print(f"" + f"" + f"" + f"" + f"") + + # Print build outcome statistics + print("") + print(f"") + + # Print test method details + print("") + print("") + + for test_method in sorted(details['test_cases'], + key=lambda x: (x.outcome_distribution.failed + x.outcome_distribution.flaky) / x.outcome_distribution.total if x.outcome_distribution.total > 0 else 0, + reverse=True): + + method_name = test_method.name.split('.')[-1] total_runs = test_method.outcome_distribution.total if total_runs > 0: - failure_rate = (test_method.outcome_distribution.failed + test_method.outcome_distribution.flaky) / total_runs - - # Extract the method name from the full test name - method_name = test_method.name.split('.')[-1] - - print(f"\n → {method_name}") - print(f" Failure Rate: {failure_rate:.2%}") - print(f" Runs: {total_runs:3d} | Failed: {test_method.outcome_distribution.failed:3d} | " - f"Flaky: {test_method.outcome_distribution.flaky:3d} | " - f"Passed: {test_method.outcome_distribution.passed:3d}") + success_rate = test_method.outcome_distribution.passed / total_runs + failure_rate = (test_method.outcome_distribution.failed + + test_method.outcome_distribution.flaky) / total_runs - # Show test method timeline + # Get most recent status + recent_status = "N/A" if test_method.timeline: - print(f"\n Recent Executions (last {min(3, len(test_method.timeline))} of {len(test_method.timeline)} runs):") - print(" Date/Time (UTC) Outcome Build ID") - print(" " + "-" * 44) - for entry in sorted(test_method.timeline, key=lambda x: x.timestamp)[-3:]: - date_str = entry.timestamp.strftime('%Y-%m-%d %H:%M') - print(f" {date_str:<17} {entry.outcome:<10} {entry.build_id}") + recent_status = test_method.timeline[-1].outcome + + print(f"" + f"" + f"" + f"" + f"") + print("") # Add spacing between classes + print("
Test ClassDays QuarantinedFailure RateRecent Failure RateTotal Runs
{full_class_name}
{details['days_quarantined']}{details['failure_rate']:.2%}{details['recent_failure_rate']:.2%}{class_result.outcome_distribution.total}
Build Outcomes:
" + f"Passed: {class_result.outcome_distribution.passed} | " + f"Failed: {class_result.outcome_distribution.failed} | " + f"Flaky: {class_result.outcome_distribution.flaky}" + f"
Test Methods (Last 7 Days):
MethodSuccess RateFailure RateTotal RunsRecent Status
{method_name}{success_rate:.2%}{failure_rate:.2%}{total_runs}{recent_status}
 
") + except Exception as e: logger.exception("Error occurred during report generation") print(f"Error occurred: {str(e)}") @@ -949,6 +1025,9 @@ def main(): # Configure logging logging.basicConfig( level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s' + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=[ + logging.FileHandler("flaky_test_report.log") + ] ) main() From bd1d83be64162aa57ff6b8dad6e76d39ffd5a79a Mon Sep 17 00:00:00 2001 From: santhoshct Date: Mon, 16 Dec 2024 22:29:03 +0530 Subject: [PATCH 2/6] formatted the detailed info into collapsible section. added develocity links to the test classes and test methods --- .github/scripts/develocity_reports.py | 130 +++++++++++++++++++++----- 1 file changed, 108 insertions(+), 22 deletions(-) diff --git a/.github/scripts/develocity_reports.py b/.github/scripts/develocity_reports.py index 81c68e7d182bd..5f1ee4c011fc9 100644 --- a/.github/scripts/develocity_reports.py +++ b/.github/scripts/develocity_reports.py @@ -768,7 +768,50 @@ def get_cleared_tests(self, project: str, results: List[TestResult], return cleared_tests -def print_summary(problematic_tests: Dict[str, Dict], flaky_regressions: Dict[str, Dict]): +def get_develocity_class_link(class_name: str, threshold_days: int) -> str: + """ + Generate Develocity link for a test class + + Args: + class_name: Name of the test class + threshold_days: Number of days to look back in search + """ + base_url = "https://ge.apache.org/scans/tests" + params = { + "search.rootProjectNames": "kafka", + "search.tags": "github,trunk", + "search.timeZoneId": "America/New_York", + "search.relativeStartTime": f"P{threshold_days}D", + "tests.container": class_name + } + return f"{base_url}?{'&'.join(f'{k}={requests.utils.quote(str(v))}' for k, v in params.items())}" + +def get_develocity_method_link(class_name: str, method_name: str, threshold_days: int) -> str: + """ + Generate Develocity link for a test method + + Args: + class_name: Name of the test class + method_name: Name of the test method + threshold_days: Number of days to look back in search + """ + base_url = "https://ge.apache.org/scans/tests" + + # Extract just the method name without the class prefix + if '.' in method_name: + method_name = method_name.split('.')[-1] + + params = { + "search.rootProjectNames": "kafka", + "search.tags": "github,trunk", + "search.timeZoneId": "America/New_York", + "search.relativeStartTime": f"P{threshold_days}D", + "tests.container": class_name, + "tests.test": method_name + } + return f"{base_url}?{'&'.join(f'{k}={requests.utils.quote(str(v))}' for k, v in params.items())}" + +def print_summary(problematic_tests: Dict[str, Dict], flaky_regressions: Dict[str, Dict], threshold_days: int): """Print a summary of the most problematic tests at the top of the report""" print("\n## Summary of Most Problematic Tests") @@ -818,11 +861,14 @@ def print_summary(problematic_tests: Dict[str, Dict], flaky_regressions: Dict[st # Print summary print("") for full_class_name, cases in by_class.items(): - print(f"") + class_link = get_develocity_class_link(full_class_name, threshold_days) + print(f"") for case in cases: method = case['method'] if method != 'N/A': - print(f"") + method_link = get_develocity_method_link(full_class_name, f"{full_class_name}.{method}", threshold_days) + print(f"" + f"") else: print(f"") print("
ClassTest CaseFailure RateBuild Scans
{full_class_name}
{full_class_name}
{method:<60}{case['failure_rate']:.2%}{case['total_runs']}
{method}{case['failure_rate']:.2%}{case['total_runs']}
{case['failure_rate']:.2%}{case['total_runs']}
") @@ -889,7 +935,7 @@ def main(): print(f"\n# Flaky Test Report for {datetime.now(pytz.UTC).strftime('%Y-%m-%d')}") print(f"This report was run on {datetime.now(pytz.UTC).strftime('%Y-%m-%d %H:%M:%S')} UTC") - print_summary(problematic_tests, flaky_regressions) + print_summary(problematic_tests, flaky_regressions, QUARANTINE_THRESHOLD_DAYS) # Print Flaky Test Regressions print("\n## Flaky Test Regressions") @@ -901,7 +947,8 @@ def main(): print("Test ClassRecent Flaky RateHistorical RateRecent Executions") for test_name, details in flaky_regressions.items(): - print(f"{test_name}") + class_link = get_develocity_class_link(test_name, QUARANTINE_THRESHOLD_DAYS) + print(f"{test_name}") print(f"{details['recent_flaky_rate']:.2%}" f"{details['historical_flaky_rate']:.2%}" f"{len(details['recent_executions'])}") @@ -918,12 +965,14 @@ def main(): if not cleared_tests: print("No tests ready to be cleared from quarantine.") else: + # Print concise table with just class and method info print("\n") print("") for test_name, details in cleared_tests.items(): # Print class-level information - print(f"") + class_link = get_develocity_class_link(test_name, QUARANTINE_THRESHOLD_DAYS) + print(f"") print(f"" f"" f"" @@ -932,25 +981,40 @@ def main(): # Print method-level information for test_case in details['test_cases']: method_name = test_case['name'].split('.')[-1] - - # Get most recent status + full_method_name = test_case['name'] + method_link = get_develocity_method_link(test_name, full_method_name, QUARANTINE_THRESHOLD_DAYS) recent_status = "N/A" if test_case['recent_executions']: recent_status = test_case['recent_executions'][-1].outcome - print(f"" + print(f"" f"" f"" f"") - - # Print recent executions for the method - print("") - for entry in test_case['recent_executions']: - date_str = entry.timestamp.strftime('%Y-%m-%d %H:%M') - print(f"") print("") # Add spacing between classes print("
Test ClassTest MethodSuccess RateTotal RunsRecent Status
{test_name}
{test_name}
Class Overall{details['success_rate']:.2%}{details['total_executions']}
{method_name}
{method_name}{test_case['success_rate']:.2%}{test_case['total_executions']}{recent_status}
Recent Executions:
{date_str} - {entry.outcome}
 
") + + # Print detailed execution history in collapsible sections + print("\n
") + print("Detailed Execution History\n") + + for test_name, details in cleared_tests.items(): + print(f"\n### {test_name}") + + for test_case in details['test_cases']: + method_name = test_case['name'].split('.')[-1] + print(f"\n#### {method_name}") + print("Recent Executions:") + print("```") + print("Date/Time (UTC) Outcome Build ID") + print("-" * 44) + for entry in test_case['recent_executions']: + date_str = entry.timestamp.strftime('%Y-%m-%d %H:%M') + print(f"{date_str:<17} {entry.outcome:<10} {entry.build_id}") + print("```") + + print("
") # Print High-Priority Quarantined Tests print("\n## High-Priority Quarantined Tests") @@ -970,9 +1034,10 @@ def main(): for full_class_name, details in sorted_tests: class_result = details['container_result'] + class_link = get_develocity_class_link(full_class_name, QUARANTINE_THRESHOLD_DAYS) # Print class-level information - print(f"{full_class_name}") + print(f"{full_class_name}") print(f"" f"{details['days_quarantined']}" f"{details['failure_rate']:.2%}" @@ -980,14 +1045,13 @@ def main(): f"{class_result.outcome_distribution.total}") # Print build outcome statistics - print("Build Outcomes:") - print(f"" + print("Build Outcomes: " f"Passed: {class_result.outcome_distribution.passed} | " f"Failed: {class_result.outcome_distribution.failed} | " f"Flaky: {class_result.outcome_distribution.flaky}" - f"") + "") - # Print test method details + # Print test method summary print("Test Methods (Last 7 Days):") print("MethodSuccess RateFailure RateTotal RunsRecent Status") @@ -996,18 +1060,18 @@ def main(): reverse=True): method_name = test_method.name.split('.')[-1] + method_link = get_develocity_method_link(full_class_name, test_method.name, QUARANTINE_THRESHOLD_DAYS) total_runs = test_method.outcome_distribution.total if total_runs > 0: success_rate = test_method.outcome_distribution.passed / total_runs failure_rate = (test_method.outcome_distribution.failed + test_method.outcome_distribution.flaky) / total_runs - # Get most recent status recent_status = "N/A" if test_method.timeline: recent_status = test_method.timeline[-1].outcome - print(f"{method_name}" + print(f"{method_name}" f"{success_rate:.2%}" f"{failure_rate:.2%}" f"{total_runs}" @@ -1015,6 +1079,28 @@ def main(): print(" ") # Add spacing between classes print("") + + # Print detailed execution history in collapsible sections + print("\n
") + print("Detailed Execution History\n") + + for full_class_name, details in sorted_tests: + print(f"\n### {full_class_name}") + + for test_method in details['test_cases']: + method_name = test_method.name.split('.')[-1] + if test_method.timeline: + print(f"\n#### {method_name}") + print("Recent Executions:") + print("```") + print("Date/Time (UTC) Outcome Build ID") + print("-" * 44) + for entry in sorted(test_method.timeline, key=lambda x: x.timestamp)[-5:]: + date_str = entry.timestamp.strftime('%Y-%m-%d %H:%M') + print(f"{date_str:<17} {entry.outcome:<10} {entry.build_id}") + print("```") + + print("
") except Exception as e: logger.exception("Error occurred during report generation") From cc0b8d4a64979c9c1816c0cebe6d7bfa16a628fa Mon Sep 17 00:00:00 2001 From: santhoshct Date: Tue, 17 Dec 2024 13:09:20 +0530 Subject: [PATCH 3/6] modified the layout of reports and removed redundant high priority quarantine test report --- .github/scripts/develocity_reports.py | 393 +++++++++++--------------- 1 file changed, 168 insertions(+), 225 deletions(-) diff --git a/.github/scripts/develocity_reports.py b/.github/scripts/develocity_reports.py index 5f1ee4c011fc9..44c5798e174df 100644 --- a/.github/scripts/develocity_reports.py +++ b/.github/scripts/develocity_reports.py @@ -811,67 +811,174 @@ def get_develocity_method_link(class_name: str, method_name: str, threshold_days } return f"{base_url}?{'&'.join(f'{k}={requests.utils.quote(str(v))}' for k, v in params.items())}" -def print_summary(problematic_tests: Dict[str, Dict], flaky_regressions: Dict[str, Dict], threshold_days: int): - """Print a summary of the most problematic tests at the top of the report""" - print("\n## Summary of Most Problematic Tests") - - # Combine and sort all test cases by failure rate - all_problem_cases = [] +def print_most_problematic_tests(problematic_tests: Dict[str, Dict], threshold_days: int): + """Print a summary of the most problematic tests""" + print("\n## Most Problematic Tests") + if not problematic_tests: + print("No high-priority problematic tests found.") + return + + print(f"Found {len(problematic_tests)} tests that have been quarantined for {threshold_days} days and are still failing frequently.") - # Process problematic quarantined tests - if len(problematic_tests) > 0: - print(f"Found {len(problematic_tests)} tests that have been quarantined for a while and are still flaky.") - for full_class_name, details in problematic_tests.items(): - for test_case in details['test_cases']: - total_runs = test_case.outcome_distribution.total - method_name = test_case.name.split('.')[-1] - if total_runs > 0: - failure_rate = (test_case.outcome_distribution.failed + - test_case.outcome_distribution.flaky) / total_runs - all_problem_cases.append({ - 'class': full_class_name, - 'method': method_name, - 'failure_rate': failure_rate, - 'total_runs': total_runs - }) + # Print concise table with key metrics + print("\n") + print("") + + for test_name, details in sorted(problematic_tests.items(), + key=lambda x: x[1]['recent_failure_rate'], + reverse=True): + class_link = get_develocity_class_link(test_name, threshold_days) + + # Find the method with highest failure rate + worst_method = max(details['test_cases'], + key=lambda x: (x.outcome_distribution.failed + x.outcome_distribution.flaky) / x.outcome_distribution.total + if x.outcome_distribution.total > 0 else 0) + + method_name = worst_method.name.split('.')[-1] + method_link = get_develocity_method_link(test_name, worst_method.name, threshold_days) + method_failure_rate = (worst_method.outcome_distribution.failed + + worst_method.outcome_distribution.flaky) / worst_method.outcome_distribution.total + + print(f"" + f"" + f"" + f"" + f"") + print("
Test ClassDays QuarantinedRecent Failure RateMost Failing MethodMethod Failure Rate
{test_name}{details['days_quarantined']}{details['recent_failure_rate']:.2%}{method_name}{method_failure_rate:.2%}
") - # Process flaky regressions - if len(flaky_regressions) > 0: - print(f"Found {len(flaky_regressions)} tests that have started recently failing.") - for test_name, details in flaky_regressions.items(): - all_problem_cases.append({ - 'class': test_name, - 'method': 'N/A', # Flaky regressions are at class level - 'failure_rate': details['recent_flaky_rate'], - 'total_runs': len(details['recent_executions']) - }) + # Print detailed execution history in collapsible section + print("\n
") + print("Detailed Execution History\n") + + for test_name, details in sorted(problematic_tests.items(), + key=lambda x: x[1]['recent_failure_rate'], + reverse=True): + print(f"\n### {test_name}") + print(f"* Days Quarantined: {details['days_quarantined']}") + print(f"* Recent Failure Rate: {details['recent_failure_rate']:.2%}") + print(f"* Total Runs: {details['container_result'].outcome_distribution.total}") + print(f"* Build Outcomes: Passed: {details['container_result'].outcome_distribution.passed} | " + f"Failed: {details['container_result'].outcome_distribution.failed} | " + f"Flaky: {details['container_result'].outcome_distribution.flaky}") + + for test_method in sorted(details['test_cases'], + key=lambda x: (x.outcome_distribution.failed + x.outcome_distribution.flaky) / x.outcome_distribution.total + if x.outcome_distribution.total > 0 else 0, + reverse=True): + method_name = test_method.name.split('.')[-1] + if test_method.timeline: + print(f"\n#### {method_name}") + print("Recent Executions:") + print("```") + print("Date/Time (UTC) Outcome Build ID") + print("-" * 44) + for entry in sorted(test_method.timeline, key=lambda x: x.timestamp)[-5:]: + date_str = entry.timestamp.strftime('%Y-%m-%d %H:%M') + print(f"{date_str:<17} {entry.outcome:<10} {entry.build_id}") + print("```") + + print("
") - # Sort by failure rate descending - sorted_cases = sorted(all_problem_cases, - key=lambda x: x['failure_rate'], - reverse=True) +def print_flaky_regressions(flaky_regressions: Dict[str, Dict], threshold_days: int): + """Print tests that have recently started showing flaky behavior""" + print("\n## Flaky Test Regressions") + if not flaky_regressions: + print("No flaky test regressions found.") + return + + print(f"Found {len(flaky_regressions)} tests that have started showing increased flaky behavior recently.") + + # Print concise table + print("\n") + print("") + + for test_name, details in sorted(flaky_regressions.items(), + key=lambda x: x[1]['recent_flaky_rate'], + reverse=True): + class_link = get_develocity_class_link(test_name, threshold_days) + rate_change = details['recent_flaky_rate'] - details['historical_flaky_rate'] + + print(f"" + f"" + f"" + f"" + f"") + print("
Test ClassRecent Flaky RateHistorical RateChangeRecent Executions
{test_name}{details['recent_flaky_rate']:.2%}{details['historical_flaky_rate']:.2%}{rate_change:+.2%}{len(details['recent_executions'])}
") + + # Print detailed history + print("\n
") + print("Detailed Execution History\n") + + for test_name, details in sorted(flaky_regressions.items(), + key=lambda x: x[1]['recent_flaky_rate'], + reverse=True): + print(f"\n### {test_name}") + print(f"* Recent Flaky Rate: {details['recent_flaky_rate']:.2%}") + print(f"* Historical Flaky Rate: {details['historical_flaky_rate']:.2%}") + print("\nRecent Executions:") + print("```") + print("Date/Time (UTC) Outcome Build ID") + print("-" * 44) + for entry in sorted(details['recent_executions'], key=lambda x: x.timestamp)[-5:]: + date_str = entry.timestamp.strftime('%Y-%m-%d %H:%M') + print(f"{date_str:<17} {entry.outcome:<10} {entry.build_id}") + print("```") - # Group by class - by_class = {} - for case in sorted_cases: - if case['class'] not in by_class: - by_class[case['class']] = [] - by_class[case['class']].append(case) + print("
") - # Print summary - print("") - for full_class_name, cases in by_class.items(): - class_link = get_develocity_class_link(full_class_name, threshold_days) - print(f"") - for case in cases: - method = case['method'] - if method != 'N/A': - method_link = get_develocity_method_link(full_class_name, f"{full_class_name}.{method}", threshold_days) - print(f"" - f"") - else: - print(f"") +def print_cleared_tests(cleared_tests: Dict[str, Dict], threshold_days: int): + """Print tests that are ready to be unquarantined""" + print("\n## Cleared Tests (Ready for Unquarantine)") + if not cleared_tests: + print("No tests ready to be cleared from quarantine.") + return + + print(f"Found {len(cleared_tests)} test classes that have been consistently passing and could be removed from quarantine.") + + # Print concise table + print("\n
ClassTest CaseFailure RateBuild Scans
{full_class_name}
{method}{case['failure_rate']:.2%}{case['total_runs']}
{case['failure_rate']:.2%}{case['total_runs']}
") + print("") + + for test_name, details in sorted(cleared_tests.items(), + key=lambda x: x[1]['success_rate'], + reverse=True): + class_link = get_develocity_class_link(test_name, threshold_days) + method_count = len(details['test_cases']) + + print(f"" + f"" + f"" + f"" + f"") print("
Test ClassSuccess RateTotal RunsDays CleanMethods
{test_name}{details['success_rate']:.2%}{details['total_executions']}{details['successful_runs']}{method_count}
") + + # Print detailed history + print("\n
") + print("Detailed Test Method History\n") + + for test_name, details in sorted(cleared_tests.items(), + key=lambda x: x[1]['success_rate'], + reverse=True): + print(f"\n### {test_name}") + print(f"* Overall Success Rate: {details['success_rate']:.2%}") + print(f"* Total Executions: {details['total_executions']}") + print(f"* Consecutive Successful Runs: {details['successful_runs']}") + + for test_case in details['test_cases']: + method_name = test_case['name'].split('.')[-1] + print(f"\n#### {method_name}") + print(f"* Success Rate: {test_case['success_rate']:.2%}") + print(f"* Total Runs: {test_case['total_executions']}") + print("\nRecent Executions:") + print("```") + print("Date/Time (UTC) Outcome Build ID") + print("-" * 44) + for entry in test_case['recent_executions']: + date_str = entry.timestamp.strftime('%Y-%m-%d %H:%M') + print(f"{date_str:<17} {entry.outcome:<10} {entry.build_id}") + print("```") + + print("
") def main(): token = None @@ -896,14 +1003,13 @@ def main(): analyzer = TestAnalyzer(BASE_URL, token) try: - # Get quarantined test results + # Get test results quarantined_results = analyzer.get_test_results( PROJECT, threshold_days=QUARANTINE_THRESHOLD_DAYS, test_type="quarantinedTest" ) - # Get regular test results for flaky regression analysis regular_results = analyzer.get_test_results( PROJECT, threshold_days=7, # Last 7 days for regular tests @@ -931,182 +1037,19 @@ def main(): success_threshold=SUCCESS_THRESHOLD ) - # Print summary first + # Print report header print(f"\n# Flaky Test Report for {datetime.now(pytz.UTC).strftime('%Y-%m-%d')}") print(f"This report was run on {datetime.now(pytz.UTC).strftime('%Y-%m-%d %H:%M:%S')} UTC") - - print_summary(problematic_tests, flaky_regressions, QUARANTINE_THRESHOLD_DAYS) - - # Print Flaky Test Regressions - print("\n## Flaky Test Regressions") - if not flaky_regressions: - print("No flaky test regressions found.") - else: - print(f"Found {len(flaky_regressions)} tests that have started showing increased flaky behavior recently.") - print("\n") - print("") - - for test_name, details in flaky_regressions.items(): - class_link = get_develocity_class_link(test_name, QUARANTINE_THRESHOLD_DAYS) - print(f"") - print(f"" - f"" - f"") - - # Add recent execution details in sub-rows - print("") - for entry in sorted(details['recent_executions'], key=lambda x: x.timestamp)[-5:]: - date_str = entry.timestamp.strftime('%Y-%m-%d %H:%M') - print(f"") - print("
Test ClassRecent Flaky RateHistorical RateRecent Executions
{test_name}
{details['recent_flaky_rate']:.2%}{details['historical_flaky_rate']:.2%}{len(details['recent_executions'])}
Recent Executions:
{date_str} - {entry.outcome}
") - # Print Cleared Tests - print("\n## Cleared Tests (Ready for Unquarantine)") - if not cleared_tests: - print("No tests ready to be cleared from quarantine.") - else: - # Print concise table with just class and method info - print("\n") - print("") - - for test_name, details in cleared_tests.items(): - # Print class-level information - class_link = get_develocity_class_link(test_name, QUARANTINE_THRESHOLD_DAYS) - print(f"") - print(f"" - f"" - f"" - f"") - - # Print method-level information - for test_case in details['test_cases']: - method_name = test_case['name'].split('.')[-1] - full_method_name = test_case['name'] - method_link = get_develocity_method_link(test_name, full_method_name, QUARANTINE_THRESHOLD_DAYS) - recent_status = "N/A" - if test_case['recent_executions']: - recent_status = test_case['recent_executions'][-1].outcome - - print(f"" - f"" - f"" - f"") - - print("") # Add spacing between classes - print("
Test ClassTest MethodSuccess RateTotal RunsRecent Status
{test_name}
Class Overall{details['success_rate']:.2%}{details['total_executions']}{details['successful_runs']} passed
{method_name}{test_case['success_rate']:.2%}{test_case['total_executions']}{recent_status}
 
") - - # Print detailed execution history in collapsible sections - print("\n
") - print("Detailed Execution History\n") - - for test_name, details in cleared_tests.items(): - print(f"\n### {test_name}") - - for test_case in details['test_cases']: - method_name = test_case['name'].split('.')[-1] - print(f"\n#### {method_name}") - print("Recent Executions:") - print("```") - print("Date/Time (UTC) Outcome Build ID") - print("-" * 44) - for entry in test_case['recent_executions']: - date_str = entry.timestamp.strftime('%Y-%m-%d %H:%M') - print(f"{date_str:<17} {entry.outcome:<10} {entry.build_id}") - print("```") - - print("
") - - # Print High-Priority Quarantined Tests - print("\n## High-Priority Quarantined Tests") - if not problematic_tests: - print("No high-priority quarantined tests found.") - else: - print("These are tests which have been quarantined for several days and need attention.") - sorted_tests = sorted( - problematic_tests.items(), - key=lambda x: (x[1]['failure_rate'], x[1]['days_quarantined']), - reverse=True - ) - - print(f"\nFound {len(sorted_tests)} high-priority quarantined test classes:") - print("\n") - print("") - - for full_class_name, details in sorted_tests: - class_result = details['container_result'] - class_link = get_develocity_class_link(full_class_name, QUARANTINE_THRESHOLD_DAYS) - - # Print class-level information - print(f"") - print(f"" - f"" - f"" - f"" - f"") - - # Print build outcome statistics - print("") - - # Print test method summary - print("") - print("") - - for test_method in sorted(details['test_cases'], - key=lambda x: (x.outcome_distribution.failed + x.outcome_distribution.flaky) / x.outcome_distribution.total if x.outcome_distribution.total > 0 else 0, - reverse=True): - - method_name = test_method.name.split('.')[-1] - method_link = get_develocity_method_link(full_class_name, test_method.name, QUARANTINE_THRESHOLD_DAYS) - total_runs = test_method.outcome_distribution.total - if total_runs > 0: - success_rate = test_method.outcome_distribution.passed / total_runs - failure_rate = (test_method.outcome_distribution.failed + - test_method.outcome_distribution.flaky) / total_runs - - recent_status = "N/A" - if test_method.timeline: - recent_status = test_method.timeline[-1].outcome - - print(f"" - f"" - f"" - f"" - f"") - - print("") # Add spacing between classes - print("
Test ClassDays QuarantinedFailure RateRecent Failure RateTotal Runs
{full_class_name}
{details['days_quarantined']}{details['failure_rate']:.2%}{details['recent_failure_rate']:.2%}{class_result.outcome_distribution.total}
Build Outcomes: " - f"Passed: {class_result.outcome_distribution.passed} | " - f"Failed: {class_result.outcome_distribution.failed} | " - f"Flaky: {class_result.outcome_distribution.flaky}" - "
Test Methods (Last 7 Days):
MethodSuccess RateFailure RateTotal RunsRecent Status
{method_name}{success_rate:.2%}{failure_rate:.2%}{total_runs}{recent_status}
 
") - - # Print detailed execution history in collapsible sections - print("\n
") - print("Detailed Execution History\n") - - for full_class_name, details in sorted_tests: - print(f"\n### {full_class_name}") - - for test_method in details['test_cases']: - method_name = test_method.name.split('.')[-1] - if test_method.timeline: - print(f"\n#### {method_name}") - print("Recent Executions:") - print("```") - print("Date/Time (UTC) Outcome Build ID") - print("-" * 44) - for entry in sorted(test_method.timeline, key=lambda x: x.timestamp)[-5:]: - date_str = entry.timestamp.strftime('%Y-%m-%d %H:%M') - print(f"{date_str:<17} {entry.outcome:<10} {entry.build_id}") - print("```") - - print("
") + # Print each section + print_most_problematic_tests(problematic_tests, QUARANTINE_THRESHOLD_DAYS) + print_flaky_regressions(flaky_regressions, QUARANTINE_THRESHOLD_DAYS) + print_cleared_tests(cleared_tests, QUARANTINE_THRESHOLD_DAYS) except Exception as e: logger.exception("Error occurred during report generation") print(f"Error occurred: {str(e)}") - if __name__ == "__main__": # Configure logging logging.basicConfig( From b482001d0d39443c487d8ad858a8a9c5d874e398 Mon Sep 17 00:00:00 2001 From: santhoshct Date: Wed, 18 Dec 2024 11:55:08 +0530 Subject: [PATCH 4/6] modified layout --- .github/scripts/develocity_reports.py | 89 +++++++++++++++------------ 1 file changed, 51 insertions(+), 38 deletions(-) diff --git a/.github/scripts/develocity_reports.py b/.github/scripts/develocity_reports.py index 44c5798e174df..17757cc2cc7a6 100644 --- a/.github/scripts/develocity_reports.py +++ b/.github/scripts/develocity_reports.py @@ -820,38 +820,35 @@ def print_most_problematic_tests(problematic_tests: Dict[str, Dict], threshold_d print(f"Found {len(problematic_tests)} tests that have been quarantined for {threshold_days} days and are still failing frequently.") - # Print concise table with key metrics + # Print table with class and method information print("\n") - print("") + print("") for test_name, details in sorted(problematic_tests.items(), - key=lambda x: x[1]['recent_failure_rate'], + key=lambda x: x[1]['failure_rate'], reverse=True): class_link = get_develocity_class_link(test_name, threshold_days) + print(f"") - # Find the method with highest failure rate - worst_method = max(details['test_cases'], - key=lambda x: (x.outcome_distribution.failed + x.outcome_distribution.flaky) / x.outcome_distribution.total - if x.outcome_distribution.total > 0 else 0) - - method_name = worst_method.name.split('.')[-1] - method_link = get_develocity_method_link(test_name, worst_method.name, threshold_days) - method_failure_rate = (worst_method.outcome_distribution.failed + - worst_method.outcome_distribution.flaky) / worst_method.outcome_distribution.total - - print(f"" - f"" - f"" - f"" - f"") + for test_case in sorted(details['test_cases'], + key=lambda x: (x.outcome_distribution.failed + x.outcome_distribution.flaky) / x.outcome_distribution.total + if x.outcome_distribution.total > 0 else 0, + reverse=True): + method_name = test_case.name.split('.')[-1] + if method_name != 'N/A': + method_link = get_develocity_method_link(test_name, test_case.name, threshold_days) + total_runs = test_case.outcome_distribution.total + failure_rate = (test_case.outcome_distribution.failed + test_case.outcome_distribution.flaky) / total_runs if total_runs > 0 else 0 + print(f"" + f"") print("
Test ClassDays QuarantinedRecent Failure RateMost Failing MethodMethod Failure Rate
ClassTest CaseFailure RateBuild Scans
{test_name}
{test_name}{details['days_quarantined']}{details['recent_failure_rate']:.2%}{method_name}{method_failure_rate:.2%}
{method_name}{failure_rate:.2%}{total_runs}
") - # Print detailed execution history in collapsible section + # Print detailed execution history print("\n
") print("Detailed Execution History\n") for test_name, details in sorted(problematic_tests.items(), - key=lambda x: x[1]['recent_failure_rate'], + key=lambda x: x[1]['failure_rate'], reverse=True): print(f"\n### {test_name}") print(f"* Days Quarantined: {details['days_quarantined']}") @@ -888,21 +885,22 @@ def print_flaky_regressions(flaky_regressions: Dict[str, Dict], threshold_days: print(f"Found {len(flaky_regressions)} tests that have started showing increased flaky behavior recently.") - # Print concise table + # Print table with test details print("\n") - print("") + print("") - for test_name, details in sorted(flaky_regressions.items(), - key=lambda x: x[1]['recent_flaky_rate'], - reverse=True): + for test_name, details in flaky_regressions.items(): class_link = get_develocity_class_link(test_name, threshold_days) - rate_change = details['recent_flaky_rate'] - details['historical_flaky_rate'] - - print(f"" - f"" + print(f"") + print(f"" f"" - f"" f"") + + # Add recent execution details in sub-rows + print("") + for entry in sorted(details['recent_executions'], key=lambda x: x.timestamp)[-5:]: + date_str = entry.timestamp.strftime('%Y-%m-%d %H:%M') + print(f"") print("
Test ClassRecent Flaky RateHistorical RateChangeRecent Executions
Test ClassRecent Flaky RateHistorical RateRecent Executions
{test_name}{details['recent_flaky_rate']:.2%}
{test_name}
{details['recent_flaky_rate']:.2%}{details['historical_flaky_rate']:.2%}{rate_change:+.2%}{len(details['recent_executions'])}
Recent Executions:
{date_str} - {entry.outcome}
") # Print detailed history @@ -933,23 +931,38 @@ def print_cleared_tests(cleared_tests: Dict[str, Dict], threshold_days: int): print("No tests ready to be cleared from quarantine.") return - print(f"Found {len(cleared_tests)} test classes that have been consistently passing and could be removed from quarantine.") + # Calculate total number of test methods + total_methods = sum(len(details['test_cases']) for details in cleared_tests.values()) + + print(f"Found {len(cleared_tests)} test classes with {total_methods} test methods that have been consistently passing. " + f"These tests could be candidates for removing quarantine annotations at either class or method level.") - # Print concise table + # Print table with class and method information print("\n") - print("") + print("") for test_name, details in sorted(cleared_tests.items(), key=lambda x: x[1]['success_rate'], reverse=True): class_link = get_develocity_class_link(test_name, threshold_days) - method_count = len(details['test_cases']) - - print(f"" + print(f"") + print(f"" f"" f"" - f"" - f"") + f"") + + for test_case in details['test_cases']: + method_name = test_case['name'].split('.')[-1] + method_link = get_develocity_method_link(test_name, test_case['name'], threshold_days) + recent_status = "N/A" + if test_case['recent_executions']: + recent_status = test_case['recent_executions'][-1].outcome + + print(f"" + f"" + f"" + f"") + print("") print("
Test ClassSuccess RateTotal RunsDays CleanMethods
Test ClassTest MethodSuccess RateTotal RunsRecent Status
{test_name}
{test_name}
Class Overall{details['success_rate']:.2%}{details['total_executions']}{details['successful_runs']}{method_count}
{details['successful_runs']} passed
{method_name}{test_case['success_rate']:.2%}{test_case['total_executions']}{recent_status}
 
") # Print detailed history From b74b21261d229811bb9c453f975fd9bbe33a15a3 Mon Sep 17 00:00:00 2001 From: santhoshct Date: Wed, 18 Dec 2024 14:47:37 +0530 Subject: [PATCH 5/6] removed hyperlinks from test names and made it simple --- .github/scripts/develocity_reports.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/scripts/develocity_reports.py b/.github/scripts/develocity_reports.py index 17757cc2cc7a6..898dabb327e55 100644 --- a/.github/scripts/develocity_reports.py +++ b/.github/scripts/develocity_reports.py @@ -828,7 +828,7 @@ def print_most_problematic_tests(problematic_tests: Dict[str, Dict], threshold_d key=lambda x: x[1]['failure_rate'], reverse=True): class_link = get_develocity_class_link(test_name, threshold_days) - print(f"{test_name}") + print(f"{test_name} ↗️") for test_case in sorted(details['test_cases'], key=lambda x: (x.outcome_distribution.failed + x.outcome_distribution.flaky) / x.outcome_distribution.total @@ -839,7 +839,7 @@ def print_most_problematic_tests(problematic_tests: Dict[str, Dict], threshold_d method_link = get_develocity_method_link(test_name, test_case.name, threshold_days) total_runs = test_case.outcome_distribution.total failure_rate = (test_case.outcome_distribution.failed + test_case.outcome_distribution.flaky) / total_runs if total_runs > 0 else 0 - print(f"{method_name}" + print(f"{method_name} ↗️" f"{failure_rate:.2%}{total_runs}") print("") @@ -891,7 +891,7 @@ def print_flaky_regressions(flaky_regressions: Dict[str, Dict], threshold_days: for test_name, details in flaky_regressions.items(): class_link = get_develocity_class_link(test_name, threshold_days) - print(f"{test_name}") + print(f"{test_name} ↗️") print(f"{details['recent_flaky_rate']:.2%}" f"{details['historical_flaky_rate']:.2%}" f"{len(details['recent_executions'])}") @@ -945,7 +945,7 @@ def print_cleared_tests(cleared_tests: Dict[str, Dict], threshold_days: int): key=lambda x: x[1]['success_rate'], reverse=True): class_link = get_develocity_class_link(test_name, threshold_days) - print(f"{test_name}") + print(f"{test_name} ↗️") print(f"Class Overall" f"{details['success_rate']:.2%}" f"{details['total_executions']}" @@ -958,7 +958,7 @@ def print_cleared_tests(cleared_tests: Dict[str, Dict], threshold_days: int): if test_case['recent_executions']: recent_status = test_case['recent_executions'][-1].outcome - print(f"{method_name}" + print(f"{method_name} ↗️" f"{test_case['success_rate']:.2%}" f"{test_case['total_executions']}" f"{recent_status}") From c1781c486fae159476e82e9457b25f4d8297c54f Mon Sep 17 00:00:00 2001 From: santhoshct Date: Wed, 18 Dec 2024 22:42:04 +0530 Subject: [PATCH 6/6] moved develocity link to right most column --- .github/scripts/develocity_reports.py | 32 ++++++++++++++------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/.github/scripts/develocity_reports.py b/.github/scripts/develocity_reports.py index 898dabb327e55..102ffba293090 100644 --- a/.github/scripts/develocity_reports.py +++ b/.github/scripts/develocity_reports.py @@ -822,13 +822,13 @@ def print_most_problematic_tests(problematic_tests: Dict[str, Dict], threshold_d # Print table with class and method information print("\n") - print("") + print("") for test_name, details in sorted(problematic_tests.items(), key=lambda x: x[1]['failure_rate'], reverse=True): class_link = get_develocity_class_link(test_name, threshold_days) - print(f"") + print(f"") for test_case in sorted(details['test_cases'], key=lambda x: (x.outcome_distribution.failed + x.outcome_distribution.flaky) / x.outcome_distribution.total @@ -839,8 +839,9 @@ def print_most_problematic_tests(problematic_tests: Dict[str, Dict], threshold_d method_link = get_develocity_method_link(test_name, test_case.name, threshold_days) total_runs = test_case.outcome_distribution.total failure_rate = (test_case.outcome_distribution.failed + test_case.outcome_distribution.flaky) / total_runs if total_runs > 0 else 0 - print(f"" - f"") + print(f"" + f"" + f"") print("
ClassTest CaseFailure RateBuild Scans
ClassTest CaseFailure RateBuild ScansLink
{test_name} ↗️
{test_name}↗️
{method_name} ↗️{failure_rate:.2%}{total_runs}
{method_name}{failure_rate:.2%}{total_runs}↗️
") # Print detailed execution history @@ -887,20 +888,20 @@ def print_flaky_regressions(flaky_regressions: Dict[str, Dict], threshold_days: # Print table with test details print("\n") - print("") + print("") for test_name, details in flaky_regressions.items(): class_link = get_develocity_class_link(test_name, threshold_days) - print(f"") + print(f"") print(f"" f"" - f"") + f"") # Add recent execution details in sub-rows - print("") + print("") for entry in sorted(details['recent_executions'], key=lambda x: x.timestamp)[-5:]: date_str = entry.timestamp.strftime('%Y-%m-%d %H:%M') - print(f"") + print(f"") print("
Test ClassRecent Flaky RateHistorical RateRecent Executions
Test ClassRecent Flaky RateHistorical RateRecent ExecutionsLink
{test_name} ↗️
{test_name}↗️
{details['recent_flaky_rate']:.2%}{details['historical_flaky_rate']:.2%}{len(details['recent_executions'])}
{len(details['recent_executions'])}
Recent Executions:
Recent Executions:
{date_str} - {entry.outcome}
{date_str} - {entry.outcome}
") # Print detailed history @@ -939,17 +940,17 @@ def print_cleared_tests(cleared_tests: Dict[str, Dict], threshold_days: int): # Print table with class and method information print("\n") - print("") + print("") for test_name, details in sorted(cleared_tests.items(), key=lambda x: x[1]['success_rate'], reverse=True): class_link = get_develocity_class_link(test_name, threshold_days) - print(f"") + print(f"") print(f"" f"" f"" - f"") + f"") for test_case in details['test_cases']: method_name = test_case['name'].split('.')[-1] @@ -958,11 +959,12 @@ def print_cleared_tests(cleared_tests: Dict[str, Dict], threshold_days: int): if test_case['recent_executions']: recent_status = test_case['recent_executions'][-1].outcome - print(f"" + print(f"" f"" f"" - f"") - print("") + f"" + f"") + print("") print("
Test ClassTest MethodSuccess RateTotal RunsRecent Status
Test ClassTest MethodSuccess RateTotal RunsRecent StatusLink
{test_name} ↗️
{test_name}↗️
Class Overall{details['success_rate']:.2%}{details['total_executions']}{details['successful_runs']} passed
{details['successful_runs']} passed
{method_name} ↗️
{method_name}{test_case['success_rate']:.2%}{test_case['total_executions']}{recent_status}
 
{recent_status}↗️
 
") # Print detailed history