From 8c9b89a801167e61e362a8a2c3e8c2aaae4a6dda Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Fri, 2 May 2025 10:39:20 -0400
Subject: [PATCH 01/36] tweak table formatting

---
 .../actions/create_workflow_report/create_workflow_report.py  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/actions/create_workflow_report/create_workflow_report.py b/.github/actions/create_workflow_report/create_workflow_report.py
index ceaac52b57e1..ce7a3dccfb5b 100755
--- a/.github/actions/create_workflow_report/create_workflow_report.py
+++ b/.github/actions/create_workflow_report/create_workflow_report.py
@@ -304,7 +304,7 @@ def get_cves(pr_number, commit_sha):
 def url_to_html_link(url: str) -> str:
     if not url:
         return ""
-    text = url.split("/")[-1]
+    text = url.split("/")[-1].replace("__", "_")
     if not text:
         text = "results"
     return f'<a href="{url}">{text}</a>'
@@ -312,7 +312,7 @@ def url_to_html_link(url: str) -> str:
 
 def format_test_name_for_linewrap(text: str) -> str:
     """Tweak the test name to improve line wrapping."""
-    return text.replace(".py::", "/")
+    return f'<span style="line-break: anywhere;">{text}</span>'
 
 
 def format_test_status(text: str) -> str:

From 49e9b1429068805933acb9dfaba7f648d506a98e Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Fri, 2 May 2025 11:08:26 -0400
Subject: [PATCH 02/36] Add report table displaying new fails in PR

---
 .../ci_run_report.html.jinja                  |  6 ++
 .../create_workflow_report.py                 | 96 +++++++++++++++++++
 2 files changed, 102 insertions(+)

diff --git a/.github/actions/create_workflow_report/ci_run_report.html.jinja b/.github/actions/create_workflow_report/ci_run_report.html.jinja
index 7a8a91c4ff29..13bded3e54b8 100644
--- a/.github/actions/create_workflow_report/ci_run_report.html.jinja
+++ b/.github/actions/create_workflow_report/ci_run_report.html.jinja
@@ -161,6 +161,7 @@
   {% endif %}
   <h2>Table of Contents</h2>
   <ul>
+    {%- if pr_number != 0 %}<li><a href="#new-fails-pr">New Fails in PR</a> ({{ counts.pr_new_fails }})</li>{% endif %}
     <li><a href="#ci-jobs-status">CI Jobs Status</a> ({{ counts.jobs_status }})</li>
     <li><a href="#checks-errors">Checks Errors</a> ({{ counts.checks_errors }})</li>
     <li><a href="#checks-fails">Checks New Fails</a> ({{ counts.checks_new_fails }})</li>
@@ -169,6 +170,11 @@
     <li><a href="#checks-known-fails">Checks Known Fails</a> ({{ counts.checks_known_fails }})</li>
   </ul>
 
+  {%- if pr_number != 0 -%}
+  <h2 id="new-fails-pr">New Fails in PR</h2>
+  {{ new_fails_html }}
+  {%- endif %}
+
   <h2 id="ci-jobs-status">CI Jobs Status</h2>
   {{ ci_jobs_status_html }}
 
diff --git a/.github/actions/create_workflow_report/create_workflow_report.py b/.github/actions/create_workflow_report/create_workflow_report.py
index ce7a3dccfb5b..a8e28baf5973 100755
--- a/.github/actions/create_workflow_report/create_workflow_report.py
+++ b/.github/actions/create_workflow_report/create_workflow_report.py
@@ -247,6 +247,91 @@ def get_regression_fails(client: Client, job_url: str):
     return df
 
 
+def get_new_fails_this_pr(
+    client: Client,
+    pr_info: dict,
+    checks_fails: pd.DataFrame,
+    regression_fails: pd.DataFrame,
+):
+    """
+    Get tests that failed in the PR but passed in the base branch.
+    Compares both checks and regression test results.
+    """
+    base_sha = pr_info.get("base", {}).get("sha")
+    if not base_sha:
+        raise Exception("No base SHA found for PR")
+
+    # Modify tables to have the same columns
+    if len(checks_fails) > 0:
+        checks_fails = checks_fails.copy().drop(columns=["job_status"])
+    if len(regression_fails) > 0:
+        regression_fails = regression_fails.copy()
+        regression_fails["job_name"] = regression_fails.apply(
+            lambda row: f"{row['arch']} {row['job_name']}".strip(), axis=1
+        )
+        regression_fails["test_status"] = regression_fails["status"]
+        regression_fails = regression_fails.drop(columns=["arch", "status"])
+
+    # Combine both types of fails
+    all_pr_fails = pd.concat([checks_fails, regression_fails], ignore_index=True)
+    if len(all_pr_fails) == 0:
+        return pd.DataFrame()
+
+    # Get all checks from the base branch that didn't fail
+    columns = (
+        "check_name as job_name, test_status, test_name, report_url as results_link"
+    )
+    base_checks_query = f"""SELECT {columns} FROM `gh-data`.checks
+                WHERE commit_sha='{base_sha}'
+                AND test_status NOT IN ('FAIL', 'ERROR')
+                AND check_status!='error'
+                ORDER BY check_name, test_name
+                """
+    base_checks = client.query_dataframe(base_checks_query)
+
+    # Get regression results from base branch that didn't fail
+    base_regression_query = f"""SELECT arch, job_name, status, test_name, results_link
+            FROM (
+               SELECT
+                    architecture as arch,
+                    test_name,
+                    argMax(result, start_time) AS status,
+                    job_url,
+                    job_name,
+                    report_url as results_link
+               FROM `gh-data`.clickhouse_regression_results
+               WHERE results_link LIKE'%/{base_sha}/%'
+               GROUP BY architecture, test_name, job_url, job_name, report_url
+               ORDER BY length(test_name) DESC
+            )
+            WHERE status NOT IN ('Fail', 'Error')
+            """
+    base_regression = client.query_dataframe(base_regression_query)
+    if len(base_regression) > 0:
+        base_regression["job_name"] = base_regression.apply(
+            lambda row: f"{row['arch']} {row['job_name']}".strip(), axis=1
+        )
+        base_regression["test_status"] = base_regression["status"]
+        base_regression = base_regression.drop(columns=["arch", "status"])
+
+    # Combine base results
+    base_results = pd.concat([base_checks, base_regression], ignore_index=True)
+
+    # Find tests that failed in PR but passed in base
+    pr_failed_tests = set(zip(all_pr_fails["job_name"], all_pr_fails["test_name"]))
+    base_passed_tests = set(zip(base_results["job_name"], base_results["test_name"]))
+
+    new_fails = pr_failed_tests.intersection(base_passed_tests)
+
+    # Filter PR results to only include new fails
+    mask = all_pr_fails.apply(
+        lambda row: (row["job_name"], row["test_name"]) in new_fails, axis=1
+    )
+    new_fails_df = all_pr_fails[mask]
+
+    return new_fails_df
+
+
 def get_cves(pr_number, commit_sha):
     """
     Fetch Grype results from S3.
@@ -349,6 +434,7 @@ def format_results_as_html_table(results) -> str:
     return html
 
 
+
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description="Create a combined CI report.")
     parser.add_argument(  # Need the full URL rather than just the ID to query the databases
@@ -400,6 +486,7 @@ def main():
         "job_statuses": get_commit_statuses(args.commit_sha),
         "checks_fails": get_checks_fails(db_client, args.actions_run_url),
         "checks_known_fails": [],
+        "pr_new_fails": [],
         "checks_errors": get_checks_errors(db_client, args.actions_run_url),
         "regression_fails": get_regression_fails(db_client, args.actions_run_url),
         "docker_images_cves": (
@@ -434,6 +521,12 @@ def main():
             pr_info_html = f"""<a href="https://github.com/{GITHUB_REPO}/pull/{pr_info["number"]}">
                     #{pr_info.get("number")} ({pr_info.get("base", {}).get('ref')} <- {pr_info.get("head", {}).get('ref')})  {pr_info.get("title")}
                     </a>"""
+            fail_results["pr_new_fails"] = get_new_fails_this_pr(
+                db_client,
+                pr_info,
+                fail_results["checks_fails"],
+                fail_results["regression_fails"],
+            )
         except Exception as e:
             pr_info_html = e
 
@@ -451,6 +544,7 @@ def main():
         "title": "ClickHouse® CI Workflow Run Report",
         "github_repo": GITHUB_REPO,
         "pr_info_html": pr_info_html,
+        "pr_number": args.pr_number,
         "workflow_id": args.actions_run_url.split("/")[-1],
         "commit_sha": args.commit_sha,
         "date": f"{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')} UTC",
@@ -466,6 +560,7 @@ def main():
                 if not args.known_fails
                 else len(fail_results["checks_known_fails"])
             ),
+            "pr_new_fails": len(fail_results["pr_new_fails"]),
         },
         "ci_jobs_status_html": format_results_as_html_table(
             fail_results["job_statuses"]
@@ -487,6 +582,7 @@ def main():
             if not args.known_fails
             else format_results_as_html_table(fail_results["checks_known_fails"])
         ),
+        "new_fails_html": format_results_as_html_table(fail_results["pr_new_fails"]),
     }
 
     # Render the template with the context

From 2553b85d54e3c807ac579f69d45bec4f347a99fa Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Fri, 2 May 2025 11:14:51 -0400
Subject: [PATCH 03/36] Add note about know fail reason conventions

---
 .../actions/create_workflow_report/ci_run_report.html.jinja | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/actions/create_workflow_report/ci_run_report.html.jinja b/.github/actions/create_workflow_report/ci_run_report.html.jinja
index 13bded3e54b8..8151cde26fdc 100644
--- a/.github/actions/create_workflow_report/ci_run_report.html.jinja
+++ b/.github/actions/create_workflow_report/ci_run_report.html.jinja
@@ -191,6 +191,12 @@
   {{ docker_images_cves_html }}
 
   <h2 id="checks-known-fails">Checks Known Fails</h2>
+  <p>
+    Fail reason conventions:<br/>
+    KNOWN - Accepted fail and fix is not planned<br/>
+    INVESTIGATE - We don't know why it fails<br/>
+    NEEDSFIX - Investigation done and a fix is needed to make it pass<br/>
+  </p>
   {{ checks_known_fails_html }}
 
   <script>

From 2582a4b323b2890df4ce22a3691232f2b3d6a5c1 Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Mon, 5 May 2025 09:25:04 -0400
Subject: [PATCH 04/36] fix broken integration tests not being handled in flaky
 check

---
 tests/ci/integration_tests_runner.py | 45 +++++++++++++++++-----------
 1 file changed, 27 insertions(+), 18 deletions(-)

diff --git a/tests/ci/integration_tests_runner.py b/tests/ci/integration_tests_runner.py
index 17bfdb3bf5c3..ab35c1a0e8b9 100755
--- a/tests/ci/integration_tests_runner.py
+++ b/tests/ci/integration_tests_runner.py
@@ -526,6 +526,27 @@ def _update_counters(main_counters, current_counters):
             for test in current_counters[state]:
                 main_counters[state].append(test)
 
+    @staticmethod
+    def _handle_broken_tests(group_counters, known_broken_tests, log_paths):
+        for fail_status in ("ERROR", "FAILED"):
+            for failed_test in group_counters[fail_status]:
+                if failed_test in known_broken_tests.keys():
+                    fail_message = known_broken_tests[failed_test].get("message")
+                    if not fail_message:
+                        mark_as_broken = True
+                    else:
+                        mark_as_broken = False
+                        for log_path in log_paths:
+                            if log_path.endswith(".log"):
+                                with open(log_path) as log_file:
+                                    if fail_message in log_file.read():
+                                        mark_as_broken = True
+                                        break
+
+                    if mark_as_broken:
+                        group_counters[fail_status].remove(failed_test)
+                        group_counters["BROKEN"].append(failed_test)
+
     def _get_runner_image_cmd(self):
         image_cmd = ""
         if self._can_run_with(
@@ -812,6 +833,7 @@ def run_flaky_check(self, build_path, should_fail=False):
         tests_times = defaultdict(float)  # type: Dict
         tests_log_paths = defaultdict(list)
         id_counter = 0
+        known_broken_tests = self._get_broken_tests_list(self.repo_path)
         for test_to_run in tests_to_run:
             tries_num = 1 if should_fail else FLAKY_TRIES_COUNT
             for i in range(tries_num):
@@ -828,6 +850,9 @@ def run_flaky_check(self, build_path, should_fail=False):
                     FLAKY_REPEAT_COUNT,
                 )
                 id_counter = id_counter + 1
+
+                self._handle_broken_tests(group_counters, known_broken_tests, log_paths)
+
                 for counter, value in group_counters.items():
                     logging.info(
                         "Tests from group %s stats, %s count %s",
@@ -841,6 +866,7 @@ def run_flaky_check(self, build_path, should_fail=False):
 
                 for test_name, test_time in group_test_times.items():
                     tests_times[test_name] = test_time
+
                 if not should_fail and (
                     group_counters["FAILED"] or group_counters["ERROR"]
                 ):
@@ -1021,24 +1047,7 @@ def run_normal_check(self, build_path):
                 group, tests, MAX_RETRY, NUM_WORKERS, 0
             )
 
-            for fail_status in ("ERROR", "FAILED"):
-                for failed_test in group_counters[fail_status]:
-                    if failed_test in known_broken_tests.keys():
-                        fail_message = known_broken_tests[failed_test].get("message")
-                        if not fail_message:
-                            mark_as_broken = True
-                        else:
-                            mark_as_broken = False
-                            for log_path in log_paths:
-                                if log_path.endswith(".log"):
-                                    with open(log_path) as log_file:
-                                        if fail_message in log_file.read():
-                                            mark_as_broken = True
-                                            break
-
-                        if mark_as_broken:
-                            group_counters[fail_status].remove(failed_test)
-                            group_counters["BROKEN"].append(failed_test)
+            self._handle_broken_tests(group_counters, known_broken_tests, log_paths)
 
             total_tests = 0
             for counter, value in group_counters.items():

From 3754c1b549ec3db4739d0f8df29dbddd0c3d0498 Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Mon, 5 May 2025 14:50:16 -0400
Subject: [PATCH 05/36] fix null columns in new fails table

---
 .../create_workflow_report/create_workflow_report.py     | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/actions/create_workflow_report/create_workflow_report.py b/.github/actions/create_workflow_report/create_workflow_report.py
index a8e28baf5973..531f2fbf3f30 100755
--- a/.github/actions/create_workflow_report/create_workflow_report.py
+++ b/.github/actions/create_workflow_report/create_workflow_report.py
@@ -270,10 +270,12 @@ def get_new_fails_this_pr(
             lambda row: f"{row['arch']} {row['job_name']}".strip(), axis=1
         )
         regression_fails["test_status"] = regression_fails["status"]
-        regression_fails = regression_fails.drop(columns=["arch", "status"])
 
-    # Combine both types of fails
-    all_pr_fails = pd.concat([checks_fails, regression_fails], ignore_index=True)
+    # Combine both types of fails and select only desired columns
+    desired_columns = ["job_name", "test_name", "test_status", "results_link"]
+    all_pr_fails = pd.concat([checks_fails, regression_fails], ignore_index=True)[
+        desired_columns
+    ]
     if len(all_pr_fails) == 0:
         return pd.DataFrame()
 
@@ -434,7 +436,6 @@ def format_results_as_html_table(results) -> str:
     return html
 
 
-
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description="Create a combined CI report.")
     parser.add_argument(  # Need the full URL rather than just the ID to query the databases

From a66516e3e298562f31444c97ce9b5b1f8312572c Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Mon, 5 May 2025 16:05:36 -0400
Subject: [PATCH 06/36] try again fixing broken test handling in flaky mode

---
 tests/ci/integration_tests_runner.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/tests/ci/integration_tests_runner.py b/tests/ci/integration_tests_runner.py
index ab35c1a0e8b9..1038344a43d8 100755
--- a/tests/ci/integration_tests_runner.py
+++ b/tests/ci/integration_tests_runner.py
@@ -528,6 +528,13 @@ def _update_counters(main_counters, current_counters):
 
     @staticmethod
     def _handle_broken_tests(group_counters, known_broken_tests, log_paths):
+
+        def get_log_paths(test_name):
+            """Could be a list of logs for all tests or a dict with test name as a key"""
+            if isinstance(log_paths, dict):
+                return log_paths[test_name]
+            return log_paths
+
         for fail_status in ("ERROR", "FAILED"):
             for failed_test in group_counters[fail_status]:
                 if failed_test in known_broken_tests.keys():
@@ -536,7 +543,7 @@ def _handle_broken_tests(group_counters, known_broken_tests, log_paths):
                         mark_as_broken = True
                     else:
                         mark_as_broken = False
-                        for log_path in log_paths:
+                        for log_path in get_log_paths(failed_test):
                             if log_path.endswith(".log"):
                                 with open(log_path) as log_file:
                                     if fail_message in log_file.read():
@@ -833,7 +840,6 @@ def run_flaky_check(self, build_path, should_fail=False):
         tests_times = defaultdict(float)  # type: Dict
         tests_log_paths = defaultdict(list)
         id_counter = 0
-        known_broken_tests = self._get_broken_tests_list(self.repo_path)
         for test_to_run in tests_to_run:
             tries_num = 1 if should_fail else FLAKY_TRIES_COUNT
             for i in range(tries_num):
@@ -850,9 +856,6 @@ def run_flaky_check(self, build_path, should_fail=False):
                     FLAKY_REPEAT_COUNT,
                 )
                 id_counter = id_counter + 1
-
-                self._handle_broken_tests(group_counters, known_broken_tests, log_paths)
-
                 for counter, value in group_counters.items():
                     logging.info(
                         "Tests from group %s stats, %s count %s",
@@ -866,7 +869,6 @@ def run_flaky_check(self, build_path, should_fail=False):
 
                 for test_name, test_time in group_test_times.items():
                     tests_times[test_name] = test_time
-
                 if not should_fail and (
                     group_counters["FAILED"] or group_counters["ERROR"]
                 ):
@@ -876,6 +878,9 @@ def run_flaky_check(self, build_path, should_fail=False):
                     )
                     break
 
+        known_broken_tests = self._get_broken_tests_list(self.repo_path)
+        self._handle_broken_tests(group_counters, known_broken_tests, tests_log_paths)
+
         if counters["FAILED"]:
             logging.info("Found failed tests: %s", " ".join(counters["FAILED"]))
             description_prefix = "Failed tests found: "

From 732692aa9b5f8201158570ea55dd886305032a32 Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Mon, 5 May 2025 20:56:29 -0400
Subject: [PATCH 07/36] fix typo

---
 tests/ci/integration_tests_runner.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/ci/integration_tests_runner.py b/tests/ci/integration_tests_runner.py
index 1038344a43d8..7cf6008bd887 100755
--- a/tests/ci/integration_tests_runner.py
+++ b/tests/ci/integration_tests_runner.py
@@ -878,8 +878,9 @@ def run_flaky_check(self, build_path, should_fail=False):
                     )
                     break
 
+        # Handle broken tests on the main counters that contain all test results
         known_broken_tests = self._get_broken_tests_list(self.repo_path)
-        self._handle_broken_tests(group_counters, known_broken_tests, tests_log_paths)
+        self._handle_broken_tests(counters, known_broken_tests, tests_log_paths)
 
         if counters["FAILED"]:
             logging.info("Found failed tests: %s", " ".join(counters["FAILED"]))

From 5392dbd94ba213408a6f7c3c3df5c68239527e75 Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Tue, 6 May 2025 08:25:59 -0400
Subject: [PATCH 08/36] add note about base sha to new fails report

---
 .github/actions/create_workflow_report/ci_run_report.html.jinja  | 1 +
 .github/actions/create_workflow_report/create_workflow_report.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/.github/actions/create_workflow_report/ci_run_report.html.jinja b/.github/actions/create_workflow_report/ci_run_report.html.jinja
index 8151cde26fdc..3fdb811ec85f 100644
--- a/.github/actions/create_workflow_report/ci_run_report.html.jinja
+++ b/.github/actions/create_workflow_report/ci_run_report.html.jinja
@@ -172,6 +172,7 @@
 
   {%- if pr_number != 0 -%}
   <h2 id="new-fails-pr">New Fails in PR</h2>
+  <p> Compared with base sha {{ base_sha }} </p>
   {{ new_fails_html }}
   {%- endif %}
 
diff --git a/.github/actions/create_workflow_report/create_workflow_report.py b/.github/actions/create_workflow_report/create_workflow_report.py
index 531f2fbf3f30..8d86c2a29733 100755
--- a/.github/actions/create_workflow_report/create_workflow_report.py
+++ b/.github/actions/create_workflow_report/create_workflow_report.py
@@ -548,6 +548,7 @@ def main():
         "pr_number": args.pr_number,
         "workflow_id": args.actions_run_url.split("/")[-1],
         "commit_sha": args.commit_sha,
+        "base_sha": pr_info.get("base", {}).get("sha"),
         "date": f"{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')} UTC",
         "is_preview": args.mark_preview,
         "counts": {

From f5c8f347e358a431462cfe8b92c77030d7f6ffed Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Tue, 6 May 2025 09:00:41 -0400
Subject: [PATCH 09/36] increase integration tests timeout

---
 tests/ci/ci_config.py                |  8 ++++++--
 tests/ci/integration_tests_runner.py | 19 +++++++++++++++++++
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
index dd3e3a028acb..acde55fa485b 100644
--- a/tests/ci/ci_config.py
+++ b/tests/ci/ci_config.py
@@ -373,11 +373,14 @@ class CI:
             required_builds=[BuildNames.PACKAGE_DEBUG], pr_only=True
         ),
         JobNames.INTEGRATION_TEST_ASAN: CommonJobConfigs.INTEGRATION_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_ASAN], num_batches=4
+            required_builds=[BuildNames.PACKAGE_ASAN],
+            num_batches=4,
+            timeout=9000,  # the job timed out with default value (7200)
         ),
         JobNames.INTEGRATION_TEST_ASAN_OLD_ANALYZER: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_ASAN],
             num_batches=6,
+            timeout=9000,  # the job timed out with default value (7200)
         ),
         JobNames.INTEGRATION_TEST_TSAN: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_TSAN],
@@ -392,7 +395,8 @@ class CI:
         JobNames.INTEGRATION_TEST: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_RELEASE],
             num_batches=4,
-            #release_only=True,
+            # release_only=True,
+            timeout=9000,  # the job timed out with default value (7200)
         ),
         JobNames.INTEGRATION_TEST_FLAKY: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_ASAN],
diff --git a/tests/ci/integration_tests_runner.py b/tests/ci/integration_tests_runner.py
index 7cf6008bd887..8b823c8403f0 100755
--- a/tests/ci/integration_tests_runner.py
+++ b/tests/ci/integration_tests_runner.py
@@ -856,6 +856,13 @@ def run_flaky_check(self, build_path, should_fail=False):
                     FLAKY_REPEAT_COUNT,
                 )
                 id_counter = id_counter + 1
+
+                # Debug logging for test statuses
+                logging.info("Test group %s statuses:", test_to_run)
+                for status, tests in group_counters.items():
+                    if tests:
+                        logging.info("  %s: %s", status, tests)
+
                 for counter, value in group_counters.items():
                     logging.info(
                         "Tests from group %s stats, %s count %s",
@@ -878,10 +885,22 @@ def run_flaky_check(self, build_path, should_fail=False):
                     )
                     break
 
+        # Debug logging before handling broken tests
+        logging.info("Statuses before handling broken tests:")
+        for status, tests in counters.items():
+            if tests:
+                logging.info("  %s: %s", status, tests)
+
         # Handle broken tests on the main counters that contain all test results
         known_broken_tests = self._get_broken_tests_list(self.repo_path)
         self._handle_broken_tests(counters, known_broken_tests, tests_log_paths)
 
+        # Debug logging after handling broken tests
+        logging.info("Statuses after handling broken tests:")
+        for status, tests in counters.items():
+            if tests:
+                logging.info("  %s: %s", status, tests)
+
         if counters["FAILED"]:
             logging.info("Found failed tests: %s", " ".join(counters["FAILED"]))
             description_prefix = "Failed tests found: "

From 9c748561a97bcf13266bb3a60f888dd4b024ed1c Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Tue, 6 May 2025 09:42:21 -0400
Subject: [PATCH 10/36] try moving broken test handling a bit later

---
 tests/ci/integration_tests_runner.py | 41 ++++++++++------------------
 1 file changed, 14 insertions(+), 27 deletions(-)

diff --git a/tests/ci/integration_tests_runner.py b/tests/ci/integration_tests_runner.py
index 8b823c8403f0..abf8b8daab45 100755
--- a/tests/ci/integration_tests_runner.py
+++ b/tests/ci/integration_tests_runner.py
@@ -527,7 +527,7 @@ def _update_counters(main_counters, current_counters):
                 main_counters[state].append(test)
 
     @staticmethod
-    def _handle_broken_tests(group_counters, known_broken_tests, log_paths):
+    def _handle_broken_tests(counters, known_broken_tests, log_paths):
 
         def get_log_paths(test_name):
             """Could be a list of logs for all tests or a dict with test name as a key"""
@@ -536,7 +536,7 @@ def get_log_paths(test_name):
             return log_paths
 
         for fail_status in ("ERROR", "FAILED"):
-            for failed_test in group_counters[fail_status]:
+            for failed_test in counters[fail_status]:
                 if failed_test in known_broken_tests.keys():
                     fail_message = known_broken_tests[failed_test].get("message")
                     if not fail_message:
@@ -551,8 +551,8 @@ def get_log_paths(test_name):
                                         break
 
                     if mark_as_broken:
-                        group_counters[fail_status].remove(failed_test)
-                        group_counters["BROKEN"].append(failed_test)
+                        counters[fail_status].remove(failed_test)
+                        counters["BROKEN"].append(failed_test)
 
     def _get_runner_image_cmd(self):
         image_cmd = ""
@@ -856,13 +856,6 @@ def run_flaky_check(self, build_path, should_fail=False):
                     FLAKY_REPEAT_COUNT,
                 )
                 id_counter = id_counter + 1
-
-                # Debug logging for test statuses
-                logging.info("Test group %s statuses:", test_to_run)
-                for status, tests in group_counters.items():
-                    if tests:
-                        logging.info("  %s: %s", status, tests)
-
                 for counter, value in group_counters.items():
                     logging.info(
                         "Tests from group %s stats, %s count %s",
@@ -885,22 +878,10 @@ def run_flaky_check(self, build_path, should_fail=False):
                     )
                     break
 
-        # Debug logging before handling broken tests
-        logging.info("Statuses before handling broken tests:")
-        for status, tests in counters.items():
-            if tests:
-                logging.info("  %s: %s", status, tests)
-
         # Handle broken tests on the main counters that contain all test results
         known_broken_tests = self._get_broken_tests_list(self.repo_path)
         self._handle_broken_tests(counters, known_broken_tests, tests_log_paths)
 
-        # Debug logging after handling broken tests
-        logging.info("Statuses after handling broken tests:")
-        for status, tests in counters.items():
-            if tests:
-                logging.info("  %s: %s", status, tests)
-
         if counters["FAILED"]:
             logging.info("Found failed tests: %s", " ".join(counters["FAILED"]))
             description_prefix = "Failed tests found: "
@@ -1040,8 +1021,6 @@ def run_normal_check(self, build_path):
             " ".join(not_found_tests[:3]),
         )
 
-        known_broken_tests = self._get_broken_tests_list(self.repo_path)
-
         grouped_tests = self.group_test_by_file(filtered_sequential_tests)
         i = 0
         for par_group in chunks(filtered_parallel_tests, PARALLEL_GROUP_SIZE):
@@ -1072,8 +1051,6 @@ def run_normal_check(self, build_path):
                 group, tests, MAX_RETRY, NUM_WORKERS, 0
             )
 
-            self._handle_broken_tests(group_counters, known_broken_tests, log_paths)
-
             total_tests = 0
             for counter, value in group_counters.items():
                 logging.info(
@@ -1096,11 +1073,21 @@ def run_normal_check(self, build_path):
             if len(counters["FAILED"]) + len(counters["ERROR"]) >= 20:
                 logging.info("Collected more than 20 failed/error tests, stopping")
                 break
+
+        # Handle broken tests on the main counters that contain all test results
+        known_broken_tests = self._get_broken_tests_list(self.repo_path)
+        self._handle_broken_tests(counters, known_broken_tests, tests_log_paths)
+
         if counters["FAILED"] or counters["ERROR"]:
             logging.info(
                 "Overall status failure, because we have tests in FAILED or ERROR state"
             )
             result_state = "failure"
+        elif len(counters["BROKEN"]) >= 20:
+            logging.info(
+                "Overall status failure, more than 20 broken tests, likely did not run all tests"
+            )
+            result_state = "failure"
         else:
             logging.info("Overall success!")
             result_state = "success"

From 5891153178849ccfe616631af35d8cbbe174ae70 Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Tue, 6 May 2025 11:53:36 -0400
Subject: [PATCH 11/36] fix typo

---
 tests/broken_tests.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/broken_tests.json b/tests/broken_tests.json
index b45a13782f7f..2086e6f467a5 100644
--- a/tests/broken_tests.json
+++ b/tests/broken_tests.json
@@ -47,7 +47,7 @@
     "03003_prql_panic": {
         "reason": "KNOWN PRQL is disabled in Antalya branch"
     },
-    "test_git_import/test.py::/test_git_import": {
+    "test_git_import/test.py::test_git_import": {
         "reason": "NEEDSFIX 403 while getting data from GitHub"
     },
     "test_storage_delta/test.py::test_types[1]": {

From 8c362c8f218ad73ad105a45afd9714655abbc195 Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Tue, 6 May 2025 13:30:06 -0400
Subject: [PATCH 12/36] log broken test handler actions verbosely to a file

---
 tests/ci/integration_tests_runner.py | 58 ++++++++++++++++++----------
 1 file changed, 38 insertions(+), 20 deletions(-)

diff --git a/tests/ci/integration_tests_runner.py b/tests/ci/integration_tests_runner.py
index abf8b8daab45..a39c9327b323 100755
--- a/tests/ci/integration_tests_runner.py
+++ b/tests/ci/integration_tests_runner.py
@@ -526,8 +526,7 @@ def _update_counters(main_counters, current_counters):
             for test in current_counters[state]:
                 main_counters[state].append(test)
 
-    @staticmethod
-    def _handle_broken_tests(counters, known_broken_tests, log_paths):
+    def _handle_broken_tests(self, counters, known_broken_tests, log_paths):
 
         def get_log_paths(test_name):
             """Could be a list of logs for all tests or a dict with test name as a key"""
@@ -535,24 +534,43 @@ def get_log_paths(test_name):
                 return log_paths[test_name]
             return log_paths
 
-        for fail_status in ("ERROR", "FAILED"):
-            for failed_test in counters[fail_status]:
-                if failed_test in known_broken_tests.keys():
-                    fail_message = known_broken_tests[failed_test].get("message")
-                    if not fail_message:
-                        mark_as_broken = True
-                    else:
-                        mark_as_broken = False
-                        for log_path in get_log_paths(failed_test):
-                            if log_path.endswith(".log"):
-                                with open(log_path) as log_file:
-                                    if fail_message in log_file.read():
-                                        mark_as_broken = True
-                                        break
-
-                    if mark_as_broken:
-                        counters[fail_status].remove(failed_test)
-                        counters["BROKEN"].append(failed_test)
+        broken_tests_log = os.path.join(self.result_path, "broken_tests_handler.log")
+
+        with open(broken_tests_log, "w") as log_file:
+            for fail_status in ("ERROR", "FAILED"):
+                for failed_test in counters[fail_status]:
+                    if failed_test in known_broken_tests.keys():
+                        fail_message = known_broken_tests[failed_test].get("message")
+                        log_file.write(
+                            f"Checking test {failed_test} (status: {fail_status})\n"
+                        )
+                        if not fail_message:
+                            log_file.write(
+                                "No fail message specified, marking as broken\n"
+                            )
+                            mark_as_broken = True
+                        else:
+                            log_file.write(
+                                f"Looking for fail message: {fail_message}\n"
+                            )
+                            mark_as_broken = False
+                            for log_path in get_log_paths(failed_test):
+                                if log_path.endswith(".log"):
+                                    log_file.write(f"Checking log file: {log_path}\n")
+                                    with open(log_path) as test_log:
+                                        if fail_message in test_log.read():
+                                            log_file.write(
+                                                "Found fail message in logs\n"
+                                            )
+                                            mark_as_broken = True
+                                            break
+
+                        if mark_as_broken:
+                            log_file.write(f"Moving test to BROKEN state\n")
+                            counters[fail_status].remove(failed_test)
+                            counters["BROKEN"].append(failed_test)
+                        else:
+                            log_file.write("Test not marked as broken\n")
 
     def _get_runner_image_cmd(self):
         image_cmd = ""

From 5ccee71d775b813a52c3a0aaf27344cbd2b3d9d7 Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Tue, 6 May 2025 13:35:31 -0400
Subject: [PATCH 13/36] add report link to RunConfig output

---
 .github/workflows/release_branches.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml
index e41d77464a09..9a4248a6eb20 100644
--- a/.github/workflows/release_branches.yml
+++ b/.github/workflows/release_branches.yml
@@ -59,6 +59,14 @@ jobs:
       - name: Re-create GH statuses for skipped jobs if any
         run: |
             python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ runner.temp }}/ci_run_data.json --update-gh-statuses
+      - name: Note report location to summary
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number || 0 }}
+          COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+        run: |
+          REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PR_NUMBER/$COMMIT_SHA/ci_run_report.html
+          echo "Workflow Run Report: [View Report]($REPORT_LINK)" >> $GITHUB_STEP_SUMMARY
+
   BuildDockers:
     needs: [RunConfig]
     if: ${{ !failure() && !cancelled() }}

From 3bca15e7f674098fbeaac6c9e8c738a8580fd8be Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Tue, 6 May 2025 13:46:18 -0400
Subject: [PATCH 14/36] more links

---
 .../actions/create_workflow_report/ci_run_report.html.jinja   | 4 ++++
 .../actions/create_workflow_report/create_workflow_report.py  | 1 +
 2 files changed, 5 insertions(+)

diff --git a/.github/actions/create_workflow_report/ci_run_report.html.jinja b/.github/actions/create_workflow_report/ci_run_report.html.jinja
index 3fdb811ec85f..a92c1aa34e3a 100644
--- a/.github/actions/create_workflow_report/ci_run_report.html.jinja
+++ b/.github/actions/create_workflow_report/ci_run_report.html.jinja
@@ -150,6 +150,10 @@
         <th class="hth no-sort">Commit</th>
         <td><a href="https://github.com/{{ github_repo }}/commit/{{ commit_sha }}">{{ commit_sha }}</a></td>
       </tr>
+      <tr>
+        <th class="hth no-sort">Build Report</th>
+        <td><a href="https://s3.amazonaws.com/{{ s3_bucket }}/{{ pr_number }}/{{ commit_sha }}/builds/report.html">Build Report</a></td>
+      </tr>
       <tr>
         <th class="hth no-sort">Date</th>
         <td> {{ date }}</td>
diff --git a/.github/actions/create_workflow_report/create_workflow_report.py b/.github/actions/create_workflow_report/create_workflow_report.py
index 8d86c2a29733..70882d8afbe1 100755
--- a/.github/actions/create_workflow_report/create_workflow_report.py
+++ b/.github/actions/create_workflow_report/create_workflow_report.py
@@ -544,6 +544,7 @@ def main():
     context = {
         "title": "ClickHouse® CI Workflow Run Report",
         "github_repo": GITHUB_REPO,
+        "s3_bucket": S3_BUCKET,
         "pr_info_html": pr_info_html,
         "pr_number": args.pr_number,
         "workflow_id": args.actions_run_url.split("/")[-1],

From 56deffb9b0734ad3d64f7418062887c35875b71d Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Tue, 6 May 2025 13:50:31 -0400
Subject: [PATCH 15/36] log not broken tests too

---
 tests/ci/integration_tests_runner.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/ci/integration_tests_runner.py b/tests/ci/integration_tests_runner.py
index a39c9327b323..cf187e0f1fef 100755
--- a/tests/ci/integration_tests_runner.py
+++ b/tests/ci/integration_tests_runner.py
@@ -539,7 +539,11 @@ def get_log_paths(test_name):
         with open(broken_tests_log, "w") as log_file:
             for fail_status in ("ERROR", "FAILED"):
                 for failed_test in counters[fail_status]:
-                    if failed_test in known_broken_tests.keys():
+                    if failed_test not in known_broken_tests.keys():
+                        log_file.write(
+                            f"Test {failed_test} is not in known broken tests\n"
+                        )
+                    else:
                         fail_message = known_broken_tests[failed_test].get("message")
                         log_file.write(
                             f"Checking test {failed_test} (status: {fail_status})\n"

From d7a1d6a788b69324b9ffc5fdd65e541d2d5c3aff Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Tue, 6 May 2025 13:59:24 -0400
Subject: [PATCH 16/36] Add image to grype scan summary

---
 .github/workflows/grype_scan.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml
index 1414129fd666..e749448b81ba 100644
--- a/.github/workflows/grype_scan.yml
+++ b/.github/workflows/grype_scan.yml
@@ -76,6 +76,7 @@ jobs:
           if: always()
           id: create_summary
           run: |
+            jq -r '"**Image**: \(.source.target.userInput)"' result.json >> $GITHUB_STEP_SUMMARY
             jq -r '.distro | "**Distro**: \(.name):\(.version)"' result.json >> $GITHUB_STEP_SUMMARY
             if jq -e '.matches | length == 0' result.json > /dev/null; then
               echo "No CVEs" >> $GITHUB_STEP_SUMMARY

From bffdc54d5ecfb67442dfae8e308f4cefd47c237c Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Tue, 6 May 2025 14:50:46 -0400
Subject: [PATCH 17/36] cross out all current fails - create a clean baseline

---
 tests/broken_tests.json | 198 +++++++++++++++++++++++++++++++---------
 1 file changed, 156 insertions(+), 42 deletions(-)

diff --git a/tests/broken_tests.json b/tests/broken_tests.json
index 2086e6f467a5..f2c29ec80499 100644
--- a/tests/broken_tests.json
+++ b/tests/broken_tests.json
@@ -1,128 +1,242 @@
 {
     "02700_s3_part_INT_MAX": {
-        "reason": "Fails on asan"
+        "reason": "NEEDSFIX - Fails on asan"
     },
     "00157_cache_dictionary": {
-        "reason": "fails with asan,debug,msan"
+        "reason": "INVESTIGATE - fails with asan,debug,msan"
     },
     "02815_no_throw_in_simple_queries": {
-        "reason": "Fails on asan,msan,tsan,debug,Aarch64"
+        "reason": "INVESTIGATE - Fails on asan,msan,tsan,debug,Aarch64"
     },
     "03206_no_exceptions_clickhouse_local": {
-        "reason": "Fails on asan,msan,tsan,debug,Aarch64"
+        "reason": "INVESTIGATE - Fails on asan,msan,tsan,debug,Aarch64"
     },
     "03203_hive_style_partitioning": {
-        "reason": "new fail in 24.12"
+        "reason": "INVESTIGATE - new fail in 24.12"
     },
     "test_attach_partition_using_copy/test.py::test_all_replicated": {
-        "reason": "Fails on ASAN, ASAN old analyzer, Release, TSAN"
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
     },
     "test_attach_partition_using_copy/test.py::test_both_mergetree": {
-        "reason": "Fails on ASAN, ASAN old analyzer, Release, TSAN"
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
     },
     "test_backup_restore_on_cluster/test_cancel_backup.py::test_shutdown_cancels_backup": {
-        "reason": "Fails on ASAN, ASAN old analyzer, Release, TSAN"
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
     },
     "test_backup_restore_on_cluster/test_different_versions.py::test_different_versions": {
-        "reason": "Fails on ASAN, ASAN old analyzer, Release, TSAN"
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
     },
     "test_grpc_protocol/test.py::test_ipv6_select_one": {
-        "reason": "Fails on ASAN, ASAN old analyzer, Release, TSAN - FIXED in 24.8"
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN - FIXED in 24.8"
     },
     "test_settings_randomization/test.py::test_settings_randomization": {
-        "reason": "Fails on ASAN, ASAN old analyzer, Release"
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release"
     },
     "test_attach_partition_using_copy/test.py::test_not_work_on_different_disk": {
-        "reason": "Fails on ASAN old analyzer, TSAN"
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
     },
     "test_attach_partition_using_copy/test.py::test_only_destination_replicated": {
-        "reason": "Fails on TSAN"
+        "reason": "INVESTIGATE - Fails on TSAN"
     },
     "02766_prql": {
-        "reason": "KNOWN PRQL is disabled in Antalya branch"
+        "reason": "KNOWN - PRQL is disabled in Antalya branch"
     },
     "02833_local_with_dialect": {
-        "reason": "KNOWN PRQL is disabled in Antalya branch"
+        "reason": "KNOWN - PRQL is disabled in Antalya branch"
     },
     "03003_prql_panic": {
-        "reason": "KNOWN PRQL is disabled in Antalya branch"
+        "reason": "KNOWN - PRQL is disabled in Antalya branch"
     },
     "test_git_import/test.py::test_git_import": {
-        "reason": "NEEDSFIX 403 while getting data from GitHub"
+        "reason": "NEEDSFIX - 403 while getting data from GitHub"
     },
     "test_storage_delta/test.py::test_types[1]": {
-        "reason": "NEEDSFIX rust problem"
+        "reason": "NEEDSFIX - rust problem"
     },
     "test_storage_delta/test.py::test_types[0]": {
-        "reason": "NEEDSFIX rust problem"
+        "reason": "NEEDSFIX - rust problem"
     },
     "test_storage_delta/test.py::test_single_log_file[1]": {
-        "reason": "NEEDSFIX rust problem"
+        "reason": "NEEDSFIX - rust problem"
     },
     "test_storage_delta/test.py::test_single_log_file[0]": {
-        "reason": "NEEDSFIX rust problem"
+        "reason": "NEEDSFIX - rust problem"
     },
     "test_storage_delta/test.py::test_restart_broken[1]": {
-        "reason": "NEEDSFIX rust problem"
+        "reason": "NEEDSFIX - rust problem"
     },
     "test_storage_delta/test.py::test_restart_broken[0]": {
-        "reason": "NEEDSFIX rust problem"
+        "reason": "NEEDSFIX - rust problem"
     },
     "test_storage_delta/test.py::test_restart_broken_table_function[1]": {
-        "reason": "NEEDSFIX rust problem"
+        "reason": "NEEDSFIX - rust problem"
     },
     "test_storage_delta/test.py::test_restart_broken_table_function[0]": {
-        "reason": "NEEDSFIX rust problem"
+        "reason": "NEEDSFIX - rust problem"
     },
     "test_storage_delta/test.py::test_replicated_database_and_unavailable_s3[1]": {
-        "reason": "NEEDSFIX rust problem"
+        "reason": "NEEDSFIX - rust problem"
     },
     "test_storage_delta/test.py::test_replicated_database_and_unavailable_s3[0]": {
-        "reason": "NEEDSFIX rust problem"
+        "reason": "NEEDSFIX - rust problem"
     },
     "test_storage_delta/test.py::test_partition_columns[1]": {
-        "reason": "NEEDSFIX rust problem"
+        "reason": "NEEDSFIX - rust problem"
     },
     "test_storage_delta/test.py::test_partition_columns[0]": {
-        "reason": "NEEDSFIX rust problem"
+        "reason": "NEEDSFIX - rust problem"
     },
     "test_storage_delta/test.py::test_partition_columns_2[1]": {
-        "reason": "NEEDSFIX rust problem"
+        "reason": "NEEDSFIX - rust problem"
     },
     "test_storage_delta/test.py::test_partition_by[1]": {
-        "reason": "NEEDSFIX rust problem"
+        "reason": "NEEDSFIX - rust problem"
     },
     "test_storage_delta/test.py::test_partition_by[0]": {
-        "reason": "NEEDSFIX rust problem"
+        "reason": "NEEDSFIX - rust problem"
     },
     "test_storage_delta/test.py::test_multiple_log_files[1]": {
-        "reason": "NEEDSFIX rust problem"
+        "reason": "NEEDSFIX - rust problem"
     },
     "test_storage_delta/test.py::test_multiple_log_files[0]": {
-        "reason": "NEEDSFIX rust problem"
+        "reason": "NEEDSFIX - rust problem"
     },
     "test_storage_delta/test.py::test_metadata[1]": {
-        "reason": "NEEDSFIX rust problem"
+        "reason": "NEEDSFIX - rust problem"
     },
     "test_storage_delta/test.py::test_metadata[0]": {
-        "reason": "NEEDSFIX rust problem"
+        "reason": "NEEDSFIX - rust problem"
     },
     "test_storage_delta/test.py::test_filesystem_cache[1-s3]": {
-        "reason": "NEEDSFIX rust problem"
+        "reason": "NEEDSFIX - rust problem"
     },
     "test_storage_delta/test.py::test_filesystem_cache[0-s3]": {
-        "reason": "NEEDSFIX rust problem"
+        "reason": "NEEDSFIX - rust problem"
     },
     "test_storage_delta/test.py::test_complex_types[1]": {
-        "reason": "NEEDSFIX rust problem"
+        "reason": "NEEDSFIX - rust problem"
     },
     "test_storage_delta/test.py::test_complex_types[0]": {
-        "reason": "NEEDSFIX rust problem"
+        "reason": "NEEDSFIX - rust problem"
     },
     "test_storage_delta/test.py::test_checkpoint[1]": {
-        "reason": "NEEDSFIX rust problem"
+        "reason": "NEEDSFIX - rust problem"
     },
     "test_storage_delta/test.py::test_checkpoint[0]": {
-        "reason": "NEEDSFIX rust problem"
+        "reason": "NEEDSFIX - rust problem"
+    },
+    "test_cow_policy/test.py::test_cow_policy[cow_policy_multi_disk]": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_cow_policy/test.py::test_cow_policy[cow_policy_multi_volume]": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_s3_assume_role/test.py::test_using_assumed_creds": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_asynchronous_metric_jemalloc_profile_active/test.py::test_asynchronous_metric_jemalloc_profile_active": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_backup_restore_new/test.py::test_incremental_backup_overflow": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_mask_sensitive_info/test.py::test_create_table": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_mask_sensitive_info/test.py::test_table_functions": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_odbc_interaction/test.py::test_postgres_odbc_hashed_dictionary_no_tty_pipe_overflow": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_storage_iceberg/test.py::test_types[s3-1]": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_storage_iceberg/test.py::test_types[azure-2]": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_storage_iceberg/test.py::test_types[local-1]": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_storage_iceberg/test.py::test_types[local-2]": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_storage_iceberg/test.py::test_types[s3-2]": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_storage_iceberg/test.py::test_types[azure-1]": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_library_bridge/test_exiled.py::test_bridge_dies_with_parent": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_merge_tree_check_part_with_cache/test.py::test_check_part_with_cache": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_odbc_interaction/test.py::test_concurrent_queries": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_odbc_interaction/test.py::test_many_connections": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_odbc_interaction/test.py::test_mysql_insert": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_odbc_interaction/test.py::test_mysql_odbc_select_nullable": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_odbc_interaction/test.py::test_mysql_simple_select_works": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_odbc_interaction/test.py::test_no_connection_pooling": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_odbc_interaction/test.py::test_odbc_cyrillic_with_varchar": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_odbc_interaction/test.py::test_odbc_long_column_names": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_odbc_interaction/test.py::test_odbc_long_text": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_odbc_interaction/test.py::test_odbc_postgres_conversions": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_odbc_interaction/test.py::test_odbc_postgres_date_data_type": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_odbc_interaction/test.py::test_postgres_insert": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_async_metrics_in_cgroup/test.py::test_normalized_user_cpu": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_async_metrics_in_cgroup/test.py::test_system_wide_metrics": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_async_metrics_in_cgroup/test.py::test_user_cpu_accounting": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_jemalloc_percpu_arena/test.py::test_jemalloc_percpu_arena": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "02354_vector_search_expansion_search": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_hedged_requests_parallel/test.py::test_combination1": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_hedged_requests_parallel/test.py::test_combination2": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_hedged_requests_parallel/test.py::test_query_with_no_data_to_sample": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_hedged_requests_parallel/test.py::test_send_data": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_hedged_requests_parallel/test.py::test_send_table_status_sleep": {
+        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
     }
 }

From 50aefaf31d4e5e7af46c1540d0bc70ca6fc66763 Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Tue, 6 May 2025 14:51:31 -0400
Subject: [PATCH 18/36] MAX_RETRY=2 takes too long in integration, change to 2

---
 tests/ci/integration_tests_runner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/ci/integration_tests_runner.py b/tests/ci/integration_tests_runner.py
index cf187e0f1fef..2cd4d36b945a 100755
--- a/tests/ci/integration_tests_runner.py
+++ b/tests/ci/integration_tests_runner.py
@@ -27,7 +27,7 @@
 from stopwatch import Stopwatch
 from tee_popen import TeePopen
 
-MAX_RETRY = 3
+MAX_RETRY = 2
 NUM_WORKERS = 10
 SLEEP_BETWEEN_RETRIES = 5
 PARALLEL_GROUP_SIZE = 100

From 75b66a46a4e50c0596a3c84befac7b01c0a3199b Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Tue, 6 May 2025 14:55:39 -0400
Subject: [PATCH 19/36] fix release report

---
 .../actions/create_workflow_report/create_workflow_report.py    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/create_workflow_report/create_workflow_report.py b/.github/actions/create_workflow_report/create_workflow_report.py
index 70882d8afbe1..8cd00f9621ee 100755
--- a/.github/actions/create_workflow_report/create_workflow_report.py
+++ b/.github/actions/create_workflow_report/create_workflow_report.py
@@ -549,7 +549,7 @@ def main():
         "pr_number": args.pr_number,
         "workflow_id": args.actions_run_url.split("/")[-1],
         "commit_sha": args.commit_sha,
-        "base_sha": pr_info.get("base", {}).get("sha"),
+        "base_sha": "" if args.pr_number == 0 else pr_info.get("base", {}).get("sha"),
         "date": f"{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')} UTC",
         "is_preview": args.mark_preview,
         "counts": {

From 8e2dac18472989a18273f42ae399ec4955786069 Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Tue, 6 May 2025 23:26:50 -0400
Subject: [PATCH 20/36] Remove 20 fail limit on test groups and log more in
 broken test handler

---
 tests/ci/integration_tests_runner.py | 29 +++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/tests/ci/integration_tests_runner.py b/tests/ci/integration_tests_runner.py
index 2cd4d36b945a..e65c31670593 100755
--- a/tests/ci/integration_tests_runner.py
+++ b/tests/ci/integration_tests_runner.py
@@ -536,8 +536,14 @@ def get_log_paths(test_name):
 
         broken_tests_log = os.path.join(self.result_path, "broken_tests_handler.log")
 
-        with open(broken_tests_log, "w") as log_file:
+        with open(broken_tests_log, "a") as log_file:
+            log_file.write(f"{len(known_broken_tests)} Known broken tests\n")
+            log_file.write(f"Total tests in BROKEN state: {len(counters['BROKEN'])}\n")
+            log_file.write(f"Total tests in ERROR state: {len(counters['ERROR'])}\n")
+            log_file.write(f"Total tests in FAILED state: {len(counters['FAILED'])}\n")
+
             for fail_status in ("ERROR", "FAILED"):
+
                 for failed_test in counters[fail_status]:
                     if failed_test not in known_broken_tests.keys():
                         log_file.write(
@@ -576,6 +582,10 @@ def get_log_paths(test_name):
                         else:
                             log_file.write("Test not marked as broken\n")
 
+            log_file.write(f"Total tests in BROKEN state: {len(counters['BROKEN'])}\n")
+            log_file.write(f"Total tests in ERROR state: {len(counters['ERROR'])}\n")
+            log_file.write(f"Total tests in FAILED state: {len(counters['FAILED'])}\n")
+
     def _get_runner_image_cmd(self):
         image_cmd = ""
         if self._can_run_with(
@@ -1092,9 +1102,10 @@ def run_normal_check(self, build_path):
             for test_name, test_time in group_test_times.items():
                 tests_times[test_name] = test_time
 
-            if len(counters["FAILED"]) + len(counters["ERROR"]) >= 20:
-                logging.info("Collected more than 20 failed/error tests, stopping")
-                break
+            # NOTE(strtgbb): Disabling while investigating broken tests not being marked as such
+            # if len(counters["FAILED"]) + len(counters["ERROR"]) >= 20:
+            #     logging.info("Collected more than 20 failed/error tests, stopping")
+            #     break
 
         # Handle broken tests on the main counters that contain all test results
         known_broken_tests = self._get_broken_tests_list(self.repo_path)
@@ -1105,11 +1116,11 @@ def run_normal_check(self, build_path):
                 "Overall status failure, because we have tests in FAILED or ERROR state"
             )
             result_state = "failure"
-        elif len(counters["BROKEN"]) >= 20:
-            logging.info(
-                "Overall status failure, more than 20 broken tests, likely did not run all tests"
-            )
-            result_state = "failure"
+        # elif len(counters["BROKEN"]) >= 20:
+        #     logging.info(
+        #         "Overall status failure, more than 20 broken tests, likely did not run all tests"
+        #     )
+        #     result_state = "failure"
         else:
             logging.info("Overall success!")
             result_state = "success"

From e5d59202d207928cbaad2816fcf9c46b8eae7711 Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Wed, 7 May 2025 10:02:59 -0400
Subject: [PATCH 21/36] increase integration batches and log more in broken
 test handler

---
 tests/ci/ci_config.py                | 10 +++++-----
 tests/ci/integration_tests_runner.py | 18 ++++++++----------
 2 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
index acde55fa485b..21de77bda594 100644
--- a/tests/ci/ci_config.py
+++ b/tests/ci/ci_config.py
@@ -374,27 +374,27 @@ class CI:
         ),
         JobNames.INTEGRATION_TEST_ASAN: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_ASAN],
-            num_batches=4,
+            num_batches=46,
             timeout=9000,  # the job timed out with default value (7200)
         ),
         JobNames.INTEGRATION_TEST_ASAN_OLD_ANALYZER: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_ASAN],
-            num_batches=6,
+            num_batches=8,
             timeout=9000,  # the job timed out with default value (7200)
         ),
         JobNames.INTEGRATION_TEST_TSAN: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_TSAN],
-            num_batches=6,
+            num_batches=8,
             timeout=9000,  # the job timed out with default value (7200)
         ),
         JobNames.INTEGRATION_TEST_AARCH64: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_AARCH64],
-            num_batches=6,
+            num_batches=8,
             runner_type=Runners.FUNC_TESTER_AARCH64,
         ),
         JobNames.INTEGRATION_TEST: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_RELEASE],
-            num_batches=4,
+            num_batches=6,
             # release_only=True,
             timeout=9000,  # the job timed out with default value (7200)
         ),
diff --git a/tests/ci/integration_tests_runner.py b/tests/ci/integration_tests_runner.py
index e65c31670593..01d894b57f56 100755
--- a/tests/ci/integration_tests_runner.py
+++ b/tests/ci/integration_tests_runner.py
@@ -538,22 +538,21 @@ def get_log_paths(test_name):
 
         with open(broken_tests_log, "a") as log_file:
             log_file.write(f"{len(known_broken_tests)} Known broken tests\n")
-            log_file.write(f"Total tests in BROKEN state: {len(counters['BROKEN'])}\n")
-            log_file.write(f"Total tests in ERROR state: {len(counters['ERROR'])}\n")
-            log_file.write(f"Total tests in FAILED state: {len(counters['FAILED'])}\n")
+            for status, tests in counters.items():
+                log_file.write(f"Total tests in {status} state: {len(tests)}\n")
 
             for fail_status in ("ERROR", "FAILED"):
-
                 for failed_test in counters[fail_status]:
+                    log_file.write(
+                        f"Checking test {failed_test} (status: {fail_status})\n"
+                    )
                     if failed_test not in known_broken_tests.keys():
                         log_file.write(
                             f"Test {failed_test} is not in known broken tests\n"
                         )
                     else:
                         fail_message = known_broken_tests[failed_test].get("message")
-                        log_file.write(
-                            f"Checking test {failed_test} (status: {fail_status})\n"
-                        )
+
                         if not fail_message:
                             log_file.write(
                                 "No fail message specified, marking as broken\n"
@@ -582,9 +581,8 @@ def get_log_paths(test_name):
                         else:
                             log_file.write("Test not marked as broken\n")
 
-            log_file.write(f"Total tests in BROKEN state: {len(counters['BROKEN'])}\n")
-            log_file.write(f"Total tests in ERROR state: {len(counters['ERROR'])}\n")
-            log_file.write(f"Total tests in FAILED state: {len(counters['FAILED'])}\n")
+            for status, tests in counters.items():
+                log_file.write(f"Total tests in {status} state: {len(tests)}\n")
 
     def _get_runner_image_cmd(self):
         image_cmd = ""

From 539955c174be2494738eadf51b2e7aeb57a0ae1c Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Wed, 7 May 2025 10:57:46 -0400
Subject: [PATCH 22/36] use twice as many batches for integration

---
 tests/ci/ci_config.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
index 21de77bda594..50fa8dfa45e3 100644
--- a/tests/ci/ci_config.py
+++ b/tests/ci/ci_config.py
@@ -374,27 +374,27 @@ class CI:
         ),
         JobNames.INTEGRATION_TEST_ASAN: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_ASAN],
-            num_batches=46,
+            num_batches=8,
             timeout=9000,  # the job timed out with default value (7200)
         ),
         JobNames.INTEGRATION_TEST_ASAN_OLD_ANALYZER: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_ASAN],
-            num_batches=8,
+            num_batches=12,
             timeout=9000,  # the job timed out with default value (7200)
         ),
         JobNames.INTEGRATION_TEST_TSAN: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_TSAN],
-            num_batches=8,
+            num_batches=12,
             timeout=9000,  # the job timed out with default value (7200)
         ),
         JobNames.INTEGRATION_TEST_AARCH64: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_AARCH64],
-            num_batches=8,
+            num_batches=12,
             runner_type=Runners.FUNC_TESTER_AARCH64,
         ),
         JobNames.INTEGRATION_TEST: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_RELEASE],
-            num_batches=6,
+            num_batches=8,
             # release_only=True,
             timeout=9000,  # the job timed out with default value (7200)
         ),

From 0542d3b4cebbd4d7c7348903c903845548a8af67 Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Wed, 7 May 2025 12:20:35 -0400
Subject: [PATCH 23/36] fix root cause of broken tests issue, add type hints

---
 tests/ci/integration_tests_runner.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/tests/ci/integration_tests_runner.py b/tests/ci/integration_tests_runner.py
index 01d894b57f56..49bab549be5f 100755
--- a/tests/ci/integration_tests_runner.py
+++ b/tests/ci/integration_tests_runner.py
@@ -526,7 +526,12 @@ def _update_counters(main_counters, current_counters):
             for test in current_counters[state]:
                 main_counters[state].append(test)
 
-    def _handle_broken_tests(self, counters, known_broken_tests, log_paths):
+    def _handle_broken_tests(
+        self,
+        counters: Dict[str, List[str]],
+        known_broken_tests: Dict[str, Dict[str, str]],
+        log_paths: Union[Dict[str, List[str]], List[str]],
+    ) -> None:
 
         def get_log_paths(test_name):
             """Could be a list of logs for all tests or a dict with test name as a key"""
@@ -542,7 +547,7 @@ def get_log_paths(test_name):
                 log_file.write(f"Total tests in {status} state: {len(tests)}\n")
 
             for fail_status in ("ERROR", "FAILED"):
-                for failed_test in counters[fail_status]:
+                for failed_test in counters[fail_status].copy():
                     log_file.write(
                         f"Checking test {failed_test} (status: {fail_status})\n"
                     )

From 87a3b65bf3470d210e12d8c8561de2abc19840ad Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Wed, 7 May 2025 12:39:50 -0400
Subject: [PATCH 24/36] move _handle_broken_tests a bit earlier

---
 tests/ci/integration_tests_runner.py | 33 ++++++++++++++--------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/tests/ci/integration_tests_runner.py b/tests/ci/integration_tests_runner.py
index 49bab549be5f..a5a57fa59172 100755
--- a/tests/ci/integration_tests_runner.py
+++ b/tests/ci/integration_tests_runner.py
@@ -874,6 +874,7 @@ def run_flaky_check(self, build_path, should_fail=False):
         }  # type: Dict
         tests_times = defaultdict(float)  # type: Dict
         tests_log_paths = defaultdict(list)
+        known_broken_tests = self._get_broken_tests_list(self.repo_path)
         id_counter = 0
         for test_to_run in tests_to_run:
             tries_num = 1 if should_fail else FLAKY_TRIES_COUNT
@@ -890,6 +891,12 @@ def run_flaky_check(self, build_path, should_fail=False):
                     1,
                     FLAKY_REPEAT_COUNT,
                 )
+
+                # Handle broken tests on the group counters that contain test results for a single group
+                self._handle_broken_tests(
+                    group_counters, known_broken_tests, tests_log_paths
+                )
+
                 id_counter = id_counter + 1
                 for counter, value in group_counters.items():
                     logging.info(
@@ -913,10 +920,6 @@ def run_flaky_check(self, build_path, should_fail=False):
                     )
                     break
 
-        # Handle broken tests on the main counters that contain all test results
-        known_broken_tests = self._get_broken_tests_list(self.repo_path)
-        self._handle_broken_tests(counters, known_broken_tests, tests_log_paths)
-
         if counters["FAILED"]:
             logging.info("Found failed tests: %s", " ".join(counters["FAILED"]))
             description_prefix = "Failed tests found: "
@@ -1073,6 +1076,7 @@ def run_normal_check(self, build_path):
         tests_times = defaultdict(float)
         tests_log_paths = defaultdict(list)
         items_to_run = list(grouped_tests.items())
+        known_broken_tests = self._get_broken_tests_list(self.repo_path)
         logging.info("Total test groups %s", len(items_to_run))
         if self.shuffle_test_groups():
             logging.info("Shuffling test groups")
@@ -1086,6 +1090,11 @@ def run_normal_check(self, build_path):
                 group, tests, MAX_RETRY, NUM_WORKERS, 0
             )
 
+            # Handle broken tests on the group counters that contain test results for a single group
+            self._handle_broken_tests(
+                group_counters, known_broken_tests, tests_log_paths
+            )
+
             total_tests = 0
             for counter, value in group_counters.items():
                 logging.info(
@@ -1105,25 +1114,15 @@ def run_normal_check(self, build_path):
             for test_name, test_time in group_test_times.items():
                 tests_times[test_name] = test_time
 
-            # NOTE(strtgbb): Disabling while investigating broken tests not being marked as such
-            # if len(counters["FAILED"]) + len(counters["ERROR"]) >= 20:
-            #     logging.info("Collected more than 20 failed/error tests, stopping")
-            #     break
-
-        # Handle broken tests on the main counters that contain all test results
-        known_broken_tests = self._get_broken_tests_list(self.repo_path)
-        self._handle_broken_tests(counters, known_broken_tests, tests_log_paths)
+            if len(counters["FAILED"]) + len(counters["ERROR"]) >= 20:
+                logging.info("Collected more than 20 failed/error tests, stopping")
+                break
 
         if counters["FAILED"] or counters["ERROR"]:
             logging.info(
                 "Overall status failure, because we have tests in FAILED or ERROR state"
             )
             result_state = "failure"
-        # elif len(counters["BROKEN"]) >= 20:
-        #     logging.info(
-        #         "Overall status failure, more than 20 broken tests, likely did not run all tests"
-        #     )
-        #     result_state = "failure"
         else:
             logging.info("Overall success!")
             result_state = "success"

From f373cb7ea440950d0a4c64afcfe9aa6adb63b660 Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Wed, 7 May 2025 14:01:22 -0400
Subject: [PATCH 25/36] fix

---
 tests/ci/integration_tests_runner.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/tests/ci/integration_tests_runner.py b/tests/ci/integration_tests_runner.py
index a5a57fa59172..8f96c72a70b8 100755
--- a/tests/ci/integration_tests_runner.py
+++ b/tests/ci/integration_tests_runner.py
@@ -16,7 +16,7 @@
 import time
 from collections import OrderedDict, defaultdict
 from itertools import chain
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
 
 import yaml  # type: ignore[import-untyped]
 
@@ -893,9 +893,7 @@ def run_flaky_check(self, build_path, should_fail=False):
                 )
 
                 # Handle broken tests on the group counters that contain test results for a single group
-                self._handle_broken_tests(
-                    group_counters, known_broken_tests, tests_log_paths
-                )
+                self._handle_broken_tests(group_counters, known_broken_tests, log_paths)
 
                 id_counter = id_counter + 1
                 for counter, value in group_counters.items():
@@ -1091,9 +1089,7 @@ def run_normal_check(self, build_path):
             )
 
             # Handle broken tests on the group counters that contain test results for a single group
-            self._handle_broken_tests(
-                group_counters, known_broken_tests, tests_log_paths
-            )
+            self._handle_broken_tests(group_counters, known_broken_tests, log_paths)
 
             total_tests = 0
             for counter, value in group_counters.items():

From c1d8e5bbaf30d15d920fd48fb4ce22360204ec42 Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Wed, 7 May 2025 18:52:49 -0400
Subject: [PATCH 26/36] tuning batch sizes

---
 .github/workflows/reusable_test.yml | 1 +
 tests/ci/ci_config.py               | 8 ++++----
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/reusable_test.yml b/.github/workflows/reusable_test.yml
index 4a8b66ca9362..df0045c11f1d 100644
--- a/.github/workflows/reusable_test.yml
+++ b/.github/workflows/reusable_test.yml
@@ -109,6 +109,7 @@ jobs:
       GITHUB_JOB_OVERRIDDEN: ${{inputs.test_name}}${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].num_batches > 1 && format('-{0}',matrix.batch) || '' }}
     strategy:
       fail-fast: false  # we always wait for the entire matrix
+      max-parallel: 6  # limit the number of parallel jobs to 6
       matrix:
         batch: ${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].batches }}
     steps:
diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
index 50fa8dfa45e3..ff6d147baf83 100644
--- a/tests/ci/ci_config.py
+++ b/tests/ci/ci_config.py
@@ -374,18 +374,18 @@ class CI:
         ),
         JobNames.INTEGRATION_TEST_ASAN: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_ASAN],
-            num_batches=8,
+            num_batches=12,
             timeout=9000,  # the job timed out with default value (7200)
         ),
         JobNames.INTEGRATION_TEST_ASAN_OLD_ANALYZER: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_ASAN],
             num_batches=12,
-            timeout=9000,  # the job timed out with default value (7200)
+            timeout=12000,  # the job timed out with default value (7200)
         ),
         JobNames.INTEGRATION_TEST_TSAN: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_TSAN],
             num_batches=12,
-            timeout=9000,  # the job timed out with default value (7200)
+            timeout=12000,  # the job timed out with default value (7200)
         ),
         JobNames.INTEGRATION_TEST_AARCH64: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_AARCH64],
@@ -394,7 +394,7 @@ class CI:
         ),
         JobNames.INTEGRATION_TEST: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_RELEASE],
-            num_batches=8,
+            num_batches=12,
             # release_only=True,
             timeout=9000,  # the job timed out with default value (7200)
         ),

From d07b9a568f7f14cce12a76145508abbfab50824a Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Thu, 8 May 2025 08:41:43 -0400
Subject: [PATCH 27/36] fix skipped test detection

---
 tests/ci/integration_tests_runner.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/tests/ci/integration_tests_runner.py b/tests/ci/integration_tests_runner.py
index 8f96c72a70b8..54e260dc1de6 100755
--- a/tests/ci/integration_tests_runner.py
+++ b/tests/ci/integration_tests_runner.py
@@ -116,16 +116,19 @@ def get_counters(fname: str) -> Dict[str, List[str]]:
             #
             #     test_mysql_protocol/test.py::test_golang_client
             #     [gw0] [  7%] ERROR test_mysql_protocol/test.py::test_golang_client
+            #     [gw1] SKIPPED test_odbc_interaction/test.py:56: Sanitizers cannot work with third-party shared libraries
             #
             # And only the line with test status should be matched
             if not (".py::" in line and " " in line):
                 continue
 
             line = line.strip()
-            # [gw0] [  7%] ERROR test_mysql_protocol/test.py::test_golang_client
-            # ^^^^^^^^^^^^^
+            # Remove worker info and progress percentage if present
             if line.strip().startswith("["):
-                line = re.sub(r"^\[[^\[\]]*\] \[[^\[\]]*\] ", "", line)
+                # Handle both formats:
+                # [gw0] [  7%] ERROR test_mysql_protocol/test.py::test_golang_client
+                # [gw1] SKIPPED test_odbc_interaction/test.py:56: Sanitizers cannot work with third-party shared libraries
+                line = re.sub(r"^\[[^\[\]]*\] (?:\[[^\[\]]*\] )?", "", line)
 
             line_arr = line.split(" ")
             if len(line_arr) < 2:
@@ -137,6 +140,7 @@ def get_counters(fname: str) -> Dict[str, List[str]]:
             #     ERROR test_mysql_protocol/test.py::test_golang_client
             #     PASSED test_replicated_users/test.py::test_rename_replicated[QUOTA]
             #     PASSED test_drop_is_lock_free/test.py::test_query_is_lock_free[detach part]
+            #     SKIPPED test_odbc_interaction/test.py:56: Sanitizers cannot work with third-party shared libraries
             #
             state = line_arr.pop(0)
             test_name = " ".join(line_arr)

From 206f20a853f147334af91fe2538724628f65f0eb Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Thu, 8 May 2025 08:43:05 -0400
Subject: [PATCH 28/36] add branch name to release report

---
 .../create_workflow_report/create_workflow_report.py        | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/actions/create_workflow_report/create_workflow_report.py b/.github/actions/create_workflow_report/create_workflow_report.py
index 8cd00f9621ee..e1945fbaa5bf 100755
--- a/.github/actions/create_workflow_report/create_workflow_report.py
+++ b/.github/actions/create_workflow_report/create_workflow_report.py
@@ -5,6 +5,7 @@
 from itertools import combinations
 import json
 from datetime import datetime
+from functools import lru_cache
 
 import pandas as pd
 from jinja2 import Environment, FileSystemLoader
@@ -124,6 +125,7 @@ def get_pr_info_from_number(pr_number: str) -> dict:
     return response.json()
 
 
+@lru_cache
 def get_run_details(run_url: str) -> dict:
     """
     Fetch run details for a given run URL.
@@ -515,7 +517,9 @@ def main():
             )
 
     if args.pr_number == 0:
-        pr_info_html = "Release"
+        run_details = get_run_details(args.actions_run_url)
+        branch_name = run_details.get("head_branch", "unknown branch")
+        pr_info_html = f"Release ({branch_name})"
     else:
         try:
             pr_info = get_pr_info_from_number(args.pr_number)

From f6dce2e39c825da1ce72f386ad39a28daec5dde3 Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Thu, 8 May 2025 12:57:20 -0400
Subject: [PATCH 29/36] add timeouts to slow tests

---
 tests/broken_tests.json                       | 23 +++++++++++--------
 .../test_attach_partition_using_copy/test.py  |  4 +++-
 tests/integration/test_cow_policy/test.py     |  4 +++-
 .../test_s3_cache_locality/test.py            | 20 +++++++++-------
 4 files changed, 31 insertions(+), 20 deletions(-)

diff --git a/tests/broken_tests.json b/tests/broken_tests.json
index f2c29ec80499..c6b9f820d99c 100644
--- a/tests/broken_tests.json
+++ b/tests/broken_tests.json
@@ -15,10 +15,16 @@
         "reason": "INVESTIGATE - new fail in 24.12"
     },
     "test_attach_partition_using_copy/test.py::test_all_replicated": {
-        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+        "reason": "NEEDSFIX - Web disk is not working"
     },
     "test_attach_partition_using_copy/test.py::test_both_mergetree": {
-        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+        "reason": "NEEDSFIX - Web disk is not working"
+    },
+    "test_attach_partition_using_copy/test.py::test_not_work_on_different_disk": {
+        "reason": "NEEDSFIX - Web disk is not working"
+    },
+    "test_attach_partition_using_copy/test.py::test_only_destination_replicated": {
+        "reason": "NEEDSFIX - Web disk is not working"
     },
     "test_backup_restore_on_cluster/test_cancel_backup.py::test_shutdown_cancels_backup": {
         "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
@@ -32,12 +38,6 @@
     "test_settings_randomization/test.py::test_settings_randomization": {
         "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release"
     },
-    "test_attach_partition_using_copy/test.py::test_not_work_on_different_disk": {
-        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
-    },
-    "test_attach_partition_using_copy/test.py::test_only_destination_replicated": {
-        "reason": "INVESTIGATE - Fails on TSAN"
-    },
     "02766_prql": {
         "reason": "KNOWN - PRQL is disabled in Antalya branch"
     },
@@ -126,10 +126,10 @@
         "reason": "NEEDSFIX - rust problem"
     },
     "test_cow_policy/test.py::test_cow_policy[cow_policy_multi_disk]": {
-        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+        "reason": "NEEDSFIX - Timeout"
     },
     "test_cow_policy/test.py::test_cow_policy[cow_policy_multi_volume]": {
-        "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+        "reason": "NEEDSFIX - Timeout"
     },
     "test_s3_assume_role/test.py::test_using_assumed_creds": {
         "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
@@ -238,5 +238,8 @@
     },
     "test_hedged_requests_parallel/test.py::test_send_table_status_sleep": {
         "reason": "INVESTIGATE - Fails on ASAN, ASAN old analyzer, Release, TSAN"
+    },
+    "test_s3_cache_locality/test.py::test_cache_locality": {
+        "reason": "INVESTIGATE - Timeout on TSAN"
     }
 }
diff --git a/tests/integration/test_attach_partition_using_copy/test.py b/tests/integration/test_attach_partition_using_copy/test.py
index 99fb2ff01530..206d339f333c 100644
--- a/tests/integration/test_attach_partition_using_copy/test.py
+++ b/tests/integration/test_attach_partition_using_copy/test.py
@@ -61,7 +61,9 @@ def create_source_table(node, table_name, replicated):
         SETTINGS disk = disk(type = web, endpoint = 'https://raw.githubusercontent.com/ClickHouse/web-tables-demo/main/web/')
         """.format(
             table_name=table_name, engine=engine
-        )
+        ),
+        timeout=60,
+        retry_count=3,
     )
 
 
diff --git a/tests/integration/test_cow_policy/test.py b/tests/integration/test_cow_policy/test.py
index 0d14eacbcb2b..db34c157f0ab 100644
--- a/tests/integration/test_cow_policy/test.py
+++ b/tests/integration/test_cow_policy/test.py
@@ -43,7 +43,9 @@ def test_cow_policy(start_cluster, storage_policy):
             ENGINE = MergeTree
             ORDER BY (postcode1, postcode2, addr1, addr2)
             SETTINGS storage_policy = '{storage_policy}'
-            """
+            """,
+            timeout=60,
+            retry_count=3,
         )
         prev_count = int(node.query("SELECT count() FROM uk_price_paid"))
         assert prev_count > 0
diff --git a/tests/integration/test_s3_cache_locality/test.py b/tests/integration/test_s3_cache_locality/test.py
index 088c24576d10..a2020d7e0568 100644
--- a/tests/integration/test_s3_cache_locality/test.py
+++ b/tests/integration/test_s3_cache_locality/test.py
@@ -75,7 +75,7 @@ def started_cluster():
 
 def check_s3_gets(cluster, node, expected_result, cluster_first, cluster_second, enable_filesystem_cache):
     for host in list(cluster.instances.values()):
-        host.query("SYSTEM DROP FILESYSTEM CACHE 'raw_s3_cache'")
+        host.query("SYSTEM DROP FILESYSTEM CACHE 'raw_s3_cache'", timeout=30)
 
     query_id_first = str(uuid.uuid4())
     result_first = node.query(
@@ -87,7 +87,8 @@ def check_s3_gets(cluster, node, expected_result, cluster_first, cluster_second,
           enable_filesystem_cache={enable_filesystem_cache},
           filesystem_cache_name='raw_s3_cache'
         """,
-        query_id=query_id_first
+        query_id=query_id_first,
+        timeout=30,
     )
     assert result_first == expected_result
     query_id_second = str(uuid.uuid4())
@@ -100,13 +101,14 @@ def check_s3_gets(cluster, node, expected_result, cluster_first, cluster_second,
           enable_filesystem_cache={enable_filesystem_cache},
           filesystem_cache_name='raw_s3_cache'
         """,
-        query_id=query_id_second
+        query_id=query_id_second,
+        timeout=30,
     )
     assert result_second == expected_result
 
-    node.query("SYSTEM FLUSH LOGS")
-    node.query(f"SYSTEM FLUSH LOGS ON CLUSTER {cluster_first}")
-    node.query(f"SYSTEM FLUSH LOGS ON CLUSTER {cluster_second}")
+    node.query("SYSTEM FLUSH LOGS", timeout=30)
+    node.query(f"SYSTEM FLUSH LOGS ON CLUSTER {cluster_first}", timeout=30)
+    node.query(f"SYSTEM FLUSH LOGS ON CLUSTER {cluster_second}", timeout=30)
 
     s3_get_first = node.query(
         f"""
@@ -114,7 +116,8 @@ def check_s3_gets(cluster, node, expected_result, cluster_first, cluster_second,
           FROM clusterAllReplicas('{cluster_first}', system.query_log)
           WHERE type='QueryFinish'
             AND initial_query_id='{query_id_first}'
-        """
+        """,
+        timeout=30,
     )
     s3_get_second = node.query(
         f"""
@@ -122,7 +125,8 @@ def check_s3_gets(cluster, node, expected_result, cluster_first, cluster_second,
           FROM clusterAllReplicas('{cluster_second}', system.query_log)
           WHERE type='QueryFinish'
             AND initial_query_id='{query_id_second}'
-        """
+        """,
+        timeout=30,
     )
 
     return int(s3_get_first), int(s3_get_second)

From 6133eabf587644a654556bb8c54f1e8bb6503fe0 Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Thu, 8 May 2025 13:08:09 -0400
Subject: [PATCH 30/36] add broken test
 test_reload_clusters_config/test.py::test_update_one_cluster

---
 tests/broken_tests.json | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/broken_tests.json b/tests/broken_tests.json
index c6b9f820d99c..c33c69a7b259 100644
--- a/tests/broken_tests.json
+++ b/tests/broken_tests.json
@@ -241,5 +241,8 @@
     },
     "test_s3_cache_locality/test.py::test_cache_locality": {
         "reason": "INVESTIGATE - Timeout on TSAN"
+    },
+    "test_reload_clusters_config/test.py::test_update_one_cluster": {
+        "reason": "INVESTIGATE - Fails on ASAN old analyzer"
     }
 }

From 45c57c3ea584509354e0cb70c6b2348c89016a71 Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Thu, 8 May 2025 14:41:38 -0400
Subject: [PATCH 31/36] Try to avoid stress test errors that may be related to
 running out of resources, re-enable azurite

---
 tests/docker_scripts/stress_runner.sh | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tests/docker_scripts/stress_runner.sh b/tests/docker_scripts/stress_runner.sh
index 09d03de475d7..fbf7ca6ea74a 100755
--- a/tests/docker_scripts/stress_runner.sh
+++ b/tests/docker_scripts/stress_runner.sh
@@ -53,10 +53,17 @@ export ZOOKEEPER_FAULT_INJECTION=1
 # available for dump via clickhouse-local
 configure
 
+# NOTE(strtgbb): Our azure_storage_conf.xml is still expecting azurite
+azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log &
+
 /repo/tests/docker_scripts/setup_minio.sh stateless # to have a proper environment
 
 config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml
 
+# NOTE(strtgbb): Trying to avoid errors that may be related to running out of resources
+export CLICKHOUSE_MAX_THREADS=8
+export CLICKHOUSE_MAX_CONCURRENT_QUERIES=4
+
 start_server
 
 setup_logs_replication

From 86af07faf51cbdb1e66c797dfb4d2f6b3094957b Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Thu, 8 May 2025 18:08:32 -0400
Subject: [PATCH 32/36] add broken tests 00988_expansion_aliases_limit
 test_overcommit_tracker/test.py::test_user_overcommit

---
 tests/broken_tests.json | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/broken_tests.json b/tests/broken_tests.json
index c33c69a7b259..c407e2c5bc79 100644
--- a/tests/broken_tests.json
+++ b/tests/broken_tests.json
@@ -14,6 +14,12 @@
     "03203_hive_style_partitioning": {
         "reason": "INVESTIGATE - new fail in 24.12"
     },
+    "00988_expansion_aliases_limit": {
+        "reason": "INVESTIGATE - fails in tsan"
+    },
+    "test_overcommit_tracker/test.py::test_user_overcommit": {
+        "reason": "INVESTIGATE - fails in tsan"
+    },
     "test_attach_partition_using_copy/test.py::test_all_replicated": {
         "reason": "NEEDSFIX - Web disk is not working"
     },

From 7de983e977f02b6c704b6791e509a803b60b35a7 Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Thu, 8 May 2025 22:42:56 -0400
Subject: [PATCH 33/36] reduce batch sizes and timeouts for integration tests

---
 .github/workflows/reusable_test.yml |  1 -
 tests/ci/ci_config.py               | 14 +++++---------
 2 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/reusable_test.yml b/.github/workflows/reusable_test.yml
index df0045c11f1d..4a8b66ca9362 100644
--- a/.github/workflows/reusable_test.yml
+++ b/.github/workflows/reusable_test.yml
@@ -109,7 +109,6 @@ jobs:
       GITHUB_JOB_OVERRIDDEN: ${{inputs.test_name}}${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].num_batches > 1 && format('-{0}',matrix.batch) || '' }}
     strategy:
       fail-fast: false  # we always wait for the entire matrix
-      max-parallel: 6  # limit the number of parallel jobs to 6
       matrix:
         batch: ${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].batches }}
     steps:
diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
index ff6d147baf83..f76f666b0641 100644
--- a/tests/ci/ci_config.py
+++ b/tests/ci/ci_config.py
@@ -374,29 +374,25 @@ class CI:
         ),
         JobNames.INTEGRATION_TEST_ASAN: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_ASAN],
-            num_batches=12,
-            timeout=9000,  # the job timed out with default value (7200)
+            num_batches=8,
         ),
         JobNames.INTEGRATION_TEST_ASAN_OLD_ANALYZER: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_ASAN],
-            num_batches=12,
-            timeout=12000,  # the job timed out with default value (7200)
+            num_batches=8,
         ),
         JobNames.INTEGRATION_TEST_TSAN: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_TSAN],
-            num_batches=12,
-            timeout=12000,  # the job timed out with default value (7200)
+            num_batches=8,
         ),
         JobNames.INTEGRATION_TEST_AARCH64: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_AARCH64],
-            num_batches=12,
+            num_batches=8,
             runner_type=Runners.FUNC_TESTER_AARCH64,
         ),
         JobNames.INTEGRATION_TEST: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_RELEASE],
-            num_batches=12,
+            num_batches=8,
             # release_only=True,
-            timeout=9000,  # the job timed out with default value (7200)
         ),
         JobNames.INTEGRATION_TEST_FLAKY: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_ASAN],

From 480ca61ccca07a3980b801a96174ba0d92aa3529 Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Fri, 9 May 2025 08:26:49 -0400
Subject: [PATCH 34/36] add broken tests 01037_polygon_dicts_correctness_fast
 03094_grouparraysorted_memory

---
 tests/broken_tests.json | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/broken_tests.json b/tests/broken_tests.json
index c407e2c5bc79..e57268f72bd2 100644
--- a/tests/broken_tests.json
+++ b/tests/broken_tests.json
@@ -20,6 +20,12 @@
     "test_overcommit_tracker/test.py::test_user_overcommit": {
         "reason": "INVESTIGATE - fails in tsan"
     },
+    "01037_polygon_dicts_correctness_fast": {
+        "reason": "INVESTIGATE - fails in tsan"
+    },
+    "03094_grouparraysorted_memory": {
+        "reason": "INVESTIGATE - fails in tsan"
+    },
     "test_attach_partition_using_copy/test.py::test_all_replicated": {
         "reason": "NEEDSFIX - Web disk is not working"
     },

From 43faf6db68233e0ba70217f317f504b471de36ab Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Fri, 9 May 2025 13:55:34 -0400
Subject: [PATCH 35/36] Properly support checks reruns in report

---
 .../create_workflow_report.py                 | 52 +++++++++++++------
 1 file changed, 36 insertions(+), 16 deletions(-)

diff --git a/.github/actions/create_workflow_report/create_workflow_report.py b/.github/actions/create_workflow_report/create_workflow_report.py
index e1945fbaa5bf..21c365a23dd1 100755
--- a/.github/actions/create_workflow_report/create_workflow_report.py
+++ b/.github/actions/create_workflow_report/create_workflow_report.py
@@ -153,13 +153,23 @@ def get_checks_fails(client: Client, job_url: str):
     Get tests that did not succeed for the given job URL.
     Exclude checks that have status 'error' as they are counted in get_checks_errors.
     """
-    columns = "check_status as job_status, check_name as job_name, test_status, test_name, report_url as results_link"
-    query = f"""SELECT {columns} FROM `gh-data`.checks
-                WHERE task_url LIKE '{job_url}%'
-                AND test_status IN ('FAIL', 'ERROR')
-                AND check_status!='error'
-                ORDER BY check_name, test_name
-                """
+    query = f"""SELECT job_status, job_name, status as test_status, test_name, results_link
+            FROM (
+                SELECT
+                    argMax(check_status, check_start_time) as job_status,
+                    check_name as job_name,
+                    argMax(test_status, check_start_time) as status,
+                    test_name,
+                    report_url as results_link,
+                    task_url
+                FROM `gh-data`.checks
+                GROUP BY check_name, test_name, report_url, task_url
+            )
+            WHERE task_url LIKE '{job_url}%'
+            AND test_status IN ('FAIL', 'ERROR')
+            AND job_status!='error'
+            ORDER BY job_name, test_name
+            """
     return client.query_dataframe(query)
 
 
@@ -195,12 +205,22 @@ def get_checks_errors(client: Client, job_url: str):
     """
     Get checks that have status 'error' for the given job URL.
     """
-    columns = "check_status as job_status, check_name as job_name, test_status, test_name, report_url as results_link"
-    query = f"""SELECT {columns} FROM `gh-data`.checks
-                WHERE task_url LIKE '{job_url}%'
-                AND check_status=='error'
-                ORDER BY check_name, test_name
-                """
+    query = f"""SELECT job_status, job_name, status as test_status, test_name, results_link
+            FROM (
+                SELECT
+                    argMax(check_status, check_start_time) as job_status,
+                    check_name as job_name,
+                    argMax(test_status, check_start_time) as status,
+                    test_name,
+                    report_url as results_link,
+                    task_url
+                FROM `gh-data`.checks
+                GROUP BY check_name, test_name, report_url, task_url
+            )
+            WHERE task_url LIKE '{job_url}%'
+            AND job_status=='error'
+            ORDER BY job_name, test_name
+            """
     return client.query_dataframe(query)
 
 
@@ -233,14 +253,14 @@ def get_regression_fails(client: Client, job_url: str):
                     architecture as arch,
                     test_name,
                     argMax(result, start_time) AS status,
-                    job_url,
                     job_name,
-                    report_url as results_link
+                    report_url as results_link,
+                    job_url
                FROM `gh-data`.clickhouse_regression_results
                GROUP BY architecture, test_name, job_url, job_name, report_url
                ORDER BY length(test_name) DESC
             )
-            WHERE job_url='{job_url}'
+            WHERE job_url LIKE '{job_url}%'
             AND status IN ('Fail', 'Error')
             """
     df = client.query_dataframe(query)

From b229c9e6f9d1e100ba32f271b6bba112d9894afb Mon Sep 17 00:00:00 2001
From: strtgbb <146047128+strtgbb@users.noreply.github.com>
Date: Sun, 11 May 2025 08:46:28 -0400
Subject: [PATCH 36/36] also support reruns in broken tests and new fails in pr

---
 .../create_workflow_report.py                 | 50 +++++++++++++------
 1 file changed, 34 insertions(+), 16 deletions(-)

diff --git a/.github/actions/create_workflow_report/create_workflow_report.py b/.github/actions/create_workflow_report/create_workflow_report.py
index 21c365a23dd1..eca0d9dac7b6 100755
--- a/.github/actions/create_workflow_report/create_workflow_report.py
+++ b/.github/actions/create_workflow_report/create_workflow_report.py
@@ -177,14 +177,26 @@ def get_checks_known_fails(client: Client, job_url: str, known_fails: dict):
     """
     Get tests that are known to fail for the given job URL.
     """
-    assert len(known_fails) > 0, "cannot query the database with empty known fails"
-    columns = "check_status as job_status, check_name as job_name, test_status, test_name, report_url as results_link"
-    query = f"""SELECT {columns} FROM `gh-data`.checks
-                WHERE task_url LIKE '{job_url}%'
-                AND test_status='BROKEN'
-                AND test_name IN ({','.join(f"'{test}'" for test in known_fails.keys())})
-                ORDER BY test_name, check_name
-                """
+    if len(known_fails) == 0:
+        return pd.DataFrame()
+
+    query = f"""SELECT job_status, job_name, status as test_status, test_name, results_link
+        FROM (
+            SELECT
+                argMax(check_status, check_start_time) as job_status,
+                check_name as job_name,
+                argMax(test_status, check_start_time) as status,
+                test_name,
+                report_url as results_link,
+                task_url
+            FROM `gh-data`.checks
+            GROUP BY check_name, test_name, report_url, task_url
+        )
+        WHERE task_url LIKE '{job_url}%'
+        AND test_status='BROKEN'
+        AND test_name IN ({','.join(f"'{test}'" for test in known_fails.keys())})
+        ORDER BY job_name, test_name
+        """
 
     df = client.query_dataframe(query)
 
@@ -302,15 +314,21 @@ def get_new_fails_this_pr(
         return pd.DataFrame()
 
     # Get all checks from the base branch that didn't fail
-    columns = (
-        "check_name as job_name, test_status, test_name, report_url as results_link"
-    )
-    base_checks_query = f"""SELECT {columns} FROM `gh-data`.checks
+    base_checks_query = f"""SELECT job_name, status as test_status, test_name, results_link
+            FROM (
+                SELECT
+                    check_name as job_name,
+                    argMax(test_status, check_start_time) as status,
+                    test_name,
+                    report_url as results_link,
+                    task_url
+                FROM `gh-data`.checks
                 WHERE commit_sha='{base_sha}'
-                AND test_status NOT IN ('FAIL', 'ERROR')
-                AND check_status!='error'
-                ORDER BY check_name, test_name
-                """
+                GROUP BY check_name, test_name, report_url, task_url
+            )
+            WHERE test_status NOT IN ('FAIL', 'ERROR')
+            ORDER BY job_name, test_name
+            """
     base_checks = client.query_dataframe(base_checks_query)
 
     # Get regression results from base branch that didn't fail