Flaky test reurun and reporting

.github/workflows/build.yaml

@@ -37,13 +37,18 @@ on:
           If 'true', trigger the test workflow after all builds complete.
         type: boolean
         default: false
+      summary-only:
+        description: "If true, skip all build jobs and run only build-summary"
+        type: boolean
+        default: false
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
   cpp-build:
+    if: ${{ !inputs.summary-only }}
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@main
     with:
@@ -53,6 +58,7 @@ jobs:
       sha: ${{ inputs.sha }}
       script: ci/build_cpp.sh
   python-build:
+    if: ${{ !inputs.summary-only }}
     needs: [cpp-build]
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@main
@@ -63,6 +69,7 @@ jobs:
       sha: ${{ inputs.sha }}
       script: ci/build_python.sh
   upload-conda:
+    if: ${{ !inputs.summary-only }}
     needs: [cpp-build, python-build]
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@main
@@ -72,6 +79,7 @@ jobs:
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
   wheel-build-cuopt-mps-parser:
+    if: ${{ !inputs.summary-only }}
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main
     with:
@@ -86,6 +94,7 @@ jobs:
       # need 1 build per Python version and arch (but CUDA version doesn't matter so choose the latest)
       matrix_filter: 'group_by([.ARCH, (.PY_VER |split(".") | map(tonumber))])|map(max_by([(.CUDA_VER|split(".")|map(tonumber))]))'
   wheel-publish-cuopt-mps-parser:
+    if: ${{ !inputs.summary-only }}
     needs: wheel-build-cuopt-mps-parser
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@main
@@ -97,6 +106,7 @@ jobs:
       package-name: cuopt_mps_parser
       package-type: python
   wheel-build-libcuopt:
+    if: ${{ !inputs.summary-only }}
     needs: wheel-build-cuopt-mps-parser
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main
@@ -110,6 +120,7 @@ jobs:
       package-type: cpp
       matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
   wheel-publish-libcuopt:
+    if: ${{ !inputs.summary-only }}
     needs: wheel-build-libcuopt
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@main
@@ -121,6 +132,7 @@ jobs:
       package-name: libcuopt
       package-type: cpp
   wheel-build-cuopt:
+    if: ${{ !inputs.summary-only }}
     needs: [wheel-build-cuopt-mps-parser, wheel-build-libcuopt]
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main
@@ -133,6 +145,7 @@ jobs:
       package-name: cuopt
       package-type: python
   wheel-publish-cuopt:
+    if: ${{ !inputs.summary-only }}
     needs: wheel-build-cuopt
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@main
@@ -144,6 +157,7 @@ jobs:
       package-name: cuopt
       package-type: python
   wheel-build-cuopt-server:
+    if: ${{ !inputs.summary-only }}
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main
     with:
@@ -158,6 +172,7 @@ jobs:
       # Only need 1 package per CUDA major version. This selects "ARCH=amd64 + the latest supported Python, 1 job per major CUDA version".
       matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]))
   wheel-publish-cuopt-server:
+    if: ${{ !inputs.summary-only }}
     needs: wheel-build-cuopt-server
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@main
@@ -169,6 +184,7 @@ jobs:
       package-name: cuopt_server
       package-type: python
   docs-build:
+    if: ${{ !inputs.summary-only }}
     needs: [python-build]
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main
@@ -184,6 +200,7 @@ jobs:
       container_image: "rapidsai/ci-conda:26.06-latest"
       script: "ci/build_docs.sh"
   wheel-build-cuopt-sh-client:
+    if: ${{ !inputs.summary-only }}
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@main
     with:
@@ -199,6 +216,7 @@ jobs:
       # only need 1 build (noarch package): this selects amd64, oldest-supported Python, latest-supported CUDA
       matrix_filter: '[map(select(.ARCH == "amd64")) | min_by((.PY_VER | split(".") | map(tonumber)), (.CUDA_VER | split(".") | map(-tonumber)))]'
   wheel-publish-cuopt-sh-client:
+    if: ${{ !inputs.summary-only }}
     needs: wheel-build-cuopt-sh-client
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@main
@@ -219,7 +237,7 @@ jobs:
       - wheel-publish-cuopt-server
       - wheel-publish-cuopt-sh-client
       - wheel-publish-libcuopt
-    if: inputs.trigger-tests
+    if: ${{ inputs.trigger-tests && !inputs.summary-only }}
     runs-on: ubuntu-latest
     # ref: https://docs.github.com/en/actions/reference/security/secure-use#use-an-intermediate-environment-variable
     env:
@@ -242,7 +260,35 @@ jobs:
             -f date="${INPUT_DATE}"             \
             -f sha="${INPUT_SHA}"
 
+  build-summary:
+    if: ${{ always() && (inputs.build_type == 'nightly') }}
+    needs:
+      - tests
+      - build-images
+      - docs-build
+    runs-on: linux-amd64-cpu4
+    container:
+      image: python:3.12-slim
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          ref: ${{ inputs.sha }}
+      - name: Install dependencies
+        run: apt-get update && apt-get install -y --no-install-recommends curl
+      - name: Send build summary
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+          GITHUB_RUN_ID: ${{ github.run_id }}
+          GITHUB_REPOSITORY: ${{ github.repository }}
+          GITHUB_SERVER_URL: ${{ github.server_url }}
+          CUOPT_SLACK_WEBHOOK_URL: ${{ secrets.CUOPT_SLACK_WEBHOOK_URL }}
+          SLACK_BOT_TOKEN: ${{ secrets.CUOPT_SLACK_BOT_TOKEN }}
+          SLACK_CHANNEL_ID: ${{ secrets.CUOPT_SLACK_CHANNEL_ID }}
+          RAPIDS_BRANCH: ${{ inputs.branch }}
+        run: bash ci/build_summary.sh
+
   build-images:
+    if: ${{ !inputs.summary-only }}
     needs:
       - wheel-publish-cuopt
       - wheel-publish-cuopt-server

.github/workflows/nightly-summary.yaml

@@ -0,0 +1,83 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+name: nightly-summary
+
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: "Branch name the run targets"
+        required: true
+        type: string
+        default: main
+      sha:
+        description: "Full git commit SHA to check out"
+        required: true
+        type: string
+      build_type:
+        description: "Build type (nightly, pull-request, branch)"
+        required: true
+        type: string
+        default: nightly
+      date:
+        description: "Date (YYYY-MM-DD) for this run. Defaults to today."
+        required: false
+        type: string
+  workflow_call:
+    inputs:
+      branch:
+        required: true
+        type: string
+      sha:
+        required: true
+        type: string
+      build_type:
+        required: true
+        type: string
+      date:
+        required: false
+        type: string
+    secrets:
+      CUOPT_DATASET_S3_URI:
+        required: true
+      CUOPT_AWS_ACCESS_KEY_ID:
+        required: true
+      CUOPT_AWS_SECRET_ACCESS_KEY:
+        required: true
+      CUOPT_SLACK_WEBHOOK_URL:
+        required: false
+      CUOPT_SLACK_BOT_TOKEN:
+        required: false
+      CUOPT_SLACK_CHANNEL_ID:
+        required: false
+
+jobs:
+  nightly-summary:
+    runs-on: linux-amd64-cpu4
+    container:
+      image: python:3.12-slim
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          ref: ${{ inputs.sha }}
+      - name: Install dependencies
+        run: |
+          apt-get update && apt-get install -y --no-install-recommends curl
+          pip install awscli
+      - name: Run nightly summary
+        env:
+          CUOPT_DATASET_S3_URI: ${{ secrets.CUOPT_DATASET_S3_URI }}
+          CUOPT_AWS_ACCESS_KEY_ID: ${{ secrets.CUOPT_AWS_ACCESS_KEY_ID }}
+          CUOPT_AWS_SECRET_ACCESS_KEY: ${{ secrets.CUOPT_AWS_SECRET_ACCESS_KEY }}
+          CUOPT_SLACK_WEBHOOK_URL: ${{ secrets.CUOPT_SLACK_WEBHOOK_URL }}
+          CUOPT_SLACK_BOT_TOKEN: ${{ secrets.CUOPT_SLACK_BOT_TOKEN }}
+          CUOPT_SLACK_CHANNEL_ID: ${{ secrets.CUOPT_SLACK_CHANNEL_ID }}
+          RAPIDS_BUILD_TYPE: ${{ inputs.build_type }}
+          RAPIDS_BRANCH: ${{ inputs.branch }}
+          RUN_DATE: ${{ inputs.date }}
+          GITHUB_TOKEN: ${{ github.token }}
+          GITHUB_RUN_ID: ${{ github.run_id }}
+          GITHUB_REPOSITORY: ${{ github.repository }}
+          GITHUB_SERVER_URL: ${{ github.server_url }}
+        run: bash ci/nightly_summary.sh

.github/workflows/test.yaml

@@ -24,9 +24,14 @@ on:
         description: "build_type: one of [branch, nightly, pull-request]"
         type: string
         default: nightly
+      summary-only:
+        description: "If true, skip all test jobs and run only nightly-summary"
+        type: boolean
+        default: false
 
 jobs:
   conda-cpp-tests:
+    if: ${{ !inputs.summary-only }}
     uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@main
     with:
       build_type: ${{ inputs.build_type }}
@@ -42,6 +47,7 @@ jobs:
       script-env-secret-3-key: CUOPT_AWS_SECRET_ACCESS_KEY
       script-env-secret-3-value: ${{ secrets.CUOPT_AWS_SECRET_ACCESS_KEY }}
   conda-python-tests:
+    if: ${{ !inputs.summary-only }}
     uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@main
     with:
       run_codecov: false
@@ -58,6 +64,7 @@ jobs:
       script-env-secret-3-key: CUOPT_AWS_SECRET_ACCESS_KEY
       script-env-secret-3-value: ${{ secrets.CUOPT_AWS_SECRET_ACCESS_KEY }}
   wheel-tests-cuopt:
+    if: ${{ !inputs.summary-only }}
     uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main
     with:
       build_type: ${{ inputs.build_type }}
@@ -73,6 +80,7 @@ jobs:
       script-env-secret-3-key: CUOPT_AWS_SECRET_ACCESS_KEY
       script-env-secret-3-value: ${{ secrets.CUOPT_AWS_SECRET_ACCESS_KEY }}
   wheel-tests-cuopt-server:
+    if: ${{ !inputs.summary-only }}
     uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main
     with:
       build_type: ${{ inputs.build_type }}
@@ -88,6 +96,7 @@ jobs:
       script-env-secret-3-key: CUOPT_AWS_SECRET_ACCESS_KEY
       script-env-secret-3-value: ${{ secrets.CUOPT_AWS_SECRET_ACCESS_KEY }}
   conda-notebook-tests:
+    if: ${{ !inputs.summary-only }}
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main
     with:
@@ -99,3 +108,24 @@ jobs:
       arch: "amd64"
       container_image: "rapidsai/ci-conda:26.06-latest"
       script: ci/test_notebooks.sh
+  nightly-summary:
+    if: ${{ always() && inputs.build_type == 'nightly' }}
+    needs:
+      - conda-cpp-tests
+      - conda-python-tests
+      - wheel-tests-cuopt
+      - wheel-tests-cuopt-server
+      - conda-notebook-tests
+    uses: ./.github/workflows/nightly-summary.yaml
+    with:
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      build_type: ${{ inputs.build_type }}
+      date: ${{ inputs.date }}
+    secrets:
+      CUOPT_DATASET_S3_URI: ${{ secrets.CUOPT_DATASET_S3_URI }}
+      CUOPT_AWS_ACCESS_KEY_ID: ${{ secrets.CUOPT_AWS_ACCESS_KEY_ID }}
+      CUOPT_AWS_SECRET_ACCESS_KEY: ${{ secrets.CUOPT_AWS_SECRET_ACCESS_KEY }}
+      CUOPT_SLACK_WEBHOOK_URL: ${{ secrets.CUOPT_SLACK_WEBHOOK_URL }}
+      CUOPT_SLACK_BOT_TOKEN: ${{ secrets.CUOPT_SLACK_BOT_TOKEN }}
+      CUOPT_SLACK_CHANNEL_ID: ${{ secrets.CUOPT_SLACK_CHANNEL_ID }}