From 61e742bc01488da51a87c8a572646809056da264 Mon Sep 17 00:00:00 2001
From: "Marcus D. Hanwell" <mhanwell@nvidia.com>
Date: Mon, 19 May 2025 15:41:35 -0400
Subject: [PATCH 1/4] Test out a simple env-vars script for build

---
 .github/workflows/build-wheel.yml | 29 +++++-----------------
 ci/tools/env-vars                 | 40 +++++++++++++++++++++++++++++++
 2 files changed, 46 insertions(+), 23 deletions(-)
 create mode 100755 ci/tools/env-vars

diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/build-wheel.yml
index 1eed96bdb8..386f166a06 100644
--- a/.github/workflows/build-wheel.yml
+++ b/.github/workflows/build-wheel.yml
@@ -59,29 +59,12 @@ jobs:
         uses: ilammy/msvc-dev-cmd@v1  # TODO: ask admin to allow pinning commits
       
       - name: Set environment variables
-        run: |
-          PYTHON_VERSION_FORMATTED=$(echo '${{ matrix.python-version }}' | tr -d '.')
-          if [[ "${{ inputs.host-platform }}" == linux* ]]; then
-            CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-manylinux*"
-            REPO_DIR=$(pwd)
-          elif [[ "${{ inputs.host-platform }}" == win* ]]; then
-            CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-win_amd64"
-            PWD=$(pwd)
-            REPO_DIR=$(cygpath -w $PWD)
-          fi
-  
-          echo "CUDA_BINDINGS_PARALLEL_LEVEL=$(nproc)" >> $GITHUB_ENV
-          CUDA_CORE_ARTIFACT_BASENAME="cuda-core-python${PYTHON_VERSION_FORMATTED}-${{ inputs.host-platform }}"
-          echo "CUDA_CORE_ARTIFACT_BASENAME=${CUDA_CORE_ARTIFACT_BASENAME}" >> $GITHUB_ENV
-          echo "CUDA_CORE_ARTIFACT_NAME=${CUDA_CORE_ARTIFACT_BASENAME}-${{ github.sha }}" >> $GITHUB_ENV
-          echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_core/dist")" >> $GITHUB_ENV
-          echo "CUDA_CORE_CYTHON_TESTS_DIR=$(realpath "$REPO_DIR/cuda_core/tests/cython")" >> $GITHUB_ENV
-          CUDA_BINDINGS_ARTIFACT_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${{ inputs.cuda-version }}-${{ inputs.host-platform }}"
-          echo "CUDA_BINDINGS_ARTIFACT_BASENAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}" >> $GITHUB_ENV
-          echo "CUDA_BINDINGS_ARTIFACT_NAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}-${{ github.sha }}" >> $GITHUB_ENV
-          echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/dist")" >> $GITHUB_ENV
-          echo "CUDA_BINDINGS_CYTHON_TESTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/tests/cython")" >> $GITHUB_ENV
-          echo "CIBW_BUILD=${CIBW_BUILD}" >> $GITHUB_ENV
+        env:
+          CUDA_VER: ${{ inputs.cuda-version }}
+          HOST_PLATFORM: ${{ inputs.host-platform }}
+          PY_VER: ${{ matrix.python-version }}
+          SHA: ${{ github.sha }}
+        run: ./ci/tools/env-vars build
 
       - name: Dump environment
         run: |
diff --git a/ci/tools/env-vars b/ci/tools/env-vars
new file mode 100755
index 0000000000..6042224fb9
--- /dev/null
+++ b/ci/tools/env-vars
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# A utility script to set up the GitHub environment variables for the CI.
+
+set -euo pipefail
+
+# Check if the script was called with exactly 1 argument
+if [[ ${#} -ne 1 ]]; then
+  echo "Error: This script requires exactly 1 argument (the build mode). You provided ${#}"
+  echo "Usage: ${0} build_mode[build or test]"
+  exit 1
+fi
+
+PYTHON_VERSION_FORMATTED=$(echo '${PY_VER}' | tr -d '.')
+if [[ "${HOST_PLATFORM}" == linux* ]]; then
+  CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-manylinux*"
+  REPO_DIR=$(pwd)
+elif [[ "${HOST_PLATFORM}" == win* ]]; then
+  CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-win_amd64"
+  PWD=$(pwd)
+  REPO_DIR=$(cygpath -w ${PWD})
+fi
+
+echo "PATH=${REPO_DIR}/ci/tools:${PATH}" >> $GITHUB_ENV
+echo "CUDA_BINDINGS_PARALLEL_LEVEL=$(nproc)" >> $GITHUB_ENV
+CUDA_CORE_ARTIFACT_BASENAME="cuda-core-python${PYTHON_VERSION_FORMATTED}-${HOST_PLATFORM}"
+echo "CUDA_CORE_ARTIFACT_BASENAME=${CUDA_CORE_ARTIFACT_BASENAME}" >> $GITHUB_ENV
+echo "CUDA_CORE_ARTIFACT_NAME=${CUDA_CORE_ARTIFACT_BASENAME}-${SHA}" >> $GITHUB_ENV
+echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "${REPO_DIR}/cuda_core/dist")" >> $GITHUB_ENV
+echo "CUDA_CORE_CYTHON_TESTS_DIR=$(realpath "${REPO_DIR}/cuda_core/tests/cython")" >> $GITHUB_ENV
+CUDA_BINDINGS_ARTIFACT_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${CUDA_VER}-${HOST_PLATFORM}"
+echo "CUDA_BINDINGS_ARTIFACT_BASENAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}" >> $GITHUB_ENV
+echo "CUDA_BINDINGS_ARTIFACT_NAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}-${SHA}" >> $GITHUB_ENV
+echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "${REPO_DIR}/cuda_bindings/dist")" >> $GITHUB_ENV
+echo "CUDA_BINDINGS_CYTHON_TESTS_DIR=$(realpath "${REPO_DIR}/cuda_bindings/tests/cython")" >> $GITHUB_ENV
+echo "CIBW_BUILD=${CIBW_BUILD}" >> $GITHUB_ENV

From 3c6da34fc40238c63ba8a7b12635b125f6af86a8 Mon Sep 17 00:00:00 2001
From: "Marcus D. Hanwell" <mhanwell@nvidia.com>
Date: Mon, 19 May 2025 15:50:04 -0400
Subject: [PATCH 2/4] Double quotes to expand out the bash var, separator

---
 ci/tools/env-vars | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/ci/tools/env-vars b/ci/tools/env-vars
index 6042224fb9..cbc1b80166 100755
--- a/ci/tools/env-vars
+++ b/ci/tools/env-vars
@@ -15,17 +15,19 @@ if [[ ${#} -ne 1 ]]; then
   exit 1
 fi
 
-PYTHON_VERSION_FORMATTED=$(echo '${PY_VER}' | tr -d '.')
+PYTHON_VERSION_FORMATTED=$(echo "${PY_VER}" | tr -d '.')
 if [[ "${HOST_PLATFORM}" == linux* ]]; then
   CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-manylinux*"
   REPO_DIR=$(pwd)
+  TOOLS_PATH="${REPO_DIR}/ci/tools"
 elif [[ "${HOST_PLATFORM}" == win* ]]; then
   CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-win_amd64"
   PWD=$(pwd)
   REPO_DIR=$(cygpath -w ${PWD})
+  TOOLS_PATH=$(cygpath -w ${PWD}/ci/tools)
 fi
 
-echo "PATH=${REPO_DIR}/ci/tools:${PATH}" >> $GITHUB_ENV
+echo "${TOOLS_PATH}" >> $GITHUB_PATH
 echo "CUDA_BINDINGS_PARALLEL_LEVEL=$(nproc)" >> $GITHUB_ENV
 CUDA_CORE_ARTIFACT_BASENAME="cuda-core-python${PYTHON_VERSION_FORMATTED}-${HOST_PLATFORM}"
 echo "CUDA_CORE_ARTIFACT_BASENAME=${CUDA_CORE_ARTIFACT_BASENAME}" >> $GITHUB_ENV
@@ -37,4 +39,7 @@ echo "CUDA_BINDINGS_ARTIFACT_BASENAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}" >> $GI
 echo "CUDA_BINDINGS_ARTIFACT_NAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}-${SHA}" >> $GITHUB_ENV
 echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "${REPO_DIR}/cuda_bindings/dist")" >> $GITHUB_ENV
 echo "CUDA_BINDINGS_CYTHON_TESTS_DIR=$(realpath "${REPO_DIR}/cuda_bindings/tests/cython")" >> $GITHUB_ENV
-echo "CIBW_BUILD=${CIBW_BUILD}" >> $GITHUB_ENV
+
+if [[ "${1}" == "build" ]]; then
+  echo "CIBW_BUILD=${CIBW_BUILD}" >> $GITHUB_ENV
+fi

From 91c07c9f6cfadcc1575296757c0ea3f2e7a574ba Mon Sep 17 00:00:00 2001
From: "Marcus D. Hanwell" <mhanwell@nvidia.com>
Date: Mon, 19 May 2025 17:11:01 -0400
Subject: [PATCH 3/4] Reuse env var code in the tests, add test runner

---
 .github/workflows/test-wheel-linux.yml   | 109 +++-----------------
 .github/workflows/test-wheel-windows.yml | 120 ++++++-----------------
 ci/tools/env-vars                        |  29 ++++++
 ci/tools/run-tests                       |  72 ++++++++++++++
 4 files changed, 149 insertions(+), 181 deletions(-)
 create mode 100755 ci/tools/run-tests

diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml
index 8a10a9ef7e..85b1b7657c 100644
--- a/.github/workflows/test-wheel-linux.yml
+++ b/.github/workflows/test-wheel-linux.yml
@@ -154,57 +154,14 @@ jobs:
           dependent_exes: "jq wget"
 
       - name: Set environment variables
-        run: |
-          PYTHON_VERSION_FORMATTED=$(echo '${{ matrix.PY_VER }}' | tr -d '.')
-          if [[ "${{ inputs.host-platform }}" == linux* ]]; then
-            REPO_DIR=$(pwd)
-          elif [[ "${{ inputs.host-platform }}" == win* ]]; then
-            PWD=$(pwd)
-            REPO_DIR=$(cygpath -w $PWD)
-          fi
-
-          BUILD_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${{ inputs.build-ctk-ver }})"
-          TEST_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${{ matrix.CUDA_VER }})"
-          if [[ $BUILD_CUDA_MAJOR != $TEST_CUDA_MAJOR ]]; then
-            SKIP_CUDA_BINDINGS_TEST=1
-            SKIP_CYTHON_TEST=1
-          else
-            SKIP_CUDA_BINDINGS_TEST=0
-            BUILD_CUDA_MINOR="$(cut -d '.' -f 2 <<< ${{ inputs.build-ctk-ver }})"
-            TEST_CUDA_MINOR="$(cut -d '.' -f 2 <<< ${{ matrix.CUDA_VER }})"
-            if [[ $BUILD_CUDA_MINOR != $TEST_CUDA_MINOR ]]; then
-              SKIP_CYTHON_TEST=1
-            else
-              SKIP_CYTHON_TEST=0
-            fi
-          fi
-
-          # We don't test compute-sanitizer on CTK<12 because backporting fixes is too much effort
-          # We only test compute-sanitizer on python 3.12 arbitrarily; we don't need to use sanitizer on the entire matrix
-          # Only local ctk installs have compute-sanitizer; there is not wheel for it
-          if [[ "${{ inputs.python-version }}" == "3.12" && "${{ inputs.cuda-version }}" != "11.8.0" && "${{ inputs.local-ctk }}" == 1 ]]; then
-            SETUP_SANITIZER=1
-            echo "LATEST_CUDA_VERSION=$(bash .github/workflows/guess_latest.sh)" >> $GITHUB_ENV
-          else
-            SETUP_SANITIZER=0
-          fi
-          echo "SETUP_SANITIZER=${SETUP_SANITIZER}" >> $GITHUB_ENV
-
-          # make outputs from the previous job as env vars
-          CUDA_CORE_ARTIFACT_BASENAME="cuda-core-python${PYTHON_VERSION_FORMATTED}-${{ inputs.host-platform }}"
-          echo "PYTHON_VERSION_FORMATTED=${PYTHON_VERSION_FORMATTED}" >> $GITHUB_ENV
-          echo "CUDA_CORE_ARTIFACT_BASENAME=${CUDA_CORE_ARTIFACT_BASENAME}" >> $GITHUB_ENV
-          echo "CUDA_CORE_ARTIFACT_NAME=${CUDA_CORE_ARTIFACT_BASENAME}-${{ github.sha }}" >> $GITHUB_ENV
-          echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_core/dist")" >> $GITHUB_ENV
-          echo "CUDA_CORE_CYTHON_TESTS_DIR=$(realpath "$REPO_DIR/cuda_core/tests/cython")" >> $GITHUB_ENV
-          CUDA_BINDINGS_ARTIFACT_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${{ inputs.build-ctk-ver }}-${{ inputs.host-platform }}"
-          echo "CUDA_BINDINGS_ARTIFACT_BASENAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}" >> $GITHUB_ENV
-          echo "CUDA_BINDINGS_ARTIFACT_NAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}-${{ github.sha }}" >> $GITHUB_ENV
-          echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/dist")" >> $GITHUB_ENV
-          echo "CUDA_BINDINGS_CYTHON_TESTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/tests/cython")" >> $GITHUB_ENV
-
-          echo "SKIP_CUDA_BINDINGS_TEST=${SKIP_CUDA_BINDINGS_TEST}" >> $GITHUB_ENV
-          echo "SKIP_CYTHON_TEST=${SKIP_CYTHON_TEST}" >> $GITHUB_ENV
+        env:
+          BUILD_CUDA_VER: ${{ inputs.build-ctk-ver }}
+          CUDA_VER: ${{ matrix.CUDA_VER }}
+          HOST_PLATFORM: ${{ inputs.host-platform }}
+          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
+          PY_VER: ${{ matrix.PY_VER }}
+          SHA: ${{ github.sha }}
+        run: ./ci/tools/env-vars test
 
       - name: Download cuda-python build artifacts
         if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0'}}
@@ -343,50 +300,16 @@ jobs:
 
       - name: Run cuda.bindings tests
         if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0' }}
-        run: |
-          pushd "${CUDA_BINDINGS_ARTIFACTS_DIR}"
-          if [[ "${{ matrix.LOCAL_CTK }}" == 1 ]]; then
-            ls $CUDA_PATH
-            pip install $(ls *.whl)[test]
-          else
-            pip install $(ls *.whl)[all,test]
-          fi
-          popd
-
-          pushd ./cuda_bindings
-          ${SANITIZER_CMD} pytest -rxXs -v tests/
-          if [[ "${SKIP_CYTHON_TEST}" == 0 ]]; then
-            ${SANITIZER_CMD} pytest -rxXs -v tests/cython
-          fi
-          popd
+        env:
+          CUDA_VER: ${{ matrix.CUDA_VER }}
+          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
+        run: run-tests bindings
 
       - name: Run cuda.core tests
-        run: |
-          # If build/test majors match: cuda.bindings is installed in the previous step.
-          # If mismatch: cuda.bindings is installed from the backport branch.
-          if [[ "${SKIP_CUDA_BINDINGS_TEST}" == 1 ]]; then
-            pushd "${CUDA_BINDINGS_ARTIFACTS_DIR}"
-            if [[ "${{ matrix.LOCAL_CTK }}" == 1 ]]; then
-              pip install *.whl
-            else
-              pip install $(ls *.whl)[all]
-            fi
-            popd
-          fi
-          TEST_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${{ matrix.CUDA_VER }})"
-          pushd "${CUDA_CORE_ARTIFACTS_DIR}"
-          pip install $(ls *.whl)["cu${TEST_CUDA_MAJOR}","test-cu${TEST_CUDA_MAJOR}"]
-          popd
-
-          pushd ./cuda_core
-          ${SANITIZER_CMD} pytest -rxXs -v tests/
-
-          # Currently our CI always installs the latest bindings (from either major version).
-          # This is not compatible with the test requirements.
-          if [[ "${SKIP_CYTHON_TEST}" == 0 ]]; then
-            ${SANITIZER_CMD} pytest -rxXs -v tests/cython
-          fi
-          popd
+        env:
+          CUDA_VER: ${{ matrix.CUDA_VER }}
+          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
+        run: run-tests core
 
       - name: Ensure cuda-python installable
         run: |
diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml
index 04ed9f341c..f5c6db6fc7 100644
--- a/.github/workflows/test-wheel-windows.yml
+++ b/.github/workflows/test-wheel-windows.yml
@@ -96,41 +96,30 @@ jobs:
       - name: Ensure GPU is working
         run: nvidia-smi
 
-      - name: Set environment variables
+      - name: Install Git for Windows
+        # the GPU runner image does not have Git Bash pre-installed...
+        env:
+          # doesn't seem there's an easy way to avoid hard-coding it?
+          GFW_EXE_URL: https://github.com/git-for-windows/git/releases/download/v2.49.0.windows.1/PortableGit-2.49.0-64-bit.7z.exe
         run: |
-          $PYTHON_VERSION_FORMATTED = '${{ matrix.PY_VER }}' -replace '\.'
-          $REPO_DIR = $PWD.Path
-
-          $BUILD_CUDA_MAJOR = '${{ inputs.build-ctk-ver }}' -split '\.' | Select-Object -First 1
-          $TEST_CUDA_MAJOR = '${{ matrix.CUDA_VER }}' -split '\.' | Select-Object -First 1
-          if ($BUILD_CUDA_MAJOR -ne $TEST_CUDA_MAJOR) {
-            $SKIP_CUDA_BINDINGS_TEST = 1
-            $SKIP_CYTHON_TEST = 1
-          } else {
-            $SKIP_CUDA_BINDINGS_TEST = 0
-            $BUILD_CUDA_MINOR = '${{ inputs.build-ctk-ver }}' -split '\.' | Select-Object -Skip 1 -First 1
-            $TEST_CUDA_MINOR = '${{ matrix.CUDA_VER }}' -split '\.' | Select-Object -Skip 1 -First 1
-            if ($BUILD_CUDA_MINOR -ne $TEST_CUDA_MINOR) {
-              $SKIP_CYTHON_TEST = 1
-            } else {
-              $SKIP_CYTHON_TEST = 0
-            }
-          }
+          Invoke-WebRequest -Uri "$env:GFW_EXE_URL" -OutFile "PortableGit.7z.exe"
+          # Self-extracting, see https://gitforwindows.org/zip-archives-extracting-the-released-archives.html
+          Start-Process .\PortableGit.7z.exe -Wait -Verbose -ArgumentList '-y -gm2'
+          ls -l PortableGit
+          echo "$((Get-Location).Path)\\PortableGit\\bin" >> $env:GITHUB_PATH
+          $env:Path += ";$((Get-Location).Path)\\PortableGit\\bin"
+          bash --version
 
-          # Make outputs from the previous job as env vars
-          $CUDA_CORE_ARTIFACT_BASENAME = "cuda-core-python${PYTHON_VERSION_FORMATTED}-${{ inputs.host-platform }}"
-          "PYTHON_VERSION_FORMATTED=${PYTHON_VERSION_FORMATTED}" >> $env:GITHUB_ENV
-          "CUDA_CORE_ARTIFACT_BASENAME=${CUDA_CORE_ARTIFACT_BASENAME}" >> $env:GITHUB_ENV
-          "CUDA_CORE_ARTIFACT_NAME=${CUDA_CORE_ARTIFACT_BASENAME}-${{ github.sha }}" >> $env:GITHUB_ENV
-          "CUDA_CORE_ARTIFACTS_DIR=$($ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath("$REPO_DIR\cuda_core\dist"))" >> $env:GITHUB_ENV
-          "CUDA_CORE_CYTHON_TESTS_DIR=$($ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath("$REPO_DIR\cuda_core\tests\cython"))" >> $env:GITHUB_ENV
-          $CUDA_BINDINGS_ARTIFACT_BASENAME = "cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${{ inputs.build-ctk-ver }}-${{ inputs.host-platform }}"
-          "CUDA_BINDINGS_ARTIFACT_BASENAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}" >> $env:GITHUB_ENV
-          "CUDA_BINDINGS_ARTIFACT_NAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}-${{ github.sha }}" >> $env:GITHUB_ENV
-          "CUDA_BINDINGS_ARTIFACTS_DIR=$($ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath("$REPO_DIR\cuda_bindings\dist"))" >> $env:GITHUB_ENV
-          "CUDA_BINDINGS_CYTHON_TESTS_DIR=$($ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath("$REPO_DIR\cuda_bindings\tests\cython"))" >> $env:GITHUB_ENV
-          "SKIP_CUDA_BINDINGS_TEST=${SKIP_CUDA_BINDINGS_TEST}" >> $env:GITHUB_ENV
-          "SKIP_CYTHON_TEST=${SKIP_CYTHON_TEST}" >> $env:GITHUB_ENV
+      - name: Set environment variables
+        env:
+          BUILD_CUDA_VER: ${{ inputs.build-ctk-ver }}
+          CUDA_VER: ${{ matrix.CUDA_VER }}
+          HOST_PLATFORM: ${{ inputs.host-platform }}
+          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
+          PY_VER: ${{ matrix.PY_VER }}
+          SHA: ${{ github.sha }}
+        shell: bash --noprofile --norc -xeuo pipefail {0}
+        run: ./ci/tools/env-vars test
 
       - name: Download cuda-python build artifacts
         if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0'}}
@@ -161,21 +150,6 @@ jobs:
           }
           gh --version
 
-      - name: Install Git for Windows
-        # the GPU runner image does not have Git Bash pre-installed...
-        if: ${{ matrix.LOCAL_CTK == '1' }}
-        env:
-          # doesn't seem there's an easy way to avoid hard-coding it?
-          GFW_EXE_URL: https://github.com/git-for-windows/git/releases/download/v2.49.0.windows.1/PortableGit-2.49.0-64-bit.7z.exe
-        run: |
-          Invoke-WebRequest -Uri "$env:GFW_EXE_URL" -OutFile "PortableGit.7z.exe"
-          # Self-extracting, see https://gitforwindows.org/zip-archives-extracting-the-released-archives.html
-          Start-Process .\PortableGit.7z.exe -Wait -Verbose -ArgumentList '-y -gm2'
-          ls -l PortableGit
-          echo "$((Get-Location).Path)\\PortableGit\\bin" >> $env:GITHUB_PATH
-          $env:Path += ";$((Get-Location).Path)\\PortableGit\\bin"
-          bash --version
-
       - name: Install zstd
         # the GPU runner image does not have zstd pre-installed... and it's needed by actions/cache
         if: ${{ matrix.LOCAL_CTK == '1' }}
@@ -278,48 +252,18 @@ jobs:
 
       - name: Run cuda.bindings tests
         if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0' }}
-        run: |
-          Push-Location $env:CUDA_BINDINGS_ARTIFACTS_DIR
-          if ('${{ matrix.LOCAL_CTK }}' -eq '1') {
-            Get-ChildItem $env:CUDA_PATH
-            echo $PATH
-            pip install "$((Get-ChildItem -Filter *.whl).FullName)[test]"
-          } else {
-            pip install "$((Get-ChildItem -Filter *.whl).FullName)[all,test]"
-          }
-          Pop-Location
-
-          Push-Location ./cuda_bindings
-          pytest -rxXs -v tests/
-          if ($env:SKIP_CYTHON_TEST -eq '0') {
-            pytest -rxXs -v tests/cython
-          }
-          Pop-Location
+        env:
+          CUDA_VER: ${{ matrix.CUDA_VER }}
+          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
+        shell: bash --noprofile --norc -xeuo pipefail {0}
+        run: run-tests bindings
 
       - name: Run cuda.core tests
-        run: |
-          # If build/test majors match: cuda.bindings is installed in the previous step.
-          # If mismatch: cuda.bindings is installed from the backport branch.
-          if ($env:SKIP_CUDA_BINDINGS_TEST -eq '1') {
-            Push-Location $env:CUDA_BINDINGS_ARTIFACTS_DIR
-            if ('${{ matrix.LOCAL_CTK }}' -eq '1') {
-              pip install (Get-ChildItem -Filter *.whl).FullName
-            } else {
-              pip install "$((Get-ChildItem -Filter *.whl).FullName)[all]"
-            }
-            Pop-Location
-          }
-          $TEST_CUDA_MAJOR = '${{ matrix.CUDA_VER }}' -split '\.' | Select-Object -First 1
-          Push-Location $env:CUDA_CORE_ARTIFACTS_DIR
-          pip install "$((Get-ChildItem -Filter *.whl).FullName)[cu${TEST_CUDA_MAJOR},test-cu${TEST_CUDA_MAJOR}]"
-          Pop-Location
-
-          Push-Location ./cuda_core
-          pytest -rxXs -v tests/
-          if ($env:SKIP_CYTHON_TEST -eq '0') {
-            pytest -rxXs -v tests/cython
-          }
-          Pop-Location
+        env:
+          CUDA_VER: ${{ matrix.CUDA_VER }}
+          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
+        shell: bash --noprofile --norc -xeuo pipefail {0}
+        run: run-tests core
 
       - name: Ensure cuda-python installable
         run: |
diff --git a/ci/tools/env-vars b/ci/tools/env-vars
index cbc1b80166..49f896ad7b 100755
--- a/ci/tools/env-vars
+++ b/ci/tools/env-vars
@@ -39,7 +39,36 @@ echo "CUDA_BINDINGS_ARTIFACT_BASENAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}" >> $GI
 echo "CUDA_BINDINGS_ARTIFACT_NAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}-${SHA}" >> $GITHUB_ENV
 echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "${REPO_DIR}/cuda_bindings/dist")" >> $GITHUB_ENV
 echo "CUDA_BINDINGS_CYTHON_TESTS_DIR=$(realpath "${REPO_DIR}/cuda_bindings/tests/cython")" >> $GITHUB_ENV
+echo "PYTHON_VERSION_FORMATTED=${PYTHON_VERSION_FORMATTED}" >> $GITHUB_ENV
 
 if [[ "${1}" == "build" ]]; then
   echo "CIBW_BUILD=${CIBW_BUILD}" >> $GITHUB_ENV
+elif [[ "${1}" == "test" ]]; then
+  BUILD_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${BUILD_CUDA_VER})"
+  TEST_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${CUDA_VER})"
+  if [[ ${BUILD_CUDA_MAJOR} != ${TEST_CUDA_MAJOR} ]]; then
+    SKIP_CUDA_BINDINGS_TEST=1
+    SKIP_CYTHON_TEST=1
+  else
+    SKIP_CUDA_BINDINGS_TEST=0
+    BUILD_CUDA_MINOR="$(cut -d '.' -f 2 <<< ${BUILD_CUDA_VER})"
+    TEST_CUDA_MINOR="$(cut -d '.' -f 2 <<< ${CUDA_VER})"
+    if [[ ${BUILD_CUDA_MINOR} != ${TEST_CUDA_MINOR} ]]; then
+      SKIP_CYTHON_TEST=1
+    else
+      SKIP_CYTHON_TEST=0
+    fi
+  fi
+  # We don't test compute-sanitizer on CTK<12 because backporting fixes is too much effort
+  # We only test compute-sanitizer on python 3.12 arbitrarily; we don't need to use sanitizer on the entire matrix
+  # Only local ctk installs have compute-sanitizer; there is no wheel for it
+  if [[ "${PY_VER}" == "3.12" && "${CUDA_VER}" != "11.8.0" && "${LOCAL_CTK}" == 1 && "${HOST_PLATFORM}" == linux* ]]; then
+    SETUP_SANITIZER=1
+    echo "LATEST_CUDA_VERSION=$(bash .github/workflows/guess_latest.sh)" >> $GITHUB_ENV
+  else
+    SETUP_SANITIZER=0
+  fi
+  echo "SETUP_SANITIZER=${SETUP_SANITIZER}" >> $GITHUB_ENV
+  echo "SKIP_CUDA_BINDINGS_TEST=${SKIP_CUDA_BINDINGS_TEST}" >> $GITHUB_ENV
+  echo "SKIP_CYTHON_TEST=${SKIP_CYTHON_TEST}" >> $GITHUB_ENV
 fi
diff --git a/ci/tools/run-tests b/ci/tools/run-tests
new file mode 100755
index 0000000000..51fcf0a6b4
--- /dev/null
+++ b/ci/tools/run-tests
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# A utility script to install the correct packages and run the tests.
+
+set -euo pipefail
+
+# Check if the script was called with exactly 1 argument
+if [[ ${#} -ne 1 && ( "${1}" == "bindings" || "${1}" == "core" ) ]]; then
+  echo "Error: This script requires exactly 1 argument (what is tested). You provided ${#}"
+  echo "Usage: ${0} test_module[bindings or core]"
+  exit 1
+fi
+
+test_module=${1}
+
+if [[ "${test_module}" == "bindings" ]]; then
+  pushd "${CUDA_BINDINGS_ARTIFACTS_DIR}"
+  echo "Installing bindings wheel"
+  pwd
+  ls
+  if [[ "${LOCAL_CTK}" == 1 ]]; then
+    ls "${CUDA_PATH}"
+    pip install $(ls *.whl)[test]
+  else
+    pip install $(ls *.whl)[all,test]
+  fi
+  popd
+  pushd ./cuda_bindings
+  echo "Running bindinds tests"
+  pwd
+  ${SANITIZER_CMD} pytest -rxXs -v tests/
+  if [[ "${SKIP_CYTHON_TEST}" == 0 ]]; then
+    ${SANITIZER_CMD} pytest -rxXs -v tests/cython
+  fi
+  popd
+elif [[ "${test_module}" == "core" ]]; then
+  # If build/test majors match: cuda.bindings is installed in the previous step.
+  # If mismatch: cuda.bindings is installed from the backport branch.
+  if [[ "${SKIP_CUDA_BINDINGS_TEST}" == 1 ]]; then
+    pushd "${CUDA_BINDINGS_ARTIFACTS_DIR}"
+    echo "Installing bindings wheel"
+    pwd
+    ls
+    if [[ "${LOCAL_CTK}" == 1 ]]; then
+      pip install *.whl
+    else
+      pip install $(ls *.whl)[all]
+    fi
+    popd
+  fi
+  TEST_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${CUDA_VER})"
+  pushd "${CUDA_CORE_ARTIFACTS_DIR}"
+  echo "Installing core wheel"
+  pwd
+  ls
+  pip install $(ls *.whl)["cu${TEST_CUDA_MAJOR}","test-cu${TEST_CUDA_MAJOR}"]
+  popd
+  pushd ./cuda_core
+  echo "Running core tests"
+  pwd
+  ${SANITIZER_CMD} pytest -rxXs -v tests/
+  # Currently our CI always installs the latest bindings (from either major version).
+  # This is not compatible with the test requirements.
+  if [[ "${SKIP_CYTHON_TEST}" == 0 ]]; then
+    ${SANITIZER_CMD} pytest -rxXs -v tests/cython
+  fi
+  popd
+fi
\ No newline at end of file

From bbeb01180e38091fd6031484b5af1f97299b7bc5 Mon Sep 17 00:00:00 2001
From: "Marcus D. Hanwell" <mhanwell@nvidia.com>
Date: Tue, 20 May 2025 13:15:31 -0400
Subject: [PATCH 4/4] Simplify compute-sanitizer setup, fix bindings name

---
 .github/workflows/test-wheel-linux.yml | 14 +-------------
 ci/tools/env-vars                      | 15 +++++++++------
 ci/tools/setup-sanitizer               | 23 +++++++++++++++++++++++
 3 files changed, 33 insertions(+), 19 deletions(-)
 create mode 100755 ci/tools/setup-sanitizer

diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml
index 85b1b7657c..05011504fa 100644
--- a/.github/workflows/test-wheel-linux.yml
+++ b/.github/workflows/test-wheel-linux.yml
@@ -284,19 +284,7 @@ jobs:
           cuda-components: "cuda_sanitizer_api"
 
       - name: Set up compute-sanitizer
-        run: |
-          if [[ "${SETUP_SANITIZER}" == 1 ]]; then
-            COMPUTE_SANITIZER="${CUDA_HOME}/bin/compute-sanitizer"
-            COMPUTE_SANITIZER_VERSION=$(${COMPUTE_SANITIZER} --version | grep -Eo "[0-9]{4}\.[0-9]\.[0-9]" | sed -e 's/\.//g')
-            SANITIZER_CMD="${COMPUTE_SANITIZER} --target-processes=all --launch-timeout=0 --tool=memcheck --error-exitcode=1"
-            if [[ "$COMPUTE_SANITIZER_VERSION" -ge 202111 ]]; then
-                SANITIZER_CMD="${SANITIZER_CMD} --padding=32"
-            fi
-            echo "CUDA_PYTHON_TESTING_WITH_COMPUTE_SANITIZER=1" >> $GITHUB_ENV
-          else
-            SANITIZER_CMD=""
-          fi
-          echo "SANITIZER_CMD=${SANITIZER_CMD}" >> $GITHUB_ENV
+        run: setup-sanitizer
 
       - name: Run cuda.bindings tests
         if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0' }}
diff --git a/ci/tools/env-vars b/ci/tools/env-vars
index 49f896ad7b..7211dc08fc 100755
--- a/ci/tools/env-vars
+++ b/ci/tools/env-vars
@@ -34,18 +34,15 @@ echo "CUDA_CORE_ARTIFACT_BASENAME=${CUDA_CORE_ARTIFACT_BASENAME}" >> $GITHUB_ENV
 echo "CUDA_CORE_ARTIFACT_NAME=${CUDA_CORE_ARTIFACT_BASENAME}-${SHA}" >> $GITHUB_ENV
 echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "${REPO_DIR}/cuda_core/dist")" >> $GITHUB_ENV
 echo "CUDA_CORE_CYTHON_TESTS_DIR=$(realpath "${REPO_DIR}/cuda_core/tests/cython")" >> $GITHUB_ENV
-CUDA_BINDINGS_ARTIFACT_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${CUDA_VER}-${HOST_PLATFORM}"
-echo "CUDA_BINDINGS_ARTIFACT_BASENAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}" >> $GITHUB_ENV
-echo "CUDA_BINDINGS_ARTIFACT_NAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}-${SHA}" >> $GITHUB_ENV
-echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "${REPO_DIR}/cuda_bindings/dist")" >> $GITHUB_ENV
-echo "CUDA_BINDINGS_CYTHON_TESTS_DIR=$(realpath "${REPO_DIR}/cuda_bindings/tests/cython")" >> $GITHUB_ENV
 echo "PYTHON_VERSION_FORMATTED=${PYTHON_VERSION_FORMATTED}" >> $GITHUB_ENV
 
 if [[ "${1}" == "build" ]]; then
   echo "CIBW_BUILD=${CIBW_BUILD}" >> $GITHUB_ENV
+  CUDA_BINDINGS_ARTIFACT_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${CUDA_VER}-${HOST_PLATFORM}"
 elif [[ "${1}" == "test" ]]; then
   BUILD_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${BUILD_CUDA_VER})"
   TEST_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${CUDA_VER})"
+  CUDA_BINDINGS_ARTIFACT_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${BUILD_CUDA_VER}-${HOST_PLATFORM}"
   if [[ ${BUILD_CUDA_MAJOR} != ${TEST_CUDA_MAJOR} ]]; then
     SKIP_CUDA_BINDINGS_TEST=1
     SKIP_CYTHON_TEST=1
@@ -63,12 +60,18 @@ elif [[ "${1}" == "test" ]]; then
   # We only test compute-sanitizer on python 3.12 arbitrarily; we don't need to use sanitizer on the entire matrix
   # Only local ctk installs have compute-sanitizer; there is no wheel for it
   if [[ "${PY_VER}" == "3.12" && "${CUDA_VER}" != "11.8.0" && "${LOCAL_CTK}" == 1 && "${HOST_PLATFORM}" == linux* ]]; then
-    SETUP_SANITIZER=1
     echo "LATEST_CUDA_VERSION=$(bash .github/workflows/guess_latest.sh)" >> $GITHUB_ENV
+    SETUP_SANITIZER=1
   else
     SETUP_SANITIZER=0
+    echo "SANITIZER_CMD=" >> $GITHUB_ENV
   fi
   echo "SETUP_SANITIZER=${SETUP_SANITIZER}" >> $GITHUB_ENV
   echo "SKIP_CUDA_BINDINGS_TEST=${SKIP_CUDA_BINDINGS_TEST}" >> $GITHUB_ENV
   echo "SKIP_CYTHON_TEST=${SKIP_CYTHON_TEST}" >> $GITHUB_ENV
 fi
+
+echo "CUDA_BINDINGS_ARTIFACT_BASENAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}" >> $GITHUB_ENV
+echo "CUDA_BINDINGS_ARTIFACT_NAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}-${SHA}" >> $GITHUB_ENV
+echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "${REPO_DIR}/cuda_bindings/dist")" >> $GITHUB_ENV
+echo "CUDA_BINDINGS_CYTHON_TESTS_DIR=$(realpath "${REPO_DIR}/cuda_bindings/tests/cython")" >> $GITHUB_ENV
diff --git a/ci/tools/setup-sanitizer b/ci/tools/setup-sanitizer
new file mode 100755
index 0000000000..7a70d8b2de
--- /dev/null
+++ b/ci/tools/setup-sanitizer
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# A utility script to set up the GitHub environment variables for the CI.
+
+set -euo pipefail
+
+# Setup comppute sanitizer if requested.
+if [[ "${SETUP_SANITIZER}" == 1 ]]; then
+  COMPUTE_SANITIZER="${CUDA_HOME}/bin/compute-sanitizer"
+  COMPUTE_SANITIZER_VERSION=$(${COMPUTE_SANITIZER} --version | grep -Eo "[0-9]{4}\.[0-9]\.[0-9]" | sed -e 's/\.//g')
+  SANITIZER_CMD="${COMPUTE_SANITIZER} --target-processes=all --launch-timeout=0 --tool=memcheck --error-exitcode=1"
+  if [[ "$COMPUTE_SANITIZER_VERSION" -ge 202111 ]]; then
+    SANITIZER_CMD="${SANITIZER_CMD} --padding=32"
+  fi
+  echo "CUDA_PYTHON_TESTING_WITH_COMPUTE_SANITIZER=1" >> $GITHUB_ENV
+else
+  SANITIZER_CMD=""
+fi
+echo "SANITIZER_CMD=${SANITIZER_CMD}" >> $GITHUB_ENV