diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/build-wheel.yml index 1eed96bdb8..386f166a06 100644 --- a/.github/workflows/build-wheel.yml +++ b/.github/workflows/build-wheel.yml @@ -59,29 +59,12 @@ jobs: uses: ilammy/msvc-dev-cmd@v1 # TODO: ask admin to allow pinning commits - name: Set environment variables - run: | - PYTHON_VERSION_FORMATTED=$(echo '${{ matrix.python-version }}' | tr -d '.') - if [[ "${{ inputs.host-platform }}" == linux* ]]; then - CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-manylinux*" - REPO_DIR=$(pwd) - elif [[ "${{ inputs.host-platform }}" == win* ]]; then - CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-win_amd64" - PWD=$(pwd) - REPO_DIR=$(cygpath -w $PWD) - fi - - echo "CUDA_BINDINGS_PARALLEL_LEVEL=$(nproc)" >> $GITHUB_ENV - CUDA_CORE_ARTIFACT_BASENAME="cuda-core-python${PYTHON_VERSION_FORMATTED}-${{ inputs.host-platform }}" - echo "CUDA_CORE_ARTIFACT_BASENAME=${CUDA_CORE_ARTIFACT_BASENAME}" >> $GITHUB_ENV - echo "CUDA_CORE_ARTIFACT_NAME=${CUDA_CORE_ARTIFACT_BASENAME}-${{ github.sha }}" >> $GITHUB_ENV - echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_core/dist")" >> $GITHUB_ENV - echo "CUDA_CORE_CYTHON_TESTS_DIR=$(realpath "$REPO_DIR/cuda_core/tests/cython")" >> $GITHUB_ENV - CUDA_BINDINGS_ARTIFACT_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${{ inputs.cuda-version }}-${{ inputs.host-platform }}" - echo "CUDA_BINDINGS_ARTIFACT_BASENAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}" >> $GITHUB_ENV - echo "CUDA_BINDINGS_ARTIFACT_NAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}-${{ github.sha }}" >> $GITHUB_ENV - echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/dist")" >> $GITHUB_ENV - echo "CUDA_BINDINGS_CYTHON_TESTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/tests/cython")" >> $GITHUB_ENV - echo "CIBW_BUILD=${CIBW_BUILD}" >> $GITHUB_ENV + env: + CUDA_VER: ${{ inputs.cuda-version }} + HOST_PLATFORM: ${{ inputs.host-platform }} + PY_VER: ${{ matrix.python-version }} + SHA: ${{ github.sha }} + run: ./ci/tools/env-vars build - name: Dump environment run: | diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index 8a10a9ef7e..05011504fa 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -154,57 +154,14 @@ jobs: dependent_exes: "jq wget" - name: Set environment variables - run: | - PYTHON_VERSION_FORMATTED=$(echo '${{ matrix.PY_VER }}' | tr -d '.') - if [[ "${{ inputs.host-platform }}" == linux* ]]; then - REPO_DIR=$(pwd) - elif [[ "${{ inputs.host-platform }}" == win* ]]; then - PWD=$(pwd) - REPO_DIR=$(cygpath -w $PWD) - fi - - BUILD_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${{ inputs.build-ctk-ver }})" - TEST_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${{ matrix.CUDA_VER }})" - if [[ $BUILD_CUDA_MAJOR != $TEST_CUDA_MAJOR ]]; then - SKIP_CUDA_BINDINGS_TEST=1 - SKIP_CYTHON_TEST=1 - else - SKIP_CUDA_BINDINGS_TEST=0 - BUILD_CUDA_MINOR="$(cut -d '.' -f 2 <<< ${{ inputs.build-ctk-ver }})" - TEST_CUDA_MINOR="$(cut -d '.' -f 2 <<< ${{ matrix.CUDA_VER }})" - if [[ $BUILD_CUDA_MINOR != $TEST_CUDA_MINOR ]]; then - SKIP_CYTHON_TEST=1 - else - SKIP_CYTHON_TEST=0 - fi - fi - - # We don't test compute-sanitizer on CTK<12 because backporting fixes is too much effort - # We only test compute-sanitizer on python 3.12 arbitrarily; we don't need to use sanitizer on the entire matrix - # Only local ctk installs have compute-sanitizer; there is not wheel for it - if [[ "${{ inputs.python-version }}" == "3.12" && "${{ inputs.cuda-version }}" != "11.8.0" && "${{ inputs.local-ctk }}" == 1 ]]; then - SETUP_SANITIZER=1 - echo "LATEST_CUDA_VERSION=$(bash .github/workflows/guess_latest.sh)" >> $GITHUB_ENV - else - SETUP_SANITIZER=0 - fi - echo "SETUP_SANITIZER=${SETUP_SANITIZER}" >> $GITHUB_ENV - - # make outputs from the previous job as env vars - CUDA_CORE_ARTIFACT_BASENAME="cuda-core-python${PYTHON_VERSION_FORMATTED}-${{ inputs.host-platform }}" - echo "PYTHON_VERSION_FORMATTED=${PYTHON_VERSION_FORMATTED}" >> $GITHUB_ENV - echo "CUDA_CORE_ARTIFACT_BASENAME=${CUDA_CORE_ARTIFACT_BASENAME}" >> $GITHUB_ENV - echo "CUDA_CORE_ARTIFACT_NAME=${CUDA_CORE_ARTIFACT_BASENAME}-${{ github.sha }}" >> $GITHUB_ENV - echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_core/dist")" >> $GITHUB_ENV - echo "CUDA_CORE_CYTHON_TESTS_DIR=$(realpath "$REPO_DIR/cuda_core/tests/cython")" >> $GITHUB_ENV - CUDA_BINDINGS_ARTIFACT_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${{ inputs.build-ctk-ver }}-${{ inputs.host-platform }}" - echo "CUDA_BINDINGS_ARTIFACT_BASENAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}" >> $GITHUB_ENV - echo "CUDA_BINDINGS_ARTIFACT_NAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}-${{ github.sha }}" >> $GITHUB_ENV - echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/dist")" >> $GITHUB_ENV - echo "CUDA_BINDINGS_CYTHON_TESTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/tests/cython")" >> $GITHUB_ENV - - echo "SKIP_CUDA_BINDINGS_TEST=${SKIP_CUDA_BINDINGS_TEST}" >> $GITHUB_ENV - echo "SKIP_CYTHON_TEST=${SKIP_CYTHON_TEST}" >> $GITHUB_ENV + env: + BUILD_CUDA_VER: ${{ inputs.build-ctk-ver }} + CUDA_VER: ${{ matrix.CUDA_VER }} + HOST_PLATFORM: ${{ inputs.host-platform }} + LOCAL_CTK: ${{ matrix.LOCAL_CTK }} + PY_VER: ${{ matrix.PY_VER }} + SHA: ${{ github.sha }} + run: ./ci/tools/env-vars test - name: Download cuda-python build artifacts if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0'}} @@ -327,66 +284,20 @@ jobs: cuda-components: "cuda_sanitizer_api" - name: Set up compute-sanitizer - run: | - if [[ "${SETUP_SANITIZER}" == 1 ]]; then - COMPUTE_SANITIZER="${CUDA_HOME}/bin/compute-sanitizer" - COMPUTE_SANITIZER_VERSION=$(${COMPUTE_SANITIZER} --version | grep -Eo "[0-9]{4}\.[0-9]\.[0-9]" | sed -e 's/\.//g') - SANITIZER_CMD="${COMPUTE_SANITIZER} --target-processes=all --launch-timeout=0 --tool=memcheck --error-exitcode=1" - if [[ "$COMPUTE_SANITIZER_VERSION" -ge 202111 ]]; then - SANITIZER_CMD="${SANITIZER_CMD} --padding=32" - fi - echo "CUDA_PYTHON_TESTING_WITH_COMPUTE_SANITIZER=1" >> $GITHUB_ENV - else - SANITIZER_CMD="" - fi - echo "SANITIZER_CMD=${SANITIZER_CMD}" >> $GITHUB_ENV + run: setup-sanitizer - name: Run cuda.bindings tests if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0' }} - run: | - pushd "${CUDA_BINDINGS_ARTIFACTS_DIR}" - if [[ "${{ matrix.LOCAL_CTK }}" == 1 ]]; then - ls $CUDA_PATH - pip install $(ls *.whl)[test] - else - pip install $(ls *.whl)[all,test] - fi - popd - - pushd ./cuda_bindings - ${SANITIZER_CMD} pytest -rxXs -v tests/ - if [[ "${SKIP_CYTHON_TEST}" == 0 ]]; then - ${SANITIZER_CMD} pytest -rxXs -v tests/cython - fi - popd + env: + CUDA_VER: ${{ matrix.CUDA_VER }} + LOCAL_CTK: ${{ matrix.LOCAL_CTK }} + run: run-tests bindings - name: Run cuda.core tests - run: | - # If build/test majors match: cuda.bindings is installed in the previous step. - # If mismatch: cuda.bindings is installed from the backport branch. - if [[ "${SKIP_CUDA_BINDINGS_TEST}" == 1 ]]; then - pushd "${CUDA_BINDINGS_ARTIFACTS_DIR}" - if [[ "${{ matrix.LOCAL_CTK }}" == 1 ]]; then - pip install *.whl - else - pip install $(ls *.whl)[all] - fi - popd - fi - TEST_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${{ matrix.CUDA_VER }})" - pushd "${CUDA_CORE_ARTIFACTS_DIR}" - pip install $(ls *.whl)["cu${TEST_CUDA_MAJOR}","test-cu${TEST_CUDA_MAJOR}"] - popd - - pushd ./cuda_core - ${SANITIZER_CMD} pytest -rxXs -v tests/ - - # Currently our CI always installs the latest bindings (from either major version). - # This is not compatible with the test requirements. - if [[ "${SKIP_CYTHON_TEST}" == 0 ]]; then - ${SANITIZER_CMD} pytest -rxXs -v tests/cython - fi - popd + env: + CUDA_VER: ${{ matrix.CUDA_VER }} + LOCAL_CTK: ${{ matrix.LOCAL_CTK }} + run: run-tests core - name: Ensure cuda-python installable run: | diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index 04ed9f341c..f5c6db6fc7 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -96,41 +96,30 @@ jobs: - name: Ensure GPU is working run: nvidia-smi - - name: Set environment variables + - name: Install Git for Windows + # the GPU runner image does not have Git Bash pre-installed... + env: + # doesn't seem there's an easy way to avoid hard-coding it? + GFW_EXE_URL: https://github.com/git-for-windows/git/releases/download/v2.49.0.windows.1/PortableGit-2.49.0-64-bit.7z.exe run: | - $PYTHON_VERSION_FORMATTED = '${{ matrix.PY_VER }}' -replace '\.' - $REPO_DIR = $PWD.Path - - $BUILD_CUDA_MAJOR = '${{ inputs.build-ctk-ver }}' -split '\.' | Select-Object -First 1 - $TEST_CUDA_MAJOR = '${{ matrix.CUDA_VER }}' -split '\.' | Select-Object -First 1 - if ($BUILD_CUDA_MAJOR -ne $TEST_CUDA_MAJOR) { - $SKIP_CUDA_BINDINGS_TEST = 1 - $SKIP_CYTHON_TEST = 1 - } else { - $SKIP_CUDA_BINDINGS_TEST = 0 - $BUILD_CUDA_MINOR = '${{ inputs.build-ctk-ver }}' -split '\.' | Select-Object -Skip 1 -First 1 - $TEST_CUDA_MINOR = '${{ matrix.CUDA_VER }}' -split '\.' | Select-Object -Skip 1 -First 1 - if ($BUILD_CUDA_MINOR -ne $TEST_CUDA_MINOR) { - $SKIP_CYTHON_TEST = 1 - } else { - $SKIP_CYTHON_TEST = 0 - } - } + Invoke-WebRequest -Uri "$env:GFW_EXE_URL" -OutFile "PortableGit.7z.exe" + # Self-extracting, see https://gitforwindows.org/zip-archives-extracting-the-released-archives.html + Start-Process .\PortableGit.7z.exe -Wait -Verbose -ArgumentList '-y -gm2' + ls -l PortableGit + echo "$((Get-Location).Path)\\PortableGit\\bin" >> $env:GITHUB_PATH + $env:Path += ";$((Get-Location).Path)\\PortableGit\\bin" + bash --version - # Make outputs from the previous job as env vars - $CUDA_CORE_ARTIFACT_BASENAME = "cuda-core-python${PYTHON_VERSION_FORMATTED}-${{ inputs.host-platform }}" - "PYTHON_VERSION_FORMATTED=${PYTHON_VERSION_FORMATTED}" >> $env:GITHUB_ENV - "CUDA_CORE_ARTIFACT_BASENAME=${CUDA_CORE_ARTIFACT_BASENAME}" >> $env:GITHUB_ENV - "CUDA_CORE_ARTIFACT_NAME=${CUDA_CORE_ARTIFACT_BASENAME}-${{ github.sha }}" >> $env:GITHUB_ENV - "CUDA_CORE_ARTIFACTS_DIR=$($ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath("$REPO_DIR\cuda_core\dist"))" >> $env:GITHUB_ENV - "CUDA_CORE_CYTHON_TESTS_DIR=$($ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath("$REPO_DIR\cuda_core\tests\cython"))" >> $env:GITHUB_ENV - $CUDA_BINDINGS_ARTIFACT_BASENAME = "cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${{ inputs.build-ctk-ver }}-${{ inputs.host-platform }}" - "CUDA_BINDINGS_ARTIFACT_BASENAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}" >> $env:GITHUB_ENV - "CUDA_BINDINGS_ARTIFACT_NAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}-${{ github.sha }}" >> $env:GITHUB_ENV - "CUDA_BINDINGS_ARTIFACTS_DIR=$($ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath("$REPO_DIR\cuda_bindings\dist"))" >> $env:GITHUB_ENV - "CUDA_BINDINGS_CYTHON_TESTS_DIR=$($ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath("$REPO_DIR\cuda_bindings\tests\cython"))" >> $env:GITHUB_ENV - "SKIP_CUDA_BINDINGS_TEST=${SKIP_CUDA_BINDINGS_TEST}" >> $env:GITHUB_ENV - "SKIP_CYTHON_TEST=${SKIP_CYTHON_TEST}" >> $env:GITHUB_ENV + - name: Set environment variables + env: + BUILD_CUDA_VER: ${{ inputs.build-ctk-ver }} + CUDA_VER: ${{ matrix.CUDA_VER }} + HOST_PLATFORM: ${{ inputs.host-platform }} + LOCAL_CTK: ${{ matrix.LOCAL_CTK }} + PY_VER: ${{ matrix.PY_VER }} + SHA: ${{ github.sha }} + shell: bash --noprofile --norc -xeuo pipefail {0} + run: ./ci/tools/env-vars test - name: Download cuda-python build artifacts if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0'}} @@ -161,21 +150,6 @@ jobs: } gh --version - - name: Install Git for Windows - # the GPU runner image does not have Git Bash pre-installed... - if: ${{ matrix.LOCAL_CTK == '1' }} - env: - # doesn't seem there's an easy way to avoid hard-coding it? - GFW_EXE_URL: https://github.com/git-for-windows/git/releases/download/v2.49.0.windows.1/PortableGit-2.49.0-64-bit.7z.exe - run: | - Invoke-WebRequest -Uri "$env:GFW_EXE_URL" -OutFile "PortableGit.7z.exe" - # Self-extracting, see https://gitforwindows.org/zip-archives-extracting-the-released-archives.html - Start-Process .\PortableGit.7z.exe -Wait -Verbose -ArgumentList '-y -gm2' - ls -l PortableGit - echo "$((Get-Location).Path)\\PortableGit\\bin" >> $env:GITHUB_PATH - $env:Path += ";$((Get-Location).Path)\\PortableGit\\bin" - bash --version - - name: Install zstd # the GPU runner image does not have zstd pre-installed... and it's needed by actions/cache if: ${{ matrix.LOCAL_CTK == '1' }} @@ -278,48 +252,18 @@ jobs: - name: Run cuda.bindings tests if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0' }} - run: | - Push-Location $env:CUDA_BINDINGS_ARTIFACTS_DIR - if ('${{ matrix.LOCAL_CTK }}' -eq '1') { - Get-ChildItem $env:CUDA_PATH - echo $PATH - pip install "$((Get-ChildItem -Filter *.whl).FullName)[test]" - } else { - pip install "$((Get-ChildItem -Filter *.whl).FullName)[all,test]" - } - Pop-Location - - Push-Location ./cuda_bindings - pytest -rxXs -v tests/ - if ($env:SKIP_CYTHON_TEST -eq '0') { - pytest -rxXs -v tests/cython - } - Pop-Location + env: + CUDA_VER: ${{ matrix.CUDA_VER }} + LOCAL_CTK: ${{ matrix.LOCAL_CTK }} + shell: bash --noprofile --norc -xeuo pipefail {0} + run: run-tests bindings - name: Run cuda.core tests - run: | - # If build/test majors match: cuda.bindings is installed in the previous step. - # If mismatch: cuda.bindings is installed from the backport branch. - if ($env:SKIP_CUDA_BINDINGS_TEST -eq '1') { - Push-Location $env:CUDA_BINDINGS_ARTIFACTS_DIR - if ('${{ matrix.LOCAL_CTK }}' -eq '1') { - pip install (Get-ChildItem -Filter *.whl).FullName - } else { - pip install "$((Get-ChildItem -Filter *.whl).FullName)[all]" - } - Pop-Location - } - $TEST_CUDA_MAJOR = '${{ matrix.CUDA_VER }}' -split '\.' | Select-Object -First 1 - Push-Location $env:CUDA_CORE_ARTIFACTS_DIR - pip install "$((Get-ChildItem -Filter *.whl).FullName)[cu${TEST_CUDA_MAJOR},test-cu${TEST_CUDA_MAJOR}]" - Pop-Location - - Push-Location ./cuda_core - pytest -rxXs -v tests/ - if ($env:SKIP_CYTHON_TEST -eq '0') { - pytest -rxXs -v tests/cython - } - Pop-Location + env: + CUDA_VER: ${{ matrix.CUDA_VER }} + LOCAL_CTK: ${{ matrix.LOCAL_CTK }} + shell: bash --noprofile --norc -xeuo pipefail {0} + run: run-tests core - name: Ensure cuda-python installable run: | diff --git a/ci/tools/env-vars b/ci/tools/env-vars new file mode 100755 index 0000000000..7211dc08fc --- /dev/null +++ b/ci/tools/env-vars @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. +# +# SPDX-License-Identifier: Apache-2.0 + +# A utility script to set up the GitHub environment variables for the CI. + +set -euo pipefail + +# Check if the script was called with exactly 1 argument +if [[ ${#} -ne 1 ]]; then + echo "Error: This script requires exactly 1 argument (the build mode). You provided ${#}" + echo "Usage: ${0} build_mode[build or test]" + exit 1 +fi + +PYTHON_VERSION_FORMATTED=$(echo "${PY_VER}" | tr -d '.') +if [[ "${HOST_PLATFORM}" == linux* ]]; then + CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-manylinux*" + REPO_DIR=$(pwd) + TOOLS_PATH="${REPO_DIR}/ci/tools" +elif [[ "${HOST_PLATFORM}" == win* ]]; then + CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-win_amd64" + PWD=$(pwd) + REPO_DIR=$(cygpath -w ${PWD}) + TOOLS_PATH=$(cygpath -w ${PWD}/ci/tools) +fi + +echo "${TOOLS_PATH}" >> $GITHUB_PATH +echo "CUDA_BINDINGS_PARALLEL_LEVEL=$(nproc)" >> $GITHUB_ENV +CUDA_CORE_ARTIFACT_BASENAME="cuda-core-python${PYTHON_VERSION_FORMATTED}-${HOST_PLATFORM}" +echo "CUDA_CORE_ARTIFACT_BASENAME=${CUDA_CORE_ARTIFACT_BASENAME}" >> $GITHUB_ENV +echo "CUDA_CORE_ARTIFACT_NAME=${CUDA_CORE_ARTIFACT_BASENAME}-${SHA}" >> $GITHUB_ENV +echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "${REPO_DIR}/cuda_core/dist")" >> $GITHUB_ENV +echo "CUDA_CORE_CYTHON_TESTS_DIR=$(realpath "${REPO_DIR}/cuda_core/tests/cython")" >> $GITHUB_ENV +echo "PYTHON_VERSION_FORMATTED=${PYTHON_VERSION_FORMATTED}" >> $GITHUB_ENV + +if [[ "${1}" == "build" ]]; then + echo "CIBW_BUILD=${CIBW_BUILD}" >> $GITHUB_ENV + CUDA_BINDINGS_ARTIFACT_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${CUDA_VER}-${HOST_PLATFORM}" +elif [[ "${1}" == "test" ]]; then + BUILD_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${BUILD_CUDA_VER})" + TEST_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${CUDA_VER})" + CUDA_BINDINGS_ARTIFACT_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${BUILD_CUDA_VER}-${HOST_PLATFORM}" + if [[ ${BUILD_CUDA_MAJOR} != ${TEST_CUDA_MAJOR} ]]; then + SKIP_CUDA_BINDINGS_TEST=1 + SKIP_CYTHON_TEST=1 + else + SKIP_CUDA_BINDINGS_TEST=0 + BUILD_CUDA_MINOR="$(cut -d '.' -f 2 <<< ${BUILD_CUDA_VER})" + TEST_CUDA_MINOR="$(cut -d '.' -f 2 <<< ${CUDA_VER})" + if [[ ${BUILD_CUDA_MINOR} != ${TEST_CUDA_MINOR} ]]; then + SKIP_CYTHON_TEST=1 + else + SKIP_CYTHON_TEST=0 + fi + fi + # We don't test compute-sanitizer on CTK<12 because backporting fixes is too much effort + # We only test compute-sanitizer on python 3.12 arbitrarily; we don't need to use sanitizer on the entire matrix + # Only local ctk installs have compute-sanitizer; there is no wheel for it + if [[ "${PY_VER}" == "3.12" && "${CUDA_VER}" != "11.8.0" && "${LOCAL_CTK}" == 1 && "${HOST_PLATFORM}" == linux* ]]; then + echo "LATEST_CUDA_VERSION=$(bash .github/workflows/guess_latest.sh)" >> $GITHUB_ENV + SETUP_SANITIZER=1 + else + SETUP_SANITIZER=0 + echo "SANITIZER_CMD=" >> $GITHUB_ENV + fi + echo "SETUP_SANITIZER=${SETUP_SANITIZER}" >> $GITHUB_ENV + echo "SKIP_CUDA_BINDINGS_TEST=${SKIP_CUDA_BINDINGS_TEST}" >> $GITHUB_ENV + echo "SKIP_CYTHON_TEST=${SKIP_CYTHON_TEST}" >> $GITHUB_ENV +fi + +echo "CUDA_BINDINGS_ARTIFACT_BASENAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}" >> $GITHUB_ENV +echo "CUDA_BINDINGS_ARTIFACT_NAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}-${SHA}" >> $GITHUB_ENV +echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "${REPO_DIR}/cuda_bindings/dist")" >> $GITHUB_ENV +echo "CUDA_BINDINGS_CYTHON_TESTS_DIR=$(realpath "${REPO_DIR}/cuda_bindings/tests/cython")" >> $GITHUB_ENV diff --git a/ci/tools/run-tests b/ci/tools/run-tests new file mode 100755 index 0000000000..51fcf0a6b4 --- /dev/null +++ b/ci/tools/run-tests @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. +# +# SPDX-License-Identifier: Apache-2.0 + +# A utility script to install the correct packages and run the tests. + +set -euo pipefail + +# Check if the script was called with exactly 1 argument +if [[ ${#} -ne 1 && ( "${1}" == "bindings" || "${1}" == "core" ) ]]; then + echo "Error: This script requires exactly 1 argument (what is tested). You provided ${#}" + echo "Usage: ${0} test_module[bindings or core]" + exit 1 +fi + +test_module=${1} + +if [[ "${test_module}" == "bindings" ]]; then + pushd "${CUDA_BINDINGS_ARTIFACTS_DIR}" + echo "Installing bindings wheel" + pwd + ls + if [[ "${LOCAL_CTK}" == 1 ]]; then + ls "${CUDA_PATH}" + pip install $(ls *.whl)[test] + else + pip install $(ls *.whl)[all,test] + fi + popd + pushd ./cuda_bindings + echo "Running bindinds tests" + pwd + ${SANITIZER_CMD} pytest -rxXs -v tests/ + if [[ "${SKIP_CYTHON_TEST}" == 0 ]]; then + ${SANITIZER_CMD} pytest -rxXs -v tests/cython + fi + popd +elif [[ "${test_module}" == "core" ]]; then + # If build/test majors match: cuda.bindings is installed in the previous step. + # If mismatch: cuda.bindings is installed from the backport branch. + if [[ "${SKIP_CUDA_BINDINGS_TEST}" == 1 ]]; then + pushd "${CUDA_BINDINGS_ARTIFACTS_DIR}" + echo "Installing bindings wheel" + pwd + ls + if [[ "${LOCAL_CTK}" == 1 ]]; then + pip install *.whl + else + pip install $(ls *.whl)[all] + fi + popd + fi + TEST_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${CUDA_VER})" + pushd "${CUDA_CORE_ARTIFACTS_DIR}" + echo "Installing core wheel" + pwd + ls + pip install $(ls *.whl)["cu${TEST_CUDA_MAJOR}","test-cu${TEST_CUDA_MAJOR}"] + popd + pushd ./cuda_core + echo "Running core tests" + pwd + ${SANITIZER_CMD} pytest -rxXs -v tests/ + # Currently our CI always installs the latest bindings (from either major version). + # This is not compatible with the test requirements. + if [[ "${SKIP_CYTHON_TEST}" == 0 ]]; then + ${SANITIZER_CMD} pytest -rxXs -v tests/cython + fi + popd +fi \ No newline at end of file diff --git a/ci/tools/setup-sanitizer b/ci/tools/setup-sanitizer new file mode 100755 index 0000000000..7a70d8b2de --- /dev/null +++ b/ci/tools/setup-sanitizer @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. +# +# SPDX-License-Identifier: Apache-2.0 + +# A utility script to set up the GitHub environment variables for the CI. + +set -euo pipefail + +# Setup comppute sanitizer if requested. +if [[ "${SETUP_SANITIZER}" == 1 ]]; then + COMPUTE_SANITIZER="${CUDA_HOME}/bin/compute-sanitizer" + COMPUTE_SANITIZER_VERSION=$(${COMPUTE_SANITIZER} --version | grep -Eo "[0-9]{4}\.[0-9]\.[0-9]" | sed -e 's/\.//g') + SANITIZER_CMD="${COMPUTE_SANITIZER} --target-processes=all --launch-timeout=0 --tool=memcheck --error-exitcode=1" + if [[ "$COMPUTE_SANITIZER_VERSION" -ge 202111 ]]; then + SANITIZER_CMD="${SANITIZER_CMD} --padding=32" + fi + echo "CUDA_PYTHON_TESTING_WITH_COMPUTE_SANITIZER=1" >> $GITHUB_ENV +else + SANITIZER_CMD="" +fi +echo "SANITIZER_CMD=${SANITIZER_CMD}" >> $GITHUB_ENV