diff --git a/.github/workflows/build-cuvs-image.yml b/.github/workflows/build-cuvs-image.yml index d6e1b366..40dc2079 100644 --- a/.github/workflows/build-cuvs-image.yml +++ b/.github/workflows/build-cuvs-image.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. +# Copyright (c) 2025-2026, NVIDIA CORPORATION. name: build-cuvs on: @@ -89,6 +89,14 @@ jobs: buildkitd-config: /etc/buildkit/buildkitd.toml driver: docker-container endpoint: builders + - name: Generate Build Args + id: generate-build-args + run: ci/compute-build-args.sh + env: + CUDA_VER: ${{ inputs.CUDA_VER }} + LINUX_VER: ${{ inputs.LINUX_VER }} + PYTHON_VER: ${{ inputs.PYTHON_VER }} + RAPIDS_VER: ${{ inputs.RAPIDS_VER }} - name: Build cuVS Benchmarks GPU image uses: docker/build-push-action@v6 with: @@ -98,10 +106,7 @@ jobs: push: true pull: true build-args: | - CUDA_VER=${{ inputs.CUDA_VER }} - LINUX_VER=${{ inputs.LINUX_VER }} - PYTHON_VER=${{ inputs.PYTHON_VER }} - RAPIDS_VER=${{ inputs.RAPIDS_VER }} + ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }} tags: ${{ inputs.CUVS_BENCH_TAG }}-${{ matrix.ARCH }} # ensure only OCI mediatypes are used: https://docs.docker.com/build/exporters/#oci-media-types outputs: type=registry,oci-mediatypes=true @@ -114,10 +119,7 @@ jobs: # push: true # pull: true # build-args: | - # CUDA_VER=${{ inputs.CUDA_VER }} - # LINUX_VER=${{ inputs.LINUX_VER }} - # PYTHON_VER=${{ inputs.PYTHON_VER }} - # RAPIDS_VER=${{ inputs.RAPIDS_VER }} + # ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }} # tags: ${{ inputs.CUVS_BENCH_DATASETS_TAG }}-${{ matrix.ARCH }} # outputs: type=registry,oci-mediatypes=true - name: Build cuVS Benchmarks CPU image @@ -130,10 +132,7 @@ jobs: push: true pull: true build-args: | - CUDA_VER=${{ inputs.CUDA_VER }} - LINUX_VER=${{ inputs.LINUX_VER }} - PYTHON_VER=${{ inputs.PYTHON_VER }} - RAPIDS_VER=${{ inputs.RAPIDS_VER }} + ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }} tags: ${{ inputs.CUVS_BENCH_CPU_TAG }}-${{ matrix.ARCH }} # ensure only OCI mediatypes are used: https://docs.docker.com/build/exporters/#oci-media-types outputs: type=registry,oci-mediatypes=true diff --git a/.github/workflows/build-rapids-image.yml b/.github/workflows/build-rapids-image.yml index b2ffd040..b4e9d3d8 100644 --- a/.github/workflows/build-rapids-image.yml +++ b/.github/workflows/build-rapids-image.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. +# Copyright (c) 2025-2026, NVIDIA CORPORATION. name: build-rapids on: @@ -88,6 +88,17 @@ jobs: driver: docker-container endpoint: builders version: 'v0.30.1' + - name: Generate Build Args + id: generate-build-args + run: ci/compute-build-args.sh + env: + CPU_ARCH: ${{ matrix.ARCH }} + CUDA_VER: ${{ inputs.CUDA_VER }} + LINUX_DISTRO: ${{ inputs.LINUX_DISTRO }} + LINUX_DISTRO_VER: ${{ inputs.LINUX_DISTRO_VER }} + LINUX_VER: ${{ inputs.LINUX_VER }} + PYTHON_VER: ${{ inputs.PYTHON_VER }} + RAPIDS_VER: ${{ inputs.RAPIDS_VER }} - name: Build base image uses: docker/build-push-action@v6 with: @@ -97,12 +108,7 @@ jobs: push: true pull: true build-args: | - CUDA_VER=${{ inputs.CUDA_VER }} - LINUX_DISTRO=${{ inputs.LINUX_DISTRO }} - LINUX_DISTRO_VER=${{ inputs.LINUX_DISTRO_VER }} - LINUX_VER=${{ inputs.LINUX_VER }} - PYTHON_VER=${{ inputs.PYTHON_VER }} - RAPIDS_VER=${{ inputs.RAPIDS_VER }} + ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }} tags: ${{ inputs.BASE_TAG }}-${{ matrix.ARCH }} # ensure only OCI mediatypes are used: https://docs.docker.com/build/exporters/#oci-media-types outputs: type=registry,oci-mediatypes=true @@ -115,12 +121,7 @@ jobs: push: true pull: true build-args: | - CUDA_VER=${{ inputs.CUDA_VER }} - LINUX_DISTRO=${{ inputs.LINUX_DISTRO }} - LINUX_DISTRO_VER=${{ inputs.LINUX_DISTRO_VER }} - LINUX_VER=${{ inputs.LINUX_VER }} - PYTHON_VER=${{ inputs.PYTHON_VER }} - RAPIDS_VER=${{ inputs.RAPIDS_VER }} + ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }} tags: ${{ inputs.NOTEBOOKS_TAG }}-${{ matrix.ARCH }} # ensure only OCI mediatypes are used: https://docs.docker.com/build/exporters/#oci-media-types outputs: type=registry,oci-mediatypes=true diff --git a/.github/workflows/build-test-publish-images.yml b/.github/workflows/build-test-publish-images.yml index b4fe73ae..771804a0 100644 --- a/.github/workflows/build-test-publish-images.yml +++ b/.github/workflows/build-test-publish-images.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2025, NVIDIA CORPORATION. +# Copyright (c) 2023-2026, NVIDIA CORPORATION. name: build and publish imgs workflow on: @@ -55,8 +55,6 @@ jobs: ci/lint-dockerfiles.sh compute-matrix: runs-on: ubuntu-latest - container: - image: rapidsai/ci-conda:latest outputs: BASE_IMAGE_REPO: ${{ steps.compute-image-repo.outputs.BASE_IMAGE_REPO }} NOTEBOOKS_IMAGE_REPO: ${{ steps.compute-image-repo.outputs.NOTEBOOKS_IMAGE_REPO }} diff --git a/.gitignore b/.gitignore index 0822f81f..f99d452a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,20 @@ .vscode +*.a +*.bz2 +*.dll +*.dylib +*.exe +*.gz +*.pdf +*.png +.ruff_cache/ +*.so +*.tar.gz +*.tgz +*.whl +*.zip -# Created by https://www.gitignore.io/api/macos,linux ### Linux ### *~ .fuse_hidden* @@ -27,5 +40,3 @@ Icon Network Trash Folder Temporary Items .apdisk - -# End of https://www.gitignore.io/api/macos,linux diff --git a/Dockerfile b/Dockerfile index 9c96a6a2..0507fd55 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,58 +1,78 @@ # syntax=docker/dockerfile:1 -# Copyright (c) 2024-2025, NVIDIA CORPORATION. +# Copyright (c) 2024-2026, NVIDIA CORPORATION. -ARG CUDA_VER=unset -ARG PYTHON_VER=unset +ARG CUDA_VER=notset ARG LINUX_DISTRO=ubuntu ARG LINUX_DISTRO_VER=22.04 ARG LINUX_VER=${LINUX_DISTRO}${LINUX_DISTRO_VER} - +ARG PYTHON_VER=notset ARG RAPIDS_VER=26.02 # Gather dependency information - -# ignore hadolint DL3007... we really do always want the latest 'rapidsai/ci-conda', -# and don't want to have to push new commits to update to it -# -# hadolint ignore=DL3007 -FROM rapidsai/ci-conda:${RAPIDS_VER}-latest AS dependencies -ARG CUDA_VER -ARG PYTHON_VER - -ARG RAPIDS_VER - +FROM python:${PYTHON_VER} AS dependencies +ARG CPU_ARCH=notset +ARG CUDA_VER=notset +ARG PYTHON_VER=notset ARG RAPIDS_BRANCH="main" +ARG RAPIDS_VER=notset +ARG YQ_VER=notset SHELL ["/bin/bash", "-euo", "pipefail", "-c"] -RUN pip install --upgrade conda-merge rapids-dependency-file-generator - COPY condarc /condarc COPY notebooks.sh /notebooks.sh +# clone RAPIDS repos and extract the following: +# +# * IPython notebooks (/notebooks) +# * a single conda env YAML with all dependencies needed to run the notebooks (at /test_notebooks_dependencies.yaml) +# RUN <=4.0 is needed for the bit in /notebooks.sh that uses load() to read channels from /condarc +wget -q https://github.com/mikefarah/yq/releases/download/v${YQ_VER}/yq_linux_${CPU_ARCH} -O /tmp/yq +mv /tmp/yq /usr/bin/yq +chmod +x /usr/bin/yq + /notebooks.sh -apt-get purge -y --auto-remove rsync + +apt-get purge -y --auto-remove \ + "${APT_PACKAGES_TO_INSTALL[@]}" + rm -rf /var/lib/apt/lists/* EOF # Base image FROM rapidsai/miniforge-cuda:${RAPIDS_VER}-cuda${CUDA_VER}-base-${LINUX_VER}-py${PYTHON_VER} AS base -ARG CUDA_VER -ARG PYTHON_VER - -ARG RAPIDS_VER +ARG CUDA_VER=notset +ARG PYTHON_VER=notset +ARG RAPIDS_VER=notset SHELL ["/bin/bash", "-euo", "pipefail", "-c"] RUN <=8.37.0' + 'rapids-cli==0.1.*' + 'openssl==3.6.0' +) rapids-mamba-retry install -y -n base \ - "rapids=${RAPIDS_VER}.*" \ - "python=${PYTHON_VER}.*" \ - "cuda-version=${CUDA_VER%.*}.*" \ - 'ipython>=8.37.0' \ - 'rapids-cli==0.1.*' \ - 'openssl==3.6.0' + "${PACKAGES_TO_INSTALL[@]}" + conda clean -afy EOF @@ -100,9 +124,9 @@ CMD ["ipython"] # Notebooks image FROM base AS notebooks -ARG CUDA_VER -ARG LINUX_DISTRO -ARG LINUX_DISTRO_VER +ARG CUDA_VER=notset +ARG LINUX_DISTRO=notset +ARG LINUX_DISTRO_VER=notset SHELL ["/bin/bash", "-euo", "pipefail", "-c"] @@ -120,10 +144,13 @@ conda clean -afy EOF RUN <=7.0.0' + 'jupyterlab-nvdashboard>=0.13.0' +) rapids-mamba-retry install -y -n base \ - "jupyterlab=4" \ - 'dask-labextension>=7.0.0' \ - 'jupyterlab-nvdashboard>=0.13.0' + "${PACKAGES_TO_INSTALL[@]}" conda clean -afy EOF diff --git a/ci/compute-build-args.sh b/ci/compute-build-args.sh new file mode 100755 index 00000000..38ffb60c --- /dev/null +++ b/ci/compute-build-args.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +# start with arguments provided by environment variables. Anything with a fallback is only used by +# a subset of images built here. +ARGS=" +CPU_ARCH: ${CPU_ARCH:-notset} +CUDA_VER: ${CUDA_VER} +LINUX_DISTRO: ${LINUX_DISTRO:-notset} +LINUX_DISTRO_VER: ${LINUX_DISTRO_VER:-notset} +LINUX_VER: ${LINUX_VER} +PYTHON_VER: ${PYTHON_VER} +RAPIDS_VER: ${RAPIDS_VER} +" +export ARGS + +# add in arguments parsed out of 'versions.yaml' +if [ -n "${GITHUB_ACTIONS:-}" ]; then +# on GitHub Actions, create a step output called 'DOCKER_BUILD_ARGS' with this information appended to it. +# +# That can be referenced like: +# +# - name: Build image +# uses: docker/build-push-action@v6 +# with: +# build-args: | +# ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }} +# +cat < "${GITHUB_OUTPUT:-/dev/stdout}" +DOCKER_BUILD_ARGS<[^ ]+) depName=(?[^\\s]+)(?:\\s+versioning=(?[^\\n]+))?\\n(?[^:]+): (?[^\\n]+)" + ], + "depNameTemplate": "{{depName}}", + "datasourceTemplate": "{{datasource}}", + "versioningTemplate": "{{#if versioning}}{{versioning}}{{else}}semver{{/if}}" + } ] } diff --git a/versions.yaml b/versions.yaml new file mode 100644 index 00000000..5ca848c8 --- /dev/null +++ b/versions.yaml @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# renovate: datasource=github-releases depName=mikefarah/yq +YQ_VER: 4.49.2