diff --git a/.github/workflows/release-to-nvstaging.yml b/.github/workflows/release-to-nvstaging.yml index 239782f0..863096d6 100644 --- a/.github/workflows/release-to-nvstaging.yml +++ b/.github/workflows/release-to-nvstaging.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2025, NVIDIA CORPORATION. +# Copyright (c) 2024-2026, NVIDIA CORPORATION. name: Publish release images to NGC diff --git a/.github/workflows/test-notebooks.yml b/.github/workflows/test-notebooks.yml index bfb9353a..1abc0713 100644 --- a/.github/workflows/test-notebooks.yml +++ b/.github/workflows/test-notebooks.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2025, NVIDIA CORPORATION. +# Copyright (c) 2023-2026, NVIDIA CORPORATION. name: Test notebooks on: @@ -45,9 +45,6 @@ permissions: security-events: none statuses: none -env: - GHA_TOOLS_DIR: /home/rapids/.local/bin # Workaround for https://github.com/actions/runner/issues/2411 - jobs: test: strategy: @@ -77,20 +74,18 @@ jobs: mamba install -n base --freeze-installed \ curl \ git - - name: Install gha-tools - run: | - mkdir -p "$GHA_TOOLS_DIR" - curl -s -L 'https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz' | tar -xz -C "$GHA_TOOLS_DIR" - name: Checkout code uses: actions/checkout@v6 with: fetch-depth: 0 + - name: Install gha-tools + run: | + context/scripts/install-gha-tools - name: Get RAPIDS GitHub Info id: get-rapids-github-info uses: rapidsai/shared-actions/rapids-github-info@main - name: Print environment run: | - PATH="$PATH:$GHA_TOOLS_DIR" rapids-print-env rapids-logger "nvidia-smi" nvidia-smi @@ -99,7 +94,6 @@ jobs: - name: Install awscli if: '!cancelled()' run: | - PATH="$PATH:$GHA_TOOLS_DIR" rapids-mamba-retry install -n base awscli - uses: aws-actions/configure-aws-credentials@v5 if: '!cancelled()' @@ -110,5 +104,4 @@ jobs: - name: Upload notebook test outputs if: '!cancelled()' run: | - PATH="$PATH:$GHA_TOOLS_DIR" rapids-upload-to-s3 test_notebooks_output_${{ inputs.ARCH }}_cuda${{ inputs.CUDA_VER }}_py${{ inputs.PYTHON_VER }}_${{ inputs.GPU }}-${{ inputs.DRIVER }}.tar.gz /home/rapids/notebooks_output diff --git a/Dockerfile b/Dockerfile index 7fab8d1b..cbcce6b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,22 +65,11 @@ ARG MINIFORGE_VER=notset FROM condaforge/miniforge3:${MINIFORGE_VER} AS miniforge-upstream -SHELL ["/bin/bash", "-euo", "pipefail", "-c"] - -RUN <==' >> /opt/conda/conda-meta/pinned - -# update everything before other environment changes, to ensure mixing -# an older conda with newer packages still works well -PATH="/opt/conda/bin:$PATH" \ - rapids-mamba-retry update --all -y -n base +RUN \ + --mount=type=bind,source=scripts,target=/tmp/build-scripts \ +< /etc/apt/apt.conf.d/warnings-as-errors -echo 'APT::Acquire::Retries "10";' > /etc/apt/apt.conf.d/retries -echo 'APT::Acquire::https::Timeout "240";' > /etc/apt/apt.conf.d/https-timeout -echo 'APT::Acquire::http::Timeout "240";' > /etc/apt/apt.conf.d/http-timeout -EOF - -# Install gha-tools -RUN <= 5)) && break; sleep 10; done - apt-get install -y --no-install-recommends wget - wget -q https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz -O - | tar -xz -C /usr/local/bin - apt-get purge -y wget && apt-get autoremove -y - rm -rf /var/lib/apt/lists/* -EOF - # Create a conda group and assign it as root's primary group RUN <' to compare decimals here, but it works as expected for the 'bash' version in these -# images, and installing 'bc' or using a Python interpreter seem heavy for this purpose. -# -# shellcheck disable=SC2072 -if [[ "$PYTHON_VERSION_PADDED" > "3.12" ]]; then - PYTHON_ABI_TAG="cp${PYTHON_MAJOR_VERSION}${PYTHON_MINOR_VERSION}" -else - PYTHON_ABI_TAG="cpython" -fi -rapids-mamba-retry install -y -n base "python>=${PYTHON_VERSION},<${PYTHON_UPPER_BOUND}=*_${PYTHON_ABI_TAG}" -rapids-mamba-retry update --all -y -n base -find /opt/conda -follow -type f -name '*.a' -delete -find /opt/conda -follow -type f -name '*.pyc' -delete -# recreate missing libstdc++ symlinks -conda clean -aiptfy -EOF - -# Reassign root's primary group to root -RUN usermod -g root root - -RUN <> /etc/skel/.bashrc -echo ". /opt/conda/etc/profile.d/conda.sh; conda activate base" >> ~/.bashrc -EOF +RUN \ + --mount=type=bind,source=scripts,target=/tmp/build-scripts \ +<' to compare decimals here, but it works as expected for the 'bash' version in these -# images, and installing 'bc' or using a Python interpreter seem heavy for this purpose. -# -# shellcheck disable=SC2072 -if [[ "${os_version}" > "24.04" ]] || [[ "${os_version}" == "24.04" ]]; then - PACKAGES_TO_INSTALL+=(tzdata-legacy) -fi - -rapids-retry apt-get update -y -apt-get upgrade -y -apt-get install -y --no-install-recommends \ - "${PACKAGES_TO_INSTALL[@]}" - -rm -rf "/var/lib/apt/lists/*" +# install tzdata system packages +/tmp/build-scripts/install-tzdata-packages EOF # --- end 'rapidsai/miniforge-cuda' --- # @@ -198,7 +120,10 @@ ARG RAPIDS_VER=26.04 SHELL ["/bin/bash", "-euo", "pipefail", "-c"] -RUN < /etc/apt/apt.conf.d/warnings-as-errors +echo 'APT::Acquire::Retries "10";' > /etc/apt/apt.conf.d/retries +echo 'APT::Acquire::https::Timeout "240";' > /etc/apt/apt.conf.d/https-timeout +echo 'APT::Acquire::http::Timeout "240";' > /etc/apt/apt.conf.d/http-timeout diff --git a/context/scripts/configure-conda-base-environment b/context/scripts/configure-conda-base-environment new file mode 100755 index 00000000..8aac04c5 --- /dev/null +++ b/context/scripts/configure-conda-base-environment @@ -0,0 +1,50 @@ +#!/bin/bash + +# [description] +# +# Configures the image's environment so users can be productive in the 'base' conda env. +# For example: +# +# * installs the expected Python version +# * ensures the 'base' environment is always activated by default +# + +set -e -u -o pipefail + +# Ensure new files are created with group write access & setgid. See https://unix.stackexchange.com/a/12845 +chmod g+ws /opt/conda + +# Ensure new files/dirs have group write permissions +umask 002 + +# install expected Python version +PYTHON_MAJOR_VERSION=${PYTHON_VERSION%%.*} +PYTHON_MINOR_VERSION=${PYTHON_VERSION#*.} +PYTHON_UPPER_BOUND="${PYTHON_MAJOR_VERSION}.$((PYTHON_MINOR_VERSION+1)).0a0" +PYTHON_MINOR_PADDED=$(printf "%02d" "$PYTHON_MINOR_VERSION") +PYTHON_VERSION_PADDED="${PYTHON_MAJOR_VERSION}.${PYTHON_MINOR_PADDED}" + +# 'shellcheck' is unhappy with the use of '>' to compare decimals here, but it works as expected for the 'bash' version in these +# images, and installing 'bc' or using a Python interpreter seem heavy for this purpose. +# +# shellcheck disable=SC2072 +if [[ "$PYTHON_VERSION_PADDED" > "3.12" ]]; then + PYTHON_ABI_TAG="cp${PYTHON_MAJOR_VERSION}${PYTHON_MINOR_VERSION}" +else + PYTHON_ABI_TAG="cpython" +fi +rapids-mamba-retry install -y -n base "python>=${PYTHON_VERSION},<${PYTHON_UPPER_BOUND}=*_${PYTHON_ABI_TAG}" +rapids-mamba-retry update --all -y -n base +find /opt/conda -follow -type f -name '*.a' -delete +find /opt/conda -follow -type f -name '*.pyc' -delete + +# recreate missing libstdc++ symlinks +conda clean -aiptfy + +# Reassign root's primary group to root +usermod -g root root + +# ensure conda environment is always activated +ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh +echo ". /opt/conda/etc/profile.d/conda.sh; conda activate base" >> /etc/skel/.bashrc +echo ". /opt/conda/etc/profile.d/conda.sh; conda activate base" >> ~/.bashrc diff --git a/context/scripts/install-gha-tools b/context/scripts/install-gha-tools new file mode 100755 index 00000000..6f7012df --- /dev/null +++ b/context/scripts/install-gha-tools @@ -0,0 +1,35 @@ +#!/bin/bash + +# [description] +# +# Installs 'gha-tools' and puts it at /usr/local/bin so no PATH modification is required. +# + +set -e -u -o pipefail + +GHA_TOOLS_INSTALL_DIR=/usr/local/bin + +# install wget +wget_preinstalled=true + +if ! type -f wget >/dev/null; then + echo "could not find 'wget', installing it" + wget_preinstalled=false + i=0 + until apt-get update -y; do + ((++i >= 5)) && break; + sleep 10; + done + apt-get install -y --no-install-recommends \ + wget +fi + +# install gha-tools +wget -q https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz -O - | tar -xz -C "${GHA_TOOLS_INSTALL_DIR}" + +# uninstall wget +if [[ "${wget_preinstalled}" != "true" ]]; then + apt-get purge -y wget + apt-get autoremove -y + rm -rf /var/lib/apt/lists/* +fi diff --git a/context/scripts/install-tzdata-packages b/context/scripts/install-tzdata-packages new file mode 100755 index 00000000..5281d05e --- /dev/null +++ b/context/scripts/install-tzdata-packages @@ -0,0 +1,30 @@ +#!/bin/bash + +# [description] +# +# tzdata is needed by the ORC library used by pyarrow, because it provides /etc/localtime +# On Ubuntu 24.04 and newer, we also need tzdata-legacy +# + +set -e -u -o pipefail + +PACKAGES_TO_INSTALL=( + tzdata +) + +os_version=$(grep 'VERSION_ID' /etc/os-release | cut -d '"' -f 2) + +# 'shellcheck' is unhappy with the use of '>' to compare decimals here, but it works as expected for the 'bash' version in these +# images, and installing 'bc' or using a Python interpreter seem heavy for this purpose. +# +# shellcheck disable=SC2072 +if [[ "${os_version}" > "24.04" ]] || [[ "${os_version}" == "24.04" ]]; then + PACKAGES_TO_INSTALL+=(tzdata-legacy) +fi + +rapids-retry apt-get update -y +apt-get upgrade -y +apt-get install -y --no-install-recommends \ + "${PACKAGES_TO_INSTALL[@]}" + +rm -rf "/var/lib/apt/lists/*" diff --git a/context/scripts/update-base-conda-environment b/context/scripts/update-base-conda-environment new file mode 100755 index 00000000..13d02528 --- /dev/null +++ b/context/scripts/update-base-conda-environment @@ -0,0 +1,30 @@ +#!/bin/bash + +set -e -u -o pipefail + +# [description] +# +# Updates the 'base' conda environment that ships in the condaforge/miniforge3 image, +# to ensure everything is up to date before it's copied into later images. +# + +# expect to find other scripts in the same directory as this one +SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" + +# Ensure new files/dirs have group write permissions +umask 002 + +# install gha-tools for rapids-mamba-retry +"${SCRIPT_DIR}/install-gha-tools" + +# Example of pinned package in case you require an override +# echo '==' >> /opt/conda/conda-meta/pinned + +# update everything before other environment changes, to ensure mixing +# an older conda with newer packages still works well +PATH="/opt/conda/bin:$PATH" \ + rapids-mamba-retry update --all -y -n base + +# clean up the package cache and other unused files, to minimize +# what's copied into later images +conda clean -aiptfy diff --git a/cuvs-bench/cpu/Dockerfile b/cuvs-bench/cpu/Dockerfile index 7da81365..b8ed05fc 100644 --- a/cuvs-bench/cpu/Dockerfile +++ b/cuvs-bench/cpu/Dockerfile @@ -22,17 +22,23 @@ chmod 777 /data echo ". /opt/conda/etc/profile.d/conda.sh; conda activate base" >> /etc/bash.bashrc EOF -# we need perl temporarily for the remaining benchmark perl scripts -RUN <==' >> /opt/conda/conda-meta/pinned - -# update everything before other environment changes, to ensure mixing -# an older conda with newer packages still works well -PATH="/opt/conda/bin:$PATH" \ - rapids-mamba-retry update --all -y -n base +RUN \ + --mount=type=bind,source=scripts,target=/tmp/build-scripts \ +< /etc/apt/apt.conf.d/warnings-as-errors -echo 'APT::Acquire::Retries "10";' > /etc/apt/apt.conf.d/retries -echo 'APT::Acquire::https::Timeout "240";' > /etc/apt/apt.conf.d/https-timeout -echo 'APT::Acquire::http::Timeout "240";' > /etc/apt/apt.conf.d/http-timeout -EOF - -# Install gha-tools -RUN <= 5)) && break; sleep 10; done - apt-get install -y --no-install-recommends wget - wget -q https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz -O - | tar -xz -C /usr/local/bin - apt-get purge -y wget && apt-get autoremove -y - rm -rf /var/lib/apt/lists/* -EOF - # Create a conda group and assign it as root's primary group RUN <' to compare decimals here, but it works as expected for the 'bash' version in these -# images, and installing 'bc' or using a Python interpreter seem heavy for this purpose. -# -# shellcheck disable=SC2072 -if [[ "$PYTHON_VERSION_PADDED" > "3.12" ]]; then - PYTHON_ABI_TAG="cp${PYTHON_MAJOR_VERSION}${PYTHON_MINOR_VERSION}" -else - PYTHON_ABI_TAG="cpython" -fi -rapids-mamba-retry install -y -n base "python>=${PYTHON_VERSION},<${PYTHON_UPPER_BOUND}=*_${PYTHON_ABI_TAG}" -rapids-mamba-retry update --all -y -n base -find /opt/conda -follow -type f -name '*.a' -delete -find /opt/conda -follow -type f -name '*.pyc' -delete -# recreate missing libstdc++ symlinks -conda clean -aiptfy -EOF - -RUN <> /etc/skel/.bashrc -echo ". /opt/conda/etc/profile.d/conda.sh; conda activate base" >> ~/.bashrc -EOF - -# tzdata is needed by the ORC library used by pyarrow, because it provides /etc/localtime -# On Ubuntu 24.04 and newer, we also need tzdata-legacy -RUN <' to compare decimals here, but it works as expected for the 'bash' version in these -# images, and installing 'bc' or using a Python interpreter seem heavy for this purpose. -# -# shellcheck disable=SC2072 -if [[ "${os_version}" > "24.04" ]] || [[ "${os_version}" == "24.04" ]]; then - PACKAGES_TO_INSTALL+=(tzdata-legacy) -fi - -rapids-retry apt-get update -y -apt-get upgrade -y -apt-get install -y --no-install-recommends \ - "${PACKAGES_TO_INSTALL[@]}" +# set up conda +/tmp/build-scripts/configure-conda-base-environment -rm -rf "/var/lib/apt/lists/*" +# install tzdata system packages +/tmp/build-scripts/install-tzdata-packages EOF # --- end 'rapidsai/miniforge-cuda' --- # @@ -152,8 +74,10 @@ chmod 777 /data echo ". /opt/conda/etc/profile.d/conda.sh; conda activate base" >> /etc/bash.bashrc EOF +RUN \ + --mount=type=bind,source=scripts,target=/tmp/build-scripts \ +<