From 6b2bdcb03c28bd2ebc50a1c396cff66453ca45e0 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 13 Aug 2025 12:42:59 -0500 Subject: [PATCH 1/9] post-processing: fix version map, use pre-commit to regenerate it --- .gitignore | 1 - .pre-commit-config.yaml | 7 +++ ci/customization/projects-to-versions.json | 58 +++++++++++++++++++ ci/download_from_s3.sh | 2 +- ...ns.sh => generate-projects-to-versions.sh} | 5 +- ci/post-process.sh | 1 - 6 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 ci/customization/projects-to-versions.json rename ci/{get-projects-to-versions.sh => generate-projects-to-versions.sh} (96%) diff --git a/.gitignore b/.gitignore index 15e57da1a36..d2e9d00015e 100644 --- a/.gitignore +++ b/.gitignore @@ -11,5 +11,4 @@ files-to-customize.txt .netlify __pycache__/ node_modules -projects-to-versions.json rapids-docs-env/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1b573d22140..ae188226a23 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,6 +35,13 @@ repos: .devcontainer/devcontainer.json ) - id: check-yaml + - repo: local + hooks: + - id: generate-projects-to-versions + name: generate-projects-to-versions + entry: ./ci/generate-projects-to-versions.sh + language: system + pass_filenames: false - repo: https://github.com/sirosen/texthooks rev: 0.7.1 hooks: diff --git a/ci/customization/projects-to-versions.json b/ci/customization/projects-to-versions.json new file mode 100644 index 00000000000..d042bf9223e --- /dev/null +++ b/ci/customization/projects-to-versions.json @@ -0,0 +1,58 @@ +{ + "cudf": { + "legacy": "25.06", + "nightly": "25.10", + "stable": "25.08" + }, + "cudf-java": { + "legacy": "25.06", + "stable": "25.08" + }, + "cuml": { + "legacy": "25.06", + "nightly": "25.10", + "stable": "25.08" + }, + "cuproj": { + "legacy": "25.02", + "stable": "25.04" + }, + "cusignal": {}, + "cuspatial": { + "legacy": "25.02", + "stable": "25.04" + }, + "dask-cudf": { + "legacy": "25.06", + "nightly": "25.10", + "stable": "25.08" + }, + "libcudf": { + "legacy": "25.06", + "nightly": "25.10", + "stable": "25.08" + }, + "libcuproj": { + "legacy": "25.02", + "stable": "25.04" + }, + "libcuspatial": { + "legacy": "25.02", + "stable": "25.04" + }, + "librmm": { + "legacy": "25.06", + "nightly": "25.10", + "stable": "25.08" + }, + "libucxx": { + "legacy": "0.44", + "nightly": "0.46", + "stable": "0.45" + }, + "rapids-cmake": { + "legacy": "25.06", + "nightly": "25.10", + "stable": "25.08" + } +} diff --git a/ci/download_from_s3.sh b/ci/download_from_s3.sh index 82185b3ccc5..37144b54ab9 100755 --- a/ci/download_from_s3.sh +++ b/ci/download_from_s3.sh @@ -71,7 +71,7 @@ download_lib_docs() { SRC VERSION_NAME VERSION_NUMBER echo "--- processing RAPIDS libraries ---" - PROJECTS_TO_VERSIONS_JSON=$(./ci/get-projects-to-versions.sh) + PROJECTS_TO_VERSIONS_JSON=$(cat "./ci/customization/projects-to-versions.json") for PROJECT in $(jq -r 'keys | .[]' <<< "${PROJECTS_TO_VERSIONS_JSON}"); do # extract the map of versions to download for this project, which will look something like: diff --git a/ci/get-projects-to-versions.sh b/ci/generate-projects-to-versions.sh similarity index 96% rename from ci/get-projects-to-versions.sh rename to ci/generate-projects-to-versions.sh index 4925addf355..fa6332caa4a 100755 --- a/ci/get-projects-to-versions.sh +++ b/ci/generate-projects-to-versions.sh @@ -35,7 +35,7 @@ # # Only that mapping is written to stdout, so this is safe to use inline like this: # -# PROJECTS_TO_VERSIONS=$(./ci/get-projects-to-versions.sh) +# PROJECTS_TO_VERSIONS=$(./ci/generate-projects-to-versions.sh) # # WARNING: no guarantees are made about the ordering of output in this mapping. # @@ -58,6 +58,7 @@ VERSION_MAP=$(jq '{ PROJECTS_TO_VERSIONS='{}' for PROJECT in $(yq -r 'keys | .[]' <<< "$PROJECT_MAP"); do + log-stderr "active: ${PROJECT}" THIS_PROJECT_MAP="{\"${PROJECT}\":{}}" for VERSION_NAME in $(jq -r 'keys | .[]' <<< "$VERSION_MAP"); do VERSION_NUMBER=$(jq -r --arg vn "$VERSION_NAME" --arg pr "$PROJECT" ' @@ -130,4 +131,4 @@ for PROJECT in $(yq -r 'keys | .[]' <<< "$INACTIVE_PROJECT_MAP"); do ) done -echo -n "${PROJECTS_TO_VERSIONS}" +echo "${PROJECTS_TO_VERSIONS}" > ./ci/customization/projects-to-versions.json diff --git a/ci/post-process.sh b/ci/post-process.sh index 0bbbbc4f7cd..5078a86a181 100755 --- a/ci/post-process.sh +++ b/ci/post-process.sh @@ -10,7 +10,6 @@ CURRENT_DIR=$(dirname $(realpath $0)) pip install -r "${CURRENT_DIR}/customization/requirements.txt" PROJECTS_TO_VERSIONS_PATH="${CURRENT_DIR}"/customization/projects-to-versions.json -"${CURRENT_DIR}"/get-projects-to-versions.sh > "${PROJECTS_TO_VERSIONS_PATH}" "${CURRENT_DIR}"/update_symlinks.sh "${PROJECTS_TO_VERSIONS_PATH}" From 74e126fbc95ef9e298153adde6146a305e6fbec8 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 13 Aug 2025 12:46:11 -0500 Subject: [PATCH 2/9] restore dropped packages --- _data/docs.yml | 121 +++++++++++++++++++++ ci/customization/projects-to-versions.json | 53 +++++++++ 2 files changed, 174 insertions(+) diff --git a/_data/docs.yml b/_data/docs.yml index 0ea4b6b2538..62a5c0f1fbe 100644 --- a/_data/docs.yml +++ b/_data/docs.yml @@ -35,6 +35,28 @@ apis: legacy: 1 stable: 1 nightly: 1 + cugraph: + name: cuGraph + path: cugraph + desc: 'cuGraph is a GPU accelerated graph analytics library, with functionality like NetworkX, which is seamlessly integrated into the RAPIDS data science platform. cuGraph supports GNNs with PyG, DGL packages, cugraph-service for analytics on a remote graph, and WHOLEGRAPH for memory management.' + ghlink: https://github.com/rapidsai/cugraph + cllink: https://github.com/rapidsai/cugraph/blob/main/CHANGELOG.md + versions: + # enable or disable links; 0 = disabled, 1 = enabled + legacy: 1 + stable: 1 + nightly: 1 + cuxfilter: + name: cuxfilter + path: cuxfilter + desc: 'cuxfilter acts as a connector library, which provides the connections between different visualization libraries and a GPU dataframe without much hassle. This also allows the user to use charts from different libraries in a single dashboard, while also providing the interaction.' + ghlink: https://github.com/rapidsai/cuxfilter + cllink: https://github.com/rapidsai/cuxfilter/blob/main/CHANGELOG.md + versions: + # enable or disable links; 0 = disabled, 1 = enabled + legacy: 1 + stable: 1 + nightly: 1 cudf-java: name: 'Java + cuDF' path: cudf-java @@ -46,6 +68,83 @@ apis: legacy: 1 stable: 1 nightly: 0 + cucim: + name: cuCIM + path: cucim + desc: 'The RAPIDS cuCIM is an extensible toolkit designed to provide GPU accelerated I/O, computer vision & image processing primitives for N-Dimensional images with a focus on biomedical imaging.' + ghlink: https://github.com/rapidsai/cucim + cllink: https://github.com/rapidsai/cucim/blob/main/CHANGELOG.md + versions: + # enable or disable links; 0 = disabled, 1 = enabled + legacy: 1 + stable: 1 + nightly: 1 + cuvs: + name: cuVS + path: cuvs + desc: 'cuVS is a library for GPU-accelerated vector search and clustering.' + ghlink: https://github.com/rapidsai/cuvs + cllink: https://github.com/rapidsai/cuvs/blob/main/CHANGELOG.md + versions: + # enable or disable links; 0 = disabled, 1 = enabled + legacy: 1 + stable: 1 + nightly: 1 + kvikio: + name: KvikIO + path: kvikio + desc: "KvikIO is a Python and C++ library for high performance file IO using GPUDirect Storage (GDS)." + ghlink: https://github.com/rapidsai/kvikio + cllink: https://github.com/rapidsai/kvikio/blob/main/CHANGELOG.md + versions: + # enable or disable links; 0 = disabled, 1 = enabled + legacy: 1 + stable: 1 + nightly: 1 + raft: + name: RAFT + path: raft + desc: "RAFT contains fundamental widely-used algorithms and primitives for vector search, machine learning, and information retrieval." + ghlink: https://github.com/rapidsai/raft + cllink: https://github.com/rapidsai/raft/blob/main/CHANGELOG.md + versions: + # enable or disable links; 0 = disabled, 1 = enabled + legacy: 1 + stable: 1 + nightly: 1 + dask-cuda: + name: Dask-CUDA + path: dask-cuda + desc: "Various utilities to improve deployment and management of Dask workers on CUDA-enabled systems." + ghlink: https://github.com/rapidsai/dask-cuda + cllink: https://github.com/rapidsai/dask-cuda/blob/main/CHANGELOG.md + versions: + # enable or disable links; 0 = disabled, 1 = enabled + legacy: 1 + stable: 1 + nightly: 1 + rmm: + name: RMM + path: rmm + desc: 'RAPIDS Memory Manager (RMM) is a central place for all device memory allocations in cuDF (C++ and Python) and other RAPIDS libraries. In addition, it is a replacement allocator for CUDA Device Memory (and CUDA Managed Memory) and a pool allocator to make CUDA device memory allocation / deallocation faster and asynchronous.' + ghlink: https://github.com/rapidsai/rmm + cllink: https://github.com/rapidsai/rmm/blob/main/CHANGELOG.md + versions: + # enable or disable links; 0 = disabled, 1 = enabled + legacy: 1 + stable: 1 + nightly: 1 + rapidsmpf: + name: RapidsMPF + path: rapidsmpf + desc: 'RAPIDS Multi-Process Foundation (rapidsmpf) is a collection of multi-GPU, distributed memory algorithms written in C++ and exposed to Python.' + ghlink: https://github.com/rapidsai/rapidsmpf + versions: + # enable or disable links; 0 = disabled, 1 = enabled + legacy: 0 + stable: 0 + nightly: 1 + # RAPIDS "Libs" - lower-level libraries that are building blocks for creating # custom tools and integrate with other libraries @@ -73,6 +172,28 @@ libs: legacy: 1 stable: 1 nightly: 1 + libcuml: + name: libcuml + path: libcuml + desc: 'libcuml is a C/C++ CUDA library for cuML.' + ghlink: https://github.com/rapidsai/cuml + cllink: https://github.com/rapidsai/cuml/blob/main/CHANGELOG.md + versions: + # enable or disable links; 0 = disabled, 1 = enabled + legacy: 1 + stable: 1 + nightly: 1 + libkvikio: + name: libkvikio + path: libkvikio + desc: "libkvikio is a C++ header-only library providing bindings to cuFile, which enables GPUDirect Storage (GDS)." + ghlink: https://github.com/rapidsai/kvikio + cllink: https://github.com/rapidsai/kvikio/blob/main/CHANGELOG.md + versions: + # enable or disable links; 0 = disabled, 1 = enabled + legacy: 1 + stable: 1 + nightly: 1 libucxx: name: libucxx path: libucxx diff --git a/ci/customization/projects-to-versions.json b/ci/customization/projects-to-versions.json index d042bf9223e..7a22c76837c 100644 --- a/ci/customization/projects-to-versions.json +++ b/ci/customization/projects-to-versions.json @@ -1,4 +1,9 @@ { + "cucim": { + "legacy": "25.06", + "nightly": "25.10", + "stable": "25.08" + }, "cudf": { "legacy": "25.06", "nightly": "25.10", @@ -8,6 +13,11 @@ "legacy": "25.06", "stable": "25.08" }, + "cugraph": { + "legacy": "25.06", + "nightly": "25.10", + "stable": "25.08" + }, "cuml": { "legacy": "25.06", "nightly": "25.10", @@ -22,16 +32,41 @@ "legacy": "25.02", "stable": "25.04" }, + "cuvs": { + "legacy": "25.06", + "nightly": "25.10", + "stable": "25.08" + }, + "cuxfilter": { + "legacy": "25.06", + "nightly": "25.10", + "stable": "25.08" + }, + "dask-cuda": { + "legacy": "25.06", + "nightly": "25.10", + "stable": "25.08" + }, "dask-cudf": { "legacy": "25.06", "nightly": "25.10", "stable": "25.08" }, + "kvikio": { + "legacy": "25.06", + "nightly": "25.10", + "stable": "25.08" + }, "libcudf": { "legacy": "25.06", "nightly": "25.10", "stable": "25.08" }, + "libcuml": { + "legacy": "25.06", + "nightly": "25.10", + "stable": "25.08" + }, "libcuproj": { "legacy": "25.02", "stable": "25.04" @@ -40,6 +75,11 @@ "legacy": "25.02", "stable": "25.04" }, + "libkvikio": { + "legacy": "25.06", + "nightly": "25.10", + "stable": "25.08" + }, "librmm": { "legacy": "25.06", "nightly": "25.10", @@ -50,9 +90,22 @@ "nightly": "0.46", "stable": "0.45" }, + "raft": { + "legacy": "25.06", + "nightly": "25.10", + "stable": "25.08" + }, "rapids-cmake": { "legacy": "25.06", "nightly": "25.10", "stable": "25.08" + }, + "rapidsmpf": { + "nightly": "25.10" + }, + "rmm": { + "legacy": "25.06", + "nightly": "25.10", + "stable": "25.08" } } From fa462c9876f4489a1daff408046e72483241497f Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 13 Aug 2025 13:21:18 -0500 Subject: [PATCH 3/9] remove debugging log line --- ci/generate-projects-to-versions.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/generate-projects-to-versions.sh b/ci/generate-projects-to-versions.sh index fa6332caa4a..514cb2b85c7 100755 --- a/ci/generate-projects-to-versions.sh +++ b/ci/generate-projects-to-versions.sh @@ -58,7 +58,6 @@ VERSION_MAP=$(jq '{ PROJECTS_TO_VERSIONS='{}' for PROJECT in $(yq -r 'keys | .[]' <<< "$PROJECT_MAP"); do - log-stderr "active: ${PROJECT}" THIS_PROJECT_MAP="{\"${PROJECT}\":{}}" for VERSION_NAME in $(jq -r 'keys | .[]' <<< "$VERSION_MAP"); do VERSION_NUMBER=$(jq -r --arg vn "$VERSION_NAME" --arg pr "$PROJECT" ' From 0b85f753d90551cc9c74acadb11fffbd771b506f Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 13 Aug 2025 14:20:06 -0500 Subject: [PATCH 4/9] try to fix pre-commit.ci --- ci/generate-projects-to-versions.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ci/generate-projects-to-versions.sh b/ci/generate-projects-to-versions.sh index 514cb2b85c7..62afcc06293 100755 --- a/ci/generate-projects-to-versions.sh +++ b/ci/generate-projects-to-versions.sh @@ -46,6 +46,24 @@ log-stderr() { echo "${1}" >&2 } +# no way to pre-install tools on pre-commit.ci, so ensure they +# get installed at runtime as needed here +if type -f jq && type -f yq ; then + log-stderr "detected 'jq' and 'yq' already installed" +else + DETECTED_OS="$(uname)" + if [[ "${DETECTED_OS}" != "Linux" ]] || ! type -f apt; then + log-stderr "'jq' and/or 'yq' not found, and not sure how to automatically them'. Install those tools." + exit 1 + else + log-stderr "'jq' and/or 'yq' not found, installing them with 'apt'" + apt-get update + apt-get install -y --no-install-recommends \ + jq \ + yq + fi +fi + PROJECT_MAP=$(yq '.apis + .libs' _data/docs.yml) INACTIVE_PROJECT_MAP=$(yq '.inactive-projects' _data/docs.yml) From 12724471f98dfe158d0ebf83dbbd22803774338e Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 13 Aug 2025 14:20:44 -0500 Subject: [PATCH 5/9] another fix --- ci/generate-projects-to-versions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/generate-projects-to-versions.sh b/ci/generate-projects-to-versions.sh index 62afcc06293..494c635908c 100755 --- a/ci/generate-projects-to-versions.sh +++ b/ci/generate-projects-to-versions.sh @@ -52,7 +52,7 @@ if type -f jq && type -f yq ; then log-stderr "detected 'jq' and 'yq' already installed" else DETECTED_OS="$(uname)" - if [[ "${DETECTED_OS}" != "Linux" ]] || ! type -f apt; then + if [[ "${DETECTED_OS}" != "Linux" ]] || ! type -f apt-get; then log-stderr "'jq' and/or 'yq' not found, and not sure how to automatically them'. Install those tools." exit 1 else From 7240013bf7427324929904c560468bbbd43063ec Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 13 Aug 2025 14:23:39 -0500 Subject: [PATCH 6/9] skip generate-projects-to-versions on pre-commit.ci --- .pre-commit-config.yaml | 3 +++ ci/generate-projects-to-versions.sh | 18 ------------------ 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ae188226a23..f0c12de92de 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,6 +6,9 @@ ci: autoupdate_branch: "" autoupdate_commit_msg: "[pre-commit.ci] pre-commit autoupdate" autoupdate_schedule: quarterly + skip: + # requires 'jq' and 'yq', which dont' come pre-installed in the pre-commit.ci image + - generate-projects-to-versions submodules: false repos: diff --git a/ci/generate-projects-to-versions.sh b/ci/generate-projects-to-versions.sh index 494c635908c..514cb2b85c7 100755 --- a/ci/generate-projects-to-versions.sh +++ b/ci/generate-projects-to-versions.sh @@ -46,24 +46,6 @@ log-stderr() { echo "${1}" >&2 } -# no way to pre-install tools on pre-commit.ci, so ensure they -# get installed at runtime as needed here -if type -f jq && type -f yq ; then - log-stderr "detected 'jq' and 'yq' already installed" -else - DETECTED_OS="$(uname)" - if [[ "${DETECTED_OS}" != "Linux" ]] || ! type -f apt-get; then - log-stderr "'jq' and/or 'yq' not found, and not sure how to automatically them'. Install those tools." - exit 1 - else - log-stderr "'jq' and/or 'yq' not found, installing them with 'apt'" - apt-get update - apt-get install -y --no-install-recommends \ - jq \ - yq - fi -fi - PROJECT_MAP=$(yq '.apis + .libs' _data/docs.yml) INACTIVE_PROJECT_MAP=$(yq '.inactive-projects' _data/docs.yml) From 7a6b1200b04e4a49c69e6250a8374bdd2747e97d Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 13 Aug 2025 14:46:13 -0500 Subject: [PATCH 7/9] Update _data/docs.yml Co-authored-by: Bradley Dice --- _data/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/docs.yml b/_data/docs.yml index 62a5c0f1fbe..1ffb432c911 100644 --- a/_data/docs.yml +++ b/_data/docs.yml @@ -142,7 +142,7 @@ apis: versions: # enable or disable links; 0 = disabled, 1 = enabled legacy: 0 - stable: 0 + stable: 1 nightly: 1 From babe5c0e60797a748ca321e82ddb052bc104ca95 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 13 Aug 2025 14:49:09 -0500 Subject: [PATCH 8/9] regenerate for rapidsmpf --- ci/customization/projects-to-versions.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/customization/projects-to-versions.json b/ci/customization/projects-to-versions.json index 7a22c76837c..97aa6020750 100644 --- a/ci/customization/projects-to-versions.json +++ b/ci/customization/projects-to-versions.json @@ -101,7 +101,8 @@ "stable": "25.08" }, "rapidsmpf": { - "nightly": "25.10" + "nightly": "25.10", + "stable": "25.08" }, "rmm": { "legacy": "25.06", From 323de86f844d0ca0e7e76dfd48bd8c7041b83cb6 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 13 Aug 2025 14:50:36 -0500 Subject: [PATCH 9/9] fix typo --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f0c12de92de..cad6c925b2a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ ci: autoupdate_commit_msg: "[pre-commit.ci] pre-commit autoupdate" autoupdate_schedule: quarterly skip: - # requires 'jq' and 'yq', which dont' come pre-installed in the pre-commit.ci image + # requires 'jq' and 'yq', which don't come pre-installed in the pre-commit.ci image - generate-projects-to-versions submodules: false