From 45d7835516149af766cada8fe972d889afaeee88 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 7 Aug 2025 11:50:51 -0500 Subject: [PATCH 1/8] working for active projects --- ci/get-projects-to-versions.sh | 58 ++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100755 ci/get-projects-to-versions.sh diff --git a/ci/get-projects-to-versions.sh b/ci/get-projects-to-versions.sh new file mode 100755 index 00000000000..7905d38287f --- /dev/null +++ b/ci/get-projects-to-versions.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# produces a YAML mapping of the form: +# +# {project}: +# stable: {version_number} +# legacy: {version_number} +# nightly: {version_number} +# +# With keys omitted based on configuration in _data/docs.yml. +# +# e.g. if a project has 'stable: 0' in that file, it will not have a '{project}.stable' +# key in the mapping produced by this script. + +set -e -u -o pipefail + +PROJECT_MAP=$(yq '.apis + .libs' _data/docs.yml) +INACTIVE_PROJECT_MAP=$(yq '.inactive-projects' _data/docs.yml) + +VERSION_MAP=$(jq '{ + "legacy": { "version": .legacy.version, "ucxx_version": .legacy.ucxx_version }, + "stable": { "version": .stable.version, "ucxx_version": .stable.ucxx_version }, + "nightly": { "version": .nightly.version, "ucxx_version": .nightly.ucxx_version } +}' _data/releases.json) + +PROJECTS_TO_VERSIONS='{}' + +for PROJECT in $(yq -r 'keys | .[]' <<< "$PROJECT_MAP"); do + THIS_PROJECT_MAP="{\"${PROJECT}\":{}}" + for VERSION_NAME in $(jq -r 'keys | .[]' <<< "$VERSION_MAP"); do + VERSION_NUMBER=$(jq -r --arg vn "$VERSION_NAME" --arg pr "$PROJECT" ' + if ($pr | contains("ucxx")) then + .[$vn].ucxx_version + else + .[$vn].version + end' <<< "$VERSION_MAP") + PROJECT_MAP_JSON=$(yq -r -o json '.' <<< "$PROJECT_MAP") + if [ "$(jq -r --arg pr "$PROJECT" --arg vn "$VERSION_NAME" '.[$pr].versions[$vn]' <<< "$PROJECT_MAP_JSON")" == "0" ]; then + echo "Skipping: $PROJECT | $VERSION_NAME | $VERSION_NUMBER" + continue + fi + THIS_PROJECT_MAP=$( + jq \ + --arg pr "${PROJECT}" \ + --arg version_name "${VERSION_NAME}" \ + --arg version_number "${VERSION_NUMBER}" \ + '.[$pr] |= . + {$version_name: $version_number}' \ + <<< "${THIS_PROJECT_MAP}" + ) + done + # add this new entry to the mapping + PROJECTS_TO_VERSIONS=$( + jq --slurp \ + 'map(to_entries) | flatten | group_by(.key) | map({key: .[0].key, value: map(.value) | add}) | from_entries' \ + <<< "${PROJECTS_TO_VERSIONS}${THIS_PROJECT_MAP}" + ) +done + +echo -n "${PROJECTS_TO_VERSIONS}" From bfe229366bdf9d6b7208b0c982a7efab285054b4 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 7 Aug 2025 15:28:00 -0500 Subject: [PATCH 2/8] fix post-processing for inactive projects --- ci/download_from_s3.sh | 71 ++++++++++++++------------- ci/get-projects-to-versions.sh | 89 +++++++++++++++++++++++++++++++--- ci/update_symlinks.sh | 77 +++++++++++++++-------------- 3 files changed, 159 insertions(+), 78 deletions(-) diff --git a/ci/download_from_s3.sh b/ci/download_from_s3.sh index eaa68bf7aee..82185b3ccc5 100755 --- a/ci/download_from_s3.sh +++ b/ci/download_from_s3.sh @@ -2,7 +2,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. # All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Copies the RAPIDS libraries' HTML files from S3 into the "_site" directory of +# +# Copies the RAPIDS projects' HTML files from S3 into the "_site" directory of # the Jekyll build. set -euo pipefail @@ -64,43 +65,44 @@ aws_cp() { } # Downloads the RAPIDS libraries' documentation files from S3 and places them -# into the "_site/api" folder. The versions that should be copied are read from -# "_data/releases.json" and the libraries that should be copied are read from -# "_data/docs.yml". +# into the "_site/api" folder. download_lib_docs() { - local DST PROJECT PROJECT_MAP \ - SRC VERSION_MAP VERSION_NAME \ - VERSION_NUMBER - - VERSION_MAP=$(jq '{ - "legacy": { "version": .legacy.version, "ucxx_version": .legacy.ucxx_version }, - "stable": { "version": .stable.version, "ucxx_version": .stable.ucxx_version }, - "nightly": { "version": .nightly.version, "ucxx_version": .nightly.ucxx_version } - }' _data/releases.json) - - PROJECT_MAP=$(yq '.apis + .libs' _data/docs.yml) - - for VERSION_NAME in $(jq -r 'keys | .[]' <<< "$VERSION_MAP"); do - for PROJECT in $(yq -r 'keys | .[]' <<< "$PROJECT_MAP"); do - VERSION_NUMBER=$(jq -r --arg vn "$VERSION_NAME" --arg pr "$PROJECT" ' - if ($pr | contains("ucxx")) then - .[$vn].ucxx_version - else - .[$vn].version - end' <<< "$VERSION_MAP") - - PROJECT_MAP_JSON=$(yq -r -o json '.' <<< "$PROJECT_MAP") - if [ "$(jq -r --arg pr "$PROJECT" --arg vn "$VERSION_NAME" '.[$pr].versions[$vn]' <<< "$PROJECT_MAP_JSON")" == "0" ]; then - echo "Skipping: $PROJECT | $VERSION_NAME | $VERSION_NUMBER" - continue - fi - + local DST PROJECT PROJECTS_TO_VERSIONS_JSON \ + SRC VERSION_NAME VERSION_NUMBER + + echo "--- processing RAPIDS libraries ---" + PROJECTS_TO_VERSIONS_JSON=$(./ci/get-projects-to-versions.sh) + for PROJECT in $(jq -r 'keys | .[]' <<< "${PROJECTS_TO_VERSIONS_JSON}"); do + + # extract the map of versions to download for this project, which will look something like: + # + # {"stable": 25.10, "nightly": 25.12, "legacy": 25.08} + # + # With keys varying based on which types of docs we want to build for this particular project. + VERSIONS_FOR_THIS_PROJECT=$( + jq \ + -r \ + --arg pr "${PROJECT}" \ + '.[$pr]' \ + <<< "${PROJECTS_TO_VERSIONS_JSON}" + ) + + # loop over 'stable', 'nightly', etc. + for VERSION_NAME in $(jq -r 'keys | .[]' <<< "${VERSIONS_FOR_THIS_PROJECT}"); do + VERSION_NUMBER=$( + jq \ + -r \ + --arg version_name "${VERSION_NAME}" \ + '.[$version_name]' \ + <<< "${VERSIONS_FOR_THIS_PROJECT}" + ) + # copy the relevant files from S3 to the local directory SRC="s3://${DOCS_BUCKET}/${PROJECT}/html/${VERSION_NUMBER}/" DST="$(yq -n 'env(GENERATED_DIRS)|.libs')/${PROJECT}/${VERSION_NUMBER}/" - aws_cp "${SRC}" "${DST}" - done - done + done # for VERSION_NAME + + done # for PROJECT } # Downloads the deployment docs from S3 and places them in the @@ -108,6 +110,7 @@ download_lib_docs() { download_deployment_docs() { local DST SRC VERSION + echo "--- processing deployment docs ---" for VERSION in nightly stable; do SRC="s3://${DOCS_BUCKET}/deployment/html/${VERSION}/" DST="$(yq -n 'env(GENERATED_DIRS)|.deployment')/${VERSION}/" diff --git a/ci/get-projects-to-versions.sh b/ci/get-projects-to-versions.sh index 7905d38287f..4925addf355 100755 --- a/ci/get-projects-to-versions.sh +++ b/ci/get-projects-to-versions.sh @@ -1,17 +1,50 @@ #!/bin/bash -# produces a YAML mapping of the form: +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 # -# {project}: -# stable: {version_number} -# legacy: {version_number} -# nightly: {version_number} +# [description] +# +# Determines: +# +# * which RAPIDS libraries to host docs for +# * what types of docs to host ('legacy', 'nightly', 'stable', etc.) +# * what versions to map to those types +# +# The libraries that should be copied are read from "_data/docs.yml". +# +# The versions that should be copied are read from a mix of sources: +# +# - active projects: "_data/releases.json" +# - inactive projects: 'version-overrides' field in entries in "_data/docs.yml" +# +# Produces a JSON mapping of the form: +# +# { +# "{project}": { +# "stable": {version_number}, +# "legacy": {version_number}, +# "nightly": {version_number} +# }, +# } # # With keys omitted based on configuration in _data/docs.yml. # # e.g. if a project has 'stable: 0' in that file, it will not have a '{project}.stable' # key in the mapping produced by this script. +# +# Only that mapping is written to stdout, so this is safe to use inline like this: +# +# PROJECTS_TO_VERSIONS=$(./ci/get-projects-to-versions.sh) +# +# WARNING: no guarantees are made about the ordering of output in this mapping. +# + +set -e -E -u -o pipefail -set -e -u -o pipefail +log-stderr() { + echo "${1}" >&2 +} PROJECT_MAP=$(yq '.apis + .libs' _data/docs.yml) INACTIVE_PROJECT_MAP=$(yq '.inactive-projects' _data/docs.yml) @@ -35,7 +68,49 @@ for PROJECT in $(yq -r 'keys | .[]' <<< "$PROJECT_MAP"); do end' <<< "$VERSION_MAP") PROJECT_MAP_JSON=$(yq -r -o json '.' <<< "$PROJECT_MAP") if [ "$(jq -r --arg pr "$PROJECT" --arg vn "$VERSION_NAME" '.[$pr].versions[$vn]' <<< "$PROJECT_MAP_JSON")" == "0" ]; then - echo "Skipping: $PROJECT | $VERSION_NAME | $VERSION_NUMBER" + log-stderr "Skipping: $PROJECT | $VERSION_NAME | $VERSION_NUMBER" + continue + fi + THIS_PROJECT_MAP=$( + jq \ + --arg pr "${PROJECT}" \ + --arg version_name "${VERSION_NAME}" \ + --arg version_number "${VERSION_NUMBER}" \ + '.[$pr] |= . + {$version_name: $version_number}' \ + <<< "${THIS_PROJECT_MAP}" + ) + done + # add this new entry to the mapping + PROJECTS_TO_VERSIONS=$( + jq --slurp \ + 'map(to_entries) | flatten | group_by(.key) | map({key: .[0].key, value: map(.value) | add}) | from_entries' \ + <<< "${PROJECTS_TO_VERSIONS}${THIS_PROJECT_MAP}" + ) +done + +# inactive projects have specific versions hard-coded in their configuration, process those separately +for PROJECT in $(yq -r 'keys | .[]' <<< "$INACTIVE_PROJECT_MAP"); do + THIS_PROJECT_MAP="{\"${PROJECT}\":{}}" + for VERSION_NAME in $(jq -r 'keys | .[]' <<< "$VERSION_MAP"); do + # do not attempt updates for any versions where the corresponding key is '0' in docs.yml + INACTIVE_PROJECT_MAP_JSON=$(yq -r -o json '.' <<< "$INACTIVE_PROJECT_MAP") + if [ "$(jq -r --arg pr "$PROJECT" --arg vn "$VERSION_NAME" '.[$pr].versions[$vn]' <<< "$INACTIVE_PROJECT_MAP_JSON")" == "0" ]; then + log-stderr "Skipping: $PROJECT | $VERSION_NAME" + continue + fi + + # get the version from the 'version-overrides' field in docs.yml, hard-coded there + # so it doesn't change from release-to-release for inactive projects + VERSION_NUMBER=$( + jq -r \ + --arg vn "$VERSION_NAME" \ + --arg pr "${PROJECT}" \ + '.[$pr]."version-overrides"[$vn]' \ + <<< "${INACTIVE_PROJECT_MAP_JSON}" + ) + PROJECT_MAP_JSON=$(yq -r -o json '.' <<< "$PROJECT_MAP") + if [ "$(jq -r --arg pr "$PROJECT" --arg vn "$VERSION_NAME" '.[$pr].versions[$vn]' <<< "$PROJECT_MAP_JSON")" == "0" ]; then + log-stderr "Skipping: $PROJECT | $VERSION_NAME | $VERSION_NUMBER" continue fi THIS_PROJECT_MAP=$( diff --git a/ci/update_symlinks.sh b/ci/update_symlinks.sh index 8540ecb4a89..2c5b62c6600 100755 --- a/ci/update_symlinks.sh +++ b/ci/update_symlinks.sh @@ -8,51 +8,54 @@ ####################################### set -euEo pipefail +# expect paths to be relative to the project root PROJ_ROOT=$(realpath "$(dirname $(realpath $0))/../") -RELEASES="${PROJ_ROOT}/_data/releases.json" +pushd "${PROJ_ROOT}" -STABLE_VERSION=$(jq -r '.stable.version' < "${RELEASES}") -LEGACY_VERSION=$(jq -r '.legacy.version' < "${RELEASES}") -NIGHTLY_VERSION=$(jq -r '.nightly.version' < "${RELEASES}") - -STABLE_UCXX_VERSION=$(jq -r '.stable.ucxx_version' < "${RELEASES}") -LEGACY_UCXX_VERSION=$(jq -r '.legacy.ucxx_version' < "${RELEASES}") -NIGHTLY_UCXX_VERSION=$(jq -r '.nightly.ucxx_version' < "${RELEASES}") +PROJECTS_TO_VERSIONS_JSON=$(./ci/get-projects-to-versions.sh) echo "Updating symlinks..." echo "" -for FOLDER in _site/api/*/ ; do - if [[ "${FOLDER}" == *"ucxx"* ]]; then - STABLE_FOLDER=$STABLE_UCXX_VERSION - LEGACY_FOLDER=$LEGACY_UCXX_VERSION - NIGHTLY_FOLDER=$NIGHTLY_UCXX_VERSION - else - STABLE_FOLDER=$STABLE_VERSION - LEGACY_FOLDER=$LEGACY_VERSION - NIGHTLY_FOLDER=$NIGHTLY_VERSION - fi - - cd ${FOLDER} +for PROJECT in $(jq -r 'keys | .[]' <<< "${PROJECTS_TO_VERSIONS_JSON}"); do + VERSIONS_FOR_THIS_PROJECT=$( + jq \ + -r \ + --arg pr "${PROJECT}" \ + '.[$pr]' \ + <<< "${PROJECTS_TO_VERSIONS_JSON}" + ) + + # expect to find a local folder, relative to the root of the repo, + # named e.g. '_site/api/cudf' + FOLDER="_site/api/${PROJECT}/" + pushd "${FOLDER}" echo "" echo "${FOLDER}--------" - if [ -d "${STABLE_FOLDER}" ]; then - ln -s ${STABLE_FOLDER} stable - ln -s ${STABLE_FOLDER} latest - echo " - stable & latest point to ${STABLE_FOLDER}" - fi - - if [ -d "${LEGACY_FOLDER}" ]; then - ln -s ${LEGACY_FOLDER} legacy - echo " - legacy points to ${LEGACY_FOLDER}" - fi - - if [ -d "${NIGHTLY_FOLDER}" ]; then - ln -s ${NIGHTLY_FOLDER} nightly - echo " - nightly points to ${NIGHTLY_FOLDER}" - fi + # loop over 'stable', 'nightly', etc. + for VERSION_NAME in $(jq -r 'keys | .[]' <<< "${VERSIONS_FOR_THIS_PROJECT}"); do + # expect to find a directory matching the version number, e.g. '_site/api/cudf/25.10' + VERSION_NUMBER=$( + jq \ + -r \ + --arg version_name "${VERSION_NAME}" \ + '.[$version_name]' \ + <<< "${VERSIONS_FOR_THIS_PROJECT}" + ) + FOLDER_FOR_THIS_VERSION="${VERSION_NUMBER}" + + # map /latest to the same version as /stable + if [[ "${VERSION_NAME}" == "stable" ]]; then + ls -s "${FOLDER_FOR_THIS_VERSION}" stable + ls -s "${FOLDER_FOR_THIS_VERSION}" latest + echo " - 'stable' and 'latest' point to '${FOLDER_FOR_THIS_VERSION}'" + else + ls -s "${FOLDER_FOR_THIS_VERSION}" "${VERSION_NAME}" + echo " - '${VERSION_NAME}' points to '${FOLDER_FOR_THIS_VERSION}'" + fi + done # for VERSION echo "---------------" echo "" - cd ${PROJ_ROOT} -done + popd +done # for PROJECT From 0087b6b7ed401c3c4e8e38722cf2db1c69a3dd81 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 7 Aug 2025 22:24:50 -0500 Subject: [PATCH 3/8] fix paths in update_symlinks.sh --- ci/update_symlinks.sh | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/ci/update_symlinks.sh b/ci/update_symlinks.sh index 2c5b62c6600..c5eae1c1f5e 100755 --- a/ci/update_symlinks.sh +++ b/ci/update_symlinks.sh @@ -27,10 +27,9 @@ for PROJECT in $(jq -r 'keys | .[]' <<< "${PROJECTS_TO_VERSIONS_JSON}"); do # expect to find a local folder, relative to the root of the repo, # named e.g. '_site/api/cudf' - FOLDER="_site/api/${PROJECT}/" - pushd "${FOLDER}" + PROJECT_FOLDER="_site/api/${PROJECT}" echo "" - echo "${FOLDER}--------" + echo "${PROJECT_FOLDER}/--------" # loop over 'stable', 'nightly', etc. for VERSION_NAME in $(jq -r 'keys | .[]' <<< "${VERSIONS_FOR_THIS_PROJECT}"); do @@ -42,20 +41,19 @@ for PROJECT in $(jq -r 'keys | .[]' <<< "${PROJECTS_TO_VERSIONS_JSON}"); do '.[$version_name]' \ <<< "${VERSIONS_FOR_THIS_PROJECT}" ) - FOLDER_FOR_THIS_VERSION="${VERSION_NUMBER}" + FOLDER_FOR_THIS_VERSION="${PROJECT_FOLDER}/${VERSION_NUMBER}" # map /latest to the same version as /stable if [[ "${VERSION_NAME}" == "stable" ]]; then - ls -s "${FOLDER_FOR_THIS_VERSION}" stable - ls -s "${FOLDER_FOR_THIS_VERSION}" latest + ln -s "${FOLDER_FOR_THIS_VERSION}" "${PROJECT_FOLDER}/stable" + ln -s "${FOLDER_FOR_THIS_VERSION}" "${PROJECT_FOLDER}/latest" echo " - 'stable' and 'latest' point to '${FOLDER_FOR_THIS_VERSION}'" else - ls -s "${FOLDER_FOR_THIS_VERSION}" "${VERSION_NAME}" + ln -s "${FOLDER_FOR_THIS_VERSION}" "${PROJECT_FOLDER}/${VERSION_NAME}" echo " - '${VERSION_NAME}' points to '${FOLDER_FOR_THIS_VERSION}'" fi done # for VERSION echo "---------------" echo "" - popd done # for PROJECT From b0e3f97eaa7bee55feca49ee72602a4bc09d15be Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 8 Aug 2025 00:00:37 -0500 Subject: [PATCH 4/8] update lockfile for arm64 macOS --- Gemfile.lock | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Gemfile.lock b/Gemfile.lock index 94634017289..1156a81a562 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -237,6 +237,8 @@ GEM uri nokogiri (1.18.8-aarch64-linux-gnu) racc (~> 1.4) + nokogiri (1.18.8-arm64-darwin) + racc (~> 1.4) nokogiri (1.18.8-x86_64-linux-gnu) racc (~> 1.4) octokit (4.25.1) @@ -276,6 +278,7 @@ GEM PLATFORMS aarch64-linux + arm64-darwin-24 x86_64-linux DEPENDENCIES From f811a0f6e28e8bab911932e3a16e996451ee4640 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 8 Aug 2025 00:07:41 -0500 Subject: [PATCH 5/8] fix symlinks --- ci/update_symlinks.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ci/update_symlinks.sh b/ci/update_symlinks.sh index c5eae1c1f5e..66e9f4fefb8 100755 --- a/ci/update_symlinks.sh +++ b/ci/update_symlinks.sh @@ -28,6 +28,7 @@ for PROJECT in $(jq -r 'keys | .[]' <<< "${PROJECTS_TO_VERSIONS_JSON}"); do # expect to find a local folder, relative to the root of the repo, # named e.g. '_site/api/cudf' PROJECT_FOLDER="_site/api/${PROJECT}" + pushd "${PROJECT_FOLDER}" echo "" echo "${PROJECT_FOLDER}/--------" @@ -41,15 +42,15 @@ for PROJECT in $(jq -r 'keys | .[]' <<< "${PROJECTS_TO_VERSIONS_JSON}"); do '.[$version_name]' \ <<< "${VERSIONS_FOR_THIS_PROJECT}" ) - FOLDER_FOR_THIS_VERSION="${PROJECT_FOLDER}/${VERSION_NUMBER}" + FOLDER_FOR_THIS_VERSION="${VERSION_NUMBER}" # map /latest to the same version as /stable if [[ "${VERSION_NAME}" == "stable" ]]; then - ln -s "${FOLDER_FOR_THIS_VERSION}" "${PROJECT_FOLDER}/stable" - ln -s "${FOLDER_FOR_THIS_VERSION}" "${PROJECT_FOLDER}/latest" + ln -s "${FOLDER_FOR_THIS_VERSION}" stable + ln -s "${FOLDER_FOR_THIS_VERSION}" latest echo " - 'stable' and 'latest' point to '${FOLDER_FOR_THIS_VERSION}'" else - ln -s "${FOLDER_FOR_THIS_VERSION}" "${PROJECT_FOLDER}/${VERSION_NAME}" + ln -s "${FOLDER_FOR_THIS_VERSION}" "${VERSION_NAME}" echo " - '${VERSION_NAME}' points to '${FOLDER_FOR_THIS_VERSION}'" fi done # for VERSION From d4e7644c9d184929e15f4ba84768df54d83b542c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 8 Aug 2025 00:39:49 -0500 Subject: [PATCH 6/8] handle empty cases like cusignal --- ci/update_symlinks.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ci/update_symlinks.sh b/ci/update_symlinks.sh index 66e9f4fefb8..0e7440e68d7 100755 --- a/ci/update_symlinks.sh +++ b/ci/update_symlinks.sh @@ -25,6 +25,11 @@ for PROJECT in $(jq -r 'keys | .[]' <<< "${PROJECTS_TO_VERSIONS_JSON}"); do <<< "${PROJECTS_TO_VERSIONS_JSON}" ) + if [[ "${VERSIONS_FOR_THIS_PROJECT}" == "{}" ]]; then + echo "skipping '${PROJECT}'... no API docs hosted for this project" + continue + fi + # expect to find a local folder, relative to the root of the repo, # named e.g. '_site/api/cudf' PROJECT_FOLDER="_site/api/${PROJECT}" @@ -55,6 +60,7 @@ for PROJECT in $(jq -r 'keys | .[]' <<< "${PROJECTS_TO_VERSIONS_JSON}"); do fi done # for VERSION + popd echo "---------------" echo "" done # for PROJECT From b4ed4e2b3582846d7c5300f96a4eed8c77e9fa6f Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 8 Aug 2025 12:32:31 -0500 Subject: [PATCH 7/8] restore 'inactive projects' stuff --- _data/docs.yml | 134 +++++++++++++++++++++++----------------- _includes/api-docs.html | 12 +++- api.md | 4 ++ 3 files changed, 94 insertions(+), 56 deletions(-) diff --git a/_data/docs.yml b/_data/docs.yml index 93ad233f9e1..c6518c4ff26 100644 --- a/_data/docs.yml +++ b/_data/docs.yml @@ -57,39 +57,6 @@ apis: legacy: 1 stable: 1 nightly: 1 - cuspatial: - name: cuSpatial - path: cuspatial - desc: 'cuSpatial is a GPU-accelerated vector GIS library including binary predicates (DE-9IM), point-in-polygon, spatial join, distances, and trajectory analysis.' - ghlink: https://github.com/rapidsai/cuspatial - cllink: https://github.com/rapidsai/cuspatial/blob/main/CHANGELOG.md - versions: - # enable or disable links; 0 = disabled, 1 = enabled - legacy: 1 - stable: 0 - nightly: 0 - cuproj: - name: cuProj - path: cuProj - desc: 'cuProj is a GPU-accelerated geographic and geodetic coordinate transformation library which supports projecting coordinates between coordinate reference systems (CRSes), compatible with PyProj.' - ghlink: https://github.com/rapidsai/cuspatial/tree/main/python/cuproj - cllink: https://github.com/rapidsai/cuspatial/blob/main/CHANGELOG.md # cuProj is housed within cuSpatial so updates remain in the cuSpatial changelog - versions: - # enable or disable links; 0 = disabled, 1 = enabled - legacy: 1 - stable: 0 - nightly: 0 - cusignal: - name: cusignal - path: cusignal - desc: 'cuSignal functionality has been moved to CuPy. Please see the CuPy documentation for more information.' - ghlink: https://github.com/cupy/cupy - cllink: https://docs.cupy.dev/en/latest/reference/scipy_signal.html - versions: - # enable or disable links; 0 = disabled, 1 = enabled - legacy: 0 - stable: 0 - nightly: 0 cudf-java: name: 'Java + cuDF' path: cudf-java @@ -205,28 +172,6 @@ libs: legacy: 1 stable: 1 nightly: 1 - libcuspatial: - name: libcuspatial - path: libcuspatial - desc: 'libcuspatial is a GPU-accelerated header-only C++ vector GIS library including binary predicates (DE-9IM), point-in-polygon, spatial join, distances, and trajectory analysis.' - ghlink: https://github.com/rapidsai/cuspatial - cllink: https://github.com/rapidsai/cuspatial/blob/main/CHANGELOG.md - versions: - # enable or disable links; 0 = disabled, 1 = enabled - legacy: 1 - stable: 0 - nightly: 0 - libcuproj: - name: libcuproj - path: libcuproj - desc: 'libcuproj is a C++ header-only library for GPU-accelerated geographic and geodetic coordinate transformation library which supports projecting coordinates between coordinate reference systems (CRSes), similar to PROJ.' - ghlink: https://github.com/rapidsai/cuspatial/tree/main/cpp/cuproj - cllink: https://github.com/rapidsai/cuspatial/blob/main/CHANGELOG.md # Shares a changelog with cuSpatial - versions: - # enable or disable links; 0 = disabled, 1 = enabled - legacy: 1 - stable: 0 - nightly: 0 libcuml: name: libcuml path: libcuml @@ -270,3 +215,82 @@ libs: legacy: 1 stable: 1 nightly: 1 + +# "inactive" RAPIDS projects, where docs links are preserved indefinitely but not +# expected to be get new releases +inactive-projects: + cuproj: + name: cuProj + path: cuProj + desc: 'cuProj is a GPU-accelerated geographic and geodetic coordinate transformation library which supports projecting coordinates between coordinate reference systems (CRSes), compatible with PyProj.' + ghlink: https://github.com/rapidsai/cuspatial/tree/main/python/cuproj + cllink: https://github.com/rapidsai/cuspatial/blob/main/CHANGELOG.md # cuProj is housed within cuSpatial so updates remain in the cuSpatial changelog + versions: + # enable or disable links; 0 = disabled, 1 = enabled + legacy: 1 + stable: 1 + nightly: 0 + # always use these specific versions for these specific paths, regardless + # of the current state of RAPIDS + version-overrides: + legacy: "25.02" + stable: "25.04" + cusignal: + name: cusignal + path: cusignal + desc: 'cuSignal functionality has been moved to CuPy. Please see the CuPy documentation for more information.' + ghlink: https://github.com/cupy/cupy + cllink: https://docs.cupy.dev/en/latest/reference/scipy_signal.html + versions: + # enable or disable links; 0 = disabled, 1 = enabled + legacy: 0 + stable: 0 + nightly: 0 + cuspatial: + name: cuSpatial + path: cuspatial + desc: 'cuSpatial is a GPU-accelerated vector GIS library including binary predicates (DE-9IM), point-in-polygon, spatial join, distances, and trajectory analysis.' + ghlink: https://github.com/rapidsai/cuspatial + cllink: https://github.com/rapidsai/cuspatial/blob/main/CHANGELOG.md + versions: + # enable or disable links; 0 = disabled, 1 = enabled + legacy: 1 + stable: 1 + nightly: 0 + # always use these specific versions for these specific paths, regardless + # of the current state of RAPIDS + version-overrides: + legacy: "25.02" + stable: "25.04" + libcuproj: + name: libcuproj + path: libcuproj + desc: 'libcuproj is a C++ header-only library for GPU-accelerated geographic and geodetic coordinate transformation library which supports projecting coordinates between coordinate reference systems (CRSes), similar to PROJ.' + ghlink: https://github.com/rapidsai/cuspatial/tree/main/cpp/cuproj + cllink: https://github.com/rapidsai/cuspatial/blob/main/CHANGELOG.md # Shares a changelog with cuSpatial + versions: + # enable or disable links; 0 = disabled, 1 = enabled + legacy: 1 + stable: 1 + nightly: 0 + # always use these specific versions for these specific paths, regardless + # of the current state of RAPIDS + version-overrides: + legacy: "25.02" + stable: "25.04" + libcuspatial: + name: libcuspatial + path: libcuspatial + desc: 'libcuspatial is a GPU-accelerated header-only C++ vector GIS library including binary predicates (DE-9IM), point-in-polygon, spatial join, distances, and trajectory analysis.' + ghlink: https://github.com/rapidsai/cuspatial + cllink: https://github.com/rapidsai/cuspatial/blob/main/CHANGELOG.md + versions: + # enable or disable links; 0 = disabled, 1 = enabled + legacy: 1 + stable: 1 + nightly: 0 + # always use these specific versions for these specific paths, regardless + # of the current state of RAPIDS + version-overrides: + legacy: "25.02" + stable: "25.04" diff --git a/_includes/api-docs.html b/_includes/api-docs.html index da5a66cc507..cf7613ccabe 100644 --- a/_includes/api-docs.html +++ b/_includes/api-docs.html @@ -8,7 +8,17 @@ {% assign versions = api.versions | sort | where_exp: "item", "item[1] == 1" | join: "" | split: "1" | reverse %} ### {{ api.name }} {{ api.desc }} -#### DOCS {% for version_name in versions %} {% if api.name == "libucxx" %} **[{{ version_name }} ({{ site.data.releases[version_name].ucxx_version }})](/api/{{ api.path }}/{{ version_name }})** {% else %} **[{{ version_name }} ({{ site.data.releases[version_name].version }})](/api/{{ api.path }}/{{ version_name }})** {% endif %} {% unless forloop.last %}|{% endunless %} {% endfor %} +#### DOCS {% for version_name in versions -%} + {%- if api.version-overrides -%} + **[{{ version_name }} ({{ api.version-overrides[version_name] }})](/api/{{ api.path }}/{{ version_name }})** + {%- elsif api.name == "libucxx" -%} + **[{{ version_name }} ({{ site.data.releases[version_name].ucxx_version }})](/api/{{ api.path }}/{{ version_name }})** + {%- else -%} + **[{{ version_name }} ({{ site.data.releases[version_name].version }})](/api/{{ api.path }}/{{ version_name }})** + {%- endif -%} + {%- unless forloop.last %} | {% endunless -%} +{%- endfor %} + #### LINKS {% if api.cllink %} **[changelog]({{ api.cllink }}){:target="_blank"}** | {% endif %} **[github]({{ api.ghlink }}){:target="_blank"}** {: .mb-7 } {% endif %} diff --git a/api.md b/api.md index e1959e0e5b9..88c8e4988bb 100644 --- a/api.md +++ b/api.md @@ -29,3 +29,7 @@ select the docs that fit your needs. ## RAPIDS Libraries {% include api-docs.html data=site.data.docs.libs %} + +## Inactive Projects + +{% include api-docs.html data=site.data.docs.inactive-projects %} From cb9b33152b06a2eb945d686ddc2e87a8223f4183 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 8 Aug 2025 14:33:18 -0500 Subject: [PATCH 8/8] more refactoring --- .gitignore | 1 + _data/docs.yml | 121 ------------------- ci/customization/customize_doc.py | 68 ++++++++--- ci/customization/customize_docs_in_folder.sh | 6 +- ci/post-process.sh | 2 + 5 files changed, 52 insertions(+), 146 deletions(-) diff --git a/.gitignore b/.gitignore index 22dfb4d0bf8..7c681d2f829 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ lib_map.json .netlify __pycache__/ node_modules +projects-to-versions.json rapids-docs-env/ diff --git a/_data/docs.yml b/_data/docs.yml index c6518c4ff26..c348e0dd614 100644 --- a/_data/docs.yml +++ b/_data/docs.yml @@ -35,28 +35,6 @@ apis: legacy: 1 stable: 1 nightly: 1 - cugraph: - name: cuGraph - path: cugraph - desc: 'cuGraph is a GPU accelerated graph analytics library, with functionality like NetworkX, which is seamlessly integrated into the RAPIDS data science platform. cuGraph supports GNNs with PyG, DGL packages, cugraph-service for analytics on a remote graph, and WHOLEGRAPH for memory management.' - ghlink: https://github.com/rapidsai/cugraph - cllink: https://github.com/rapidsai/cugraph/blob/main/CHANGELOG.md - versions: - # enable or disable links; 0 = disabled, 1 = enabled - legacy: 1 - stable: 1 - nightly: 1 - cuxfilter: - name: cuxfilter - path: cuxfilter - desc: 'cuxfilter acts as a connector library, which provides the connections between different visualization libraries and a GPU dataframe without much hassle. This also allows the user to use charts from different libraries in a single dashboard, while also providing the interaction.' - ghlink: https://github.com/rapidsai/cuxfilter - cllink: https://github.com/rapidsai/cuxfilter/blob/main/CHANGELOG.md - versions: - # enable or disable links; 0 = disabled, 1 = enabled - legacy: 1 - stable: 1 - nightly: 1 cudf-java: name: 'Java + cuDF' path: cudf-java @@ -68,83 +46,6 @@ apis: legacy: 1 stable: 0 nightly: 0 - cucim: - name: cuCIM - path: cucim - desc: 'The RAPIDS cuCIM is an extensible toolkit designed to provide GPU accelerated I/O, computer vision & image processing primitives for N-Dimensional images with a focus on biomedical imaging.' - ghlink: https://github.com/rapidsai/cucim - cllink: https://github.com/rapidsai/cucim/blob/main/CHANGELOG.md - versions: - # enable or disable links; 0 = disabled, 1 = enabled - legacy: 1 - stable: 1 - nightly: 1 - cuvs: - name: cuVS - path: cuvs - desc: 'cuVS is a library for GPU-accelerated vector search and clustering.' - ghlink: https://github.com/rapidsai/cuvs - cllink: https://github.com/rapidsai/cuvs/blob/main/CHANGELOG.md - versions: - # enable or disable links; 0 = disabled, 1 = enabled - legacy: 1 - stable: 1 - nightly: 1 - kvikio: - name: KvikIO - path: kvikio - desc: "KvikIO is a Python and C++ library for high performance file IO using GPUDirect Storage (GDS)." - ghlink: https://github.com/rapidsai/kvikio - cllink: https://github.com/rapidsai/kvikio/blob/main/CHANGELOG.md - versions: - # enable or disable links; 0 = disabled, 1 = enabled - legacy: 1 - stable: 1 - nightly: 1 - raft: - name: RAFT - path: raft - desc: "RAFT contains fundamental widely-used algorithms and primitives for vector search, machine learning, and information retrieval." - ghlink: https://github.com/rapidsai/raft - cllink: https://github.com/rapidsai/raft/blob/main/CHANGELOG.md - versions: - # enable or disable links; 0 = disabled, 1 = enabled - legacy: 1 - stable: 1 - nightly: 1 - dask-cuda: - name: Dask-CUDA - path: dask-cuda - desc: "Various utilities to improve deployment and management of Dask workers on CUDA-enabled systems." - ghlink: https://github.com/rapidsai/dask-cuda - cllink: https://github.com/rapidsai/dask-cuda/blob/main/CHANGELOG.md - versions: - # enable or disable links; 0 = disabled, 1 = enabled - legacy: 1 - stable: 1 - nightly: 1 - rmm: - name: RMM - path: rmm - desc: 'RAPIDS Memory Manager (RMM) is a central place for all device memory allocations in cuDF (C++ and Python) and other RAPIDS libraries. In addition, it is a replacement allocator for CUDA Device Memory (and CUDA Managed Memory) and a pool allocator to make CUDA device memory allocation / deallocation faster and asynchronous.' - ghlink: https://github.com/rapidsai/rmm - cllink: https://github.com/rapidsai/rmm/blob/main/CHANGELOG.md - versions: - # enable or disable links; 0 = disabled, 1 = enabled - legacy: 1 - stable: 1 - nightly: 1 - rapidsmpf: - name: RapidsMPF - path: rapidsmpf - desc: 'RAPIDS Multi-Process Foundation (rapidsmpf) is a collection of multi-GPU, distributed memory algorithms written in C++ and exposed to Python.' - ghlink: https://github.com/rapidsai/rapidsmpf - versions: - # enable or disable links; 0 = disabled, 1 = enabled - legacy: 0 - stable: 0 - nightly: 1 - # RAPIDS "Libs" - lower-level libraries that are building blocks for creating # custom tools and integrate with other libraries @@ -172,28 +73,6 @@ libs: legacy: 1 stable: 1 nightly: 1 - libcuml: - name: libcuml - path: libcuml - desc: 'libcuml is a C/C++ CUDA library for cuML.' - ghlink: https://github.com/rapidsai/cuml - cllink: https://github.com/rapidsai/cuml/blob/main/CHANGELOG.md - versions: - # enable or disable links; 0 = disabled, 1 = enabled - legacy: 1 - stable: 1 - nightly: 1 - libkvikio: - name: libkvikio - path: libkvikio - desc: "libkvikio is a C++ header-only library providing bindings to cuFile, which enables GPUDirect Storage (GDS)." - ghlink: https://github.com/rapidsai/kvikio - cllink: https://github.com/rapidsai/kvikio/blob/main/CHANGELOG.md - versions: - # enable or disable links; 0 = disabled, 1 = enabled - legacy: 1 - stable: 1 - nightly: 1 libucxx: name: libucxx path: libucxx diff --git a/ci/customization/customize_doc.py b/ci/customization/customize_doc.py index a8fa5494449..714d635aceb 100644 --- a/ci/customization/customize_doc.py +++ b/ci/customization/customize_doc.py @@ -14,24 +14,26 @@ from util import r_versions FILEPATH = sys.argv[1] -IS_UCXX_FILE = len(sys.argv) > 2 and sys.argv[2] == "--is-ucxx" LIB_MAP_PATH = os.path.join(os.path.dirname(__file__), "lib_map.json") RELEASES_PATH = os.path.join( os.path.dirname(__file__), "../", "../", "_data", "releases.json" ) +PROJECTS_TO_VERSIONS_PATH = os.path.join(os.path.dirname(__file__), "projects-to-versions.json") with open(LIB_MAP_PATH) as fp: LIB_PATH_DICT = json.load(fp) +# get version overrides for inactive projects +with open(PROJECTS_TO_VERSIONS_PATH) as f: + PROJECTS_TO_VERSIONS_DICT = json.load(f) + with open(RELEASES_PATH) as fp: release_data = json.load(fp) VERSIONS_DICT = { - "nightly": release_data["nightly"][ - "ucxx_version" if IS_UCXX_FILE else "version" - ], - "stable": release_data["stable"]["ucxx_version" if IS_UCXX_FILE else "version"], - "legacy": release_data["legacy"]["ucxx_version" if IS_UCXX_FILE else "version"], + "nightly": release_data["nightly"]["version"], + "stable": release_data["stable"]["version"], + "legacy": release_data["legacy"]["version"], } SCRIPT_TAG_ID = "rapids-selector-js" @@ -41,7 +43,7 @@ FA_TAG_ID = "rapids-fa-tag" -def get_version_from_fp(): +def get_version_from_fp(*, stable_version: str): """ Determines if the current HTML document is for legacy, stable, or nightly versions based on the file path @@ -49,22 +51,21 @@ def get_version_from_fp(): match = re.search(r"/(\d?\d\.\d\d)/", FILEPATH) version_number_str = r_versions(match.group(1)) version_name = "stable" - if version_number_str.is_greater_than(VERSIONS_DICT["stable"]): + if version_number_str.is_greater_than(stable_version): version_name = "nightly" - if version_number_str.is_less_than(VERSIONS_DICT["stable"]): + if version_number_str.is_less_than(stable_version): version_name = "legacy" return {"name": version_name, "number": version_number_str} -def get_lib_from_fp(): +def get_lib_from_fp(*, filepath: str): """ Determines the current RAPIDS library based on the file path """ - for lib in LIB_PATH_DICT.keys(): - if re.search(f"(^{lib}/|/{lib}/)", FILEPATH): + if re.search(f"(^{lib}/|/{lib}/)", filepath): return lib - raise Exception(f"Couldn't find valid library name in {FILEPATH}.") + raise ValueError(f"Couldn't find valid library name in '{filepath}'.") def create_home_container(soup): @@ -94,20 +95,25 @@ def add_font_awesome(soup): soup.head.append(fa_tag) -def create_version_options(): +def create_version_options( + *, + stable_version: str | None, + legacy_version: str | None, + nightly_version: str | None, +) -> list[dict[str, bool | str]]: """ Creates options for legacy/stable/nightly selector """ options = [] - doc_version = get_version_from_fp() - doc_lib = get_lib_from_fp() + doc_version = get_version_from_fp(stable_version=stable_version) + doc_lib = get_lib_from_fp(filepath=FILEPATH) doc_is_extra_legacy = ( # extra legacy means the doc version is older then current legacy doc_version["name"] == "legacy" - and VERSIONS_DICT["legacy"] != doc_version["number"] + and legacy_version != doc_version["number"] ) doc_is_extra_nightly = ( # extra nightly means the doc version is newer then current nightly doc_version["name"] == "nightly" - and VERSIONS_DICT["nightly"] != doc_version["number"] + and nightly_version != doc_version["number"] ) for version_name, version_path in [ (_, path) for _, path in LIB_PATH_DICT[doc_lib].items() if path is not None @@ -135,7 +141,7 @@ def create_library_options(): """ Creates options for library selector """ - doc_lib = get_lib_from_fp() + doc_lib = get_lib_from_fp(filepath=FILEPATH) options = [] for lib, lib_versions in LIB_PATH_DICT.items(): @@ -294,6 +300,18 @@ def main(): parse the file and add library/version selectors and a Home button """ print(f"--- {FILEPATH} ---") + + # exit early for projects with no API docs (like cusignal) + project_name = get_lib_from_fp(filepath=FILEPATH) + if not PROJECTS_TO_VERSIONS_DICT[project_name]: + print(f"'{project_name}': no API docs requested. Skipping customization.") + return + + # figure out versions + stable_version = PROJECTS_TO_VERSIONS_DICT[project_name].get("stable") + nightly_version = PROJECTS_TO_VERSIONS_DICT[project_name].get("nightly") + legacy_version = PROJECTS_TO_VERSIONS_DICT[project_name].get("legacy") + with open(FILEPATH) as fp: soup = BeautifulSoup(fp, "html5lib") @@ -309,7 +327,14 @@ def main(): # Create new elements home_btn_container = create_home_container(soup) library_selector = create_selector(soup, create_library_options()) - version_selector = create_selector(soup, create_version_options()) + version_selector = create_selector( + soup, + create_version_options( + stable_version=stable_version, + legacy_version=legacy_version, + nightly_version=nightly_version, + ) + ) container = soup.new_tag("div", id=f"rapids-{doc_type}-container") script_tag = create_script_tag(soup) [pix_head_tag, pix_body_tag] = create_pixel_tags(soup) @@ -331,5 +356,8 @@ def main(): fp.write(soup.decode(formatter="html5")) +import pdb +pdb.set_trace() + if __name__ == "__main__": main() diff --git a/ci/customization/customize_docs_in_folder.sh b/ci/customization/customize_docs_in_folder.sh index 73bedb10eb2..30410e49a04 100755 --- a/ci/customization/customize_docs_in_folder.sh +++ b/ci/customization/customize_docs_in_folder.sh @@ -50,11 +50,7 @@ for FILE in $(grep "${JTD_SEARCH_TERM}\|${DOXYGEN_SEARCH_TERM}\|${PYDATA_SEARCH_ --exclude-dir=legacy \ --exclude-dir=cudf-java \ ${FOLDER_TO_CUSTOMIZE} ); do - if [[ ${FILE} == *"ucxx"* ]]; then - python ${SCRIPT_SRC_FOLDER}/customize_doc.py $(realpath ${FILE}) --is-ucxx - else - python ${SCRIPT_SRC_FOLDER}/customize_doc.py $(realpath ${FILE}) - fi + python ${SCRIPT_SRC_FOLDER}/customize_doc.py $(realpath ${FILE}) echo "" # line break for readability done IFS="$OIFS" diff --git a/ci/post-process.sh b/ci/post-process.sh index 740e163c590..c79233e1a43 100755 --- a/ci/post-process.sh +++ b/ci/post-process.sh @@ -13,4 +13,6 @@ pip install -r "${CURRENT_DIR}/customization/requirements.txt" "${CURRENT_DIR}"/customization/lib_map.sh +"${CURRENT_DIR}"/get-projects-to-versions.sh > "${CURRENT_DIR}"/customization/projects-to-versions.json + "${CURRENT_DIR}"/customization/customize_docs_in_folder.sh "_site/api"