From ce40ff86c8727c512a059bb58bfbdc4e59b3edc1 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 22 Aug 2025 11:08:54 -0500 Subject: [PATCH 01/16] Add notebook workflow to PR --- .github/workflows/pr.yaml | 10 ++++++++++ .github/workflows/test.yaml | 12 ++++++++++++ ci/test_notebooks.sh | 39 ++++++++++++++----------------------- ci/utils/nbtest.sh | 2 +- 4 files changed, 38 insertions(+), 25 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index b23bf5cfc4..6cdc51a590 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -147,6 +147,16 @@ jobs: run_codecov: false build_type: pull-request script: ci/test_python.sh + conda-notebook-tests: + needs: [conda-python-build] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.10 + with: + build_type: pull-request + node_type: "gpu-l4-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:25.10-latest" + script: ci/test_notebooks.sh docs-build: needs: conda-python-build secrets: inherit diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 80ba1f869f..371c4a7056 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -69,3 +69,15 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} script: ci/test_wheel_cuopt_server.sh + conda-notebook-tests: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.10 + with: + build_type: ${{ inputs.build_type }} + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + node_type: "gpu-l4-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:25.10-latest" + script: ci/test_notebooks.sh \ No newline at end of file diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index feb711e78e..e9d052a038 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -21,13 +21,22 @@ set -euo pipefail CUOPT_VERSION="$(rapids-version)" +rapids-logger "Downloading artifacts from previous jobs" +CPP_CHANNEL=$(rapids-download-conda-from-github cpp) +PYTHON_CHANNEL=$(rapids-download-conda-from-github python) + rapids-logger "Generate notebook testing dependencies" + +ENV_YAML_DIR="$(mktemp -d)" + rapids-dependency-file-generator \ --output conda \ --file-key test_notebooks \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + --prepend-channel "${CPP_CHANNEL}" \ + --prepend-channel "${PYTHON_CHANNEL}" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${ENV_YAML_DIR}/env.yaml" -rapids-mamba-retry env create --yes -f env.yaml -n test +rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n test # Temporarily allow unbound variables for conda activation. set +u @@ -36,25 +45,15 @@ set -u rapids-print-env -rapids-logger "Downloading artifacts from previous jobs" -CPP_CHANNEL=$(rapids-download-conda-from-github cpp) -PYTHON_CHANNEL=$(rapids-download-conda-from-github python) - -rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - --channel "${PYTHON_CHANNEL}" \ - "libcuopt=${CUOPT_VERSION}" \ - "cuopt=${CUOPT_VERSION}" \ - "cuopt-server=${CUOPT_VERSION}" - -pip install python/cuopt_self_hosted/ +EXAMPLES_BRANCH="branch-${CUOPT_VERSION%.*}" +rapids-logger "Cloning cuopt-examples repository for branch: ${EXAMPLES_BRANCH}" +git clone --single-branch --branch "${EXAMPLES_BRANCH}" https://github.com/NVIDIA/cuopt-examples.git NBTEST="$(realpath "$(dirname "$0")/utils/nbtest.sh")" NBLIST_PATH="$(realpath "$(dirname "$0")/utils/notebook_list.py")" NBLIST=$(python "${NBLIST_PATH}") -SERVER_WAIT_DELAY=10 -pushd notebooks +pushd cuopt-examples EXITCODE=0 trap "EXITCODE=1" ERR @@ -62,12 +61,6 @@ trap "EXITCODE=1" ERR rapids-logger "Start cuopt-server" set +e -#python -c "from cuopt_server.cuopt_service import run_server; run_server()" & - -python -m cuopt_server.cuopt_service & -export SERVER_PID=$! -sleep "${SERVER_WAIT_DELAY}" -curl http://0.0.0.0:5000/cuopt/health rapids-logger "Start notebooks tests" for nb in ${NBLIST}; do @@ -76,6 +69,4 @@ for nb in ${NBLIST}; do done rapids-logger "Notebook test script exiting with value: $EXITCODE" -kill -s SIGTERM $SERVER_PID -wait $SERVER_PID exit ${EXITCODE} diff --git a/ci/utils/nbtest.sh b/ci/utils/nbtest.sh index 1b99d68247..064dec8dd0 100755 --- a/ci/utils/nbtest.sh +++ b/ci/utils/nbtest.sh @@ -50,7 +50,7 @@ get_ipython().run_cell_magic=my_run_cell_magic NO_COLORS=--colors=NoColor EXITCODE=0 -# PWD is REPO_ROOT/notebooks +# PWD is REPO_ROOT/cuopt-examples NBTMPDIR="${PWD}/../tmp" mkdir -p "${NBTMPDIR}" NBUTILS="${PWD}/external" From 2c7aeb1bb9af81b0d2e709818443429d1043178d Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 22 Aug 2025 12:14:53 -0500 Subject: [PATCH 02/16] fix checks --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 371c4a7056..080d81a7ae 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -80,4 +80,4 @@ jobs: node_type: "gpu-l4-latest-1" arch: "amd64" container_image: "rapidsai/ci-conda:25.10-latest" - script: ci/test_notebooks.sh \ No newline at end of file + script: ci/test_notebooks.sh From a0f51c9fe833db78f055d2116f4a24e35d9cb8e3 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 22 Aug 2025 12:16:44 -0500 Subject: [PATCH 03/16] fix pr --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 6cdc51a590..378fe1be24 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -35,7 +35,7 @@ jobs: - conda-python-tests - docs-build - wheel-build-libcuopt - # - conda-notebook-tests + - conda-notebook-tests - wheel-build-cuopt - wheel-tests-cuopt - wheel-build-cuopt-server @@ -148,7 +148,7 @@ jobs: build_type: pull-request script: ci/test_python.sh conda-notebook-tests: - needs: [conda-python-build] + needs: [conda-python-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.10 with: From 5d04191ff92711e115645a499fdb3d2860696728 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 25 Aug 2025 18:45:39 -0500 Subject: [PATCH 04/16] fixes --- ci/test_notebooks.sh | 20 +++++++- ci/utils/nbtest.sh | 109 +++++++++++++++++++++++-------------------- dependencies.yaml | 9 +--- 3 files changed, 78 insertions(+), 60 deletions(-) diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index e9d052a038..31abce8910 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -45,16 +45,28 @@ set -u rapids-print-env -EXAMPLES_BRANCH="branch-${CUOPT_VERSION%.*}" + +#EXAMPLES_BRANCH="branch-${CUOPT_VERSION%.*}" +EXAMPLES_BRANCH="fix_testing" rapids-logger "Cloning cuopt-examples repository for branch: ${EXAMPLES_BRANCH}" + +# Remove any existing cuopt-examples directory +rm -rf cuopt-examples git clone --single-branch --branch "${EXAMPLES_BRANCH}" https://github.com/NVIDIA/cuopt-examples.git NBTEST="$(realpath "$(dirname "$0")/utils/nbtest.sh")" NBLIST_PATH="$(realpath "$(dirname "$0")/utils/notebook_list.py")" -NBLIST=$(python "${NBLIST_PATH}") pushd cuopt-examples +NBLIST=$(python "${NBLIST_PATH}") + +# Install any wheels if they are present in any subdirectory +find . -type f -name "*.whl" | while read whl; do + echo "Installing wheel: $whl" + pip install "$whl" +done + EXITCODE=0 trap "EXITCODE=1" ERR @@ -66,6 +78,10 @@ rapids-logger "Start notebooks tests" for nb in ${NBLIST}; do nvidia-smi ${NBTEST} "${nb}" + if [ $? -ne 0 ]; then + echo "Notebook ${nb} failed to execute. Exiting." + exit 1 + fi done rapids-logger "Notebook test script exiting with value: $EXITCODE" diff --git a/ci/utils/nbtest.sh b/ci/utils/nbtest.sh index 064dec8dd0..97ecc7d177 100755 --- a/ci/utils/nbtest.sh +++ b/ci/utils/nbtest.sh @@ -12,75 +12,82 @@ # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# See the License for the specific language governing permissions and limitations. +# +# This script executes Jupyter notebooks directly using nbconvert. set +e # do not abort the script on error set -o pipefail # piped commands propagate their error set -E # ERR traps are inherited by subcommands trap "EXITCODE=1" ERR -# Prepend the following code to all scripts generated from nbconvert. This -# allows all cell and line magic code to run and update the namespace as if -# running in jupyter, but will also tolerate failures due to running in a -# non-jupyter env. -# Note: depending on the assumptions of the notebook script, ignoring failures -# may not be acceptable (meaning the converted notebook simply cannot run -# outside of jupyter as-is), hence the warning. -MAGIC_OVERRIDE_CODE=" -def my_run_line_magic(*args, **kwargs): - g=globals() - l={} - for a in args: - try: - exec(str(a),g,l) - except Exception as e: - print('WARNING: %s\n While executing this magic function code:\n%s\n continuing...\n' % (e, a)) - else: - g.update(l) - -def my_run_cell_magic(*args, **kwargs): - my_run_line_magic(*args, **kwargs) - -get_ipython().run_line_magic=my_run_line_magic -get_ipython().run_cell_magic=my_run_cell_magic +# Save the original directory +ORIGINAL_DIR=$(pwd) -" - -NO_COLORS=--colors=NoColor EXITCODE=0 -# PWD is REPO_ROOT/cuopt-examples -NBTMPDIR="${PWD}/../tmp" -mkdir -p "${NBTMPDIR}" -NBUTILS="${PWD}/external" -cp -r "${NBUTILS}/python/utils" "${NBTMPDIR}/." -cp -r "${NBUTILS}/server/notebook_utils" "${NBTMPDIR}/." -cp -r "${NBUTILS}/dli/helper_function" "${NBTMPDIR}/." -cd "${NBTMPDIR}" || exit 1 - for nb in "$@"; do NBFILENAME=$nb NBNAME=${NBFILENAME%.*} NBNAME=${NBNAME##*/} - NBTESTSCRIPT=${NBTMPDIR}/${NBNAME}-test.py - shift + + # Get the directory where the notebook is located + NBDIR=$(dirname "$NBFILENAME") + + echo "Changing to directory: ${NBDIR}" + cd "${NBDIR}" + + # Output the executed notebook in the same folder + EXECUTED_NOTEBOOK="${NBNAME}-executed.ipynb" echo -------------------------------------------------------------------------------- echo STARTING: "${NBNAME}" echo -------------------------------------------------------------------------------- - jupyter nbconvert --to script ../"${NBFILENAME}" --output "${NBTMPDIR}"/"${NBNAME}"-test - python "${PWD}/../ci/utils/dli_nb_strip.py" "${NBTESTSCRIPT}" - echo "${MAGIC_OVERRIDE_CODE}" > "${NBTMPDIR}"/tmpfile - cat "${NBTESTSCRIPT}" >> "${NBTMPDIR}"/tmpfile - mv "${NBTMPDIR}"/tmpfile "${NBTESTSCRIPT}" - echo "Running \"ipython ${NO_COLORS} ${NBTESTSCRIPT}\" on $(date)" - echo - time timeout 30m bash -c "ipython ${NO_COLORS} ${NBTESTSCRIPT}; EC=\$?; echo -------------------------------------------------------------------------------- ; echo DONE: ${NBNAME}; exit \$EC" - NBEXITCODE=$? - echo EXIT CODE: ${NBEXITCODE} - echo + echo "Executing notebook: ${NBFILENAME}" + echo "Output will be saved to: ${EXECUTED_NOTEBOOK}" + + # Extract and execute pip install commands from notebook + echo "Checking for pip install commands in notebook..." + PIP_COMMANDS=$(grep -h "pip install" "$NBNAME.ipynb" 2>/dev/null | grep -v "#" | sed 's/^[[:space:]]*//' | sed 's/^["'"'"']*//' | sed 's/["'"'"']*$//' || true) + + if [ -n "$PIP_COMMANDS" ]; then + echo "Found pip install commands:" + echo "$PIP_COMMANDS" + echo "Executing pip install commands..." + echo "$PIP_COMMANDS" | while read -r cmd; do + echo "Processing command: '$cmd'" + if [[ "$cmd" =~ ^!?pip[[:space:]]+install ]]; then + echo "Running: $cmd" + # Remove the ! prefix if present for execution + EXEC_CMD="${cmd#!}" + # Clean up escaped quotes and extra quotes + EXEC_CMD=$(echo "$EXEC_CMD" | sed 's/\\"/"/g' | sed 's/^"//' | sed 's/"$//') + echo "Executing: $EXEC_CMD" + eval "$EXEC_CMD" + if [ $? -eq 0 ]; then + echo "✓ Successfully executed: $cmd" + else + echo "✗ Failed to execute: $cmd" + fi + else + echo "Command '$cmd' did not match pip install pattern" + fi + done + fi + + # Execute notebook with default kernel + jupyter nbconvert --execute "${NBNAME}.ipynb" --to notebook --output "${EXECUTED_NOTEBOOK}" --ExecutePreprocessor.kernel_name="python3" + + if [ $? -eq 0 ]; then + echo "Notebook executed successfully: ${EXECUTED_NOTEBOOK}" + else + echo "ERROR: Failed to execute notebook: ${NBFILENAME}" + EXITCODE=1 + fi + + echo "Returning to original directory: ${ORIGINAL_DIR}" + cd "${ORIGINAL_DIR}" done exit ${EXITCODE} diff --git a/dependencies.yaml b/dependencies.yaml index 07de525333..c038534374 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -69,6 +69,8 @@ files: - cuda_version - notebooks - py_version + - depends_on_cuopt_server + - depends_on_cuopt_sh_client checks: output: none includes: @@ -770,15 +772,8 @@ dependencies: common: - output_types: [conda, requirements] packages: - - breathe - - folium - - geopandas - ipython - - matplotlib - notebook - - polyline - - scipy - - libgdal<3.9.0 - output_types: [conda] packages: - *jsonref From 7968bab2afb1c90408c740bba76f2e37bfaae063 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Tue, 26 Aug 2025 09:53:14 -0500 Subject: [PATCH 05/16] fix style --- ci/utils/nbtest.sh | 12 ++++++------ conda/environments/all_cuda-128_arch-aarch64.yaml | 6 ------ conda/environments/all_cuda-128_arch-x86_64.yaml | 6 ------ 3 files changed, 6 insertions(+), 18 deletions(-) diff --git a/ci/utils/nbtest.sh b/ci/utils/nbtest.sh index 97ecc7d177..8c201ad079 100755 --- a/ci/utils/nbtest.sh +++ b/ci/utils/nbtest.sh @@ -30,7 +30,7 @@ for nb in "$@"; do NBFILENAME=$nb NBNAME=${NBFILENAME%.*} NBNAME=${NBNAME##*/} - + # Get the directory where the notebook is located NBDIR=$(dirname "$NBFILENAME") @@ -61,8 +61,8 @@ for nb in "$@"; do echo "Running: $cmd" # Remove the ! prefix if present for execution EXEC_CMD="${cmd#!}" - # Clean up escaped quotes and extra quotes - EXEC_CMD=$(echo "$EXEC_CMD" | sed 's/\\"/"/g' | sed 's/^"//' | sed 's/"$//') + # Clean up escaped quotes, extra quotes, and newlines + EXEC_CMD=$(echo "$EXEC_CMD" | sed 's/\\"/"/g' | sed 's/^"//' | sed 's/"$//' | tr -d '\n\r') echo "Executing: $EXEC_CMD" eval "$EXEC_CMD" if [ $? -eq 0 ]; then @@ -75,17 +75,17 @@ for nb in "$@"; do fi done fi - + # Execute notebook with default kernel jupyter nbconvert --execute "${NBNAME}.ipynb" --to notebook --output "${EXECUTED_NOTEBOOK}" --ExecutePreprocessor.kernel_name="python3" - + if [ $? -eq 0 ]; then echo "Notebook executed successfully: ${EXECUTED_NOTEBOOK}" else echo "ERROR: Failed to execute notebook: ${NBFILENAME}" EXITCODE=1 fi - + echo "Returning to original directory: ${ORIGINAL_DIR}" cd "${ORIGINAL_DIR}" done diff --git a/conda/environments/all_cuda-128_arch-aarch64.yaml b/conda/environments/all_cuda-128_arch-aarch64.yaml index ba6b6945e3..796f4a215a 100644 --- a/conda/environments/all_cuda-128_arch-aarch64.yaml +++ b/conda/environments/all_cuda-128_arch-aarch64.yaml @@ -27,9 +27,7 @@ dependencies: - doxygen=1.9.1 - exhale - fastapi -- folium - gcc_linux-aarch64=13.* -- geopandas - gmock - gtest - httpx @@ -38,11 +36,9 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libgdal<3.9.0 - libraft-headers==25.10.* - librmm==25.10.* - make -- matplotlib - msgpack-numpy==0.4.8 - msgpack-python==1.1.0 - myst-nb @@ -56,7 +52,6 @@ dependencies: - pandas>=2.0 - pexpect - pip -- polyline - pre-commit - psutil>=5.9,<6.0a0 - pylibraft==25.10.*,>=0.0.0a0 @@ -71,7 +66,6 @@ dependencies: - requests - rmm==25.10.*,>=0.0.0a0 - scikit-build-core>=0.10.0 -- scipy - sphinx - sphinx-copybutton - sphinx-design diff --git a/conda/environments/all_cuda-128_arch-x86_64.yaml b/conda/environments/all_cuda-128_arch-x86_64.yaml index f91c229570..bb00ec1d11 100644 --- a/conda/environments/all_cuda-128_arch-x86_64.yaml +++ b/conda/environments/all_cuda-128_arch-x86_64.yaml @@ -27,9 +27,7 @@ dependencies: - doxygen=1.9.1 - exhale - fastapi -- folium - gcc_linux-64=13.* -- geopandas - gmock - gtest - httpx @@ -38,11 +36,9 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libgdal<3.9.0 - libraft-headers==25.10.* - librmm==25.10.* - make -- matplotlib - msgpack-numpy==0.4.8 - msgpack-python==1.1.0 - myst-nb @@ -56,7 +52,6 @@ dependencies: - pandas>=2.0 - pexpect - pip -- polyline - pre-commit - psutil>=5.9,<6.0a0 - pylibraft==25.10.*,>=0.0.0a0 @@ -71,7 +66,6 @@ dependencies: - requests - rmm==25.10.*,>=0.0.0a0 - scikit-build-core>=0.10.0 -- scipy - sphinx - sphinx-copybutton - sphinx-design From 7cf79c01ae504f10cf9c8e19de9e04cf5aaf5bf8 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Tue, 26 Aug 2025 10:08:52 -0500 Subject: [PATCH 06/16] fix style --- ci/test_notebooks.sh | 2 +- ci/utils/nbtest.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index 31abce8910..844a97ded9 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -19,7 +19,7 @@ set -euo pipefail . /opt/conda/etc/profile.d/conda.sh -CUOPT_VERSION="$(rapids-version)" +# CUOPT_VERSION variable removed as it was unused rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-github cpp) diff --git a/ci/utils/nbtest.sh b/ci/utils/nbtest.sh index 8c201ad079..9d622fe617 100755 --- a/ci/utils/nbtest.sh +++ b/ci/utils/nbtest.sh @@ -35,7 +35,7 @@ for nb in "$@"; do NBDIR=$(dirname "$NBFILENAME") echo "Changing to directory: ${NBDIR}" - cd "${NBDIR}" + cd "${NBDIR}" || exit 1 # Output the executed notebook in the same folder EXECUTED_NOTEBOOK="${NBNAME}-executed.ipynb" @@ -87,7 +87,7 @@ for nb in "$@"; do fi echo "Returning to original directory: ${ORIGINAL_DIR}" - cd "${ORIGINAL_DIR}" + cd "${ORIGINAL_DIR}" || exit 1 done exit ${EXITCODE} From 43e6418b1cfc01fe8aa3c4ce2db715d673976cb3 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Tue, 26 Aug 2025 17:35:45 -0500 Subject: [PATCH 07/16] fix pip install --- ci/utils/nbtest.sh | 53 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/ci/utils/nbtest.sh b/ci/utils/nbtest.sh index 9d622fe617..40795fea4e 100755 --- a/ci/utils/nbtest.sh +++ b/ci/utils/nbtest.sh @@ -47,35 +47,64 @@ for nb in "$@"; do echo "Executing notebook: ${NBFILENAME}" echo "Output will be saved to: ${EXECUTED_NOTEBOOK}" - # Extract and execute pip install commands from notebook echo "Checking for pip install commands in notebook..." - PIP_COMMANDS=$(grep -h "pip install" "$NBNAME.ipynb" 2>/dev/null | grep -v "#" | sed 's/^[[:space:]]*//' | sed 's/^["'"'"']*//' | sed 's/["'"'"']*$//' || true) + + # if [[ "$NBNAME" != *trnsport* ]]; then + # echo "Skipping notebook '${NBNAME}' as it does not contain '01_optimization' in the name." + # cd "$ORIGINAL_DIR" + # continue + # fi + + # Extract pip install lines (approximate method) + PIP_COMMANDS=$(grep -h "pip install" "$NBNAME.ipynb" 2>/dev/null \ + | grep -v "#" \ + | sed 's/^[[:space:]]*//' \ + | sed 's/^["'"'"']*//' \ + | sed 's/["'"'"']*$//' \ + | sed 's/\\$//' \ + || true) if [ -n "$PIP_COMMANDS" ]; then echo "Found pip install commands:" echo "$PIP_COMMANDS" echo "Executing pip install commands..." - echo "$PIP_COMMANDS" | while read -r cmd; do - echo "Processing command: '$cmd'" - if [[ "$cmd" =~ ^!?pip[[:space:]]+install ]]; then + + # Split multiple commands by common separators and process each one + echo "$PIP_COMMANDS" | awk -F'\\n",' '{for(i=1;i<=NF;i++) print $i}' | while IFS= read -r cmd; do + # Clean up the command and remove any remaining quote artifacts and trailing commas + cmd=$(echo "$cmd" | tr -d '"' | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$//' | sed 's/\\n//g' | sed 's/\\r//g' | sed 's/,$//') + + if [ -n "$cmd" ] && [[ "$cmd" =~ ^!?pip[[:space:]]+install ]]; then + echo "Processing command: '$cmd'" echo "Running: $cmd" # Remove the ! prefix if present for execution EXEC_CMD="${cmd#!}" - # Clean up escaped quotes, extra quotes, and newlines - EXEC_CMD=$(echo "$EXEC_CMD" | sed 's/\\"/"/g' | sed 's/^"//' | sed 's/"$//' | tr -d '\n\r') + + # Clean up quotes, backslashes, and newline artifacts using safer methods + EXEC_CMD=$(echo "$EXEC_CMD" | tr -d '"' | tr -d '\n\r' | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$//' | sed 's/,$//') + echo "DEBUG: Original command: '$cmd'" + echo "DEBUG: Cleaned command: '$EXEC_CMD'" echo "Executing: $EXEC_CMD" + + # INSERT_YOUR_CODE + # Add --pre to EXEC_CMD if not already present + if [[ "$EXEC_CMD" =~ ^pip[[:space:]]+install ]] && [[ ! "$EXEC_CMD" =~ [[:space:]]--pre([[:space:]]|$) ]]; then + EXEC_CMD="$EXEC_CMD --pre --extra-index-url https://pypi.anaconda.org/rapidsai-nightly/simple" + fi + eval "$EXEC_CMD" if [ $? -eq 0 ]; then echo "✓ Successfully executed: $cmd" - else - echo "✗ Failed to execute: $cmd" - fi - else + else + echo "✗ Failed to execute: $cmd" + fi + elif [ -n "$cmd" ]; then echo "Command '$cmd' did not match pip install pattern" fi done fi + # Execute notebook with default kernel jupyter nbconvert --execute "${NBNAME}.ipynb" --to notebook --output "${EXECUTED_NOTEBOOK}" --ExecutePreprocessor.kernel_name="python3" @@ -90,4 +119,4 @@ for nb in "$@"; do cd "${ORIGINAL_DIR}" || exit 1 done -exit ${EXITCODE} +exit ${EXITCODE} \ No newline at end of file From 891408f8d86970ea79449dc9bf760c9da5a4b036 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Tue, 26 Aug 2025 17:37:21 -0500 Subject: [PATCH 08/16] update --- ci/utils/nbtest.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/utils/nbtest.sh b/ci/utils/nbtest.sh index 40795fea4e..e7e2d3ed8d 100755 --- a/ci/utils/nbtest.sh +++ b/ci/utils/nbtest.sh @@ -68,12 +68,12 @@ for nb in "$@"; do echo "Found pip install commands:" echo "$PIP_COMMANDS" echo "Executing pip install commands..." - + # Split multiple commands by common separators and process each one echo "$PIP_COMMANDS" | awk -F'\\n",' '{for(i=1;i<=NF;i++) print $i}' | while IFS= read -r cmd; do # Clean up the command and remove any remaining quote artifacts and trailing commas cmd=$(echo "$cmd" | tr -d '"' | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$//' | sed 's/\\n//g' | sed 's/\\r//g' | sed 's/,$//') - + if [ -n "$cmd" ] && [[ "$cmd" =~ ^!?pip[[:space:]]+install ]]; then echo "Processing command: '$cmd'" echo "Running: $cmd" @@ -119,4 +119,4 @@ for nb in "$@"; do cd "${ORIGINAL_DIR}" || exit 1 done -exit ${EXITCODE} \ No newline at end of file +exit ${EXITCODE} From 1c5f22fd38855c7d556bf707cae1e150a50423cf Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Thu, 28 Aug 2025 15:02:44 -0500 Subject: [PATCH 09/16] fixed script --- ci/test_notebooks.sh | 18 ++-- ci/utils/nbtest.sh | 193 ++++++++++++++++++++++++++++++++++++------- 2 files changed, 172 insertions(+), 39 deletions(-) diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index 844a97ded9..feeccb90a7 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -22,21 +22,21 @@ set -euo pipefail # CUOPT_VERSION variable removed as it was unused rapids-logger "Downloading artifacts from previous jobs" -CPP_CHANNEL=$(rapids-download-conda-from-github cpp) -PYTHON_CHANNEL=$(rapids-download-conda-from-github python) +#CPP_CHANNEL=$(rapids-download-conda-from-github cpp) +#PYTHON_CHANNEL=$(rapids-download-conda-from-github python) rapids-logger "Generate notebook testing dependencies" ENV_YAML_DIR="$(mktemp -d)" -rapids-dependency-file-generator \ - --output conda \ - --file-key test_notebooks \ - --prepend-channel "${CPP_CHANNEL}" \ - --prepend-channel "${PYTHON_CHANNEL}" \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${ENV_YAML_DIR}/env.yaml" +#rapids-dependency-file-generator \ +# --output conda \ +# --file-key test_notebooks \ +# --prepend-channel "${CPP_CHANNEL}" \ +# --prepend-channel "${PYTHON_CHANNEL}" \ +# --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${ENV_YAML_DIR}/env.yaml" -rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n test +#rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n test # Temporarily allow unbound variables for conda activation. set +u diff --git a/ci/utils/nbtest.sh b/ci/utils/nbtest.sh index e7e2d3ed8d..a2a9ebfe01 100755 --- a/ci/utils/nbtest.sh +++ b/ci/utils/nbtest.sh @@ -30,12 +30,18 @@ for nb in "$@"; do NBFILENAME=$nb NBNAME=${NBFILENAME%.*} NBNAME=${NBNAME##*/} + echo "DEBUG: NBFILENAME=$NBFILENAME" + echo "DEBUG: NBNAME=$NBNAME" # Get the directory where the notebook is located NBDIR=$(dirname "$NBFILENAME") echo "Changing to directory: ${NBDIR}" + echo "Original directory was: ${ORIGINAL_DIR}" + echo "Target directory: ${NBDIR}" + echo "Full notebook path: ${NBFILENAME}" cd "${NBDIR}" || exit 1 + echo "Current directory after cd: $(pwd)" # Output the executed notebook in the same folder EXECUTED_NOTEBOOK="${NBNAME}-executed.ipynb" @@ -48,62 +54,189 @@ for nb in "$@"; do echo "Output will be saved to: ${EXECUTED_NOTEBOOK}" echo "Checking for pip install commands in notebook..." + echo "Notebook file: $NBNAME.ipynb" + echo "Current directory: $(pwd)" + if [ -f "$NBNAME.ipynb" ]; then + echo "Notebook file exists and is readable" + echo "First few lines of notebook:" + head -20 "$NBNAME.ipynb" | grep -E "(pip install|!pip)" || echo "No pip install lines found in first 20 lines" + else + echo "ERROR: Notebook file not found or not readable" + fi + + if [[ "$NBNAME" = *trnsport* ]] || [[ "$NBNAME" = *Pulp* ]]; then + echo "Skipping notebook '${NBNAME}' as it does not contain '01_optimization' in the name." + cd "$ORIGINAL_DIR" + continue + fi + - # if [[ "$NBNAME" != *trnsport* ]]; then - # echo "Skipping notebook '${NBNAME}' as it does not contain '01_optimization' in the name." - # cd "$ORIGINAL_DIR" - # continue - # fi - - # Extract pip install lines (approximate method) - PIP_COMMANDS=$(grep -h "pip install" "$NBNAME.ipynb" 2>/dev/null \ - | grep -v "#" \ - | sed 's/^[[:space:]]*//' \ - | sed 's/^["'"'"']*//' \ - | sed 's/["'"'"']*$//' \ - | sed 's/\\$//' \ - || true) + # Extract pip install lines by parsing the notebook JSON properly + echo "Extracting pip install commands from notebook JSON..." + + # Use python to properly parse the notebook JSON and extract pip commands + PIP_COMMANDS=$(python3 -c " +import json +import sys + +try: + with open('$NBNAME.ipynb', 'r') as f: + notebook = json.load(f) + + pip_commands = [] + for cell in notebook.get('cells', []): + if cell.get('cell_type') == 'code': + source = ''.join(cell.get('source', [])) + lines = source.split('\n') + for line in lines: + line = line.strip() + if line.startswith('!pip install') or line.startswith('pip install'): + # Clean up the line but preserve quotes + clean_line = line.strip() + if clean_line: + pip_commands.append(clean_line) + + # Print each command on a separate line for processing + for cmd in pip_commands: + print(cmd) + +except Exception as e: + print(f'Error parsing notebook: {e}', file=sys.stderr) + sys.exit(1) +" 2>/dev/null || true) if [ -n "$PIP_COMMANDS" ]; then echo "Found pip install commands:" - echo "$PIP_COMMANDS" + echo "Raw commands: '$PIP_COMMANDS'" + echo "Number of commands found: $(echo "$PIP_COMMANDS" | wc -l)" echo "Executing pip install commands..." - # Split multiple commands by common separators and process each one - echo "$PIP_COMMANDS" | awk -F'\\n",' '{for(i=1;i<=NF;i++) print $i}' | while IFS= read -r cmd; do - # Clean up the command and remove any remaining quote artifacts and trailing commas - cmd=$(echo "$cmd" | tr -d '"' | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$//' | sed 's/\\n//g' | sed 's/\\r//g' | sed 's/,$//') - + # Process each pip install command + echo "$PIP_COMMANDS" | while IFS= read -r cmd; do if [ -n "$cmd" ] && [[ "$cmd" =~ ^!?pip[[:space:]]+install ]]; then echo "Processing command: '$cmd'" echo "Running: $cmd" # Remove the ! prefix if present for execution EXEC_CMD="${cmd#!}" - # Clean up quotes, backslashes, and newline artifacts using safer methods - EXEC_CMD=$(echo "$EXEC_CMD" | tr -d '"' | tr -d '\n\r' | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$//' | sed 's/,$//') echo "DEBUG: Original command: '$cmd'" echo "DEBUG: Cleaned command: '$EXEC_CMD'" + echo "DEBUG: Command length: ${#EXEC_CMD}" + echo "DEBUG: Command contains 'numpy': $([[ "$EXEC_CMD" == *numpy* ]] && echo "YES" || echo "NO")" echo "Executing: $EXEC_CMD" - # INSERT_YOUR_CODE # Add --pre to EXEC_CMD if not already present if [[ "$EXEC_CMD" =~ ^pip[[:space:]]+install ]] && [[ ! "$EXEC_CMD" =~ [[:space:]]--pre([[:space:]]|$) ]]; then - EXEC_CMD="$EXEC_CMD --pre --extra-index-url https://pypi.anaconda.org/rapidsai-nightly/simple" + # Check if --extra-index-url is already present + if [[ "$EXEC_CMD" =~ [[:space:]]--extra-index-url ]]; then + EXEC_CMD="$EXEC_CMD --pre" + else + EXEC_CMD="$EXEC_CMD --pre --extra-index-url https://pypi.anaconda.org/rapidsai-nightly/simple" + fi fi - eval "$EXEC_CMD" - if [ $? -eq 0 ]; then - echo "✓ Successfully executed: $cmd" - else - echo "✗ Failed to execute: $cmd" - fi + # Execute pip install commands using eval to properly handle quoted arguments + if [[ "$EXEC_CMD" =~ ^pip[[:space:]]+install ]]; then + echo "Executing pip install command with eval to handle quoted arguments..." + if eval "$EXEC_CMD"; then + echo "✓ Successfully executed: $cmd" + else + echo "✗ Failed to execute: $cmd" + fi + else + echo "✗ Invalid pip install command format: $EXEC_CMD" + fi elif [ -n "$cmd" ]; then echo "Command '$cmd' did not match pip install pattern" fi done fi + # Extract and execute other shell commands (wget, curl, git, etc.) using proper JSON parsing + echo "Checking for other shell commands in notebook..." + OTHER_COMMANDS=$(python3 -c " +import json +import sys + +try: + with open('$NBNAME.ipynb', 'r') as f: + notebook = json.load(f) + + shell_commands = [] + for cell in notebook.get('cells', []): + if cell.get('cell_type') == 'code': + source = ''.join(cell.get('source', [])) + lines = source.split('\n') + for line in lines: + line = line.strip() + if line.startswith('!'): + # Check if it's a shell command we want to execute + cmd = line[1:].strip().split()[0] if line[1:].strip() else '' + if cmd in ['wget', 'curl', 'git', 'python', 'jupyter', 'ls', 'pwd', 'cd', 'mkdir', 'rm', 'cp', 'mv', 'chmod', 'unzip', 'tar', 'apt', 'yum', 'brew', 'conda', 'npm', 'yarn', 'docker', 'kubectl', 'helm', 'aws', 'gcloud', 'az']: + shell_commands.append(line) + + # Print each command on a separate line for processing + for cmd in shell_commands: + print(cmd) + +except Exception as e: + print(f'Error parsing notebook: {e}', file=sys.stderr) + sys.exit(1) +" 2>/dev/null || true) + + if [ -n "$OTHER_COMMANDS" ]; then + echo "Found other shell commands:" + echo "Raw commands: '$OTHER_COMMANDS'" + echo "Executing other shell commands..." + + # Process each shell command + echo "$OTHER_COMMANDS" | while IFS= read -r cmd; do + if [ -n "$cmd" ] && [[ "$cmd" =~ ^! ]]; then + echo "Processing command: '$cmd'" + echo "Running: $cmd" + # Remove the ! prefix for execution + EXEC_CMD="${cmd#!}" + + echo "DEBUG: Original command: '$cmd'" + echo "DEBUG: Cleaned command: '$EXEC_CMD'" + echo "DEBUG: Command length: ${#EXEC_CMD}" + echo "Executing: $EXEC_CMD" + + # Use eval for better argument handling, but with safety checks + if [ -n "$EXEC_CMD" ]; then + # Skip potentially dangerous commands + if [[ "$EXEC_CMD" =~ ^(chmod|chown|sudo|su) ]]; then + echo "⚠ Skipping potentially dangerous command: $cmd" + continue + fi + + echo "Executing shell command with eval to handle quoted arguments..." + if $EXEC_CMD; then + echo "✓ Successfully executed: $cmd" + else + echo "✗ Failed to execute: $cmd" + fi + else + echo "✗ Invalid command format: $EXEC_CMD" + fi + elif [ -n "$cmd" ]; then + echo "Command '$cmd' did not match shell command pattern" + fi + done + fi + + # Summary of executed commands + if [ -n "$PIP_COMMANDS" ] || [ -n "$OTHER_COMMANDS" ]; then + echo "------------------------------------------------------------------------" + echo "SUMMARY: Commands executed for notebook ${NBNAME}:" + if [ -n "$PIP_COMMANDS" ]; then + echo " Pip install commands: $(echo "$PIP_COMMANDS" | wc -l)" + fi + if [ -n "$OTHER_COMMANDS" ]; then + echo " Other shell commands: $(echo "$OTHER_COMMANDS" | wc -l)" + fi + echo "------------------------------------------------------------------------" + fi # Execute notebook with default kernel jupyter nbconvert --execute "${NBNAME}.ipynb" --to notebook --output "${EXECUTED_NOTEBOOK}" --ExecutePreprocessor.kernel_name="python3" From 010e6a9681d7de42a30954ac9dc9ea999659a8fe Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 29 Aug 2025 12:19:50 -0500 Subject: [PATCH 10/16] minimize and move python script --- ci/test_notebooks.sh | 18 +- ci/utils/nbtest.sh | 212 +++-------------------- ci/utils/notebook_command_extractor.py | 228 +++++++++++++++++++++++++ 3 files changed, 256 insertions(+), 202 deletions(-) create mode 100644 ci/utils/notebook_command_extractor.py diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index feeccb90a7..844a97ded9 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -22,21 +22,21 @@ set -euo pipefail # CUOPT_VERSION variable removed as it was unused rapids-logger "Downloading artifacts from previous jobs" -#CPP_CHANNEL=$(rapids-download-conda-from-github cpp) -#PYTHON_CHANNEL=$(rapids-download-conda-from-github python) +CPP_CHANNEL=$(rapids-download-conda-from-github cpp) +PYTHON_CHANNEL=$(rapids-download-conda-from-github python) rapids-logger "Generate notebook testing dependencies" ENV_YAML_DIR="$(mktemp -d)" -#rapids-dependency-file-generator \ -# --output conda \ -# --file-key test_notebooks \ -# --prepend-channel "${CPP_CHANNEL}" \ -# --prepend-channel "${PYTHON_CHANNEL}" \ -# --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${ENV_YAML_DIR}/env.yaml" +rapids-dependency-file-generator \ + --output conda \ + --file-key test_notebooks \ + --prepend-channel "${CPP_CHANNEL}" \ + --prepend-channel "${PYTHON_CHANNEL}" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${ENV_YAML_DIR}/env.yaml" -#rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n test +rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n test # Temporarily allow unbound variables for conda activation. set +u diff --git a/ci/utils/nbtest.sh b/ci/utils/nbtest.sh index a2a9ebfe01..0085f797a8 100755 --- a/ci/utils/nbtest.sh +++ b/ci/utils/nbtest.sh @@ -30,18 +30,11 @@ for nb in "$@"; do NBFILENAME=$nb NBNAME=${NBFILENAME%.*} NBNAME=${NBNAME##*/} - echo "DEBUG: NBFILENAME=$NBFILENAME" - echo "DEBUG: NBNAME=$NBNAME" # Get the directory where the notebook is located NBDIR=$(dirname "$NBFILENAME") - echo "Changing to directory: ${NBDIR}" - echo "Original directory was: ${ORIGINAL_DIR}" - echo "Target directory: ${NBDIR}" - echo "Full notebook path: ${NBFILENAME}" cd "${NBDIR}" || exit 1 - echo "Current directory after cd: $(pwd)" # Output the executed notebook in the same folder EXECUTED_NOTEBOOK="${NBNAME}-executed.ipynb" @@ -50,194 +43,28 @@ for nb in "$@"; do echo STARTING: "${NBNAME}" echo -------------------------------------------------------------------------------- - echo "Executing notebook: ${NBFILENAME}" - echo "Output will be saved to: ${EXECUTED_NOTEBOOK}" - - echo "Checking for pip install commands in notebook..." - echo "Notebook file: $NBNAME.ipynb" - echo "Current directory: $(pwd)" - if [ -f "$NBNAME.ipynb" ]; then - echo "Notebook file exists and is readable" - echo "First few lines of notebook:" - head -20 "$NBNAME.ipynb" | grep -E "(pip install|!pip)" || echo "No pip install lines found in first 20 lines" - else - echo "ERROR: Notebook file not found or not readable" - fi - - if [[ "$NBNAME" = *trnsport* ]] || [[ "$NBNAME" = *Pulp* ]]; then - echo "Skipping notebook '${NBNAME}' as it does not contain '01_optimization' in the name." - cd "$ORIGINAL_DIR" - continue - fi - - - # Extract pip install lines by parsing the notebook JSON properly - echo "Extracting pip install commands from notebook JSON..." - - # Use python to properly parse the notebook JSON and extract pip commands - PIP_COMMANDS=$(python3 -c " -import json -import sys - -try: - with open('$NBNAME.ipynb', 'r') as f: - notebook = json.load(f) - - pip_commands = [] - for cell in notebook.get('cells', []): - if cell.get('cell_type') == 'code': - source = ''.join(cell.get('source', [])) - lines = source.split('\n') - for line in lines: - line = line.strip() - if line.startswith('!pip install') or line.startswith('pip install'): - # Clean up the line but preserve quotes - clean_line = line.strip() - if clean_line: - pip_commands.append(clean_line) - - # Print each command on a separate line for processing - for cmd in pip_commands: - print(cmd) - -except Exception as e: - print(f'Error parsing notebook: {e}', file=sys.stderr) - sys.exit(1) -" 2>/dev/null || true) - - if [ -n "$PIP_COMMANDS" ]; then - echo "Found pip install commands:" - echo "Raw commands: '$PIP_COMMANDS'" - echo "Number of commands found: $(echo "$PIP_COMMANDS" | wc -l)" - echo "Executing pip install commands..." - - # Process each pip install command - echo "$PIP_COMMANDS" | while IFS= read -r cmd; do - if [ -n "$cmd" ] && [[ "$cmd" =~ ^!?pip[[:space:]]+install ]]; then - echo "Processing command: '$cmd'" - echo "Running: $cmd" - # Remove the ! prefix if present for execution - EXEC_CMD="${cmd#!}" - - echo "DEBUG: Original command: '$cmd'" - echo "DEBUG: Cleaned command: '$EXEC_CMD'" - echo "DEBUG: Command length: ${#EXEC_CMD}" - echo "DEBUG: Command contains 'numpy': $([[ "$EXEC_CMD" == *numpy* ]] && echo "YES" || echo "NO")" - echo "Executing: $EXEC_CMD" - - # Add --pre to EXEC_CMD if not already present - if [[ "$EXEC_CMD" =~ ^pip[[:space:]]+install ]] && [[ ! "$EXEC_CMD" =~ [[:space:]]--pre([[:space:]]|$) ]]; then - # Check if --extra-index-url is already present - if [[ "$EXEC_CMD" =~ [[:space:]]--extra-index-url ]]; then - EXEC_CMD="$EXEC_CMD --pre" - else - EXEC_CMD="$EXEC_CMD --pre --extra-index-url https://pypi.anaconda.org/rapidsai-nightly/simple" - fi - fi - - # Execute pip install commands using eval to properly handle quoted arguments - if [[ "$EXEC_CMD" =~ ^pip[[:space:]]+install ]]; then - echo "Executing pip install command with eval to handle quoted arguments..." - if eval "$EXEC_CMD"; then - echo "✓ Successfully executed: $cmd" - else - echo "✗ Failed to execute: $cmd" - fi - else - echo "✗ Invalid pip install command format: $EXEC_CMD" - fi - elif [ -n "$cmd" ]; then - echo "Command '$cmd' did not match pip install pattern" - fi - done - fi + # Skip notebooks that are not yet supported + SKIP_NOTEBOOKS=( + "trnsport_cuopt", + "Production_Planning_Example_Pulp" + "Simple_LP_pulp" + "Simple_MIP_pulp" + "Sudoku_pulp" + ) + + for skip in "${SKIP_NOTEBOOKS[@]}"; do + if [[ "$NBNAME" == "$skip"* ]]; then + echo "Skipping notebook '${NBNAME}' as it matches skip pattern '${skip}'" + cd "$ORIGINAL_DIR" + continue 2 + fi + done - # Extract and execute other shell commands (wget, curl, git, etc.) using proper JSON parsing - echo "Checking for other shell commands in notebook..." - OTHER_COMMANDS=$(python3 -c " -import json -import sys - -try: - with open('$NBNAME.ipynb', 'r') as f: - notebook = json.load(f) - - shell_commands = [] - for cell in notebook.get('cells', []): - if cell.get('cell_type') == 'code': - source = ''.join(cell.get('source', [])) - lines = source.split('\n') - for line in lines: - line = line.strip() - if line.startswith('!'): - # Check if it's a shell command we want to execute - cmd = line[1:].strip().split()[0] if line[1:].strip() else '' - if cmd in ['wget', 'curl', 'git', 'python', 'jupyter', 'ls', 'pwd', 'cd', 'mkdir', 'rm', 'cp', 'mv', 'chmod', 'unzip', 'tar', 'apt', 'yum', 'brew', 'conda', 'npm', 'yarn', 'docker', 'kubectl', 'helm', 'aws', 'gcloud', 'az']: - shell_commands.append(line) - - # Print each command on a separate line for processing - for cmd in shell_commands: - print(cmd) - -except Exception as e: - print(f'Error parsing notebook: {e}', file=sys.stderr) - sys.exit(1) -" 2>/dev/null || true) - - if [ -n "$OTHER_COMMANDS" ]; then - echo "Found other shell commands:" - echo "Raw commands: '$OTHER_COMMANDS'" - echo "Executing other shell commands..." - - # Process each shell command - echo "$OTHER_COMMANDS" | while IFS= read -r cmd; do - if [ -n "$cmd" ] && [[ "$cmd" =~ ^! ]]; then - echo "Processing command: '$cmd'" - echo "Running: $cmd" - # Remove the ! prefix for execution - EXEC_CMD="${cmd#!}" - - echo "DEBUG: Original command: '$cmd'" - echo "DEBUG: Cleaned command: '$EXEC_CMD'" - echo "DEBUG: Command length: ${#EXEC_CMD}" - echo "Executing: $EXEC_CMD" - - # Use eval for better argument handling, but with safety checks - if [ -n "$EXEC_CMD" ]; then - # Skip potentially dangerous commands - if [[ "$EXEC_CMD" =~ ^(chmod|chown|sudo|su) ]]; then - echo "⚠ Skipping potentially dangerous command: $cmd" - continue - fi - - echo "Executing shell command with eval to handle quoted arguments..." - if $EXEC_CMD; then - echo "✓ Successfully executed: $cmd" - else - echo "✗ Failed to execute: $cmd" - fi - else - echo "✗ Invalid command format: $EXEC_CMD" - fi - elif [ -n "$cmd" ]; then - echo "Command '$cmd' did not match shell command pattern" - fi - done - fi + rapids-logger "Running commands from notebook: ${NBNAME}.ipynb" - # Summary of executed commands - if [ -n "$PIP_COMMANDS" ] || [ -n "$OTHER_COMMANDS" ]; then - echo "------------------------------------------------------------------------" - echo "SUMMARY: Commands executed for notebook ${NBNAME}:" - if [ -n "$PIP_COMMANDS" ]; then - echo " Pip install commands: $(echo "$PIP_COMMANDS" | wc -l)" - fi - if [ -n "$OTHER_COMMANDS" ]; then - echo " Other shell commands: $(echo "$OTHER_COMMANDS" | wc -l)" - fi - echo "------------------------------------------------------------------------" - fi + python3 "$ORIGINAL_DIR/../ci/utils/notebook_command_extractor.py" "$NBNAME.ipynb" --verbose + rapids-logger "Executing notebook: ${NBNAME}.ipynb" # Execute notebook with default kernel jupyter nbconvert --execute "${NBNAME}.ipynb" --to notebook --output "${EXECUTED_NOTEBOOK}" --ExecutePreprocessor.kernel_name="python3" @@ -248,7 +75,6 @@ except Exception as e: EXITCODE=1 fi - echo "Returning to original directory: ${ORIGINAL_DIR}" cd "${ORIGINAL_DIR}" || exit 1 done diff --git a/ci/utils/notebook_command_extractor.py b/ci/utils/notebook_command_extractor.py new file mode 100644 index 0000000000..4827dcaae6 --- /dev/null +++ b/ci/utils/notebook_command_extractor.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 + +""" +Notebook Command Extractor + +This script extracts pip install and other shell commands from Jupyter notebooks +and can optionally execute them. It's designed to be used by the nbtest.sh script. +""" + +import json +import sys +import subprocess +import argparse +from typing import List, Tuple + + +def extract_pip_commands(notebook_path: str) -> List[str]: + """Extract pip install commands from a Jupyter notebook.""" + try: + with open(notebook_path, 'r') as f: + notebook = json.load(f) + + pip_commands = [] + for cell in notebook.get('cells', []): + if cell.get('cell_type') == 'code': + source = ''.join(cell.get('source', [])) + lines = source.split('\n') + for line in lines: + line = line.strip() + if line.startswith('!pip install') or line.startswith('pip install'): + # Clean up the line but preserve quotes + clean_line = line.strip() + if clean_line: + pip_commands.append(clean_line) + + return pip_commands + + except Exception as e: + print(f'Error parsing notebook: {e}', file=sys.stderr) + return [] + + +def extract_shell_commands(notebook_path: str) -> List[str]: + """Extract other shell commands from a Jupyter notebook.""" + try: + with open(notebook_path, 'r') as f: + notebook = json.load(f) + + shell_commands = [] + allowed_commands = [ + 'wget', 'curl', 'git', 'python', 'jupyter', 'ls', 'pwd', 'cd', + 'mkdir', 'rm', 'cp', 'mv', 'chmod', 'unzip', 'tar', 'apt', 'yum', + 'brew', 'conda', 'npm', 'yarn', 'docker', 'kubectl', 'helm', + 'aws', 'gcloud', 'az' + ] + + for cell in notebook.get('cells', []): + if cell.get('cell_type') == 'code': + source = ''.join(cell.get('source', [])) + lines = source.split('\n') + for line in lines: + line = line.strip() + if line.startswith('!'): + # Check if it's a shell command we want to execute + cmd = line[1:].strip().split()[0] if line[1:].strip() else '' + if cmd in allowed_commands: + shell_commands.append(line) + + return shell_commands + + except Exception as e: + print(f'Error parsing notebook: {e}', file=sys.stderr) + return [] + + +def execute_pip_command(cmd: str, verbose: bool = False) -> bool: + """Execute a pip install command.""" + if verbose: + print(f"Processing command: '{cmd}'") + + # Remove the ! prefix if present for execution + exec_cmd = cmd.lstrip('!').strip() + + if verbose: + print(f"DEBUG: Original command: '{cmd}'") + print(f"DEBUG: Cleaned command: '{exec_cmd}'") + print(f"DEBUG: Command length: {len(exec_cmd)}") + print(f"DEBUG: Command contains 'numpy': {'YES' if 'numpy' in exec_cmd else 'NO'}") + print(f"Executing: {exec_cmd}") + + # Add --pre to exec_cmd if not already present + if exec_cmd.startswith('pip install') and '--pre' not in exec_cmd: + # Check if --extra-index-url is already present + if '--extra-index-url' in exec_cmd: + exec_cmd += ' --pre' + else: + exec_cmd += ' --pre --extra-index-url https://pypi.anaconda.org/rapidsai-nightly/simple' + + if verbose: + print(f"Final command: {exec_cmd}") + + try: + # Execute pip install commands + if exec_cmd.startswith('pip install'): + # Use shell=True for pip install to handle quoted arguments properly + # This is safe since we're only executing pip install commands + result = subprocess.run(exec_cmd, shell=True, capture_output=True, text=True) + if result.returncode == 0: + if verbose: + print(f"✓ Successfully executed: {cmd}") + return True + else: + if verbose: + print(f"✗ Failed to execute: {cmd}") + print(f"Error: {result.stderr}") + return False + else: + if verbose: + print(f"✗ Invalid pip install command format: {exec_cmd}") + return False + except Exception as e: + if verbose: + print(f"✗ Exception executing {cmd}: {e}") + return False + + +def execute_shell_command(cmd: str, verbose: bool = False) -> bool: + """Execute a shell command.""" + if verbose: + print(f"Processing command: '{cmd}'") + + # Remove the ! prefix for execution + exec_cmd = cmd.lstrip('!').strip() + + if verbose: + print(f"DEBUG: Original command: '{cmd}'") + print(f"DEBUG: Cleaned command: '{exec_cmd}'") + print(f"DEBUG: Command length: {len(exec_cmd)}") + print(f"Executing: {exec_cmd}") + + # Skip potentially dangerous commands + dangerous_commands = ['chmod', 'chown', 'sudo', 'su'] + if any(exec_cmd.startswith(dangerous) for dangerous in dangerous_commands): + if verbose: + print(f"⚠ Skipping potentially dangerous command: {cmd}") + return False + + try: + if verbose: + print("Executing shell command...") + + result = subprocess.run(exec_cmd, shell=True, capture_output=True, text=True) + if result.returncode == 0: + if verbose: + print(f"✓ Successfully executed: {cmd}") + return True + else: + if verbose: + print(f"✗ Failed to execute: {cmd}") + print(f"Error: {result.stderr}") + return False + except Exception as e: + if verbose: + print(f"✗ Exception executing {cmd}: {e}") + return False + + +def main(): + parser = argparse.ArgumentParser(description='Extract and optionally execute commands from Jupyter notebooks') + parser.add_argument('notebook_path', help='Path to the Jupyter notebook') + parser.add_argument('--extract-only', action='store_true', help='Only extract commands, do not execute') + parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output') + parser.add_argument('--output-format', choices=['json', 'text'], default='text', + help='Output format for extracted commands') + + args = parser.parse_args() + + # Extract commands + pip_commands = extract_pip_commands(args.notebook_path) + shell_commands = extract_shell_commands(args.notebook_path) + + if args.output_format == 'json': + # Output as JSON for shell script processing + output = { + 'pip_commands': pip_commands, + 'shell_commands': shell_commands + } + print(json.dumps(output)) + else: + # Output as text (default) + if pip_commands: + print("PIP_COMMANDS:") + for cmd in pip_commands: + print(cmd) + + if shell_commands: + print("SHELL_COMMANDS:") + for cmd in shell_commands: + print(cmd) + + # Execute commands if not extract-only mode + if not args.extract_only: + success_count = 0 + total_count = 0 + + if pip_commands: + print(f"\nExecuting {len(pip_commands)} pip install commands...") + for cmd in pip_commands: + if execute_pip_command(cmd, args.verbose): + success_count += 1 + total_count += 1 + + if shell_commands: + print(f"\nExecuting {len(shell_commands)} shell commands...") + for cmd in shell_commands: + if execute_shell_command(cmd, args.verbose): + success_count += 1 + total_count += 1 + + if total_count > 0: + print(f"\nExecution summary: {success_count}/{total_count} commands succeeded") + return 0 if success_count == total_count else 1 + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) From 65eac1782a7f6476401601828fafe01848f943de Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 29 Aug 2025 12:24:24 -0500 Subject: [PATCH 11/16] fix style --- ci/utils/nbtest.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/utils/nbtest.sh b/ci/utils/nbtest.sh index 0085f797a8..356e92795f 100755 --- a/ci/utils/nbtest.sh +++ b/ci/utils/nbtest.sh @@ -45,7 +45,7 @@ for nb in "$@"; do # Skip notebooks that are not yet supported SKIP_NOTEBOOKS=( - "trnsport_cuopt", + "trnsport_cuopt" "Production_Planning_Example_Pulp" "Simple_LP_pulp" "Simple_MIP_pulp" @@ -55,7 +55,7 @@ for nb in "$@"; do for skip in "${SKIP_NOTEBOOKS[@]}"; do if [[ "$NBNAME" == "$skip"* ]]; then echo "Skipping notebook '${NBNAME}' as it matches skip pattern '${skip}'" - cd "$ORIGINAL_DIR" + cd "$ORIGINAL_DIR" || exit 1 continue 2 fi done From bba7a04ad9c4c89a6b5df0dcc9254d38d2577418 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 29 Aug 2025 12:29:51 -0500 Subject: [PATCH 12/16] fix check --- ci/utils/notebook_command_extractor.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ci/utils/notebook_command_extractor.py b/ci/utils/notebook_command_extractor.py index 0a646e3364..af33b1f964 100644 --- a/ci/utils/notebook_command_extractor.py +++ b/ci/utils/notebook_command_extractor.py @@ -1,5 +1,21 @@ #!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + """ Notebook Command Extractor From 46fa644ea1ef237c14d0f26e35362db3426f99fc Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 29 Aug 2025 15:22:27 -0500 Subject: [PATCH 13/16] remove redundant usage --- ci/test_notebooks.sh | 33 ++++++++++---------------- ci/utils/notebook_command_extractor.py | 25 ++++--------------- ci/utils/notebook_list.py | 21 ---------------- 3 files changed, 17 insertions(+), 62 deletions(-) diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index 844a97ded9..d71402b534 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -19,24 +19,24 @@ set -euo pipefail . /opt/conda/etc/profile.d/conda.sh -# CUOPT_VERSION variable removed as it was unused +CUOPT_VERSION="$(rapids-version)" rapids-logger "Downloading artifacts from previous jobs" -CPP_CHANNEL=$(rapids-download-conda-from-github cpp) -PYTHON_CHANNEL=$(rapids-download-conda-from-github python) +# CPP_CHANNEL=$(rapids-download-conda-from-github cpp) +# PYTHON_CHANNEL=$(rapids-download-conda-from-github python) rapids-logger "Generate notebook testing dependencies" ENV_YAML_DIR="$(mktemp -d)" -rapids-dependency-file-generator \ - --output conda \ - --file-key test_notebooks \ - --prepend-channel "${CPP_CHANNEL}" \ - --prepend-channel "${PYTHON_CHANNEL}" \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${ENV_YAML_DIR}/env.yaml" +#rapids-dependency-file-generator \ +# --output conda \ +# --file-key test_notebooks \ +# --prepend-channel "${CPP_CHANNEL}" \ +# --prepend-channel "${PYTHON_CHANNEL}" \ +# --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${ENV_YAML_DIR}/env.yaml" -rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n test +# rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n test # Temporarily allow unbound variables for conda activation. set +u @@ -45,12 +45,11 @@ set -u rapids-print-env - -#EXAMPLES_BRANCH="branch-${CUOPT_VERSION%.*}" -EXAMPLES_BRANCH="fix_testing" -rapids-logger "Cloning cuopt-examples repository for branch: ${EXAMPLES_BRANCH}" +EXAMPLES_BRANCH="branch-${CUOPT_VERSION%.*}" # Remove any existing cuopt-examples directory + +rapids-logger "Cloning cuopt-examples repository for branch: ${EXAMPLES_BRANCH}" rm -rf cuopt-examples git clone --single-branch --branch "${EXAMPLES_BRANCH}" https://github.com/NVIDIA/cuopt-examples.git @@ -61,12 +60,6 @@ pushd cuopt-examples NBLIST=$(python "${NBLIST_PATH}") -# Install any wheels if they are present in any subdirectory -find . -type f -name "*.whl" | while read whl; do - echo "Installing wheel: $whl" - pip install "$whl" -done - EXITCODE=0 trap "EXITCODE=1" ERR diff --git a/ci/utils/notebook_command_extractor.py b/ci/utils/notebook_command_extractor.py index af33b1f964..7b443c7bd2 100644 --- a/ci/utils/notebook_command_extractor.py +++ b/ci/utils/notebook_command_extractor.py @@ -70,29 +70,13 @@ def extract_shell_commands(notebook_path: str) -> List[str]: "curl", "git", "python", - "jupyter", - "ls", - "pwd", "cd", "mkdir", "rm", "cp", "mv", - "chmod", "unzip", "tar", - "apt", - "yum", - "brew", - "conda", - "npm", - "yarn", - "docker", - "kubectl", - "helm", - "aws", - "gcloud", - "az", ] for cell in notebook.get("cells", []): @@ -137,11 +121,7 @@ def execute_pip_command(cmd: str, verbose: bool = False) -> bool: # Add --pre to exec_cmd if not already present if exec_cmd.startswith("pip install") and "--pre" not in exec_cmd: - # Check if --extra-index-url is already present - if "--extra-index-url" in exec_cmd: - exec_cmd += " --pre" - else: - exec_cmd += " --pre --extra-index-url https://pypi.anaconda.org/rapidsai-nightly/simple" + exec_cmd += " --pre --extra-index-url https://pypi.anaconda.org/rapidsai-nightly/simple" if verbose: print(f"Final command: {exec_cmd}") @@ -242,6 +222,9 @@ def main(): pip_commands = extract_pip_commands(args.notebook_path) shell_commands = extract_shell_commands(args.notebook_path) + print(f"Pip commands: {pip_commands}") + print(f"Shell commands: {shell_commands}") + if args.output_format == "json": # Output as JSON for shell script processing output = { diff --git a/ci/utils/notebook_list.py b/ci/utils/notebook_list.py index 0ecbddcc3b..1b4e5a26ba 100644 --- a/ci/utils/notebook_list.py +++ b/ci/utils/notebook_list.py @@ -41,27 +41,6 @@ skip = True print(f"SKIPPING {filename} (marked as skip)", file=sys.stderr) break - elif re.search("dask", line): - print( - f"SKIPPING {filename} (suspected Dask usage, not currently automatable)", - file=sys.stderr, - ) - skip = True - break - elif pascal and re.search("# Does not run on Pascal", line): - print( - f"SKIPPING {filename} (does not run on Pascal)", - file=sys.stderr, - ) - skip = True - break - elif re.search("CVRPTW Exercise", line): - print( - f"SKIPPING {filename} (user exercise notebook)", - file=sys.stderr, - ) - skip = True - break if not skip: print(filename) From 98c2aa31bfdecc037d9c7fae37d889cd71385862 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Tue, 2 Sep 2025 09:49:59 -0500 Subject: [PATCH 14/16] restore commented parts --- ci/test_notebooks.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index d71402b534..7ff6f2424b 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -22,21 +22,21 @@ set -euo pipefail CUOPT_VERSION="$(rapids-version)" rapids-logger "Downloading artifacts from previous jobs" -# CPP_CHANNEL=$(rapids-download-conda-from-github cpp) -# PYTHON_CHANNEL=$(rapids-download-conda-from-github python) +CPP_CHANNEL=$(rapids-download-conda-from-github cpp) +PYTHON_CHANNEL=$(rapids-download-conda-from-github python) rapids-logger "Generate notebook testing dependencies" ENV_YAML_DIR="$(mktemp -d)" -#rapids-dependency-file-generator \ -# --output conda \ -# --file-key test_notebooks \ -# --prepend-channel "${CPP_CHANNEL}" \ -# --prepend-channel "${PYTHON_CHANNEL}" \ -# --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${ENV_YAML_DIR}/env.yaml" +rapids-dependency-file-generator \ + --output conda \ + --file-key test_notebooks \ + --prepend-channel "${CPP_CHANNEL}" \ + --prepend-channel "${PYTHON_CHANNEL}" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${ENV_YAML_DIR}/env.yaml" -# rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n test +rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n test # Temporarily allow unbound variables for conda activation. set +u From c25cc71a798bdc6772ea4ffe5efa2c33edeb3d96 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 5 Sep 2025 15:02:06 -0500 Subject: [PATCH 15/16] remove notebooks testing from PR workflow --- .github/workflows/pr.yaml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 378fe1be24..8f7a89364e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -35,7 +35,6 @@ jobs: - conda-python-tests - docs-build - wheel-build-libcuopt - - conda-notebook-tests - wheel-build-cuopt - wheel-tests-cuopt - wheel-build-cuopt-server @@ -147,16 +146,6 @@ jobs: run_codecov: false build_type: pull-request script: ci/test_python.sh - conda-notebook-tests: - needs: [conda-python-build, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.10 - with: - build_type: pull-request - node_type: "gpu-l4-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:25.10-latest" - script: ci/test_notebooks.sh docs-build: needs: conda-python-build secrets: inherit From 40208e2233abf1db69923bd9b4f7f43d6ea81a80 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 5 Sep 2025 15:48:15 -0500 Subject: [PATCH 16/16] fix style --- conda/environments/all_cuda-130_arch-aarch64.yaml | 7 +------ conda/environments/all_cuda-130_arch-x86_64.yaml | 7 +------ 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/conda/environments/all_cuda-130_arch-aarch64.yaml b/conda/environments/all_cuda-130_arch-aarch64.yaml index 5648d8799a..cf306f74e6 100644 --- a/conda/environments/all_cuda-130_arch-aarch64.yaml +++ b/conda/environments/all_cuda-130_arch-aarch64.yaml @@ -27,9 +27,7 @@ dependencies: - doxygen=1.9.1 - exhale - fastapi -- folium - gcc_linux-aarch64=14.* -- geopandas - gmock - gtest - httpx @@ -38,11 +36,9 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libgdal<3.9.0 - libraft-headers==25.10.* - librmm==25.10.* - make -- matplotlib - msgpack-numpy==0.4.8 - msgpack-python==1.1.0 - myst-nb @@ -56,7 +52,6 @@ dependencies: - pandas>=2.0 - pexpect - pip -- polyline - pre-commit - psutil>=5.9,<6.0a0 - pylibraft==25.10.*,>=0.0.0a0 @@ -71,7 +66,6 @@ dependencies: - requests - rmm==25.10.*,>=0.0.0a0 - scikit-build-core>=0.10.0 -- scipy - sphinx - sphinx-copybutton - sphinx-design @@ -80,6 +74,7 @@ dependencies: - sphinxcontrib-openapi - sphinxcontrib-websupport - sysroot_linux-aarch64==2.28 +- tbb-devel - uvicorn==0.34.* - zlib - pip: diff --git a/conda/environments/all_cuda-130_arch-x86_64.yaml b/conda/environments/all_cuda-130_arch-x86_64.yaml index 36e522b235..c9ba5beba0 100644 --- a/conda/environments/all_cuda-130_arch-x86_64.yaml +++ b/conda/environments/all_cuda-130_arch-x86_64.yaml @@ -27,9 +27,7 @@ dependencies: - doxygen=1.9.1 - exhale - fastapi -- folium - gcc_linux-64=14.* -- geopandas - gmock - gtest - httpx @@ -38,11 +36,9 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libgdal<3.9.0 - libraft-headers==25.10.* - librmm==25.10.* - make -- matplotlib - msgpack-numpy==0.4.8 - msgpack-python==1.1.0 - myst-nb @@ -56,7 +52,6 @@ dependencies: - pandas>=2.0 - pexpect - pip -- polyline - pre-commit - psutil>=5.9,<6.0a0 - pylibraft==25.10.*,>=0.0.0a0 @@ -71,7 +66,6 @@ dependencies: - requests - rmm==25.10.*,>=0.0.0a0 - scikit-build-core>=0.10.0 -- scipy - sphinx - sphinx-copybutton - sphinx-design @@ -80,6 +74,7 @@ dependencies: - sphinxcontrib-openapi - sphinxcontrib-websupport - sysroot_linux-64==2.28 +- tbb-devel - uvicorn==0.34.* - zlib - pip: