diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e13e02b88a..51203186fa 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -35,7 +35,6 @@ jobs: - conda-python-tests - docs-build - wheel-build-libcuopt - # - conda-notebook-tests - wheel-build-cuopt - wheel-tests-cuopt - wheel-build-cuopt-server diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 80ba1f869f..080d81a7ae 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -69,3 +69,15 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} script: ci/test_wheel_cuopt_server.sh + conda-notebook-tests: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.10 + with: + build_type: ${{ inputs.build_type }} + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + node_type: "gpu-l4-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:25.10-latest" + script: ci/test_notebooks.sh diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index feb711e78e..7ff6f2424b 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -21,13 +21,22 @@ set -euo pipefail CUOPT_VERSION="$(rapids-version)" +rapids-logger "Downloading artifacts from previous jobs" +CPP_CHANNEL=$(rapids-download-conda-from-github cpp) +PYTHON_CHANNEL=$(rapids-download-conda-from-github python) + rapids-logger "Generate notebook testing dependencies" + +ENV_YAML_DIR="$(mktemp -d)" + rapids-dependency-file-generator \ --output conda \ --file-key test_notebooks \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + --prepend-channel "${CPP_CHANNEL}" \ + --prepend-channel "${PYTHON_CHANNEL}" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${ENV_YAML_DIR}/env.yaml" -rapids-mamba-retry env create --yes -f env.yaml -n test +rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n test # Temporarily allow unbound variables for conda activation. set +u @@ -36,25 +45,20 @@ set -u rapids-print-env -rapids-logger "Downloading artifacts from previous jobs" -CPP_CHANNEL=$(rapids-download-conda-from-github cpp) -PYTHON_CHANNEL=$(rapids-download-conda-from-github python) +EXAMPLES_BRANCH="branch-${CUOPT_VERSION%.*}" -rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - --channel "${PYTHON_CHANNEL}" \ - "libcuopt=${CUOPT_VERSION}" \ - "cuopt=${CUOPT_VERSION}" \ - "cuopt-server=${CUOPT_VERSION}" +# Remove any existing cuopt-examples directory -pip install python/cuopt_self_hosted/ +rapids-logger "Cloning cuopt-examples repository for branch: ${EXAMPLES_BRANCH}" +rm -rf cuopt-examples +git clone --single-branch --branch "${EXAMPLES_BRANCH}" https://github.com/NVIDIA/cuopt-examples.git NBTEST="$(realpath "$(dirname "$0")/utils/nbtest.sh")" NBLIST_PATH="$(realpath "$(dirname "$0")/utils/notebook_list.py")" -NBLIST=$(python "${NBLIST_PATH}") -SERVER_WAIT_DELAY=10 -pushd notebooks +pushd cuopt-examples + +NBLIST=$(python "${NBLIST_PATH}") EXITCODE=0 trap "EXITCODE=1" ERR @@ -62,20 +66,16 @@ trap "EXITCODE=1" ERR rapids-logger "Start cuopt-server" set +e -#python -c "from cuopt_server.cuopt_service import run_server; run_server()" & - -python -m cuopt_server.cuopt_service & -export SERVER_PID=$! -sleep "${SERVER_WAIT_DELAY}" -curl http://0.0.0.0:5000/cuopt/health rapids-logger "Start notebooks tests" for nb in ${NBLIST}; do nvidia-smi ${NBTEST} "${nb}" + if [ $? -ne 0 ]; then + echo "Notebook ${nb} failed to execute. Exiting." + exit 1 + fi done rapids-logger "Notebook test script exiting with value: $EXITCODE" -kill -s SIGTERM $SERVER_PID -wait $SERVER_PID exit ${EXITCODE} diff --git a/ci/utils/nbtest.sh b/ci/utils/nbtest.sh index 1b99d68247..356e92795f 100755 --- a/ci/utils/nbtest.sh +++ b/ci/utils/nbtest.sh @@ -12,75 +12,70 @@ # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# See the License for the specific language governing permissions and limitations. +# +# This script executes Jupyter notebooks directly using nbconvert. set +e # do not abort the script on error set -o pipefail # piped commands propagate their error set -E # ERR traps are inherited by subcommands trap "EXITCODE=1" ERR -# Prepend the following code to all scripts generated from nbconvert. This -# allows all cell and line magic code to run and update the namespace as if -# running in jupyter, but will also tolerate failures due to running in a -# non-jupyter env. -# Note: depending on the assumptions of the notebook script, ignoring failures -# may not be acceptable (meaning the converted notebook simply cannot run -# outside of jupyter as-is), hence the warning. -MAGIC_OVERRIDE_CODE=" -def my_run_line_magic(*args, **kwargs): - g=globals() - l={} - for a in args: - try: - exec(str(a),g,l) - except Exception as e: - print('WARNING: %s\n While executing this magic function code:\n%s\n continuing...\n' % (e, a)) - else: - g.update(l) - -def my_run_cell_magic(*args, **kwargs): - my_run_line_magic(*args, **kwargs) - -get_ipython().run_line_magic=my_run_line_magic -get_ipython().run_cell_magic=my_run_cell_magic - -" - -NO_COLORS=--colors=NoColor -EXITCODE=0 +# Save the original directory +ORIGINAL_DIR=$(pwd) -# PWD is REPO_ROOT/notebooks -NBTMPDIR="${PWD}/../tmp" -mkdir -p "${NBTMPDIR}" -NBUTILS="${PWD}/external" -cp -r "${NBUTILS}/python/utils" "${NBTMPDIR}/." -cp -r "${NBUTILS}/server/notebook_utils" "${NBTMPDIR}/." -cp -r "${NBUTILS}/dli/helper_function" "${NBTMPDIR}/." -cd "${NBTMPDIR}" || exit 1 +EXITCODE=0 for nb in "$@"; do NBFILENAME=$nb NBNAME=${NBFILENAME%.*} NBNAME=${NBNAME##*/} - NBTESTSCRIPT=${NBTMPDIR}/${NBNAME}-test.py - shift + + # Get the directory where the notebook is located + NBDIR=$(dirname "$NBFILENAME") + + cd "${NBDIR}" || exit 1 + + # Output the executed notebook in the same folder + EXECUTED_NOTEBOOK="${NBNAME}-executed.ipynb" echo -------------------------------------------------------------------------------- echo STARTING: "${NBNAME}" echo -------------------------------------------------------------------------------- - jupyter nbconvert --to script ../"${NBFILENAME}" --output "${NBTMPDIR}"/"${NBNAME}"-test - python "${PWD}/../ci/utils/dli_nb_strip.py" "${NBTESTSCRIPT}" - echo "${MAGIC_OVERRIDE_CODE}" > "${NBTMPDIR}"/tmpfile - cat "${NBTESTSCRIPT}" >> "${NBTMPDIR}"/tmpfile - mv "${NBTMPDIR}"/tmpfile "${NBTESTSCRIPT}" - - echo "Running \"ipython ${NO_COLORS} ${NBTESTSCRIPT}\" on $(date)" - echo - time timeout 30m bash -c "ipython ${NO_COLORS} ${NBTESTSCRIPT}; EC=\$?; echo -------------------------------------------------------------------------------- ; echo DONE: ${NBNAME}; exit \$EC" - NBEXITCODE=$? - echo EXIT CODE: ${NBEXITCODE} - echo + + # Skip notebooks that are not yet supported + SKIP_NOTEBOOKS=( + "trnsport_cuopt" + "Production_Planning_Example_Pulp" + "Simple_LP_pulp" + "Simple_MIP_pulp" + "Sudoku_pulp" + ) + + for skip in "${SKIP_NOTEBOOKS[@]}"; do + if [[ "$NBNAME" == "$skip"* ]]; then + echo "Skipping notebook '${NBNAME}' as it matches skip pattern '${skip}'" + cd "$ORIGINAL_DIR" || exit 1 + continue 2 + fi + done + + rapids-logger "Running commands from notebook: ${NBNAME}.ipynb" + + python3 "$ORIGINAL_DIR/../ci/utils/notebook_command_extractor.py" "$NBNAME.ipynb" --verbose + + rapids-logger "Executing notebook: ${NBNAME}.ipynb" + # Execute notebook with default kernel + jupyter nbconvert --execute "${NBNAME}.ipynb" --to notebook --output "${EXECUTED_NOTEBOOK}" --ExecutePreprocessor.kernel_name="python3" + + if [ $? -eq 0 ]; then + echo "Notebook executed successfully: ${EXECUTED_NOTEBOOK}" + else + echo "ERROR: Failed to execute notebook: ${NBFILENAME}" + EXITCODE=1 + fi + + cd "${ORIGINAL_DIR}" || exit 1 done exit ${EXITCODE} diff --git a/ci/utils/notebook_command_extractor.py b/ci/utils/notebook_command_extractor.py new file mode 100644 index 0000000000..7b443c7bd2 --- /dev/null +++ b/ci/utils/notebook_command_extractor.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Notebook Command Extractor + +This script extracts pip install and other shell commands from Jupyter notebooks +and can optionally execute them. It's designed to be used by the nbtest.sh script. +""" + +import argparse +import json +import subprocess +import sys +from typing import List, Tuple + + +def extract_pip_commands(notebook_path: str) -> List[str]: + """Extract pip install commands from a Jupyter notebook.""" + try: + with open(notebook_path, "r") as f: + notebook = json.load(f) + + pip_commands = [] + for cell in notebook.get("cells", []): + if cell.get("cell_type") == "code": + source = "".join(cell.get("source", [])) + lines = source.split("\n") + for line in lines: + line = line.strip() + if line.startswith("!pip install") or line.startswith( + "pip install" + ): + # Clean up the line but preserve quotes + clean_line = line.strip() + if clean_line: + pip_commands.append(clean_line) + + return pip_commands + + except Exception as e: + print(f"Error parsing notebook: {e}", file=sys.stderr) + return [] + + +def extract_shell_commands(notebook_path: str) -> List[str]: + """Extract other shell commands from a Jupyter notebook.""" + try: + with open(notebook_path, "r") as f: + notebook = json.load(f) + + shell_commands = [] + allowed_commands = [ + "wget", + "curl", + "git", + "python", + "cd", + "mkdir", + "rm", + "cp", + "mv", + "unzip", + "tar", + ] + + for cell in notebook.get("cells", []): + if cell.get("cell_type") == "code": + source = "".join(cell.get("source", [])) + lines = source.split("\n") + for line in lines: + line = line.strip() + if line.startswith("!"): + # Check if it's a shell command we want to execute + cmd = ( + line[1:].strip().split()[0] + if line[1:].strip() + else "" + ) + if cmd in allowed_commands: + shell_commands.append(line) + + return shell_commands + + except Exception as e: + print(f"Error parsing notebook: {e}", file=sys.stderr) + return [] + + +def execute_pip_command(cmd: str, verbose: bool = False) -> bool: + """Execute a pip install command.""" + if verbose: + print(f"Processing command: '{cmd}'") + + # Remove the ! prefix if present for execution + exec_cmd = cmd.lstrip("!").strip() + + if verbose: + print(f"DEBUG: Original command: '{cmd}'") + print(f"DEBUG: Cleaned command: '{exec_cmd}'") + print(f"DEBUG: Command length: {len(exec_cmd)}") + print( + f"DEBUG: Command contains 'numpy': {'YES' if 'numpy' in exec_cmd else 'NO'}" + ) + print(f"Executing: {exec_cmd}") + + # Add --pre to exec_cmd if not already present + if exec_cmd.startswith("pip install") and "--pre" not in exec_cmd: + exec_cmd += " --pre --extra-index-url https://pypi.anaconda.org/rapidsai-nightly/simple" + + if verbose: + print(f"Final command: {exec_cmd}") + + try: + # Execute pip install commands + if exec_cmd.startswith("pip install"): + # Use shell=True for pip install to handle quoted arguments properly + # This is safe since we're only executing pip install commands + result = subprocess.run( + exec_cmd, shell=True, capture_output=True, text=True + ) + if result.returncode == 0: + if verbose: + print(f"✓ Successfully executed: {cmd}") + return True + else: + if verbose: + print(f"✗ Failed to execute: {cmd}") + print(f"Error: {result.stderr}") + return False + else: + if verbose: + print(f"✗ Invalid pip install command format: {exec_cmd}") + return False + except Exception as e: + if verbose: + print(f"✗ Exception executing {cmd}: {e}") + return False + + +def execute_shell_command(cmd: str, verbose: bool = False) -> bool: + """Execute a shell command.""" + if verbose: + print(f"Processing command: '{cmd}'") + + # Remove the ! prefix for execution + exec_cmd = cmd.lstrip("!").strip() + + if verbose: + print(f"DEBUG: Original command: '{cmd}'") + print(f"DEBUG: Cleaned command: '{exec_cmd}'") + print(f"DEBUG: Command length: {len(exec_cmd)}") + print(f"Executing: {exec_cmd}") + + # Skip potentially dangerous commands + dangerous_commands = ["chmod", "chown", "sudo", "su"] + if any(exec_cmd.startswith(dangerous) for dangerous in dangerous_commands): + if verbose: + print(f"⚠ Skipping potentially dangerous command: {cmd}") + return False + + try: + if verbose: + print("Executing shell command...") + + result = subprocess.run( + exec_cmd, shell=True, capture_output=True, text=True + ) + if result.returncode == 0: + if verbose: + print(f"✓ Successfully executed: {cmd}") + return True + else: + if verbose: + print(f"✗ Failed to execute: {cmd}") + print(f"Error: {result.stderr}") + return False + except Exception as e: + if verbose: + print(f"✗ Exception executing {cmd}: {e}") + return False + + +def main(): + parser = argparse.ArgumentParser( + description="Extract and optionally execute commands from Jupyter notebooks" + ) + parser.add_argument("notebook_path", help="Path to the Jupyter notebook") + parser.add_argument( + "--extract-only", + action="store_true", + help="Only extract commands, do not execute", + ) + parser.add_argument( + "--verbose", "-v", action="store_true", help="Verbose output" + ) + parser.add_argument( + "--output-format", + choices=["json", "text"], + default="text", + help="Output format for extracted commands", + ) + + args = parser.parse_args() + + # Extract commands + pip_commands = extract_pip_commands(args.notebook_path) + shell_commands = extract_shell_commands(args.notebook_path) + + print(f"Pip commands: {pip_commands}") + print(f"Shell commands: {shell_commands}") + + if args.output_format == "json": + # Output as JSON for shell script processing + output = { + "pip_commands": pip_commands, + "shell_commands": shell_commands, + } + print(json.dumps(output)) + else: + # Output as text (default) + if pip_commands: + print("PIP_COMMANDS:") + for cmd in pip_commands: + print(cmd) + + if shell_commands: + print("SHELL_COMMANDS:") + for cmd in shell_commands: + print(cmd) + + # Execute commands if not extract-only mode + if not args.extract_only: + success_count = 0 + total_count = 0 + + if pip_commands: + print(f"\nExecuting {len(pip_commands)} pip install commands...") + for cmd in pip_commands: + if execute_pip_command(cmd, args.verbose): + success_count += 1 + total_count += 1 + + if shell_commands: + print(f"\nExecuting {len(shell_commands)} shell commands...") + for cmd in shell_commands: + if execute_shell_command(cmd, args.verbose): + success_count += 1 + total_count += 1 + + if total_count > 0: + print( + f"\nExecution summary: {success_count}/{total_count} commands succeeded" + ) + return 0 if success_count == total_count else 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/utils/notebook_list.py b/ci/utils/notebook_list.py index 0ecbddcc3b..1b4e5a26ba 100644 --- a/ci/utils/notebook_list.py +++ b/ci/utils/notebook_list.py @@ -41,27 +41,6 @@ skip = True print(f"SKIPPING {filename} (marked as skip)", file=sys.stderr) break - elif re.search("dask", line): - print( - f"SKIPPING {filename} (suspected Dask usage, not currently automatable)", - file=sys.stderr, - ) - skip = True - break - elif pascal and re.search("# Does not run on Pascal", line): - print( - f"SKIPPING {filename} (does not run on Pascal)", - file=sys.stderr, - ) - skip = True - break - elif re.search("CVRPTW Exercise", line): - print( - f"SKIPPING {filename} (user exercise notebook)", - file=sys.stderr, - ) - skip = True - break if not skip: print(filename) diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml index 5353890305..2a3120f5a3 100644 --- a/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -27,9 +27,7 @@ dependencies: - doxygen=1.9.1 - exhale - fastapi -- folium - gcc_linux-aarch64=14.* -- geopandas - gmock - gtest - httpx @@ -38,11 +36,9 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libgdal<3.9.0 - libraft-headers==25.10.* - librmm==25.10.* - make -- matplotlib - msgpack-numpy==0.4.8 - msgpack-python==1.1.0 - myst-nb @@ -56,7 +52,6 @@ dependencies: - pandas>=2.0 - pexpect - pip -- polyline - pre-commit - psutil>=5.9,<6.0a0 - pylibraft==25.10.*,>=0.0.0a0 @@ -71,7 +66,6 @@ dependencies: - requests - rmm==25.10.*,>=0.0.0a0 - scikit-build-core>=0.10.0 -- scipy - sphinx - sphinx-copybutton - sphinx-design diff --git a/conda/environments/all_cuda-129_arch-x86_64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml index 63fb69d765..da6d19c193 100644 --- a/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -27,9 +27,7 @@ dependencies: - doxygen=1.9.1 - exhale - fastapi -- folium - gcc_linux-64=14.* -- geopandas - gmock - gtest - httpx @@ -38,11 +36,9 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libgdal<3.9.0 - libraft-headers==25.10.* - librmm==25.10.* - make -- matplotlib - msgpack-numpy==0.4.8 - msgpack-python==1.1.0 - myst-nb @@ -56,7 +52,6 @@ dependencies: - pandas>=2.0 - pexpect - pip -- polyline - pre-commit - psutil>=5.9,<6.0a0 - pylibraft==25.10.*,>=0.0.0a0 @@ -71,7 +66,6 @@ dependencies: - requests - rmm==25.10.*,>=0.0.0a0 - scikit-build-core>=0.10.0 -- scipy - sphinx - sphinx-copybutton - sphinx-design diff --git a/conda/environments/all_cuda-130_arch-aarch64.yaml b/conda/environments/all_cuda-130_arch-aarch64.yaml index c473a9713b..cf306f74e6 100644 --- a/conda/environments/all_cuda-130_arch-aarch64.yaml +++ b/conda/environments/all_cuda-130_arch-aarch64.yaml @@ -27,9 +27,7 @@ dependencies: - doxygen=1.9.1 - exhale - fastapi -- folium - gcc_linux-aarch64=14.* -- geopandas - gmock - gtest - httpx @@ -38,11 +36,9 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libgdal<3.9.0 - libraft-headers==25.10.* - librmm==25.10.* - make -- matplotlib - msgpack-numpy==0.4.8 - msgpack-python==1.1.0 - myst-nb @@ -56,7 +52,6 @@ dependencies: - pandas>=2.0 - pexpect - pip -- polyline - pre-commit - psutil>=5.9,<6.0a0 - pylibraft==25.10.*,>=0.0.0a0 @@ -71,7 +66,6 @@ dependencies: - requests - rmm==25.10.*,>=0.0.0a0 - scikit-build-core>=0.10.0 -- scipy - sphinx - sphinx-copybutton - sphinx-design diff --git a/conda/environments/all_cuda-130_arch-x86_64.yaml b/conda/environments/all_cuda-130_arch-x86_64.yaml index 0eddbfeeea..c9ba5beba0 100644 --- a/conda/environments/all_cuda-130_arch-x86_64.yaml +++ b/conda/environments/all_cuda-130_arch-x86_64.yaml @@ -27,9 +27,7 @@ dependencies: - doxygen=1.9.1 - exhale - fastapi -- folium - gcc_linux-64=14.* -- geopandas - gmock - gtest - httpx @@ -38,11 +36,9 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libgdal<3.9.0 - libraft-headers==25.10.* - librmm==25.10.* - make -- matplotlib - msgpack-numpy==0.4.8 - msgpack-python==1.1.0 - myst-nb @@ -56,7 +52,6 @@ dependencies: - pandas>=2.0 - pexpect - pip -- polyline - pre-commit - psutil>=5.9,<6.0a0 - pylibraft==25.10.*,>=0.0.0a0 @@ -71,7 +66,6 @@ dependencies: - requests - rmm==25.10.*,>=0.0.0a0 - scikit-build-core>=0.10.0 -- scipy - sphinx - sphinx-copybutton - sphinx-design diff --git a/dependencies.yaml b/dependencies.yaml index d1dbc07f2e..6508a60a69 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -69,6 +69,8 @@ files: - cuda_version - notebooks - py_version + - depends_on_cuopt_server + - depends_on_cuopt_sh_client checks: output: none includes: @@ -817,15 +819,8 @@ dependencies: common: - output_types: [conda, requirements] packages: - - breathe - - folium - - geopandas - ipython - - matplotlib - notebook - - polyline - - scipy - - libgdal<3.9.0 - output_types: [conda] packages: - *jsonref