From 8c9a2de7044e6daaf3d1a75a16b03f71a53cf12d Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sat, 3 May 2025 20:40:13 -0700 Subject: [PATCH 01/24] WIP (search priority updated in README.md but not in code) --- .../cuda/bindings/_path_finder/README.md | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/cuda_bindings/cuda/bindings/_path_finder/README.md b/cuda_bindings/cuda/bindings/_path_finder/README.md index 94b80499ff..ae18e86a4e 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/README.md +++ b/cuda_bindings/cuda/bindings/_path_finder/README.md @@ -24,17 +24,21 @@ strategy for locating NVIDIA shared libraries: The absolute path of the already loaded library will be returned, along with the handle to the library. -1. **Python Package Ecosystem** - - Scans `sys.path` to find libraries installed via NVIDIA Python wheels. +1. **Environment variables** + - Relies on `CUDA_HOME`/`CUDA_PATH` environment variables if set. -2. **Conda Environments** - - Leverages Conda-specific paths through our fork of `get_cuda_paths()` - from numba-cuda. +2. **NVIDIA Python wheels** + - Scans all site-packages to find libraries installed via NVIDIA Python wheels. -3. **Environment variables** - - Relies on `CUDA_HOME`/`CUDA_PATH` environment variables if set. +3. **OS default mechanisms / Conda environments** + - Falls back to native loader: + - `dlopen()` on Linux + - `LoadLibraryW()` on Windows + - Conda environments are expected to be covered by OS default mechanisms: + - Based on `$ORIGIN/../lib` `RPATH` on Linux + - Based on `%CONDA_PREFIX%\Library\bin` on the system `PATH` on Windows -4. **System Installations** +5. **System Installations** - Checks traditional system locations through these paths: - Linux: `/usr/local/cuda/lib64` - Windows: `C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\vX.Y\bin` @@ -44,11 +48,6 @@ strategy for locating NVIDIA shared libraries: - Distribution-specific packages (RPM/DEB) EXCEPT Debian's `nvidia-cuda-toolkit` -5. **OS Default Mechanisms** - - Falls back to native loader: - - `dlopen()` on Linux - - `LoadLibraryW()` on Windows - Note that the search is done on a per-library basis. There is no centralized mechanism that ensures all libraries are found in the same way. From 2cf3fa2e52f02711459d7fb25189635b989e740e Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sun, 4 May 2025 15:25:54 -0700 Subject: [PATCH 02/24] Completely replace cuda_paths.py to achieve the desired Search Priority (see updated README.md). --- .../cuda/bindings/_path_finder/README.md | 40 +- .../cuda/bindings/_path_finder/cuda_paths.py | 573 ------------------ .../find_nvidia_dynamic_library.py | 138 +++-- .../cuda/bindings/_path_finder/findlib.py | 97 --- .../bindings/_path_finder/load_dl_common.py | 20 + .../load_nvidia_dynamic_library.py | 32 +- ..._find_load.py => test_path_finder_load.py} | 32 +- toolshed/run_cuda_bindings_path_finder.py | 7 +- 8 files changed, 148 insertions(+), 791 deletions(-) delete mode 100644 cuda_bindings/cuda/bindings/_path_finder/cuda_paths.py delete mode 100644 cuda_bindings/cuda/bindings/_path_finder/findlib.py rename cuda_bindings/tests/{test_path_finder_find_load.py => test_path_finder_load.py} (75%) diff --git a/cuda_bindings/cuda/bindings/_path_finder/README.md b/cuda_bindings/cuda/bindings/_path_finder/README.md index ae18e86a4e..5096246b87 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/README.md +++ b/cuda_bindings/cuda/bindings/_path_finder/README.md @@ -25,7 +25,8 @@ strategy for locating NVIDIA shared libraries: with the handle to the library. 1. **Environment variables** - - Relies on `CUDA_HOME`/`CUDA_PATH` environment variables if set. + - Relies on `CUDA_HOME` or `CUDA_PATH` environment variables if set + (in that order). 2. **NVIDIA Python wheels** - Scans all site-packages to find libraries installed via NVIDIA Python wheels. @@ -34,41 +35,16 @@ strategy for locating NVIDIA shared libraries: - Falls back to native loader: - `dlopen()` on Linux - `LoadLibraryW()` on Windows - - Conda environments are expected to be covered by OS default mechanisms: - - Based on `$ORIGIN/../lib` `RPATH` on Linux - - Based on `%CONDA_PREFIX%\Library\bin` on the system `PATH` on Windows - -5. **System Installations** - - Checks traditional system locations through these paths: - - Linux: `/usr/local/cuda/lib64` - - Windows: `C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\vX.Y\bin` - (where X.Y is the CTK version) - - **Notably does NOT search**: - - Versioned CUDA directories like `/usr/local/cuda-12.3` - - Distribution-specific packages (RPM/DEB) - EXCEPT Debian's `nvidia-cuda-toolkit` + - CTK installations with system config updates are expected to be discovered: + - Linux: Via `/etc/ld.so.conf.d/*cuda*.conf` + - Windows: Via `C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\vX.Y\bin` on system `PATH` + - Conda installations are expected to be discovered: + - Linux: Via `$ORIGIN/../lib` on `RPATH` (of the `python` binary) + - Windows: Via `%CONDA_PREFIX%\Library\bin` on system `PATH` Note that the search is done on a per-library basis. There is no centralized mechanism that ensures all libraries are found in the same way. -## Implementation Philosophy - -The current implementation balances stability and evolution: - -- **Baseline Foundation:** Uses a fork of numba-cuda's `cuda_paths.py` that has been - battle-tested in production environments. - -- **Validation Infrastructure:** Comprehensive CI testing matrix being developed to cover: - - Various Linux/Windows environments - - Python packaging formats (wheels, conda) - - CUDA Toolkit versions - -- **Roadmap:** Planned refactoring to: - - Unify library discovery logic - - Improve maintainability - - Better enforce search priority - - Expand platform support - ## Maintenance Requirements These key components must be updated for new CUDA Toolkit releases: diff --git a/cuda_bindings/cuda/bindings/_path_finder/cuda_paths.py b/cuda_bindings/cuda/bindings/_path_finder/cuda_paths.py deleted file mode 100644 index 80f4e0149f..0000000000 --- a/cuda_bindings/cuda/bindings/_path_finder/cuda_paths.py +++ /dev/null @@ -1,573 +0,0 @@ -# Copyright 2025 NVIDIA Corporation. All rights reserved. -# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE - -# Forked from: -# https://github.com/NVIDIA/numba-cuda/blob/8c9c9d0cb901c06774a9abea6d12b6a4b0287e5e/numba_cuda/numba/cuda/cuda_paths.py - -# The numba-cuda version in turn was forked from: -# https://github.com/numba/numba/blob/6c8a71ffc3eaa1c68e1bac927b80ee7469002b3f/numba/cuda/cuda_paths.py -# SPDX-License-Identifier: BSD-2-Clause -# -# Original Numba LICENSE: -# Copyright (c) 2012, Anaconda, Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import ctypes -import os -import platform -import re -import site -import sys -import traceback -import warnings -from collections import namedtuple -from pathlib import Path - -from cuda.bindings._path_finder.findlib import find_lib - -IS_WIN32 = sys.platform.startswith("win32") - -_env_path_tuple = namedtuple("_env_path_tuple", ["by", "info"]) - - -def _get_numba_CUDA_INCLUDE_PATH(): - # From numba/numba/core/config.py - - def _readenv(name, ctor, default): - value = os.environ.get(name) - if value is None: - return default() if callable(default) else default - try: - return ctor(value) - except Exception: - warnings.warn( # noqa: B028 - f"Environment variable '{name}' is defined but " - f"its associated value '{value}' could not be " - "parsed.\nThe parse failed with exception:\n" - f"{traceback.format_exc()}", - RuntimeWarning, - ) - return default - - if IS_WIN32: - cuda_path = os.environ.get("CUDA_PATH") - if cuda_path: # noqa: SIM108 - default_cuda_include_path = os.path.join(cuda_path, "include") - else: - default_cuda_include_path = "cuda_include_not_found" - else: - default_cuda_include_path = os.path.join(os.sep, "usr", "local", "cuda", "include") - CUDA_INCLUDE_PATH = _readenv("NUMBA_CUDA_INCLUDE_PATH", str, default_cuda_include_path) - return CUDA_INCLUDE_PATH - - -config_CUDA_INCLUDE_PATH = _get_numba_CUDA_INCLUDE_PATH() - -SEARCH_PRIORITY = [ - "Conda environment", - "Conda environment (NVIDIA package)", - "NVIDIA NVCC Wheel", - "CUDA_HOME", - "System", - "Debian package", -] - - -def _priority_index(label): - if label in SEARCH_PRIORITY: - return SEARCH_PRIORITY.index(label) - else: - raise ValueError(f"Can't determine search priority for {label}") - - -def _find_first_valid_lazy(options): - sorted_options = sorted(options, key=lambda x: _priority_index(x[0])) - for label, fn in sorted_options: - value = fn() - if value: - return label, value - return "", None - - -def _build_options(pairs): - """Sorts and returns a list of (label, value) tuples according to SEARCH_PRIORITY.""" - priority_index = {label: i for i, label in enumerate(SEARCH_PRIORITY)} - return sorted(pairs, key=lambda pair: priority_index.get(pair[0], float("inf"))) - - -def _find_valid_path(options): - """Find valid path from *options*, which is a list of 2-tuple of - (name, path). Return first pair where *path* is not None. - If no valid path is found, return ('', None) - """ - for by, data in options: - if data is not None: - return by, data - else: - return "", None - - -def _get_libdevice_path_decision(): - options = _build_options( - [ - ("Conda environment", get_conda_ctk), - ("Conda environment (NVIDIA package)", get_nvidia_libdevice_ctk), - ("CUDA_HOME", lambda: get_cuda_home("nvvm", "libdevice")), - ("NVIDIA NVCC Wheel", get_libdevice_wheel), - ("System", lambda: get_system_ctk("nvvm", "libdevice")), - ("Debian package", get_debian_pkg_libdevice), - ] - ) - return _find_first_valid_lazy(options) - - -def _nvvm_lib_dir(): - if IS_WIN32: - return "nvvm", "bin" - else: - return "nvvm", "lib64" - - -def _get_nvvm_path_decision(): - options = [ - ("Conda environment", get_conda_ctk), - ("Conda environment (NVIDIA package)", get_nvidia_nvvm_ctk), - ("NVIDIA NVCC Wheel", _get_nvvm_wheel), - ("CUDA_HOME", lambda: get_cuda_home(*_nvvm_lib_dir())), - ("System", lambda: get_system_ctk(*_nvvm_lib_dir())), - ] - return _find_first_valid_lazy(options) - - -def _get_nvrtc_system_ctk(): - sys_path = get_system_ctk("bin" if IS_WIN32 else "lib64") - candidates = find_lib("nvrtc", sys_path) - if candidates: - return max(candidates) - - -def _get_nvrtc_path_decision(): - options = _build_options( - [ - ("CUDA_HOME", lambda: get_cuda_home("nvrtc")), - ("Conda environment", get_conda_ctk), - ("Conda environment (NVIDIA package)", get_nvidia_cudalib_ctk), - ("NVIDIA NVCC Wheel", _get_nvrtc_wheel), - ("System", _get_nvrtc_system_ctk), - ] - ) - return _find_first_valid_lazy(options) - - -def _get_nvvm_wheel(): - platform_map = { - "linux": ("lib64", "libnvvm.so"), - "win32": ("bin", "nvvm64_40_0.dll"), - } - - for plat, (dso_dir, dso_path) in platform_map.items(): - if sys.platform.startswith(plat): - break - else: - raise NotImplementedError("Unsupported platform") - - site_paths = [site.getusersitepackages()] + site.getsitepackages() - - for sp in filter(None, site_paths): - nvvm_path = Path(sp, "nvidia", "cuda_nvcc", "nvvm", dso_dir, dso_path) - if nvvm_path.exists(): - return str(nvvm_path.parent) - - return None - - -def get_nvrtc_dso_path(): - site_paths = [site.getusersitepackages()] + site.getsitepackages() - for sp in site_paths: - lib_dir = os.path.join( - sp, - "nvidia", - "cuda_nvrtc", - ("bin" if IS_WIN32 else "lib") if sp else None, - ) - if lib_dir and os.path.exists(lib_dir): - for major in (12, 11): - if major == 11: - cu_ver = "112" if IS_WIN32 else "11.2" - elif major == 12: - cu_ver = "120" if IS_WIN32 else "12" - else: - raise NotImplementedError(f"CUDA {major} is not supported") - - dso_path = os.path.join( - lib_dir, - f"nvrtc64_{cu_ver}_0.dll" if IS_WIN32 else f"libnvrtc.so.{cu_ver}", - ) - if os.path.isfile(dso_path): - return dso_path - return None - - -def _get_nvrtc_wheel(): - dso_path = get_nvrtc_dso_path() - if dso_path: - try: - result = ctypes.CDLL(dso_path, mode=ctypes.RTLD_GLOBAL) - except OSError: - pass - else: - if IS_WIN32: - import win32api - - # This absolute path will - # always be correct regardless of the package source - nvrtc_path = win32api.GetModuleFileNameW(result._handle) - dso_dir = os.path.dirname(nvrtc_path) - builtins_path = os.path.join( - dso_dir, - [f for f in os.listdir(dso_dir) if re.match("^nvrtc-builtins.*.dll$", f)][0], - ) - if not os.path.exists(builtins_path): - raise RuntimeError(f'Path does not exist: "{builtins_path}"') - return Path(dso_path) - - -def _get_libdevice_paths(): - by, libdir = _get_libdevice_path_decision() - if not libdir: - return _env_path_tuple(by, None) - out = os.path.join(libdir, "libdevice.10.bc") - return _env_path_tuple(by, out) - - -def _cudalib_path(): - if IS_WIN32: - return "bin" - else: - return "lib64" - - -def _cuda_home_static_cudalib_path(): - if IS_WIN32: - return ("lib", "x64") - else: - return ("lib64",) - - -def _get_cudalib_wheel(): - """Get the cudalib path from the NVCC wheel.""" - site_paths = [site.getusersitepackages()] + site.getsitepackages() - libdir = "bin" if IS_WIN32 else "lib" - for sp in filter(None, site_paths): - cudalib_path = Path(sp, "nvidia", "cuda_runtime", libdir) - if cudalib_path.exists(): - return str(cudalib_path) - return None - - -def _get_cudalib_dir_path_decision(): - options = _build_options( - [ - ("Conda environment", get_conda_ctk), - ("Conda environment (NVIDIA package)", get_nvidia_cudalib_ctk), - ("NVIDIA NVCC Wheel", _get_cudalib_wheel), - ("CUDA_HOME", lambda: get_cuda_home(_cudalib_path())), - ("System", lambda: get_system_ctk(_cudalib_path())), - ] - ) - return _find_first_valid_lazy(options) - - -def _get_static_cudalib_dir_path_decision(): - options = _build_options( - [ - ("Conda environment", get_conda_ctk), - ( - "Conda environment (NVIDIA package)", - get_nvidia_static_cudalib_ctk, - ), - ( - "CUDA_HOME", - lambda: get_cuda_home(*_cuda_home_static_cudalib_path()), - ), - ("System", lambda: get_system_ctk(_cudalib_path())), - ] - ) - return _find_first_valid_lazy(options) - - -def _get_cudalib_dir(): - by, libdir = _get_cudalib_dir_path_decision() - return _env_path_tuple(by, libdir) - - -def _get_static_cudalib_dir(): - by, libdir = _get_static_cudalib_dir_path_decision() - return _env_path_tuple(by, libdir) - - -def get_system_ctk(*subdirs): - """Return path to system-wide cudatoolkit; or, None if it doesn't exist.""" - # Linux? - if not IS_WIN32: - # Is cuda alias to /usr/local/cuda? - # We are intentionally not getting versioned cuda installation. - result = os.path.join("/usr/local/cuda", *subdirs) - if os.path.exists(result): - return result - - -def get_conda_ctk(): - """Return path to directory containing the shared libraries of cudatoolkit.""" - is_conda_env = os.path.exists(os.path.join(sys.prefix, "conda-meta")) - if not is_conda_env: - return - # Assume the existence of NVVM to imply cudatoolkit installed - paths = find_lib("nvvm") - if not paths: - return - # Use the directory name of the max path - return os.path.dirname(max(paths)) - - -def get_nvidia_nvvm_ctk(): - """Return path to directory containing the NVVM shared library.""" - is_conda_env = os.path.exists(os.path.join(sys.prefix, "conda-meta")) - if not is_conda_env: - return - - # Assume the existence of NVVM in the conda env implies that a CUDA toolkit - # conda package is installed. - - # First, try the location used on Linux and the Windows 11.x packages - libdir = os.path.join(sys.prefix, "nvvm", _cudalib_path()) - if not os.path.exists(libdir) or not os.path.isdir(libdir): - # If that fails, try the location used for Windows 12.x packages - libdir = os.path.join(sys.prefix, "Library", "nvvm", _cudalib_path()) - if not os.path.exists(libdir) or not os.path.isdir(libdir): - # If that doesn't exist either, assume we don't have the NVIDIA - # conda package - return - - paths = find_lib("nvvm", libdir=libdir) - if not paths: - return - # Use the directory name of the max path - return os.path.dirname(max(paths)) - - -def get_nvidia_libdevice_ctk(): - """Return path to directory containing the libdevice library.""" - nvvm_ctk = get_nvidia_nvvm_ctk() - if not nvvm_ctk: - return - nvvm_dir = os.path.dirname(nvvm_ctk) - return os.path.join(nvvm_dir, "libdevice") - - -def get_nvidia_cudalib_ctk(): - """Return path to directory containing the shared libraries of cudatoolkit.""" - nvvm_ctk = get_nvidia_nvvm_ctk() - if not nvvm_ctk: - return - env_dir = os.path.dirname(os.path.dirname(nvvm_ctk)) - subdir = "bin" if IS_WIN32 else "lib" - return os.path.join(env_dir, subdir) - - -def get_nvidia_static_cudalib_ctk(): - """Return path to directory containing the static libraries of cudatoolkit.""" - nvvm_ctk = get_nvidia_nvvm_ctk() - if not nvvm_ctk: - return - - if IS_WIN32 and ("Library" not in nvvm_ctk): # noqa: SIM108 - # Location specific to CUDA 11.x packages on Windows - dirs = ("Lib", "x64") - else: - # Linux, or Windows with CUDA 12.x packages - dirs = ("lib",) - - env_dir = os.path.dirname(os.path.dirname(nvvm_ctk)) - return os.path.join(env_dir, *dirs) - - -def get_cuda_home(*subdirs): - """Get paths of CUDA_HOME. - If *subdirs* are the subdirectory name to be appended in the resulting - path. - """ - cuda_home = os.environ.get("CUDA_HOME") - if cuda_home is None: - # Try Windows CUDA installation without Anaconda - cuda_home = os.environ.get("CUDA_PATH") - if cuda_home is not None: - return os.path.join(cuda_home, *subdirs) - - -def _get_nvvm_path(): - by, path = _get_nvvm_path_decision() - - if by == "NVIDIA NVCC Wheel": - platform_map = { - "linux": "libnvvm.so", - "win32": "nvvm64_40_0.dll", - } - - for plat, dso_name in platform_map.items(): - if sys.platform.startswith(plat): - break - else: - raise NotImplementedError("Unsupported platform") - - path = os.path.join(path, dso_name) - else: - candidates = find_lib("nvvm", path) - path = max(candidates) if candidates else None - return _env_path_tuple(by, path) - - -def _get_nvrtc_path(): - by, path = _get_nvrtc_path_decision() - if by == "NVIDIA NVCC Wheel": - path = str(path) - elif by == "System": - return _env_path_tuple(by, path) - else: - candidates = find_lib("nvrtc", path) - path = max(candidates) if candidates else None - return _env_path_tuple(by, path) - - -def get_cuda_paths(): - """Returns a dictionary mapping component names to a 2-tuple - of (source_variable, info). - - The returned dictionary will have the following keys and infos: - - "nvvm": file_path - - "libdevice": List[Tuple[arch, file_path]] - - "cudalib_dir": directory_path - - Note: The result of the function is cached. - """ - # Check cache - if hasattr(get_cuda_paths, "_cached_result"): - return get_cuda_paths._cached_result - else: - # Not in cache - d = { - "nvvm": _get_nvvm_path(), - "nvrtc": _get_nvrtc_path(), - "libdevice": _get_libdevice_paths(), - "cudalib_dir": _get_cudalib_dir(), - "static_cudalib_dir": _get_static_cudalib_dir(), - "include_dir": _get_include_dir(), - } - # Cache result - get_cuda_paths._cached_result = d - return d - - -def get_debian_pkg_libdevice(): - """ - Return the Debian NVIDIA Maintainers-packaged libdevice location, if it - exists. - """ - pkg_libdevice_location = "/usr/lib/nvidia-cuda-toolkit/libdevice" - if not os.path.exists(pkg_libdevice_location): - return None - return pkg_libdevice_location - - -def get_libdevice_wheel(): - nvvm_path = _get_nvvm_wheel() - if nvvm_path is None: - return None - nvvm_path = Path(nvvm_path) - libdevice_path = nvvm_path.parent / "libdevice" - - return str(libdevice_path) - - -def get_current_cuda_target_name(): - """Determine conda's CTK target folder based on system and machine arch. - - CTK's conda package delivers headers based on its architecture type. For example, - `x86_64` machine places header under `$CONDA_PREFIX/targets/x86_64-linux`, and - `aarch64` places under `$CONDA_PREFIX/targets/sbsa-linux`. Read more about the - nuances at cudart's conda feedstock: - https://github.com/conda-forge/cuda-cudart-feedstock/blob/main/recipe/meta.yaml#L8-L11 # noqa: E501 - """ - system = platform.system() - machine = platform.machine() - - if system == "Linux": - arch_to_targets = {"x86_64": "x86_64-linux", "aarch64": "sbsa-linux"} - elif system == "Windows": - arch_to_targets = { - "AMD64": "x64", - } - else: - arch_to_targets = {} - - return arch_to_targets.get(machine, None) - - -def get_conda_include_dir(): - """ - Return the include directory in the current conda environment, if one - is active and it exists. - """ - is_conda_env = os.path.exists(os.path.join(sys.prefix, "conda-meta")) - if not is_conda_env: - return - - if platform.system() == "Windows": - include_dir = os.path.join(sys.prefix, "Library", "include") - elif target_name := get_current_cuda_target_name(): - include_dir = os.path.join(sys.prefix, "targets", target_name, "include") - else: - # A fallback when target cannot determined - # though usually it shouldn't. - include_dir = os.path.join(sys.prefix, "include") - - if ( - os.path.exists(include_dir) - and os.path.isdir(include_dir) - and os.path.exists(os.path.join(include_dir, "cuda_device_runtime_api.h")) - ): - return include_dir - return - - -def _get_include_dir(): - """Find the root include directory.""" - options = [ - ("Conda environment (NVIDIA package)", get_conda_include_dir()), - ("CUDA_INCLUDE_PATH Config Entry", config_CUDA_INCLUDE_PATH), - # TODO: add others - ] - by, include_dir = _find_valid_path(options) - return _env_path_tuple(by, include_dir) diff --git a/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py b/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py index af9f42fbf8..20908d971b 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py +++ b/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py @@ -6,7 +6,6 @@ import os import sys -from cuda.bindings._path_finder.cuda_paths import get_cuda_paths from cuda.bindings._path_finder.find_sub_dirs import find_sub_dirs_all_sitepackages from cuda.bindings._path_finder.supported_libs import is_suppressed_dll_file @@ -61,59 +60,77 @@ def _find_dll_using_nvidia_bin_dirs(libname, lib_searched_for, error_messages, a return None -def _get_cuda_paths_info(key, error_messages): - env_path_tuple = get_cuda_paths()[key] - if not env_path_tuple: - error_messages.append(f'Failure obtaining get_cuda_paths()["{key}"]') - return None - if not env_path_tuple.info: - error_messages.append(f'Failure obtaining get_cuda_paths()["{key}"].info') - return None - return env_path_tuple.info +def _get_cuda_home(): + cuda_home = os.environ.get("CUDA_HOME") + if cuda_home is None: + cuda_home = os.environ.get("CUDA_PATH") + return cuda_home -def _find_so_using_cudalib_dir(so_basename, error_messages, attachments): - cudalib_dir = _get_cuda_paths_info("cudalib_dir", error_messages) - if cudalib_dir is None: +def _find_lib_dir_using_cuda_home(libname): + cuda_home = _get_cuda_home() + if cuda_home is None: return None - primary_so_dir = cudalib_dir + "/" - candidate_so_dirs = [primary_so_dir] - libs = ["/lib/", "/lib64/"] - for _ in range(2): - alt_dir = libs[0].join(primary_so_dir.rsplit(libs[1], 1)) - if alt_dir not in candidate_so_dirs: - candidate_so_dirs.append(alt_dir) - libs.reverse() - candidate_so_names = [so_dirname + so_basename for so_dirname in candidate_so_dirs] - for so_name in candidate_so_names: - if os.path.isfile(so_name): - return so_name - error_messages.append(f"No such file: {so_name}") - for so_dirname in candidate_so_dirs: - attachments.append(f' listdir("{so_dirname}"):') - if not os.path.isdir(so_dirname): - attachments.append(" DIRECTORY DOES NOT EXIST") + if sys.platform == "win32": + if libname == "nvvm": # noqa: SIM108 + subdirs = (os.path.join("nvvm", "bin"),) + else: + subdirs = ("bin",) + else: + if libname == "nvvm": # noqa: SIM108 + subdirs = (os.path.join("nvvm", "lib64"),) else: - for node in sorted(os.listdir(so_dirname)): - attachments.append(f" {node}") + subdirs = ( + "lib64", # CTK + "lib", # Conda + ) + for subdir in subdirs: + dirname = os.path.join(cuda_home, subdir) + if os.path.isdir(dirname): + return dirname return None -def _find_dll_using_cudalib_dir(libname, error_messages, attachments): - cudalib_dir = _get_cuda_paths_info("cudalib_dir", error_messages) - if cudalib_dir is None: - return None +def _find_so_using_lib_dir(lib_dir, so_basename, error_messages, attachments): + so_name = os.path.join(lib_dir, so_basename) + if os.path.isfile(so_name): + return so_name + error_messages.append(f"No such file: {so_name}") + attachments.append(f' listdir("{lib_dir}"):') + if not os.path.isdir(lib_dir): + attachments.append(" DIRECTORY DOES NOT EXIST") + else: + for node in sorted(os.listdir(lib_dir)): + attachments.append(f" {node}") + return None + + +def _find_dll_using_lib_dir(lib_dir, libname, error_messages, attachments): file_wild = libname + "*.dll" - dll_name = _find_dll_under_dir(cudalib_dir, file_wild) + dll_name = _find_dll_under_dir(lib_dir, file_wild) if dll_name is not None: return dll_name error_messages.append(f"No such file: {file_wild}") - attachments.append(f' listdir("{cudalib_dir}"):') - for node in sorted(os.listdir(cudalib_dir)): + attachments.append(f' listdir("{lib_dir}"):') + for node in sorted(os.listdir(lib_dir)): attachments.append(f" {node}") return None +def _find_nvvm_lib_dir_from_other_abs_path(other_abs_path): + if sys.platform == "win32": + nvvm_subdir = "bin" + else: + nvvm_subdir = "lib64" + while other_abs_path: + if os.path.isdir(other_abs_path): + nvvm_lib_dir = os.path.join(other_abs_path, "nvvm", nvvm_subdir) + if os.path.isdir(nvvm_lib_dir): + return nvvm_lib_dir + other_abs_path = os.path.dirname(other_abs_path) + return None + + class _find_nvidia_dynamic_library: def __init__(self, libname: str): self.libname = libname @@ -121,28 +138,39 @@ def __init__(self, libname: str): self.attachments = [] self.abs_path = None + cuda_home_lib_dir = _find_lib_dir_using_cuda_home(libname) if sys.platform == "win32": self.lib_searched_for = f"{libname}*.dll" - self.abs_path = _find_dll_using_nvidia_bin_dirs( - libname, self.lib_searched_for, self.error_messages, self.attachments - ) + if cuda_home_lib_dir is not None: + self.abs_path = _find_dll_using_lib_dir( + cuda_home_lib_dir, libname, self.error_messages, self.attachments + ) if self.abs_path is None: - if libname == "nvvm": - self.abs_path = _get_cuda_paths_info("nvvm", self.error_messages) - else: - self.abs_path = _find_dll_using_cudalib_dir(libname, self.error_messages, self.attachments) + self.abs_path = _find_dll_using_nvidia_bin_dirs( + libname, self.lib_searched_for, self.error_messages, self.attachments + ) else: self.lib_searched_for = f"lib{libname}.so" - self.abs_path = _find_so_using_nvidia_lib_dirs( - libname, self.lib_searched_for, self.error_messages, self.attachments - ) + if cuda_home_lib_dir is not None: + self.abs_path = _find_so_using_lib_dir( + cuda_home_lib_dir, self.lib_searched_for, self.error_messages, self.attachments + ) if self.abs_path is None: - if libname == "nvvm": - self.abs_path = _get_cuda_paths_info("nvvm", self.error_messages) - else: - self.abs_path = _find_so_using_cudalib_dir( - self.lib_searched_for, self.error_messages, self.attachments - ) + self.abs_path = _find_so_using_nvidia_lib_dirs( + libname, self.lib_searched_for, self.error_messages, self.attachments + ) + + def retry_with_other_abs_path(self, other_abs_path): + assert self.libname == "nvvm" + nvvm_lib_dir = _find_nvvm_lib_dir_from_other_abs_path(other_abs_path) + if nvvm_lib_dir is None: + return + if sys.platform == "win32": + self.abs_path = _find_dll_using_lib_dir(nvvm_lib_dir, self.libname, self.error_messages, self.attachments) + else: + self.abs_path = _find_so_using_lib_dir( + nvvm_lib_dir, self.lib_searched_for, self.error_messages, self.attachments + ) def raise_if_abs_path_is_None(self): if self.abs_path: diff --git a/cuda_bindings/cuda/bindings/_path_finder/findlib.py b/cuda_bindings/cuda/bindings/_path_finder/findlib.py deleted file mode 100644 index 992a3940e5..0000000000 --- a/cuda_bindings/cuda/bindings/_path_finder/findlib.py +++ /dev/null @@ -1,97 +0,0 @@ -# SPDX-License-Identifier: BSD-2-Clause -# -# Forked from: -# https://github.com/numba/numba/blob/f0d24824fcd6a454827e3c108882395d00befc04/numba/misc/findlib.py -# -# Original LICENSE: -# Copyright (c) 2012, Anaconda, Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import os -import re -import sys - - -def get_lib_dirs(): - """ - Anaconda specific - """ - if sys.platform == "win32": - # on windows, historically `DLLs` has been used for CUDA libraries, - # since approximately CUDA 9.2, `Library\bin` has been used. - dirnames = ["DLLs", os.path.join("Library", "bin")] - else: - dirnames = [ - "lib", - ] - libdirs = [os.path.join(sys.prefix, x) for x in dirnames] - return libdirs - - -DLLNAMEMAP = { - "linux": r"lib%(name)s\.so\.%(ver)s$", - "linux2": r"lib%(name)s\.so\.%(ver)s$", - "linux-static": r"lib%(name)s\.a$", - "darwin": r"lib%(name)s\.%(ver)s\.dylib$", - "win32": r"%(name)s%(ver)s\.dll$", - "win32-static": r"%(name)s\.lib$", - "bsd": r"lib%(name)s\.so\.%(ver)s$", -} - -RE_VER = r"[0-9]*([_\.][0-9]+)*" - - -def find_lib(libname, libdir=None, platform=None, static=False): - platform = platform or sys.platform - platform = "bsd" if "bsd" in platform else platform - if static: - platform = f"{platform}-static" - if platform not in DLLNAMEMAP: - # Return empty list if platform name is undefined. - # Not all platforms define their static library paths. - return [] - pat = DLLNAMEMAP[platform] % {"name": libname, "ver": RE_VER} - regex = re.compile(pat) - return find_file(regex, libdir) - - -def find_file(pat, libdir=None): - if libdir is None: - libdirs = get_lib_dirs() - elif isinstance(libdir, str): - libdirs = [ - libdir, - ] - else: - libdirs = list(libdir) - files = [] - for ldir in libdirs: - try: - entries = os.listdir(ldir) - except FileNotFoundError: - continue - candidates = [os.path.join(ldir, ent) for ent in entries if pat.match(ent)] - files.extend([c for c in candidates if os.path.isfile(c)]) - return files diff --git a/cuda_bindings/cuda/bindings/_path_finder/load_dl_common.py b/cuda_bindings/cuda/bindings/_path_finder/load_dl_common.py index 4592f6c335..717bdc00c5 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/load_dl_common.py +++ b/cuda_bindings/cuda/bindings/_path_finder/load_dl_common.py @@ -1,6 +1,8 @@ # Copyright 2025 NVIDIA Corporation. All rights reserved. # SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE +import subprocess # nosec B404 +import sys from dataclasses import dataclass from typing import Callable, Optional @@ -38,3 +40,21 @@ def load_dependencies(libname: str, load_func: Callable[[str], LoadedDL]) -> Non """ for dep in DIRECT_DEPENDENCIES.get(libname, ()): load_func(dep) + + +def load_in_subprocess(python_code, timeout=30): + # This is to avoid loading libraries into the parent process. + return subprocess.run( # nosec B603 + [sys.executable, "-c", python_code], + capture_output=True, + encoding="utf-8", + timeout=timeout, # Ensure this does not hang for an excessive amount of time. + ) + + +def build_subprocess_failed_for_libname_message(libname, result): + return ( + f"Subprocess failed for {libname=!r} with exit code {result.returncode}\n" + f"--- stdout-from-subprocess ---\n{result.stdout}\n" + f"--- stderr-from-subprocess ---\n{result.stderr}\n" + ) diff --git a/cuda_bindings/cuda/bindings/_path_finder/load_nvidia_dynamic_library.py b/cuda_bindings/cuda/bindings/_path_finder/load_nvidia_dynamic_library.py index 015c4cdf85..8e9df6f12f 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/load_nvidia_dynamic_library.py +++ b/cuda_bindings/cuda/bindings/_path_finder/load_nvidia_dynamic_library.py @@ -2,10 +2,16 @@ # SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE import functools +import json import sys from cuda.bindings._path_finder.find_nvidia_dynamic_library import _find_nvidia_dynamic_library -from cuda.bindings._path_finder.load_dl_common import LoadedDL, load_dependencies +from cuda.bindings._path_finder.load_dl_common import ( + LoadedDL, + build_subprocess_failed_for_libname_message, + load_dependencies, + load_in_subprocess, +) if sys.platform == "win32": from cuda.bindings._path_finder.load_dl_windows import ( @@ -21,6 +27,21 @@ ) +def _load_other_in_subprocess(libname, error_messages): + code = f"""\ +from cuda.bindings._path_finder.load_nvidia_dynamic_library import load_nvidia_dynamic_library +import json +import sys +loaded = load_nvidia_dynamic_library({libname!r}) +sys.stdout.write(json.dumps(loaded.abs_path, ensure_ascii=True)) +""" + result = load_in_subprocess(code) + if result.returncode == 0: + return json.loads(result.stdout) + error_messages.extend(build_subprocess_failed_for_libname_message(libname, result).splitlines()) + return None + + def _load_nvidia_dynamic_library_no_cache(libname: str) -> LoadedDL: # Check whether the library is already loaded into the current process by # some other component. This check uses OS-level mechanisms (e.g., @@ -38,6 +59,15 @@ def _load_nvidia_dynamic_library_no_cache(libname: str) -> LoadedDL: loaded = load_with_system_search(libname, found.lib_searched_for) if loaded is not None: return loaded + if libname == "nvvm": + # Use cudart as anchor point (libcudart.so.12 is only ~720K, cudart64_12.dll ~560K). + loaded_cudart = check_if_already_loaded_from_elsewhere("cudart") + if loaded_cudart is not None: + found.retry_with_other_abs_path(loaded_cudart.abs_path) + else: + cudart_abs_path = _load_other_in_subprocess("cudart", found.error_messages) + if cudart_abs_path is not None: + found.retry_with_other_abs_path(cudart_abs_path) found.raise_if_abs_path_is_None() # Load the library from the found path diff --git a/cuda_bindings/tests/test_path_finder_find_load.py b/cuda_bindings/tests/test_path_finder_load.py similarity index 75% rename from cuda_bindings/tests/test_path_finder_find_load.py rename to cuda_bindings/tests/test_path_finder_load.py index 2a5f887fdc..f665ca0229 100644 --- a/cuda_bindings/tests/test_path_finder_find_load.py +++ b/cuda_bindings/tests/test_path_finder_load.py @@ -2,13 +2,13 @@ # SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE import os -import subprocess # nosec B404 import sys import pytest from cuda.bindings import path_finder from cuda.bindings._path_finder import supported_libs +from cuda.bindings._path_finder.load_dl_common import build_subprocess_failed_for_libname_message, load_in_subprocess ALL_LIBNAMES = path_finder._SUPPORTED_LIBNAMES + supported_libs.PARTIALLY_SUPPORTED_LIBNAMES_ALL ALL_LIBNAMES_LINUX = path_finder._SUPPORTED_LIBNAMES + supported_libs.PARTIALLY_SUPPORTED_LIBNAMES_LINUX @@ -38,17 +38,8 @@ def test_all_libnames_expected_lib_symbols_consistency(): assert tuple(sorted(ALL_LIBNAMES)) == tuple(sorted(supported_libs.EXPECTED_LIB_SYMBOLS.keys())) -def _build_subprocess_failed_for_libname_message(libname, result): - return ( - f"Subprocess failed for {libname=!r} with exit code {result.returncode}\n" - f"--- stdout-from-subprocess ---\n{result.stdout}\n" - f"--- stderr-from-subprocess ---\n{result.stderr}\n" - ) - - -@pytest.mark.parametrize("api", ("find", "load")) @pytest.mark.parametrize("libname", TEST_FIND_OR_LOAD_LIBNAMES) -def test_find_or_load_nvidia_dynamic_library(info_summary_append, api, libname): +def test_find_or_load_nvidia_dynamic_library(info_summary_append, libname): # We intentionally run each dynamic library operation in a subprocess # to ensure isolation of global dynamic linking state (e.g., dlopen handles). # Without subprocesses, loading/unloading libraries during testing could @@ -56,14 +47,7 @@ def test_find_or_load_nvidia_dynamic_library(info_summary_append, api, libname): # # Defining the subprocess code snippets as strings ensures each subprocess # runs a minimal, independent script tailored to the specific libname and API being tested. - if api == "find": - code = f"""\ -from cuda.bindings._path_finder.find_nvidia_dynamic_library import find_nvidia_dynamic_library -abs_path = find_nvidia_dynamic_library({libname!r}) -print(f"{{abs_path!r}}") -""" - else: - code = f"""\ + code = f"""\ from cuda.bindings.path_finder import _load_nvidia_dynamic_library from cuda.bindings._path_finder.load_nvidia_dynamic_library import _load_nvidia_dynamic_library_no_cache @@ -83,14 +67,8 @@ def test_find_or_load_nvidia_dynamic_library(info_summary_append, api, libname): print(f"{{loaded_dl_fresh.abs_path!r}}") """ - result = subprocess.run( # nosec B603 - [sys.executable, "-c", code], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding="utf-8", - timeout=30, # Ensure CI testing does not hang for an excessive amount of time. - ) + result = load_in_subprocess(code) if result.returncode == 0: info_summary_append(f"abs_path={result.stdout.rstrip()}") else: - raise RuntimeError(_build_subprocess_failed_for_libname_message(libname, result)) + raise RuntimeError(build_subprocess_failed_for_libname_message(libname, result)) diff --git a/toolshed/run_cuda_bindings_path_finder.py b/toolshed/run_cuda_bindings_path_finder.py index 19f43c2881..cf173aa415 100644 --- a/toolshed/run_cuda_bindings_path_finder.py +++ b/toolshed/run_cuda_bindings_path_finder.py @@ -6,7 +6,7 @@ import traceback from cuda.bindings import path_finder -from cuda.bindings._path_finder import cuda_paths, supported_libs +from cuda.bindings._path_finder import supported_libs ALL_LIBNAMES = ( path_finder._SUPPORTED_LIBNAMES + supported_libs.PARTIALLY_SUPPORTED_LIBNAMES @@ -16,11 +16,6 @@ def run(args): assert len(args) == 0 - paths = cuda_paths.get_cuda_paths() - for k, v in paths.items(): - print(f"{k}: {v}", flush=True) - print() - for libname in ALL_LIBNAMES: print(f"{libname=}") try: From 2b740226335b788e699e244007c241f082a46c3b Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sun, 4 May 2025 22:01:10 -0700 Subject: [PATCH 03/24] Define `IS_WINDOWS = sys.platform == "win32"` in supported_libs.py --- .../_path_finder/find_nvidia_dynamic_library.py | 14 +++++--------- .../_path_finder/load_nvidia_dynamic_library.py | 4 ++-- .../cuda/bindings/_path_finder/supported_libs.py | 4 +++- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py b/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py index 20908d971b..7df8276c6d 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py +++ b/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py @@ -4,10 +4,9 @@ import functools import glob import os -import sys from cuda.bindings._path_finder.find_sub_dirs import find_sub_dirs_all_sitepackages -from cuda.bindings._path_finder.supported_libs import is_suppressed_dll_file +from cuda.bindings._path_finder.supported_libs import IS_WINDOWS, is_suppressed_dll_file def _no_such_file_in_sub_dirs(sub_dirs, file_wild, error_messages, attachments): @@ -71,7 +70,7 @@ def _find_lib_dir_using_cuda_home(libname): cuda_home = _get_cuda_home() if cuda_home is None: return None - if sys.platform == "win32": + if IS_WINDOWS: if libname == "nvvm": # noqa: SIM108 subdirs = (os.path.join("nvvm", "bin"),) else: @@ -118,10 +117,7 @@ def _find_dll_using_lib_dir(lib_dir, libname, error_messages, attachments): def _find_nvvm_lib_dir_from_other_abs_path(other_abs_path): - if sys.platform == "win32": - nvvm_subdir = "bin" - else: - nvvm_subdir = "lib64" + nvvm_subdir = "bin" if IS_WINDOWS else "lib64" while other_abs_path: if os.path.isdir(other_abs_path): nvvm_lib_dir = os.path.join(other_abs_path, "nvvm", nvvm_subdir) @@ -139,7 +135,7 @@ def __init__(self, libname: str): self.abs_path = None cuda_home_lib_dir = _find_lib_dir_using_cuda_home(libname) - if sys.platform == "win32": + if IS_WINDOWS: self.lib_searched_for = f"{libname}*.dll" if cuda_home_lib_dir is not None: self.abs_path = _find_dll_using_lib_dir( @@ -165,7 +161,7 @@ def retry_with_other_abs_path(self, other_abs_path): nvvm_lib_dir = _find_nvvm_lib_dir_from_other_abs_path(other_abs_path) if nvvm_lib_dir is None: return - if sys.platform == "win32": + if IS_WINDOWS: self.abs_path = _find_dll_using_lib_dir(nvvm_lib_dir, self.libname, self.error_messages, self.attachments) else: self.abs_path = _find_so_using_lib_dir( diff --git a/cuda_bindings/cuda/bindings/_path_finder/load_nvidia_dynamic_library.py b/cuda_bindings/cuda/bindings/_path_finder/load_nvidia_dynamic_library.py index 8e9df6f12f..b368f5d9ae 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/load_nvidia_dynamic_library.py +++ b/cuda_bindings/cuda/bindings/_path_finder/load_nvidia_dynamic_library.py @@ -3,7 +3,6 @@ import functools import json -import sys from cuda.bindings._path_finder.find_nvidia_dynamic_library import _find_nvidia_dynamic_library from cuda.bindings._path_finder.load_dl_common import ( @@ -12,8 +11,9 @@ load_dependencies, load_in_subprocess, ) +from cuda.bindings._path_finder.supported_libs import IS_WINDOWS -if sys.platform == "win32": +if IS_WINDOWS: from cuda.bindings._path_finder.load_dl_windows import ( check_if_already_loaded_from_elsewhere, load_with_abs_path, diff --git a/cuda_bindings/cuda/bindings/_path_finder/supported_libs.py b/cuda_bindings/cuda/bindings/_path_finder/supported_libs.py index 6852c7fcea..e5d4c28c18 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/supported_libs.py +++ b/cuda_bindings/cuda/bindings/_path_finder/supported_libs.py @@ -5,6 +5,8 @@ import sys +IS_WINDOWS = sys.platform == "win32" + SUPPORTED_LIBNAMES = ( # Core CUDA Runtime and Compiler "nvJitLink", @@ -65,7 +67,7 @@ + PARTIALLY_SUPPORTED_LIBNAMES_WINDOWS_ONLY ) -if sys.platform == "win32": +if IS_WINDOWS: PARTIALLY_SUPPORTED_LIBNAMES = PARTIALLY_SUPPORTED_LIBNAMES_WINDOWS else: PARTIALLY_SUPPORTED_LIBNAMES = PARTIALLY_SUPPORTED_LIBNAMES_LINUX From 27db0a78f839c9a9ad9e3a4456a84b2c862838c1 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 5 May 2025 00:19:30 -0700 Subject: [PATCH 04/24] Use os.path.samefile() to resolve issues with doubled backslashes. --- cuda_bindings/tests/test_path_finder_load.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cuda_bindings/tests/test_path_finder_load.py b/cuda_bindings/tests/test_path_finder_load.py index f665ca0229..8fc117df5b 100644 --- a/cuda_bindings/tests/test_path_finder_load.py +++ b/cuda_bindings/tests/test_path_finder_load.py @@ -48,6 +48,7 @@ def test_find_or_load_nvidia_dynamic_library(info_summary_append, libname): # Defining the subprocess code snippets as strings ensures each subprocess # runs a minimal, independent script tailored to the specific libname and API being tested. code = f"""\ +import os from cuda.bindings.path_finder import _load_nvidia_dynamic_library from cuda.bindings._path_finder.load_nvidia_dynamic_library import _load_nvidia_dynamic_library_no_cache @@ -62,8 +63,8 @@ def test_find_or_load_nvidia_dynamic_library(info_summary_append, libname): loaded_dl_no_cache = _load_nvidia_dynamic_library_no_cache({libname!r}) if not loaded_dl_no_cache.was_already_loaded_from_elsewhere: raise RuntimeError("loaded_dl_no_cache.was_already_loaded_from_elsewhere") -if loaded_dl_no_cache.abs_path != loaded_dl_fresh.abs_path: - raise RuntimeError(f"{{loaded_dl_no_cache.abs_path=!r}} != {{loaded_dl_fresh.abs_path=!r}}") +if not os.path.samefile(loaded_dl_no_cache.abs_path, loaded_dl_fresh.abs_path): + raise RuntimeError(f"not os.path.samefile({{loaded_dl_no_cache.abs_path=!r}}, {{loaded_dl_fresh.abs_path=!r}})") print(f"{{loaded_dl_fresh.abs_path!r}}") """ From 7d8ab70b8fd6e07e39c13f01a046c46ec9553ec1 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 5 May 2025 00:50:21 -0700 Subject: [PATCH 05/24] Change nvJitLink, nvrtc, nvvm bindings to use path_finder --- .github/actions/fetch_ctk/action.yml | 3 +- .../cuda/bindings/_bindings/cynvrtc.pyx.in | 64 +++---------------- .../bindings/_internal/nvjitlink_linux.pyx | 20 ++---- .../bindings/_internal/nvjitlink_windows.pyx | 53 +++------------ .../cuda/bindings/_internal/nvvm_linux.pyx | 18 ++---- .../cuda/bindings/_internal/nvvm_windows.pyx | 61 +++--------------- .../cuda/bindings/_internal/utils.pxd | 3 - .../cuda/bindings/_internal/utils.pyx | 14 ---- 8 files changed, 37 insertions(+), 199 deletions(-) diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml index f6b78701a5..5f37a5f80a 100644 --- a/.github/actions/fetch_ctk/action.yml +++ b/.github/actions/fetch_ctk/action.yml @@ -151,14 +151,13 @@ runs: # mimics actual CTK installation if [[ "${{ inputs.host-platform }}" == linux* ]]; then CUDA_PATH=$(realpath "./cuda_toolkit") - echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:${CUDA_PATH}/lib:${CUDA_PATH}/nvvm/lib64" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:${CUDA_PATH}/lib" >> $GITHUB_ENV elif [[ "${{ inputs.host-platform }}" == win* ]]; then function normpath() { echo "$(echo $(cygpath -w $1) | sed 's/\\/\\\\/g')" } CUDA_PATH=$(normpath $(realpath "./cuda_toolkit")) echo "$(normpath ${CUDA_PATH}/bin)" >> $GITHUB_PATH - echo "$(normpath $CUDA_PATH/nvvm/bin)" >> $GITHUB_PATH fi echo "CUDA_PATH=${CUDA_PATH}" >> $GITHUB_ENV echo "CUDA_HOME=${CUDA_PATH}" >> $GITHUB_ENV diff --git a/cuda_bindings/cuda/bindings/_bindings/cynvrtc.pyx.in b/cuda_bindings/cuda/bindings/_bindings/cynvrtc.pyx.in index d102901cf3..2aed45e648 100644 --- a/cuda_bindings/cuda/bindings/_bindings/cynvrtc.pyx.in +++ b/cuda_bindings/cuda/bindings/_bindings/cynvrtc.pyx.in @@ -9,13 +9,12 @@ # This code was automatically generated with version 12.9.0. Do not modify it directly. {{if 'Windows' == platform.system()}} import os -import site -import struct import win32api -from pywintypes import error {{else}} cimport cuda.bindings._lib.dlfcn as dlfcn +from libc.stdint cimport uintptr_t {{endif}} +from cuda.bindings import path_finder from libc.stdint cimport intptr_t @@ -48,65 +47,18 @@ cdef bint __cuPythonInit = False {{if 'nvrtcSetFlowCallback' in found_functions}}cdef void *__nvrtcSetFlowCallback = NULL{{endif}} cdef int cuPythonInit() except -1 nogil: + {{if 'Windows' != platform.system()}} + cdef void* handle = NULL + {{endif}} + global __cuPythonInit if __cuPythonInit: return 0 __cuPythonInit = True - # Load library - {{if 'Windows' == platform.system()}} - with gil: - # First check if the DLL has been loaded by 3rd parties - try: - handle = win32api.GetModuleHandle("nvrtc64_120_0.dll") - except: - handle = None - - # Check if DLLs can be found within pip installations - if not handle: - LOAD_LIBRARY_SEARCH_DEFAULT_DIRS = 0x00001000 - LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR = 0x00000100 - site_packages = [site.getusersitepackages()] + site.getsitepackages() - for sp in site_packages: - mod_path = os.path.join(sp, "nvidia", "cuda_nvrtc", "bin") - if os.path.isdir(mod_path): - os.add_dll_directory(mod_path) - try: - handle = win32api.LoadLibraryEx( - # Note: LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR needs an abs path... - os.path.join(mod_path, "nvrtc64_120_0.dll"), - 0, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR) - - # Note: nvrtc64_120_0.dll calls into nvrtc-builtins64_*.dll which is - # located in the same mod_path. - # Update PATH environ so that the two dlls can find each other - os.environ["PATH"] = os.pathsep.join((os.environ.get("PATH", ""), mod_path)) - except: - pass - else: - break - else: - # Else try default search - # Only reached if DLL wasn't found in any site-package path - LOAD_LIBRARY_SAFE_CURRENT_DIRS = 0x00002000 - try: - handle = win32api.LoadLibraryEx("nvrtc64_120_0.dll", 0, LOAD_LIBRARY_SAFE_CURRENT_DIRS) - except: - pass - - if not handle: - raise RuntimeError('Failed to LoadLibraryEx nvrtc64_120_0.dll') - {{else}} - handle = dlfcn.dlopen('libnvrtc.so.12', dlfcn.RTLD_NOW) - if handle == NULL: - with gil: - raise RuntimeError('Failed to dlopen libnvrtc.so.12') - {{endif}} - - - # Load function {{if 'Windows' == platform.system()}} with gil: + handle = path_finder._load_nvidia_dynamic_library("nvrtc").handle {{if 'nvrtcGetErrorString' in found_functions}} try: global __nvrtcGetErrorString @@ -291,6 +243,8 @@ cdef int cuPythonInit() except -1 nogil: {{endif}} {{else}} + with gil: + handle = path_finder._load_nvidia_dynamic_library("nvrtc").handle {{if 'nvrtcGetErrorString' in found_functions}} global __nvrtcGetErrorString __nvrtcGetErrorString = dlfcn.dlsym(handle, 'nvrtcGetErrorString') diff --git a/cuda_bindings/cuda/bindings/_internal/nvjitlink_linux.pyx b/cuda_bindings/cuda/bindings/_internal/nvjitlink_linux.pyx index bb61a3e225..36bdcb4f45 100644 --- a/cuda_bindings/cuda/bindings/_internal/nvjitlink_linux.pyx +++ b/cuda_bindings/cuda/bindings/_internal/nvjitlink_linux.pyx @@ -4,12 +4,12 @@ # # This code was automatically generated across versions from 12.0.1 to 12.9.0. Do not modify it directly. -from libc.stdint cimport intptr_t - -from .utils cimport get_nvjitlink_dso_version_suffix +from libc.stdint cimport intptr_t, uintptr_t from .utils import FunctionNotFoundError, NotSupportedError +from cuda.bindings import path_finder + ############################################################################### # Extern ############################################################################### @@ -52,17 +52,9 @@ cdef void* __nvJitLinkGetInfoLog = NULL cdef void* __nvJitLinkVersion = NULL -cdef void* load_library(const int driver_ver) except* with gil: - cdef void* handle - for suffix in get_nvjitlink_dso_version_suffix(driver_ver): - so_name = "libnvJitLink.so" + (f".{suffix}" if suffix else suffix) - handle = dlopen(so_name.encode(), RTLD_NOW | RTLD_GLOBAL) - if handle != NULL: - break - else: - err_msg = dlerror() - raise RuntimeError(f'Failed to dlopen libnvJitLink ({err_msg.decode()})') - return handle +cdef void* load_library(int driver_ver) except* with gil: + cdef uintptr_t handle = path_finder._load_nvidia_dynamic_library("nvJitLink").handle + return handle cdef int _check_or_init_nvjitlink() except -1 nogil: diff --git a/cuda_bindings/cuda/bindings/_internal/nvjitlink_windows.pyx b/cuda_bindings/cuda/bindings/_internal/nvjitlink_windows.pyx index a2f77ca2e2..fb29ac8b70 100644 --- a/cuda_bindings/cuda/bindings/_internal/nvjitlink_windows.pyx +++ b/cuda_bindings/cuda/bindings/_internal/nvjitlink_windows.pyx @@ -6,12 +6,9 @@ from libc.stdint cimport intptr_t -from .utils cimport get_nvjitlink_dso_version_suffix - from .utils import FunctionNotFoundError, NotSupportedError -import os -import site +from cuda.bindings import path_finder import win32api @@ -42,44 +39,9 @@ cdef void* __nvJitLinkGetInfoLog = NULL cdef void* __nvJitLinkVersion = NULL -cdef inline list get_site_packages(): - return [site.getusersitepackages()] + site.getsitepackages() - - -cdef load_library(const int driver_ver): - handle = 0 - - for suffix in get_nvjitlink_dso_version_suffix(driver_ver): - if len(suffix) == 0: - continue - dll_name = f"nvJitLink_{suffix}0_0.dll" - - # First check if the DLL has been loaded by 3rd parties - try: - return win32api.GetModuleHandle(dll_name) - except: - pass - - # Next, check if DLLs are installed via pip - for sp in get_site_packages(): - mod_path = os.path.join(sp, "nvidia", "nvJitLink", "bin") - if os.path.isdir(mod_path): - os.add_dll_directory(mod_path) - try: - return win32api.LoadLibraryEx( - # Note: LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR needs an abs path... - os.path.join(mod_path, dll_name), - 0, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR) - except: - pass - # Finally, try default search - # Only reached if DLL wasn't found in any site-package path - try: - return win32api.LoadLibrary(dll_name) - except: - pass - - raise RuntimeError('Failed to load nvJitLink') +cdef void* load_library(int driver_ver) except* with gil: + cdef intptr_t handle = path_finder._load_nvidia_dynamic_library("nvJitLink").handle + return handle cdef int _check_or_init_nvjitlink() except -1 nogil: @@ -88,15 +50,16 @@ cdef int _check_or_init_nvjitlink() except -1 nogil: return 0 cdef int err, driver_ver + cdef intptr_t handle with gil: # Load driver to check version try: - handle = win32api.LoadLibraryEx("nvcuda.dll", 0, LOAD_LIBRARY_SEARCH_SYSTEM32) + nvcuda_handle = win32api.LoadLibraryEx("nvcuda.dll", 0, LOAD_LIBRARY_SEARCH_SYSTEM32) except Exception as e: raise NotSupportedError(f'CUDA driver is not found ({e})') global __cuDriverGetVersion if __cuDriverGetVersion == NULL: - __cuDriverGetVersion = win32api.GetProcAddress(handle, 'cuDriverGetVersion') + __cuDriverGetVersion = win32api.GetProcAddress(nvcuda_handle, 'cuDriverGetVersion') if __cuDriverGetVersion == NULL: raise RuntimeError('something went wrong') err = (__cuDriverGetVersion)(&driver_ver) @@ -104,7 +67,7 @@ cdef int _check_or_init_nvjitlink() except -1 nogil: raise RuntimeError('something went wrong') # Load library - handle = load_library(driver_ver) + handle = load_library(driver_ver) # Load function global __nvJitLinkCreate diff --git a/cuda_bindings/cuda/bindings/_internal/nvvm_linux.pyx b/cuda_bindings/cuda/bindings/_internal/nvvm_linux.pyx index 53675b094e..8759096a4c 100644 --- a/cuda_bindings/cuda/bindings/_internal/nvvm_linux.pyx +++ b/cuda_bindings/cuda/bindings/_internal/nvvm_linux.pyx @@ -4,12 +4,12 @@ # # This code was automatically generated across versions from 11.0.3 to 12.9.0. Do not modify it directly. -from libc.stdint cimport intptr_t - -from .utils cimport get_nvvm_dso_version_suffix +from libc.stdint cimport intptr_t, uintptr_t from .utils import FunctionNotFoundError, NotSupportedError +from cuda.bindings import path_finder + ############################################################################### # Extern ############################################################################### @@ -51,16 +51,8 @@ cdef void* __nvvmGetProgramLog = NULL cdef void* load_library(const int driver_ver) except* with gil: - cdef void* handle - for suffix in get_nvvm_dso_version_suffix(driver_ver): - so_name = "libnvvm.so" + (f".{suffix}" if suffix else suffix) - handle = dlopen(so_name.encode(), RTLD_NOW | RTLD_GLOBAL) - if handle != NULL: - break - else: - err_msg = dlerror() - raise RuntimeError(f'Failed to dlopen libnvvm ({err_msg.decode()})') - return handle + cdef uintptr_t handle = path_finder._load_nvidia_dynamic_library("nvvm").handle + return handle cdef int _check_or_init_nvvm() except -1 nogil: diff --git a/cuda_bindings/cuda/bindings/_internal/nvvm_windows.pyx b/cuda_bindings/cuda/bindings/_internal/nvvm_windows.pyx index 3f9f54a4db..abf75ba1d4 100644 --- a/cuda_bindings/cuda/bindings/_internal/nvvm_windows.pyx +++ b/cuda_bindings/cuda/bindings/_internal/nvvm_windows.pyx @@ -6,12 +6,9 @@ from libc.stdint cimport intptr_t -from .utils cimport get_nvvm_dso_version_suffix - from .utils import FunctionNotFoundError, NotSupportedError -import os -import site +from cuda.bindings import path_finder import win32api @@ -40,52 +37,9 @@ cdef void* __nvvmGetProgramLogSize = NULL cdef void* __nvvmGetProgramLog = NULL -cdef inline list get_site_packages(): - return [site.getusersitepackages()] + site.getsitepackages() + ["conda"] - - -cdef load_library(const int driver_ver): - handle = 0 - - for suffix in get_nvvm_dso_version_suffix(driver_ver): - if len(suffix) == 0: - continue - dll_name = "nvvm64_40_0.dll" - - # First check if the DLL has been loaded by 3rd parties - try: - return win32api.GetModuleHandle(dll_name) - except: - pass - - # Next, check if DLLs are installed via pip or conda - for sp in get_site_packages(): - if sp == "conda": - # nvvm is not under $CONDA_PREFIX/lib, so it's not in the default search path - conda_prefix = os.environ.get("CONDA_PREFIX") - if conda_prefix is None: - continue - mod_path = os.path.join(conda_prefix, "Library", "nvvm", "bin") - else: - mod_path = os.path.join(sp, "nvidia", "cuda_nvcc", "nvvm", "bin") - if os.path.isdir(mod_path): - os.add_dll_directory(mod_path) - try: - return win32api.LoadLibraryEx( - # Note: LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR needs an abs path... - os.path.join(mod_path, dll_name), - 0, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR) - except: - pass - - # Finally, try default search - # Only reached if DLL wasn't found in any site-package path - try: - return win32api.LoadLibrary(dll_name) - except: - pass - - raise RuntimeError('Failed to load nvvm') +cdef void* load_library(int driver_ver) except* with gil: + cdef intptr_t handle = path_finder._load_nvidia_dynamic_library("nvvm").handle + return handle cdef int _check_or_init_nvvm() except -1 nogil: @@ -94,15 +48,16 @@ cdef int _check_or_init_nvvm() except -1 nogil: return 0 cdef int err, driver_ver + cdef intptr_t handle with gil: # Load driver to check version try: - handle = win32api.LoadLibraryEx("nvcuda.dll", 0, LOAD_LIBRARY_SEARCH_SYSTEM32) + nvcuda_handle = win32api.LoadLibraryEx("nvcuda.dll", 0, LOAD_LIBRARY_SEARCH_SYSTEM32) except Exception as e: raise NotSupportedError(f'CUDA driver is not found ({e})') global __cuDriverGetVersion if __cuDriverGetVersion == NULL: - __cuDriverGetVersion = win32api.GetProcAddress(handle, 'cuDriverGetVersion') + __cuDriverGetVersion = win32api.GetProcAddress(nvcuda_handle, 'cuDriverGetVersion') if __cuDriverGetVersion == NULL: raise RuntimeError('something went wrong') err = (__cuDriverGetVersion)(&driver_ver) @@ -110,7 +65,7 @@ cdef int _check_or_init_nvvm() except -1 nogil: raise RuntimeError('something went wrong') # Load library - handle = load_library(driver_ver) + handle = load_library(driver_ver) # Load function global __nvvmVersion diff --git a/cuda_bindings/cuda/bindings/_internal/utils.pxd b/cuda_bindings/cuda/bindings/_internal/utils.pxd index cac7846ff7..a4b71c5314 100644 --- a/cuda_bindings/cuda/bindings/_internal/utils.pxd +++ b/cuda_bindings/cuda/bindings/_internal/utils.pxd @@ -165,6 +165,3 @@ cdef int get_nested_resource_ptr(nested_resource[ResT] &in_out_ptr, object obj, cdef bint is_nested_sequence(data) cdef void* get_buffer_pointer(buf, Py_ssize_t size, readonly=*) except* - -cdef tuple get_nvjitlink_dso_version_suffix(int driver_ver) -cdef tuple get_nvvm_dso_version_suffix(int driver_ver) diff --git a/cuda_bindings/cuda/bindings/_internal/utils.pyx b/cuda_bindings/cuda/bindings/_internal/utils.pyx index 0a693c052a..7fc77b22c2 100644 --- a/cuda_bindings/cuda/bindings/_internal/utils.pyx +++ b/cuda_bindings/cuda/bindings/_internal/utils.pyx @@ -127,17 +127,3 @@ cdef int get_nested_resource_ptr(nested_resource[ResT] &in_out_ptr, object obj, class FunctionNotFoundError(RuntimeError): pass class NotSupportedError(RuntimeError): pass - - -cdef tuple get_nvjitlink_dso_version_suffix(int driver_ver): - if 12000 <= driver_ver < 13000: - return ('12', '') - raise NotSupportedError(f'CUDA driver version {driver_ver} is not supported') - - -cdef tuple get_nvvm_dso_version_suffix(int driver_ver): - if 11000 <= driver_ver < 11020: - return ('3', '') - if 11020 <= driver_ver < 13000: - return ('4', '') - raise NotSupportedError(f'CUDA driver version {driver_ver} is not supported') From 1f728c0cb4a5a99788e230edb739cc29a12ded7a Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 5 May 2025 11:05:52 -0700 Subject: [PATCH 06/24] `load_in_subprocess(): Pass current environment --- cuda_bindings/cuda/bindings/_path_finder/load_dl_common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cuda_bindings/cuda/bindings/_path_finder/load_dl_common.py b/cuda_bindings/cuda/bindings/_path_finder/load_dl_common.py index 717bdc00c5..268d04295f 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/load_dl_common.py +++ b/cuda_bindings/cuda/bindings/_path_finder/load_dl_common.py @@ -1,6 +1,7 @@ # Copyright 2025 NVIDIA Corporation. All rights reserved. # SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE +import os import subprocess # nosec B404 import sys from dataclasses import dataclass @@ -49,6 +50,7 @@ def load_in_subprocess(python_code, timeout=30): capture_output=True, encoding="utf-8", timeout=timeout, # Ensure this does not hang for an excessive amount of time. + env=os.environ, # Pass current environment ) From 0d23bb697ec353a0dd991ad8a3a66e3612fe1da7 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 5 May 2025 11:53:06 -0700 Subject: [PATCH 07/24] Add run_python_code_safely.py as generated by perplexity, plus ruff format, bandit nosec --- .../_path_finder/run_python_code_safely.py | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 cuda_bindings/cuda/bindings/_path_finder/run_python_code_safely.py diff --git a/cuda_bindings/cuda/bindings/_path_finder/run_python_code_safely.py b/cuda_bindings/cuda/bindings/_path_finder/run_python_code_safely.py new file mode 100644 index 0000000000..6b4dbd1919 --- /dev/null +++ b/cuda_bindings/cuda/bindings/_path_finder/run_python_code_safely.py @@ -0,0 +1,64 @@ +import multiprocessing +import subprocess # nosec B404 +import sys +from io import StringIO + + +def run_python_code_safely(python_code, timeout=None): + """Replacement for subprocess.run that forces 'spawn' context""" + ctx = multiprocessing.get_context("spawn") + result_queue = ctx.Queue() + + def worker(): + # Capture stdout/stderr + old_stdout = sys.stdout + old_stderr = sys.stderr + sys.stdout = StringIO() + sys.stderr = StringIO() + + returncode = 0 + try: + exec(python_code, {"__name__": "__main__"}) # nosec B102 + except SystemExit as e: # Handle sys.exit() + returncode = e.code if isinstance(e.code, int) else 0 + except Exception: # Capture other exceptions + import traceback + + traceback.print_exc() + returncode = 1 + finally: + # Collect outputs and restore streams + stdout = sys.stdout.getvalue() + stderr = sys.stderr.getvalue() + sys.stdout = old_stdout + sys.stderr = old_stderr + result_queue.put((returncode, stdout, stderr)) + + process = ctx.Process(target=worker) + process.start() + + try: + # Wait with timeout support + process.join(timeout) + if process.is_alive(): + process.terminate() + process.join() + raise subprocess.TimeoutExpired([sys.executable, "-c", python_code], timeout) + + # Get results from queue + if result_queue.empty(): + return subprocess.CompletedProcess( + [sys.executable, "-c", python_code], + returncode=-999, + stdout="", + stderr="Process failed to return results", + ) + + returncode, stdout, stderr = result_queue.get() + return subprocess.CompletedProcess( + [sys.executable, "-c", python_code], returncode=returncode, stdout=stdout, stderr=stderr + ) + finally: + # Cleanup if needed + if process.is_alive(): + process.kill() From b1a5e9d8adc20ffabe5d35d7f3d562a7b1f50b71 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 5 May 2025 11:57:07 -0700 Subject: [PATCH 08/24] Replace subprocess.run with run_python_code_safely --- .../cuda/bindings/_path_finder/load_dl_common.py | 12 ++---------- .../bindings/_path_finder/run_python_code_safely.py | 2 +- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/cuda_bindings/cuda/bindings/_path_finder/load_dl_common.py b/cuda_bindings/cuda/bindings/_path_finder/load_dl_common.py index 268d04295f..c5595f1dee 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/load_dl_common.py +++ b/cuda_bindings/cuda/bindings/_path_finder/load_dl_common.py @@ -1,12 +1,10 @@ # Copyright 2025 NVIDIA Corporation. All rights reserved. # SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE -import os -import subprocess # nosec B404 -import sys from dataclasses import dataclass from typing import Callable, Optional +from cuda.bindings._path_finder.run_python_code_safely import run_python_code_safely from cuda.bindings._path_finder.supported_libs import DIRECT_DEPENDENCIES @@ -45,13 +43,7 @@ def load_dependencies(libname: str, load_func: Callable[[str], LoadedDL]) -> Non def load_in_subprocess(python_code, timeout=30): # This is to avoid loading libraries into the parent process. - return subprocess.run( # nosec B603 - [sys.executable, "-c", python_code], - capture_output=True, - encoding="utf-8", - timeout=timeout, # Ensure this does not hang for an excessive amount of time. - env=os.environ, # Pass current environment - ) + return run_python_code_safely(python_code, timeout=timeout) def build_subprocess_failed_for_libname_message(libname, result): diff --git a/cuda_bindings/cuda/bindings/_path_finder/run_python_code_safely.py b/cuda_bindings/cuda/bindings/_path_finder/run_python_code_safely.py index 6b4dbd1919..a1a93a34fc 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/run_python_code_safely.py +++ b/cuda_bindings/cuda/bindings/_path_finder/run_python_code_safely.py @@ -4,7 +4,7 @@ from io import StringIO -def run_python_code_safely(python_code, timeout=None): +def run_python_code_safely(python_code, *, timeout=None): """Replacement for subprocess.run that forces 'spawn' context""" ctx = multiprocessing.get_context("spawn") result_queue = ctx.Queue() From 8e9c7b14e4352517c90dfcf013255f73dfcb0dba Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 5 May 2025 12:31:59 -0700 Subject: [PATCH 09/24] Factor out `class Worker` to fix pickle issue. --- .../_path_finder/run_python_code_safely.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/cuda_bindings/cuda/bindings/_path_finder/run_python_code_safely.py b/cuda_bindings/cuda/bindings/_path_finder/run_python_code_safely.py index a1a93a34fc..9b84885986 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/run_python_code_safely.py +++ b/cuda_bindings/cuda/bindings/_path_finder/run_python_code_safely.py @@ -4,12 +4,12 @@ from io import StringIO -def run_python_code_safely(python_code, *, timeout=None): - """Replacement for subprocess.run that forces 'spawn' context""" - ctx = multiprocessing.get_context("spawn") - result_queue = ctx.Queue() +class Worker: + def __init__(self, python_code, result_queue): + self.python_code = python_code + self.result_queue = result_queue - def worker(): + def __call__(self): # Capture stdout/stderr old_stdout = sys.stdout old_stderr = sys.stderr @@ -18,7 +18,7 @@ def worker(): returncode = 0 try: - exec(python_code, {"__name__": "__main__"}) # nosec B102 + exec(self.python_code, {"__name__": "__main__"}) # nosec B102 except SystemExit as e: # Handle sys.exit() returncode = e.code if isinstance(e.code, int) else 0 except Exception: # Capture other exceptions @@ -32,9 +32,14 @@ def worker(): stderr = sys.stderr.getvalue() sys.stdout = old_stdout sys.stderr = old_stderr - result_queue.put((returncode, stdout, stderr)) + self.result_queue.put((returncode, stdout, stderr)) + - process = ctx.Process(target=worker) +def run_python_code_safely(python_code, *, timeout=None): + """Replacement for subprocess.run that forces 'spawn' context""" + ctx = multiprocessing.get_context("spawn") + result_queue = ctx.Queue() + process = ctx.Process(target=Worker(python_code, result_queue)) process.start() try: From 5977b9d59493edcdbea080247a6ae8e7158f8d54 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 5 May 2025 12:42:57 -0700 Subject: [PATCH 10/24] ChatGPT revisions based on Deep research: https://chatgpt.com/share/681914ce-f274-8008-9e9f-4538716b4ed7 --- .../_path_finder/run_python_code_safely.py | 43 +++++++++++++------ 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/cuda_bindings/cuda/bindings/_path_finder/run_python_code_safely.py b/cuda_bindings/cuda/bindings/_path_finder/run_python_code_safely.py index 9b84885986..e311e14bb7 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/run_python_code_safely.py +++ b/cuda_bindings/cuda/bindings/_path_finder/run_python_code_safely.py @@ -1,6 +1,7 @@ import multiprocessing import subprocess # nosec B404 import sys +import traceback from io import StringIO @@ -16,14 +17,12 @@ def __call__(self): sys.stdout = StringIO() sys.stderr = StringIO() - returncode = 0 try: exec(self.python_code, {"__name__": "__main__"}) # nosec B102 + returncode = 0 except SystemExit as e: # Handle sys.exit() returncode = e.code if isinstance(e.code, int) else 0 - except Exception: # Capture other exceptions - import traceback - + except BaseException: traceback.print_exc() returncode = 1 finally: @@ -32,38 +31,54 @@ def __call__(self): stderr = sys.stderr.getvalue() sys.stdout = old_stdout sys.stderr = old_stderr - self.result_queue.put((returncode, stdout, stderr)) + try: # noqa: SIM105 + self.result_queue.put((returncode, stdout, stderr)) + except Exception: # nosec B110 + # If the queue is broken (e.g., parent gone), best effort logging + pass def run_python_code_safely(python_code, *, timeout=None): - """Replacement for subprocess.run that forces 'spawn' context""" + """Run Python code in a spawned subprocess, capturing stdout/stderr/output.""" ctx = multiprocessing.get_context("spawn") - result_queue = ctx.Queue() + result_queue = ctx.SimpleQueue() process = ctx.Process(target=Worker(python_code, result_queue)) process.start() try: - # Wait with timeout support process.join(timeout) if process.is_alive(): process.terminate() process.join() - raise subprocess.TimeoutExpired([sys.executable, "-c", python_code], timeout) + return subprocess.CompletedProcess( + args=[sys.executable, "-c", python_code], + returncode=-9, + stdout="", + stderr=f"Process timed out after {timeout} seconds and was terminated.", + ) - # Get results from queue if result_queue.empty(): return subprocess.CompletedProcess( - [sys.executable, "-c", python_code], + args=[sys.executable, "-c", python_code], returncode=-999, stdout="", - stderr="Process failed to return results", + stderr="Process exited without returning results.", ) returncode, stdout, stderr = result_queue.get() return subprocess.CompletedProcess( - [sys.executable, "-c", python_code], returncode=returncode, stdout=stdout, stderr=stderr + args=[sys.executable, "-c", python_code], + returncode=returncode, + stdout=stdout, + stderr=stderr, ) + finally: - # Cleanup if needed + try: + result_queue.close() + result_queue.join_thread() + except Exception: # nosec B110 + pass if process.is_alive(): process.kill() + process.join() From 9b474bc80b079f5977712970d63439310a9f3319 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 5 May 2025 13:42:39 -0700 Subject: [PATCH 11/24] Fix race condition in result queue handling by using timeout-based get() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous implementation checked result_queue.empty() before calling get(), which introduces a classic race condition: the queue may become non-empty immediately after the check, resulting in missed results or misleading errors. This patch replaces the empty() check with result_queue.get(timeout=1.0), allowing the parent process to robustly wait for results with a bounded delay. Also switches from ctx.SimpleQueue() to ctx.Queue() for compatibility with timeout-based get(), which SimpleQueue does not support on Python ≤3.12. Note: The race condition was discovered by Gemini 2.5 --- .../bindings/_path_finder/run_python_code_safely.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cuda_bindings/cuda/bindings/_path_finder/run_python_code_safely.py b/cuda_bindings/cuda/bindings/_path_finder/run_python_code_safely.py index e311e14bb7..316cb38858 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/run_python_code_safely.py +++ b/cuda_bindings/cuda/bindings/_path_finder/run_python_code_safely.py @@ -1,4 +1,5 @@ import multiprocessing +import queue # for Empty import subprocess # nosec B404 import sys import traceback @@ -41,7 +42,7 @@ def __call__(self): def run_python_code_safely(python_code, *, timeout=None): """Run Python code in a spawned subprocess, capturing stdout/stderr/output.""" ctx = multiprocessing.get_context("spawn") - result_queue = ctx.SimpleQueue() + result_queue = ctx.Queue() process = ctx.Process(target=Worker(python_code, result_queue)) process.start() @@ -57,15 +58,16 @@ def run_python_code_safely(python_code, *, timeout=None): stderr=f"Process timed out after {timeout} seconds and was terminated.", ) - if result_queue.empty(): + try: + returncode, stdout, stderr = result_queue.get(timeout=1.0) + except (queue.Empty, EOFError): return subprocess.CompletedProcess( args=[sys.executable, "-c", python_code], returncode=-999, stdout="", - stderr="Process exited without returning results.", + stderr="Process exited or crashed before returning results.", ) - returncode, stdout, stderr = result_queue.get() return subprocess.CompletedProcess( args=[sys.executable, "-c", python_code], returncode=returncode, From ab00a872181d0a84cd20bb0459e2fe20b3dbcc37 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 5 May 2025 14:41:05 -0700 Subject: [PATCH 12/24] Resolve SIM108 --- .../find_nvidia_dynamic_library.py | 24 +++++++------------ 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py b/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py index 7df8276c6d..d266559a6e 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py +++ b/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py @@ -18,10 +18,7 @@ def _no_such_file_in_sub_dirs(sub_dirs, file_wild, error_messages, attachments): def _find_so_using_nvidia_lib_dirs(libname, so_basename, error_messages, attachments): - if libname == "nvvm": # noqa: SIM108 - nvidia_sub_dirs = ("nvidia", "*", "nvvm", "lib64") - else: - nvidia_sub_dirs = ("nvidia", "*", "lib") + nvidia_sub_dirs = ("nvidia", "*", "nvvm", "lib64") if libname == "nvvm" else ("nvidia", "*", "lib") file_wild = so_basename + "*" for lib_dir in find_sub_dirs_all_sitepackages(nvidia_sub_dirs): # First look for an exact match @@ -47,10 +44,7 @@ def _find_dll_under_dir(dirpath, file_wild): def _find_dll_using_nvidia_bin_dirs(libname, lib_searched_for, error_messages, attachments): - if libname == "nvvm": # noqa: SIM108 - nvidia_sub_dirs = ("nvidia", "*", "nvvm", "bin") - else: - nvidia_sub_dirs = ("nvidia", "*", "bin") + nvidia_sub_dirs = ("nvidia", "*", "nvvm", "bin") if libname == "nvvm" else ("nvidia", "*", "bin") for bin_dir in find_sub_dirs_all_sitepackages(nvidia_sub_dirs): dll_name = _find_dll_under_dir(bin_dir, lib_searched_for) if dll_name is not None: @@ -71,18 +65,16 @@ def _find_lib_dir_using_cuda_home(libname): if cuda_home is None: return None if IS_WINDOWS: - if libname == "nvvm": # noqa: SIM108 - subdirs = (os.path.join("nvvm", "bin"),) - else: - subdirs = ("bin",) + subdirs = (os.path.join("nvvm", "bin"),) if libname == "nvvm" else ("bin",) else: - if libname == "nvvm": # noqa: SIM108 - subdirs = (os.path.join("nvvm", "lib64"),) - else: - subdirs = ( + subdirs = ( + (os.path.join("nvvm", "lib64"),) + if libname == "nvvm" + else ( "lib64", # CTK "lib", # Conda ) + ) for subdir in subdirs: dirname = os.path.join(cuda_home, subdir) if os.path.isdir(dirname): From 2a039d2a7ec44e1d77b68b7fcc610897fcb4bd4f Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 5 May 2025 15:21:56 -0700 Subject: [PATCH 13/24] Change to "nppc" as ANCHOR_LIBNAME --- .../find_nvidia_dynamic_library.py | 14 +++++++------- .../load_nvidia_dynamic_library.py | 18 ++++++++++-------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py b/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py index d266559a6e..12bed47fb7 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py +++ b/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py @@ -108,14 +108,14 @@ def _find_dll_using_lib_dir(lib_dir, libname, error_messages, attachments): return None -def _find_nvvm_lib_dir_from_other_abs_path(other_abs_path): +def _find_nvvm_lib_dir_from_anchor_abs_path(anchor_abs_path): nvvm_subdir = "bin" if IS_WINDOWS else "lib64" - while other_abs_path: - if os.path.isdir(other_abs_path): - nvvm_lib_dir = os.path.join(other_abs_path, "nvvm", nvvm_subdir) + while anchor_abs_path: + if os.path.isdir(anchor_abs_path): + nvvm_lib_dir = os.path.join(anchor_abs_path, "nvvm", nvvm_subdir) if os.path.isdir(nvvm_lib_dir): return nvvm_lib_dir - other_abs_path = os.path.dirname(other_abs_path) + anchor_abs_path = os.path.dirname(anchor_abs_path) return None @@ -148,9 +148,9 @@ def __init__(self, libname: str): libname, self.lib_searched_for, self.error_messages, self.attachments ) - def retry_with_other_abs_path(self, other_abs_path): + def retry_with_anchor_abs_path(self, anchor_abs_path): assert self.libname == "nvvm" - nvvm_lib_dir = _find_nvvm_lib_dir_from_other_abs_path(other_abs_path) + nvvm_lib_dir = _find_nvvm_lib_dir_from_anchor_abs_path(anchor_abs_path) if nvvm_lib_dir is None: return if IS_WINDOWS: diff --git a/cuda_bindings/cuda/bindings/_path_finder/load_nvidia_dynamic_library.py b/cuda_bindings/cuda/bindings/_path_finder/load_nvidia_dynamic_library.py index b368f5d9ae..e9195e7b55 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/load_nvidia_dynamic_library.py +++ b/cuda_bindings/cuda/bindings/_path_finder/load_nvidia_dynamic_library.py @@ -26,8 +26,11 @@ load_with_system_search, ) +# "nvvm" is found from this anchor +ANCHOR_LIBNAME = "nppc" # libnppc.so.12 1.6M, nppc64_12.dll 288K -def _load_other_in_subprocess(libname, error_messages): + +def _load_anchor_in_subprocess(libname, error_messages): code = f"""\ from cuda.bindings._path_finder.load_nvidia_dynamic_library import load_nvidia_dynamic_library import json @@ -60,14 +63,13 @@ def _load_nvidia_dynamic_library_no_cache(libname: str) -> LoadedDL: if loaded is not None: return loaded if libname == "nvvm": - # Use cudart as anchor point (libcudart.so.12 is only ~720K, cudart64_12.dll ~560K). - loaded_cudart = check_if_already_loaded_from_elsewhere("cudart") - if loaded_cudart is not None: - found.retry_with_other_abs_path(loaded_cudart.abs_path) + loaded_anchor = check_if_already_loaded_from_elsewhere(ANCHOR_LIBNAME) + if loaded_anchor is not None: + found.retry_with_anchor_abs_path(loaded_anchor.abs_path) else: - cudart_abs_path = _load_other_in_subprocess("cudart", found.error_messages) - if cudart_abs_path is not None: - found.retry_with_other_abs_path(cudart_abs_path) + anchor_abs_path = _load_anchor_in_subprocess(ANCHOR_LIBNAME, found.error_messages) + if anchor_abs_path is not None: + found.retry_with_anchor_abs_path(anchor_abs_path) found.raise_if_abs_path_is_None() # Load the library from the found path From f978e67ece5503b814f0b4c5984fc87310343a6f Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 5 May 2025 20:25:52 -0700 Subject: [PATCH 14/24] Implement CUDA_PYTHON_CUDA_HOME_PRIORITY first, last, with default first --- .../find_nvidia_dynamic_library.py | 28 ++++++++++++++++--- .../load_nvidia_dynamic_library.py | 4 ++- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py b/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py index 12bed47fb7..8f7bd9e0c6 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py +++ b/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py @@ -53,15 +53,23 @@ def _find_dll_using_nvidia_bin_dirs(libname, lib_searched_for, error_messages, a return None -def _get_cuda_home(): +def _get_cuda_home(priority): + supported_priorities = ("first", "last") + assert priority in supported_priorities + env_priority = os.environ.get("CUDA_PYTHON_CUDA_HOME_PRIORITY") + if env_priority: + if env_priority not in supported_priorities: + raise RuntimeError(f"Invalid CUDA_PYTHON_CUDA_HOME_PRIORITY {env_priority!r} ({supported_priorities=})") + if priority != env_priority: + return None cuda_home = os.environ.get("CUDA_HOME") if cuda_home is None: cuda_home = os.environ.get("CUDA_PATH") return cuda_home -def _find_lib_dir_using_cuda_home(libname): - cuda_home = _get_cuda_home() +def _find_lib_dir_using_cuda_home(libname, priority): + cuda_home = _get_cuda_home(priority) if cuda_home is None: return None if IS_WINDOWS: @@ -126,7 +134,7 @@ def __init__(self, libname: str): self.attachments = [] self.abs_path = None - cuda_home_lib_dir = _find_lib_dir_using_cuda_home(libname) + cuda_home_lib_dir = _find_lib_dir_using_cuda_home(libname, "first") if IS_WINDOWS: self.lib_searched_for = f"{libname}*.dll" if cuda_home_lib_dir is not None: @@ -148,6 +156,18 @@ def __init__(self, libname: str): libname, self.lib_searched_for, self.error_messages, self.attachments ) + def retry_with_cuda_home_priority_last(self): + cuda_home_lib_dir = _find_lib_dir_using_cuda_home(self.libname, "last") + if cuda_home_lib_dir is not None: + if IS_WINDOWS: + self.abs_path = _find_dll_using_lib_dir( + cuda_home_lib_dir, self.libname, self.error_messages, self.attachments + ) + else: + self.abs_path = _find_so_using_lib_dir( + cuda_home_lib_dir, self.lib_searched_for, self.error_messages, self.attachments + ) + def retry_with_anchor_abs_path(self, anchor_abs_path): assert self.libname == "nvvm" nvvm_lib_dir = _find_nvvm_lib_dir_from_anchor_abs_path(anchor_abs_path) diff --git a/cuda_bindings/cuda/bindings/_path_finder/load_nvidia_dynamic_library.py b/cuda_bindings/cuda/bindings/_path_finder/load_nvidia_dynamic_library.py index e9195e7b55..b9514e068e 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/load_nvidia_dynamic_library.py +++ b/cuda_bindings/cuda/bindings/_path_finder/load_nvidia_dynamic_library.py @@ -62,7 +62,7 @@ def _load_nvidia_dynamic_library_no_cache(libname: str) -> LoadedDL: loaded = load_with_system_search(libname, found.lib_searched_for) if loaded is not None: return loaded - if libname == "nvvm": + if libname == "nvvm" and ANCHOR_LIBNAME is not None: loaded_anchor = check_if_already_loaded_from_elsewhere(ANCHOR_LIBNAME) if loaded_anchor is not None: found.retry_with_anchor_abs_path(loaded_anchor.abs_path) @@ -70,6 +70,8 @@ def _load_nvidia_dynamic_library_no_cache(libname: str) -> LoadedDL: anchor_abs_path = _load_anchor_in_subprocess(ANCHOR_LIBNAME, found.error_messages) if anchor_abs_path is not None: found.retry_with_anchor_abs_path(anchor_abs_path) + if found.abs_path is None: + found.retry_with_cuda_home_priority_last() found.raise_if_abs_path_is_None() # Load the library from the found path From 782fcf6f0ba07ad48f1649907f8e28de7c18c7d6 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 5 May 2025 20:30:46 -0700 Subject: [PATCH 15/24] Remove retry_with_anchor_abs_path() and make retry_with_cuda_home_priority_last() the default. --- .../find_nvidia_dynamic_library.py | 25 +++---------------- .../load_nvidia_dynamic_library.py | 14 +---------- 2 files changed, 5 insertions(+), 34 deletions(-) diff --git a/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py b/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py index 8f7bd9e0c6..f58c314202 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py +++ b/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py @@ -53,23 +53,15 @@ def _find_dll_using_nvidia_bin_dirs(libname, lib_searched_for, error_messages, a return None -def _get_cuda_home(priority): - supported_priorities = ("first", "last") - assert priority in supported_priorities - env_priority = os.environ.get("CUDA_PYTHON_CUDA_HOME_PRIORITY") - if env_priority: - if env_priority not in supported_priorities: - raise RuntimeError(f"Invalid CUDA_PYTHON_CUDA_HOME_PRIORITY {env_priority!r} ({supported_priorities=})") - if priority != env_priority: - return None +def _get_cuda_home(): cuda_home = os.environ.get("CUDA_HOME") if cuda_home is None: cuda_home = os.environ.get("CUDA_PATH") return cuda_home -def _find_lib_dir_using_cuda_home(libname, priority): - cuda_home = _get_cuda_home(priority) +def _find_lib_dir_using_cuda_home(libname): + cuda_home = _get_cuda_home() if cuda_home is None: return None if IS_WINDOWS: @@ -134,30 +126,21 @@ def __init__(self, libname: str): self.attachments = [] self.abs_path = None - cuda_home_lib_dir = _find_lib_dir_using_cuda_home(libname, "first") if IS_WINDOWS: self.lib_searched_for = f"{libname}*.dll" - if cuda_home_lib_dir is not None: - self.abs_path = _find_dll_using_lib_dir( - cuda_home_lib_dir, libname, self.error_messages, self.attachments - ) if self.abs_path is None: self.abs_path = _find_dll_using_nvidia_bin_dirs( libname, self.lib_searched_for, self.error_messages, self.attachments ) else: self.lib_searched_for = f"lib{libname}.so" - if cuda_home_lib_dir is not None: - self.abs_path = _find_so_using_lib_dir( - cuda_home_lib_dir, self.lib_searched_for, self.error_messages, self.attachments - ) if self.abs_path is None: self.abs_path = _find_so_using_nvidia_lib_dirs( libname, self.lib_searched_for, self.error_messages, self.attachments ) def retry_with_cuda_home_priority_last(self): - cuda_home_lib_dir = _find_lib_dir_using_cuda_home(self.libname, "last") + cuda_home_lib_dir = _find_lib_dir_using_cuda_home(self.libname) if cuda_home_lib_dir is not None: if IS_WINDOWS: self.abs_path = _find_dll_using_lib_dir( diff --git a/cuda_bindings/cuda/bindings/_path_finder/load_nvidia_dynamic_library.py b/cuda_bindings/cuda/bindings/_path_finder/load_nvidia_dynamic_library.py index b9514e068e..3587858521 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/load_nvidia_dynamic_library.py +++ b/cuda_bindings/cuda/bindings/_path_finder/load_nvidia_dynamic_library.py @@ -26,9 +26,6 @@ load_with_system_search, ) -# "nvvm" is found from this anchor -ANCHOR_LIBNAME = "nppc" # libnppc.so.12 1.6M, nppc64_12.dll 288K - def _load_anchor_in_subprocess(libname, error_messages): code = f"""\ @@ -62,16 +59,7 @@ def _load_nvidia_dynamic_library_no_cache(libname: str) -> LoadedDL: loaded = load_with_system_search(libname, found.lib_searched_for) if loaded is not None: return loaded - if libname == "nvvm" and ANCHOR_LIBNAME is not None: - loaded_anchor = check_if_already_loaded_from_elsewhere(ANCHOR_LIBNAME) - if loaded_anchor is not None: - found.retry_with_anchor_abs_path(loaded_anchor.abs_path) - else: - anchor_abs_path = _load_anchor_in_subprocess(ANCHOR_LIBNAME, found.error_messages) - if anchor_abs_path is not None: - found.retry_with_anchor_abs_path(anchor_abs_path) - if found.abs_path is None: - found.retry_with_cuda_home_priority_last() + found.retry_with_cuda_home_priority_last() found.raise_if_abs_path_is_None() # Load the library from the found path From ad9e994df879c5a4e68f6ac39554bcb1c6663b01 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 5 May 2025 20:36:18 -0700 Subject: [PATCH 16/24] Restore nvvm-related LD_LIBRARY_PATH, PATH manipulations from main branch. --- .github/actions/fetch_ctk/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml index 5f37a5f80a..f6b78701a5 100644 --- a/.github/actions/fetch_ctk/action.yml +++ b/.github/actions/fetch_ctk/action.yml @@ -151,13 +151,14 @@ runs: # mimics actual CTK installation if [[ "${{ inputs.host-platform }}" == linux* ]]; then CUDA_PATH=$(realpath "./cuda_toolkit") - echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:${CUDA_PATH}/lib" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:${CUDA_PATH}/lib:${CUDA_PATH}/nvvm/lib64" >> $GITHUB_ENV elif [[ "${{ inputs.host-platform }}" == win* ]]; then function normpath() { echo "$(echo $(cygpath -w $1) | sed 's/\\/\\\\/g')" } CUDA_PATH=$(normpath $(realpath "./cuda_toolkit")) echo "$(normpath ${CUDA_PATH}/bin)" >> $GITHUB_PATH + echo "$(normpath $CUDA_PATH/nvvm/bin)" >> $GITHUB_PATH fi echo "CUDA_PATH=${CUDA_PATH}" >> $GITHUB_ENV echo "CUDA_HOME=${CUDA_PATH}" >> $GITHUB_ENV From a1b553b832159dba5c58123edbbf54de767eb6a6 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 5 May 2025 20:40:58 -0700 Subject: [PATCH 17/24] Update README.md to reflect new search priority --- cuda_bindings/cuda/bindings/_path_finder/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cuda_bindings/cuda/bindings/_path_finder/README.md b/cuda_bindings/cuda/bindings/_path_finder/README.md index 5096246b87..fa51b56fa7 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/README.md +++ b/cuda_bindings/cuda/bindings/_path_finder/README.md @@ -24,14 +24,10 @@ strategy for locating NVIDIA shared libraries: The absolute path of the already loaded library will be returned, along with the handle to the library. -1. **Environment variables** - - Relies on `CUDA_HOME` or `CUDA_PATH` environment variables if set - (in that order). - -2. **NVIDIA Python wheels** +1. **NVIDIA Python wheels** - Scans all site-packages to find libraries installed via NVIDIA Python wheels. -3. **OS default mechanisms / Conda environments** +2. **OS default mechanisms / Conda environments** - Falls back to native loader: - `dlopen()` on Linux - `LoadLibraryW()` on Windows @@ -42,6 +38,10 @@ strategy for locating NVIDIA shared libraries: - Linux: Via `$ORIGIN/../lib` on `RPATH` (of the `python` binary) - Windows: Via `%CONDA_PREFIX%\Library\bin` on system `PATH` +3. **Environment variables** + - Relies on `CUDA_HOME` or `CUDA_PATH` environment variables if set + (in that order). + Note that the search is done on a per-library basis. There is no centralized mechanism that ensures all libraries are found in the same way. From 676ecb2d8fe88b78c6002ed4e4f5d8888b97f2ec Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 5 May 2025 20:40:58 -0700 Subject: [PATCH 18/24] Update README.md to reflect new search priority --- cuda_bindings/cuda/bindings/_path_finder/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cuda_bindings/cuda/bindings/_path_finder/README.md b/cuda_bindings/cuda/bindings/_path_finder/README.md index 5096246b87..fa51b56fa7 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/README.md +++ b/cuda_bindings/cuda/bindings/_path_finder/README.md @@ -24,14 +24,10 @@ strategy for locating NVIDIA shared libraries: The absolute path of the already loaded library will be returned, along with the handle to the library. -1. **Environment variables** - - Relies on `CUDA_HOME` or `CUDA_PATH` environment variables if set - (in that order). - -2. **NVIDIA Python wheels** +1. **NVIDIA Python wheels** - Scans all site-packages to find libraries installed via NVIDIA Python wheels. -3. **OS default mechanisms / Conda environments** +2. **OS default mechanisms / Conda environments** - Falls back to native loader: - `dlopen()` on Linux - `LoadLibraryW()` on Windows @@ -42,6 +38,10 @@ strategy for locating NVIDIA shared libraries: - Linux: Via `$ORIGIN/../lib` on `RPATH` (of the `python` binary) - Windows: Via `%CONDA_PREFIX%\Library\bin` on system `PATH` +3. **Environment variables** + - Relies on `CUDA_HOME` or `CUDA_PATH` environment variables if set + (in that order). + Note that the search is done on a per-library basis. There is no centralized mechanism that ensures all libraries are found in the same way. From 73498c06e2176d3cfac0e198fe80e2d85f7ef2a2 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 5 May 2025 21:24:27 -0700 Subject: [PATCH 19/24] SUPPORTED_LINUX_SONAMES does not need updates for CTK 12.9.0 --- cuda_bindings/cuda/bindings/_path_finder/supported_libs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cuda_bindings/cuda/bindings/_path_finder/supported_libs.py b/cuda_bindings/cuda/bindings/_path_finder/supported_libs.py index e5d4c28c18..06023b21bc 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/supported_libs.py +++ b/cuda_bindings/cuda/bindings/_path_finder/supported_libs.py @@ -111,6 +111,7 @@ # cuda_12.5.1_555.42.06_linux.run # cuda_12.6.2_560.35.03_linux.run # cuda_12.8.0_570.86.10_linux.run +# cuda_12.9.0_575.51.03_linux.run # Generated with toolshed/build_path_finder_sonames.py SUPPORTED_LINUX_SONAMES = { "cublas": ( From 7661c13b4feff9157a08de4b7b04d3bc8748950b Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 5 May 2025 21:30:31 -0700 Subject: [PATCH 20/24] The only addition to SUPPORTED_WINDOWS_DLLS for CTK 12.9.0 is nvvm70.dll --- cuda_bindings/cuda/bindings/_path_finder/supported_libs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cuda_bindings/cuda/bindings/_path_finder/supported_libs.py b/cuda_bindings/cuda/bindings/_path_finder/supported_libs.py index 06023b21bc..14dc98a968 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/supported_libs.py +++ b/cuda_bindings/cuda/bindings/_path_finder/supported_libs.py @@ -233,6 +233,7 @@ # cuda_12.5.1_555.85_windows.exe # cuda_12.6.2_560.94_windows.exe # cuda_12.8.1_572.61_windows.exe +# cuda_12.9.0_576.02_windows.txt # Generated with toolshed/build_path_finder_dlls.py (WITH MANUAL EDITS) SUPPORTED_WINDOWS_DLLS = { "cublas": ( @@ -340,6 +341,7 @@ "nvvm64.dll", "nvvm64_33_0.dll", "nvvm64_40_0.dll", + "nvvm70.dll", ), } From ddea021832bb36249abbacf8b0f6a070c6cd7bb1 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 5 May 2025 21:39:11 -0700 Subject: [PATCH 21/24] Make OSError in load_dl_windows.py abs_path_for_dynamic_library() more informative. --- .../cuda/bindings/_path_finder/load_dl_windows.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cuda_bindings/cuda/bindings/_path_finder/load_dl_windows.py b/cuda_bindings/cuda/bindings/_path_finder/load_dl_windows.py index 1f0c9c7e22..7004500dca 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/load_dl_windows.py +++ b/cuda_bindings/cuda/bindings/_path_finder/load_dl_windows.py @@ -35,7 +35,7 @@ def add_dll_directory(dll_abs_path: str) -> None: os.environ["PATH"] = dirpath if curr_path is None else os.pathsep.join((curr_path, dirpath)) -def abs_path_for_dynamic_library(handle: int) -> str: +def abs_path_for_dynamic_library(libname: str, handle: int) -> str: """Get the absolute path of a loaded dynamic library on Windows. Args: @@ -57,7 +57,8 @@ def abs_path_for_dynamic_library(handle: int) -> str: if n_chars == 0: raise OSError( - "GetModuleFileNameW failed. Long paths may require enabling the " + f"GetModuleFileNameW failed ({libname=!r}, {buf_size=}). " + "Long paths may require enabling the " "Windows 10+ long path registry setting. See: " "https://docs.python.org/3/using/windows.html#removing-the-max-path-limitation" ) @@ -99,7 +100,7 @@ def check_if_already_loaded_from_elsewhere(libname: str) -> Optional[LoadedDL]: except pywintypes.error: continue else: - return LoadedDL(handle, abs_path_for_dynamic_library(handle), True) + return LoadedDL(handle, abs_path_for_dynamic_library(libname, handle), True) return None @@ -118,7 +119,7 @@ def load_with_system_search(libname: str, _unused: str) -> Optional[LoadedDL]: for dll_name in SUPPORTED_WINDOWS_DLLS.get(libname, ()): handle = ctypes.windll.kernel32.LoadLibraryW(ctypes.c_wchar_p(dll_name)) if handle: - return LoadedDL(handle, abs_path_for_dynamic_library(handle), False) + return LoadedDL(handle, abs_path_for_dynamic_library(libname, handle), False) return None From 55583d93a8543637cdc04fe238c3f1ceae946c23 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 6 May 2025 01:40:42 -0700 Subject: [PATCH 22/24] run_cuda_bindings_path_finder.py: optionally use args as libnames (to aid debugging) --- toolshed/run_cuda_bindings_path_finder.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/toolshed/run_cuda_bindings_path_finder.py b/toolshed/run_cuda_bindings_path_finder.py index cf173aa415..ca2193a816 100644 --- a/toolshed/run_cuda_bindings_path_finder.py +++ b/toolshed/run_cuda_bindings_path_finder.py @@ -14,9 +14,12 @@ def run(args): - assert len(args) == 0 + if args: + libnames = args + else: + libnames = ALL_LIBNAMES - for libname in ALL_LIBNAMES: + for libname in libnames: print(f"{libname=}") try: loaded_dl = path_finder._load_nvidia_dynamic_library(libname) From a576327216b91ead833a64fcdf278fc6c5cccd6b Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 6 May 2025 02:06:18 -0700 Subject: [PATCH 23/24] Bug fix in load_dl_windows.py: ctypes.windll.kernel32.LoadLibraryW() returns an incompatible `handle`. Use win32api.LoadLibraryEx() instead to ensure self-consistency. --- .../bindings/_path_finder/load_dl_common.py | 22 ++++++++----------- .../bindings/_path_finder/load_dl_windows.py | 9 +++++--- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/cuda_bindings/cuda/bindings/_path_finder/load_dl_common.py b/cuda_bindings/cuda/bindings/_path_finder/load_dl_common.py index c5595f1dee..7d896e10bf 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/load_dl_common.py +++ b/cuda_bindings/cuda/bindings/_path_finder/load_dl_common.py @@ -5,23 +5,19 @@ from typing import Callable, Optional from cuda.bindings._path_finder.run_python_code_safely import run_python_code_safely -from cuda.bindings._path_finder.supported_libs import DIRECT_DEPENDENCIES +from cuda.bindings._path_finder.supported_libs import DIRECT_DEPENDENCIES, IS_WINDOWS +if IS_WINDOWS: + import pywintypes -@dataclass -class LoadedDL: - """Represents a loaded dynamic library. + HandleType = pywintypes.HANDLE +else: + HandleType = int - Attributes: - handle: The library handle (can be converted to void* in Cython) - abs_path: The absolute path to the library file - was_already_loaded_from_elsewhere: Whether the library was already loaded - """ - # ATTENTION: To convert `handle` back to `void*` in cython: - # Linux: `cdef void* ptr = ` - # Windows: `cdef void* ptr = ` - handle: int +@dataclass +class LoadedDL: + handle: HandleType abs_path: Optional[str] was_already_loaded_from_elsewhere: bool diff --git a/cuda_bindings/cuda/bindings/_path_finder/load_dl_windows.py b/cuda_bindings/cuda/bindings/_path_finder/load_dl_windows.py index 7004500dca..ec305be927 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/load_dl_windows.py +++ b/cuda_bindings/cuda/bindings/_path_finder/load_dl_windows.py @@ -35,7 +35,7 @@ def add_dll_directory(dll_abs_path: str) -> None: os.environ["PATH"] = dirpath if curr_path is None else os.pathsep.join((curr_path, dirpath)) -def abs_path_for_dynamic_library(libname: str, handle: int) -> str: +def abs_path_for_dynamic_library(libname: str, handle: pywintypes.HANDLE) -> str: """Get the absolute path of a loaded dynamic library on Windows. Args: @@ -117,8 +117,11 @@ def load_with_system_search(libname: str, _unused: str) -> Optional[LoadedDL]: from cuda.bindings._path_finder.supported_libs import SUPPORTED_WINDOWS_DLLS for dll_name in SUPPORTED_WINDOWS_DLLS.get(libname, ()): - handle = ctypes.windll.kernel32.LoadLibraryW(ctypes.c_wchar_p(dll_name)) - if handle: + try: + handle = win32api.LoadLibraryEx(dll_name, 0, 0) + except pywintypes.error: + continue + else: return LoadedDL(handle, abs_path_for_dynamic_library(libname, handle), False) return None From 5fb2d1fec4932b7b86b4111d87ba73ce8a3c1f71 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 6 May 2025 08:37:11 -0700 Subject: [PATCH 24/24] Remove _find_nvidia_dynamic_library.retry_with_anchor_abs_path() method. Move run_python_code_safely.py to test/ directory. --- .../find_nvidia_dynamic_library.py | 23 ------------------- .../bindings/_path_finder/load_dl_common.py | 14 ----------- .../load_nvidia_dynamic_library.py | 23 +------------------ .../run_python_code_safely.py | 0 cuda_bindings/tests/test_path_finder_load.py | 12 ++++++++-- 5 files changed, 11 insertions(+), 61 deletions(-) rename cuda_bindings/{cuda/bindings/_path_finder => tests}/run_python_code_safely.py (100%) diff --git a/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py b/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py index f58c314202..9835b72d0e 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py +++ b/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_dynamic_library.py @@ -108,17 +108,6 @@ def _find_dll_using_lib_dir(lib_dir, libname, error_messages, attachments): return None -def _find_nvvm_lib_dir_from_anchor_abs_path(anchor_abs_path): - nvvm_subdir = "bin" if IS_WINDOWS else "lib64" - while anchor_abs_path: - if os.path.isdir(anchor_abs_path): - nvvm_lib_dir = os.path.join(anchor_abs_path, "nvvm", nvvm_subdir) - if os.path.isdir(nvvm_lib_dir): - return nvvm_lib_dir - anchor_abs_path = os.path.dirname(anchor_abs_path) - return None - - class _find_nvidia_dynamic_library: def __init__(self, libname: str): self.libname = libname @@ -151,18 +140,6 @@ def retry_with_cuda_home_priority_last(self): cuda_home_lib_dir, self.lib_searched_for, self.error_messages, self.attachments ) - def retry_with_anchor_abs_path(self, anchor_abs_path): - assert self.libname == "nvvm" - nvvm_lib_dir = _find_nvvm_lib_dir_from_anchor_abs_path(anchor_abs_path) - if nvvm_lib_dir is None: - return - if IS_WINDOWS: - self.abs_path = _find_dll_using_lib_dir(nvvm_lib_dir, self.libname, self.error_messages, self.attachments) - else: - self.abs_path = _find_so_using_lib_dir( - nvvm_lib_dir, self.lib_searched_for, self.error_messages, self.attachments - ) - def raise_if_abs_path_is_None(self): if self.abs_path: return self.abs_path diff --git a/cuda_bindings/cuda/bindings/_path_finder/load_dl_common.py b/cuda_bindings/cuda/bindings/_path_finder/load_dl_common.py index 7d896e10bf..034b9d4330 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/load_dl_common.py +++ b/cuda_bindings/cuda/bindings/_path_finder/load_dl_common.py @@ -4,7 +4,6 @@ from dataclasses import dataclass from typing import Callable, Optional -from cuda.bindings._path_finder.run_python_code_safely import run_python_code_safely from cuda.bindings._path_finder.supported_libs import DIRECT_DEPENDENCIES, IS_WINDOWS if IS_WINDOWS: @@ -35,16 +34,3 @@ def load_dependencies(libname: str, load_func: Callable[[str], LoadedDL]) -> Non """ for dep in DIRECT_DEPENDENCIES.get(libname, ()): load_func(dep) - - -def load_in_subprocess(python_code, timeout=30): - # This is to avoid loading libraries into the parent process. - return run_python_code_safely(python_code, timeout=timeout) - - -def build_subprocess_failed_for_libname_message(libname, result): - return ( - f"Subprocess failed for {libname=!r} with exit code {result.returncode}\n" - f"--- stdout-from-subprocess ---\n{result.stdout}\n" - f"--- stderr-from-subprocess ---\n{result.stderr}\n" - ) diff --git a/cuda_bindings/cuda/bindings/_path_finder/load_nvidia_dynamic_library.py b/cuda_bindings/cuda/bindings/_path_finder/load_nvidia_dynamic_library.py index 3587858521..f8fe5ce4a4 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/load_nvidia_dynamic_library.py +++ b/cuda_bindings/cuda/bindings/_path_finder/load_nvidia_dynamic_library.py @@ -2,15 +2,9 @@ # SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE import functools -import json from cuda.bindings._path_finder.find_nvidia_dynamic_library import _find_nvidia_dynamic_library -from cuda.bindings._path_finder.load_dl_common import ( - LoadedDL, - build_subprocess_failed_for_libname_message, - load_dependencies, - load_in_subprocess, -) +from cuda.bindings._path_finder.load_dl_common import LoadedDL, load_dependencies from cuda.bindings._path_finder.supported_libs import IS_WINDOWS if IS_WINDOWS: @@ -27,21 +21,6 @@ ) -def _load_anchor_in_subprocess(libname, error_messages): - code = f"""\ -from cuda.bindings._path_finder.load_nvidia_dynamic_library import load_nvidia_dynamic_library -import json -import sys -loaded = load_nvidia_dynamic_library({libname!r}) -sys.stdout.write(json.dumps(loaded.abs_path, ensure_ascii=True)) -""" - result = load_in_subprocess(code) - if result.returncode == 0: - return json.loads(result.stdout) - error_messages.extend(build_subprocess_failed_for_libname_message(libname, result).splitlines()) - return None - - def _load_nvidia_dynamic_library_no_cache(libname: str) -> LoadedDL: # Check whether the library is already loaded into the current process by # some other component. This check uses OS-level mechanisms (e.g., diff --git a/cuda_bindings/cuda/bindings/_path_finder/run_python_code_safely.py b/cuda_bindings/tests/run_python_code_safely.py similarity index 100% rename from cuda_bindings/cuda/bindings/_path_finder/run_python_code_safely.py rename to cuda_bindings/tests/run_python_code_safely.py diff --git a/cuda_bindings/tests/test_path_finder_load.py b/cuda_bindings/tests/test_path_finder_load.py index 8fc117df5b..5c21e8a058 100644 --- a/cuda_bindings/tests/test_path_finder_load.py +++ b/cuda_bindings/tests/test_path_finder_load.py @@ -5,10 +5,10 @@ import sys import pytest +from run_python_code_safely import run_python_code_safely from cuda.bindings import path_finder from cuda.bindings._path_finder import supported_libs -from cuda.bindings._path_finder.load_dl_common import build_subprocess_failed_for_libname_message, load_in_subprocess ALL_LIBNAMES = path_finder._SUPPORTED_LIBNAMES + supported_libs.PARTIALLY_SUPPORTED_LIBNAMES_ALL ALL_LIBNAMES_LINUX = path_finder._SUPPORTED_LIBNAMES + supported_libs.PARTIALLY_SUPPORTED_LIBNAMES_LINUX @@ -38,6 +38,14 @@ def test_all_libnames_expected_lib_symbols_consistency(): assert tuple(sorted(ALL_LIBNAMES)) == tuple(sorted(supported_libs.EXPECTED_LIB_SYMBOLS.keys())) +def build_subprocess_failed_for_libname_message(libname, result): + return ( + f"Subprocess failed for {libname=!r} with exit code {result.returncode}\n" + f"--- stdout-from-subprocess ---\n{result.stdout}\n" + f"--- stderr-from-subprocess ---\n{result.stderr}\n" + ) + + @pytest.mark.parametrize("libname", TEST_FIND_OR_LOAD_LIBNAMES) def test_find_or_load_nvidia_dynamic_library(info_summary_append, libname): # We intentionally run each dynamic library operation in a subprocess @@ -68,7 +76,7 @@ def test_find_or_load_nvidia_dynamic_library(info_summary_append, libname): print(f"{{loaded_dl_fresh.abs_path!r}}") """ - result = load_in_subprocess(code) + result = run_python_code_safely(code, timeout=30) if result.returncode == 0: info_summary_append(f"abs_path={result.stdout.rstrip()}") else: