Use Legion with CMake's native CUDA language

.github/workflows/ci-gh-docs.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches-ignore:
       - gh-pages  # deployment target branch (this workflow should not exist on that branch anyway)
+      - "pull-request/[0-9]+"
   pull_request:
     branches-ignore:
       - gh-pages  # deployment target branch (this workflow should not exist on that branch anyway)
@@ -44,14 +45,14 @@ jobs:
         env:
           MATRIX_CONTEXT: ${{ toJSON(matrix) }}
         run: echo "$MATRIX_CONTEXT"
-      
+
       ####################################
       # Actual build process starts here #
       ####################################
-      
+
       - name: Checkout
         uses: actions/checkout@v3
-      
+
       - name: Generate YAML file listing dependencies
         run: scripts/generate-conda-envs.py --python 3.10 --ctk 11.8 --os linux --compilers --openmpi
 
@@ -61,7 +62,7 @@ jobs:
       - name: Build doxygen documentation through install.py
         run: |
           conda run -n legate /bin/bash -c "./install.py --docs"
-      
+
       - name: Build documentation using Makefile
         working-directory: ./docs/legate/core
         run: |

cmake/Modules/cuda_arch_helpers.cmake

@@ -17,34 +17,34 @@
 function(set_cuda_arch_from_names)
   set(cuda_archs "")
   # translate legacy arch names into numbers
-  if(CMAKE_CUDA_ARCHITECTURES MATCHES "fermi")
+  if(Legion_CUDA_ARCH MATCHES "fermi")
     list(APPEND cuda_archs 20)
   endif()
-  if(CMAKE_CUDA_ARCHITECTURES MATCHES "kepler")
+  if(Legion_CUDA_ARCH MATCHES "kepler")
     list(APPEND cuda_archs 30)
   endif()
-  if(CMAKE_CUDA_ARCHITECTURES MATCHES "k20")
+  if(Legion_CUDA_ARCH MATCHES "k20")
     list(APPEND cuda_archs 35)
   endif()
-  if(CMAKE_CUDA_ARCHITECTURES MATCHES "k80")
+  if(Legion_CUDA_ARCH MATCHES "k80")
     list(APPEND cuda_archs 37)
   endif()
-  if(CMAKE_CUDA_ARCHITECTURES MATCHES "maxwell")
+  if(Legion_CUDA_ARCH MATCHES "maxwell")
     list(APPEND cuda_archs 52)
   endif()
-  if(CMAKE_CUDA_ARCHITECTURES MATCHES "pascal")
+  if(Legion_CUDA_ARCH MATCHES "pascal")
     list(APPEND cuda_archs 60)
   endif()
-  if(CMAKE_CUDA_ARCHITECTURES MATCHES "volta")
+  if(Legion_CUDA_ARCH MATCHES "volta")
     list(APPEND cuda_archs 70)
   endif()
-  if(CMAKE_CUDA_ARCHITECTURES MATCHES "turing")
+  if(Legion_CUDA_ARCH MATCHES "turing")
     list(APPEND cuda_archs 75)
   endif()
-  if(CMAKE_CUDA_ARCHITECTURES MATCHES "ampere")
+  if(Legion_CUDA_ARCH MATCHES "ampere")
     list(APPEND cuda_archs 80)
   endif()
-  if(CMAKE_CUDA_ARCHITECTURES MATCHES "hopper")
+  if(Legion_CUDA_ARCH MATCHES "hopper")
     list(APPEND cuda_archs 90)
   endif()
 
@@ -60,19 +60,24 @@ function(set_cuda_arch_from_names)
     else()
       list(TRANSFORM cuda_archs APPEND "-real")
     endif()
-    set(CMAKE_CUDA_ARCHITECTURES ${cuda_archs} PARENT_SCOPE)
+    set(Legion_CUDA_ARCH ${cuda_archs} PARENT_SCOPE)
   endif()
 endfunction()
 
-function(add_cuda_architecture_defines defs)
-  message(VERBOSE "legate.core: CMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}")
+function(add_cuda_architecture_defines)
+  set(options )
+  set(oneValueArgs DEFS)
+  set(multiValueArgs ARCHS)
+  cmake_parse_arguments(cuda "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
 
-  set(_defs ${${defs}})
+  message(VERBOSE "legate.core: CUDA_ARCHITECTURES=${cuda_ARCHS}")
+
+  set(_defs ${${cuda_DEFS}})
 
   macro(add_def_if_arch_enabled arch def)
-    if("${arch}" IN_LIST CMAKE_CUDA_ARCHITECTURES OR
-      ("${arch}-real" IN_LIST CMAKE_CUDA_ARCHITECTURES) OR
-      ("${arch}-virtual" IN_LIST CMAKE_CUDA_ARCHITECTURES))
+    if("${arch}" IN_LIST cuda_ARCHS OR
+      ("${arch}-real" IN_LIST cuda_ARCHS) OR
+      ("${arch}-virtual" IN_LIST cuda_ARCHS))
       list(APPEND _defs ${def})
     endif()
   endmacro()
@@ -88,5 +93,5 @@ function(add_cuda_architecture_defines defs)
   add_def_if_arch_enabled("80" "AMPERE_ARCH")
   add_def_if_arch_enabled("90" "HOPPER_ARCH")
 
-  set(${defs} ${_defs} PARENT_SCOPE)
+  set(${cuda_DEFS} ${_defs} PARENT_SCOPE)
 endfunction()

cmake/Modules/legate_core_options.cmake

@@ -87,7 +87,7 @@ option(legate_core_BUILD_DOCS "Build doxygen docs" OFF)
 set_or_default(NCCL_DIR NCCL_PATH)
 set_or_default(Thrust_DIR THRUST_PATH)
 set_or_default(CUDA_TOOLKIT_ROOT_DIR CUDA)
-set_or_default(CMAKE_CUDA_ARCHITECTURES GPU_ARCH NATIVE)
+set_or_default(Legion_CUDA_ARCH GPU_ARCH all-major)
 set_or_default(Legion_HIJACK_CUDART USE_CUDART_HIJACK OFF)
 
 include(CMakeDependentOption)

cmake/thirdparty/get_legion.cmake

@@ -39,13 +39,6 @@ function(find_or_configure_legion)
     set(git_repo "${PKG_REPOSITORY}")
   endif()
 
-  set(Legion_CUDA_ARCH "")
-  if(Legion_USE_CUDA)
-    set(Legion_CUDA_ARCH ${CMAKE_CUDA_ARCHITECTURES})
-    list(TRANSFORM Legion_CUDA_ARCH REPLACE "-real" "")
-    list(TRANSFORM Legion_CUDA_ARCH REPLACE "-virtual" "")
-  endif()
-
   set(FIND_PKG_ARGS
       GLOBAL_TARGETS     Legion::Realm
                          Legion::Regent
@@ -75,10 +68,13 @@ function(find_or_configure_legion)
     include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/cpm_helpers.cmake)
     get_cpm_git_args(legion_cpm_git_args REPOSITORY ${git_repo} BRANCH ${git_branch})
     if(NOT DEFINED Legion_PYTHON_EXTRA_INSTALL_ARGS)
-      set(Legion_PYTHON_EXTRA_INSTALL_ARGS "--single-version-externally-managed --root=/")
+      set(Legion_PYTHON_EXTRA_INSTALL_ARGS "--root / --prefix \"\${CMAKE_INSTALL_PREFIX}\"")
     endif()
 
-    set(_cuda_path "")
+    # Support comma and semicolon delimited lists
+    string(REPLACE "," " " Legion_PYTHON_EXTRA_INSTALL_ARGS "${Legion_PYTHON_EXTRA_INSTALL_ARGS}")
+    string(REPLACE ";" " " Legion_PYTHON_EXTRA_INSTALL_ARGS "${Legion_PYTHON_EXTRA_INSTALL_ARGS}")
+
     set(_legion_cuda_options "")
 
     # Set CMAKE_CXX_STANDARD and CMAKE_CUDA_STANDARD for Legion builds. Legion's FindCUDA.cmake
@@ -95,66 +91,13 @@ function(find_or_configure_legion)
         set(_cuda_std ${_cxx_std})
       endif()
 
-      if(NOT CUDA_NVCC_FLAGS)
-        list(APPEND CUDA_NVCC_FLAGS "${CUDAFLAGS}")
-      endif()
-
-      set(_nvcc_flags ${CUDA_NVCC_FLAGS})
-      if(NOT "${_nvcc_flags}" MATCHES "-std=")
-        list(APPEND _nvcc_flags "-std=c++${_cuda_std}")
-      endif()
-
-      # Get the `stubs/libcuda.so` path so we can set CMAKE_LIBRARY_PATH for FindCUDA.cmake
-      set(_libdir "lib64")
-      if(CMAKE_SIZEOF_VOID_P LESS 8)
-        set(_libdir "lib")
-      endif()
+      list(APPEND _legion_cuda_options "CMAKE_CUDA_STANDARD ${_cuda_std}")
 
-      if(EXISTS "${CUDAToolkit_LIBRARY_DIR}/stubs/libcuda.so")
-        # This might be the path to the `$CONDA_PREFIX/lib`
-        # If it is (and it has the libcuda.so driver stub),
-        # then we know we're using the cuda-toolkit package
-        # and should link to that driver stub instead of the
-        # one potentially in `/usr/local/cuda/lib[64]/stubs`
-        list(APPEND _cuda_stubs "${CUDAToolkit_LIBRARY_DIR}/stubs")
-      elseif(EXISTS "${CUDAToolkit_TARGET_DIR}/${_libdir}/stubs/libcuda.so")
-        # Otherwise assume stubs are relative to the CUDA toolkit root dir
-        list(APPEND _cuda_stubs "${CUDAToolkit_TARGET_DIR}/${_libdir}/stubs")
-      elseif(EXISTS "${CUDAToolkit_LIBRARY_ROOT}/${_libdir}/stubs/libcuda.so")
-        list(APPEND _cuda_stubs "${CUDAToolkit_LIBRARY_ROOT}/${_libdir}/stubs")
-      elseif(DEFINED ENV{CUDA_PATH} AND EXISTS "$ENV{CUDA_PATH}/${_libdir}/stubs/libcuda.so")
-        # Use CUDA_PATH envvar (if set)
-        list(APPEND _cuda_stubs "$ENV{CUDA_PATH}/${_libdir}/stubs/libcuda.so")
-      elseif(DEFINED ENV{CUDA_LIB_PATH} AND EXISTS "$ENV{CUDA_LIB_PATH}/stubs/libcuda.so")
-        # Use CUDA_LIB_PATH envvar (if set)
-        list(APPEND _cuda_stubs "$ENV{CUDA_LIB_PATH}/stubs/libcuda.so")
-      elseif(DEFINED ENV{LIBRARY_PATH} AND
-            ("$ENV{LIBRARY_PATH}" STREQUAL "/usr/local/cuda/${_libdir}/stubs"))
-        # LIBRARY_PATH is set in the `nvidia/cuda` containers to /usr/local/cuda/lib64/stubs
-        list(APPEND _cuda_stubs "$ENV{LIBRARY_PATH}")
+      if(legate_core_STATIC_CUDA_RUNTIME)
+        list(APPEND _legion_cuda_options "CMAKE_CUDA_RUNTIME_LIBRARY STATIC")
       else()
-        message(FATAL_ERROR "Could not find the libcuda.so driver stub. "
-                            "Please reconfigure with -DCUDAToolkit_ROOT= "
-                            "set to a valid CUDA Toolkit installation.")
+        list(APPEND _legion_cuda_options "CMAKE_CUDA_RUNTIME_LIBRARY SHARED")
       endif()
-
-      message(VERBOSE "legate.core: Path(s) to CUDA stubs: ${_cuda_stubs}")
-
-      list(APPEND _legion_cuda_options "CUDA_NVCC_FLAGS ${_nvcc_flags}")
-      list(APPEND _legion_cuda_options "CMAKE_CUDA_STANDARD ${_cuda_std}")
-      # Set this so Legion correctly finds the CUDA toolkit.
-      list(APPEND _legion_cuda_options "CMAKE_LIBRARY_PATH ${_cuda_stubs}")
-
-      # Set these as cache variables for the legacy FindCUDA.cmake
-      set(CUDA_VERBOSE_BUILD ON CACHE BOOL "" FORCE)
-      set(CUDA_USE_STATIC_CUDA_RUNTIME ${legate_core_STATIC_CUDA_RUNTIME} CACHE BOOL "" FORCE)
-
-      # Ensure `${_cuda_stubs}/libcuda.so` doesn't end up in the RPATH of the legion_python binary
-      list(APPEND CMAKE_C_IMPLICIT_LINK_DIRECTORIES "${_cuda_stubs}")
-      list(APPEND CMAKE_CXX_IMPLICIT_LINK_DIRECTORIES "${_cuda_stubs}")
-      list(APPEND CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES "${_cuda_stubs}")
-      set(legate_core_cuda_stubs_path "${_cuda_stubs}" PARENT_SCOPE)
-      set(legate_core_cuda_stubs_path "${_cuda_stubs}" CACHE STRING "" FORCE)
     endif()
 
     # Because legion sets these as cache variables, we need to force set this as a cache variable here
@@ -166,8 +109,6 @@ function(find_or_configure_legion)
     set(Legion_MAX_DIM ${Legion_MAX_DIM} CACHE STRING "The max number of dimensions for Legion" FORCE)
     set(Legion_MAX_FIELDS ${Legion_MAX_FIELDS} CACHE STRING "The max number of fields for Legion" FORCE)
     set(Legion_DEFAULT_LOCAL_FIELDS ${Legion_DEFAULT_LOCAL_FIELDS} CACHE STRING "Number of local fields for Legion" FORCE)
-    set(Legion_CUDA_ARCH ${Legion_CUDA_ARCH} CACHE STRING
-      "Comma-separated list of CUDA architectures to build for (e.g. 60,70)" FORCE)
 
     message(VERBOSE "legate.core: Legion version: ${version}")
     message(VERBOSE "legate.core: Legion git_repo: ${git_repo}")
@@ -189,33 +130,24 @@ function(find_or_configure_legion)
                                  "CMAKE_CXX_STANDARD ${_cxx_std}"
                                  "Legion_VERSION ${version}"
                                  "Legion_BUILD_BINDINGS ON"
-                                 "Legion_BUILD_APPS OFF"
-                                 "Legion_BUILD_TESTS OFF"
-                                 "Legion_BUILD_TUTORIAL OFF"
                                  "Legion_REDOP_HALF ON"
                                  "Legion_REDOP_COMPLEX ON"
-                                 "Legion_GPU_REDUCTIONS OFF"
                                  "Legion_BUILD_RUST_PROFILER ON"
-                                 "Legion_BACKTRACE_USE_LIBDW ${Legion_BACKTRACE_USE_LIBDW}"
-                                 "Legion_SPY ${Legion_SPY}"
-                                 "Legion_USE_LLVM ${Legion_USE_LLVM}"
-                                 "Legion_USE_HDF5 ${Legion_USE_HDF5}"
-                                 "Legion_USE_CUDA ${Legion_USE_CUDA}"
-                                 "Legion_NETWORKS ${Legion_NETWORKS}"
-                                 "Legion_USE_OpenMP ${Legion_USE_OpenMP}"
-                                 "Legion_USE_Python ${Legion_USE_Python}"
-                                 "Legion_BOUNDS_CHECKS ${Legion_BOUNDS_CHECKS}"
     )
   endif()
 
   set(Legion_USE_CUDA ${Legion_USE_CUDA} PARENT_SCOPE)
   set(Legion_USE_OpenMP ${Legion_USE_OpenMP} PARENT_SCOPE)
   set(Legion_USE_Python ${Legion_USE_Python} PARENT_SCOPE)
+  set(Legion_CUDA_ARCH ${Legion_CUDA_ARCH} PARENT_SCOPE)
+  set(Legion_BOUNDS_CHECKS ${Legion_BOUNDS_CHECKS} PARENT_SCOPE)
   set(Legion_NETWORKS ${Legion_NETWORKS} PARENT_SCOPE)
 
   message(VERBOSE "Legion_USE_CUDA=${Legion_USE_CUDA}")
   message(VERBOSE "Legion_USE_OpenMP=${Legion_USE_OpenMP}")
   message(VERBOSE "Legion_USE_Python=${Legion_USE_Python}")
+  message(VERBOSE "Legion_CUDA_ARCH=${Legion_CUDA_ARCH}")
+  message(VERBOSE "Legion_BOUNDS_CHECKS=${Legion_BOUNDS_CHECKS}")
   message(VERBOSE "Legion_NETWORKS=${Legion_NETWORKS}")
 
 endfunction()

cmake/versions.json

@@ -7,7 +7,7 @@
     },
     "Legion": {
       "git_url" : "https://gitlab.com/StanfordLegion/legion.git",
-      "git_tag" : "e770f4b212d3c2634427a408422697a9f47b3d76"
+      "git_tag" : "039d79a60f65307d0533b595bd21f5e5ccc6c44c"
     }
   }
 }

conda/conda-build/build.sh

@@ -1,57 +1,11 @@
-#!/bin/bash
+# Install legate_core C++ libs
+tar -C "$PREFIX" --exclude="*.a" --strip-components=1 -xf /tmp/out/legate_core-*-Linux.tar.gz;
+sed -E -i "s@/home/coder/\.conda/envs/legate@$PREFIX@g" "$PREFIX/share/Legion/cmake/LegionConfigCommon.cmake";
+sed -E -i "s@/home/coder/legate/build/_CPack_Packages/Linux/TGZ/legate_core-(.*)-Linux@$PREFIX@g" "$SP_DIR/legion_canonical_cffi.py";
 
-# Rewrite conda's -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY to
-#                 -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH
-CMAKE_ARGS="$(echo "$CMAKE_ARGS" | sed -r "s@_INCLUDE=ONLY@_INCLUDE=BOTH@g")"
+# Install legate_core Python wheel
+pip install --no-deps --root / --prefix "$PREFIX" /tmp/out/legate.core-*.whl;
 
-# Add our options to conda's CMAKE_ARGS
-CMAKE_ARGS+="
---log-level=VERBOSE
--DBUILD_MARCH=haswell
--DLegion_USE_OpenMP=ON
--DLegion_USE_Python=ON
--DLegion_BUILD_BINDINGS=ON"
-
-# We rely on an environment variable to determine if we need to build cpu-only bits
-if [ -z "$CPU_ONLY" ]; then
-  CMAKE_ARGS+="
--DLegion_USE_CUDA=ON
--DCMAKE_CUDA_ARCHITECTURES:LIST=60-real;70-real;75-real;80-real;90
-"
-fi
-
-# Do not compile with NDEBUG until Legion handles it without warnings
-export CFLAGS="-UNDEBUG"
-export CXXFLAGS="-UNDEBUG"
-export CPPFLAGS="-UNDEBUG"
-export CUDAFLAGS="-UNDEBUG"
-export CMAKE_GENERATOR=Ninja
-export CUDAHOSTCXX=${CXX}
-export OPENSSL_DIR="$CONDA_PREFIX"
-
-echo "Build starting on $(date)"
-
-cmake -S . -B build ${CMAKE_ARGS}
-cmake --build build -j$CPU_COUNT
-cmake --install build --prefix "$PREFIX"
-
-CMAKE_ARGS="
--DFIND_LEGATE_CORE_CPP=ON
--Dlegate_core_ROOT=$PREFIX
-"
-
-SKBUILD_BUILD_OPTIONS=-j$CPU_COUNT \
-$PYTHON -m pip install             \
-  --root /                         \
-  --no-deps                        \
-  --prefix "$PREFIX"               \
-  --no-build-isolation             \
-  --cache-dir "$PIP_CACHE_DIR"     \
-  --disable-pip-version-check      \
-  . -vv
-
-echo "Build ending on $(date)"
-
-# Legion leaves an egg-info file which will confuse conda trying to pick up the information
-# Remove it so the legate-core is the only egg-info file added
-rm -rf $SP_DIR/legion*egg-info
+# Legion leaves .egg-info files, which confuses conda trying to pick up the information
+# Remove them so legate-core is the only egg-info file added.
+rm -rf "$SP_DIR"/legion*egg-info;

conda/conda-build/conda_build_config.yaml

@@ -12,3 +12,9 @@ numpy_version:
 
 cmake_version:
   - ">=3.20.1,!=3.23.0"
+
+numpy:
+  - 1.22
+
+package_version:
+  - "23.09.00"

conda/conda-build/meta.yaml

@@ -98,15 +98,15 @@ requirements:
     - cmake {{ cmake_version }}
     - {{ compiler('c') }} =11.2
     - {{ compiler('cxx') }} =11.2
-    - elfutils # [linux]
-    - libdwarf # [linux]
   host:
     - zlib
     - python
     - cython
     - llvm-openmp
     - scikit-build
     - numpy {{ numpy_version }}
+    - elfutils # [linux]
+    - libdwarf # [linux]
 {% if gpu_enabled_bool %}
     - nccl
     - cuda-nvcc ={{ cuda_version }}
@@ -122,6 +122,8 @@ requirements:
     - llvm-openmp
     - numpy {{ numpy_version }}
     - typing_extensions
+    - elfutils # [linux]
+    - libdwarf # [linux]
 {% if gpu_enabled_bool %}
     - cuda-cudart  >={{ cuda_version }},<{{ cuda_major+1 }}
     - cuda-version >={{ cuda_version }},<{{ cuda_major+1 }}