From ffe2177aeab7becfa66a2c8c9d5e09cfea1b3945 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 12 May 2023 11:47:40 -0700 Subject: [PATCH 01/84] set Legion_CUDA_ARCH instead of CMAKE_CUDA_ARCHITECTURES --- cmake/Modules/cuda_arch_helpers.cmake | 41 +++++++++++-------- cmake/Modules/legate_core_options.cmake | 2 +- conda/conda-build/build.sh | 2 +- install.py | 3 +- legate_core_cpp.cmake | 7 +++- scripts/build-install.sh | 2 +- scripts/build-no-install.sh | 2 +- scripts/build-separately-no-install.sh | 2 +- scripts/build-with-legion-no-install.sh | 2 +- ...build-with-legion-separately-no-install.sh | 2 +- 10 files changed, 38 insertions(+), 27 deletions(-) diff --git a/cmake/Modules/cuda_arch_helpers.cmake b/cmake/Modules/cuda_arch_helpers.cmake index 9a2206f69d..97d0ee8402 100644 --- a/cmake/Modules/cuda_arch_helpers.cmake +++ b/cmake/Modules/cuda_arch_helpers.cmake @@ -17,34 +17,34 @@ function(set_cuda_arch_from_names) set(cuda_archs "") # translate legacy arch names into numbers - if(CMAKE_CUDA_ARCHITECTURES MATCHES "fermi") + if(Legion_CUDA_ARCH MATCHES "fermi") list(APPEND cuda_archs 20) endif() - if(CMAKE_CUDA_ARCHITECTURES MATCHES "kepler") + if(Legion_CUDA_ARCH MATCHES "kepler") list(APPEND cuda_archs 30) endif() - if(CMAKE_CUDA_ARCHITECTURES MATCHES "k20") + if(Legion_CUDA_ARCH MATCHES "k20") list(APPEND cuda_archs 35) endif() - if(CMAKE_CUDA_ARCHITECTURES MATCHES "k80") + if(Legion_CUDA_ARCH MATCHES "k80") list(APPEND cuda_archs 37) endif() - if(CMAKE_CUDA_ARCHITECTURES MATCHES "maxwell") + if(Legion_CUDA_ARCH MATCHES "maxwell") list(APPEND cuda_archs 52) endif() - if(CMAKE_CUDA_ARCHITECTURES MATCHES "pascal") + if(Legion_CUDA_ARCH MATCHES "pascal") list(APPEND cuda_archs 60) endif() - if(CMAKE_CUDA_ARCHITECTURES MATCHES "volta") + if(Legion_CUDA_ARCH MATCHES "volta") list(APPEND cuda_archs 70) endif() - if(CMAKE_CUDA_ARCHITECTURES MATCHES "turing") + if(Legion_CUDA_ARCH MATCHES "turing") list(APPEND cuda_archs 75) endif() - if(CMAKE_CUDA_ARCHITECTURES MATCHES "ampere") + if(Legion_CUDA_ARCH MATCHES "ampere") list(APPEND cuda_archs 80) endif() - if(CMAKE_CUDA_ARCHITECTURES MATCHES "hopper") + if(Legion_CUDA_ARCH MATCHES "hopper") list(APPEND cuda_archs 90) endif() @@ -60,19 +60,24 @@ function(set_cuda_arch_from_names) else() list(TRANSFORM cuda_archs APPEND "-real") endif() - set(CMAKE_CUDA_ARCHITECTURES ${cuda_archs} PARENT_SCOPE) + set(Legion_CUDA_ARCH ${cuda_archs} PARENT_SCOPE) endif() endfunction() -function(add_cuda_architecture_defines defs) - message(VERBOSE "legate.core: CMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}") +function(add_cuda_architecture_defines) + set(options ) + set(oneValueArgs DEFS) + set(multiValueArgs ARCHS) + cmake_parse_arguments(cuda "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - set(_defs ${${defs}}) + message(VERBOSE "legate.core: CUDA_ARCHITECTURES=${cuda_ARCHS}") + + set(_defs ${${cuda_DEFS}}) macro(add_def_if_arch_enabled arch def) - if("${arch}" IN_LIST CMAKE_CUDA_ARCHITECTURES OR - ("${arch}-real" IN_LIST CMAKE_CUDA_ARCHITECTURES) OR - ("${arch}-virtual" IN_LIST CMAKE_CUDA_ARCHITECTURES)) + if("${arch}" IN_LIST cuda_ARCHS OR + ("${arch}-real" IN_LIST cuda_ARCHS) OR + ("${arch}-virtual" IN_LIST cuda_ARCHS)) list(APPEND _defs ${def}) endif() endmacro() @@ -88,5 +93,5 @@ function(add_cuda_architecture_defines defs) add_def_if_arch_enabled("80" "AMPERE_ARCH") add_def_if_arch_enabled("90" "HOPPER_ARCH") - set(${defs} ${_defs} PARENT_SCOPE) + set(${cuda_DEFS} ${_defs} PARENT_SCOPE) endfunction() diff --git a/cmake/Modules/legate_core_options.cmake b/cmake/Modules/legate_core_options.cmake index 62fd1e7592..8e5a0fddc1 100644 --- a/cmake/Modules/legate_core_options.cmake +++ b/cmake/Modules/legate_core_options.cmake @@ -85,7 +85,7 @@ option(legate_core_BUILD_DOCS "Build doxygen docs" OFF) set_or_default(NCCL_DIR NCCL_PATH) set_or_default(Thrust_DIR THRUST_PATH) set_or_default(CUDA_TOOLKIT_ROOT_DIR CUDA) -set_or_default(CMAKE_CUDA_ARCHITECTURES GPU_ARCH NATIVE) +set_or_default(Legion_CUDA_ARCH GPU_ARCH all-major) set_or_default(Legion_HIJACK_CUDART USE_CUDART_HIJACK OFF) include(CMakeDependentOption) diff --git a/conda/conda-build/build.sh b/conda/conda-build/build.sh index 317947dc4d..bcc16acd27 100644 --- a/conda/conda-build/build.sh +++ b/conda/conda-build/build.sh @@ -16,7 +16,7 @@ CMAKE_ARGS+=" if [ -z "$CPU_ONLY" ]; then CMAKE_ARGS+=" -DLegion_USE_CUDA=ON --DCMAKE_CUDA_ARCHITECTURES:LIST=60-real;70-real;75-real;80-real;90 +-DLegion_CUDA_ARCH:LIST=60-real;70-real;75-real;80-real;90 " fi diff --git a/install.py b/install.py index 25bf4e7992..a6e09790a4 100755 --- a/install.py +++ b/install.py @@ -426,7 +426,6 @@ def validate_path(path): )} -DBUILD_SHARED_LIBS=ON -DBUILD_MARCH={march} --DCMAKE_CUDA_ARCHITECTURES={arch} -DLegion_MAX_DIM={str(maxdim)} -DLegion_MAX_FIELDS={str(maxfields)} -DLegion_SPY={("ON" if spy else "OFF")} @@ -445,6 +444,8 @@ def validate_path(path): -DLegion_EMBED_GASNet_CONFIGURE_ARGS="--with-ibv-max-hcas=8" """.splitlines() + if cuda: + cmake_flags += [f"-DLegion_CUDA_ARCH={arch}"] if nccl_dir: cmake_flags += [f"-DNCCL_DIR={nccl_dir}"] if gasnet_dir: diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index 05f6394b40..cac7f3a888 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -155,7 +155,7 @@ if(Legion_USE_CUDA) list(APPEND legate_core_CXX_DEFS LEGATE_USE_CUDA) list(APPEND legate_core_CUDA_DEFS LEGATE_USE_CUDA) - add_cuda_architecture_defines(legate_core_CUDA_DEFS) + add_cuda_architecture_defines(legate_core_CUDA_DEFS ARCHS ${Legion_CUDA_ARCH}) list(APPEND legate_core_CUDA_OPTIONS -Xfatbin=-compress-all) list(APPEND legate_core_CUDA_OPTIONS --expt-extended-lambda) @@ -257,6 +257,10 @@ set_target_properties(legate_core INTERFACE_POSITION_INDEPENDENT_CODE ON LIBRARY_OUTPUT_DIRECTORY lib) +if(Legion_USE_CUDA) + set_property(TARGET legate_core PROPERTY CUDA_ARCHITECTURES ${Legion_CUDA_ARCH}) +endif() + # Add Conda library, and include paths if specified if(TARGET conda_env) target_link_libraries(legate_core PRIVATE conda_env) @@ -474,6 +478,7 @@ endif() "set(Legion_USE_CUDA ${Legion_USE_CUDA})" "set(Legion_USE_OpenMP ${Legion_USE_OpenMP})" "set(Legion_USE_Python ${Legion_USE_Python})" + "set(Legion_CUDA_ARCH ${Legion_CUDA_ARCH})" "set(Legion_NETWORKS ${Legion_NETWORKS})" "set(Legion_BOUNDS_CHECKS ${Legion_BOUNDS_CHECKS})" [=[ diff --git a/scripts/build-install.sh b/scripts/build-install.sh index a4d671cc36..7601ed4797 100755 --- a/scripts/build-install.sh +++ b/scripts/build-install.sh @@ -22,7 +22,7 @@ if [[ -n "$(which ninja)" ]]; then cmake_args+=" -GNinja"; fi cmake_args+=" -D Legion_USE_CUDA=ON -D Legion_USE_OpenMP=ON --D CMAKE_CUDA_ARCHITECTURES=NATIVE +-D Legion_CUDA_ARCH=native "; # Use all but 2 threads to compile diff --git a/scripts/build-no-install.sh b/scripts/build-no-install.sh index 50827ce7ad..37a2a356d8 100755 --- a/scripts/build-no-install.sh +++ b/scripts/build-no-install.sh @@ -20,7 +20,7 @@ if [[ -n "$(which ninja)" ]]; then cmake_args+=" -GNinja"; fi cmake_args+=" -D Legion_USE_CUDA=ON -D Legion_USE_OpenMP=ON --D CMAKE_CUDA_ARCHITECTURES=NATIVE +-D Legion_CUDA_ARCH=native "; # Use all but 2 threads to compile diff --git a/scripts/build-separately-no-install.sh b/scripts/build-separately-no-install.sh index e8d1e64c58..ea6e094a46 100755 --- a/scripts/build-separately-no-install.sh +++ b/scripts/build-separately-no-install.sh @@ -22,7 +22,7 @@ cmake_args+=" -D Legion_USE_OpenMP=ON -D Legion_USE_Python=ON -D Legion_BUILD_BINDINGS=ON --D CMAKE_CUDA_ARCHITECTURES=NATIVE +-D Legion_CUDA_ARCH=native "; # Use all but 2 threads to compile diff --git a/scripts/build-with-legion-no-install.sh b/scripts/build-with-legion-no-install.sh index 2f3a1e3974..27f4160aac 100755 --- a/scripts/build-with-legion-no-install.sh +++ b/scripts/build-with-legion-no-install.sh @@ -35,7 +35,7 @@ if [[ -n "$(which ninja)" ]]; then cmake_args+=" -GNinja"; fi cmake_args+=" -D Legion_USE_CUDA=ON -D Legion_USE_OpenMP=ON --D CMAKE_CUDA_ARCHITECTURES=NATIVE +-D Legion_CUDA_ARCH=native -D Legion_ROOT:STRING=\"$Legion_ROOT\" "; diff --git a/scripts/build-with-legion-separately-no-install.sh b/scripts/build-with-legion-separately-no-install.sh index d15180dc0a..1961fdc625 100755 --- a/scripts/build-with-legion-separately-no-install.sh +++ b/scripts/build-with-legion-separately-no-install.sh @@ -33,7 +33,7 @@ if [[ -n "$(which ninja)" ]]; then cmake_args+=" -GNinja"; fi # Add other build options here as desired cmake_args+=" --D CMAKE_CUDA_ARCHITECTURES=NATIVE +-D Legion_CUDA_ARCH=native -D Legion_ROOT:STRING=\"$Legion_ROOT\" "; From ca1233b95b76dd726171a71b2a4989517e3675d2 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 12 May 2023 11:48:50 -0700 Subject: [PATCH 02/84] define options to enable building tests and examples --- install.py | 12 +++++++++++- legate_core_cpp.cmake | 16 +++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/install.py b/install.py index a6e09790a4..161cc3b6e1 100755 --- a/install.py +++ b/install.py @@ -256,6 +256,7 @@ def install( check_bounds, clean_first, extra_flags, + build_tests, build_examples, editable, build_isolation, @@ -470,8 +471,10 @@ def validate_path(path): cmake_flags += [f"-Dlegate_core_LEGION_BRANCH={legion_branch}"] if build_docs: cmake_flags += ["-Dlegate_core_BUILD_DOCS=ON"] + if build_tests: + cmake_flags += ["-Dlegate_core_BUILD_TESTS=ON"] if build_examples: - cmake_flags += ["-Dlegate_core_EXAMPLE_BUILD_TESTS=ON"] + cmake_flags += ["-Dlegate_core_BUILD_EXAMPLES=ON"] cmake_flags += extra_flags build_flags = [f"-j{str(thread_count)}"] @@ -690,6 +693,13 @@ def driver(): default=[], help="Extra CMake flags.", ) + parser.add_argument( + "--build-tests", + dest="build_tests", + action=BooleanFlag, + default=False, + help="Whether to build the tests", + ) parser.add_argument( "--build-examples", dest="build_examples", diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index cac7f3a888..8e4c370ad1 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -17,6 +17,8 @@ ############################################################################## # - User Options ------------------------------------------------------------ +option(legate_core_BUILD_TESTS OFF) +option(legate_core_BUILD_EXAMPLES OFF) include(cmake/Modules/legate_core_options.cmake) ############################################################################## @@ -516,11 +518,15 @@ rapids_export( FINAL_CODE_BLOCK code_string LANGUAGES ${ENABLED_LANGUAES} ) -option(legate_core_EXAMPLE_BUILD_TESTS OFF) + include(cmake/legate_helper_functions.cmake) -if (legate_core_EXAMPLE_BUILD_TESTS) - set(legate_core_ROOT ${CMAKE_CURRENT_BINARY_DIR}) + +set(legate_core_ROOT ${CMAKE_CURRENT_BINARY_DIR}) + +if(legate_core_BUILD_TESTS) + add_subdirectory(tests/integration) +endif() + +if(legate_core_BUILD_EXAMPLES) add_subdirectory(examples) endif() -set(legate_core_ROOT ${CMAKE_CURRENT_BINARY_DIR}) -add_subdirectory(tests/integration) From 20bc692155b29b8f618a71b8384f4ede7b1c103f Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 12 May 2023 11:49:13 -0700 Subject: [PATCH 03/84] remove the custom linker script --- legate_core_cpp.cmake | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index 8e4c370ad1..a7143a465a 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -302,20 +302,6 @@ target_include_directories(legate_core $ ) -if(Legion_USE_CUDA) - file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/fatbin.ld" - [=[ -SECTIONS -{ -.nvFatBinSegment : { *(.nvFatBinSegment) } -.nv_fatbin : { *(.nv_fatbin) } -} -]=]) - - # ensure CUDA symbols aren't relocated to the middle of the debug build binaries - target_link_options(legate_core PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/fatbin.ld") -endif() - ############################################################################## # - Doxygen target------------------------------------------------------------ From b251da6a373007377ae8bc05681e2ab012db1c8f Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 12 May 2023 11:49:42 -0700 Subject: [PATCH 04/84] update to Legion branch that uses CMake's native CUDA language --- cmake/thirdparty/get_legion.cmake | 82 +++---------------------------- cmake/versions.json | 4 +- install.py | 2 +- legate_core_cpp.cmake | 69 ++++++++------------------ 4 files changed, 31 insertions(+), 126 deletions(-) diff --git a/cmake/thirdparty/get_legion.cmake b/cmake/thirdparty/get_legion.cmake index 71ce33fb1e..8d39f7f26b 100644 --- a/cmake/thirdparty/get_legion.cmake +++ b/cmake/thirdparty/get_legion.cmake @@ -39,13 +39,6 @@ function(find_or_configure_legion) set(git_repo "${PKG_REPOSITORY}") endif() - set(Legion_CUDA_ARCH "") - if(Legion_USE_CUDA) - set(Legion_CUDA_ARCH ${CMAKE_CUDA_ARCHITECTURES}) - list(TRANSFORM Legion_CUDA_ARCH REPLACE "-real" "") - list(TRANSFORM Legion_CUDA_ARCH REPLACE "-virtual" "") - endif() - set(FIND_PKG_ARGS GLOBAL_TARGETS Legion::Realm Legion::Regent @@ -95,66 +88,13 @@ function(find_or_configure_legion) set(_cuda_std ${_cxx_std}) endif() - if(NOT CUDA_NVCC_FLAGS) - list(APPEND CUDA_NVCC_FLAGS "${CUDAFLAGS}") - endif() - - set(_nvcc_flags ${CUDA_NVCC_FLAGS}) - if(NOT "${_nvcc_flags}" MATCHES "-std=") - list(APPEND _nvcc_flags "-std=c++${_cuda_std}") - endif() - - # Get the `stubs/libcuda.so` path so we can set CMAKE_LIBRARY_PATH for FindCUDA.cmake - set(_libdir "lib64") - if(CMAKE_SIZEOF_VOID_P LESS 8) - set(_libdir "lib") - endif() + list(APPEND _legion_cuda_options "CMAKE_CUDA_STANDARD ${_cuda_std}") - if(EXISTS "${CUDAToolkit_LIBRARY_DIR}/stubs/libcuda.so") - # This might be the path to the `$CONDA_PREFIX/lib` - # If it is (and it has the libcuda.so driver stub), - # then we know we're using the cuda-toolkit package - # and should link to that driver stub instead of the - # one potentially in `/usr/local/cuda/lib[64]/stubs` - list(APPEND _cuda_stubs "${CUDAToolkit_LIBRARY_DIR}/stubs") - elseif(EXISTS "${CUDAToolkit_TARGET_DIR}/${_libdir}/stubs/libcuda.so") - # Otherwise assume stubs are relative to the CUDA toolkit root dir - list(APPEND _cuda_stubs "${CUDAToolkit_TARGET_DIR}/${_libdir}/stubs") - elseif(EXISTS "${CUDAToolkit_LIBRARY_ROOT}/${_libdir}/stubs/libcuda.so") - list(APPEND _cuda_stubs "${CUDAToolkit_LIBRARY_ROOT}/${_libdir}/stubs") - elseif(DEFINED ENV{CUDA_PATH} AND EXISTS "$ENV{CUDA_PATH}/${_libdir}/stubs/libcuda.so") - # Use CUDA_PATH envvar (if set) - list(APPEND _cuda_stubs "$ENV{CUDA_PATH}/${_libdir}/stubs/libcuda.so") - elseif(DEFINED ENV{CUDA_LIB_PATH} AND EXISTS "$ENV{CUDA_LIB_PATH}/stubs/libcuda.so") - # Use CUDA_LIB_PATH envvar (if set) - list(APPEND _cuda_stubs "$ENV{CUDA_LIB_PATH}/stubs/libcuda.so") - elseif(DEFINED ENV{LIBRARY_PATH} AND - ("$ENV{LIBRARY_PATH}" STREQUAL "/usr/local/cuda/${_libdir}/stubs")) - # LIBRARY_PATH is set in the `nvidia/cuda` containers to /usr/local/cuda/lib64/stubs - list(APPEND _cuda_stubs "$ENV{LIBRARY_PATH}") + if(legate_core_STATIC_CUDA_RUNTIME) + list(APPEND _legion_cuda_options "CMAKE_CUDA_RUNTIME_LIBRARY STATIC") else() - message(FATAL_ERROR "Could not find the libcuda.so driver stub. " - "Please reconfigure with -DCUDAToolkit_ROOT= " - "set to a valid CUDA Toolkit installation.") + list(APPEND _legion_cuda_options "CMAKE_CUDA_RUNTIME_LIBRARY SHARED") endif() - - message(VERBOSE "legate.core: Path(s) to CUDA stubs: ${_cuda_stubs}") - - list(APPEND _legion_cuda_options "CUDA_NVCC_FLAGS ${_nvcc_flags}") - list(APPEND _legion_cuda_options "CMAKE_CUDA_STANDARD ${_cuda_std}") - # Set this so Legion correctly finds the CUDA toolkit. - list(APPEND _legion_cuda_options "CMAKE_LIBRARY_PATH ${_cuda_stubs}") - - # Set these as cache variables for the legacy FindCUDA.cmake - set(CUDA_VERBOSE_BUILD ON CACHE BOOL "" FORCE) - set(CUDA_USE_STATIC_CUDA_RUNTIME ${legate_core_STATIC_CUDA_RUNTIME} CACHE BOOL "" FORCE) - - # Ensure `${_cuda_stubs}/libcuda.so` doesn't end up in the RPATH of the legion_python binary - list(APPEND CMAKE_C_IMPLICIT_LINK_DIRECTORIES "${_cuda_stubs}") - list(APPEND CMAKE_CXX_IMPLICIT_LINK_DIRECTORIES "${_cuda_stubs}") - list(APPEND CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES "${_cuda_stubs}") - set(legate_core_cuda_stubs_path "${_cuda_stubs}" PARENT_SCOPE) - set(legate_core_cuda_stubs_path "${_cuda_stubs}" CACHE STRING "" FORCE) endif() # Because legion sets these as cache variables, we need to force set this as a cache variable here @@ -165,8 +105,6 @@ function(find_or_configure_legion) # https://discourse.cmake.org/t/fetchcontent-cache-variables/1538/8 set(Legion_MAX_DIM ${Legion_MAX_DIM} CACHE STRING "The max number of dimensions for Legion" FORCE) set(Legion_MAX_FIELDS ${Legion_MAX_FIELDS} CACHE STRING "The max number of fields for Legion" FORCE) - set(Legion_CUDA_ARCH ${Legion_CUDA_ARCH} CACHE STRING - "Comma-separated list of CUDA architectures to build for (e.g. 60,70)" FORCE) message(VERBOSE "legate.core: Legion version: ${version}") message(VERBOSE "legate.core: Legion git_repo: ${git_repo}") @@ -189,25 +127,21 @@ function(find_or_configure_legion) "Legion_REDOP_COMPLEX ON" "Legion_GPU_REDUCTIONS OFF" "Legion_BUILD_RUST_PROFILER ON" - "Legion_SPY ${Legion_SPY}" - "Legion_USE_LLVM ${Legion_USE_LLVM}" - "Legion_USE_HDF5 ${Legion_USE_HDF5}" - "Legion_USE_CUDA ${Legion_USE_CUDA}" - "Legion_NETWORKS ${Legion_NETWORKS}" - "Legion_USE_OpenMP ${Legion_USE_OpenMP}" - "Legion_USE_Python ${Legion_USE_Python}" - "Legion_BOUNDS_CHECKS ${Legion_BOUNDS_CHECKS}" ) endif() set(Legion_USE_CUDA ${Legion_USE_CUDA} PARENT_SCOPE) set(Legion_USE_OpenMP ${Legion_USE_OpenMP} PARENT_SCOPE) set(Legion_USE_Python ${Legion_USE_Python} PARENT_SCOPE) + set(Legion_CUDA_ARCH ${Legion_CUDA_ARCH} PARENT_SCOPE) + set(Legion_BOUNDS_CHECKS ${Legion_BOUNDS_CHECKS} PARENT_SCOPE) set(Legion_NETWORKS ${Legion_NETWORKS} PARENT_SCOPE) message(VERBOSE "Legion_USE_CUDA=${Legion_USE_CUDA}") message(VERBOSE "Legion_USE_OpenMP=${Legion_USE_OpenMP}") message(VERBOSE "Legion_USE_Python=${Legion_USE_Python}") + message(VERBOSE "Legion_CUDA_ARCH=${Legion_CUDA_ARCH}") + message(VERBOSE "Legion_BOUNDS_CHECKS=${Legion_BOUNDS_CHECKS}") message(VERBOSE "Legion_NETWORKS=${Legion_NETWORKS}") endfunction() diff --git a/cmake/versions.json b/cmake/versions.json index 67044502d5..d4fa5ef5c2 100644 --- a/cmake/versions.json +++ b/cmake/versions.json @@ -6,8 +6,8 @@ "git_tag" : "1.17.0" }, "Legion": { - "git_url" : "https://gitlab.com/StanfordLegion/legion.git", - "git_tag" : "915f481e5daa9485344866062441d4ce4d7fb365" + "git_url" : "https://gitlab.com/inlineptx/legion.git", + "git_tag" : "0de432af68dc8a8274dd5412945e946cea08f3bc" } } } diff --git a/install.py b/install.py index 161cc3b6e1..7b301f74c3 100755 --- a/install.py +++ b/install.py @@ -590,7 +590,7 @@ def driver(): dest="arch", action="store", required=False, - default="NATIVE", + default="all-major", help="Specify the target GPU architecture.", ) parser.add_argument( diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index a7143a465a..cbd922df4f 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -36,10 +36,6 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # Set a default build type if none was specified rapids_cmake_build_type(Release) -# ################################################################################################## -# * conda environment ----------------------------------------------------------------------------- -rapids_cmake_support_conda_env(conda_env MODIFY_PREFIX_PATH) - ############################################################################## # - Dependencies ------------------------------------------------------------- @@ -56,41 +52,6 @@ macro(_find_package_Python3) message(VERBOSE "legate.core: Python 3 version: ${Python3_VERSION}") endmacro() -# CUDA initialization might need to happen at different times depending on -# how Legion is built. If building Legion inline, CUDA must be enabled -# BEFORE get_legion.cmake because of how Legion handles CMAKE_CUDA_ARCHITECURES. -# If using an external Legion, CUDA must be enabled AFTER get_legion.cmake. -# This function executes all the enable CUDA functions with a boolean guard -# to make sure it is only executed once. -macro(_enable_cuda_language) - if (NOT legate_core_CUDA_ENABLED) - include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/cuda_arch_helpers.cmake) - # Needs to run before `rapids_cuda_init_architectures` - set_cuda_arch_from_names() - # Must come before `enable_language(CUDA)` - rapids_cuda_init_architectures(legate_core) - # Enable the CUDA language - enable_language(CUDA) - # Since legate_core only enables CUDA optionally we need to manually include - # the file that rapids_cuda_init_architectures relies on `project` calling - if(CMAKE_PROJECT_legate_core_INCLUDE) - include("${CMAKE_PROJECT_legate_core_INCLUDE}") - endif() - # Must come after `enable_language(CUDA)` - # Use `-isystem ` instead of `-isystem=` - # because the former works with clangd intellisense - set(CMAKE_INCLUDE_SYSTEM_FLAG_CUDA "-isystem ") - # set to TRUE so the macro does not repeat if called again. - set(legate_core_CUDA_ENABLED TRUE) - # Find the CUDAToolkit - rapids_find_package( - CUDAToolkit REQUIRED - BUILD_EXPORT_SET legate-core-exports - INSTALL_EXPORT_SET legate-core-exports - ) - endif() -endmacro() - if(Legion_USE_Python) _find_package_Python3() if(Python3_FOUND AND Python3_VERSION) @@ -98,8 +59,11 @@ if(Legion_USE_Python) endif() endif() +include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/cuda_arch_helpers.cmake) + if(Legion_USE_CUDA) - _enable_cuda_language() + # Needs to run before find_package(Legion) + set_cuda_arch_from_names() endif() ### @@ -123,8 +87,18 @@ if(Legion_NETWORKS) endif() if(Legion_USE_CUDA) - # If CUDA has not yet been enabled, make sure it is now. - _enable_cuda_language() + # Enable the CUDA language + enable_language(CUDA) + # Must come after `enable_language(CUDA)` + # Use `-isystem ` instead of `-isystem=` + # because the former works with clangd intellisense + set(CMAKE_INCLUDE_SYSTEM_FLAG_CUDA "-isystem ") + # Find the CUDAToolkit + rapids_find_package( + CUDAToolkit REQUIRED + BUILD_EXPORT_SET legate-core-exports + INSTALL_EXPORT_SET legate-core-exports + ) # Find NCCL include(cmake/thirdparty/get_nccl.cmake) endif() @@ -263,6 +237,10 @@ if(Legion_USE_CUDA) set_property(TARGET legate_core PROPERTY CUDA_ARCHITECTURES ${Legion_CUDA_ARCH}) endif() +# ################################################################################################## +# * conda environment ----------------------------------------------------------------------------- +rapids_cmake_support_conda_env(conda_env) + # Add Conda library, and include paths if specified if(TARGET conda_env) target_link_libraries(legate_core PRIVATE conda_env) @@ -477,13 +455,6 @@ endif() "${helper_functions}" ) -if(DEFINED legate_core_cuda_stubs_path) - string(JOIN "\n" code_string "${code_string}" - "list(APPEND CMAKE_C_IMPLICIT_LINK_DIRECTORIES ${legate_core_cuda_stubs_path})" - "list(APPEND CMAKE_CXX_IMPLICIT_LINK_DIRECTORIES ${legate_core_cuda_stubs_path})" - "list(APPEND CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES ${legate_core_cuda_stubs_path})") -endif() - rapids_export( INSTALL legate_core EXPORT_SET legate-core-exports From be8404f4e0dd2b25439ee27a768784ef9f3f117f Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 30 May 2023 12:05:03 -0700 Subject: [PATCH 05/84] set -DLegion_CUDA_ARCH=all-major in build-legate-cpp --- continuous_integration/home/coder/.local/bin/build-legate-cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous_integration/home/coder/.local/bin/build-legate-cpp b/continuous_integration/home/coder/.local/bin/build-legate-cpp index f7516253a0..fbbc33e212 100755 --- a/continuous_integration/home/coder/.local/bin/build-legate-cpp +++ b/continuous_integration/home/coder/.local/bin/build-legate-cpp @@ -8,7 +8,7 @@ build_legate_cpp() { cmake_args+=(-DBUILD_SHARED_LIBS=ON); cmake_args+=(-DBUILD_MARCH=${BUILD_MARCH:-haswell}); cmake_args+=(-DCMAKE_BUILD_TYPE=Release); - cmake_args+=(-DCMAKE_CUDA_ARCHITECTURES=RAPIDS); + cmake_args+=(-DLegion_CUDA_ARCH=all-major); cmake_args+=(-DLegion_NETWORKS=); cmake_args+=(-DLegion_USE_Python=ON); cmake_args+=(-DLegion_REDOP_HALF=ON); From 15514dbc2e31f75071fc1a3d07ee6442f38f007a Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 1 Jun 2023 17:04:35 -0700 Subject: [PATCH 06/84] * upload /tmp/conda-build dir * run on pull_request events if PR is to a fork * use rapids-sccache-devs when building on a fork * test prerelease devcontainer --- .github/workflows/ci-gh1.yml | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-gh1.yml b/.github/workflows/ci-gh1.yml index 6d263473d8..8521334e55 100644 --- a/.github/workflows/ci-gh1.yml +++ b/.github/workflows/ci-gh1.yml @@ -5,13 +5,31 @@ concurrency: cancel-in-progress: true on: + pull_request: push: branches: - "pull-request/[0-9]+" - "branch-*" jobs: + check-event: + name: Check PR Event + runs-on: ubuntu-latest + outputs: + ok: ${{ steps.check_pr_event.outputs.ok }} + steps: + - id: check_pr_event + name: Check PR Event + shell: bash + run: | + [[ '${{ github.event_name }}' == 'push' && '${{ github.repository }}' == 'nv-legate/legate.core' ]] || \ + [[ '${{ github.event_name }}' == 'pull_request' && '${{ github.repository }}' != 'nv-legate/legate.core' ]] \ + && echo "ok=true" | tee -a $GITHUB_OUTPUT \ + || echo "ok=false" | tee -a $GITHUB_OUTPUT; + build: + needs: check-event + if: needs.check-event.outputs.ok == 'true' permissions: id-token: write # This is required for configure-aws-credentials contents: read # This is required for actions/checkout @@ -20,14 +38,14 @@ jobs: runs-on: ${{ github.repository == 'nv-legate/legate.core' && 'linux-amd64-cpu4' || 'ubuntu-latest' }} container: options: -u root - image: rapidsai/devcontainers:23.06-cpp-cuda11.8-mambaforge-ubuntu22.04 + image: pauletaylor/devcontainers:23.06-cpp-cuda11.8-mambaforge-ubuntu22.04 volumes: - ${{ github.workspace }}/out:/tmp/out env: DEFAULT_CONDA_ENV: legate PYTHONDONTWRITEBYTECODE: 1 SCCACHE_REGION: us-east-2 - SCCACHE_BUCKET: rapids-sccache-east + SCCACHE_BUCKET: ${{ secrets.PERSONAL_ACCESS_TOKEN && 'rapids-sccache-devs' || 'rapids-sccache-east' }} SCCACHE_S3_KEY_PREFIX: legate-cunumeric-dev GH_TOKEN: "${{ secrets.PERSONAL_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}" GITHUB_TOKEN: "${{ secrets.PERSONAL_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}" @@ -80,4 +98,5 @@ jobs: uses: actions/upload-artifact@v3 with: name: "legate.core-${{ github.sha }}" - path: ./out/* + path: /tmp/conda-build + if-no-files-found: warn From bd3923177e3be9d403cc880f5ff50bcf04947e49 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 1 Jun 2023 18:26:51 -0700 Subject: [PATCH 07/84] switch image back --- .github/workflows/ci-gh1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-gh1.yml b/.github/workflows/ci-gh1.yml index 8521334e55..0e7f3c50e0 100644 --- a/.github/workflows/ci-gh1.yml +++ b/.github/workflows/ci-gh1.yml @@ -38,7 +38,7 @@ jobs: runs-on: ${{ github.repository == 'nv-legate/legate.core' && 'linux-amd64-cpu4' || 'ubuntu-latest' }} container: options: -u root - image: pauletaylor/devcontainers:23.06-cpp-cuda11.8-mambaforge-ubuntu22.04 + image: rapidsai/devcontainers:23.06-cpp-cuda11.8-mambaforge-ubuntu22.04 volumes: - ${{ github.workspace }}/out:/tmp/out env: From 47f279426413d460583375f1c9e3d8f1163802bd Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 7 Jun 2023 09:30:11 -0700 Subject: [PATCH 08/84] bump legion commit sha --- cmake/versions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/versions.json b/cmake/versions.json index d4fa5ef5c2..82625f4e0c 100644 --- a/cmake/versions.json +++ b/cmake/versions.json @@ -7,7 +7,7 @@ }, "Legion": { "git_url" : "https://gitlab.com/inlineptx/legion.git", - "git_tag" : "0de432af68dc8a8274dd5412945e946cea08f3bc" + "git_tag" : "67529d05d26d17560fb5957a7424d8ae94eab40e" } } } From c59f7c5bd3ef66623d08a05a01775ce835996be7 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 23 Jun 2023 18:40:10 -0700 Subject: [PATCH 09/84] update legion commit sha --- cmake/versions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/versions.json b/cmake/versions.json index 82625f4e0c..fea9cd9154 100644 --- a/cmake/versions.json +++ b/cmake/versions.json @@ -7,7 +7,7 @@ }, "Legion": { "git_url" : "https://gitlab.com/inlineptx/legion.git", - "git_tag" : "67529d05d26d17560fb5957a7424d8ae94eab40e" + "git_tag" : "e05c36c7867641a4c2e8e167c078e459ac83d596" } } } From a9cd5c44e0b1d9c1ea2ba64adb1351a02cfe3b9e Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 23 Jun 2023 19:56:11 -0700 Subject: [PATCH 10/84] fix path to type_traits.h --- legate_core_cpp.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index 360540b2f4..b1421a7619 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -290,6 +290,7 @@ if (legate_core_BUILD_DOCS) list(APPEND legate_core_DOC_SOURCES # type src/core/type/type_info.h + src/core/type/type_traits.h # task src/core/task/task.h src/core/task/registrar.h @@ -314,7 +315,6 @@ if (legate_core_BUILD_DOCS) # utilities src/core/utilities/debug.h src/core/utilities/dispatch.h - src/core/utilities/type_traits.h # main page src/legate.h ) From 06848de9f3fbb93aa981ad3103d19890798e9bf1 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 23 Jun 2023 19:56:24 -0700 Subject: [PATCH 11/84] install libssl-dev for rust --- .github/workflows/gh-build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index b40d17cecd..d8dbfc5179 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -49,6 +49,8 @@ jobs: - name: Setup shell: bash -eo pipefail {0} run: | + apt update && apt install -y --no-install-recommends libssl-dev; + cp -ar legate/continuous_integration/home/coder/.gitconfig /home/coder/; cp -ar legate/continuous_integration/home/coder/.local /home/coder/; mv legate /home/coder/legate; From c05a24561422f292edf09b4c2d7cca14b5d27fb9 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 23 Jun 2023 20:26:10 -0700 Subject: [PATCH 12/84] set OPENSSL_DIR to CONDA_PREFIX --- .github/workflows/gh-build.yml | 2 -- .../home/coder/.local/bin/build-legate-cpp | 6 ++++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index d8dbfc5179..b40d17cecd 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -49,8 +49,6 @@ jobs: - name: Setup shell: bash -eo pipefail {0} run: | - apt update && apt install -y --no-install-recommends libssl-dev; - cp -ar legate/continuous_integration/home/coder/.gitconfig /home/coder/; cp -ar legate/continuous_integration/home/coder/.local /home/coder/; mv legate /home/coder/legate; diff --git a/continuous_integration/home/coder/.local/bin/build-legate-cpp b/continuous_integration/home/coder/.local/bin/build-legate-cpp index fbbc33e212..51f05ed855 100755 --- a/continuous_integration/home/coder/.local/bin/build-legate-cpp +++ b/continuous_integration/home/coder/.local/bin/build-legate-cpp @@ -31,7 +31,8 @@ build_legate_cpp() { sccache --stop-server >/dev/null 2>&1 || true; sccache --show-stats; - time CMAKE_BUILD_PARALLEL_LEVEL=${JOBS:-$(nproc --ignore=1)} \ + time OPENSSL_DIR="${CONDA_PREFIX}" \ + CMAKE_BUILD_PARALLEL_LEVEL=${JOBS:-$(nproc --ignore=1)} \ cmake -S ~/legate -B ~/legate/build "${cmake_args[@]}" -GNinja; sed -i \ @@ -40,7 +41,8 @@ build_legate_cpp() { sccache --show-stats; - time CMAKE_BUILD_PARALLEL_LEVEL=${JOBS:-$(nproc --ignore=1)} \ + time OPENSSL_DIR="${CONDA_PREFIX}" \ + CMAKE_BUILD_PARALLEL_LEVEL=${JOBS:-$(nproc --ignore=1)} \ cmake --build ~/legate/build --verbose --parallel ${JOBS:-$(nproc --ignore=1)}; sccache --show-stats; From 02091c142609fb47764df13b88364c7c72411b7c Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 23 Jun 2023 20:41:49 -0700 Subject: [PATCH 13/84] set OPENSSL_DIR to CONDA_PREFIX in install.py --- install.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install.py b/install.py index ab3c571abd..84c1fbf6d0 100755 --- a/install.py +++ b/install.py @@ -497,6 +497,9 @@ def validate_path(path): } ) + if ("CONDA_PREFIX" in os.environ) and not ("OPENSSL_DIR" in os.environ): + cmd_env.update({"OPENSSL_DIR": os.environ["CONDA_PREFIX"]}) + # execute python -m pip install . execute_command(pip_install_cmd, verbose, cwd=legate_core_dir, env=cmd_env) From e95d45f20b6b6f8255010429765c9bde889ba31a Mon Sep 17 00:00:00 2001 From: ptaylor Date: Sat, 24 Jun 2023 16:41:06 -0700 Subject: [PATCH 14/84] use mamba to install legate-core --- .github/workflows/gh-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-test.yml b/.github/workflows/gh-test.yml index aa300a585a..f1bfecf7c1 100644 --- a/.github/workflows/gh-test.yml +++ b/.github/workflows/gh-test.yml @@ -76,7 +76,7 @@ jobs: - name: Install legate-core and cunumeric in the legate conda environment shell: bash -eo pipefail {0} run: | - conda install -y -n legate -c nvidia -c conda-forge -c $(pwd)/artifacts/conda-build/legate_core legate-core + mamba install -y -n legate -c nvidia -c conda-forge -c $(pwd)/artifacts/conda-build/legate_core legate-core - if: inputs.test-scope == 'unit' name: Run legate.core unit tests From 2aebec166dd69d5120fd110ed257c46b4b636ef4 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Sat, 24 Jun 2023 16:46:03 -0700 Subject: [PATCH 15/84] set openssldir during conda build --- install.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/install.py b/install.py index 84c1fbf6d0..863f7a0a67 100755 --- a/install.py +++ b/install.py @@ -150,7 +150,11 @@ def get_install_dir(): # cmake-install dir # Install into conda prefix if defined - if "CONDA_PREFIX" in os.environ: + if "PREFIX" in os.environ and ( + os.environ.get("CONDA_BUILD", "0") == "1" + ): + return os.environ["PREFIX"] + elif "CONDA_PREFIX" in os.environ: return os.environ["CONDA_PREFIX"] import site @@ -497,8 +501,13 @@ def validate_path(path): } ) - if ("CONDA_PREFIX" in os.environ) and not ("OPENSSL_DIR" in os.environ): - cmd_env.update({"OPENSSL_DIR": os.environ["CONDA_PREFIX"]}) + if not ("OPENSSL_DIR" in os.environ): + if "PREFIX" in os.environ and ( + os.environ.get("CONDA_BUILD", "0") == "1" + ): + cmd_env.update({"OPENSSL_DIR": os.environ["PREFIX"]}) + elif "CONDA_PREFIX" in os.environ: + cmd_env.update({"OPENSSL_DIR": os.environ["CONDA_PREFIX"]}) # execute python -m pip install . execute_command(pip_install_cmd, verbose, cwd=legate_core_dir, env=cmd_env) From d6babb875ae907cf73a9986c8f30e12aebb9bb01 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 24 Jun 2023 23:47:22 +0000 Subject: [PATCH 16/84] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- install.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/install.py b/install.py index 863f7a0a67..07d36237f3 100755 --- a/install.py +++ b/install.py @@ -150,9 +150,7 @@ def get_install_dir(): # cmake-install dir # Install into conda prefix if defined - if "PREFIX" in os.environ and ( - os.environ.get("CONDA_BUILD", "0") == "1" - ): + if "PREFIX" in os.environ and (os.environ.get("CONDA_BUILD", "0") == "1"): return os.environ["PREFIX"] elif "CONDA_PREFIX" in os.environ: return os.environ["CONDA_PREFIX"] From a812676ec36cb8b4c8f2da2ecb120d081b7b5227 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Sat, 24 Jun 2023 16:52:48 -0700 Subject: [PATCH 17/84] run commands as non-root user --- .github/workflows/gh-test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gh-test.yml b/.github/workflows/gh-test.yml index f1bfecf7c1..5da8d5c41e 100644 --- a/.github/workflows/gh-test.yml +++ b/.github/workflows/gh-test.yml @@ -62,9 +62,10 @@ jobs: chown -R coder:coder /home/coder/ mkdir -p /tmp/out chown -R coder:coder /tmp/out + chown -R coder:coder artifacts - name: Recreate the legate conda environment - shell: bash -eo pipefail {0} + shell: su coder {0} run: | pattern="artifacts/out/*.yaml" files=( $pattern ) @@ -74,7 +75,7 @@ jobs: mamba env create -n "${DEFAULT_CONDA_ENV:-legate}" -f "${yaml_file}"; - name: Install legate-core and cunumeric in the legate conda environment - shell: bash -eo pipefail {0} + shell: su coder {0} run: | mamba install -y -n legate -c nvidia -c conda-forge -c $(pwd)/artifacts/conda-build/legate_core legate-core From 0211d9094cee38d551e0c6f257589077055c24d8 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Sat, 24 Jun 2023 17:30:56 -0700 Subject: [PATCH 18/84] ignore pull-request/* branches in docs and build jobs --- .github/workflows/ci-gh-docs.yml | 9 +++++---- .github/workflows/ci.yml | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-gh-docs.yml b/.github/workflows/ci-gh-docs.yml index 267fbd3d87..722a44347a 100644 --- a/.github/workflows/ci-gh-docs.yml +++ b/.github/workflows/ci-gh-docs.yml @@ -4,6 +4,7 @@ on: push: branches-ignore: - gh-pages # deployment target branch (this workflow should not exist on that branch anyway) + - "pull-request/[0-9]+" pull_request: branches-ignore: - gh-pages # deployment target branch (this workflow should not exist on that branch anyway) @@ -44,14 +45,14 @@ jobs: env: MATRIX_CONTEXT: ${{ toJSON(matrix) }} run: echo "$MATRIX_CONTEXT" - + #################################### # Actual build process starts here # #################################### - + - name: Checkout uses: actions/checkout@v3 - + - name: Generate YAML file listing dependencies run: scripts/generate-conda-envs.py --python 3.10 --ctk 11.8 --os linux --compilers --openmpi @@ -61,7 +62,7 @@ jobs: - name: Build doxygen documentation through install.py run: | conda run -n legate /bin/bash -c "./install.py --docs" - + - name: Build documentation using Makefile working-directory: ./docs/legate/core run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d01e3b5f9..1470110058 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,6 +3,7 @@ on: push: branches-ignore: - gh-pages # deployment target branch (this workflow should not exist on that branch anyway) + - "pull-request/[0-9]+" pull_request: branches-ignore: - gh-pages # deployment target branch (this workflow should not exist on that branch anyway) From 86879d42f19aabddd1b2236d38400d93eddb5ff1 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 27 Jun 2023 11:57:07 -0700 Subject: [PATCH 19/84] overwrite conda_build_config.yaml --- .../home/coder/.local/bin/build-legate-conda | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/continuous_integration/home/coder/.local/bin/build-legate-conda b/continuous_integration/home/coder/.local/bin/build-legate-conda index 4bab333ed6..6b26b131f1 100755 --- a/continuous_integration/home/coder/.local/bin/build-legate-conda +++ b/continuous_integration/home/coder/.local/bin/build-legate-conda @@ -14,7 +14,7 @@ build_legate_conda_package() { conda_build_args+=(-c conda-forge); conda_build_args+=(-c nvidia); conda_build_args+=(--croot /tmp/conda-build/legate_core); - conda_build_args+=(--numpy 1.22); + conda_build_args+=(--numpy "1.22"); conda_build_args+=(--python ${python_version}); conda_build_args+=(--no-test); conda_build_args+=(--no-verify); @@ -26,18 +26,18 @@ build_legate_conda_package() { GPU_ENABLED=true [ "${USE_CUDA:-}" = "OFF" ] && GPU_ENABLED=false - conda_build_args+=(--variants "{gpu_enabled:${GPU_ENABLED},python:${python_version}}"); - rm -rf /tmp/conda-build/legate_core; mkdir -p /tmp/conda-build/legate_core; # Synthesize new legate_core conda-build build.sh script - cat <> ~/legate/conda/conda-build/conda_build_config.yaml - + cat < ~/legate/conda/conda-build/conda_build_config.yaml numpy: - - 1.22 - + - "1.22" +python: + - "${python_version}" +gpu_enabled: + - "${GPU_ENABLED}" package_version: - "$(git -C ~/legate describe --abbrev=0 --tags | sed 's/[a-zA-Z]//g' | cut -d '.' -f -2).00" EOF From 49d55acc0ea1d00244d936e44c3bdfb88ce0d266 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 27 Jun 2023 13:44:57 -0700 Subject: [PATCH 20/84] add cmake_version --- continuous_integration/home/coder/.local/bin/build-legate-conda | 2 ++ 1 file changed, 2 insertions(+) diff --git a/continuous_integration/home/coder/.local/bin/build-legate-conda b/continuous_integration/home/coder/.local/bin/build-legate-conda index 6b26b131f1..ce63374d46 100755 --- a/continuous_integration/home/coder/.local/bin/build-legate-conda +++ b/continuous_integration/home/coder/.local/bin/build-legate-conda @@ -38,6 +38,8 @@ python: - "${python_version}" gpu_enabled: - "${GPU_ENABLED}" +cmake_version: + - ">=3.20.1,!=3.23.0" package_version: - "$(git -C ~/legate describe --abbrev=0 --tags | sed 's/[a-zA-Z]//g' | cut -d '.' -f -2).00" EOF From 9cea0a399894cea608530f4f40b18ec09a1db1d2 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 27 Jun 2023 14:11:40 -0700 Subject: [PATCH 21/84] define and unpin numpy_version --- .../home/coder/.local/bin/build-legate-conda | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/continuous_integration/home/coder/.local/bin/build-legate-conda b/continuous_integration/home/coder/.local/bin/build-legate-conda index ce63374d46..c4da064f74 100755 --- a/continuous_integration/home/coder/.local/bin/build-legate-conda +++ b/continuous_integration/home/coder/.local/bin/build-legate-conda @@ -14,7 +14,7 @@ build_legate_conda_package() { conda_build_args+=(-c conda-forge); conda_build_args+=(-c nvidia); conda_build_args+=(--croot /tmp/conda-build/legate_core); - conda_build_args+=(--numpy "1.22"); + conda_build_args+=(--numpy ">=1.22"); conda_build_args+=(--python ${python_version}); conda_build_args+=(--no-test); conda_build_args+=(--no-verify); @@ -32,10 +32,12 @@ build_legate_conda_package() { # Synthesize new legate_core conda-build build.sh script cat < ~/legate/conda/conda-build/conda_build_config.yaml -numpy: - - "1.22" python: - "${python_version}" +numpy: + - ">=1.22" +numpy_version: + - ">=1.22" gpu_enabled: - "${GPU_ENABLED}" cmake_version: From 1b81596b724c6505f7d6b21f1a6b99f3cf3dc042 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 27 Jun 2023 15:19:07 -0700 Subject: [PATCH 22/84] remove --numpy arg --- .../home/coder/.local/bin/build-legate-conda | 3 --- 1 file changed, 3 deletions(-) diff --git a/continuous_integration/home/coder/.local/bin/build-legate-conda b/continuous_integration/home/coder/.local/bin/build-legate-conda index c4da064f74..75b53a9c29 100755 --- a/continuous_integration/home/coder/.local/bin/build-legate-conda +++ b/continuous_integration/home/coder/.local/bin/build-legate-conda @@ -14,7 +14,6 @@ build_legate_conda_package() { conda_build_args+=(-c conda-forge); conda_build_args+=(-c nvidia); conda_build_args+=(--croot /tmp/conda-build/legate_core); - conda_build_args+=(--numpy ">=1.22"); conda_build_args+=(--python ${python_version}); conda_build_args+=(--no-test); conda_build_args+=(--no-verify); @@ -34,8 +33,6 @@ build_legate_conda_package() { cat < ~/legate/conda/conda-build/conda_build_config.yaml python: - "${python_version}" -numpy: - - ">=1.22" numpy_version: - ">=1.22" gpu_enabled: From 8f55d5205058eafa4495fbc69cf6a2b97292e621 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 27 Jun 2023 17:19:25 -0700 Subject: [PATCH 23/84] ensure dirs exist and are clean --- .../home/coder/.local/bin/build-legate-cpp | 2 ++ .../home/coder/.local/bin/build-legate-wheel | 2 ++ continuous_integration/home/coder/.local/bin/make-conda-env | 5 ++++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/continuous_integration/home/coder/.local/bin/build-legate-cpp b/continuous_integration/home/coder/.local/bin/build-legate-cpp index 51f05ed855..2c927c6e30 100755 --- a/continuous_integration/home/coder/.local/bin/build-legate-cpp +++ b/continuous_integration/home/coder/.local/bin/build-legate-cpp @@ -31,6 +31,8 @@ build_legate_cpp() { sccache --stop-server >/dev/null 2>&1 || true; sccache --show-stats; + rm -rf ~/legate/build; + time OPENSSL_DIR="${CONDA_PREFIX}" \ CMAKE_BUILD_PARALLEL_LEVEL=${JOBS:-$(nproc --ignore=1)} \ cmake -S ~/legate -B ~/legate/build "${cmake_args[@]}" -GNinja; diff --git a/continuous_integration/home/coder/.local/bin/build-legate-wheel b/continuous_integration/home/coder/.local/bin/build-legate-wheel index 3537b28714..34128fec46 100755 --- a/continuous_integration/home/coder/.local/bin/build-legate-wheel +++ b/continuous_integration/home/coder/.local/bin/build-legate-wheel @@ -5,6 +5,8 @@ build_legate_wheel() { mkdir -p /tmp/out; + rm -rf ~/legate/_skbuild; + local pip_args=(-vv); pip_args+=(--wheel-dir /tmp/out); diff --git a/continuous_integration/home/coder/.local/bin/make-conda-env b/continuous_integration/home/coder/.local/bin/make-conda-env index 41ebb23dcf..9a6cb01964 100755 --- a/continuous_integration/home/coder/.local/bin/make-conda-env +++ b/continuous_integration/home/coder/.local/bin/make-conda-env @@ -12,6 +12,8 @@ make_conda_env() { python_version="$(python3 --version 2>&1 | cut -d' ' -f2 | cut -d'.' -f3 --complement)"; fi + cd ~/; + local yaml_file=~/"$( \ ~/legate/scripts/generate-conda-envs.py \ --os linux \ @@ -27,7 +29,8 @@ make_conda_env() { echo " - boa" >> "${yaml_file}"; cat "${yaml_file}"; - cp "${yaml_file}" /tmp/out/ + mkdir -p /tmp/out/; + cp "${yaml_file}" /tmp/out/; mamba env create -n "${DEFAULT_CONDA_ENV:-legate}" -f "${yaml_file}"; From 6c6ce71f96b53bb4188047747e048c3b32f23114 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 28 Jun 2023 11:56:18 -0700 Subject: [PATCH 24/84] save conda build output to /tmp/out/legate_core --- .github/workflows/gh-build.yml | 1 - .github/workflows/gh-test.yml | 2 +- .../home/coder/.local/bin/build-legate-conda | 3 +++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index b40d17cecd..a9f3bd0ba6 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -87,5 +87,4 @@ jobs: with: name: "legate.core-${{ inputs.build-target }}-${{ inputs.sha }}" path: | - /tmp/conda-build /tmp/out diff --git a/.github/workflows/gh-test.yml b/.github/workflows/gh-test.yml index 5da8d5c41e..8556577314 100644 --- a/.github/workflows/gh-test.yml +++ b/.github/workflows/gh-test.yml @@ -77,7 +77,7 @@ jobs: - name: Install legate-core and cunumeric in the legate conda environment shell: su coder {0} run: | - mamba install -y -n legate -c nvidia -c conda-forge -c $(pwd)/artifacts/conda-build/legate_core legate-core + mamba install -y -n legate -c nvidia -c conda-forge -c $(pwd)/artifacts/out/legate_core legate-core - if: inputs.test-scope == 'unit' name: Run legate.core unit tests diff --git a/continuous_integration/home/coder/.local/bin/build-legate-conda b/continuous_integration/home/coder/.local/bin/build-legate-conda index 75b53a9c29..05a59843ba 100755 --- a/continuous_integration/home/coder/.local/bin/build-legate-conda +++ b/continuous_integration/home/coder/.local/bin/build-legate-conda @@ -21,10 +21,13 @@ build_legate_conda_package() { conda_build_args+=("--build-id-pat=''"); conda_build_args+=(--no-include-recipe); conda_build_args+=(--no-anaconda-upload); + conda_build_args+=(--output-folder /tmp/out/legate_core); GPU_ENABLED=true [ "${USE_CUDA:-}" = "OFF" ] && GPU_ENABLED=false + rm -rf /tmp/out/legate_core; + mkdir -p /tmp/out/legate_core; rm -rf /tmp/conda-build/legate_core; mkdir -p /tmp/conda-build/legate_core; From 94f79ff6033fb2ee16d6a32a8e9dcb5bd99ac19b Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 28 Jun 2023 11:56:58 -0700 Subject: [PATCH 25/84] unpin numpy in run requirements --- conda/conda-build/meta.yaml | 2 +- .../home/coder/.local/bin/build-legate-conda | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/conda/conda-build/meta.yaml b/conda/conda-build/meta.yaml index e384affcf7..b5479de8df 100644 --- a/conda/conda-build/meta.yaml +++ b/conda/conda-build/meta.yaml @@ -118,7 +118,7 @@ requirements: run: - cffi - llvm-openmp - - numpy {{ numpy_version }} + - numpy - typing_extensions {% if gpu_enabled_bool %} - cuda-cudart >={{ cuda_version }},<{{ cuda_major+1 }} diff --git a/continuous_integration/home/coder/.local/bin/build-legate-conda b/continuous_integration/home/coder/.local/bin/build-legate-conda index 05a59843ba..466db7f1a4 100755 --- a/continuous_integration/home/coder/.local/bin/build-legate-conda +++ b/continuous_integration/home/coder/.local/bin/build-legate-conda @@ -14,6 +14,7 @@ build_legate_conda_package() { conda_build_args+=(-c conda-forge); conda_build_args+=(-c nvidia); conda_build_args+=(--croot /tmp/conda-build/legate_core); + conda_build_args+=(--numpy 1.22); conda_build_args+=(--python ${python_version}); conda_build_args+=(--no-test); conda_build_args+=(--no-verify); @@ -34,10 +35,14 @@ build_legate_conda_package() { # Synthesize new legate_core conda-build build.sh script cat < ~/legate/conda/conda-build/conda_build_config.yaml +numpy: + - 1.22 python: - "${python_version}" numpy_version: - ">=1.22" +use_local_path: + - "true" gpu_enabled: - "${GPU_ENABLED}" cmake_version: @@ -69,8 +74,6 @@ EOF git -C ~/legate reset --hard HEAD~1; - cp /tmp/conda-build/legate_core/linux-64/legate-*.tar.bz2 /tmp/out/; - { set +x; } 2>/dev/null; } From 78edac38d0eb5a9f608eeb61ffa797961060eeec Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 28 Jun 2023 11:57:08 -0700 Subject: [PATCH 26/84] pin cuda-version package --- conda/conda-build/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda/conda-build/meta.yaml b/conda/conda-build/meta.yaml index b5479de8df..090969c495 100644 --- a/conda/conda-build/meta.yaml +++ b/conda/conda-build/meta.yaml @@ -111,6 +111,7 @@ requirements: - cuda-nvtx ={{ cuda_version }} - cuda-cccl ={{ cuda_version }} - cuda-cudart ={{ cuda_version }} + - cuda-version ={{ cuda_version }} - cuda-nvml-dev ={{ cuda_version }} - cuda-driver-dev ={{ cuda_version }} - cuda-cudart-dev ={{ cuda_version }} From 0fd2a070f58397d6d13743cde8ad7f6250c96515 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 28 Jun 2023 11:59:20 -0700 Subject: [PATCH 27/84] change channel order --- .github/workflows/gh-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gh-test.yml b/.github/workflows/gh-test.yml index 8556577314..6244283dc2 100644 --- a/.github/workflows/gh-test.yml +++ b/.github/workflows/gh-test.yml @@ -74,10 +74,10 @@ jobs: mamba env create -n "${DEFAULT_CONDA_ENV:-legate}" -f "${yaml_file}"; - - name: Install legate-core and cunumeric in the legate conda environment + - name: Install legate-core in the legate conda environment shell: su coder {0} run: | - mamba install -y -n legate -c nvidia -c conda-forge -c $(pwd)/artifacts/out/legate_core legate-core + mamba install -y -n legate -c $(pwd)/artifacts/out/legate_core -c conda-forge -c nvidia legate-core - if: inputs.test-scope == 'unit' name: Run legate.core unit tests From 53fd3207ef01da10d3cc7f0dd61d30f2f5bde701 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 28 Jun 2023 12:08:23 -0700 Subject: [PATCH 28/84] remove numpy_version pin --- conda/conda-build/meta.yaml | 2 +- continuous_integration/home/coder/.local/bin/build-legate-conda | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/conda/conda-build/meta.yaml b/conda/conda-build/meta.yaml index 090969c495..f20dffb8ca 100644 --- a/conda/conda-build/meta.yaml +++ b/conda/conda-build/meta.yaml @@ -104,7 +104,7 @@ requirements: - cython - llvm-openmp - scikit-build - - numpy {{ numpy_version }} + - numpy {% if gpu_enabled_bool %} - nccl - cuda-nvcc ={{ cuda_version }} diff --git a/continuous_integration/home/coder/.local/bin/build-legate-conda b/continuous_integration/home/coder/.local/bin/build-legate-conda index 466db7f1a4..7458015348 100755 --- a/continuous_integration/home/coder/.local/bin/build-legate-conda +++ b/continuous_integration/home/coder/.local/bin/build-legate-conda @@ -39,8 +39,6 @@ numpy: - 1.22 python: - "${python_version}" -numpy_version: - - ">=1.22" use_local_path: - "true" gpu_enabled: From 310c9bb4aed43a453c33e4cfa7353a62ccd249ad Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 28 Jun 2023 12:22:38 -0700 Subject: [PATCH 29/84] fix artifacts path after removing dir --- .github/workflows/gh-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gh-test.yml b/.github/workflows/gh-test.yml index 6244283dc2..6f1b708b56 100644 --- a/.github/workflows/gh-test.yml +++ b/.github/workflows/gh-test.yml @@ -67,7 +67,7 @@ jobs: - name: Recreate the legate conda environment shell: su coder {0} run: | - pattern="artifacts/out/*.yaml" + pattern="artifacts/*.yaml" files=( $pattern ) yaml_file="${files[0]}" echo "YAML file: ${yaml_file}" @@ -77,7 +77,7 @@ jobs: - name: Install legate-core in the legate conda environment shell: su coder {0} run: | - mamba install -y -n legate -c $(pwd)/artifacts/out/legate_core -c conda-forge -c nvidia legate-core + mamba install -y -n legate -c $(pwd)/artifacts/legate_core -c conda-forge -c nvidia legate-core - if: inputs.test-scope == 'unit' name: Run legate.core unit tests From 9eae6e6a4b45092e7d30a50e981d7b489da8012f Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 28 Jun 2023 13:19:28 -0700 Subject: [PATCH 30/84] pin numpy to <1.25 --- conda/conda-build/meta.yaml | 4 ++-- .../home/coder/.local/bin/build-legate-conda | 2 ++ scripts/generate-conda-envs.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/conda/conda-build/meta.yaml b/conda/conda-build/meta.yaml index f20dffb8ca..e4436849b1 100644 --- a/conda/conda-build/meta.yaml +++ b/conda/conda-build/meta.yaml @@ -104,7 +104,7 @@ requirements: - cython - llvm-openmp - scikit-build - - numpy + - numpy {{ numpy_version }} {% if gpu_enabled_bool %} - nccl - cuda-nvcc ={{ cuda_version }} @@ -119,7 +119,7 @@ requirements: run: - cffi - llvm-openmp - - numpy + - numpy {{ numpy_version }} - typing_extensions {% if gpu_enabled_bool %} - cuda-cudart >={{ cuda_version }},<{{ cuda_major+1 }} diff --git a/continuous_integration/home/coder/.local/bin/build-legate-conda b/continuous_integration/home/coder/.local/bin/build-legate-conda index 7458015348..e610bffb64 100755 --- a/continuous_integration/home/coder/.local/bin/build-legate-conda +++ b/continuous_integration/home/coder/.local/bin/build-legate-conda @@ -39,6 +39,8 @@ numpy: - 1.22 python: - "${python_version}" +numpy_version: + - ">=1.22,<1.25" use_local_path: - "true" gpu_enabled: diff --git a/scripts/generate-conda-envs.py b/scripts/generate-conda-envs.py index 9374ac63f6..e556afd18d 100755 --- a/scripts/generate-conda-envs.py +++ b/scripts/generate-conda-envs.py @@ -139,7 +139,7 @@ def conda(self) -> Reqs: return ( "cffi", "llvm-openmp", - "numpy>=1.22", + "numpy>=1.22,<1.25", "libblas=*=*openblas*", "openblas=*=*openmp*", "opt_einsum", From df0e32bb8df3de2130c44e597179f7a27754b141 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 5 Jul 2023 16:11:40 -0700 Subject: [PATCH 31/84] bump legion commit hash --- cmake/versions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/versions.json b/cmake/versions.json index fea9cd9154..dc20afa6c0 100644 --- a/cmake/versions.json +++ b/cmake/versions.json @@ -7,7 +7,7 @@ }, "Legion": { "git_url" : "https://gitlab.com/inlineptx/legion.git", - "git_tag" : "e05c36c7867641a4c2e8e167c078e459ac83d596" + "git_tag" : "9f25e30a1f4a0c86789d27d55310934df3b82ba3" } } } From 242c326da40d924a24fa34f80dfefb408d3d69d8 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 5 Jul 2023 17:42:40 -0700 Subject: [PATCH 32/84] bump legion commit hash --- cmake/versions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/versions.json b/cmake/versions.json index dc20afa6c0..b4804be718 100644 --- a/cmake/versions.json +++ b/cmake/versions.json @@ -7,7 +7,7 @@ }, "Legion": { "git_url" : "https://gitlab.com/inlineptx/legion.git", - "git_tag" : "9f25e30a1f4a0c86789d27d55310934df3b82ba3" + "git_tag" : "23c0f1ec77b0cabc0ef251ae232e3545c83cd217" } } } From 2333bbd526c420a6ee4bec06f1d200943b7e81ba Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 6 Jul 2023 12:04:01 -0700 Subject: [PATCH 33/84] bump legion commit hash and switch back to StanfordLegion fork --- cmake/versions.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/versions.json b/cmake/versions.json index b4804be718..0004e33c5c 100644 --- a/cmake/versions.json +++ b/cmake/versions.json @@ -6,8 +6,8 @@ "git_tag" : "1.17.0" }, "Legion": { - "git_url" : "https://gitlab.com/inlineptx/legion.git", - "git_tag" : "23c0f1ec77b0cabc0ef251ae232e3545c83cd217" + "git_url" : "https://gitlab.com/StanfordLegion/legion.git", + "git_tag" : "11b66c83a13e427d15a0b8bc19c4026e950979f5" } } } From 53a97aea86ed9a843574dfda00e169bc24775434 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 10 Jul 2023 13:45:48 -0700 Subject: [PATCH 34/84] bump legion commit hash --- cmake/versions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/versions.json b/cmake/versions.json index 0004e33c5c..155b2b838b 100644 --- a/cmake/versions.json +++ b/cmake/versions.json @@ -7,7 +7,7 @@ }, "Legion": { "git_url" : "https://gitlab.com/StanfordLegion/legion.git", - "git_tag" : "11b66c83a13e427d15a0b8bc19c4026e950979f5" + "git_tag" : "4396cde515e26fc1410a9a9ff54aa7fbc3c3315c" } } } From 5b31bc85b8c781aaec35d80b4727538e02944349 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 10 Jul 2023 13:47:38 -0700 Subject: [PATCH 35/84] support passing Legion_PYTHON_EXTRA_INSTALL_ARGS with comma and semicolon separators --- cmake/thirdparty/get_legion.cmake | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/thirdparty/get_legion.cmake b/cmake/thirdparty/get_legion.cmake index ccd384ebb8..dc97c7dd76 100644 --- a/cmake/thirdparty/get_legion.cmake +++ b/cmake/thirdparty/get_legion.cmake @@ -68,9 +68,13 @@ function(find_or_configure_legion) include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/cpm_helpers.cmake) get_cpm_git_args(legion_cpm_git_args REPOSITORY ${git_repo} BRANCH ${git_branch}) if(NOT DEFINED Legion_PYTHON_EXTRA_INSTALL_ARGS) - set(Legion_PYTHON_EXTRA_INSTALL_ARGS "--single-version-externally-managed --root=/") + set(Legion_PYTHON_EXTRA_INSTALL_ARGS "--root / --prefix \"\${CMAKE_INSTALL_PREFIX}\"") endif() + # Support comma and semicolon delimited lists + string(REPLACE "," " " Legion_PYTHON_EXTRA_INSTALL_ARGS "${Legion_PYTHON_EXTRA_INSTALL_ARGS}") + string(REPLACE ";" " " Legion_PYTHON_EXTRA_INSTALL_ARGS "${Legion_PYTHON_EXTRA_INSTALL_ARGS}") + set(_cuda_path "") set(_legion_cuda_options "") @@ -121,12 +125,8 @@ function(find_or_configure_legion) "CMAKE_CXX_STANDARD ${_cxx_std}" "Legion_VERSION ${version}" "Legion_BUILD_BINDINGS ON" - "Legion_BUILD_APPS OFF" - "Legion_BUILD_TESTS OFF" - "Legion_BUILD_TUTORIAL OFF" "Legion_REDOP_HALF ON" "Legion_REDOP_COMPLEX ON" - "Legion_GPU_REDUCTIONS OFF" "Legion_BUILD_RUST_PROFILER ON" ) endif() From b65eb445774d85408e71527ede0fa789b021a608 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 10 Jul 2023 13:54:01 -0700 Subject: [PATCH 36/84] remove custom entrypoint script --- .github/workflows/gh-build.yml | 10 +++++----- .github/workflows/gh-test.yml | 2 +- .../home/coder/.local/bin/entrypoint | 13 ------------- 3 files changed, 6 insertions(+), 19 deletions(-) delete mode 100755 continuous_integration/home/coder/.local/bin/entrypoint diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index a9f3bd0ba6..41c7b33c3d 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -25,7 +25,7 @@ jobs: container: options: -u root - image: rapidsai/devcontainers:23.06-cpp-cuda11.8-mambaforge-ubuntu22.04 + image: rapidsai/devcontainers:23.08-cpp-cuda11.8-mambaforge-ubuntu22.04 env: USE_CUDA: ${{ (inputs.build-target == 'cpu' && 'OFF') || 'ON' }} DEFAULT_CONDA_ENV: legate @@ -68,19 +68,19 @@ jobs: - name: Create conda env shell: su coder {0} - run: cd ~/; exec entrypoint make-conda-env; + run: exec devcontainer-utils-post-attach-command-entrypoint make-conda-env; - name: Build legate.core C++ library shell: su coder {0} - run: cd ~/; exec entrypoint build-legate-cpp; + run: exec devcontainer-utils-post-attach-command-entrypoint build-legate-cpp; - name: Build legate.core Python Wheel shell: su coder {0} - run: cd ~/; exec entrypoint build-legate-wheel; + run: exec devcontainer-utils-post-attach-command-entrypoint build-legate-wheel; - name: Build legate.core Conda Package shell: su coder {0} - run: cd ~/; exec entrypoint build-legate-conda; + run: exec devcontainer-utils-post-attach-command-entrypoint build-legate-conda; - name: Upload build artifacts uses: actions/upload-artifact@v3 diff --git a/.github/workflows/gh-test.yml b/.github/workflows/gh-test.yml index 4eecc75a68..f9481e53e4 100644 --- a/.github/workflows/gh-test.yml +++ b/.github/workflows/gh-test.yml @@ -23,7 +23,7 @@ jobs: runs-on: ${{ inputs.runs-on }} container: options: -u root - image: rapidsai/devcontainers:23.06-cpp-cuda11.8-mambaforge-ubuntu22.04 + image: rapidsai/devcontainers:23.08-cpp-cuda11.8-mambaforge-ubuntu22.04 env: DEFAULT_CONDA_ENV: legate PYTHONDONTWRITEBYTECODE: 1 diff --git a/continuous_integration/home/coder/.local/bin/entrypoint b/continuous_integration/home/coder/.local/bin/entrypoint deleted file mode 100755 index 45100c97aa..0000000000 --- a/continuous_integration/home/coder/.local/bin/entrypoint +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -entrypoint() { - - . devcontainer-utils-post-attach-command; - - sccache --stop-server >/dev/null 2>&1 || true; - sccache --show-stats; - - exec "$@"; -} - -entrypoint "$@"; From 901d9710b9a1040508d9041008ff19752410c649 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 10 Jul 2023 13:57:57 -0700 Subject: [PATCH 37/84] rename pip wheel to `legate-core` to align with conda package --- install.py | 2 +- pyproject.toml | 8 ++++---- scripts/build-install.sh | 2 +- scripts/build-no-install.sh | 2 +- scripts/build-separately-no-install.sh | 4 ++-- scripts/build-with-legion-no-install.sh | 2 +- scripts/build-with-legion-separately-no-install.sh | 2 +- setup.py | 8 ++++++-- 8 files changed, 17 insertions(+), 13 deletions(-) diff --git a/install.py b/install.py index 4e7624f942..7ce0bc7f9d 100755 --- a/install.py +++ b/install.py @@ -377,7 +377,7 @@ def validate_path(path): shutil.rmtree(join(legate_core_dir, "dist"), ignore_errors=True) shutil.rmtree(join(legate_core_dir, "build"), ignore_errors=True) shutil.rmtree( - join(legate_core_dir, "legate.core.egg-info"), + join(legate_core_dir, "legate_core.egg-info"), ignore_errors=True, ) diff --git a/pyproject.toml b/pyproject.toml index c9f0e8cb3b..5554c63b52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,12 +14,12 @@ [build-system] requires = [ - "wheel", - "ninja", - "setuptools", - "scikit-build>=0.13.1", "cmake>=3.22.1,!=3.23.0,!=3.25.0", "cython", + "ninja", + "scikit-build>=0.13.1", + "setuptools>=60", + "wheel", ] [tool.black] diff --git a/scripts/build-install.sh b/scripts/build-install.sh index 7601ed4797..ac6c896e39 100755 --- a/scripts/build-install.sh +++ b/scripts/build-install.sh @@ -10,7 +10,7 @@ source ./scripts/util/compiler-flags.sh source ./scripts/util/uninstall-global-legion-and-legate-core.sh # Remove existing build artifacts -rm -rf ./{build,_skbuild,dist,legate.core.egg-info} +rm -rf ./{build,_skbuild,dist,legate_core.egg-info} # Define CMake configuration arguments cmake_args="${CMAKE_ARGS:-}" diff --git a/scripts/build-no-install.sh b/scripts/build-no-install.sh index 37a2a356d8..b6ec10b55c 100755 --- a/scripts/build-no-install.sh +++ b/scripts/build-no-install.sh @@ -8,7 +8,7 @@ source ./scripts/util/build-caching.sh source ./scripts/util/compiler-flags.sh # Remove existing build artifacts -rm -rf ./{build,_skbuild,dist,legate.core.egg-info} +rm -rf ./{build,_skbuild,dist,legate_core.egg-info} # Define CMake configuration arguments cmake_args="${CMAKE_ARGS:-}" diff --git a/scripts/build-separately-no-install.sh b/scripts/build-separately-no-install.sh index ea6e094a46..3ef1c65aad 100755 --- a/scripts/build-separately-no-install.sh +++ b/scripts/build-separately-no-install.sh @@ -8,7 +8,7 @@ source ./scripts/util/build-caching.sh source ./scripts/util/compiler-flags.sh # Remove existing build artifacts -rm -rf ./{build,_skbuild,dist,legate.core.egg-info} +rm -rf ./{build,_skbuild,dist,legate_core.egg-info} # Define CMake configuration arguments cmake_args="${CMAKE_ARGS:-}" @@ -21,7 +21,7 @@ cmake_args+=" -D Legion_USE_CUDA=ON -D Legion_USE_OpenMP=ON -D Legion_USE_Python=ON --D Legion_BUILD_BINDINGS=ON +-D Legion_Python_Version=$(python3 --version 2>&1 | cut -d' ' -f2 | cut -d'.' -f3 --complement) -D Legion_CUDA_ARCH=native "; diff --git a/scripts/build-with-legion-no-install.sh b/scripts/build-with-legion-no-install.sh index 27f4160aac..c3d340b556 100755 --- a/scripts/build-with-legion-no-install.sh +++ b/scripts/build-with-legion-no-install.sh @@ -10,7 +10,7 @@ source ./scripts/util/compiler-flags.sh source ./scripts/util/read-legion-root.sh "$0" # Remove existing build artifacts -rm -rf ./{build,_skbuild,dist,legate.core.egg-info} +rm -rf ./{build,_skbuild,dist,legate_core.egg-info} # Use all but 2 threads to compile ninja_args="-j$(nproc --ignore=2)" diff --git a/scripts/build-with-legion-separately-no-install.sh b/scripts/build-with-legion-separately-no-install.sh index 1961fdc625..f224a8f44a 100755 --- a/scripts/build-with-legion-separately-no-install.sh +++ b/scripts/build-with-legion-separately-no-install.sh @@ -10,7 +10,7 @@ source ./scripts/util/compiler-flags.sh source ./scripts/util/read-legion-root.sh "$0" # Remove existing build artifacts -rm -rf ./{build,_skbuild,dist,legate.core.egg-info} +rm -rf ./{build,_skbuild,dist,legate_core.egg-info} # Use all but 2 threads to compile ninja_args="-j$(nproc --ignore=2)" diff --git a/setup.py b/setup.py index 83912f31f8..e46b61cd67 100755 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ import versioneer setup( - name="legate.core", + name="legate-core", version=versioneer.get_version(), description="legate.core - The Foundation for All Legate Libraries", url="https://github.com/nv-legate/legate.core", @@ -69,6 +69,10 @@ }, scripts=["bind.sh"], cmdclass=versioneer.get_cmdclass(), - install_requires=["numpy>=1.22"], + install_requires=[ + "cffi", + "numpy>=1.22", + "typing_extensions", + ], zip_safe=False, ) From ffbdc78cb6159bcfe4e18807527ecd30a5c0bdd8 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 10 Jul 2023 13:58:57 -0700 Subject: [PATCH 38/84] fix cython install rpaths for both editable and regular install modes --- legate/install_info.py.in | 6 ++++-- legate_core_cpp.cmake | 2 +- legate_core_python.cmake | 17 +++++++++++------ 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/legate/install_info.py.in b/legate/install_info.py.in index d416e48d24..fc60b09999 100644 --- a/legate/install_info.py.in +++ b/legate/install_info.py.in @@ -32,8 +32,10 @@ def get_libpath(): return libdir return None + from .util.fs import get_legate_paths + return ( - find_liblgcore(join(lg_path, "build", "lib")) or + find_liblgcore(get_legate_paths().legate_lib_path) or find_liblgcore(join(dirname(dirname(dirname(lg_path))), "lib")) or find_liblgcore(join(dirname(dirname(sys.executable)), "lib")) or "" @@ -54,4 +56,4 @@ ON, OFF = True, False use_cuda: bool = @Legion_USE_CUDA@ -use_openmp: bool = @Legion_USE_OpenMP@ \ No newline at end of file +use_openmp: bool = @Legion_USE_OpenMP@ diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index acbae890fb..6e8fa884d4 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -1,5 +1,5 @@ #============================================================================= -# Copyright 2022 NVIDIA Corporation +# Copyright 2022-2023 NVIDIA Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/legate_core_python.cmake b/legate_core_python.cmake index d23296d86f..dea81ff21f 100644 --- a/legate_core_python.cmake +++ b/legate_core_python.cmake @@ -38,10 +38,7 @@ endif() if(NOT legate_core_FOUND) set(SKBUILD OFF) set(Legion_USE_Python ON) - set(Legion_BUILD_BINDINGS ON) - add_subdirectory(. "${CMAKE_CURRENT_SOURCE_DIR}/build") - get_target_property(cython_lib_dir legate_core LIBRARY_OUTPUT_DIRECTORY) - set(cython_lib_dir "${CMAKE_CURRENT_SOURCE_DIR}/build/${cython_lib_dir}") + add_subdirectory(. legate-core-cpp) set(SKBUILD ON) endif() @@ -66,10 +63,18 @@ rapids_cython_init() add_subdirectory(legate/core/_lib) -if(DEFINED cython_lib_dir) - rapids_cython_add_rpath_entries(TARGET legate_core PATHS "${cython_lib_dir}") +set(cython_lib_dir "../../") + +if(CMAKE_INSTALL_RPATH_USE_LINK_PATH) + get_target_property(cython_lib_dir legate_core LIBRARY_OUTPUT_DIRECTORY) + get_target_property(legate_cpp_dir legate_core BINARY_DIR) + if(legate_cpp_dir) + set(cython_lib_dir "${legate_cpp_dir}/${cython_lib_dir}") + endif() endif() +rapids_cython_add_rpath_entries(TARGET legate_core PATHS "${cython_lib_dir}") + ############################################################################## # - conda environment -------------------------------------------------------- From 173d0990bef8ec6e38f8f67407bf8f06af73bab0 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 10 Jul 2023 13:59:50 -0700 Subject: [PATCH 39/84] support the `pip install --user` flag --- install.py | 120 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 85 insertions(+), 35 deletions(-) diff --git a/install.py b/install.py index 7ce0bc7f9d..72885b3390 100755 --- a/install.py +++ b/install.py @@ -68,9 +68,11 @@ def __call__(self, parser, namespace, values, option_string): setattr(namespace, self.dest, not option_string.startswith("--no")) -def execute_command(args, verbose, **kwargs): +def execute_command(args, verbose, ignore_errors=False, **kwargs): if verbose: print(f"Executing: {' '.join(args)} with {kwargs}") + if ignore_errors: + subprocess.call(args, **kwargs) subprocess.check_call(args, **kwargs) @@ -144,29 +146,61 @@ def was_previously_built_with_different_build_isolation( return False -def get_install_dir(): +def get_install_dir(args: list): # Infer the location where to install the Legion Python bindings, # otherwise they'll only be installed into the local scikit-build # cmake-install dir # Install into conda prefix if defined if "PREFIX" in os.environ and (os.environ.get("CONDA_BUILD", "0") == "1"): - return os.environ["PREFIX"] + install_dir = os.environ["PREFIX"] + return (install_dir, ["--root", "/", "--prefix", install_dir]) elif "CONDA_PREFIX" in os.environ: - return os.environ["CONDA_PREFIX"] + install_dir = os.environ["CONDA_PREFIX"] + return (install_dir, ["--root", "/", "--prefix", install_dir]) + + def get_arg(flag: str): + idx = args.index(flag) if flag in args else -1 + if idx > -1: + args.remove(flag) + if idx + 1 < len(args): + args.remove(args[idx + 1]) + return args[idx + 1] + return None + + install_dir = None + install_args = [] + if root_dir := get_arg("--root"): + install_dir = root_dir + install_args += ["--root", root_dir] + + if prefix_dir := get_arg("--prefix"): + install_dir = [install_dir, prefix_dir] + install_dir = filter(lambda x: x, install_dir) + install_dir = os.path.join(*install_dir, "") + install_args += ["--prefix", prefix_dir] + + if install_dir and os.path.exists(install_dir): + return (install_dir, install_args) import site # Try to install into user site packages first? - if site.ENABLE_USER_SITE and os.path.exists( - user_site_pkgs := site.getusersitepackages() + if ("--user" in args or site.ENABLE_USER_SITE) and os.path.exists( + user_base := site.getuserbase() ): - return user_site_pkgs + if "--user" in args: + args.remove("--user") + return (user_base, ["--user"]) # Otherwise fallback to regular site-packages? for site_pkgs in site.getsitepackages(): - if os.path.exists(site_pkgs): - return site_pkgs + install_dir = os.path.join(site_pkgs, '..', '..', '..') + install_dir = os.path.realpath(install_dir) + if install_dir != "/" and os.path.exists(install_dir): + return (install_dir, ["--root", "/", "--prefix", install_dir]) + + return (None, []) def install_legion_python_bindings( @@ -372,6 +406,29 @@ def validate_path(path): print("Performing a clean build to accommodate build isolation.") clean_first = True + cmd_env = dict(os.environ.items()) + + # Explicitly uninstall legate.core if doing a clean/isolated build. + # + # A prior installation may have built and installed legate.core C++ + # dependencies (like Legion). + # + # CMake will find and use them for the current build, which would normally + # be correct, but pip uninstalls files from any existing installation as + # the last step of the install process, including the libraries found by + # CMake during the current build. + # + # Therefore this uninstall step must occur *before* CMake attempts to find + # these dependencies, triggering CMake to build and install them again. + if clean_first or (build_isolation and not editable): + execute_command( + [sys.executable, "-m", "pip", "uninstall", "-y", "legate-core"], + verbose, + ignore_errors=True, + cwd=legate_core_dir, + env=cmd_env, + ) + if clean_first: shutil.rmtree(skbuild_dir, ignore_errors=True) shutil.rmtree(join(legate_core_dir, "dist"), ignore_errors=True) @@ -383,49 +440,41 @@ def validate_path(path): # Configure and build legate.core via setup.py pip_install_cmd = [sys.executable, "-m", "pip", "install"] - cmd_env = dict(os.environ.items()) + + # Use preexisting CMAKE_ARGS from conda if set + cmake_flags = cmd_env.get("CMAKE_ARGS", "").split(" ") if "OPENSSL_DIR" not in cmd_env and "CONDA_PREFIX" in cmd_env: cmd_env.update({"OPENSSL_DIR": cmd_env["CONDA_PREFIX"]}) - if unknown is not None: - try: - prefix_loc = unknown.index("--prefix") - prefix_dir = validate_path(unknown[prefix_loc + 1]) - if prefix_dir is not None: - install_dir = prefix_dir - unknown = unknown[:prefix_loc] + unknown[prefix_loc + 2 :] - except Exception: - pass + if unknown is None: + unknown = [] - install_dir = get_install_dir() + install_dir, install_args = get_install_dir(unknown) + + pip_install_cmd += install_args + pip_install_cmd += unknown if verbose: print(f"install_dir: {install_dir}") + print("install_args:", install_args) - if install_dir is not None: - pip_install_cmd += ["--root", "/", "--prefix", str(install_dir)] + # editable implies build_isolation = False + if editable or not build_isolation: + pip_install_cmd += ["--no-deps", "--no-build-isolation"] if editable: - # editable implies build_isolation = False - pip_install_cmd += ["--no-deps", "--no-build-isolation", "--editable"] cmd_env.update({"SETUPTOOLS_ENABLE_FEATURES": "legacy-editable"}) + cmake_flags += ["-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON"] + pip_install_cmd += ["--editable"] else: - if not build_isolation: - pip_install_cmd += ["--no-deps", "--no-build-isolation"] pip_install_cmd += ["--upgrade"] - if unknown is not None: - pip_install_cmd += unknown - pip_install_cmd += ["."] if verbose: pip_install_cmd += ["-vv"] - # Also use preexisting CMAKE_ARGS from conda if set - cmake_flags = cmd_env.get("CMAKE_ARGS", "").split(" ") - if debug or verbose: cmake_flags += [f"--log-level={'DEBUG' if debug else 'VERBOSE'}"] @@ -446,9 +495,6 @@ def validate_path(path): -DLegion_USE_HDF5={("ON" if hdf else "OFF")} -DLegion_USE_Python=ON -DLegion_Python_Version={pyversion} --DLegion_REDOP_COMPLEX=ON --DLegion_REDOP_HALF=ON --DLegion_BUILD_BINDINGS=ON -DLegion_BUILD_JUPYTER=ON -DLegion_EMBED_GASNet_CONFIGURE_ARGS="--with-ibv-max-hcas=8" """.splitlines() @@ -485,6 +531,10 @@ def validate_path(path): cmake_flags += ["-Dlegate_core_BUILD_TESTS=ON"] if build_examples: cmake_flags += ["-Dlegate_core_BUILD_EXAMPLES=ON"] + if len(install_args) > 0: + cmake_flags += [ + f'-DLegion_PYTHON_EXTRA_INSTALL_ARGS="{";".join(install_args)}"' + ] cmake_flags += extra_flags build_flags = [f"-j{str(thread_count)}"] From f07ce363f97aa8646dcdbd805b3d6990d34a39c0 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 10 Jul 2023 14:00:14 -0700 Subject: [PATCH 40/84] run the build commands via mamba run --- continuous_integration/home/coder/.local/bin/build-all | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/continuous_integration/home/coder/.local/bin/build-all b/continuous_integration/home/coder/.local/bin/build-all index 2a91bcb298..91b3d554a7 100755 --- a/continuous_integration/home/coder/.local/bin/build-all +++ b/continuous_integration/home/coder/.local/bin/build-all @@ -6,12 +6,12 @@ build_all() { cd ~/; if type conda >/dev/null 2>&1; then - . make-conda-env; + make-conda-env; fi - build-legate-cpp; - build-legate-wheel; - build-legate-conda; + mamba run -n "${DEFAULT_CONDA_ENV:-legate}" --cwd ~/ --live-stream build-legate-cpp; + mamba run -n "${DEFAULT_CONDA_ENV:-legate}" --cwd ~/ --live-stream build-legate-wheel; + mamba run -n "${DEFAULT_CONDA_ENV:-legate}" --cwd ~/ --live-stream build-legate-conda; } (build_all "$@"); From 9a47baf7ab6419a8c30692cc23e928d8aad61aff Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 10 Jul 2023 14:00:54 -0700 Subject: [PATCH 41/84] pass `-DLegion_Python_VERSION` to CMake --- conda/conda-build/build.sh | 2 +- .../home/coder/.local/bin/build-legate-cpp | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/conda/conda-build/build.sh b/conda/conda-build/build.sh index 7d198b3404..768497beaa 100644 --- a/conda/conda-build/build.sh +++ b/conda/conda-build/build.sh @@ -10,7 +10,7 @@ CMAKE_ARGS+=" -DBUILD_MARCH=haswell -DLegion_USE_OpenMP=ON -DLegion_USE_Python=ON --DLegion_BUILD_BINDINGS=ON" +-DLegion_Python_Version=$($PYTHON --version 2>&1 | cut -d' ' -f2 | cut -d'.' -f3 --complement)" # We rely on an environment variable to determine if we need to build cpu-only bits if [ -z "$CPU_ONLY" ]; then diff --git a/continuous_integration/home/coder/.local/bin/build-legate-cpp b/continuous_integration/home/coder/.local/bin/build-legate-cpp index 2c927c6e30..c52e9bfaff 100755 --- a/continuous_integration/home/coder/.local/bin/build-legate-cpp +++ b/continuous_integration/home/coder/.local/bin/build-legate-cpp @@ -11,10 +11,9 @@ build_legate_cpp() { cmake_args+=(-DLegion_CUDA_ARCH=all-major); cmake_args+=(-DLegion_NETWORKS=); cmake_args+=(-DLegion_USE_Python=ON); - cmake_args+=(-DLegion_REDOP_HALF=ON); - cmake_args+=(-DLegion_REDOP_COMPLEX=ON); + cmake_args+=(-DLegion_PYTHON_EXTRA_INSTALL_ARGS="--root;/;--prefix;\"\${CMAKE_INSTALL_PREFIX}\""); + cmake_args+=(-DLegion_Python_Version=$(python3 --version 2>&1 | cut -d' ' -f2 | cut -d'.' -f3 --complement)); cmake_args+=(-DLegion_BUILD_JUPYTER=ON); - cmake_args+=(-DLegion_BUILD_BINDINGS=ON); cmake_args+=(-DLegion_BOUNDS_CHECKS=${BOUNDS_CHECKS:-OFF}); cmake_args+=(-DLegion_EMBED_GASNet_CONFIGURE_ARGS=${GASNet_CONFIGURE_ARGS:---with-ibv-max-hcas=8}); cmake_args+=(-DLegion_MAX_DIM=${MAX_DIM:-4}); @@ -37,10 +36,6 @@ build_legate_cpp() { CMAKE_BUILD_PARALLEL_LEVEL=${JOBS:-$(nproc --ignore=1)} \ cmake -S ~/legate -B ~/legate/build "${cmake_args[@]}" -GNinja; - sed -i \ - 's/= unknown/= unknown + ["--prefix", args.prefix]/' \ - ~/legate/build/_deps/legion-src/bindings/python/setup.py; - sccache --show-stats; time OPENSSL_DIR="${CONDA_PREFIX}" \ From bf0e4145c8c82b617afa0ab52c29373be8ef3813 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 10 Jul 2023 14:01:19 -0700 Subject: [PATCH 42/84] add scripts to launch conda and pip based containers for replicating CI locally --- scripts/launch-devcontainer-conda.sh | 8 +++++ scripts/launch-devcontainer-pip.sh | 8 +++++ scripts/launch-devcontainer.sh | 54 ++++++++++++++++++++++++++++ scripts/replicate-ci.sh | 23 ++++++++++++ 4 files changed, 93 insertions(+) create mode 100755 scripts/launch-devcontainer-conda.sh create mode 100755 scripts/launch-devcontainer-pip.sh create mode 100755 scripts/launch-devcontainer.sh create mode 100755 scripts/replicate-ci.sh diff --git a/scripts/launch-devcontainer-conda.sh b/scripts/launch-devcontainer-conda.sh new file mode 100755 index 0000000000..f2f13447df --- /dev/null +++ b/scripts/launch-devcontainer-conda.sh @@ -0,0 +1,8 @@ +#! /usr/bin/env bash + +# cd to the repo root +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."; + +exec ./scripts/launch-devcontainer.sh \ + rapidsai/devcontainers:23.08-cpp-cuda11.8-mambaforge-ubuntu22.04 \ + $(if ! test -z "${@}"; then echo "${@}"; else echo "bash -li"; fi); diff --git a/scripts/launch-devcontainer-pip.sh b/scripts/launch-devcontainer-pip.sh new file mode 100755 index 0000000000..2904467e4a --- /dev/null +++ b/scripts/launch-devcontainer-pip.sh @@ -0,0 +1,8 @@ +#! /usr/bin/env bash + +# cd to the repo root +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."; + +exec ./scripts/launch-devcontainer.sh \ + rapidsai/devcontainers:23.08-cpp-rust-cuda11.8-ubuntu22.04 \ + $(if ! test -z "${@}"; then echo "${@}"; else echo "bash -li"; fi); diff --git a/scripts/launch-devcontainer.sh b/scripts/launch-devcontainer.sh new file mode 100755 index 0000000000..060ad0f7e3 --- /dev/null +++ b/scripts/launch-devcontainer.sh @@ -0,0 +1,54 @@ +#! /usr/bin/env bash + +launch_devcontainer() { + + set -Eeuox pipefail; + + # cd to the repo root + cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."; + + local cwd="$(pwd)"; + + local args=(); + args+=(--rm); + args+=(--tty); + args+=(--interactive); + args+=(--gpus all); + args+=(--user coder); + args+=(--workdir /home/coder/legate); + args+=(--entrypoint devcontainer-utils-post-attach-command-entrypoint); + + local vars=(); + vars+=(-e "PYTHONSAFEPATH=1"); + vars+=(-e "DEFAULT_CONDA_ENV=legate"); + vars+=(-e "PYTHONDONTWRITEBYTECODE=1"); + + vars+=(-e "HISTFILE=/home/coder/.local/state/._bash_history"); + + vars+=(-e "SCCACHE_REGION=us-east-2"); + vars+=(-e "SCCACHE_BUCKET=rapids-sccache-devs"); + vars+=(-e "SCCACHE_S3_KEY_PREFIX=legate-cunumeric-dev"); + vars+=(-e "VAULT_HOST=https://vault.ops.k8s.rapids.ai"); + + local vols=(); + vols+=(-v "${cwd}:/home/coder/legate"); + vols+=(-v "${cwd}/continuous_integration/home/coder/.local/bin/make-conda-env:/home/coder/.local/bin/make-conda-env"); + vols+=(-v "${cwd}/continuous_integration/home/coder/.local/bin/build-legate-cpp:/home/coder/.local/bin/build-legate-cpp"); + vols+=(-v "${cwd}/continuous_integration/home/coder/.local/bin/build-legate-wheel:/home/coder/.local/bin/build-legate-wheel"); + vols+=(-v "${cwd}/continuous_integration/home/coder/.local/bin/build-legate-conda:/home/coder/.local/bin/build-legate-conda"); + + for x in ".aws" ".cache" ".cargo" ".config" ".conda/pkgs" ".local/state" "legion"; do + if test -d "${cwd}/../${x}"; then + vols+=(-v "${cwd}/../${x}:/home/coder/${x}"); + fi + done + + if test -n "${SSH_AUTH_SOCK:-}"; then + vars+=(-e "SSH_AUTH_SOCK=/tmp/ssh-auth-sock"); + vols+=(-v "${SSH_AUTH_SOCK}:/tmp/ssh-auth-sock"); + fi + + exec docker run ${args[@]} ${vars[@]} ${vols[@]} ${@}; +} + +launch_devcontainer "$@"; diff --git a/scripts/replicate-ci.sh b/scripts/replicate-ci.sh new file mode 100755 index 0000000000..33b279f485 --- /dev/null +++ b/scripts/replicate-ci.sh @@ -0,0 +1,23 @@ +#! /usr/bin/env bash + +# cd to the repo root +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."; + +exec ./scripts/launch-devcontainer-conda.sh bash -licx ' +# Create build env + build libs and conda package +make-conda-env; +mamba run -n "${DEFAULT_CONDA_ENV:-legate}" --cwd ~/ --live-stream build-legate-cpp; +mamba run -n "${DEFAULT_CONDA_ENV:-legate}" --cwd ~/ --live-stream build-legate-wheel; +mamba run -n "${DEFAULT_CONDA_ENV:-legate}" --cwd ~/ --live-stream build-legate-conda; + +# Uncreate build env +mv ~/.conda/envs/${DEFAULT_CONDA_ENV:-legate}{,.bak}; + +# Recreate env from conda package +mamba create -n "${DEFAULT_CONDA_ENV:-legate}" \ + -c /tmp/out/legate_core -c conda-forge -c nvidia \ + legate-core pytest pytest-mock ipython jupyter_client; + +# Test package +mamba run -n legate --cwd ~/legate/tests/unit --live-stream pytest; +'; From 7a6741e52a4beae606fba6dc537c7de9557135b9 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 10 Jul 2023 14:05:22 -0700 Subject: [PATCH 43/84] update legate_core wheel name --- continuous_integration/home/coder/.local/bin/build-legate-conda | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous_integration/home/coder/.local/bin/build-legate-conda b/continuous_integration/home/coder/.local/bin/build-legate-conda index e610bffb64..99ef9c1912 100755 --- a/continuous_integration/home/coder/.local/bin/build-legate-conda +++ b/continuous_integration/home/coder/.local/bin/build-legate-conda @@ -58,7 +58,7 @@ sed -E -i "s@$HOME/\.conda/envs/legate@\$PREFIX@g" "\$PREFIX/share/Legion/cmake/ sed -E -i "s@$HOME/legate/build/_CPack_Packages/Linux/TGZ/legate_core-(.*)-Linux@\$PREFIX@g" "\$SP_DIR/legion_canonical_cffi.py"; # Install legate_core Python wheel -pip install --no-deps --root / --prefix "\$PREFIX" /tmp/out/legate.core-*.whl; +pip install --no-deps --root / --prefix "\$PREFIX" /tmp/out/legate_core-*.whl; # Legion leaves .egg-info files, which confuses conda trying to pick up the information # Remove them so legate-core is the only egg-info file added. From 01d22bcc8c775089ed1e22b567f9b3a231f1c6d3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 10 Jul 2023 21:01:57 +0000 Subject: [PATCH 44/84] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.py b/install.py index 72885b3390..dbedd7b58a 100755 --- a/install.py +++ b/install.py @@ -195,7 +195,7 @@ def get_arg(flag: str): # Otherwise fallback to regular site-packages? for site_pkgs in site.getsitepackages(): - install_dir = os.path.join(site_pkgs, '..', '..', '..') + install_dir = os.path.join(site_pkgs, "..", "..", "..") install_dir = os.path.realpath(install_dir) if install_dir != "/" and os.path.exists(install_dir): return (install_dir, ["--root", "/", "--prefix", install_dir]) From 900b4c44e29a19b6e47b3c8458be60449ea913f4 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 10 Jul 2023 14:14:47 -0700 Subject: [PATCH 45/84] add -y flag --- scripts/replicate-ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/replicate-ci.sh b/scripts/replicate-ci.sh index 33b279f485..4bc5e75ae2 100755 --- a/scripts/replicate-ci.sh +++ b/scripts/replicate-ci.sh @@ -14,7 +14,7 @@ mamba run -n "${DEFAULT_CONDA_ENV:-legate}" --cwd ~/ --live-stream build-legate- mv ~/.conda/envs/${DEFAULT_CONDA_ENV:-legate}{,.bak}; # Recreate env from conda package -mamba create -n "${DEFAULT_CONDA_ENV:-legate}" \ +mamba create -y -n "${DEFAULT_CONDA_ENV:-legate}" \ -c /tmp/out/legate_core -c conda-forge -c nvidia \ legate-core pytest pytest-mock ipython jupyter_client; From 5aeb1be7db88b7717b1994ff74985814731777bf Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 10 Jul 2023 16:38:02 -0700 Subject: [PATCH 46/84] remove custom .gitconfig --- .github/workflows/gh-build.yml | 1 - .github/workflows/gh-test.yml | 1 - continuous_integration/home/coder/.gitconfig | 3 --- 3 files changed, 5 deletions(-) delete mode 100644 continuous_integration/home/coder/.gitconfig diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index 41c7b33c3d..e91d2fba01 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -49,7 +49,6 @@ jobs: - name: Setup shell: bash -eo pipefail {0} run: | - cp -ar legate/continuous_integration/home/coder/.gitconfig /home/coder/; cp -ar legate/continuous_integration/home/coder/.local /home/coder/; mv legate /home/coder/legate; diff --git a/.github/workflows/gh-test.yml b/.github/workflows/gh-test.yml index f9481e53e4..d01dd23c61 100644 --- a/.github/workflows/gh-test.yml +++ b/.github/workflows/gh-test.yml @@ -54,7 +54,6 @@ jobs: - name: Setup shell: bash -eo pipefail {0} run: | - cp -ar legate/continuous_integration/home/coder/.gitconfig /home/coder/ cp -ar legate/continuous_integration/home/coder/.local /home/coder/ mv legate /home/coder/legate diff --git a/continuous_integration/home/coder/.gitconfig b/continuous_integration/home/coder/.gitconfig deleted file mode 100644 index dcb11a5e7e..0000000000 --- a/continuous_integration/home/coder/.gitconfig +++ /dev/null @@ -1,3 +0,0 @@ -[user] - email = users.noreply.github.com - name = anon From c46a73c8bdf47d61ee647ec1fee30f5957f12767 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 11 Jul 2023 11:35:31 -0700 Subject: [PATCH 47/84] restrict gcc version to 11.* --- scripts/generate-conda-envs.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/scripts/generate-conda-envs.py b/scripts/generate-conda-envs.py index 0693bf991b..0a6ae09159 100755 --- a/scripts/generate-conda-envs.py +++ b/scripts/generate-conda-envs.py @@ -71,16 +71,21 @@ def conda(self) -> Reqs: "pynvml", # tests ) - # gcc 11.3 is incompatible with nvcc <= 11.5. if ( self.compilers and self.os == "linux" - and (V(self.ctk_version) <= V("11.5")) ): - deps += ( - "gcc_linux-64<=11.2", - "gxx_linux-64<=11.2", - ) + # gcc 11.3 is incompatible with nvcc <= 11.5. + if V(self.ctk_version) <= V("11.5"): + deps += ( + "gcc_linux-64<=11.2", + "gxx_linux-64<=11.2", + ) + else: + deps += ( + "gcc_linux-64=11.*", + "gxx_linux-64=11.*", + ) return deps From 8295c429dd9afa7a0753f83a762c4fa9a9eef797 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 11 Jul 2023 18:35:56 +0000 Subject: [PATCH 48/84] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/generate-conda-envs.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/generate-conda-envs.py b/scripts/generate-conda-envs.py index 0a6ae09159..d311e76654 100755 --- a/scripts/generate-conda-envs.py +++ b/scripts/generate-conda-envs.py @@ -71,10 +71,7 @@ def conda(self) -> Reqs: "pynvml", # tests ) - if ( - self.compilers - and self.os == "linux" - ): + if self.compilers and self.os == "linux": # gcc 11.3 is incompatible with nvcc <= 11.5. if V(self.ctk_version) <= V("11.5"): deps += ( From 0def7f8e45368bd75133fb36579a63095dfc0e82 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 11 Jul 2023 15:37:54 -0700 Subject: [PATCH 49/84] bump legion commit sha --- cmake/versions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/versions.json b/cmake/versions.json index 155b2b838b..1d0adeb015 100644 --- a/cmake/versions.json +++ b/cmake/versions.json @@ -7,7 +7,7 @@ }, "Legion": { "git_url" : "https://gitlab.com/StanfordLegion/legion.git", - "git_tag" : "4396cde515e26fc1410a9a9ff54aa7fbc3c3315c" + "git_tag" : "c8fbf636dfae0ceb6977bf0a2734feba24d7ee57" } } } From ffcfc9cd16100a4f20015906d4266ccda3dfc685 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 25 Jul 2023 10:28:46 -0700 Subject: [PATCH 50/84] fix typo --- .github/workflows/gh-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index c1261ec8f5..87d542b8fa 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -69,7 +69,7 @@ jobs: rm -rf .creds; - name: Login to GitHub Container Registry + - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: registry: ghcr.io From 712001a1735ae3772ad0e9037754035d4f0eb7db Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 25 Jul 2023 10:47:31 -0700 Subject: [PATCH 51/84] print if image build has secrets --- .github/workflows/gh-build.yml | 8 ++++---- continuous_integration/Dockerfile | 5 +++++ continuous_integration/home/coder/.local/bin/build-all | 6 +++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index 87d542b8fa..5e50882f49 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -58,10 +58,10 @@ jobs: --progress=plain \ --build-arg USE_CUDA="${USE_CUDA}" \ --build-arg AWS_REGION="${AWS_REGION}" \ - --secret "id=GH_TOKEN,src=${PWD}/.creds/GH_TOKEN" \ - --secret "id=AWS_SESSION_TOKEN,src=${PWD}/.creds/AWS_SESSION_TOKEN" \ - --secret "id=AWS_ACCESS_KEY_ID,src=${PWD}/.creds/AWS_ACCESS_KEY_ID" \ - --secret "id=AWS_SECRET_ACCESS_KEY,src=${PWD}/.creds/AWS_SECRET_ACCESS_KEY" \ + --secret "id=GH_TOKEN,src=$(pwd)/.creds/GH_TOKEN" \ + --secret "id=AWS_SESSION_TOKEN,src=$(pwd)/.creds/AWS_SESSION_TOKEN" \ + --secret "id=AWS_ACCESS_KEY_ID,src=$(pwd)/.creds/AWS_ACCESS_KEY_ID" \ + --secret "id=AWS_SECRET_ACCESS_KEY,src=$(pwd)/.creds/AWS_SECRET_ACCESS_KEY" \ --tag=$IMAGE_NAME:${{ inputs.sha }} \ --label "git-commit=${{ inputs.sha }}" \ -f continuous_integration/Dockerfile \ diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile index b4f76101ae..8de961467c 100644 --- a/continuous_integration/Dockerfile +++ b/continuous_integration/Dockerfile @@ -55,6 +55,11 @@ if ! test -n "${GH_TOKEN:-}" \ export SCCACHE_BUCKET=""; fi +if test -n "${GH_TOKEN:-}"; then echo "Has GH_TOKEN"; fi +if test -n "${AWS_ACCESS_KEY_ID:-}"; then echo "Has AWS_ACCESS_KEY_ID"; fi +if test -n "${AWS_SECRET_ACCESS_KEY:-}"; then echo "Has AWS_SECRET_ACCESS_KEY"; fi +if test -n "${SCCACHE_BUCKET:-}"; then echo "Has SCCACHE_BUCKET"; fi + . devcontainer-utils-post-attach-command; build-all; diff --git a/continuous_integration/home/coder/.local/bin/build-all b/continuous_integration/home/coder/.local/bin/build-all index 12afd9f7ee..e79ae785e6 100755 --- a/continuous_integration/home/coder/.local/bin/build-all +++ b/continuous_integration/home/coder/.local/bin/build-all @@ -9,9 +9,9 @@ build_all() { set -euo pipefail; - mamba run -n "${DEFAULT_CONDA_ENV:-legate}" --cwd ~/ --live-stream build-legate-cpp; - mamba run -n "${DEFAULT_CONDA_ENV:-legate}" --cwd ~/ --live-stream build-legate-wheel; - mamba run -n "${DEFAULT_CONDA_ENV:-legate}" --cwd ~/ --live-stream build-legate-conda; + build-legate-cpp; + build-legate-wheel; + build-legate-conda; } (build_all "$@"); From bb106b4d2ce37debb292b1ffbd127d4b8a427c2b Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 25 Jul 2023 11:33:11 -0700 Subject: [PATCH 52/84] show sccache stats --- continuous_integration/Dockerfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile index 8de961467c..a06d0521f0 100644 --- a/continuous_integration/Dockerfile +++ b/continuous_integration/Dockerfile @@ -55,13 +55,10 @@ if ! test -n "${GH_TOKEN:-}" \ export SCCACHE_BUCKET=""; fi -if test -n "${GH_TOKEN:-}"; then echo "Has GH_TOKEN"; fi -if test -n "${AWS_ACCESS_KEY_ID:-}"; then echo "Has AWS_ACCESS_KEY_ID"; fi -if test -n "${AWS_SECRET_ACCESS_KEY:-}"; then echo "Has AWS_SECRET_ACCESS_KEY"; fi -if test -n "${SCCACHE_BUCKET:-}"; then echo "Has SCCACHE_BUCKET"; fi - . devcontainer-utils-post-attach-command; +sccache --show-stats; + build-all; EOF From 3cfc2668b002838398a24d24b9909abf33561c3d Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 25 Jul 2023 12:24:40 -0700 Subject: [PATCH 53/84] start sccache in the foreground for debugging --- continuous_integration/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile index a06d0521f0..42a9079f7e 100644 --- a/continuous_integration/Dockerfile +++ b/continuous_integration/Dockerfile @@ -57,7 +57,8 @@ fi . devcontainer-utils-post-attach-command; -sccache --show-stats; +sccache --stop-server >/dev/null 2>&1 || true; +SCCACHE_NO_DAEMON=1 SCCACHE_LOG=debug sccache --show-stats; build-all; From 0c1193073c2b554fa79f761e8bfde45c4e33516e Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 25 Jul 2023 12:26:31 -0700 Subject: [PATCH 54/84] don't redefine sccache_region --- continuous_integration/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile index 42a9079f7e..c8ee057984 100644 --- a/continuous_integration/Dockerfile +++ b/continuous_integration/Dockerfile @@ -9,17 +9,17 @@ ENV BUILD_MARCH=${BUILD_MARCH} ARG AWS_REGION ENV AWS_REGION=${AWS_REGION} -ENV SCCACHE_REGION=${AWS_REGION:-${SCCACHE_REGION}} ENV DEFAULT_CONDA_ENV=legate ENV PYTHONDONTWRITEBYTECODE=1 -ENV SCCACHE_REGION="us-east-2" +ENV SCCACHE_REGION=${AWS_REGION:-"us-east-2"} ENV SCCACHE_BUCKET="rapids-sccache-devs" ENV SCCACHE_S3_KEY_PREFIX=legate-cunumeric-dev ENV VAULT_HOST=https://vault.ops.k8s.rapids.ai # 8 hours ENV VAULT_S3_TTL="28800s" + SHELL ["/bin/bash", "-Eeox", "pipefail", "-c"] USER coder @@ -59,6 +59,7 @@ fi sccache --stop-server >/dev/null 2>&1 || true; SCCACHE_NO_DAEMON=1 SCCACHE_LOG=debug sccache --show-stats; +sccache --show-stats; build-all; From d319801304e177140a74ff5a95a71ba70cff8e2b Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 25 Jul 2023 13:02:47 -0700 Subject: [PATCH 55/84] define SCCACHE_S3_USE_SSL=true --- continuous_integration/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile index c8ee057984..93c52a5944 100644 --- a/continuous_integration/Dockerfile +++ b/continuous_integration/Dockerfile @@ -12,6 +12,7 @@ ENV AWS_REGION=${AWS_REGION} ENV DEFAULT_CONDA_ENV=legate ENV PYTHONDONTWRITEBYTECODE=1 +ENV SCCACHE_S3_USE_SSL=true ENV SCCACHE_REGION=${AWS_REGION:-"us-east-2"} ENV SCCACHE_BUCKET="rapids-sccache-devs" ENV SCCACHE_S3_KEY_PREFIX=legate-cunumeric-dev From 898d772ef5dffa96920bd6bf35b55b463759eb4c Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 25 Jul 2023 13:14:05 -0700 Subject: [PATCH 56/84] dockerignore is not the same as gitignore --- .dockerignore | 45 ++++++++++++++++++++++++++++++++++++++++++++- .gitignore | 2 -- 2 files changed, 44 insertions(+), 3 deletions(-) mode change 120000 => 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore deleted file mode 120000 index 3e4e48b0b5..0000000000 --- a/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -.gitignore \ No newline at end of file diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..fcefbdfefc --- /dev/null +++ b/.dockerignore @@ -0,0 +1,44 @@ +*~ +\#*# +*/typestubs/* +.#* +.DS_Store +*.a +*.d +*.o +*.so +*.gz +*.gcno +*.gcda +*.gcov +*.fluid +*.pyc +*.swp +*.log +*.bbl +*.blg +*.aux +*.json +*.dylib +install_info.py +/dist +/build +/legion +/install* +/_skbuild +config.mk +/docs/legate/core/build +/docs/legate/core/source/api/generated +*.egg-info +.creds +.cache +.coverage +.vscode +_cmake_test_compile +!cmake/versions.json +legate.core.code-workspace +*.prof +.legate-test-last-failed +out/ +.github/ +continuous_integration/Dockerfile diff --git a/.gitignore b/.gitignore index fcefbdfefc..29e4649b7c 100644 --- a/.gitignore +++ b/.gitignore @@ -40,5 +40,3 @@ legate.core.code-workspace *.prof .legate-test-last-failed out/ -.github/ -continuous_integration/Dockerfile From 75bd4ea8c1ac2ad2f6a0c05778fdd37fea7258fc Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 25 Jul 2023 13:17:13 -0700 Subject: [PATCH 57/84] mount .aws dir from build context --- .github/workflows/gh-build.yml | 25 +++++++++++++++++++++---- continuous_integration/Dockerfile | 3 ++- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index 5e50882f49..bd1de42592 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -48,26 +48,43 @@ jobs: echo BUILD_TARGET: ${{ inputs.build-target }} echo USE_CUDA: ${USE_CUDA} - mkdir .creds; - cat <<< "${AWS_SESSION_TOKEN}" > .creds/AWS_SESSION_TOKEN; + mkdir .{aws,creds}; + cat <<< "${AWS_ACCESS_KEY_ID}" > .creds/AWS_ACCESS_KEY_ID; + cat <<< "${AWS_SESSION_TOKEN}" > .creds/AWS_SESSION_TOKEN; cat <<< "${AWS_SECRET_ACCESS_KEY}" > .creds/AWS_SECRET_ACCESS_KEY; cat <<< "${{ secrets.PERSONAL_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}" > .creds/GH_TOKEN; + cat < .aws/config + [default] + bucket=rapids-sccache-devs + region=${AWS_REGION:-"us-east-2"} + EOF + + cat < .aws/credentials + [default] + aws_access_key_id=${AWS_ACCESS_KEY_ID} + aws_session_token=${AWS_SESSION_TOKEN} + aws_secret_access_key=${AWS_SECRET_ACCESS_KEY} + EOF + + chmod 0600 .aws/credentials; + docker build \ + --network host \ --progress=plain \ --build-arg USE_CUDA="${USE_CUDA}" \ --build-arg AWS_REGION="${AWS_REGION}" \ --secret "id=GH_TOKEN,src=$(pwd)/.creds/GH_TOKEN" \ - --secret "id=AWS_SESSION_TOKEN,src=$(pwd)/.creds/AWS_SESSION_TOKEN" \ --secret "id=AWS_ACCESS_KEY_ID,src=$(pwd)/.creds/AWS_ACCESS_KEY_ID" \ + --secret "id=AWS_SESSION_TOKEN,src=$(pwd)/.creds/AWS_SESSION_TOKEN" \ --secret "id=AWS_SECRET_ACCESS_KEY,src=$(pwd)/.creds/AWS_SECRET_ACCESS_KEY" \ --tag=$IMAGE_NAME:${{ inputs.sha }} \ --label "git-commit=${{ inputs.sha }}" \ -f continuous_integration/Dockerfile \ . - rm -rf .creds; + rm -rf .{aws,creds}; - name: Login to GitHub Container Registry uses: docker/login-action@v2 diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile index 93c52a5944..90012dc9b6 100644 --- a/continuous_integration/Dockerfile +++ b/continuous_integration/Dockerfile @@ -40,7 +40,8 @@ RUN make-conda-env FROM setup as build # Mount in secrets -RUN --mount=type=secret,id=GH_TOKEN,uid=1000,gid=1000 \ +RUN --mount=type=bind,source=.aws,target=/home/coder/.aws \ + --mount=type=secret,id=GH_TOKEN,uid=1000,gid=1000 \ --mount=type=secret,id=AWS_SESSION_TOKEN,uid=1000,gid=1000 \ --mount=type=secret,id=AWS_ACCESS_KEY_ID,uid=1000,gid=1000 \ --mount=type=secret,id=AWS_SECRET_ACCESS_KEY,uid=1000,gid=1000 \ From e156cd53456f48c148d3e5d064d38d588f4bb7bf Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 25 Jul 2023 13:46:57 -0700 Subject: [PATCH 58/84] add AWS_DEFAULT_REGION, remove .aws mount --- .github/workflows/gh-build.yml | 20 +++----------------- continuous_integration/Dockerfile | 23 +++++++++++++++++------ 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index bd1de42592..be533ad2be 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -48,33 +48,19 @@ jobs: echo BUILD_TARGET: ${{ inputs.build-target }} echo USE_CUDA: ${USE_CUDA} - mkdir .{aws,creds}; + mkdir .creds; cat <<< "${AWS_ACCESS_KEY_ID}" > .creds/AWS_ACCESS_KEY_ID; cat <<< "${AWS_SESSION_TOKEN}" > .creds/AWS_SESSION_TOKEN; cat <<< "${AWS_SECRET_ACCESS_KEY}" > .creds/AWS_SECRET_ACCESS_KEY; cat <<< "${{ secrets.PERSONAL_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}" > .creds/GH_TOKEN; - cat < .aws/config - [default] - bucket=rapids-sccache-devs - region=${AWS_REGION:-"us-east-2"} - EOF - - cat < .aws/credentials - [default] - aws_access_key_id=${AWS_ACCESS_KEY_ID} - aws_session_token=${AWS_SESSION_TOKEN} - aws_secret_access_key=${AWS_SECRET_ACCESS_KEY} - EOF - - chmod 0600 .aws/credentials; - docker build \ --network host \ --progress=plain \ --build-arg USE_CUDA="${USE_CUDA}" \ --build-arg AWS_REGION="${AWS_REGION}" \ + --build-arg AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" \ --secret "id=GH_TOKEN,src=$(pwd)/.creds/GH_TOKEN" \ --secret "id=AWS_ACCESS_KEY_ID,src=$(pwd)/.creds/AWS_ACCESS_KEY_ID" \ --secret "id=AWS_SESSION_TOKEN,src=$(pwd)/.creds/AWS_SESSION_TOKEN" \ @@ -84,7 +70,7 @@ jobs: -f continuous_integration/Dockerfile \ . - rm -rf .{aws,creds}; + rm -rf .creds; - name: Login to GitHub Container Registry uses: docker/login-action@v2 diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile index 90012dc9b6..27791364e7 100644 --- a/continuous_integration/Dockerfile +++ b/continuous_integration/Dockerfile @@ -7,13 +7,16 @@ ENV USE_CUDA=${USE_CUDA} ARG BUILD_MARCH=nocona ENV BUILD_MARCH=${BUILD_MARCH} +ARG AWS_DEFAULT_REGION +ENV AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-"us-east-2"} + ARG AWS_REGION -ENV AWS_REGION=${AWS_REGION} +ENV AWS_REGION=${AWS_REGION:-${AWS_DEFAULT_REGION}} ENV DEFAULT_CONDA_ENV=legate ENV PYTHONDONTWRITEBYTECODE=1 ENV SCCACHE_S3_USE_SSL=true -ENV SCCACHE_REGION=${AWS_REGION:-"us-east-2"} +ENV SCCACHE_REGION=${AWS_REGION} ENV SCCACHE_BUCKET="rapids-sccache-devs" ENV SCCACHE_S3_KEY_PREFIX=legate-cunumeric-dev ENV VAULT_HOST=https://vault.ops.k8s.rapids.ai @@ -40,10 +43,9 @@ RUN make-conda-env FROM setup as build # Mount in secrets -RUN --mount=type=bind,source=.aws,target=/home/coder/.aws \ - --mount=type=secret,id=GH_TOKEN,uid=1000,gid=1000 \ - --mount=type=secret,id=AWS_SESSION_TOKEN,uid=1000,gid=1000 \ +RUN --mount=type=secret,id=GH_TOKEN,uid=1000,gid=1000 \ --mount=type=secret,id=AWS_ACCESS_KEY_ID,uid=1000,gid=1000 \ + --mount=type=secret,id=AWS_SESSION_TOKEN,uid=1000,gid=1000 \ --mount=type=secret,id=AWS_SECRET_ACCESS_KEY,uid=1000,gid=1000 \ </dev/null 2>&1 || true; -SCCACHE_NO_DAEMON=1 SCCACHE_LOG=debug sccache --show-stats; + +SCCACHE_LOG=trace \ +SCCACHE_NO_DAEMON=1 \ +SCCACHE_REGION="${SCCACHE_REGION:-}" \ +SCCACHE_BUCKET="${SCCACHE_BUCKET:-}" \ +AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-}" \ +AWS_SESSION_TOKEN="${AWS_SESSION_TOKEN:-}" \ +AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-}" \ +sccache --show-stats; + sccache --show-stats; build-all; From 8bb3a9a5ab2f397f8e8793f1c35d68a43b575f1d Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 25 Jul 2023 14:28:47 -0700 Subject: [PATCH 59/84] ensure new creds have propagated --- .github/workflows/gh-build.yml | 4 +-- continuous_integration/Dockerfile | 30 ++++++------------- .../home/coder/.local/bin/build-legate-cpp | 1 - 3 files changed, 10 insertions(+), 25 deletions(-) diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index be533ad2be..0e47459631 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -56,11 +56,9 @@ jobs: cat <<< "${{ secrets.PERSONAL_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}" > .creds/GH_TOKEN; docker build \ - --network host \ - --progress=plain \ + --progress plain \ --build-arg USE_CUDA="${USE_CUDA}" \ --build-arg AWS_REGION="${AWS_REGION}" \ - --build-arg AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" \ --secret "id=GH_TOKEN,src=$(pwd)/.creds/GH_TOKEN" \ --secret "id=AWS_ACCESS_KEY_ID,src=$(pwd)/.creds/AWS_ACCESS_KEY_ID" \ --secret "id=AWS_SESSION_TOKEN,src=$(pwd)/.creds/AWS_SESSION_TOKEN" \ diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile index 27791364e7..9968ddbe1a 100644 --- a/continuous_integration/Dockerfile +++ b/continuous_integration/Dockerfile @@ -15,7 +15,6 @@ ENV AWS_REGION=${AWS_REGION:-${AWS_DEFAULT_REGION}} ENV DEFAULT_CONDA_ENV=legate ENV PYTHONDONTWRITEBYTECODE=1 -ENV SCCACHE_S3_USE_SSL=true ENV SCCACHE_REGION=${AWS_REGION} ENV SCCACHE_BUCKET="rapids-sccache-devs" ENV SCCACHE_S3_KEY_PREFIX=legate-cunumeric-dev @@ -23,7 +22,6 @@ ENV VAULT_HOST=https://vault.ops.k8s.rapids.ai # 8 hours ENV VAULT_S3_TTL="28800s" - SHELL ["/bin/bash", "-Eeox", "pipefail", "-c"] USER coder @@ -35,12 +33,7 @@ COPY --chown=coder:coder continuous_integration/home/coder/.local/bin/* /home/co COPY --chown=coder:coder . /home/coder/legate #--------------------------------------------------- -FROM stage0 as setup - -RUN make-conda-env - -#--------------------------------------------------- -FROM setup as build +FROM stage0 as build # Mount in secrets RUN --mount=type=secret,id=GH_TOKEN,uid=1000,gid=1000 \ @@ -53,26 +46,21 @@ RUN --mount=type=secret,id=GH_TOKEN,uid=1000,gid=1000 \ eval "export $(find /run/secrets/ -type f -exec bash -c 'echo ${0/\/run\/secrets\//}=$(<${0})' {} \;)"; { set -xo history; } 2>/dev/null; -if ! test -n "${GH_TOKEN:-}" \ -&& ! test -n "${AWS_ACCESS_KEY_ID:-}" \ -&& ! test -n "${AWS_SECRET_ACCESS_KEY:-}" ; then +if grep -qE "^$" <<< "${GH_TOKEN:-}" \ +&& grep -qE "^$" <<< "${AWS_ACCESS_KEY_ID:-}" \ +&& grep -qE "^$" <<< "${AWS_SECRET_ACCESS_KEY:-}"; then export SCCACHE_BUCKET=""; fi . devcontainer-utils-post-attach-command; -sccache --stop-server >/dev/null 2>&1 || true; +devcontainer-utils-vault-s3-creds-propagate || true; + +SCCACHE_NO_DAEMON=1 SCCACHE_LOG=trace sccache --show-stats; -SCCACHE_LOG=trace \ -SCCACHE_NO_DAEMON=1 \ -SCCACHE_REGION="${SCCACHE_REGION:-}" \ -SCCACHE_BUCKET="${SCCACHE_BUCKET:-}" \ -AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-}" \ -AWS_SESSION_TOKEN="${AWS_SESSION_TOKEN:-}" \ -AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-}" \ -sccache --show-stats; +sccache --start-server >/dev/null 2>&1 || true; -sccache --show-stats; +make-conda-env; build-all; diff --git a/continuous_integration/home/coder/.local/bin/build-legate-cpp b/continuous_integration/home/coder/.local/bin/build-legate-cpp index 3213210514..f0cb38ff44 100755 --- a/continuous_integration/home/coder/.local/bin/build-legate-cpp +++ b/continuous_integration/home/coder/.local/bin/build-legate-cpp @@ -27,7 +27,6 @@ build_legate_cpp() { cmake_args+=(-DCMAKE_BUILD_PARALLEL_LEVEL=${JOBS:-$(nproc --ignore=1)}); cmake_args+=(${@}); - sccache --stop-server >/dev/null 2>&1 || true; sccache --show-stats; rm -rf ~/legate/build; From ad27fa17f6aed86889229521abba382fcced9987 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 25 Jul 2023 14:34:14 -0700 Subject: [PATCH 60/84] exit if credentials don't propagate --- continuous_integration/Dockerfile | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile index 9968ddbe1a..4b1acbc241 100644 --- a/continuous_integration/Dockerfile +++ b/continuous_integration/Dockerfile @@ -54,11 +54,13 @@ fi . devcontainer-utils-post-attach-command; -devcontainer-utils-vault-s3-creds-propagate || true; - -SCCACHE_NO_DAEMON=1 SCCACHE_LOG=trace sccache --show-stats; - -sccache --start-server >/dev/null 2>&1 || true; +if ! grep -qE "^$" <<< "${GH_TOKEN:-}" \ +|| ! grep -qE "^$" <<< "${AWS_ACCESS_KEY_ID:-}" \ +|| ! grep -qE "^$" <<< "${AWS_SECRET_ACCESS_KEY:-}"; then + devcontainer-utils-vault-s3-creds-propagate; + SCCACHE_NO_DAEMON=1 SCCACHE_LOG=trace sccache --show-stats; + sccache --start-server >/dev/null 2>&1 || true; +fi make-conda-env; From ec4c37379727281861c8675d2cb2cc66ce1322b8 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 25 Jul 2023 14:43:39 -0700 Subject: [PATCH 61/84] print debugging info --- continuous_integration/Dockerfile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile index 4b1acbc241..e12d648edc 100644 --- a/continuous_integration/Dockerfile +++ b/continuous_integration/Dockerfile @@ -52,16 +52,17 @@ if grep -qE "^$" <<< "${GH_TOKEN:-}" \ export SCCACHE_BUCKET=""; fi -. devcontainer-utils-post-attach-command; - -if ! grep -qE "^$" <<< "${GH_TOKEN:-}" \ -|| ! grep -qE "^$" <<< "${AWS_ACCESS_KEY_ID:-}" \ -|| ! grep -qE "^$" <<< "${AWS_SECRET_ACCESS_KEY:-}"; then - devcontainer-utils-vault-s3-creds-propagate; +if ! grep -qE "^$" <<< "${SCCACHE_BUCKET:-}" \ +&& ! grep -qE "^$" <<< "${AWS_ACCESS_KEY_ID:-}" \ +&& ! grep -qE "^$" <<< "${AWS_SECRET_ACCESS_KEY:-}"; then + devcontainer_utils_debug=1 \ + devcontainer-utils-vault-s3-creds-propagate; SCCACHE_NO_DAEMON=1 SCCACHE_LOG=trace sccache --show-stats; sccache --start-server >/dev/null 2>&1 || true; fi +. devcontainer-utils-post-attach-command; + make-conda-env; build-all; From 17e87eb4b39d099efeb126a4633c3268a4bbbd2f Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 25 Jul 2023 14:46:22 -0700 Subject: [PATCH 62/84] export envvars --- continuous_integration/Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile index e12d648edc..1a03e70a51 100644 --- a/continuous_integration/Dockerfile +++ b/continuous_integration/Dockerfile @@ -52,6 +52,12 @@ if grep -qE "^$" <<< "${GH_TOKEN:-}" \ export SCCACHE_BUCKET=""; fi +if ! grep -qE "^$" <<< "${SCCACHE_BUCKET:-}"; then export SCCACHE_BUCKET="${SCCACHE_BUCKET:-}"; fi +if ! grep -qE "^$" <<< "${SCCACHE_REGION:-}"; then export SCCACHE_REGION="${SCCACHE_REGION:-}"; fi +if ! grep -qE "^$" <<< "${AWS_ACCESS_KEY_ID:-}"; then export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-}"; fi +if ! grep -qE "^$" <<< "${AWS_SESSION_TOKEN:-}"; then export AWS_SESSION_TOKEN="${AWS_SESSION_TOKEN:-}"; fi +if ! grep -qE "^$" <<< "${AWS_SECRET_ACCESS_KEY:-}"; then export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-}"; fi + if ! grep -qE "^$" <<< "${SCCACHE_BUCKET:-}" \ && ! grep -qE "^$" <<< "${AWS_ACCESS_KEY_ID:-}" \ && ! grep -qE "^$" <<< "${AWS_SECRET_ACCESS_KEY:-}"; then From db3e743e11fed8f5ecf0888c00290a60917fb5ac Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 25 Jul 2023 15:05:22 -0700 Subject: [PATCH 63/84] export each secret individually --- .github/workflows/gh-build.yml | 2 +- continuous_integration/Dockerfile | 14 ++++++-------- .../home/coder/.local/bin/build-all | 6 +++--- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index 0e47459631..507ebade55 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -93,7 +93,7 @@ jobs: run: | mkdir -p artifacts docker run -v "$(pwd)/artifacts:/artifacts" --rm -t -u coder \ - $IMAGE_NAME:${{ inputs.sha }} cp -ar ~/.artifacts/* /artifacts + $IMAGE_NAME:${{ inputs.sha }} cp -ar /home/coder/.artifacts/* /artifacts echo --------- DOCKER HISTORY START ----------- docker history $IMAGE_NAME:${{ inputs.sha }} echo --------- DOCKER HISTORY END ----------- diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile index 1a03e70a51..92d0a220c6 100644 --- a/continuous_integration/Dockerfile +++ b/continuous_integration/Dockerfile @@ -43,7 +43,11 @@ RUN --mount=type=secret,id=GH_TOKEN,uid=1000,gid=1000 \ </dev/null; -eval "export $(find /run/secrets/ -type f -exec bash -c 'echo ${0/\/run\/secrets\//}=$(<${0})' {} \;)"; +eval "$( \ + find /run/secrets/ -type f -exec \ + bash -c 'echo ${0/\/run\/secrets\//}=$(<${0})' {} \; \ + | xargs -r -d'\n' -I% echo -n export %\; \ +)"; { set -xo history; } 2>/dev/null; if grep -qE "^$" <<< "${GH_TOKEN:-}" \ @@ -52,17 +56,11 @@ if grep -qE "^$" <<< "${GH_TOKEN:-}" \ export SCCACHE_BUCKET=""; fi -if ! grep -qE "^$" <<< "${SCCACHE_BUCKET:-}"; then export SCCACHE_BUCKET="${SCCACHE_BUCKET:-}"; fi -if ! grep -qE "^$" <<< "${SCCACHE_REGION:-}"; then export SCCACHE_REGION="${SCCACHE_REGION:-}"; fi -if ! grep -qE "^$" <<< "${AWS_ACCESS_KEY_ID:-}"; then export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-}"; fi -if ! grep -qE "^$" <<< "${AWS_SESSION_TOKEN:-}"; then export AWS_SESSION_TOKEN="${AWS_SESSION_TOKEN:-}"; fi -if ! grep -qE "^$" <<< "${AWS_SECRET_ACCESS_KEY:-}"; then export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-}"; fi - if ! grep -qE "^$" <<< "${SCCACHE_BUCKET:-}" \ && ! grep -qE "^$" <<< "${AWS_ACCESS_KEY_ID:-}" \ && ! grep -qE "^$" <<< "${AWS_SECRET_ACCESS_KEY:-}"; then devcontainer_utils_debug=1 \ - devcontainer-utils-vault-s3-creds-propagate; + devcontainer-utils-vault-s3-creds-propagate; SCCACHE_NO_DAEMON=1 SCCACHE_LOG=trace sccache --show-stats; sccache --start-server >/dev/null 2>&1 || true; fi diff --git a/continuous_integration/home/coder/.local/bin/build-all b/continuous_integration/home/coder/.local/bin/build-all index e79ae785e6..12afd9f7ee 100755 --- a/continuous_integration/home/coder/.local/bin/build-all +++ b/continuous_integration/home/coder/.local/bin/build-all @@ -9,9 +9,9 @@ build_all() { set -euo pipefail; - build-legate-cpp; - build-legate-wheel; - build-legate-conda; + mamba run -n "${DEFAULT_CONDA_ENV:-legate}" --cwd ~/ --live-stream build-legate-cpp; + mamba run -n "${DEFAULT_CONDA_ENV:-legate}" --cwd ~/ --live-stream build-legate-wheel; + mamba run -n "${DEFAULT_CONDA_ENV:-legate}" --cwd ~/ --live-stream build-legate-conda; } (build_all "$@"); From 0f452175a9a580d445078478cb8b5927d730c39e Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 25 Jul 2023 15:25:04 -0700 Subject: [PATCH 64/84] fix bad merge --- conda/conda-build/build.sh | 64 +++++++++++++++++++---- conda/conda-build/conda_build_config.yaml | 20 +++---- 2 files changed, 65 insertions(+), 19 deletions(-) diff --git a/conda/conda-build/build.sh b/conda/conda-build/build.sh index 1e3c4f89f7..768497beaa 100644 --- a/conda/conda-build/build.sh +++ b/conda/conda-build/build.sh @@ -1,11 +1,57 @@ -# Install legate_core C++ libs -tar -C "$PREFIX" --exclude="*.a" --strip-components=1 -xf ~/.artifacts/legate_core-*-Linux.tar.gz; -sed -E -i "s@/home/coder/\.conda/envs/legate@$PREFIX@g" "$PREFIX/share/Legion/cmake/LegionConfigCommon.cmake"; -sed -E -i "s@/home/coder/legate/build/_CPack_Packages/Linux/TGZ/legate_core-(.*)-Linux@$PREFIX@g" "$SP_DIR/legion_canonical_cffi.py"; +#!/bin/bash -# Install legate_core Python wheel -pip install --no-deps --root / --prefix "$PREFIX" ~/.artifacts/legate_core-*.whl; +# Rewrite conda's -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY to +# -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH +CMAKE_ARGS="$(echo "$CMAKE_ARGS" | sed -r "s@_INCLUDE=ONLY@_INCLUDE=BOTH@g")" -# Legion leaves .egg-info files, which confuses conda trying to pick up the information -# Remove them so legate-core is the only egg-info file added. -rm -rf "$SP_DIR"/legion*egg-info; +# Add our options to conda's CMAKE_ARGS +CMAKE_ARGS+=" +--log-level=VERBOSE +-DBUILD_MARCH=haswell +-DLegion_USE_OpenMP=ON +-DLegion_USE_Python=ON +-DLegion_Python_Version=$($PYTHON --version 2>&1 | cut -d' ' -f2 | cut -d'.' -f3 --complement)" + +# We rely on an environment variable to determine if we need to build cpu-only bits +if [ -z "$CPU_ONLY" ]; then + CMAKE_ARGS+=" +-DLegion_USE_CUDA=ON +-DLegion_CUDA_ARCH:LIST=60-real;70-real;75-real;80-real;90 +" +fi + +# Do not compile with NDEBUG until Legion handles it without warnings +export CFLAGS="-UNDEBUG" +export CXXFLAGS="-UNDEBUG" +export CPPFLAGS="-UNDEBUG" +export CUDAFLAGS="-UNDEBUG" +export CMAKE_GENERATOR=Ninja +export CUDAHOSTCXX=${CXX} +export OPENSSL_DIR="$CONDA_PREFIX" + +echo "Build starting on $(date)" + +cmake -S . -B build ${CMAKE_ARGS} +cmake --build build -j$CPU_COUNT +cmake --install build --prefix "$PREFIX" + +CMAKE_ARGS=" +-DFIND_LEGATE_CORE_CPP=ON +-Dlegate_core_ROOT=$PREFIX +" + +SKBUILD_BUILD_OPTIONS=-j$CPU_COUNT \ +$PYTHON -m pip install \ + --root / \ + --no-deps \ + --prefix "$PREFIX" \ + --no-build-isolation \ + --cache-dir "$PIP_CACHE_DIR" \ + --disable-pip-version-check \ + . -vv + +echo "Build ending on $(date)" + +# Legion leaves an egg-info file which will confuse conda trying to pick up the information +# Remove it so the legate-core is the only egg-info file added +rm -rf $SP_DIR/legion*egg-info diff --git a/conda/conda-build/conda_build_config.yaml b/conda/conda-build/conda_build_config.yaml index bda2048916..a508a6ed16 100644 --- a/conda/conda-build/conda_build_config.yaml +++ b/conda/conda-build/conda_build_config.yaml @@ -1,14 +1,14 @@ -numpy: - - 1.22 +gpu_enabled: + - true + - false + python: - - "3.10" + - 3.9 + - 3.10 + - 3.11 + numpy_version: - - ">=1.22,<1.25" -use_local_path: - - "true" -gpu_enabled: - - "true" + - ">=1.22" + cmake_version: - ">=3.20.1,!=3.23.0" -package_version: - - "23.07.00" From 778ba8ae559faf43161d989f4dd997b57dbe0487 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 25 Jul 2023 15:29:56 -0700 Subject: [PATCH 65/84] fix copy artifacts step --- .github/workflows/gh-build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index 507ebade55..50e2b86d62 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -63,7 +63,7 @@ jobs: --secret "id=AWS_ACCESS_KEY_ID,src=$(pwd)/.creds/AWS_ACCESS_KEY_ID" \ --secret "id=AWS_SESSION_TOKEN,src=$(pwd)/.creds/AWS_SESSION_TOKEN" \ --secret "id=AWS_SECRET_ACCESS_KEY,src=$(pwd)/.creds/AWS_SECRET_ACCESS_KEY" \ - --tag=$IMAGE_NAME:${{ inputs.sha }} \ + --tag $IMAGE_NAME:${{ inputs.sha }} \ --label "git-commit=${{ inputs.sha }}" \ -f continuous_integration/Dockerfile \ . @@ -91,12 +91,12 @@ jobs: - name: Copy artifacts back to the host run: | - mkdir -p artifacts - docker run -v "$(pwd)/artifacts:/artifacts" --rm -t -u coder \ - $IMAGE_NAME:${{ inputs.sha }} cp -ar /home/coder/.artifacts/* /artifacts + mkdir -p out + docker run -v "$(pwd)/out:/out" --rm -t -u coder \ + $IMAGE_NAME:${{ inputs.sha }} cp -ar /home/coder/.artifacts /out/ echo --------- DOCKER HISTORY START ----------- docker history $IMAGE_NAME:${{ inputs.sha }} - echo --------- DOCKER HISTORY END ----------- + echo --------- DOCKER HISTORY END ------------- - name: Display structure of workdir run: ls -R @@ -105,4 +105,4 @@ jobs: uses: actions/upload-artifact@v3 with: name: "legate.core-${{ inputs.build-target }}-${{ inputs.sha }}" - path: artifacts + path: out/.artifacts From 263bb62ca1f436ebce641b9bc5a80bc62ff9f888 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 25 Jul 2023 15:36:44 -0700 Subject: [PATCH 66/84] remove debugging code --- continuous_integration/Dockerfile | 9 --------- 1 file changed, 9 deletions(-) diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile index 92d0a220c6..3447daad42 100644 --- a/continuous_integration/Dockerfile +++ b/continuous_integration/Dockerfile @@ -56,15 +56,6 @@ if grep -qE "^$" <<< "${GH_TOKEN:-}" \ export SCCACHE_BUCKET=""; fi -if ! grep -qE "^$" <<< "${SCCACHE_BUCKET:-}" \ -&& ! grep -qE "^$" <<< "${AWS_ACCESS_KEY_ID:-}" \ -&& ! grep -qE "^$" <<< "${AWS_SECRET_ACCESS_KEY:-}"; then - devcontainer_utils_debug=1 \ - devcontainer-utils-vault-s3-creds-propagate; - SCCACHE_NO_DAEMON=1 SCCACHE_LOG=trace sccache --show-stats; - sccache --start-server >/dev/null 2>&1 || true; -fi - . devcontainer-utils-post-attach-command; make-conda-env; From 1c1fc06c8821f60515316afb693532beaa4fd9c0 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 25 Jul 2023 15:36:56 -0700 Subject: [PATCH 67/84] install mypy --- scripts/replicate-ci.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/replicate-ci.sh b/scripts/replicate-ci.sh index 8b2e8d989c..7cecaf5399 100755 --- a/scripts/replicate-ci.sh +++ b/scripts/replicate-ci.sh @@ -16,8 +16,14 @@ mv ~/.conda/envs/${DEFAULT_CONDA_ENV:-legate}{,.bak}; # Recreate env from conda package mamba create -y -n "${DEFAULT_CONDA_ENV:-legate}" \ - -c ~/.artifacts/legate_core -c conda-forge -c nvidia \ - legate-core pytest pytest-mock ipython jupyter_client; + `# local legate_core channel first` \ + -c ~/.artifacts/legate_core \ + `# then conda-forge` \ + -c conda-forge \ + `# and finally nvidia` \ + -c nvidia \ + legate-core \ + mypy pytest pytest-mock ipython jupyter_client ; # Check types type-check-legate-python; From 018a20043ffd896cd323f708933a81e477003c47 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 25 Jul 2023 16:15:05 -0700 Subject: [PATCH 68/84] fix copy artifacts step again --- .github/workflows/gh-build.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index 50e2b86d62..3ec7259ed2 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -91,9 +91,10 @@ jobs: - name: Copy artifacts back to the host run: | - mkdir -p out - docker run -v "$(pwd)/out:/out" --rm -t -u coder \ - $IMAGE_NAME:${{ inputs.sha }} cp -ar /home/coder/.artifacts /out/ + mkdir -p out; + docker run -v "$(pwd)/out:/out" --rm -t -u root $IMAGE_NAME:${{ inputs.sha }} \ + sh -c "cp -ar /home/coder/.artifacts /out/ && chown -R $(id -u):$(id -g) /out"; + mv out/.artifacts artifacts && rmdir out; echo --------- DOCKER HISTORY START ----------- docker history $IMAGE_NAME:${{ inputs.sha }} echo --------- DOCKER HISTORY END ------------- @@ -105,4 +106,4 @@ jobs: uses: actions/upload-artifact@v3 with: name: "legate.core-${{ inputs.build-target }}-${{ inputs.sha }}" - path: out/.artifacts + path: artifacts From dffc8e27bf7669579104e713620b56a2efe38cd0 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 25 Jul 2023 16:49:09 -0700 Subject: [PATCH 69/84] exclude legate/jupyter from mypy type checking --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 5554c63b52..6cad7eb533 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,5 +98,6 @@ module = [ "legate.__main__", "legate.install_info", "legate._sphinxext.*", + "legate.jupyter.*", ] ignore_errors = true From a05239201d364e87ffe75a9db9505cfa3836dfe9 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 28 Jul 2023 12:37:55 -0700 Subject: [PATCH 70/84] bump legion commit hash --- cmake/versions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/versions.json b/cmake/versions.json index 1d0adeb015..d45a2e32ca 100644 --- a/cmake/versions.json +++ b/cmake/versions.json @@ -7,7 +7,7 @@ }, "Legion": { "git_url" : "https://gitlab.com/StanfordLegion/legion.git", - "git_tag" : "c8fbf636dfae0ceb6977bf0a2734feba24d7ee57" + "git_tag" : "d8e90f9e9b02d212edd7a5a70778c6e19430b4c6" } } } From 426a1ee958248102ac5f2977898bf4fd494154b9 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 31 Jul 2023 15:09:51 -0700 Subject: [PATCH 71/84] copy legate to /tmp before running git commands in preparation for conda build --- .../home/coder/.local/bin/build-legate-conda | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/continuous_integration/home/coder/.local/bin/build-legate-conda b/continuous_integration/home/coder/.local/bin/build-legate-conda index 82c1d5ee0a..236ccb5353 100755 --- a/continuous_integration/home/coder/.local/bin/build-legate-conda +++ b/continuous_integration/home/coder/.local/bin/build-legate-conda @@ -27,6 +27,8 @@ build_legate_conda_package() { GPU_ENABLED=true [ "${USE_CUDA:-}" = "OFF" ] && GPU_ENABLED=false + rm -rf /tmp/legate; + cp -ar ~/legate /tmp/legate; rm -rf ~/.artifacts/legate_core; mkdir -p ~/.artifacts/legate_core; rm -rf /tmp/conda-build/legate_core; @@ -34,7 +36,7 @@ build_legate_conda_package() { # Synthesize new legate_core conda-build build.sh script - cat < ~/legate/conda/conda-build/conda_build_config.yaml + cat < /tmp/legate/conda/conda-build/conda_build_config.yaml numpy: - 1.22 python: @@ -48,10 +50,10 @@ gpu_enabled: cmake_version: - ">=3.20.1,!=3.23.0" package_version: - - "$(git -C ~/legate describe --abbrev=0 --tags | sed 's/[a-zA-Z]//g' | cut -d '.' -f -2).00" + - "$(git -C /tmp/legate describe --abbrev=0 --tags | sed 's/[a-zA-Z]//g' | cut -d '.' -f -2).00" EOF - cat < ~/legate/conda/conda-build/build.sh + cat < /tmp/legate/conda/conda-build/build.sh # Install legate_core C++ libs tar -C "\$PREFIX" --exclude="*.a" --strip-components=1 -xf ~/.artifacts/legate_core-*-Linux.tar.gz; sed -E -i "s@$HOME/\.conda/envs/legate@\$PREFIX@g" "\$PREFIX/share/Legion/cmake/LegionConfigCommon.cmake"; @@ -65,14 +67,16 @@ pip install --no-deps --root / --prefix "\$PREFIX" ~/.artifacts/legate_core-*.wh rm -rf "\$SP_DIR"/legion*egg-info; EOF - git -C ~/legate add .; - git -C ~/legate commit --allow-empty --allow-empty-message -n -m ""; + git -C /tmp/legate add .; + git -C /tmp/legate commit --allow-empty --allow-empty-message -n -m ""; # Build legate_core conda package time CUDA=${CUDA_VERSION_MAJOR}.${CUDA_VERSION_MINOR} \ - conda mambabuild ${conda_build_args[@]} ~/legate/conda/conda-build; + conda mambabuild ${conda_build_args[@]} /tmp/legate/conda/conda-build; - git -C ~/legate reset --hard HEAD~1; + git -C /tmp/legate reset --hard HEAD~1; + + rm -rf /tmp/legate; { set +x; } 2>/dev/null; } From 0617ed49e55902af9e34f607b4b5673714395d44 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 2 Aug 2023 16:19:36 -0700 Subject: [PATCH 72/84] bump legion commit hash --- cmake/versions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/versions.json b/cmake/versions.json index d45a2e32ca..086c05de94 100644 --- a/cmake/versions.json +++ b/cmake/versions.json @@ -7,7 +7,7 @@ }, "Legion": { "git_url" : "https://gitlab.com/StanfordLegion/legion.git", - "git_tag" : "d8e90f9e9b02d212edd7a5a70778c6e19430b4c6" + "git_tag" : "6399fdd9287f0c766f120a91d9159abff47a0f67" } } } From 8ebfa8335469897fdcdff01e1164e73714a1a25f Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 11 Aug 2023 10:32:29 -0700 Subject: [PATCH 73/84] bump legion hash --- cmake/versions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/versions.json b/cmake/versions.json index 086c05de94..d6bc8adf63 100644 --- a/cmake/versions.json +++ b/cmake/versions.json @@ -7,7 +7,7 @@ }, "Legion": { "git_url" : "https://gitlab.com/StanfordLegion/legion.git", - "git_tag" : "6399fdd9287f0c766f120a91d9159abff47a0f67" + "git_tag" : "6646f248982be5b6c28195e8fe019f22af4f1ef4" } } } From 878fde0e550c17d07a33e9bde2e0e1c20f41e599 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 11 Aug 2023 10:47:26 -0700 Subject: [PATCH 74/84] fix workflow env --- .github/workflows/gh-build.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index d2c8da6a96..e8415c178d 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -15,7 +15,7 @@ on: env: BASE_IMAGE: rapidsai/devcontainers:23.08-cpp-cuda11.8-mambaforge-ubuntu22.04 - IMAGE_NAME: legate.core-${{ inputs.build-target }} + IMAGE_TAG: "legate.core-${{ inputs.build-target }}:${{ inputs.sha }}" USE_CUDA: ${{ (inputs.build-target == 'cpu' && 'OFF') || 'ON' }} jobs: @@ -29,9 +29,6 @@ jobs: runs-on: ${{ inputs.runs-on }} - env: - IMAGE_TAG: "${{ env.IMAGE_NAME }}:${{ inputs.sha }}" - steps: - name: Checkout legate.core (= this repo) uses: actions/checkout@v3 From fe268748c87a3a4b504fbe8559a6ec191ab1a8fa Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 11 Aug 2023 10:54:06 -0700 Subject: [PATCH 75/84] don't error on undefined envvars --- continuous_integration/build-docker-image | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/continuous_integration/build-docker-image b/continuous_integration/build-docker-image index f30fa0c6fa..9f124098a2 100755 --- a/continuous_integration/build-docker-image +++ b/continuous_integration/build-docker-image @@ -27,9 +27,9 @@ set -e docker build \ --progress plain \ --tag "$IMAGE_TAG" \ - --build-arg USE_CUDA="$USE_CUDA" \ - --build-arg AWS_REGION="$AWS_REGION" \ - --build-arg BASE_IMAGE="$BASE_IMAGE" \ + --build-arg USE_CUDA="${USE_CUDA:-}" \ + --build-arg AWS_REGION="${AWS_REGION:-}" \ + --build-arg BASE_IMAGE="${BASE_IMAGE:-}" \ --secret id=GH_TOKEN,src=<(cat <<< "${GH_TOKEN:-}") \ --secret id=AWS_ACCESS_KEY_ID,src=<(cat <<< "${AWS_ACCESS_KEY_ID:-}") \ --secret id=AWS_SESSION_TOKEN,src=<(cat <<< "${AWS_SESSION_TOKEN:-}") \ From e77692979552378cbd55dba97994dfe5568ac5ba Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 11 Aug 2023 11:00:08 -0700 Subject: [PATCH 76/84] update build-all build-legate-all --- scripts/launch-devcontainer.sh | 2 +- scripts/replicate-ci.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/launch-devcontainer.sh b/scripts/launch-devcontainer.sh index bb843d114b..78ef1b86cb 100755 --- a/scripts/launch-devcontainer.sh +++ b/scripts/launch-devcontainer.sh @@ -33,7 +33,7 @@ launch_devcontainer() { local vols=(); vols+=(-v "${cwd}:/home/coder/legate"); vols+=(-v "${cwd}/continuous_integration/home/coder/.local/bin/make-conda-env:/home/coder/.local/bin/make-conda-env"); - vols+=(-v "${cwd}/continuous_integration/home/coder/.local/bin/build-all:/home/coder/.local/bin/build-all"); + vols+=(-v "${cwd}/continuous_integration/home/coder/.local/bin/build-legate-all:/home/coder/.local/bin/build-legate-all"); vols+=(-v "${cwd}/continuous_integration/home/coder/.local/bin/build-legate-cpp:/home/coder/.local/bin/build-legate-cpp"); vols+=(-v "${cwd}/continuous_integration/home/coder/.local/bin/build-legate-wheel:/home/coder/.local/bin/build-legate-wheel"); vols+=(-v "${cwd}/continuous_integration/home/coder/.local/bin/build-legate-conda:/home/coder/.local/bin/build-legate-conda"); diff --git a/scripts/replicate-ci.sh b/scripts/replicate-ci.sh index 7cecaf5399..7cb578b93b 100755 --- a/scripts/replicate-ci.sh +++ b/scripts/replicate-ci.sh @@ -8,7 +8,7 @@ exec ./scripts/launch-devcontainer-conda.sh bash -licx ' make-conda-env; # Build libs + conda package -build-all; +build-legate-all; # Uncreate build env rm -rf ~/.conda/envs/${DEFAULT_CONDA_ENV:-legate}.bak; From af7713bdd9e844aa49cd3697ceabb47206793f4a Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 16 Aug 2023 23:10:33 +0000 Subject: [PATCH 77/84] add MODIFY_PREFIX_PATH argument back --- cmake/thirdparty/get_legion.cmake | 1 - legate_core_cpp.cmake | 3 ++- legate_core_python.cmake | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/thirdparty/get_legion.cmake b/cmake/thirdparty/get_legion.cmake index 485c0a6204..b990253e97 100644 --- a/cmake/thirdparty/get_legion.cmake +++ b/cmake/thirdparty/get_legion.cmake @@ -75,7 +75,6 @@ function(find_or_configure_legion) string(REPLACE "," " " Legion_PYTHON_EXTRA_INSTALL_ARGS "${Legion_PYTHON_EXTRA_INSTALL_ARGS}") string(REPLACE ";" " " Legion_PYTHON_EXTRA_INSTALL_ARGS "${Legion_PYTHON_EXTRA_INSTALL_ARGS}") - set(_cuda_path "") set(_legion_cuda_options "") # Set CMAKE_CXX_STANDARD and CMAKE_CUDA_STANDARD for Legion builds. Legion's FindCUDA.cmake diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index 6e8fa884d4..393670e997 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -39,7 +39,7 @@ rapids_cmake_build_type(Release) ############################################################################## # - conda environment -------------------------------------------------------- -rapids_cmake_support_conda_env(conda_env) +rapids_cmake_support_conda_env(conda_env MODIFY_PREFIX_PATH) # We're building python extension libraries, which must always be installed # under lib/, even if the system normally uses lib64/. Rapids-cmake currently @@ -150,6 +150,7 @@ if(Legion_USE_CUDA) list(APPEND legate_core_CUDA_OPTIONS -Xfatbin=-compress-all) list(APPEND legate_core_CUDA_OPTIONS --expt-extended-lambda) list(APPEND legate_core_CUDA_OPTIONS --expt-relaxed-constexpr) + list(APPEND legate_core_CUDA_OPTIONS -Wno-deprecated-gpu-targets) endif() if(Legion_USE_OpenMP) diff --git a/legate_core_python.cmake b/legate_core_python.cmake index dea81ff21f..bfa6d8296a 100644 --- a/legate_core_python.cmake +++ b/legate_core_python.cmake @@ -65,7 +65,7 @@ add_subdirectory(legate/core/_lib) set(cython_lib_dir "../../") -if(CMAKE_INSTALL_RPATH_USE_LINK_PATH) +if(CMAKE_INSTALL_RPATH_USE_LINK_PATH AND (TARGET legate_core)) get_target_property(cython_lib_dir legate_core LIBRARY_OUTPUT_DIRECTORY) get_target_property(legate_cpp_dir legate_core BINARY_DIR) if(legate_cpp_dir) @@ -78,7 +78,7 @@ rapids_cython_add_rpath_entries(TARGET legate_core PATHS "${cython_lib_dir}") ############################################################################## # - conda environment -------------------------------------------------------- -rapids_cmake_support_conda_env(conda_env) +rapids_cmake_support_conda_env(conda_env MODIFY_PREFIX_PATH) # We're building python extension libraries, which must always be installed # under lib/, even if the system normally uses lib64/. Rapids-cmake currently From 8f586271c1e9589b457b7278f60801eccfdd2acd Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 16 Aug 2023 23:10:58 +0000 Subject: [PATCH 78/84] support generating envs for CUDA 12.0, add nvcc --- scripts/generate-conda-envs.py | 35 +++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/scripts/generate-conda-envs.py b/scripts/generate-conda-envs.py index 3bfe004ee7..3e0ee5dce3 100755 --- a/scripts/generate-conda-envs.py +++ b/scripts/generate-conda-envs.py @@ -71,18 +71,30 @@ def conda(self) -> Reqs: "pynvml", # tests ) - if self.compilers and self.os == "linux": - # gcc 11.3 is incompatible with nvcc <= 11.5. - if V(self.ctk_version) <= V("11.5"): - deps += ( - "gcc_linux-64<=11.2", - "gxx_linux-64<=11.2", - ) + if V(self.ctk_version) >= V("12.0"): + deps += ( + "cuda-cudart-dev", + "cuda-nvtx-dev", + ) + + if self.compilers: + if V(self.ctk_version) < V("12.0"): + deps += (f"nvcc_linux-64={self.ctk_version}",) else: - deps += ( - "gcc_linux-64=11.*", - "gxx_linux-64=11.*", - ) + deps += ("cuda-nvcc",) + + if self.os == "linux": + # gcc 11.3 is incompatible with nvcc <= 11.5. + if V(self.ctk_version) <= V("11.5"): + deps += ( + "gcc_linux-64<=11.2", + "gxx_linux-64<=11.2", + ) + else: + deps += ( + "gcc_linux-64=11.*", + "gxx_linux-64=11.*", + ) return deps @@ -279,6 +291,7 @@ def filename(self) -> str: name: legate-{use} channels: - conda-forge + - nvidia dependencies: - python={python},!=3.9.7 # avoid https://bugs.python.org/issue45121 From ae9a2761029f524dce42a54b49bc53ad59b3c46a Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 17 Aug 2023 12:03:38 -0700 Subject: [PATCH 79/84] add conda-utils and entrypoint back in for cunumeric CI --- continuous_integration/Dockerfile | 22 +-------- .../home/coder/.local/bin/conda-utils | 12 +++++ .../home/coder/.local/bin/entrypoint | 47 +++++++++++++++++++ 3 files changed, 61 insertions(+), 20 deletions(-) create mode 100755 continuous_integration/home/coder/.local/bin/conda-utils create mode 100755 continuous_integration/home/coder/.local/bin/entrypoint diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile index 2b54da6422..b9a46771bb 100644 --- a/continuous_integration/Dockerfile +++ b/continuous_integration/Dockerfile @@ -1,7 +1,7 @@ ARG BASE_IMAGE=rapidsai/devcontainers:23.08-cpp-cuda11.8-mambaforge-ubuntu22.04 FROM ${BASE_IMAGE} as stage0 -SHELL ["/bin/bash", "-Eeox", "pipefail", "-c"] +SHELL ["/bin/bash", "-Eeou", "pipefail", "-c"] ARG USE_CUDA=1 ENV USE_CUDA=${USE_CUDA} @@ -41,27 +41,9 @@ RUN --mount=type=secret,id=GH_TOKEN,uid=1000,gid=1000 \ --mount=type=secret,id=AWS_SESSION_TOKEN,uid=1000,gid=1000 \ --mount=type=secret,id=AWS_SECRET_ACCESS_KEY,uid=1000,gid=1000 \ </dev/null; -eval "$( \ - find /run/secrets/ -type f -exec \ - bash -c 'echo ${0/\/run\/secrets\//}=$(<${0})' {} \; \ - | xargs -r -d'\n' -I% echo -n export %\; \ -)"; -{ set -xo history; } 2>/dev/null; - -if grep -qE "^$" <<< "${GH_TOKEN:-}" \ -&& grep -qE "^$" <<< "${AWS_ACCESS_KEY_ID:-}" \ -&& grep -qE "^$" <<< "${AWS_SECRET_ACCESS_KEY:-}"; then - export SCCACHE_BUCKET=""; -fi - -. devcontainer-utils-post-attach-command; - +. entrypoint; make-conda-env; - build-legate-all; - EOF #--------------------------------------------------- diff --git a/continuous_integration/home/coder/.local/bin/conda-utils b/continuous_integration/home/coder/.local/bin/conda-utils new file mode 100755 index 0000000000..2f5027afe5 --- /dev/null +++ b/continuous_integration/home/coder/.local/bin/conda-utils @@ -0,0 +1,12 @@ +get_yaml_and_make_conda_env() { + make-conda-env; +} + +install_legate_core_with_war() { + # WAR: legate-core depends on a different version of numpy than what is already installed. + # The correct version will be installed when legate-core is installed below. + # See github issue: https://github.com/nv-legate/legate.core/issues/812 + mamba uninstall -y -n "${DEFAULT_CONDA_ENV:-legate}" numpy; + + mamba install -y -n "${DEFAULT_CONDA_ENV:-legate}" -c nvidia -c conda-forge -c /tmp/conda-build/legate_core legate-core; +} diff --git a/continuous_integration/home/coder/.local/bin/entrypoint b/continuous_integration/home/coder/.local/bin/entrypoint new file mode 100755 index 0000000000..2c77fd68ae --- /dev/null +++ b/continuous_integration/home/coder/.local/bin/entrypoint @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +entrypoint() { + set -euo pipefail; + + # disable xtrace and history + { set +xo history; } 2>/dev/null; + + mkdir -p /home/coder/.cache; + + # export run secrets as shell vars + eval "$( \ + find /run/secrets/ -type f -exec \ + bash -c 'echo ${0/\/run\/secrets\//}=$(<${0})' {} \; \ + | xargs -r -d'\n' -I% echo -n export %\; \ + )"; + + # set SCCACHE_BUCKET to an empty string if we don't have creds + if grep -qE "^$" <<< "${GH_TOKEN:-}" \ + && grep -qE "^$" <<< "${AWS_ACCESS_KEY_ID:-}" \ + && grep -qE "^$" <<< "${AWS_SECRET_ACCESS_KEY:-}"; then + export SCCACHE_BUCKET=""; + fi + + . devcontainer-utils-post-attach-command; + + # report sccache variables for debugging + for x in "GH_TOKEN" \ + "SCCACHE_BUCKET" \ + "SCCACHE_REGION" \ + "AWS_SESSION_TOKEN" \ + "AWS_ACCESS_KEY_ID" \ + "AWS_SECRET_ACCESS_KEY"; do + if ! grep -qE "^$" <<< "${!x:-}"; then + echo "${x}=${!x:-}"; + fi + done + + # reenable xtrace and history + { set -xo history; } 2>/dev/null; + + if test "${#@}" -gt 0; then + exec; + fi +} + +entrypoint "$@"; From defe9e66cee233c635dfabb330e4769aa8561d48 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 18 Aug 2023 12:18:11 -0700 Subject: [PATCH 80/84] reusable workflows should inherit secrets --- .github/workflows/ci-gh-cpu-build-and-test.yml | 12 ++++++------ .github/workflows/ci-gh-gpu-build-and-test.yml | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci-gh-cpu-build-and-test.yml b/.github/workflows/ci-gh-cpu-build-and-test.yml index dbba96523d..d2592a9520 100644 --- a/.github/workflows/ci-gh-cpu-build-and-test.yml +++ b/.github/workflows/ci-gh-cpu-build-and-test.yml @@ -19,8 +19,8 @@ jobs: build-cpu: needs: filter-by-triggering-event if: needs.filter-by-triggering-event.outputs.ok == 'true' - uses: - ./.github/workflows/gh-build.yml + secrets: inherit + uses: ./.github/workflows/gh-build.yml with: build-target: cpu # Ref: https://docs.rapids.ai/resources/github-actions/#cpu-labels for `linux-amd64-cpu4` @@ -37,8 +37,8 @@ jobs: - { name: pytest, test-scope: test } - { name: mypy, test-scope: type-check } name: ${{ matrix.name }} - uses: - ./.github/workflows/gh-test.yml + secrets: inherit + uses: ./.github/workflows/gh-test.yml with: build-target: cpu runs-on: ${{ github.repository == 'nv-legate/legate.core' && 'linux-amd64-cpu4' || 'ubuntu-latest' }} @@ -52,8 +52,8 @@ jobs: - test-cpu # This ensures the cleanup job runs even if previous jobs fail or the workflow is cancelled. if: always() - uses: - ./.github/workflows/gh-cleanup.yml + secrets: inherit + uses: ./.github/workflows/gh-cleanup.yml with: build-target: cpu sha: ${{ github.sha }} diff --git a/.github/workflows/ci-gh-gpu-build-and-test.yml b/.github/workflows/ci-gh-gpu-build-and-test.yml index e2a3edab9b..36bb4bba2b 100644 --- a/.github/workflows/ci-gh-gpu-build-and-test.yml +++ b/.github/workflows/ci-gh-gpu-build-and-test.yml @@ -19,8 +19,8 @@ jobs: build-gpu: needs: filter-by-triggering-event if: needs.filter-by-triggering-event.outputs.ok == 'true' - uses: - ./.github/workflows/gh-build.yml + secrets: inherit + uses: ./.github/workflows/gh-build.yml with: build-target: gpu # Ref: https://docs.rapids.ai/resources/github-actions/#cpu-labels for `linux-amd64-cpu4` @@ -39,8 +39,8 @@ jobs: - { name: pytest, test-scope: test, runner: linux-amd64-2gpu } - { name: mypy, test-scope: type-check, runner: linux-amd64-gpu-v100-latest-1 } name: ${{ matrix.name }} - uses: - ./.github/workflows/gh-test.yml + secrets: inherit + uses: ./.github/workflows/gh-test.yml with: build-target: gpu runs-on: ${{ matrix.runner }} @@ -54,8 +54,8 @@ jobs: - test-gpu # This ensures the cleanup job runs even if previous jobs fail or the workflow is cancelled. if: always() - uses: - ./.github/workflows/gh-cleanup.yml + secrets: inherit + uses: ./.github/workflows/gh-cleanup.yml with: build-target: gpu sha: ${{ github.sha }} From 7d2344fbf605f78373ad115a19fc25d375f1a41f Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 18 Aug 2023 12:18:37 -0700 Subject: [PATCH 81/84] update devcontainer versions --- .github/workflows/gh-build.yml | 2 +- continuous_integration/Dockerfile | 8 ++++++-- scripts/launch-devcontainer-conda.sh | 2 +- scripts/launch-devcontainer-pip.sh | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index e8415c178d..885bf23d3a 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -14,7 +14,7 @@ on: type: string env: - BASE_IMAGE: rapidsai/devcontainers:23.08-cpp-cuda11.8-mambaforge-ubuntu22.04 + BASE_IMAGE: rapidsai/devcontainers:23.10-cpp-cuda11.8-mambaforge-ubuntu22.04 IMAGE_TAG: "legate.core-${{ inputs.build-target }}:${{ inputs.sha }}" USE_CUDA: ${{ (inputs.build-target == 'cpu' && 'OFF') || 'ON' }} diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile index b9a46771bb..7308786006 100644 --- a/continuous_integration/Dockerfile +++ b/continuous_integration/Dockerfile @@ -1,9 +1,9 @@ -ARG BASE_IMAGE=rapidsai/devcontainers:23.08-cpp-cuda11.8-mambaforge-ubuntu22.04 +ARG BASE_IMAGE=rapidsai/devcontainers:23.10-cpp-cuda11.8-mambaforge-ubuntu22.04 FROM ${BASE_IMAGE} as stage0 SHELL ["/bin/bash", "-Eeou", "pipefail", "-c"] -ARG USE_CUDA=1 +ARG USE_CUDA="ON" ENV USE_CUDA=${USE_CUDA} ARG BUILD_MARCH=nocona @@ -41,9 +41,13 @@ RUN --mount=type=secret,id=GH_TOKEN,uid=1000,gid=1000 \ --mount=type=secret,id=AWS_SESSION_TOKEN,uid=1000,gid=1000 \ --mount=type=secret,id=AWS_SECRET_ACCESS_KEY,uid=1000,gid=1000 \ < Date: Fri, 18 Aug 2023 12:19:00 -0700 Subject: [PATCH 82/84] don't store git credentials --- .github/workflows/gh-build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index 885bf23d3a..ad0840bb16 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -33,8 +33,9 @@ jobs: - name: Checkout legate.core (= this repo) uses: actions/checkout@v3 with: - ref: ${{ inputs.sha }} fetch-depth: 0 + persist-credentials: false + ref: ${{ inputs.sha }} - if: github.repository_owner == 'nv-legate' name: Get AWS credentials for sccache bucket From 858856a675b18a884c16bdf9ed1ed16fc99f5120 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 21 Aug 2023 10:14:34 -0700 Subject: [PATCH 83/84] add elfutils and libdwarf to conda host and run sections --- conda/conda-build/meta.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/conda/conda-build/meta.yaml b/conda/conda-build/meta.yaml index 64934e8a29..4d35154954 100644 --- a/conda/conda-build/meta.yaml +++ b/conda/conda-build/meta.yaml @@ -98,8 +98,6 @@ requirements: - cmake {{ cmake_version }} - {{ compiler('c') }} =11.2 - {{ compiler('cxx') }} =11.2 - - elfutils # [linux] - - libdwarf # [linux] host: - zlib - python @@ -107,6 +105,8 @@ requirements: - llvm-openmp - scikit-build - numpy {{ numpy_version }} + - elfutils # [linux] + - libdwarf # [linux] {% if gpu_enabled_bool %} - nccl - cuda-nvcc ={{ cuda_version }} @@ -123,6 +123,8 @@ requirements: - llvm-openmp - numpy {{ numpy_version }} - typing_extensions + - elfutils # [linux] + - libdwarf # [linux] {% if gpu_enabled_bool %} - cuda-cudart >={{ cuda_version }},<{{ cuda_major+1 }} - cuda-version >={{ cuda_version }},<{{ cuda_major+1 }} From c89552ecddc3f8834c48921f04761f903eb2b836 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 28 Aug 2023 18:25:02 -0700 Subject: [PATCH 84/84] bump legion hash --- cmake/versions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/versions.json b/cmake/versions.json index d6bc8adf63..eac50916a2 100644 --- a/cmake/versions.json +++ b/cmake/versions.json @@ -7,7 +7,7 @@ }, "Legion": { "git_url" : "https://gitlab.com/StanfordLegion/legion.git", - "git_tag" : "6646f248982be5b6c28195e8fe019f22af4f1ef4" + "git_tag" : "92c45583c07cb887fd5898e59c5c2f54a12d6d91" } } }