diff --git a/.dockerignore b/.dockerignore deleted file mode 120000 index 3e4e48b0b5..0000000000 --- a/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -.gitignore \ No newline at end of file diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..fcefbdfefc --- /dev/null +++ b/.dockerignore @@ -0,0 +1,44 @@ +*~ +\#*# +*/typestubs/* +.#* +.DS_Store +*.a +*.d +*.o +*.so +*.gz +*.gcno +*.gcda +*.gcov +*.fluid +*.pyc +*.swp +*.log +*.bbl +*.blg +*.aux +*.json +*.dylib +install_info.py +/dist +/build +/legion +/install* +/_skbuild +config.mk +/docs/legate/core/build +/docs/legate/core/source/api/generated +*.egg-info +.creds +.cache +.coverage +.vscode +_cmake_test_compile +!cmake/versions.json +legate.core.code-workspace +*.prof +.legate-test-last-failed +out/ +.github/ +continuous_integration/Dockerfile diff --git a/.github/workflows/ci-gh-cpu-build-and-test.yml b/.github/workflows/ci-gh-cpu-build-and-test.yml index a2202dbce4..d2592a9520 100644 --- a/.github/workflows/ci-gh-cpu-build-and-test.yml +++ b/.github/workflows/ci-gh-cpu-build-and-test.yml @@ -5,15 +5,22 @@ concurrency: cancel-in-progress: true on: + pull_request: push: branches: - "pull-request/[0-9]+" - "branch-*" jobs: + filter-by-triggering-event: + name: Filter by triggering event + uses: ./.github/workflows/filter-by-triggering-event.yml + build-cpu: - uses: - ./.github/workflows/gh-build.yml + needs: filter-by-triggering-event + if: needs.filter-by-triggering-event.outputs.ok == 'true' + secrets: inherit + uses: ./.github/workflows/gh-build.yml with: build-target: cpu # Ref: https://docs.rapids.ai/resources/github-actions/#cpu-labels for `linux-amd64-cpu4` @@ -27,15 +34,16 @@ jobs: fail-fast: false matrix: include: - - { name: Pytest Unit Tests, test-scope: unit } - - { name: mypy, test-scope: mypy } + - { name: pytest, test-scope: test } + - { name: mypy, test-scope: type-check } name: ${{ matrix.name }} - uses: - ./.github/workflows/gh-test.yml + secrets: inherit + uses: ./.github/workflows/gh-test.yml with: build-target: cpu runs-on: ${{ github.repository == 'nv-legate/legate.core' && 'linux-amd64-cpu4' || 'ubuntu-latest' }} sha: ${{ github.sha }} + test-name: ${{ matrix.name }} test-scope: ${{ matrix.test-scope }} cleanup: @@ -44,8 +52,8 @@ jobs: - test-cpu # This ensures the cleanup job runs even if previous jobs fail or the workflow is cancelled. if: always() - uses: - ./.github/workflows/gh-cleanup.yml + secrets: inherit + uses: ./.github/workflows/gh-cleanup.yml with: build-target: cpu sha: ${{ github.sha }} diff --git a/.github/workflows/ci-gh-docs.yml b/.github/workflows/ci-gh-docs.yml index 267fbd3d87..722a44347a 100644 --- a/.github/workflows/ci-gh-docs.yml +++ b/.github/workflows/ci-gh-docs.yml @@ -4,6 +4,7 @@ on: push: branches-ignore: - gh-pages # deployment target branch (this workflow should not exist on that branch anyway) + - "pull-request/[0-9]+" pull_request: branches-ignore: - gh-pages # deployment target branch (this workflow should not exist on that branch anyway) @@ -44,14 +45,14 @@ jobs: env: MATRIX_CONTEXT: ${{ toJSON(matrix) }} run: echo "$MATRIX_CONTEXT" - + #################################### # Actual build process starts here # #################################### - + - name: Checkout uses: actions/checkout@v3 - + - name: Generate YAML file listing dependencies run: scripts/generate-conda-envs.py --python 3.10 --ctk 11.8 --os linux --compilers --openmpi @@ -61,7 +62,7 @@ jobs: - name: Build doxygen documentation through install.py run: | conda run -n legate /bin/bash -c "./install.py --docs" - + - name: Build documentation using Makefile working-directory: ./docs/legate/core run: | diff --git a/.github/workflows/ci-gh-gpu-build-and-test.yml b/.github/workflows/ci-gh-gpu-build-and-test.yml index ccb599ca87..36bb4bba2b 100644 --- a/.github/workflows/ci-gh-gpu-build-and-test.yml +++ b/.github/workflows/ci-gh-gpu-build-and-test.yml @@ -5,15 +5,22 @@ concurrency: cancel-in-progress: true on: + pull_request: push: branches: - "pull-request/[0-9]+" - "branch-*" jobs: + filter-by-triggering-event: + name: Filter by triggering event + uses: ./.github/workflows/filter-by-triggering-event.yml + build-gpu: - uses: - ./.github/workflows/gh-build.yml + needs: filter-by-triggering-event + if: needs.filter-by-triggering-event.outputs.ok == 'true' + secrets: inherit + uses: ./.github/workflows/gh-build.yml with: build-target: gpu # Ref: https://docs.rapids.ai/resources/github-actions/#cpu-labels for `linux-amd64-cpu4` @@ -21,22 +28,24 @@ jobs: sha: ${{ github.sha }} test-gpu: + if: github.repository == 'nv-legate/legate.core' needs: - build-gpu strategy: fail-fast: false matrix: include: - - { name: Pytest Unit Tests, test-scope: unit, runner: linux-amd64-gpu-v100-latest-1 } - - { name: Pytest Unit Tests, test-scope: unit, runner: linux-amd64-2gpu } - - { name: mypy, test-scope: mypy, runner: 'linux-amd64-gpu-v100-latest-1' } + - { name: pytest, test-scope: test, runner: linux-amd64-gpu-v100-latest-1 } + - { name: pytest, test-scope: test, runner: linux-amd64-2gpu } + - { name: mypy, test-scope: type-check, runner: linux-amd64-gpu-v100-latest-1 } name: ${{ matrix.name }} - uses: - ./.github/workflows/gh-test.yml + secrets: inherit + uses: ./.github/workflows/gh-test.yml with: build-target: gpu runs-on: ${{ matrix.runner }} sha: ${{ github.sha }} + test-name: ${{ matrix.name }} test-scope: ${{ matrix.test-scope }} cleanup: @@ -45,8 +54,8 @@ jobs: - test-gpu # This ensures the cleanup job runs even if previous jobs fail or the workflow is cancelled. if: always() - uses: - ./.github/workflows/gh-cleanup.yml + secrets: inherit + uses: ./.github/workflows/gh-cleanup.yml with: build-target: gpu sha: ${{ github.sha }} diff --git a/.github/workflows/filter-by-triggering-event.yml b/.github/workflows/filter-by-triggering-event.yml new file mode 100644 index 0000000000..af69a62068 --- /dev/null +++ b/.github/workflows/filter-by-triggering-event.yml @@ -0,0 +1,30 @@ +name: Filter by triggering event + +on: + workflow_call: + outputs: + ok: + value: ${{ jobs.filter-by-triggering-event.outputs.ok }} + +jobs: + filter-by-triggering-event: + name: Filter by triggering event + runs-on: ubuntu-latest + outputs: + ok: ${{ steps.filter-by-triggering-event.outputs.ok }} + steps: + - id: filter-by-triggering-event + name: Filter by triggering event + shell: bash + run: | + ### + # 1. If `event_name == 'push'` and the repo is the official nv-legate repo. + # 2. If `event_name == 'pull_request'` and teh repo is a fork of the official nv-legate repo. + ### + + if [[ '${{ github.event_name }}' == 'push' && '${{ github.repository }}' == 'nv-legate/legate.core' ]] \ + || [[ '${{ github.event_name }}' == 'pull_request' && '${{ github.repository }}' != 'nv-legate/legate.core' ]]; then + echo "ok=true" | tee -a "${GITHUB_OUTPUT}"; + else + echo "ok=false" | tee -a "${GITHUB_OUTPUT}"; + fi diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index d5e8393420..ad0840bb16 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -14,9 +14,8 @@ on: type: string env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BASE_IMAGE: rapidsai/devcontainers:23.06-cpp-cuda11.8-mambaforge-ubuntu22.04 - IMAGE_NAME: legate.core-${{ inputs.build-target }} + BASE_IMAGE: rapidsai/devcontainers:23.10-cpp-cuda11.8-mambaforge-ubuntu22.04 + IMAGE_TAG: "legate.core-${{ inputs.build-target }}:${{ inputs.sha }}" USE_CUDA: ${{ (inputs.build-target == 'cpu' && 'OFF') || 'ON' }} jobs: @@ -35,6 +34,8 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 0 + persist-credentials: false + ref: ${{ inputs.sha }} - if: github.repository_owner == 'nv-legate' name: Get AWS credentials for sccache bucket @@ -50,11 +51,11 @@ jobs: docker system prune --all --force - name: Build docker image + env: + GH_TOKEN: "${{ secrets.PERSONAL_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}" run: | echo BUILD_TARGET: ${{ inputs.build-target }} - echo USE_CUDA: ${{ env.USE_CUDA }} - - IMAGE_TAG=${{ env.IMAGE_NAME }}:${{ inputs.sha }} + echo USE_CUDA: ${USE_CUDA} continuous_integration/build-docker-image \ --base-image "$BASE_IMAGE" \ @@ -63,20 +64,17 @@ jobs: - name: Dump docker history of image before upload run: | - IMAGE_TAG=${{ env.IMAGE_NAME }}:${{ inputs.sha }} docker history $IMAGE_TAG - - name: Dump docker history of image before upload - run: | - IMAGE_TAG=${{ env.IMAGE_NAME_CUNUMERIC }}:${{ inputs.sha }} - - - name: Log in to container image registry - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Push image run: | - IMAGE_TAG=${{ env.IMAGE_NAME }}:${{ inputs.sha }} - IMAGE_ID=ghcr.io/${{ github.repository_owner }} # Change all uppercase to lowercase @@ -89,10 +87,10 @@ jobs: - name: Copy artifacts back to the host run: | - IMAGE_TAG=${{ env.IMAGE_NAME }}:${{ inputs.sha }} - mkdir -p artifacts - docker run -v "$(pwd)/artifacts:/home/coder/.artifacts" --rm -t $IMAGE_TAG copy-artifacts - + mkdir -p out; + docker run -v "$(pwd)/out:/out" --rm -t -u root $IMAGE_TAG \ + sh -c "cp -ar /home/coder/.artifacts /out/ && chown -R $(id -u):$(id -g) /out"; + mv out/.artifacts artifacts && rmdir out; - name: Display structure of workdir run: ls -R diff --git a/.github/workflows/gh-test.yml b/.github/workflows/gh-test.yml index ebce23089b..23cb12892a 100644 --- a/.github/workflows/gh-test.yml +++ b/.github/workflows/gh-test.yml @@ -12,6 +12,9 @@ on: sha: required: true type: string + test-name: + required: true + type: string test-scope: required: true type: string @@ -33,6 +36,6 @@ jobs: name: Run nvidia-smi to make sure GPU is working run: nvidia-smi - - name: Run legate.core test / analysis + - name: Run ${{ inputs.test-name }} on legate.core shell: su coder {0} - run: run-test-or-analysis ${{ inputs.test-scope }} + run: ${{ inputs.test-scope }}-legate-python diff --git a/.gitignore b/.gitignore index 2db6b9e9fc..29e4649b7c 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ config.mk /docs/legate/core/build /docs/legate/core/source/api/generated *.egg-info +.creds .cache .coverage .vscode diff --git a/cmake/Modules/cuda_arch_helpers.cmake b/cmake/Modules/cuda_arch_helpers.cmake index 9a2206f69d..97d0ee8402 100644 --- a/cmake/Modules/cuda_arch_helpers.cmake +++ b/cmake/Modules/cuda_arch_helpers.cmake @@ -17,34 +17,34 @@ function(set_cuda_arch_from_names) set(cuda_archs "") # translate legacy arch names into numbers - if(CMAKE_CUDA_ARCHITECTURES MATCHES "fermi") + if(Legion_CUDA_ARCH MATCHES "fermi") list(APPEND cuda_archs 20) endif() - if(CMAKE_CUDA_ARCHITECTURES MATCHES "kepler") + if(Legion_CUDA_ARCH MATCHES "kepler") list(APPEND cuda_archs 30) endif() - if(CMAKE_CUDA_ARCHITECTURES MATCHES "k20") + if(Legion_CUDA_ARCH MATCHES "k20") list(APPEND cuda_archs 35) endif() - if(CMAKE_CUDA_ARCHITECTURES MATCHES "k80") + if(Legion_CUDA_ARCH MATCHES "k80") list(APPEND cuda_archs 37) endif() - if(CMAKE_CUDA_ARCHITECTURES MATCHES "maxwell") + if(Legion_CUDA_ARCH MATCHES "maxwell") list(APPEND cuda_archs 52) endif() - if(CMAKE_CUDA_ARCHITECTURES MATCHES "pascal") + if(Legion_CUDA_ARCH MATCHES "pascal") list(APPEND cuda_archs 60) endif() - if(CMAKE_CUDA_ARCHITECTURES MATCHES "volta") + if(Legion_CUDA_ARCH MATCHES "volta") list(APPEND cuda_archs 70) endif() - if(CMAKE_CUDA_ARCHITECTURES MATCHES "turing") + if(Legion_CUDA_ARCH MATCHES "turing") list(APPEND cuda_archs 75) endif() - if(CMAKE_CUDA_ARCHITECTURES MATCHES "ampere") + if(Legion_CUDA_ARCH MATCHES "ampere") list(APPEND cuda_archs 80) endif() - if(CMAKE_CUDA_ARCHITECTURES MATCHES "hopper") + if(Legion_CUDA_ARCH MATCHES "hopper") list(APPEND cuda_archs 90) endif() @@ -60,19 +60,24 @@ function(set_cuda_arch_from_names) else() list(TRANSFORM cuda_archs APPEND "-real") endif() - set(CMAKE_CUDA_ARCHITECTURES ${cuda_archs} PARENT_SCOPE) + set(Legion_CUDA_ARCH ${cuda_archs} PARENT_SCOPE) endif() endfunction() -function(add_cuda_architecture_defines defs) - message(VERBOSE "legate.core: CMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}") +function(add_cuda_architecture_defines) + set(options ) + set(oneValueArgs DEFS) + set(multiValueArgs ARCHS) + cmake_parse_arguments(cuda "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - set(_defs ${${defs}}) + message(VERBOSE "legate.core: CUDA_ARCHITECTURES=${cuda_ARCHS}") + + set(_defs ${${cuda_DEFS}}) macro(add_def_if_arch_enabled arch def) - if("${arch}" IN_LIST CMAKE_CUDA_ARCHITECTURES OR - ("${arch}-real" IN_LIST CMAKE_CUDA_ARCHITECTURES) OR - ("${arch}-virtual" IN_LIST CMAKE_CUDA_ARCHITECTURES)) + if("${arch}" IN_LIST cuda_ARCHS OR + ("${arch}-real" IN_LIST cuda_ARCHS) OR + ("${arch}-virtual" IN_LIST cuda_ARCHS)) list(APPEND _defs ${def}) endif() endmacro() @@ -88,5 +93,5 @@ function(add_cuda_architecture_defines defs) add_def_if_arch_enabled("80" "AMPERE_ARCH") add_def_if_arch_enabled("90" "HOPPER_ARCH") - set(${defs} ${_defs} PARENT_SCOPE) + set(${cuda_DEFS} ${_defs} PARENT_SCOPE) endfunction() diff --git a/cmake/Modules/legate_core_options.cmake b/cmake/Modules/legate_core_options.cmake index 96a8d7ec5f..175b965b27 100644 --- a/cmake/Modules/legate_core_options.cmake +++ b/cmake/Modules/legate_core_options.cmake @@ -87,7 +87,7 @@ option(legate_core_BUILD_DOCS "Build doxygen docs" OFF) set_or_default(NCCL_DIR NCCL_PATH) set_or_default(Thrust_DIR THRUST_PATH) set_or_default(CUDA_TOOLKIT_ROOT_DIR CUDA) -set_or_default(CMAKE_CUDA_ARCHITECTURES GPU_ARCH NATIVE) +set_or_default(Legion_CUDA_ARCH GPU_ARCH all-major) set_or_default(Legion_HIJACK_CUDART USE_CUDART_HIJACK OFF) include(CMakeDependentOption) diff --git a/cmake/thirdparty/get_legion.cmake b/cmake/thirdparty/get_legion.cmake index 902727c944..b990253e97 100644 --- a/cmake/thirdparty/get_legion.cmake +++ b/cmake/thirdparty/get_legion.cmake @@ -39,13 +39,6 @@ function(find_or_configure_legion) set(git_repo "${PKG_REPOSITORY}") endif() - set(Legion_CUDA_ARCH "") - if(Legion_USE_CUDA) - set(Legion_CUDA_ARCH ${CMAKE_CUDA_ARCHITECTURES}) - list(TRANSFORM Legion_CUDA_ARCH REPLACE "-real" "") - list(TRANSFORM Legion_CUDA_ARCH REPLACE "-virtual" "") - endif() - set(FIND_PKG_ARGS GLOBAL_TARGETS Legion::Realm Legion::Regent @@ -75,10 +68,13 @@ function(find_or_configure_legion) include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/cpm_helpers.cmake) get_cpm_git_args(legion_cpm_git_args REPOSITORY ${git_repo} BRANCH ${git_branch}) if(NOT DEFINED Legion_PYTHON_EXTRA_INSTALL_ARGS) - set(Legion_PYTHON_EXTRA_INSTALL_ARGS "--single-version-externally-managed --root=/") + set(Legion_PYTHON_EXTRA_INSTALL_ARGS "--root / --prefix \"\${CMAKE_INSTALL_PREFIX}\"") endif() - set(_cuda_path "") + # Support comma and semicolon delimited lists + string(REPLACE "," " " Legion_PYTHON_EXTRA_INSTALL_ARGS "${Legion_PYTHON_EXTRA_INSTALL_ARGS}") + string(REPLACE ";" " " Legion_PYTHON_EXTRA_INSTALL_ARGS "${Legion_PYTHON_EXTRA_INSTALL_ARGS}") + set(_legion_cuda_options "") # Set CMAKE_CXX_STANDARD and CMAKE_CUDA_STANDARD for Legion builds. Legion's FindCUDA.cmake @@ -95,66 +91,13 @@ function(find_or_configure_legion) set(_cuda_std ${_cxx_std}) endif() - if(NOT CUDA_NVCC_FLAGS) - list(APPEND CUDA_NVCC_FLAGS "${CUDAFLAGS}") - endif() - - set(_nvcc_flags ${CUDA_NVCC_FLAGS}) - if(NOT "${_nvcc_flags}" MATCHES "-std=") - list(APPEND _nvcc_flags "-std=c++${_cuda_std}") - endif() - - # Get the `stubs/libcuda.so` path so we can set CMAKE_LIBRARY_PATH for FindCUDA.cmake - set(_libdir "lib64") - if(CMAKE_SIZEOF_VOID_P LESS 8) - set(_libdir "lib") - endif() + list(APPEND _legion_cuda_options "CMAKE_CUDA_STANDARD ${_cuda_std}") - if(EXISTS "${CUDAToolkit_LIBRARY_DIR}/stubs/libcuda.so") - # This might be the path to the `$CONDA_PREFIX/lib` - # If it is (and it has the libcuda.so driver stub), - # then we know we're using the cuda-toolkit package - # and should link to that driver stub instead of the - # one potentially in `/usr/local/cuda/lib[64]/stubs` - list(APPEND _cuda_stubs "${CUDAToolkit_LIBRARY_DIR}/stubs") - elseif(EXISTS "${CUDAToolkit_TARGET_DIR}/${_libdir}/stubs/libcuda.so") - # Otherwise assume stubs are relative to the CUDA toolkit root dir - list(APPEND _cuda_stubs "${CUDAToolkit_TARGET_DIR}/${_libdir}/stubs") - elseif(EXISTS "${CUDAToolkit_LIBRARY_ROOT}/${_libdir}/stubs/libcuda.so") - list(APPEND _cuda_stubs "${CUDAToolkit_LIBRARY_ROOT}/${_libdir}/stubs") - elseif(DEFINED ENV{CUDA_PATH} AND EXISTS "$ENV{CUDA_PATH}/${_libdir}/stubs/libcuda.so") - # Use CUDA_PATH envvar (if set) - list(APPEND _cuda_stubs "$ENV{CUDA_PATH}/${_libdir}/stubs/libcuda.so") - elseif(DEFINED ENV{CUDA_LIB_PATH} AND EXISTS "$ENV{CUDA_LIB_PATH}/stubs/libcuda.so") - # Use CUDA_LIB_PATH envvar (if set) - list(APPEND _cuda_stubs "$ENV{CUDA_LIB_PATH}/stubs/libcuda.so") - elseif(DEFINED ENV{LIBRARY_PATH} AND - ("$ENV{LIBRARY_PATH}" STREQUAL "/usr/local/cuda/${_libdir}/stubs")) - # LIBRARY_PATH is set in the `nvidia/cuda` containers to /usr/local/cuda/lib64/stubs - list(APPEND _cuda_stubs "$ENV{LIBRARY_PATH}") + if(legate_core_STATIC_CUDA_RUNTIME) + list(APPEND _legion_cuda_options "CMAKE_CUDA_RUNTIME_LIBRARY STATIC") else() - message(FATAL_ERROR "Could not find the libcuda.so driver stub. " - "Please reconfigure with -DCUDAToolkit_ROOT= " - "set to a valid CUDA Toolkit installation.") + list(APPEND _legion_cuda_options "CMAKE_CUDA_RUNTIME_LIBRARY SHARED") endif() - - message(VERBOSE "legate.core: Path(s) to CUDA stubs: ${_cuda_stubs}") - - list(APPEND _legion_cuda_options "CUDA_NVCC_FLAGS ${_nvcc_flags}") - list(APPEND _legion_cuda_options "CMAKE_CUDA_STANDARD ${_cuda_std}") - # Set this so Legion correctly finds the CUDA toolkit. - list(APPEND _legion_cuda_options "CMAKE_LIBRARY_PATH ${_cuda_stubs}") - - # Set these as cache variables for the legacy FindCUDA.cmake - set(CUDA_VERBOSE_BUILD ON CACHE BOOL "" FORCE) - set(CUDA_USE_STATIC_CUDA_RUNTIME ${legate_core_STATIC_CUDA_RUNTIME} CACHE BOOL "" FORCE) - - # Ensure `${_cuda_stubs}/libcuda.so` doesn't end up in the RPATH of the legion_python binary - list(APPEND CMAKE_C_IMPLICIT_LINK_DIRECTORIES "${_cuda_stubs}") - list(APPEND CMAKE_CXX_IMPLICIT_LINK_DIRECTORIES "${_cuda_stubs}") - list(APPEND CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES "${_cuda_stubs}") - set(legate_core_cuda_stubs_path "${_cuda_stubs}" PARENT_SCOPE) - set(legate_core_cuda_stubs_path "${_cuda_stubs}" CACHE STRING "" FORCE) endif() # Because legion sets these as cache variables, we need to force set this as a cache variable here @@ -166,8 +109,6 @@ function(find_or_configure_legion) set(Legion_MAX_DIM ${Legion_MAX_DIM} CACHE STRING "The max number of dimensions for Legion" FORCE) set(Legion_MAX_FIELDS ${Legion_MAX_FIELDS} CACHE STRING "The max number of fields for Legion" FORCE) set(Legion_DEFAULT_LOCAL_FIELDS ${Legion_DEFAULT_LOCAL_FIELDS} CACHE STRING "Number of local fields for Legion" FORCE) - set(Legion_CUDA_ARCH ${Legion_CUDA_ARCH} CACHE STRING - "Comma-separated list of CUDA architectures to build for (e.g. 60,70)" FORCE) message(VERBOSE "legate.core: Legion version: ${version}") message(VERBOSE "legate.core: Legion git_repo: ${git_repo}") @@ -189,33 +130,24 @@ function(find_or_configure_legion) "CMAKE_CXX_STANDARD ${_cxx_std}" "Legion_VERSION ${version}" "Legion_BUILD_BINDINGS ON" - "Legion_BUILD_APPS OFF" - "Legion_BUILD_TESTS OFF" - "Legion_BUILD_TUTORIAL OFF" "Legion_REDOP_HALF ON" "Legion_REDOP_COMPLEX ON" - "Legion_GPU_REDUCTIONS OFF" "Legion_BUILD_RUST_PROFILER ON" - "Legion_BACKTRACE_USE_LIBDW ${Legion_BACKTRACE_USE_LIBDW}" - "Legion_SPY ${Legion_SPY}" - "Legion_USE_LLVM ${Legion_USE_LLVM}" - "Legion_USE_HDF5 ${Legion_USE_HDF5}" - "Legion_USE_CUDA ${Legion_USE_CUDA}" - "Legion_NETWORKS ${Legion_NETWORKS}" - "Legion_USE_OpenMP ${Legion_USE_OpenMP}" - "Legion_USE_Python ${Legion_USE_Python}" - "Legion_BOUNDS_CHECKS ${Legion_BOUNDS_CHECKS}" ) endif() set(Legion_USE_CUDA ${Legion_USE_CUDA} PARENT_SCOPE) set(Legion_USE_OpenMP ${Legion_USE_OpenMP} PARENT_SCOPE) set(Legion_USE_Python ${Legion_USE_Python} PARENT_SCOPE) + set(Legion_CUDA_ARCH ${Legion_CUDA_ARCH} PARENT_SCOPE) + set(Legion_BOUNDS_CHECKS ${Legion_BOUNDS_CHECKS} PARENT_SCOPE) set(Legion_NETWORKS ${Legion_NETWORKS} PARENT_SCOPE) message(VERBOSE "Legion_USE_CUDA=${Legion_USE_CUDA}") message(VERBOSE "Legion_USE_OpenMP=${Legion_USE_OpenMP}") message(VERBOSE "Legion_USE_Python=${Legion_USE_Python}") + message(VERBOSE "Legion_CUDA_ARCH=${Legion_CUDA_ARCH}") + message(VERBOSE "Legion_BOUNDS_CHECKS=${Legion_BOUNDS_CHECKS}") message(VERBOSE "Legion_NETWORKS=${Legion_NETWORKS}") endfunction() diff --git a/cmake/versions.json b/cmake/versions.json index 75a7b90fd6..eac50916a2 100644 --- a/cmake/versions.json +++ b/cmake/versions.json @@ -7,7 +7,7 @@ }, "Legion": { "git_url" : "https://gitlab.com/StanfordLegion/legion.git", - "git_tag" : "e770f4b212d3c2634427a408422697a9f47b3d76" + "git_tag" : "92c45583c07cb887fd5898e59c5c2f54a12d6d91" } } } diff --git a/conda/conda-build/build.sh b/conda/conda-build/build.sh index 95d3de9af2..768497beaa 100644 --- a/conda/conda-build/build.sh +++ b/conda/conda-build/build.sh @@ -10,13 +10,13 @@ CMAKE_ARGS+=" -DBUILD_MARCH=haswell -DLegion_USE_OpenMP=ON -DLegion_USE_Python=ON --DLegion_BUILD_BINDINGS=ON" +-DLegion_Python_Version=$($PYTHON --version 2>&1 | cut -d' ' -f2 | cut -d'.' -f3 --complement)" # We rely on an environment variable to determine if we need to build cpu-only bits if [ -z "$CPU_ONLY" ]; then CMAKE_ARGS+=" -DLegion_USE_CUDA=ON --DCMAKE_CUDA_ARCHITECTURES:LIST=60-real;70-real;75-real;80-real;90 +-DLegion_CUDA_ARCH:LIST=60-real;70-real;75-real;80-real;90 " fi diff --git a/conda/conda-build/meta.yaml b/conda/conda-build/meta.yaml index 40ec170915..4d35154954 100644 --- a/conda/conda-build/meta.yaml +++ b/conda/conda-build/meta.yaml @@ -98,8 +98,6 @@ requirements: - cmake {{ cmake_version }} - {{ compiler('c') }} =11.2 - {{ compiler('cxx') }} =11.2 - - elfutils # [linux] - - libdwarf # [linux] host: - zlib - python @@ -107,12 +105,15 @@ requirements: - llvm-openmp - scikit-build - numpy {{ numpy_version }} + - elfutils # [linux] + - libdwarf # [linux] {% if gpu_enabled_bool %} - nccl - cuda-nvcc ={{ cuda_version }} - cuda-nvtx ={{ cuda_version }} - cuda-cccl ={{ cuda_version }} - cuda-cudart ={{ cuda_version }} + - cuda-version ={{ cuda_version }} - cuda-nvml-dev ={{ cuda_version }} - cuda-driver-dev ={{ cuda_version }} - cuda-cudart-dev ={{ cuda_version }} @@ -122,6 +123,8 @@ requirements: - llvm-openmp - numpy {{ numpy_version }} - typing_extensions + - elfutils # [linux] + - libdwarf # [linux] {% if gpu_enabled_bool %} - cuda-cudart >={{ cuda_version }},<{{ cuda_major+1 }} - cuda-version >={{ cuda_version }},<{{ cuda_major+1 }} diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile index 03a3005d09..e5470c1140 100644 --- a/continuous_integration/Dockerfile +++ b/continuous_integration/Dockerfile @@ -1,70 +1,73 @@ -ARG BASE_IMAGE +ARG BASE_IMAGE=rapidsai/devcontainers:23.10-cpp-cuda11.8-mambaforge-ubuntu22.04 FROM ${BASE_IMAGE} as stage0 -SHELL ["/bin/bash", "-c"] +SHELL ["/bin/bash", "-Eeou", "pipefail", "-c"] -ENV PYTHONDONTWRITEBYTECODE=1 -ENV SCCACHE_REGION="us-east-2" -ENV SCCACHE_BUCKET="rapids-sccache-east" -ENV SCCACHE_S3_KEY_PREFIX=legate-cunumeric-dev -ENV VAULT_HOST=https://vault.ops.k8s.rapids.ai -ENV HISTFILE=/home/coder/.cache/._bash_history -ENV DEFAULT_CONDA_ENV=legate - -ARG USE_CUDA +ARG USE_CUDA=ON ENV USE_CUDA=${USE_CUDA} -ENV USE_OPENMP=ON +ARG USE_OPENMP=ON +ENV USE_OPENMP=${USE_OPENMP} -ENV BUILD_MARCH=nocona +ARG BUILD_MARCH=nocona +ENV BUILD_MARCH=${BUILD_MARCH} -ENV PATH="${PATH}:/home/coder/.local/bin" +ARG AWS_DEFAULT_REGION +ENV AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-"us-east-2"} + +ARG AWS_REGION +ENV AWS_REGION=${AWS_REGION:-${AWS_DEFAULT_REGION}} + +ENV DEFAULT_CONDA_ENV=legate +ENV PYTHONDONTWRITEBYTECODE=1 +ENV SCCACHE_REGION=${AWS_REGION} +ENV SCCACHE_BUCKET="rapids-sccache-devs" +ENV SCCACHE_S3_KEY_PREFIX=legate-cunumeric-dev +ENV VAULT_HOST=https://vault.ops.k8s.rapids.ai +# 8 hours +ENV VAULT_S3_TTL="28800s" USER coder -WORKDIR /home/coder/.cache WORKDIR /home/coder/.artifacts WORKDIR /home/coder -COPY --chown=coder:coder continuous_integration/home/coder/.gitconfig /home/coder/ COPY --chown=coder:coder continuous_integration/home/coder/.local/bin/* /home/coder/.local/bin/ COPY --chown=coder:coder . /home/coder/legate -RUN chmod a+x /home/coder/.local/bin/* && \ - mkdir -p /tmp/out && \ - chown -R coder:coder /tmp/out - #--------------------------------------------------- -FROM stage0 as setup - -USER coder -WORKDIR /home/coder +FROM stage0 as build -RUN set -x && . conda-utils && get_yaml_and_make_conda_env +# Mount in secrets +RUN --mount=type=secret,id=GH_TOKEN,uid=1000,gid=1000 \ + --mount=type=secret,id=AWS_ACCESS_KEY_ID,uid=1000,gid=1000 \ + --mount=type=secret,id=AWS_SESSION_TOKEN,uid=1000,gid=1000 \ + --mount=type=secret,id=AWS_SECRET_ACCESS_KEY,uid=1000,gid=1000 \ +<> ~/legate/conda/conda-build/conda_build_config.yaml - + cat < /tmp/legate/conda/conda-build/conda_build_config.yaml numpy: - 1.22 - +python: + - "${python_version}" +numpy_version: + - ">=1.22,<1.25" +use_local_path: + - "true" +gpu_enabled: + - "${GPU_ENABLED}" +cmake_version: + - ">=3.20.1,!=3.23.0" package_version: - - "$(git -C ~/legate describe --abbrev=0 --tags | sed 's/[a-zA-Z]//g' | cut -d '.' -f -2).00" + - "$(git -C /tmp/legate describe --abbrev=0 --tags | sed 's/[a-zA-Z]//g' | cut -d '.' -f -2).00" EOF - cat < ~/legate/conda/conda-build/build.sh + cat < /tmp/legate/conda/conda-build/build.sh # Install legate_core C++ libs -tar -C "\$PREFIX" --exclude="*.a" --strip-components=1 -xf /tmp/out/legate_core-*-Linux.tar.gz; +tar -C "\$PREFIX" --exclude="*.a" --strip-components=1 -xf ~/.artifacts/legate_core-*-Linux.tar.gz; sed -E -i "s@$HOME/\.conda/envs/legate@\$PREFIX@g" "\$PREFIX/share/Legion/cmake/LegionConfigCommon.cmake"; sed -E -i "s@$HOME/legate/build/_CPack_Packages/Linux/TGZ/legate_core-(.*)-Linux@\$PREFIX@g" "\$SP_DIR/legion_canonical_cffi.py"; # Install legate_core Python wheel -pip install --no-deps --root / --prefix "\$PREFIX" /tmp/out/legate.core-*.whl; +pip install --no-deps --root / --prefix "\$PREFIX" ~/.artifacts/legate_core-*.whl; # Legion leaves .egg-info files, which confuses conda trying to pick up the information # Remove them so legate-core is the only egg-info file added. rm -rf "\$SP_DIR"/legion*egg-info; EOF - git -C ~/legate add .; - git -C ~/legate commit --allow-empty --allow-empty-message -n -m ""; + git -C /tmp/legate add .; + git -C /tmp/legate commit --allow-empty --allow-empty-message -n -m ""; # Build legate_core conda package - CUDA=${CUDA_VERSION_MAJOR}.${CUDA_VERSION_MINOR} \ - conda mambabuild ${conda_build_args[@]} ~/legate/conda/conda-build; + time CUDA=${CUDA_VERSION_MAJOR}.${CUDA_VERSION_MINOR} \ + conda mambabuild ${conda_build_args[@]} /tmp/legate/conda/conda-build; - git -C ~/legate reset --hard HEAD~1; + git -C /tmp/legate reset --hard HEAD~1; - cp /tmp/conda-build/legate_core/linux-64/legate-*.tar.bz2 /tmp/out/; + rm -rf /tmp/legate; { set +x; } 2>/dev/null; } diff --git a/continuous_integration/home/coder/.local/bin/build-legate-cpp b/continuous_integration/home/coder/.local/bin/build-legate-cpp index f7516253a0..f0cb38ff44 100755 --- a/continuous_integration/home/coder/.local/bin/build-legate-cpp +++ b/continuous_integration/home/coder/.local/bin/build-legate-cpp @@ -8,13 +8,12 @@ build_legate_cpp() { cmake_args+=(-DBUILD_SHARED_LIBS=ON); cmake_args+=(-DBUILD_MARCH=${BUILD_MARCH:-haswell}); cmake_args+=(-DCMAKE_BUILD_TYPE=Release); - cmake_args+=(-DCMAKE_CUDA_ARCHITECTURES=RAPIDS); + cmake_args+=(-DLegion_CUDA_ARCH=all-major); cmake_args+=(-DLegion_NETWORKS=); cmake_args+=(-DLegion_USE_Python=ON); - cmake_args+=(-DLegion_REDOP_HALF=ON); - cmake_args+=(-DLegion_REDOP_COMPLEX=ON); + cmake_args+=(-DLegion_PYTHON_EXTRA_INSTALL_ARGS="--root;/;--prefix;\"\${CMAKE_INSTALL_PREFIX}\""); + cmake_args+=(-DLegion_Python_Version=$(python3 --version 2>&1 | cut -d' ' -f2 | cut -d'.' -f3 --complement)); cmake_args+=(-DLegion_BUILD_JUPYTER=ON); - cmake_args+=(-DLegion_BUILD_BINDINGS=ON); cmake_args+=(-DLegion_BOUNDS_CHECKS=${BOUNDS_CHECKS:-OFF}); cmake_args+=(-DLegion_EMBED_GASNet_CONFIGURE_ARGS=${GASNet_CONFIGURE_ARGS:---with-ibv-max-hcas=8}); cmake_args+=(-DLegion_MAX_DIM=${MAX_DIM:-4}); @@ -28,28 +27,27 @@ build_legate_cpp() { cmake_args+=(-DCMAKE_BUILD_PARALLEL_LEVEL=${JOBS:-$(nproc --ignore=1)}); cmake_args+=(${@}); - sccache --stop-server >/dev/null 2>&1 || true; sccache --show-stats; - time CMAKE_BUILD_PARALLEL_LEVEL=${JOBS:-$(nproc --ignore=1)} \ - cmake -S ~/legate -B ~/legate/build "${cmake_args[@]}" -GNinja; + rm -rf ~/legate/build; - sed -i \ - 's/= unknown/= unknown + ["--prefix", args.prefix]/' \ - ~/legate/build/_deps/legion-src/bindings/python/setup.py; + time OPENSSL_DIR="${CONDA_PREFIX}" \ + CMAKE_BUILD_PARALLEL_LEVEL=${JOBS:-$(nproc --ignore=1)} \ + cmake -S ~/legate -B ~/legate/build "${cmake_args[@]}" -GNinja; sccache --show-stats; - time CMAKE_BUILD_PARALLEL_LEVEL=${JOBS:-$(nproc --ignore=1)} \ + time OPENSSL_DIR="${CONDA_PREFIX}" \ + CMAKE_BUILD_PARALLEL_LEVEL=${JOBS:-$(nproc --ignore=1)} \ cmake --build ~/legate/build --verbose --parallel ${JOBS:-$(nproc --ignore=1)}; sccache --show-stats; ( - mkdir -p /tmp/out; + mkdir -p ~/.artifacts; cd ~/legate/build; cpack -G TGZ; - cp ./*-Linux.tar.gz /tmp/out/; + cp ./*-Linux.tar.gz ~/.artifacts/; ); { set +x; } 2>/dev/null; diff --git a/continuous_integration/home/coder/.local/bin/build-legate-wheel b/continuous_integration/home/coder/.local/bin/build-legate-wheel index 3537b28714..7f54550ccd 100755 --- a/continuous_integration/home/coder/.local/bin/build-legate-wheel +++ b/continuous_integration/home/coder/.local/bin/build-legate-wheel @@ -3,10 +3,12 @@ build_legate_wheel() { set -xeuo pipefail; - mkdir -p /tmp/out; + mkdir -p ~/.artifacts; + + rm -rf ~/legate/_skbuild; local pip_args=(-vv); - pip_args+=(--wheel-dir /tmp/out); + pip_args+=(--wheel-dir ~/.artifacts); if type conda 2>&1 >/dev/null; then pip_args+=(--no-deps); @@ -18,7 +20,7 @@ build_legate_wheel() { cmake_args+=("-Dlegate_core_ROOT=$HOME/legate/build"); # Build + package legate.core Python wheel - CMAKE_ARGS="${cmake_args[@]}" \ + time CMAKE_ARGS="${cmake_args[@]}" \ pip wheel ${pip_args[@]} ~/legate; { set +x; } 2>/dev/null; diff --git a/continuous_integration/home/coder/.local/bin/conda-utils b/continuous_integration/home/coder/.local/bin/conda-utils old mode 100644 new mode 100755 index a7c7b29f7e..2f5027afe5 --- a/continuous_integration/home/coder/.local/bin/conda-utils +++ b/continuous_integration/home/coder/.local/bin/conda-utils @@ -1,61 +1,5 @@ -make_conda_env_from_yaml() { - mamba env create -v -n "${DEFAULT_CONDA_ENV:-legate}" -f "${yaml_file}"; -} - -generate_yaml_file() { - local cuda_version="${CUDA_VERSION:-${CUDA_VERSION_MAJOR}.${CUDA_VERSION_MINOR}}"; - cuda_version="$(echo "${cuda_version}" | cut -d'.' -f3 --complement)"; - - local python_version="${PYTHON_VERSION:-}"; - - if [ -z "${python_version}" ]; then - python_version="$(python3 --version 2>&1 | cut -d' ' -f2 | cut -d'.' -f3 --complement)"; - fi - - yaml_file=~/"$( \ - ~/legate/scripts/generate-conda-envs.py \ - --os linux \ - --compilers \ - --ctk ${cuda_version} \ - --python ${python_version} \ - --openmpi \ - --no-ucx \ - | head -n1 | cut -d' ' -f3 \ - )" - - sed -i -re "s/legate-test/${DEFAULT_CONDA_ENV:-legate}/g" "${yaml_file}"; - echo " - boa" >> "${yaml_file}"; - - cp "${yaml_file}" /tmp/out/ - mkdir -p /tmp/env_yaml - cp "${yaml_file}" /tmp/env_yaml -} - -find_yaml_file() { - pattern="/tmp/env_yaml/*.yaml"; - files=( $pattern ); - yaml_file="${files[0]}"; - - if [ -z "${yaml_file:-}" ] || [ ! -f "$yaml_file" ]; then - return 1; - fi - - return 0; -} - get_yaml_and_make_conda_env() { - set -e; - - local yaml_file=""; - - if ! find_yaml_file; then - generate_yaml_file; - fi - - echo YAML file: ${yaml_file} - cat "${yaml_file}"; - - make_conda_env_from_yaml; + make-conda-env; } install_legate_core_with_war() { @@ -66,10 +10,3 @@ install_legate_core_with_war() { mamba install -y -n "${DEFAULT_CONDA_ENV:-legate}" -c nvidia -c conda-forge -c /tmp/conda-build/legate_core legate-core; } - -activate_conda_env() { - . /opt/conda/etc/profile.d/conda.sh; - . /opt/conda/etc/profile.d/mamba.sh; - - conda activate ${DEFAULT_CONDA_ENV:-legate}; -} \ No newline at end of file diff --git a/continuous_integration/home/coder/.local/bin/copy-artifacts b/continuous_integration/home/coder/.local/bin/copy-artifacts deleted file mode 100644 index 9ca293535d..0000000000 --- a/continuous_integration/home/coder/.local/bin/copy-artifacts +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -copy_artifacts() { - set -xeuo pipefail - sudo chown coder:coder ~/.artifacts - cp -r /tmp/out ~/.artifacts - cp -r /tmp/conda-build ~/.artifacts -} - -(copy_artifacts "$@"); diff --git a/continuous_integration/home/coder/.local/bin/entrypoint b/continuous_integration/home/coder/.local/bin/entrypoint index 8e9178dd97..2c77fd68ae 100755 --- a/continuous_integration/home/coder/.local/bin/entrypoint +++ b/continuous_integration/home/coder/.local/bin/entrypoint @@ -1,45 +1,47 @@ #!/usr/bin/env bash -sccache_stop_server_and_show_stats() { - sccache --stop-server || true && sccache --show-stats; -} +entrypoint() { + set -euo pipefail; -init_devcontainer() { # disable xtrace and history - local xtrace_enabled=$(echo "${SHELLOPTS:-}" | grep -q 'xtrace'; echo $?); - local history_enabled=$(echo "${SHELLOPTS:-}" | grep -q 'history'; echo $?); { set +xo history; } 2>/dev/null; - eval "export $(find /run/secrets/ -type f -exec bash -c 'echo ${0/\/run\/secrets\//}=$(<${0})' {} \;)"; - if [ "${history_enabled}" -eq "0" ]; then { set -o history; } 2>/dev/null; fi; - if [ "${xtrace_enabled}" -eq "0" ]; then { set -o xtrace; } 2>/dev/null; fi; - - . devcontainer-utils-post-attach-command; - - sleep 10; - . devcontainer-utils-vault-s3-test; - . devcontainer-utils-vault-s3-export 0; -} - -entrypoint() { - set -x - - echo AWS_REGION=${AWS_REGION:-} - echo AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN:-} - echo AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-} - echo AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-} mkdir -p /home/coder/.cache; - local secrets_dir=/run/secrets - - if [ -d "$secrets_dir" ] && [ "$(ls -A $secrets_dir)" ]; then - init_devcontainer - else - sccache_stop_server_and_show_stats + # export run secrets as shell vars + eval "$( \ + find /run/secrets/ -type f -exec \ + bash -c 'echo ${0/\/run\/secrets\//}=$(<${0})' {} \; \ + | xargs -r -d'\n' -I% echo -n export %\; \ + )"; + + # set SCCACHE_BUCKET to an empty string if we don't have creds + if grep -qE "^$" <<< "${GH_TOKEN:-}" \ + && grep -qE "^$" <<< "${AWS_ACCESS_KEY_ID:-}" \ + && grep -qE "^$" <<< "${AWS_SECRET_ACCESS_KEY:-}"; then + export SCCACHE_BUCKET=""; fi - exec "$@"; + . devcontainer-utils-post-attach-command; + + # report sccache variables for debugging + for x in "GH_TOKEN" \ + "SCCACHE_BUCKET" \ + "SCCACHE_REGION" \ + "AWS_SESSION_TOKEN" \ + "AWS_ACCESS_KEY_ID" \ + "AWS_SECRET_ACCESS_KEY"; do + if ! grep -qE "^$" <<< "${!x:-}"; then + echo "${x}=${!x:-}"; + fi + done + + # reenable xtrace and history + { set -xo history; } 2>/dev/null; + + if test "${#@}" -gt 0; then + exec; + fi } entrypoint "$@"; - diff --git a/continuous_integration/home/coder/.local/bin/make-conda-env b/continuous_integration/home/coder/.local/bin/make-conda-env new file mode 100755 index 0000000000..b8ffc2c33c --- /dev/null +++ b/continuous_integration/home/coder/.local/bin/make-conda-env @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +get_yaml_and_make_conda_env() { + set -e; + + local cuda_version="${CUDA_VERSION:-${CUDA_VERSION_MAJOR}.${CUDA_VERSION_MINOR}}"; + cuda_version="$(echo "${cuda_version}" | cut -d'.' -f3 --complement)"; + + local python_version="${PYTHON_VERSION:-}"; + + if [ -z "${python_version}" ]; then + python_version="$(python3 --version 2>&1 | cut -d' ' -f2 | cut -d'.' -f3 --complement)"; + fi + + mkdir -p ~/.artifacts; + cd ~/.artifacts; + + local yaml_file=~/.artifacts/"$( \ + ~/legate/scripts/generate-conda-envs.py \ + --os linux \ + --compilers \ + --ctk ${cuda_version} \ + --python ${python_version} \ + --openmpi \ + --no-ucx \ + | head -n1 | cut -d' ' -f3 \ + )" + + sed -i -re "s/legate-test/${DEFAULT_CONDA_ENV:-legate}/g" "${yaml_file}"; + echo " - boa" >> "${yaml_file}"; + + echo YAML file: "${yaml_file}" + cat "${yaml_file}"; + + mamba env create -v -n "${DEFAULT_CONDA_ENV:-legate}" -f "${yaml_file}"; +} + +get_yaml_and_make_conda_env "$@"; diff --git a/continuous_integration/home/coder/.local/bin/run-test-or-analysis b/continuous_integration/home/coder/.local/bin/run-test-or-analysis deleted file mode 100644 index 155288f613..0000000000 --- a/continuous_integration/home/coder/.local/bin/run-test-or-analysis +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -. conda-utils - -run_test_or_analysis() { - set -x - cd ~/ - - install_legate_core_with_war; - - activate_conda_env; - - conda info; - - set -xeuo pipefail - - case "$1" in - "unit") - echo "Executing unit tests..." - mamba install -y -n "${DEFAULT_CONDA_ENV:-legate}" -c conda-forge pytest pytest-mock ipython jupyter_client - cd ~/legate/tests/unit - pytest - ;; - "mypy") - echo "Executing mypy..." - mamba install -y -n "${DEFAULT_CONDA_ENV:-legate}" mypy - cd ~/legate - mypy legate - ;; - *) - echo "Invalid command: $1" - return 1 - ;; - esac - - return 0 -} - -(run_test_or_analysis "$@"); \ No newline at end of file diff --git a/continuous_integration/home/coder/.local/bin/test-legate-python b/continuous_integration/home/coder/.local/bin/test-legate-python new file mode 100755 index 0000000000..5797213eab --- /dev/null +++ b/continuous_integration/home/coder/.local/bin/test-legate-python @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +run_legate_pytest() { + set -euo pipefail; + echo "Executing unit tests..." + mamba run \ + -n "${DEFAULT_CONDA_ENV:-legate}" \ + --cwd ~/legate/tests/unit \ + --live-stream \ + pytest; +} + +run_legate_pytest "$@"; diff --git a/continuous_integration/home/coder/.local/bin/type-check-legate-python b/continuous_integration/home/coder/.local/bin/type-check-legate-python new file mode 100755 index 0000000000..565c271189 --- /dev/null +++ b/continuous_integration/home/coder/.local/bin/type-check-legate-python @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +run_legate_pytest() { + set -euo pipefail; + echo "Executing mypy..."; + mamba run \ + -n "${DEFAULT_CONDA_ENV:-legate}" \ + --cwd ~/legate \ + --live-stream \ + mypy legate; +} + +run_legate_pytest "$@"; diff --git a/install.py b/install.py index 22a6e2cc3a..dbedd7b58a 100755 --- a/install.py +++ b/install.py @@ -68,9 +68,11 @@ def __call__(self, parser, namespace, values, option_string): setattr(namespace, self.dest, not option_string.startswith("--no")) -def execute_command(args, verbose, **kwargs): +def execute_command(args, verbose, ignore_errors=False, **kwargs): if verbose: print(f"Executing: {' '.join(args)} with {kwargs}") + if ignore_errors: + subprocess.call(args, **kwargs) subprocess.check_call(args, **kwargs) @@ -144,27 +146,61 @@ def was_previously_built_with_different_build_isolation( return False -def get_install_dir(): +def get_install_dir(args: list): # Infer the location where to install the Legion Python bindings, # otherwise they'll only be installed into the local scikit-build # cmake-install dir # Install into conda prefix if defined - if "CONDA_PREFIX" in os.environ: - return os.environ["CONDA_PREFIX"] + if "PREFIX" in os.environ and (os.environ.get("CONDA_BUILD", "0") == "1"): + install_dir = os.environ["PREFIX"] + return (install_dir, ["--root", "/", "--prefix", install_dir]) + elif "CONDA_PREFIX" in os.environ: + install_dir = os.environ["CONDA_PREFIX"] + return (install_dir, ["--root", "/", "--prefix", install_dir]) + + def get_arg(flag: str): + idx = args.index(flag) if flag in args else -1 + if idx > -1: + args.remove(flag) + if idx + 1 < len(args): + args.remove(args[idx + 1]) + return args[idx + 1] + return None + + install_dir = None + install_args = [] + if root_dir := get_arg("--root"): + install_dir = root_dir + install_args += ["--root", root_dir] + + if prefix_dir := get_arg("--prefix"): + install_dir = [install_dir, prefix_dir] + install_dir = filter(lambda x: x, install_dir) + install_dir = os.path.join(*install_dir, "") + install_args += ["--prefix", prefix_dir] + + if install_dir and os.path.exists(install_dir): + return (install_dir, install_args) import site # Try to install into user site packages first? - if site.ENABLE_USER_SITE and os.path.exists( - user_site_pkgs := site.getusersitepackages() + if ("--user" in args or site.ENABLE_USER_SITE) and os.path.exists( + user_base := site.getuserbase() ): - return user_site_pkgs + if "--user" in args: + args.remove("--user") + return (user_base, ["--user"]) # Otherwise fallback to regular site-packages? for site_pkgs in site.getsitepackages(): - if os.path.exists(site_pkgs): - return site_pkgs + install_dir = os.path.join(site_pkgs, "..", "..", "..") + install_dir = os.path.realpath(install_dir) + if install_dir != "/" and os.path.exists(install_dir): + return (install_dir, ["--root", "/", "--prefix", install_dir]) + + return (None, []) def install_legion_python_bindings( @@ -258,6 +294,7 @@ def install( check_bounds, clean_first, extra_flags, + build_tests, build_examples, editable, build_isolation, @@ -369,60 +406,75 @@ def validate_path(path): print("Performing a clean build to accommodate build isolation.") clean_first = True + cmd_env = dict(os.environ.items()) + + # Explicitly uninstall legate.core if doing a clean/isolated build. + # + # A prior installation may have built and installed legate.core C++ + # dependencies (like Legion). + # + # CMake will find and use them for the current build, which would normally + # be correct, but pip uninstalls files from any existing installation as + # the last step of the install process, including the libraries found by + # CMake during the current build. + # + # Therefore this uninstall step must occur *before* CMake attempts to find + # these dependencies, triggering CMake to build and install them again. + if clean_first or (build_isolation and not editable): + execute_command( + [sys.executable, "-m", "pip", "uninstall", "-y", "legate-core"], + verbose, + ignore_errors=True, + cwd=legate_core_dir, + env=cmd_env, + ) + if clean_first: shutil.rmtree(skbuild_dir, ignore_errors=True) shutil.rmtree(join(legate_core_dir, "dist"), ignore_errors=True) shutil.rmtree(join(legate_core_dir, "build"), ignore_errors=True) shutil.rmtree( - join(legate_core_dir, "legate.core.egg-info"), + join(legate_core_dir, "legate_core.egg-info"), ignore_errors=True, ) # Configure and build legate.core via setup.py pip_install_cmd = [sys.executable, "-m", "pip", "install"] - cmd_env = dict(os.environ.items()) + + # Use preexisting CMAKE_ARGS from conda if set + cmake_flags = cmd_env.get("CMAKE_ARGS", "").split(" ") if "OPENSSL_DIR" not in cmd_env and "CONDA_PREFIX" in cmd_env: cmd_env.update({"OPENSSL_DIR": cmd_env["CONDA_PREFIX"]}) - if unknown is not None: - try: - prefix_loc = unknown.index("--prefix") - prefix_dir = validate_path(unknown[prefix_loc + 1]) - if prefix_dir is not None: - install_dir = prefix_dir - unknown = unknown[:prefix_loc] + unknown[prefix_loc + 2 :] - except Exception: - pass + if unknown is None: + unknown = [] + + install_dir, install_args = get_install_dir(unknown) - install_dir = get_install_dir() + pip_install_cmd += install_args + pip_install_cmd += unknown if verbose: print(f"install_dir: {install_dir}") + print("install_args:", install_args) - if install_dir is not None: - pip_install_cmd += ["--root", "/", "--prefix", str(install_dir)] + # editable implies build_isolation = False + if editable or not build_isolation: + pip_install_cmd += ["--no-deps", "--no-build-isolation"] if editable: - # editable implies build_isolation = False - pip_install_cmd += ["--no-deps", "--no-build-isolation", "--editable"] cmd_env.update({"SETUPTOOLS_ENABLE_FEATURES": "legacy-editable"}) + cmake_flags += ["-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON"] + pip_install_cmd += ["--editable"] else: - if not build_isolation: - pip_install_cmd += ["--no-deps", "--no-build-isolation"] pip_install_cmd += ["--upgrade"] - if unknown is not None: - pip_install_cmd += unknown - pip_install_cmd += ["."] if verbose: pip_install_cmd += ["-vv"] - # Also use preexisting CMAKE_ARGS from conda if set - cmake_flags = cmd_env.get("CMAKE_ARGS", "").split(" ") - if debug or verbose: cmake_flags += [f"--log-level={'DEBUG' if debug else 'VERBOSE'}"] @@ -432,7 +484,6 @@ def validate_path(path): )} -DBUILD_SHARED_LIBS=ON -DBUILD_MARCH={march} --DCMAKE_CUDA_ARCHITECTURES={arch} -DLegion_MAX_DIM={str(maxdim)} -DLegion_MAX_FIELDS={str(maxfields)} -DLegion_SPY={("ON" if spy else "OFF")} @@ -444,13 +495,12 @@ def validate_path(path): -DLegion_USE_HDF5={("ON" if hdf else "OFF")} -DLegion_USE_Python=ON -DLegion_Python_Version={pyversion} --DLegion_REDOP_COMPLEX=ON --DLegion_REDOP_HALF=ON --DLegion_BUILD_BINDINGS=ON -DLegion_BUILD_JUPYTER=ON -DLegion_EMBED_GASNet_CONFIGURE_ARGS="--with-ibv-max-hcas=8" """.splitlines() + if cuda: + cmake_flags += [f"-DLegion_CUDA_ARCH={arch}"] if nccl_dir: cmake_flags += [f"-DNCCL_DIR={nccl_dir}"] if gasnet_dir: @@ -477,8 +527,14 @@ def validate_path(path): cmake_flags += [f"-Dlegate_core_LEGION_BRANCH={legion_branch}"] if build_docs: cmake_flags += ["-Dlegate_core_BUILD_DOCS=ON"] + if build_tests: + cmake_flags += ["-Dlegate_core_BUILD_TESTS=ON"] if build_examples: - cmake_flags += ["-Dlegate_core_EXAMPLE_BUILD_TESTS=ON"] + cmake_flags += ["-Dlegate_core_BUILD_EXAMPLES=ON"] + if len(install_args) > 0: + cmake_flags += [ + f'-DLegion_PYTHON_EXTRA_INSTALL_ARGS="{";".join(install_args)}"' + ] cmake_flags += extra_flags build_flags = [f"-j{str(thread_count)}"] @@ -496,6 +552,14 @@ def validate_path(path): } ) + if not ("OPENSSL_DIR" in os.environ): + if "PREFIX" in os.environ and ( + os.environ.get("CONDA_BUILD", "0") == "1" + ): + cmd_env.update({"OPENSSL_DIR": os.environ["PREFIX"]}) + elif "CONDA_PREFIX" in os.environ: + cmd_env.update({"OPENSSL_DIR": os.environ["CONDA_PREFIX"]}) + # execute python -m pip install . execute_command(pip_install_cmd, verbose, cwd=legate_core_dir, env=cmd_env) @@ -594,7 +658,7 @@ def driver(): dest="arch", action="store", required=False, - default="NATIVE", + default="all-major", help="Specify the target GPU architecture.", ) parser.add_argument( @@ -705,6 +769,13 @@ def driver(): default=[], help="Extra CMake flags.", ) + parser.add_argument( + "--build-tests", + dest="build_tests", + action=BooleanFlag, + default=False, + help="Whether to build the tests", + ) parser.add_argument( "--build-examples", dest="build_examples", diff --git a/legate/install_info.py.in b/legate/install_info.py.in index d416e48d24..fc60b09999 100644 --- a/legate/install_info.py.in +++ b/legate/install_info.py.in @@ -32,8 +32,10 @@ def get_libpath(): return libdir return None + from .util.fs import get_legate_paths + return ( - find_liblgcore(join(lg_path, "build", "lib")) or + find_liblgcore(get_legate_paths().legate_lib_path) or find_liblgcore(join(dirname(dirname(dirname(lg_path))), "lib")) or find_liblgcore(join(dirname(dirname(sys.executable)), "lib")) or "" @@ -54,4 +56,4 @@ ON, OFF = True, False use_cuda: bool = @Legion_USE_CUDA@ -use_openmp: bool = @Legion_USE_OpenMP@ \ No newline at end of file +use_openmp: bool = @Legion_USE_OpenMP@ diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index 4ec1b29086..393670e997 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -1,5 +1,5 @@ #============================================================================= -# Copyright 2022 NVIDIA Corporation +# Copyright 2022-2023 NVIDIA Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,6 +17,8 @@ ############################################################################## # - User Options ------------------------------------------------------------ +option(legate_core_BUILD_TESTS OFF) +option(legate_core_BUILD_EXAMPLES OFF) include(cmake/Modules/legate_core_options.cmake) ############################################################################## @@ -64,41 +66,6 @@ macro(_find_package_Python3) message(VERBOSE "legate.core: Python 3 version: ${Python3_VERSION}") endmacro() -# CUDA initialization might need to happen at different times depending on -# how Legion is built. If building Legion inline, CUDA must be enabled -# BEFORE get_legion.cmake because of how Legion handles CMAKE_CUDA_ARCHITECURES. -# If using an external Legion, CUDA must be enabled AFTER get_legion.cmake. -# This function executes all the enable CUDA functions with a boolean guard -# to make sure it is only executed once. -macro(_enable_cuda_language) - if (NOT legate_core_CUDA_ENABLED) - include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/cuda_arch_helpers.cmake) - # Needs to run before `rapids_cuda_init_architectures` - set_cuda_arch_from_names() - # Must come before `enable_language(CUDA)` - rapids_cuda_init_architectures(legate_core) - # Enable the CUDA language - enable_language(CUDA) - # Since legate_core only enables CUDA optionally we need to manually include - # the file that rapids_cuda_init_architectures relies on `project` calling - if(CMAKE_PROJECT_legate_core_INCLUDE) - include("${CMAKE_PROJECT_legate_core_INCLUDE}") - endif() - # Must come after `enable_language(CUDA)` - # Use `-isystem ` instead of `-isystem=` - # because the former works with clangd intellisense - set(CMAKE_INCLUDE_SYSTEM_FLAG_CUDA "-isystem ") - # set to TRUE so the macro does not repeat if called again. - set(legate_core_CUDA_ENABLED TRUE) - # Find the CUDAToolkit - rapids_find_package( - CUDAToolkit REQUIRED - BUILD_EXPORT_SET legate-core-exports - INSTALL_EXPORT_SET legate-core-exports - ) - endif() -endmacro() - if(Legion_USE_Python) _find_package_Python3() if(Python3_FOUND AND Python3_VERSION) @@ -106,8 +73,11 @@ if(Legion_USE_Python) endif() endif() +include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/cuda_arch_helpers.cmake) + if(Legion_USE_CUDA) - _enable_cuda_language() + # Needs to run before find_package(Legion) + set_cuda_arch_from_names() endif() ### @@ -131,8 +101,18 @@ if(Legion_NETWORKS) endif() if(Legion_USE_CUDA) - # If CUDA has not yet been enabled, make sure it is now. - _enable_cuda_language() + # Enable the CUDA language + enable_language(CUDA) + # Must come after `enable_language(CUDA)` + # Use `-isystem ` instead of `-isystem=` + # because the former works with clangd intellisense + set(CMAKE_INCLUDE_SYSTEM_FLAG_CUDA "-isystem ") + # Find the CUDAToolkit + rapids_find_package( + CUDAToolkit REQUIRED + BUILD_EXPORT_SET legate-core-exports + INSTALL_EXPORT_SET legate-core-exports + ) # Find NCCL include(cmake/thirdparty/get_nccl.cmake) endif() @@ -165,11 +145,12 @@ if(Legion_USE_CUDA) list(APPEND legate_core_CXX_DEFS LEGATE_USE_CUDA) list(APPEND legate_core_CUDA_DEFS LEGATE_USE_CUDA) - add_cuda_architecture_defines(legate_core_CUDA_DEFS) + add_cuda_architecture_defines(legate_core_CUDA_DEFS ARCHS ${Legion_CUDA_ARCH}) list(APPEND legate_core_CUDA_OPTIONS -Xfatbin=-compress-all) list(APPEND legate_core_CUDA_OPTIONS --expt-extended-lambda) list(APPEND legate_core_CUDA_OPTIONS --expt-relaxed-constexpr) + list(APPEND legate_core_CUDA_OPTIONS -Wno-deprecated-gpu-targets) endif() if(Legion_USE_OpenMP) @@ -267,6 +248,10 @@ set_target_properties(legate_core INTERFACE_POSITION_INDEPENDENT_CODE ON LIBRARY_OUTPUT_DIRECTORY lib) +if(Legion_USE_CUDA) + set_property(TARGET legate_core PROPERTY CUDA_ARCHITECTURES ${Legion_CUDA_ARCH}) +endif() + # Add Conda library, and include paths if specified if(TARGET conda_env) target_link_libraries(legate_core PRIVATE conda_env) @@ -306,20 +291,6 @@ target_include_directories(legate_core $ ) -if(Legion_USE_CUDA) - file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/fatbin.ld" - [=[ -SECTIONS -{ -.nvFatBinSegment : { *(.nvFatBinSegment) } -.nv_fatbin : { *(.nv_fatbin) } -} -]=]) - - # ensure CUDA symbols aren't relocated to the middle of the debug build binaries - target_link_options(legate_core PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/fatbin.ld") -endif() - ############################################################################## # - Doxygen target------------------------------------------------------------ @@ -330,6 +301,7 @@ if (legate_core_BUILD_DOCS) list(APPEND legate_core_DOC_SOURCES # type src/core/type/type_info.h + src/core/type/type_traits.h # task src/core/task/task.h src/core/task/registrar.h @@ -354,7 +326,6 @@ if (legate_core_BUILD_DOCS) # utilities src/core/utilities/debug.h src/core/utilities/dispatch.h - src/core/utilities/type_traits.h # main page src/legate.h ) @@ -483,6 +454,7 @@ endif() "set(Legion_USE_CUDA ${Legion_USE_CUDA})" "set(Legion_USE_OpenMP ${Legion_USE_OpenMP})" "set(Legion_USE_Python ${Legion_USE_Python})" + "set(Legion_CUDA_ARCH ${Legion_CUDA_ARCH})" "set(Legion_NETWORKS ${Legion_NETWORKS})" "set(Legion_BOUNDS_CHECKS ${Legion_BOUNDS_CHECKS})" [=[ @@ -493,13 +465,6 @@ endif() "${helper_functions}" ) -if(DEFINED legate_core_cuda_stubs_path) - string(JOIN "\n" code_string "${code_string}" - "list(APPEND CMAKE_C_IMPLICIT_LINK_DIRECTORIES ${legate_core_cuda_stubs_path})" - "list(APPEND CMAKE_CXX_IMPLICIT_LINK_DIRECTORIES ${legate_core_cuda_stubs_path})" - "list(APPEND CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES ${legate_core_cuda_stubs_path})") -endif() - rapids_export( INSTALL legate_core EXPORT_SET legate-core-exports @@ -520,11 +485,15 @@ rapids_export( FINAL_CODE_BLOCK code_string LANGUAGES ${ENABLED_LANGUAES} ) -option(legate_core_EXAMPLE_BUILD_TESTS OFF) + include(cmake/legate_helper_functions.cmake) -if (legate_core_EXAMPLE_BUILD_TESTS) - set(legate_core_ROOT ${CMAKE_CURRENT_BINARY_DIR}) + +set(legate_core_ROOT ${CMAKE_CURRENT_BINARY_DIR}) + +if(legate_core_BUILD_TESTS) + add_subdirectory(tests/integration) +endif() + +if(legate_core_BUILD_EXAMPLES) add_subdirectory(examples) endif() -set(legate_core_ROOT ${CMAKE_CURRENT_BINARY_DIR}) -add_subdirectory(tests/integration) diff --git a/legate_core_python.cmake b/legate_core_python.cmake index d23296d86f..bfa6d8296a 100644 --- a/legate_core_python.cmake +++ b/legate_core_python.cmake @@ -38,10 +38,7 @@ endif() if(NOT legate_core_FOUND) set(SKBUILD OFF) set(Legion_USE_Python ON) - set(Legion_BUILD_BINDINGS ON) - add_subdirectory(. "${CMAKE_CURRENT_SOURCE_DIR}/build") - get_target_property(cython_lib_dir legate_core LIBRARY_OUTPUT_DIRECTORY) - set(cython_lib_dir "${CMAKE_CURRENT_SOURCE_DIR}/build/${cython_lib_dir}") + add_subdirectory(. legate-core-cpp) set(SKBUILD ON) endif() @@ -66,14 +63,22 @@ rapids_cython_init() add_subdirectory(legate/core/_lib) -if(DEFINED cython_lib_dir) - rapids_cython_add_rpath_entries(TARGET legate_core PATHS "${cython_lib_dir}") +set(cython_lib_dir "../../") + +if(CMAKE_INSTALL_RPATH_USE_LINK_PATH AND (TARGET legate_core)) + get_target_property(cython_lib_dir legate_core LIBRARY_OUTPUT_DIRECTORY) + get_target_property(legate_cpp_dir legate_core BINARY_DIR) + if(legate_cpp_dir) + set(cython_lib_dir "${legate_cpp_dir}/${cython_lib_dir}") + endif() endif() +rapids_cython_add_rpath_entries(TARGET legate_core PATHS "${cython_lib_dir}") + ############################################################################## # - conda environment -------------------------------------------------------- -rapids_cmake_support_conda_env(conda_env) +rapids_cmake_support_conda_env(conda_env MODIFY_PREFIX_PATH) # We're building python extension libraries, which must always be installed # under lib/, even if the system normally uses lib64/. Rapids-cmake currently diff --git a/pyproject.toml b/pyproject.toml index c9f0e8cb3b..6cad7eb533 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,12 +14,12 @@ [build-system] requires = [ - "wheel", - "ninja", - "setuptools", - "scikit-build>=0.13.1", "cmake>=3.22.1,!=3.23.0,!=3.25.0", "cython", + "ninja", + "scikit-build>=0.13.1", + "setuptools>=60", + "wheel", ] [tool.black] @@ -98,5 +98,6 @@ module = [ "legate.__main__", "legate.install_info", "legate._sphinxext.*", + "legate.jupyter.*", ] ignore_errors = true diff --git a/scripts/build-install.sh b/scripts/build-install.sh index a4d671cc36..ac6c896e39 100755 --- a/scripts/build-install.sh +++ b/scripts/build-install.sh @@ -10,7 +10,7 @@ source ./scripts/util/compiler-flags.sh source ./scripts/util/uninstall-global-legion-and-legate-core.sh # Remove existing build artifacts -rm -rf ./{build,_skbuild,dist,legate.core.egg-info} +rm -rf ./{build,_skbuild,dist,legate_core.egg-info} # Define CMake configuration arguments cmake_args="${CMAKE_ARGS:-}" @@ -22,7 +22,7 @@ if [[ -n "$(which ninja)" ]]; then cmake_args+=" -GNinja"; fi cmake_args+=" -D Legion_USE_CUDA=ON -D Legion_USE_OpenMP=ON --D CMAKE_CUDA_ARCHITECTURES=NATIVE +-D Legion_CUDA_ARCH=native "; # Use all but 2 threads to compile diff --git a/scripts/build-no-install.sh b/scripts/build-no-install.sh index 50827ce7ad..b6ec10b55c 100755 --- a/scripts/build-no-install.sh +++ b/scripts/build-no-install.sh @@ -8,7 +8,7 @@ source ./scripts/util/build-caching.sh source ./scripts/util/compiler-flags.sh # Remove existing build artifacts -rm -rf ./{build,_skbuild,dist,legate.core.egg-info} +rm -rf ./{build,_skbuild,dist,legate_core.egg-info} # Define CMake configuration arguments cmake_args="${CMAKE_ARGS:-}" @@ -20,7 +20,7 @@ if [[ -n "$(which ninja)" ]]; then cmake_args+=" -GNinja"; fi cmake_args+=" -D Legion_USE_CUDA=ON -D Legion_USE_OpenMP=ON --D CMAKE_CUDA_ARCHITECTURES=NATIVE +-D Legion_CUDA_ARCH=native "; # Use all but 2 threads to compile diff --git a/scripts/build-separately-no-install.sh b/scripts/build-separately-no-install.sh index e8d1e64c58..3ef1c65aad 100755 --- a/scripts/build-separately-no-install.sh +++ b/scripts/build-separately-no-install.sh @@ -8,7 +8,7 @@ source ./scripts/util/build-caching.sh source ./scripts/util/compiler-flags.sh # Remove existing build artifacts -rm -rf ./{build,_skbuild,dist,legate.core.egg-info} +rm -rf ./{build,_skbuild,dist,legate_core.egg-info} # Define CMake configuration arguments cmake_args="${CMAKE_ARGS:-}" @@ -21,8 +21,8 @@ cmake_args+=" -D Legion_USE_CUDA=ON -D Legion_USE_OpenMP=ON -D Legion_USE_Python=ON --D Legion_BUILD_BINDINGS=ON --D CMAKE_CUDA_ARCHITECTURES=NATIVE +-D Legion_Python_Version=$(python3 --version 2>&1 | cut -d' ' -f2 | cut -d'.' -f3 --complement) +-D Legion_CUDA_ARCH=native "; # Use all but 2 threads to compile diff --git a/scripts/build-with-legion-no-install.sh b/scripts/build-with-legion-no-install.sh index 2f3a1e3974..c3d340b556 100755 --- a/scripts/build-with-legion-no-install.sh +++ b/scripts/build-with-legion-no-install.sh @@ -10,7 +10,7 @@ source ./scripts/util/compiler-flags.sh source ./scripts/util/read-legion-root.sh "$0" # Remove existing build artifacts -rm -rf ./{build,_skbuild,dist,legate.core.egg-info} +rm -rf ./{build,_skbuild,dist,legate_core.egg-info} # Use all but 2 threads to compile ninja_args="-j$(nproc --ignore=2)" @@ -35,7 +35,7 @@ if [[ -n "$(which ninja)" ]]; then cmake_args+=" -GNinja"; fi cmake_args+=" -D Legion_USE_CUDA=ON -D Legion_USE_OpenMP=ON --D CMAKE_CUDA_ARCHITECTURES=NATIVE +-D Legion_CUDA_ARCH=native -D Legion_ROOT:STRING=\"$Legion_ROOT\" "; diff --git a/scripts/build-with-legion-separately-no-install.sh b/scripts/build-with-legion-separately-no-install.sh index d15180dc0a..f224a8f44a 100755 --- a/scripts/build-with-legion-separately-no-install.sh +++ b/scripts/build-with-legion-separately-no-install.sh @@ -10,7 +10,7 @@ source ./scripts/util/compiler-flags.sh source ./scripts/util/read-legion-root.sh "$0" # Remove existing build artifacts -rm -rf ./{build,_skbuild,dist,legate.core.egg-info} +rm -rf ./{build,_skbuild,dist,legate_core.egg-info} # Use all but 2 threads to compile ninja_args="-j$(nproc --ignore=2)" @@ -33,7 +33,7 @@ if [[ -n "$(which ninja)" ]]; then cmake_args+=" -GNinja"; fi # Add other build options here as desired cmake_args+=" --D CMAKE_CUDA_ARCHITECTURES=NATIVE +-D Legion_CUDA_ARCH=native -D Legion_ROOT:STRING=\"$Legion_ROOT\" "; diff --git a/scripts/generate-conda-envs.py b/scripts/generate-conda-envs.py index 00c7e11b5f..3e0ee5dce3 100755 --- a/scripts/generate-conda-envs.py +++ b/scripts/generate-conda-envs.py @@ -71,18 +71,31 @@ def conda(self) -> Reqs: "pynvml", # tests ) - if self.compilers and self.os == "linux": - # gcc 11.3 is incompatible with nvcc <= 11.5. - if V(self.ctk_version) <= V("11.5"): - deps += ( - "gcc_linux-64<=11.2", - "gxx_linux-64<=11.2", - ) + if V(self.ctk_version) >= V("12.0"): + deps += ( + "cuda-cudart-dev", + "cuda-nvtx-dev", + ) + + if self.compilers: + if V(self.ctk_version) < V("12.0"): + deps += (f"nvcc_linux-64={self.ctk_version}",) else: - deps += ( - "gcc_linux-64=11.*", - "gxx_linux-64=11.*", - ) + deps += ("cuda-nvcc",) + + if self.os == "linux": + # gcc 11.3 is incompatible with nvcc <= 11.5. + if V(self.ctk_version) <= V("11.5"): + deps += ( + "gcc_linux-64<=11.2", + "gxx_linux-64<=11.2", + ) + else: + deps += ( + "gcc_linux-64=11.*", + "gxx_linux-64=11.*", + ) + return deps def __str__(self) -> str: @@ -145,7 +158,7 @@ def conda(self) -> Reqs: return ( "cffi", "llvm-openmp", - "numpy>=1.22", + "numpy>=1.22,<1.25", "libblas=*=*openblas*", "openblas=*=*openmp*", # work around https://github.com/StanfordLegion/legion/issues/1500 @@ -278,6 +291,7 @@ def filename(self) -> str: name: legate-{use} channels: - conda-forge + - nvidia dependencies: - python={python},!=3.9.7 # avoid https://bugs.python.org/issue45121 diff --git a/scripts/launch-devcontainer-conda.sh b/scripts/launch-devcontainer-conda.sh new file mode 100755 index 0000000000..fb2ef84280 --- /dev/null +++ b/scripts/launch-devcontainer-conda.sh @@ -0,0 +1,10 @@ +#! /usr/bin/env bash + +# cd to the repo root +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."; + +if test ${#@} -le 0; then set -- bash -li; fi + +exec ./scripts/launch-devcontainer.sh \ + rapidsai/devcontainers:23.10-cpp-cuda11.8-mambaforge-ubuntu22.04 \ + "${@}"; diff --git a/scripts/launch-devcontainer-pip.sh b/scripts/launch-devcontainer-pip.sh new file mode 100755 index 0000000000..020d78b2df --- /dev/null +++ b/scripts/launch-devcontainer-pip.sh @@ -0,0 +1,10 @@ +#! /usr/bin/env bash + +# cd to the repo root +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."; + +if test ${#@} -le 0; then set -- bash -li; fi + +exec ./scripts/launch-devcontainer.sh \ + rapidsai/devcontainers:23.10-cpp-rust-cuda11.8-ubuntu22.04 \ + "${@}"; diff --git a/scripts/launch-devcontainer.sh b/scripts/launch-devcontainer.sh new file mode 100755 index 0000000000..78ef1b86cb --- /dev/null +++ b/scripts/launch-devcontainer.sh @@ -0,0 +1,57 @@ +#! /usr/bin/env bash + +launch_devcontainer() { + + set -Eeuox pipefail; + + # cd to the repo root + cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."; + + local cwd="$(pwd)"; + + local args=(); + args+=(--rm); + args+=(--tty); + args+=(--interactive); + args+=(--gpus all); + args+=(--user coder); + args+=(--workdir /home/coder/legate); + args+=(--entrypoint devcontainer-utils-post-attach-command-entrypoint); + + local vars=(); + vars+=(-e "PYTHONSAFEPATH=1"); + vars+=(-e "DEFAULT_CONDA_ENV=legate"); + vars+=(-e "PYTHONDONTWRITEBYTECODE=1"); + + vars+=(-e "HISTFILE=/home/coder/.local/state/._bash_history"); + + vars+=(-e "SCCACHE_REGION=us-east-2"); + vars+=(-e "SCCACHE_BUCKET=rapids-sccache-devs"); + vars+=(-e "SCCACHE_S3_KEY_PREFIX=legate-cunumeric-dev"); + vars+=(-e "VAULT_HOST=https://vault.ops.k8s.rapids.ai"); + + local vols=(); + vols+=(-v "${cwd}:/home/coder/legate"); + vols+=(-v "${cwd}/continuous_integration/home/coder/.local/bin/make-conda-env:/home/coder/.local/bin/make-conda-env"); + vols+=(-v "${cwd}/continuous_integration/home/coder/.local/bin/build-legate-all:/home/coder/.local/bin/build-legate-all"); + vols+=(-v "${cwd}/continuous_integration/home/coder/.local/bin/build-legate-cpp:/home/coder/.local/bin/build-legate-cpp"); + vols+=(-v "${cwd}/continuous_integration/home/coder/.local/bin/build-legate-wheel:/home/coder/.local/bin/build-legate-wheel"); + vols+=(-v "${cwd}/continuous_integration/home/coder/.local/bin/build-legate-conda:/home/coder/.local/bin/build-legate-conda"); + vols+=(-v "${cwd}/continuous_integration/home/coder/.local/bin/test-legate-python:/home/coder/.local/bin/test-legate-python"); + vols+=(-v "${cwd}/continuous_integration/home/coder/.local/bin/type-check-legate-python:/home/coder/.local/bin/type-check-legate-python"); + + for x in ".aws" ".cache" ".cargo" ".config" ".conda/pkgs" ".local/state" "legion"; do + if test -d "${cwd}/../${x}"; then + vols+=(-v "${cwd}/../${x}:/home/coder/${x}"); + fi + done + + if test -n "${SSH_AUTH_SOCK:-}"; then + vars+=(-e "SSH_AUTH_SOCK=/tmp/ssh-auth-sock"); + vols+=(-v "${SSH_AUTH_SOCK}:/tmp/ssh-auth-sock"); + fi + + exec docker run ${args[@]} ${vars[@]} ${vols[@]} "${@}"; +} + +launch_devcontainer "$@"; diff --git a/scripts/replicate-ci.sh b/scripts/replicate-ci.sh new file mode 100755 index 0000000000..7cb578b93b --- /dev/null +++ b/scripts/replicate-ci.sh @@ -0,0 +1,37 @@ +#! /usr/bin/env bash + +# cd to the repo root +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."; + +exec ./scripts/launch-devcontainer-conda.sh bash -licx ' +# Create build env +make-conda-env; + +# Build libs + conda package +build-legate-all; + +# Uncreate build env +rm -rf ~/.conda/envs/${DEFAULT_CONDA_ENV:-legate}.bak; +mv ~/.conda/envs/${DEFAULT_CONDA_ENV:-legate}{,.bak}; + +# Recreate env from conda package +mamba create -y -n "${DEFAULT_CONDA_ENV:-legate}" \ + `# local legate_core channel first` \ + -c ~/.artifacts/legate_core \ + `# then conda-forge` \ + -c conda-forge \ + `# and finally nvidia` \ + -c nvidia \ + legate-core \ + mypy pytest pytest-mock ipython jupyter_client ; + +# Check types +type-check-legate-python; + +# Test package +test-legate-python; + +# Restore build env +rm -rf ~/.conda/envs/${DEFAULT_CONDA_ENV:-legate}/; +mv ~/.conda/envs/${DEFAULT_CONDA_ENV:-legate}{.bak,}; +'; diff --git a/setup.py b/setup.py index 83912f31f8..e46b61cd67 100755 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ import versioneer setup( - name="legate.core", + name="legate-core", version=versioneer.get_version(), description="legate.core - The Foundation for All Legate Libraries", url="https://github.com/nv-legate/legate.core", @@ -69,6 +69,10 @@ }, scripts=["bind.sh"], cmdclass=versioneer.get_cmdclass(), - install_requires=["numpy>=1.22"], + install_requires=[ + "cffi", + "numpy>=1.22", + "typing_extensions", + ], zip_safe=False, )