diff --git a/.github/workflows/spack_cpu_build.yaml b/.github/workflows/spack_cpu_build.yaml new file mode 100644 index 000000000..409bb3db7 --- /dev/null +++ b/.github/workflows/spack_cpu_build.yaml @@ -0,0 +1,137 @@ +# https://spack.readthedocs.io/en/latest/binary_caches.html#spack-build-cache-for-github-actions +name: Spack Ubunutu x86_64 Buildcache + +env: + SPACK_COLOR: always + REGISTRY: ghcr.io/ornl + # Our repo name contains upper case characters, so we can't use ${{ github.repository }} + IMAGE_NAME: resolve + USERNAME: resolve-bot + BASE_VERSION: ubuntu-22.04-fortran + +# Until we remove the need to clone submodules to build, this should on be in PRs +on: [pull_request] + +jobs: + base_image_build: + runs-on: ubuntu-22.04 + permissions: + packages: write + contents: read + + name: Build Custom Base Image + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # Once we move submodule deps into spack, we can do some more builds + # Also need to change build script to use spack from base image + submodules: true + + # Need to build custom base image with gfortran + - name: Create Dockerfile heredoc + run: | + cat << EOF > Dockerfile + FROM ubuntu:22.04 + RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + software-properties-common \ + gpg-agent \ + openssh-client \ + openssh-server \ + && rm -rf /var/lib/apt/lists/* + RUN add-apt-repository ppa:ubuntu-toolchain-r/test && \ + apt-get install -y --no-install-recommends \ + gcc \ + libstdc++6 \ + && rm -rf /var/lib/apt/lists/* + EOF + + # https://docs.github.com/en/actions/publishing-packages/publishing-docker-images + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ env.USERNAME }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + labels: org.opencontainers.image.version=${{ env.BASE_VERSION }} + + - name: Build and push Docker base image + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BASE_VERSION }} + labels: ${{ steps.meta.outputs.labels }} + + resolve_spack_builds: + needs: base_image_build + runs-on: ubuntu-22.04 + permissions: + packages: write + contents: read + + strategy: + matrix: + # Minimal Build(s) - GHCR mirror speeds these up a lot! + spack_spec: + - resolve@develop+klu + - resolve@develop~klu + + name: Build ReSolve with Spack + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # Once we move submodule deps into spack, we can do some more builds + # Also need to change build script to use spack from base image + submodules: true + + - name: Setup Spack + run: echo "$PWD/buildsystem/spack/spack/bin" >> "$GITHUB_PATH" + + - name: Create heredoc spack.yaml + run: | + cat << EOF > spack.yaml + spack: + specs: + - ${{ matrix.spack_spec }} target=x86_64_v2 + concretizer: + reuse: dependencies + config: + install_tree: + root: /opt/spack + padded_length: 128 + mirrors: + local-buildcache: oci://${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + spack: https://binaries.spack.io/develop + + - name: Configure GHCR mirror + run: spack -e . mirror set --oci-username ${{ env.USERNAME }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache + + - name: Trust keys + run: spack -e . buildcache keys --install --trust + + - name: Find external packages + run: spack -e . external find --all + + - name: Spack develop ReSolve + run: spack -e . develop --path=$(pwd) resolve@develop + + - name: Concretize + run: spack -e . concretize + + - name: Install + run: spack -e . install --no-check-signature + + # Push with force to override existing binaries... + - name: Push to binaries to buildcache + run: | + spack -e . buildcache push --force --base-image ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BASE_VERSION }} --unsigned --update-index local-buildcache + if: ${{ !cancelled() }} diff --git a/buildsystem/spack/spack b/buildsystem/spack/spack index f120cada5..b1ab01280 160000 --- a/buildsystem/spack/spack +++ b/buildsystem/spack/spack @@ -1 +1 @@ -Subproject commit f120cada59dbc5115d94c2fce3cbffc946b72bb0 +Subproject commit b1ab01280af1ec3065594b739dc2bde3f2107510 diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 86e8f8573..ffe6ade6e 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -6,17 +6,19 @@ ]] -# Build example with KLU factorization and KLU refactorization -add_executable(klu_klu.exe r_KLU_KLU.cpp) -target_link_libraries(klu_klu.exe PRIVATE ReSolve) +if(RESOLVE_USE_KLU) + # Build example with KLU factorization and KLU refactorization + add_executable(klu_klu.exe r_KLU_KLU.cpp) + target_link_libraries(klu_klu.exe PRIVATE ReSolve) -# Read ONE matrix,solve with KLU -add_executable(klu_klu_standalone.exe r_KLU_KLU_standalone.cpp) -target_link_libraries(klu_klu_standalone.exe PRIVATE ReSolve) + # Read ONE matrix,solve with KLU + add_executable(klu_klu_standalone.exe r_KLU_KLU_standalone.cpp) + target_link_libraries(klu_klu_standalone.exe PRIVATE ReSolve) -# Build example with a configurable system solver -add_executable(system.exe r_SysSolver.cpp) -target_link_libraries(system.exe PRIVATE ReSolve) + # Build example with a configurable system solver + add_executable(system.exe r_SysSolver.cpp) + target_link_libraries(system.exe PRIVATE ReSolve) +endif(RESOLVE_USE_KLU) # Create CUDA examples if(RESOLVE_USE_CUDA) @@ -81,15 +83,19 @@ if(RESOLVE_USE_HIP) target_link_libraries(system_hip_fgmres.exe PRIVATE ReSolve) endif(RESOLVE_USE_HIP) +set(installable_executables "") + # Install all examples in bin directory -set(installable_executables klu_klu.exe klu_klu_standalone.exe system.exe) +if(RESOLVE_USE_KLU) + list(APPEND installable_executables klu_klu.exe klu_klu_standalone.exe system.exe) +endif() if(RESOLVE_USE_CUDA) - set(installable_executables ${installable_executables} klu_glu.exe klu_rf.exe klu_rf_fgmres.exe klu_glu_values_update.exe gmres_cusparse_rand.exe klu_cusolverrf_check_redo.exe) + list(APPEND installable_executables klu_glu.exe klu_rf.exe klu_rf_fgmres.exe klu_glu_values_update.exe gmres_cusparse_rand.exe klu_cusolverrf_check_redo.exe) endif(RESOLVE_USE_CUDA) if(RESOLVE_USE_HIP) - set(installable_executables ${installable_executables} klu_rocsolverrf.exe klu_rocsolverrf_fgmres.exe klu_rocsolverrf_check_redo.exe gmres_rocsparse_rand.exe) + list(APPEND installable_executables klu_rocsolverrf.exe klu_rocsolverrf_fgmres.exe klu_rocsolverrf_check_redo.exe gmres_rocsparse_rand.exe) endif(RESOLVE_USE_HIP) install(TARGETS ${installable_executables} @@ -102,13 +108,14 @@ set(CONSUMER_PATH ${CMAKE_INSTALL_PREFIX}/share/examples) install(PROGRAMS test.sh DESTINATION ${CONSUMER_PATH}) # Select consumer app -# TODO - have an outer loop that adds a unique consumer test for each backend supproted if(RESOLVE_USE_CUDA) set(RESOLVE_CONSUMER_APP "testKLU_Rf_FGMRES.cpp") elseif(RESOLVE_USE_HIP) set(RESOLVE_CONSUMER_APP "testKLU_RocSolver.cpp") -else() +elseif(RESOLVE_USE_KLU) set(RESOLVE_CONSUMER_APP "testKLU.cpp") +else() + set(RESOLVE_CONSUMER_APP "testVersion.cpp") endif() # Install directory with example on how to consume ReSolve diff --git a/resolve/CMakeLists.txt b/resolve/CMakeLists.txt index a02ffdb63..948db7881 100644 --- a/resolve/CMakeLists.txt +++ b/resolve/CMakeLists.txt @@ -11,19 +11,22 @@ add_subdirectory(utilities) # C++ files set(ReSolve_SRC LinSolver.cpp - LinSolverDirectKLU.cpp - SystemSolver.cpp + GramSchmidt.cpp + LinSolverIterativeFGMRES.cpp ) # Temporary until there is CPU-only option for FGMRES set(ReSolve_GPU_SRC - GramSchmidt.cpp - LinSolverIterativeFGMRES.cpp LinSolverIterativeRandFGMRES.cpp RandSketchingManager.cpp RandSketchingCountSketch.cpp RandSketchingFWHT.cpp - ) +) + +set(ReSolve_KLU_SRC + LinSolverDirectKLU.cpp + SystemSolver.cpp +) # C++ code that links to CUDA SDK libraries set(ReSolve_CUDASDK_SRC @@ -31,6 +34,7 @@ set(ReSolve_CUDASDK_SRC LinSolverDirectCuSolverRf.cpp LinSolverDirectCuSparseILU0.cpp ) + # HIP files set(ReSolve_ROCM_SRC LinSolverDirectRocSolverRf.cpp @@ -41,18 +45,30 @@ set(ReSolve_HEADER_INSTALL Common.hpp cusolver_defs.hpp LinSolver.hpp - LinSolverDirectCuSolverGLU.hpp - LinSolverDirectCuSolverRf.hpp - LinSolverDirectRocSolverRf.hpp - LinSolverDirectRocSparseILU0.hpp - LinSolverDirectCuSparseILU0.hpp - LinSolverDirectKLU.hpp LinSolverIterativeFGMRES.hpp RefactorizationSolver.hpp SystemSolver.hpp GramSchmidt.hpp - MemoryUtils.hpp - RandSketchingManager.hpp) + MemoryUtils.hpp) + +set(ReSolve_KLU_HEADER_INSTALL + LinSolverDirectKLU.hpp +) + +set(ReSolve_GPU_HEADER_INSTALL + RandSketchingManager.hpp +) + +set(ReSolve_CUDA_HEADER_INSTALL + LinSolverDirectCuSolverGLU.hpp + LinSolverDirectCuSolverRf.hpp + LinSolverDirectCuSparseILU0.hpp +) + +set(ReSolve_ROCM_HEADER_INSTALL + LinSolverDirectRocSolverRf.hpp + LinSolverDirectRocSparseILU0.hpp +) # Now, build workspaces add_subdirectory(workspace) @@ -65,8 +81,10 @@ add_subdirectory(matrix) add_library(resolve_tpl INTERFACE) if(RESOLVE_USE_KLU) - target_link_libraries(resolve_tpl INTERFACE KLU) -endif(RESOLVE_USE_KLU) + target_link_libraries(resolve_tpl INTERFACE KLU) + list(APPEND ReSolve_SRC ${ReSolve_KLU_SRC}) + list(APPEND ReSolve_HEADER_INSTALL ${ReSolve_KLU_HEADER_INSTALL}) +endif() set(ReSolve_Targets_List resolve_matrix @@ -78,29 +96,32 @@ set(ReSolve_Targets_List # Temporary until there is CPU-only option for FGMRES if(RESOLVE_USE_GPU) - set(ReSolve_SRC ${ReSolve_SRC} ${ReSolve_GPU_SRC}) + list(APPEND ReSolve_SRC ${ReSolve_GPU_SRC}) + list(APPEND ReSolve_HEADER_INSTALL ${ReSolve_GPU_HEADER_INSTALL}) endif() # If CUDA support is enabled add CUDA SDK specific code and dependencies if(RESOLVE_USE_CUDA) add_subdirectory(cuda) target_link_libraries(resolve_tpl INTERFACE resolve_cuda) - set(ReSolve_SRC ${ReSolve_SRC} ${ReSolve_CUDASDK_SRC}) - set(ReSolve_Targets_List ${ReSolve_Targets_List} resolve_backend_cuda) + list(APPEND ReSolve_SRC ${ReSolve_CUDASDK_SRC}) + list(APPEND ReSolve_Targets_List resolve_backend_cuda) + list(APPEND ReSolve_HEADER_INSTALL ${ReSolve_CUDA_HEADER_INSTALL}) endif() # If HIP support is enabled add HIP SDK specific code and dependencies if(RESOLVE_USE_HIP) add_subdirectory(hip) target_link_libraries(resolve_tpl INTERFACE resolve_hip) - set(ReSolve_SRC ${ReSolve_SRC} ${ReSolve_ROCM_SRC}) - set(ReSolve_Targets_List ${ReSolve_Targets_List} resolve_backend_hip) + list(APPEND ReSolve_SRC ${ReSolve_ROCM_SRC}) + list(APPEND ReSolve_Targets_List resolve_backend_hip) + list(APPEND ReSolve_HEADER_INSTALL ${ReSolve_ROCM_HEADER_INSTALL}) endif() # If no GPU support is enabled, link to dummy device backend if(NOT RESOLVE_USE_GPU) add_subdirectory(cpu) - set(ReSolve_Targets_List ${ReSolve_Targets_List} resolve_backend_cpu) + list(APPEND ReSolve_Targets_List resolve_backend_cpu) endif() # Set installable targets diff --git a/resolve/SystemSolver.cpp b/resolve/SystemSolver.cpp index e62eb22c3..29a9e9255 100644 --- a/resolve/SystemSolver.cpp +++ b/resolve/SystemSolver.cpp @@ -1,9 +1,12 @@ #include +#include #include #include +#ifdef RESOLVE_USE_KLU #include +#endif #ifdef RESOLVE_USE_CUDA #include diff --git a/resolve/matrix/CMakeLists.txt b/resolve/matrix/CMakeLists.txt index 565fa7c94..f8a6481a7 100644 --- a/resolve/matrix/CMakeLists.txt +++ b/resolve/matrix/CMakeLists.txt @@ -37,26 +37,18 @@ set(Matrix_HEADER_INSTALL MatrixHandler.hpp ) -# Add CUDA matrix handler if CUDA support is enabled -if(RESOLVE_USE_CUDA) - set(Matrix_SRC ${Matrix_SRC} ${Matrix_CUDASDK_SRC}) -endif() - -if(RESOLVE_USE_HIP) - set(Matrix_SRC ${Matrix_SRC} ${Matrix_ROCM_SRC}) -endif() - - # Build shared library ReSolve::matrix add_library(resolve_matrix SHARED ${Matrix_SRC}) target_link_libraries(resolve_matrix PRIVATE resolve_logger resolve_vector) # Link to CUDA ReSolve backend if CUDA is support enabled if (RESOLVE_USE_CUDA) + target_sources(resolve_matrix PRIVATE ${Matrix_CUDASDK_SRC}) target_link_libraries(resolve_matrix PUBLIC resolve_backend_cuda) endif() if (RESOLVE_USE_HIP) + target_sources(resolve_matrix PRIVATE ${Matrix_ROCM_SRC}) target_link_libraries(resolve_matrix PUBLIC resolve_backend_hip) endif() @@ -65,19 +57,9 @@ if (NOT RESOLVE_USE_GPU) target_link_libraries(resolve_matrix PUBLIC resolve_backend_cpu) endif() - target_include_directories(resolve_matrix INTERFACE $ $ ) -# # TODO: Make this PRIVATE dependency (requires refactoring ReSolve code) -# target_link_libraries(ReSolve PUBLIC resolve_tpl) - -# install(TARGETS ReSolve -# EXPORT ReSolveTargets -# ARCHIVE DESTINATION lib -# LIBRARY DESTINATION lib) -# install include headers install(FILES ${Matrix_HEADER_INSTALL} DESTINATION include/resolve/matrix) - diff --git a/resolve/vector/CMakeLists.txt b/resolve/vector/CMakeLists.txt index 89b1abc8a..fbc3e29d5 100644 --- a/resolve/vector/CMakeLists.txt +++ b/resolve/vector/CMakeLists.txt @@ -19,12 +19,10 @@ set(Vector_CUDASDK_SRC ) #and hip - set(Vector_ROCM_SRC VectorHandlerHip.cpp ) - # Header files to be installed set(Vector_HEADER_INSTALL Vector.hpp @@ -32,25 +30,18 @@ set(Vector_HEADER_INSTALL VectorKernels.hpp ) -# Add CUDA vector handler if CUDA support is enabled -if(RESOLVE_USE_CUDA) - set(Vector_SRC ${Vector_SRC} ${Vector_CUDASDK_SRC}) -endif() - -# and hip -if(RESOLVE_USE_HIP) - set(Vector_SRC ${Vector_SRC} ${Vector_ROCM_SRC}) -endif() - add_library(resolve_vector SHARED ${Vector_SRC}) target_link_libraries(resolve_vector PRIVATE resolve_logger) -# Link to ReSolve CUDA backend if CUDA is enabled -if (RESOLVE_USE_CUDA) +# Add CUDA vector handler if CUDA support is enabled +if(RESOLVE_USE_CUDA) + target_sources(resolve_vector PRIVATE ${Vector_CUDASDK_SRC}) target_link_libraries(resolve_vector PUBLIC resolve_backend_cuda) endif() -if (RESOLVE_USE_HIP) +# and hip +if(RESOLVE_USE_HIP) + target_sources(resolve_vector PRIVATE ${Vector_ROCM_SRC}) target_link_libraries(resolve_vector PUBLIC resolve_backend_hip) endif() @@ -59,19 +50,9 @@ if(NOT RESOLVE_USE_GPU) target_link_libraries(resolve_vector PUBLIC resolve_backend_cpu) endif(NOT RESOLVE_USE_GPU) - target_include_directories(resolve_vector INTERFACE $ $ ) -# # TODO: Make this PRIVATE dependency (requires refactoring ReSolve code) -# target_link_libraries(ReSolve PUBLIC resolve_tpl) - -# install(TARGETS ReSolve -# EXPORT ReSolveTargets -# ARCHIVE DESTINATION lib -# LIBRARY DESTINATION lib) -# install include headers install(FILES ${Vector_HEADER_INSTALL} DESTINATION include/resolve/vector) - diff --git a/resolve/workspace/CMakeLists.txt b/resolve/workspace/CMakeLists.txt index a44f74f81..73dd2a1e7 100644 --- a/resolve/workspace/CMakeLists.txt +++ b/resolve/workspace/CMakeLists.txt @@ -27,25 +27,18 @@ set(ReSolve_Workspace_HEADER_INSTALL LinAlgWorkspaceHIP.hpp ) -# If cuda is enabled, add CUDA SDK workspace files -if(RESOLVE_USE_CUDA) - set(ReSolve_Workspace_SRC ${ReSolve_Workspace_SRC} ${ReSolve_Workspace_CUDASDK_SRC}) -endif() - -if(RESOLVE_USE_HIP) - set(ReSolve_Workspace_SRC ${ReSolve_Workspace_SRC} ${ReSolve_Workspace_ROCM_SRC}) -endif() - add_library(resolve_workspace SHARED ${ReSolve_Workspace_SRC}) -# If CUDA is enabled, link to ReSolve CUDA backend +# If cuda is enabled, add CUDA SDK workspace files if(RESOLVE_USE_CUDA) + target_sources(resolve_workspace PRIVATE ${ReSolve_Workspace_CUDASDK_SRC}) target_link_libraries(resolve_workspace PUBLIC resolve_backend_cuda) -endif(RESOLVE_USE_CUDA) +endif() if(RESOLVE_USE_HIP) + target_sources(resolve_workspace PRIVATE ${ReSolve_Workspace_ROCM_SRC}) target_link_libraries(resolve_workspace PUBLIC resolve_backend_hip) -endif(RESOLVE_USE_HIP) +endif() target_include_directories(resolve_workspace PUBLIC $ diff --git a/tests/functionality/CMakeLists.txt b/tests/functionality/CMakeLists.txt index c606d5ff1..93af3c3f3 100644 --- a/tests/functionality/CMakeLists.txt +++ b/tests/functionality/CMakeLists.txt @@ -10,9 +10,11 @@ add_executable(version.exe testVersion.cpp) target_link_libraries(version.exe PRIVATE ReSolve) -# Build KLU+KLU test -add_executable(klu_klu_test.exe testKLU.cpp) -target_link_libraries(klu_klu_test.exe PRIVATE ReSolve) +if(RESOLVE_USE_KLU) + # Build KLU+KLU test + add_executable(klu_klu_test.exe testKLU.cpp) + target_link_libraries(klu_klu_test.exe PRIVATE ReSolve) +endif(RESOLVE_USE_KLU) if(RESOLVE_USE_CUDA) @@ -59,22 +61,24 @@ if(RESOLVE_USE_HIP) endif(RESOLVE_USE_HIP) +set(installable_tests "") + # Install tests -set(installable_tests klu_klu_test.exe version.exe) +if(RESOLVE_USE_KLU) + list(APPEND installable_tests klu_klu_test.exe version.exe) +endif() if(RESOLVE_USE_CUDA) - set(installable_tests ${installable_tests} - klu_rf_test.exe - klu_rf_fgmres_test.exe - klu_glu_test.exe - rand_gmres_cuda_test.exe) + list(APPEND installable_tests klu_rf_test.exe + klu_rf_fgmres_test.exe + klu_glu_test.exe + rand_gmres_cuda_test.exe) endif(RESOLVE_USE_CUDA) if(RESOLVE_USE_HIP) - set(installable_tests ${installable_tests} - rocsolver_rf_test.exe - rocsolver_rf_fgmres_test.exe - rand_gmres_hip_test.exe) + list(APPEND installable_tests rocsolver_rf_test.exe + rocsolver_rf_fgmres_test.exe + rand_gmres_hip_test.exe) endif(RESOLVE_USE_HIP) install(TARGETS ${installable_tests} @@ -86,7 +90,11 @@ install(DIRECTORY data DESTINATION bin/resolve/tests/functionality) set(test_data_dir ${CMAKE_SOURCE_DIR}/tests/functionality/) add_test(NAME version COMMAND $ "${test_data_dir}") -add_test(NAME klu_klu_test COMMAND $ "${test_data_dir}") + +if(RESOLVE_USE_KLU) + add_test(NAME klu_klu_test COMMAND $ "${test_data_dir}") +endif() + if(RESOLVE_USE_CUDA) add_test(NAME klu_rf_test COMMAND $ "${test_data_dir}") add_test(NAME klu_rf_fgmres_test COMMAND $ "${test_data_dir}")