Cleanup + Cosmetic changes

.github/workflows/actions.yml

@@ -27,9 +27,14 @@ jobs:
       matrix:
         device: [cpu, amd-gpu, nvidia-gpu]
         precision: [double, single]
+        mpi: [serial, parallel]
         exclude: # my AMD GPU doesn't support fp64 atomics : (
         - device: amd-gpu 
           precision: double
+        - device: amd-gpu
+          mpi: parallel
+        - device: nvidia-gpu
+          mpi: parallel
     runs-on: [self-hosted, "${{ matrix.device }}"]
     steps:
       - name: Checkout
@@ -47,7 +52,7 @@ jobs:
             fi
           elif [ "${{ matrix.device }}" = "amd-gpu" ]; then
             FLAGS="-D Kokkos_ENABLE_HIP=ON -D Kokkos_ARCH_AMD_GFX1100=ON"
-          elif [ "${{ matrix.device }}" = "cpu" ]; then
+          elif [ "${{ matrix.mpi }}" = "parallel" ]; then
             FLAGS="-D mpi=ON"
           fi
           cmake -B build -D TESTS=ON -D output=ON -D precision=${{ matrix.precision }} $FLAGS

.gitmodules

@@ -4,7 +4,6 @@
 [submodule "extern/adios2"]
 	path = extern/adios2
 	url = https://github.com/ornladios/ADIOS2.git
-	branch = master
 [submodule "extern/Kokkos"]
 	path = extern/Kokkos
 	url = https://github.com/kokkos/kokkos.git

CMakeLists.txt

@@ -1,3 +1,5 @@
+# cmake-lint: disable=C0103,C0111,E1120,R0913,R0915
+
 cmake_minimum_required(VERSION 3.16)
 cmake_policy(SET CMP0110 NEW)
 
@@ -8,10 +10,10 @@ project(
   VERSION 1.2.0
   LANGUAGES CXX C)
 add_compile_options("-D ENTITY_VERSION=\"${PROJECT_VERSION}\"")
+set(hash_cmd "git diff --quiet src/ && echo $(git rev-parse HEAD) ")
+string(APPEND hash_cmd "|| echo $(git rev-parse HEAD)-mod")
 execute_process(
-  COMMAND
-    bash -c
-    "git diff --quiet src/ && echo $(git rev-parse HEAD) || echo $(git rev-parse HEAD)-mod"
+  COMMAND bash -c ${hash_cmd}
   WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
   OUTPUT_VARIABLE GIT_HASH
   ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
@@ -55,9 +57,8 @@ if(${DEBUG} STREQUAL "OFF")
   set(CMAKE_BUILD_TYPE
       Release
       CACHE STRING "CMake build type")
-  set(CMAKE_CXX_FLAGS
-      "${CMAKE_CXX_FLAGS} -DNDEBUG -Wno-unused-local-typedefs -Wno-unknown-cuda-version"
-  )
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DNDEBUG "
+                      "-Wno-unused-local-typedefs -Wno-unknown-cuda-version")
 else()
   set(CMAKE_BUILD_TYPE
       Debug
@@ -79,11 +80,10 @@ set(BUILD_TESTING
     CACHE BOOL "Build tests")
 
 # ------------------------ Third-party dependencies ------------------------ #
-include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/kokkosConfig.cmake)
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/dependencies.cmake)
 
-find_or_fetch_dependency(Kokkos FALSE)
-find_or_fetch_dependency(plog TRUE)
+find_or_fetch_dependency(Kokkos FALSE QUIET)
+find_or_fetch_dependency(plog TRUE QUIET)
 set(DEPENDENCIES Kokkos::kokkos)
 include_directories(${plog_SRC}/include)
 
@@ -92,25 +92,16 @@ set_precision(${precision})
 
 # MPI
 if(${mpi})
-  include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/MPIConfig.cmake)
+  find_or_fetch_dependency(MPI FALSE REQUIRED)
+  include_directories(${MPI_CXX_INCLUDE_PATH})
+  add_compile_options("-D MPI_ENABLED")
   set(DEPENDENCIES ${DEPENDENCIES} MPI::MPI_CXX)
 endif()
 
 # Output
 if(${output})
-  include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/adios2Config.cmake)
-  find_or_fetch_dependency(adios2 FALSE)
-  if(NOT DEFINED ENV{HDF5_ROOT})
-    if(DEFINED ENV{CONDA_PREFIX})
-      execute_process(COMMAND bash -c "conda list | grep \"hdf5\" -q"
-                      RESULT_VARIABLE HDF5_INSTALLED)
-      if(HDF5_INSTALLED EQUAL 0)
-        set(HDF5_ROOT $ENV{CONDA_PREFIX})
-      endif()
-    endif()
-  endif()
-  find_package(HDF5 REQUIRED)
-
+  find_or_fetch_dependency(adios2 FALSE QUIET)
+  add_compile_options("-D OUTPUT_ENABLED")
   if(${mpi})
     set(DEPENDENCIES ${DEPENDENCIES} adios2::cxx11_mpi)
   else()
@@ -129,11 +120,12 @@ elseif(BENCHMARK)
 else()
   # ----------------------------------- GUI ---------------------------------- #
   if(${gui})
-    find_or_fetch_dependency(nttiny FALSE)
+    find_or_fetch_dependency(nttiny FALSE QUIET)
   endif()
 
   # ------------------------------- Main source ------------------------------ #
   set_problem_generator(${pgen})
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src src)
-  include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/report.cmake)
 endif()
+
+include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/report.cmake)

cmake/adios2Config.cmake

@@ -12,16 +12,18 @@ set(ADIOS2_USE_Fortran
     CACHE BOOL "Use Fortran for ADIOS2")
 
 # Format/compression support
-set(ADIOS2_USE_ZeroMQ
-    OFF
-    CACHE BOOL "Use ZeroMQ for ADIOS2")
+set(ADIOS2_USE_HDF5
+    ON
+    CACHE BOOL "Use HDF5 for ADIOS2")
 
 set(ADIOS2_USE_MPI
     ${mpi}
     CACHE BOOL "Use MPI for ADIOS2")
 
+set(ADIOS2_USE_ZeroMQ
+    OFF
+    CACHE BOOL "Use ZeroMQ for ADIOS2")
+
 set(ADIOS2_USE_CUDA
     OFF
     CACHE BOOL "Use CUDA for ADIOS2")
-
-add_compile_options("-D OUTPUT_ENABLED")

cmake/benchmark.cmake

@@ -1,3 +1,5 @@
+# cmake-lint: disable=C0103
+
 set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src)
 
 add_subdirectory(${SRC_DIR}/global ${CMAKE_CURRENT_BINARY_DIR}/global)

cmake/config.cmake

@@ -1,3 +1,5 @@
+# cmake-lint: disable=C0103
+
 # -------------------------------- Precision ------------------------------- #
 function(set_precision precision_name)
   list(FIND precisions ${precision_name} PRECISION_FOUND)
@@ -28,10 +30,8 @@ function(set_problem_generator pgen_name)
   endforeach()
   list(FIND PGEN_NAMES ${pgen_name} PGEN_FOUND)
   if(NOT ${pgen_name} STREQUAL "." AND ${PGEN_FOUND} EQUAL -1)
-    message(
-      FATAL_ERROR
-        "Invalid problem generator: ${pgen_name}\nValid options are: ${PGEN_NAMES}"
-    )
+    message(FATAL_ERROR "Invalid problem generator: "
+                        "${pgen_name}\nValid options are: ${PGEN_NAMES}")
   endif()
   set(PGEN
       ${pgen_name}

cmake/defaults.cmake

@@ -1,3 +1,5 @@
+# cmake-lint: disable=C0103
+
 # ----------------------------- Defaults ---------------------------------- #
 if(DEFINED ENV{Entity_ENABLE_DEBUG})
   set(default_debug
@@ -33,7 +35,7 @@ if(DEFINED ENV{Entity_ENABLE_OUTPUT})
       CACHE INTERNAL "Default flag for output")
 else()
   set(default_output
-      OFF
+      ON
       CACHE INTERNAL "Default flag for output")
 endif()
 
@@ -51,42 +53,6 @@ endif()
 
 set_property(CACHE default_gui PROPERTY TYPE BOOL)
 
-if(DEFINED ENV{Kokkos_ENABLE_CUDA})
-  set(default_KOKKOS_ENABLE_CUDA
-      $ENV{Kokkos_ENABLE_CUDA}
-      CACHE INTERNAL "Default flag for CUDA")
-else()
-  set(default_KOKKOS_ENABLE_CUDA
-      OFF
-      CACHE INTERNAL "Default flag for CUDA")
-endif()
-
-set_property(CACHE default_KOKKOS_ENABLE_CUDA PROPERTY TYPE BOOL)
-
-if(DEFINED ENV{Kokkos_ENABLE_HIP})
-  set(default_KOKKOS_ENABLE_HIP
-      $ENV{Kokkos_ENABLE_HIP}
-      CACHE INTERNAL "Default flag for HIP")
-else()
-  set(default_KOKKOS_ENABLE_HIP
-      OFF
-      CACHE INTERNAL "Default flag for HIP")
-endif()
-
-set_property(CACHE default_KOKKOS_ENABLE_HIP PROPERTY TYPE BOOL)
-
-if(DEFINED ENV{Kokkos_ENABLE_OPENMP})
-  set(default_KOKKOS_ENABLE_OPENMP
-      $ENV{Kokkos_ENABLE_OPENMP}
-      CACHE INTERNAL "Default flag for OpenMP")
-else()
-  set(default_KOKKOS_ENABLE_OPENMP
-      OFF
-      CACHE INTERNAL "Default flag for OpenMP")
-endif()
-
-set_property(CACHE default_KOKKOS_ENABLE_OPENMP PROPERTY TYPE BOOL)
-
 if(DEFINED ENV{Entity_ENABLE_MPI})
   set(default_mpi
       $ENV{Entity_ENABLE_MPI}

cmake/dependencies.cmake

@@ -1,12 +1,15 @@
+# cmake-lint: disable=C0103,C0111,R0915,R0912
+
 set(Kokkos_REPOSITORY
     https://github.com/kokkos/kokkos.git
     CACHE STRING "Kokkos repository")
 set(plog_REPOSITORY
     https://github.com/SergiusTheBest/plog.git
     CACHE STRING "plog repository")
+set(adios2_REPOSITORY
+    https://github.com/ornladios/ADIOS2.git
+    CACHE STRING "ADIOS2 repository")
 
-# set (adios2_REPOSITORY https://github.com/ornladios/ADIOS2.git CACHE STRING
-# "ADIOS2 repository")
 function(check_internet_connection)
   if(OFFLINE STREQUAL "ON")
     set(FETCHCONTENT_FULLY_DISCONNECTED
@@ -34,25 +37,28 @@ function(check_internet_connection)
   endif()
 endfunction()
 
-function(find_or_fetch_dependency package_name header_only)
+function(find_or_fetch_dependency package_name header_only mode)
   if(NOT header_only)
-    find_package(${package_name} QUIET)
+    find_package(${package_name} ${mode})
   endif()
 
   if(NOT ${package_name}_FOUND)
+    if(${package_name} STREQUAL "Kokkos")
+      include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/kokkosConfig.cmake)
+    elseif(${package_name} STREQUAL "adios2")
+      include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/adios2Config.cmake)
+    endif()
     if(DEFINED ${package_name}_REPOSITORY AND NOT
                                               FETCHCONTENT_FULLY_DISCONNECTED)
       # fetching package
-      message(
-        STATUS
-          "${Blue}${package_name} not found. Fetching from ${${package_name}_REPOSITORY}${ColorReset}"
-      )
+      message(STATUS "${Blue}${package_name} not found. "
+                     "Fetching from ${${package_name}_REPOSITORY}${ColorReset}")
       include(FetchContent)
       if(${package_name} STREQUAL "Kokkos")
         FetchContent_Declare(
           ${package_name}
           GIT_REPOSITORY ${${package_name}_REPOSITORY}
-          GIT_TAG 4.3.00)
+          GIT_TAG 4.5.01)
       else()
         FetchContent_Declare(${package_name}
                              GIT_REPOSITORY ${${package_name}_REPOSITORY})
@@ -65,6 +71,9 @@ function(find_or_fetch_dependency package_name header_only)
       set(${package_name}_SRC
           ${CMAKE_CURRENT_BINARY_DIR}/_deps/${lower_pckg_name}-src
           CACHE PATH "Path to ${package_name} src")
+      set(${package_name}_BUILD_DIR
+          ${CMAKE_CURRENT_BINARY_DIR}/_deps/${lower_pckg_name}-build
+          CACHE PATH "Path to ${package_name} build")
       set(${package_name}_FETCHED
           TRUE
           CACHE BOOL "Whether ${package_name} was fetched")
@@ -127,7 +136,35 @@ function(find_or_fetch_dependency package_name header_only)
           ${Kokkos_VERSION}
           CACHE INTERNAL "Kokkos version")
     endif()
+    if(NOT DEFINED Kokkos_ARCH
+       OR Kokkos_ARCH STREQUAL ""
+       OR NOT DEFINED Kokkos_DEVICES
+       OR Kokkos_DEVICES STREQUAL "")
+      if(${Kokkos_FOUND})
+        include(${Kokkos_DIR}/KokkosConfigCommon.cmake)
+      elseif(NOT ${Kokkos_BUILD_DIR} STREQUAL "")
+        include(${Kokkos_BUILD_DIR}/KokkosConfigCommon.cmake)
+      else()
+        message(
+          STATUS "${Red}Kokkos_DIR and Kokkos_BUILD_DIR not set.${ColorReset}")
+      endif()
+    endif()
+    set(Kokkos_ARCH
+        ${Kokkos_ARCH}
+        PARENT_SCOPE)
+    set(Kokkos_DEVICES
+        ${Kokkos_DEVICES}
+        PARENT_SCOPE)
   endif()
+  set(${package_name}_FOUND
+      ${${package_name}_FOUND}
+      PARENT_SCOPE)
+  set(${package_name}_FETCHED
+      ${${package_name}_FETCHED}
+      PARENT_SCOPE)
+  set(${package_name}_BUILD_DIR
+      ${${package_name}_BUILD_DIR}
+      PARENT_SCOPE)
 endfunction()
 
 check_internet_connection()

cmake/kokkosConfig.cmake

@@ -1,3 +1,5 @@
+# cmake-lint: disable=C0103
+
 # ----------------------------- Kokkos settings ---------------------------- #
 if(${DEBUG} STREQUAL "OFF")
   set(Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION
@@ -27,51 +29,6 @@ else()
       CACHE BOOL "Kokkos debug bounds check")
 endif()
 
-set(Kokkos_ENABLE_HIP
-    ${default_KOKKOS_ENABLE_HIP}
-    CACHE BOOL "Enable HIP")
-set(Kokkos_ENABLE_CUDA
-    ${default_KOKKOS_ENABLE_CUDA}
-    CACHE BOOL "Enable CUDA")
-set(Kokkos_ENABLE_OPENMP
-    ${default_KOKKOS_ENABLE_OPENMP}
-    CACHE BOOL "Enable OpenMP")
-
-# set memory space
-if(${Kokkos_ENABLE_CUDA})
-  add_compile_definitions(CUDA_ENABLED)
-  set(ACC_MEM_SPACE Kokkos::CudaSpace)
-elseif(${Kokkos_ENABLE_HIP})
-  add_compile_definitions(HIP_ENABLED)
-  set(ACC_MEM_SPACE Kokkos::HIPSpace)
-else()
-  set(ACC_MEM_SPACE Kokkos::HostSpace)
-endif()
-
-set(HOST_MEM_SPACE Kokkos::HostSpace)
-
-# set execution space
-if(${Kokkos_ENABLE_CUDA})
-  set(ACC_EXE_SPACE Kokkos::Cuda)
-elseif(${Kokkos_ENABLE_HIP})
-  set(ACC_EXE_SPACE Kokkos::HIP)
-elseif(${Kokkos_ENABLE_OPENMP})
-  set(ACC_EXE_SPACE Kokkos::OpenMP)
-else()
-  set(ACC_EXE_SPACE Kokkos::Serial)
-endif()
-
-if(${Kokkos_ENABLE_OPENMP})
-  set(HOST_EXE_SPACE Kokkos::OpenMP)
-else()
-  set(HOST_EXE_SPACE Kokkos::Serial)
-endif()
-
-add_compile_options("-D AccelExeSpace=${ACC_EXE_SPACE}")
-add_compile_options("-D AccelMemSpace=${ACC_MEM_SPACE}")
-add_compile_options("-D HostExeSpace=${HOST_EXE_SPACE}")
-add_compile_options("-D HostMemSpace=${HOST_MEM_SPACE}")
-
 if(${BUILD_TESTING} STREQUAL "OFF")
   set(Kokkos_ENABLE_TESTS
       OFF