Supported Problem Generators cleaned

CMakeLists.txt

@@ -48,6 +48,10 @@ set(mpi
     ${default_mpi}
     CACHE BOOL "Use MPI")
 
+set(gpu_aware_mpi
+    ${default_gpu_aware_mpi}
+    CACHE BOOL "Enable GPU-aware MPI")
+
 # -------------------------- Compilation settings -------------------------- #
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -57,8 +61,7 @@ if(${DEBUG} STREQUAL "OFF")
   set(CMAKE_BUILD_TYPE
       Release
       CACHE STRING "CMake build type")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DNDEBUG "
-                      "-Wno-unused-local-typedefs -Wno-unknown-cuda-version")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DNDEBUG")
 else()
   set(CMAKE_BUILD_TYPE
       Debug
@@ -89,13 +92,45 @@ include_directories(${plog_SRC}/include)
 
 # -------------------------------- Main code ------------------------------- #
 set_precision(${precision})
+if("${Kokkos_DEVICES}" MATCHES "CUDA")
+  add_compile_options("-D CUDA_ENABLED")
+  set(DEVICE_ENABLED ON)
+  add_compile_options("-D DEVICE_ENABLED")
+elseif("${Kokkos_DEVICES}" MATCHES "HIP")
+  add_compile_options("-D HIP_ENABLED")
+  set(DEVICE_ENABLED ON)
+  add_compile_options("-D DEVICE_ENABLED")
+elseif("${Kokkos_DEVICES}" MATCHES "SYCL")
+  add_compile_options("-D SYCL_ENABLED")
+  set(DEVICE_ENABLED ON)
+  add_compile_options("-D DEVICE_ENABLED")
+else()
+  set(DEVICE_ENABLED OFF)
+endif()
+
+if(("${Kokkos_DEVICES}" MATCHES "CUDA")
+   OR ("${Kokkos_DEVICES}" MATCHES "HIP")
+   OR ("${Kokkos_DEVICES}" MATCHES "SYCL"))
+  set(DEVICE_ENABLED ON)
+else()
+  set(DEVICE_ENABLED OFF)
+endif()
 
 # MPI
 if(${mpi})
   find_or_fetch_dependency(MPI FALSE REQUIRED)
   include_directories(${MPI_CXX_INCLUDE_PATH})
   add_compile_options("-D MPI_ENABLED")
   set(DEPENDENCIES ${DEPENDENCIES} MPI::MPI_CXX)
+  if(${DEVICE_ENABLED})
+    if(${gpu_aware_mpi})
+      add_compile_options("-D GPU_AWARE_MPI")
+    endif()
+  else()
+    set(gpu_aware_mpi
+        OFF
+        CACHE BOOL "Use explicit copy when using MPI + GPU")
+  endif()
 endif()
 
 # Output

README.md

@@ -10,33 +10,20 @@ Our [detailed documentation](https://entity-toolkit.github.io/) includes everyth
 
 [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause)
 
-## Lead developers
-
-☕ __Hayk Hakobyan__ {[@haykh](https://github.com/haykh)}
-
-🥔 __Jens Mahlmann__ {[@jmahlmann](https://github.com/jmahlmann)}
-
-💁‍♂️ __Alexander Chernoglazov__ {[@SChernoglazov](https://github.com/SChernoglazov)}
-
-🧋 __Alisa Galishnikova__ {[@alisagk](https://github.com/alisagk)}
-
-🐬 __Sasha Philippov__ {[@sashaph](https://github.com/sashaph)}
-
 ## Contributors (alphabetical)
 
-🎸 __Ludwig Böss__ {[@LudwigBoess](https://github.com/LudwigBoess): PIC, framework}
-
-👀 __Yangyang Cai__ {[@StaticObserver](https://github.com/StaticObserver): GRPIC}
-
-🍵 __Benjamin Crinquand__ {[@bcrinquand](https://github.com/bcrinquand): GRPIC, cubed-sphere}
-
-🚂 __Evgeny Gorbunov__ {[@Alcauchy](https://github.com/Alcauchy): PIC, framework}
-
-:radio: __Siddhant Solanki__ {[@sidruns30](https://github.com/sidruns30): framework}
-
-🤷 __Arno Vanthieghem__ {[@vanthieg](https://github.com/vanthieg): framework, PIC}
-
-😺 __Muni Zhou__ {[@munizhou](https://github.com/munizhou): PIC}
+* :guitar: Ludwig Böss {[@LudwigBoess](https://github.com/LudwigBoess)}
+* :eyes: Yangyang Cai {[@StaticObserver](https://github.com/StaticObserver)}
+* :person_tipping_hand: Alexander Chernoglazov {[@SChernoglazov](https://github.com/SChernoglazov)}
+* :tea: Benjamin Crinquand {[@bcrinquand](https://github.com/bcrinquand)}
+* :bubble_tea: Alisa Galishnikova {[@alisagk](https://github.com/alisagk)}
+* :locomotive: Evgeny Gorbunov {[@Alcauchy](https://github.com/Alcauchy)}
+* :coffee: Hayk Hakobyan {[@haykh](https://github.com/haykh)}
+* :potato: Jens Mahlmann {[@jmahlmann](https://github.com/jmahlmann)}
+* :dolphin: Sasha Philippov {[@sashaph](https://github.com/sashaph)}
+* :radio: Siddhant Solanki {[@sidruns30](https://github.com/sidruns30)}
+* :shrug: Arno Vanthieghem {[@vanthieg](https://github.com/vanthieg)}
+* :cat: Muni Zhou {[@munizhou](https://github.com/munizhou)}
 
 ## Branch policy
 

cmake/defaults.cmake

@@ -63,4 +63,26 @@ else()
       CACHE INTERNAL "Default flag for MPI")
 endif()
 
+if(DEFINED ENV{Entity_MPI_DEVICE_COPY})
+  set(default_mpi_device_copy
+      $ENV{Entity_MPI_DEVICE_COPY}
+      CACHE INTERNAL "Default flag for copying from device to host for MPI")
+else()
+  set(default_mpi_device_copy
+      OFF
+      CACHE INTERNAL "Default flag for copying from device to host for MPI")
+endif()
+
 set_property(CACHE default_mpi PROPERTY TYPE BOOL)
+
+if(DEFINED ENV{Entity_ENABLE_GPU_AWARE_MPI})
+  set(default_gpu_aware_mpi
+      $ENV{Entity_ENABLE_GPU_AWARE_MPI}
+      CACHE INTERNAL "Default flag for GPU-aware MPI")
+else()
+  set(default_gpu_aware_mpi
+      ON
+      CACHE INTERNAL "Default flag for GPU-aware MPI")
+endif()
+
+set_property(CACHE default_gpu_aware_mpi PROPERTY TYPE BOOL)

cmake/dependencies.cmake

@@ -58,7 +58,7 @@ function(find_or_fetch_dependency package_name header_only mode)
         FetchContent_Declare(
           ${package_name}
           GIT_REPOSITORY ${${package_name}_REPOSITORY}
-          GIT_TAG 4.5.01)
+          GIT_TAG 4.6.01)
       else()
         FetchContent_Declare(${package_name}
                              GIT_REPOSITORY ${${package_name}_REPOSITORY})

cmake/report.cmake

@@ -36,7 +36,7 @@ printchoices(
   ${default_precision}
   "${Blue}"
   PRECISION_REPORT
-  36)
+  46)
 printchoices(
   "Output"
   "output"
@@ -45,7 +45,7 @@ printchoices(
   ${default_output}
   "${Green}"
   OUTPUT_REPORT
-  36)
+  46)
 printchoices(
   "MPI"
   "mpi"
@@ -54,7 +54,18 @@ printchoices(
   OFF
   "${Green}"
   MPI_REPORT
-  36)
+  46)
+if(${mpi} AND ${DEVICE_ENABLED})
+  printchoices(
+    "GPU-aware MPI"
+    "gpu_aware_mpi"
+    "${ON_OFF_VALUES}"
+    ${gpu_aware_mpi}
+    OFF
+    "${Green}"
+    GPU_AWARE_MPI_REPORT
+    46)
+endif()
 printchoices(
   "Debug mode"
   "DEBUG"
@@ -63,7 +74,7 @@ printchoices(
   OFF
   "${Green}"
   DEBUG_REPORT
-  36)
+  46)
 
 if(NOT ${PROJECT_VERSION_TWEAK} EQUAL 0)
   set(VERSION_SYMBOL "v${PROJECT_VERSION_MAJOR}." "${PROJECT_VERSION_MINOR}.")
@@ -111,13 +122,23 @@ string(REPLACE ";" "+" Kokkos_DEVICES "${Kokkos_DEVICES}")
 string(
   APPEND
   REPORT_TEXT
-  "  - ARCH [${Magenta}Kokkos_ARCH_***${ColorReset}]:         ${Kokkos_ARCH}"
+  "  - ARCH [${Magenta}Kokkos_ARCH_***${ColorReset}]:                   "
+  "${Kokkos_ARCH}"
   "\n"
-  "  - DEVICES [${Magenta}Kokkos_ENABLE_***${ColorReset}]:    ${Kokkos_DEVICES}"
+  "  - DEVICES [${Magenta}Kokkos_ENABLE_***${ColorReset}]:              "
+  "${Kokkos_DEVICES}"
   "\n"
   "  "
   ${MPI_REPORT}
-  "\n"
+  "\n")
+
+if(${mpi} AND ${DEVICE_ENABLED})
+  string(APPEND REPORT_TEXT "  " ${GPU_AWARE_MPI_REPORT} "\n")
+endif()
+
+string(
+  APPEND
+  REPORT_TEXT
   "  "
   ${DEBUG_REPORT}
   "\n"
@@ -155,7 +176,6 @@ if(${Kokkos_DEVICES} MATCHES "CUDA")
     COMMAND bash -c ${cmd}
     OUTPUT_VARIABLE CUDACOMP_VERSION
     OUTPUT_STRIP_TRAILING_WHITESPACE)
-  message(STATUS "CUDACOMP: ${CUDACOMP_VERSION}")
   string(
     APPEND
     REPORT_TEXT

cmake/styling.cmake

@@ -86,8 +86,6 @@ function(
   if(${Padding} EQUAL 0)
     list(LENGTH "${Choices}" nchoices)
     math(EXPR lastchoice "${nchoices} - 1")
-    set(ncols 4)
-    math(EXPR lastcol "${ncols} - 1")
 
     set(longest 0)
     foreach(ch IN LISTS Choices)
@@ -97,6 +95,13 @@ function(
       endif()
     endforeach()
 
+    if(longest GREATER 20)
+      set(ncols 3)
+    else()
+      set(ncols 4)
+    endif()
+    math(EXPR lastcol "${ncols} - 1")
+
     set(counter 0)
     foreach(ch IN LISTS Choices)
       if(NOT ${Value} STREQUAL "")

cmake/tests.cmake

@@ -8,8 +8,8 @@ add_subdirectory(${SRC_DIR}/metrics ${CMAKE_CURRENT_BINARY_DIR}/metrics)
 add_subdirectory(${SRC_DIR}/kernels ${CMAKE_CURRENT_BINARY_DIR}/kernels)
 add_subdirectory(${SRC_DIR}/archetypes ${CMAKE_CURRENT_BINARY_DIR}/archetypes)
 add_subdirectory(${SRC_DIR}/framework ${CMAKE_CURRENT_BINARY_DIR}/framework)
+add_subdirectory(${SRC_DIR}/output ${CMAKE_CURRENT_BINARY_DIR}/output)
 if(${output})
-  add_subdirectory(${SRC_DIR}/output ${CMAKE_CURRENT_BINARY_DIR}/output)
   add_subdirectory(${SRC_DIR}/checkpoint ${CMAKE_CURRENT_BINARY_DIR}/checkpoint)
 endif()
 
@@ -25,8 +25,9 @@ elseif(${mpi} AND ${output})
   list(APPEND TEST_DIRECTORIES framework)
 endif()
 
+list(APPEND TEST_DIRECTORIES output)
+
 if(${output})
-  list(APPEND TEST_DIRECTORIES output)
   list(APPEND TEST_DIRECTORIES checkpoint)
 endif()
 

dev/nix/kokkos.nix

@@ -1,12 +1,13 @@
 {
   pkgs ? import <nixpkgs> { },
+  stdenv,
   arch,
   gpu,
 }:
 
 let
   name = "kokkos";
-  version = "4.5.01";
+  pversion = "4.6.01";
   compilerPkgs = {
     "HIP" = with pkgs.rocmPackages; [
       rocm-core
@@ -16,15 +17,26 @@ let
       rocminfo
       rocm-smi
     ];
+    "CUDA" = with pkgs.cudaPackages; [
+      cudatoolkit
+      cuda_cudart
+    ];
     "NONE" = [
       pkgs.gcc13
     ];
   };
-  cmakeFlags = {
+  cmakeExtraFlags = {
     "HIP" = [
+      "-D Kokkos_ENABLE_HIP=ON"
+      "-D Kokkos_ARCH_${getArch { }}=ON"
       "-D CMAKE_C_COMPILER=hipcc"
       "-D CMAKE_CXX_COMPILER=hipcc"
     ];
+    "CUDA" = [
+      "-D Kokkos_ENABLE_CUDA=ON"
+      "-D Kokkos_ARCH_${getArch { }}=ON"
+      "-D CMAKE_CXX_COMPILER=$WRAPPER_PATH"
+    ];
     "NONE" = [ ];
   };
   getArch =
@@ -35,13 +47,13 @@ let
       arch;
 
 in
-pkgs.stdenv.mkDerivation {
+pkgs.stdenv.mkDerivation rec {
   pname = "${name}";
-  version = "${version}";
+  version = "${pversion}";
   src = pkgs.fetchgit {
     url = "https://github.com/kokkos/kokkos/";
-    rev = "${version}";
-    sha256 = "sha256-cI2p+6J+8BRV5fXTDxxHTfh6P5PeeLUiF73o5zVysHQ=";
+    rev = "${pversion}";
+    sha256 = "sha256-+yszUbdHqhIkJZiGLZ9Ln4DYUosuJWKhO8FkbrY0/tY=";
   };
 
   nativeBuildInputs = with pkgs; [
@@ -50,14 +62,42 @@ pkgs.stdenv.mkDerivation {
 
   propagatedBuildInputs = compilerPkgs.${gpu};
 
-  cmakeFlags = [
-    "-D CMAKE_CXX_STANDARD=17"
-    "-D CMAKE_CXX_EXTENSIONS=OFF"
-    "-D CMAKE_POSITION_INDEPENDENT_CODE=TRUE"
-    "-D Kokkos_ARCH_${getArch { }}=ON"
-    (if gpu != "none" then "-D Kokkos_ENABLE_${gpu}=ON" else "")
-    "-D CMAKE_BUILD_TYPE=Release"
-  ] ++ cmakeFlags.${gpu};
+  patchPhase =
+    if gpu == "CUDA" then
+      ''
+        export WRAPPER_PATH="$(mktemp -d)/nvcc_wrapper"
+        cp ${src}/bin/nvcc_wrapper $WRAPPER_PATH
+        substituteInPlace $WRAPPER_PATH --replace-fail "#!/usr/bin/env bash" "#!${stdenv.shell}"
+        chmod +x "$WRAPPER_PATH"
+      ''
+    else
+      "";
+
+  configurePhase = ''
+    cmake -B build -D CMAKE_BUILD_TYPE=Release \
+      -D CMAKE_CXX_STANDARD=17 \
+      -D CMAKE_CXX_EXTENSIONS=OFF \
+      -D CMAKE_POSITION_INDEPENDENT_CODE=TRUE \
+      ${pkgs.lib.concatStringsSep " " cmakeExtraFlags.${gpu}} \
+      -D CMAKE_INSTALL_PREFIX=$out
+  '';
+
+  buildPhase = ''
+    cmake --build build -j
+  '';
+
+  installPhase = ''
+    cmake --install build
+  '';
+
+  # cmakeFlags = [
+  #   "-D CMAKE_CXX_STANDARD=17"
+  #   "-D CMAKE_CXX_EXTENSIONS=OFF"
+  #   "-D CMAKE_POSITION_INDEPENDENT_CODE=TRUE"
+  #   "-D Kokkos_ARCH_${getArch { }}=ON"
+  #   (if gpu != "none" then "-D Kokkos_ENABLE_${gpu}=ON" else "")
+  #   "-D CMAKE_BUILD_TYPE=Release"
+  # ] ++ (cmakeExtraFlags.${gpu} src);
 
-  enableParallelBuilding = true;
+  # enableParallelBuilding = true;
 }