v1.2.0 release candidate

.clang-format

@@ -106,6 +106,8 @@ IncludeCategories:
     Priority: 4
   - Regex: '^"engines\/.*\.h"'
     Priority: 4
+  - Regex: '^"checkpoint\/.*\.h"'
+    Priority: 4
   - Regex: '^"output\/.*\.h"'
     Priority: 4
   - Regex: '^"archetypes\/.*\.h"'

.github/workflows/actions.yml

@@ -1,26 +1,44 @@
 name: Unit tests
 
 on:
-  pull_request:
-    branches:
-      - '**rc'
-      - 'master'
+  push:
 
 jobs:
+  check-commit:
+    runs-on: ubuntu-latest
+    outputs:
+      run_tests: ${{ steps.check_message.outputs.run_tests }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Check commit message
+        id: check_message
+        run: |
+          if git log -1 --pretty=%B | grep -q "RUNTEST"; then
+            echo "run_tests=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "run_tests=false" >> "$GITHUB_OUTPUT"
+          fi
   tests:
+    needs: check-commit
+    if: needs.check-commit.outputs.run_tests == 'true'
     strategy:
       fail-fast: false
       matrix:
-        device: [amd-gpu, nvidia-gpu]
+        device: [cpu, amd-gpu, nvidia-gpu]
         precision: [double, single]
-        exclude: 
+        mpi: [serial, parallel]
+        exclude: # my AMD GPU doesn't support fp64 atomics : (
         - device: amd-gpu 
           precision: double
-        # my AMD GPUs doesn't support fp64 atomics : (
+        - device: amd-gpu
+          mpi: parallel
+        - device: nvidia-gpu
+          mpi: parallel
     runs-on: [self-hosted, "${{ matrix.device }}"]
     steps:
       - name: Checkout
-        uses: actions/checkout@v3.3.0
+        uses: actions/checkout@v4
       - name: Configure
         run: |
           if [ "${{ matrix.device }}" = "nvidia-gpu" ]; then
@@ -34,6 +52,8 @@ jobs:
             fi
           elif [ "${{ matrix.device }}" = "amd-gpu" ]; then
             FLAGS="-D Kokkos_ENABLE_HIP=ON -D Kokkos_ARCH_AMD_GFX1100=ON"
+          elif [ "${{ matrix.mpi }}" = "parallel" ]; then
+            FLAGS="-D mpi=ON"
           fi
           cmake -B build -D TESTS=ON -D output=ON -D precision=${{ matrix.precision }} $FLAGS
       - name: Compile

.gitignore

@@ -51,6 +51,10 @@ venv/
 # CMake testing files
 Testing/
 
+tags
+.clangd
 .schema.json
 *_old/
 action-token
+*.vim
+ignore-*

.gitmodules

@@ -1,13 +1,12 @@
-[submodule "extern/toml11"]
-	path = extern/toml11
-	url = https://github.com/ToruNiina/toml11.git
 [submodule "extern/plog"]
 	path = extern/plog
 	url = https://github.com/SergiusTheBest/plog.git
 [submodule "extern/adios2"]
 	path = extern/adios2
 	url = https://github.com/ornladios/ADIOS2.git
-	branch = master
 [submodule "extern/Kokkos"]
 	path = extern/Kokkos
 	url = https://github.com/kokkos/kokkos.git
+[submodule "extern/entity-pgens"]
+	path = extern/entity-pgens
+	url = https://github.com/entity-toolkit/entity-pgens.git

.taplo.toml

@@ -0,0 +1,6 @@
+[formatting]
+  align_entries    = true
+  indent_tables    = true
+  indent_entries   = true
+  trailing_newline = true
+  align_comments   = true

.vscode/.taplo.toml

@@ -1,4 +1,4 @@
-include = ["input.example.toml", "setups/**/*.toml"]
+include = ["input.example.toml", "pgens/**/*.toml"]
 exclude = [".taplo.toml"]
 
 [formatting]

CMakeLists.txt

@@ -1,19 +1,22 @@
+# cmake-lint: disable=C0103,C0111,E1120,R0913,R0915
+
 cmake_minimum_required(VERSION 3.16)
 cmake_policy(SET CMP0110 NEW)
 
 set(PROJECT_NAME entity)
 
 project(
   ${PROJECT_NAME}
-  VERSION 1.1.1
+  VERSION 1.2.0
   LANGUAGES CXX C)
 add_compile_options("-D ENTITY_VERSION=\"${PROJECT_VERSION}\"")
-execute_process(COMMAND
-  bash -c "git diff --quiet src/ && echo $(git rev-parse HEAD) || echo $(git rev-parse HEAD)-mod"
+set(hash_cmd "git diff --quiet src/ && echo $(git rev-parse HEAD) ")
+string(APPEND hash_cmd "|| echo $(git rev-parse HEAD)-mod")
+execute_process(
+  COMMAND bash -c ${hash_cmd}
   WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
   OUTPUT_VARIABLE GIT_HASH
-  ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE
-)
+  ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
 message(STATUS "Git hash: ${GIT_HASH}")
 add_compile_options("-D ENTITY_GIT_HASH=\"${GIT_HASH}\"")
 
@@ -25,100 +28,115 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/defaults.cmake)
 
 # defaults
 set(DEBUG
-  ${default_debug}
-  CACHE BOOL "Debug mode")
+    ${default_debug}
+    CACHE BOOL "Debug mode")
 
 set(precision
-  ${default_precision}
-  CACHE STRING "Precision")
+    ${default_precision}
+    CACHE STRING "Precision")
 set(pgen
-  ${default_pgen}
-  CACHE STRING "Problem generator")
+    ${default_pgen}
+    CACHE STRING "Problem generator")
 
 set(gui
-  ${default_gui}
-  CACHE BOOL "Use GUI [nttiny]")
+    ${default_gui}
+    CACHE BOOL "Use GUI [nttiny]")
 set(output
-  ${default_output}
-  CACHE BOOL "Enable output")
+    ${default_output}
+    CACHE BOOL "Enable output")
 set(mpi
-  ${default_mpi}
-  CACHE BOOL "Use MPI")
+    ${default_mpi}
+    CACHE BOOL "Use MPI")
+
+set(gpu_aware_mpi
+    ${default_gpu_aware_mpi}
+    CACHE BOOL "Enable GPU-aware MPI")
 
 # -------------------------- Compilation settings -------------------------- #
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 if(${DEBUG} STREQUAL "OFF")
   set(CMAKE_BUILD_TYPE
-    Release
-    CACHE STRING "CMake build type")
+      Release
+      CACHE STRING "CMake build type")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DNDEBUG")
 else()
   set(CMAKE_BUILD_TYPE
-    Debug
-    CACHE STRING "CMake build type")
+      Debug
+      CACHE STRING "CMake build type")
   set(CMAKE_CXX_FLAGS
-    "${CMAKE_CXX_FLAGS} -DDEBUG -Wall -Wextra -Wno-unknown-pragmas")
+      "${CMAKE_CXX_FLAGS} -DDEBUG -Wall -Wextra -Wno-unknown-pragmas")
 endif()
 
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-local-typedefs")
-
 # options
 set(precisions
-  "single" "double"
-  CACHE STRING "Precisions")
+    "single" "double"
+    CACHE STRING "Precisions")
 
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/config.cmake)
 
 # ------------------------- Third-Party Tests ------------------------------ #
 set(BUILD_TESTING
-  OFF
-  CACHE BOOL "Build tests")
+    OFF
+    CACHE BOOL "Build tests")
 
 # ------------------------ Third-party dependencies ------------------------ #
-include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/kokkosConfig.cmake)
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/dependencies.cmake)
 
-find_or_fetch_dependency(Kokkos FALSE)
-find_or_fetch_dependency(plog TRUE)
-find_or_fetch_dependency(toml11 TRUE)
+find_or_fetch_dependency(Kokkos FALSE QUIET)
+find_or_fetch_dependency(plog TRUE QUIET)
 set(DEPENDENCIES Kokkos::kokkos)
 include_directories(${plog_SRC}/include)
-include_directories(${toml11_SRC})
 
 # -------------------------------- Main code ------------------------------- #
 set_precision(${precision})
+if("${Kokkos_DEVICES}" MATCHES "CUDA")
+  add_compile_options("-D CUDA_ENABLED")
+  set(DEVICE_ENABLED ON)
+  add_compile_options("-D DEVICE_ENABLED")
+elseif("${Kokkos_DEVICES}" MATCHES "HIP")
+  add_compile_options("-D HIP_ENABLED")
+  set(DEVICE_ENABLED ON)
+  add_compile_options("-D DEVICE_ENABLED")
+elseif("${Kokkos_DEVICES}" MATCHES "SYCL")
+  add_compile_options("-D SYCL_ENABLED")
+  set(DEVICE_ENABLED ON)
+  add_compile_options("-D DEVICE_ENABLED")
+else()
+  set(DEVICE_ENABLED OFF)
+endif()
+
+if(("${Kokkos_DEVICES}" MATCHES "CUDA")
+   OR ("${Kokkos_DEVICES}" MATCHES "HIP")
+   OR ("${Kokkos_DEVICES}" MATCHES "SYCL"))
+  set(DEVICE_ENABLED ON)
+else()
+  set(DEVICE_ENABLED OFF)
+endif()
 
 # MPI
 if(${mpi})
-  include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/MPIConfig.cmake)
+  find_or_fetch_dependency(MPI FALSE REQUIRED)
+  include_directories(${MPI_CXX_INCLUDE_PATH})
+  add_compile_options("-D MPI_ENABLED")
   set(DEPENDENCIES ${DEPENDENCIES} MPI::MPI_CXX)
+  if(${DEVICE_ENABLED})
+    if(${gpu_aware_mpi})
+      add_compile_options("-D GPU_AWARE_MPI")
+    endif()
+  else()
+    set(gpu_aware_mpi
+        OFF
+        CACHE BOOL "Use explicit copy when using MPI + GPU")
+  endif()
 endif()
 
 # Output
 if(${output})
-  include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/adios2Config.cmake)
-  find_or_fetch_dependency(adios2 FALSE)
-  if (NOT DEFINED ENV{HDF5_ROOT})
-    set(USE_CUSTOM_HDF5 OFF)
-    if (DEFINED ENV{CONDA_PREFIX})
-      execute_process(COMMAND bash -c "conda list | grep \"hdf5\" -q"
-        RESULT_VARIABLE HDF5_INSTALLED)
-      if (HDF5_INSTALLED EQUAL 0)
-        set(HDF5_ROOT $ENV{CONDA_PREFIX})
-      else()
-        set(USE_CUSTOM_HDF5 ON)
-      endif()
-    else()
-      set(USE_CUSTOM_HDF5 ON)
-    endif()
-    if (USE_CUSTOM_HDF5)
-      message(FATAL_ERROR "HDF5_ROOT is not set. Please set it to the root of the HDF5 installation")
-    endif()
-  endif()
-  find_package(HDF5 REQUIRED)
-
+  find_or_fetch_dependency(adios2 FALSE QUIET)
+  add_compile_options("-D OUTPUT_ENABLED")
   if(${mpi})
     set(DEPENDENCIES ${DEPENDENCIES} adios2::cxx11_mpi)
   else()
@@ -131,14 +149,18 @@ link_libraries(${DEPENDENCIES})
 if(TESTS)
   # ---------------------------------- Tests --------------------------------- #
   include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/tests.cmake)
+elseif(BENCHMARK)
+  # ------------------------------ Benchmark --------------------------------- #
+  include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/benchmark.cmake)
 else()
   # ----------------------------------- GUI ---------------------------------- #
   if(${gui})
-    find_or_fetch_dependency(nttiny FALSE)
+    find_or_fetch_dependency(nttiny FALSE QUIET)
   endif()
 
   # ------------------------------- Main source ------------------------------ #
   set_problem_generator(${pgen})
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src src)
-  include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/report.cmake)
 endif()
+
+include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/report.cmake)

CODE_OF_CONDUCT.md

@@ -0,0 +1,29 @@
+This code of conduct outlines shared principles and expectations for all participants in our open-source project. It's here to keep things open, respectful, and simple for everyone. Anyone involved with using and developing the code are expected to follow these guidelines starting from the release version `1.2.0`.
+
+# Open-source Principles
+- This project is fully open-source and does not belong to any individual or institution. 
+- It is developed and maintained by our team of incredibly talented people, whose goal is to make it accessible to everyone with no expectation of credits.
+- Anyone is free to use, copy, modify, or distribute the code under the project's open license.
+- If you contribute something to the repository, it becomes part of the project and thus will also be regarded as open-source.
+
+# Contributions and Credit
+- The only attribution we strongly encourage is a citation of either the code repository, or the corresponding method papers (coming soon).
+- All contributions are made voluntarily, and there is no expectation of recognition of isolated individuals.
+- There's no built-in expectation of credit or authorship for modules or changes pushed to the repository. Anyone is free to use any part of the code with no attribution to the author of any specific module or algorithm. 
+- The code is there for everyone to use, and its only goal is to enable the community to produce exciting science!
+
+# Roles in the Project
+- There are three informal roles:
+  - users: anyone using the code;
+  - contributors: users who have write access to the repository;
+  - maintainers: contributors who also take care of organizational, administrative, and tech-support duties.
+- Role changes (user -> contributor or vice versa) are easy and open, and can be done by asking the maintainers.
+- Contributors are also expected to follow any shared guidelines -- whether discussed informally or written down -- around code-development things, such as committing, merging, and creating pull requests.
+- Maintainers have the most strict obligations of keeping the repository clean, managing pull-requests, issuing releases, documenting all the features and additions, writing unit tests, helping other users and contributors with any problems they encounter, etc.
+- To reiterate, regardless of the extent of involvement and help, there is absolutely no expectation of recognition or attribution on maintainers' end! 
+
+# Community and Participation
+- Everyone is welcome in the community.
+- Joining meetings on Zoom, using the Slack workspace, giving feedback, taking part in decision making and planning, or following development doesn't require any special status -- it's open to all.
+
+Finally, while we cannot enforce it, we strongly encourage any projects that build on this code to be open source too. If you build upon this project, we welcome transparency and openness in spirit. You may contribute to the code as much or as little as you like; all effort is appreciated, none is required.

LICENSE

@@ -1,19 +1,8 @@
 BSD 3-Clause License
 
-Copyright (c) 2024, Princeton University (PU), and
-Princeton Plasma Physics Lab (PPPL).
+Copyright (c) 2021-present, Entity development team.
 All rights reserved.
 
-Core developers (alphabetical order):
-* Alexander Chernoglazov
-* Benjamin Crinquand
-* Alisa Galishnikova
-* Hayk Hakobyan
-* Jens Mahlmann
-* Sasha Philippov
-* Arno Vanthieghem
-* Muni Zhou
-
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met: