Checkpoint dumping (restart files)

.clang-format

@@ -106,6 +106,8 @@ IncludeCategories:
     Priority: 4
   - Regex: '^"engines\/.*\.h"'
     Priority: 4
+  - Regex: '^"checkpoint\/.*\.h"'
+    Priority: 4
   - Regex: '^"output\/.*\.h"'
     Priority: 4
   - Regex: '^"archetypes\/.*\.h"'

.github/workflows/actions.yml

@@ -16,8 +16,9 @@ jobs:
         exclude: 
         - device: amd-gpu 
           precision: double
-        # my AMD GPUs doesn't support fp64 atomics : (
+        # my AMD GPU doesn't support fp64 atomics : (
     runs-on: [self-hosted, "${{ matrix.device }}"]
+    if: contains(github.event.head_commit.message, 'totest')
     steps:
       - name: Checkout
         uses: actions/checkout@v3.3.0

.gitignore

@@ -54,3 +54,5 @@ Testing/
 .schema.json
 *_old/
 action-token
+*.vim
+ignore-*

.gitmodules

@@ -1,6 +1,3 @@
-[submodule "extern/toml11"]
-	path = extern/toml11
-	url = https://github.com/ToruNiina/toml11.git
 [submodule "extern/plog"]
 	path = extern/plog
 	url = https://github.com/SergiusTheBest/plog.git

CMakeLists.txt

@@ -82,10 +82,8 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/dependencies.cmake)
 
 find_or_fetch_dependency(Kokkos FALSE)
 find_or_fetch_dependency(plog TRUE)
-find_or_fetch_dependency(toml11 TRUE)
 set(DEPENDENCIES Kokkos::kokkos)
 include_directories(${plog_SRC}/include)
-include_directories(${toml11_SRC})
 
 # -------------------------------- Main code ------------------------------- #
 set_precision(${precision})

README.md

@@ -12,6 +12,8 @@ Our [detailed documentation](https://entity-toolkit.github.io/) includes everyth
 
 ## Core developers (alphabetical)
 
+👀 __Yangyang Cai__ {[@StaticObserver](https://github.com/StaticObserver): GRPIC}
+
 💁‍♂️ __Alexander Chernoglazov__ {[@SChernoglazov](https://github.com/SChernoglazov): PIC}
 
 🍵 __Benjamin Crinquand__ {[@bcrinquand](https://github.com/bcrinquand): GRPIC, cubed-sphere}

TASKLIST.md

@@ -1,33 +1,3 @@
-# v0.8
-
-- [x] thick layer boundary for the monopole
-- [x] test with filters
-- [x] add diagnostics for nans in fields and particles
-- [x] add gravitationally bound atmosphere
-- [x] rewrite UniformInjector with global random pool
-- [x] add particle deletion routine
-- [x] make more user-friendly and understandable boundary conditions
-- [x] refine output
-- [x] add different moments (momX, momY, momZ, meanGamma)
-- [x] add charge
-- [x] add per species densities
-
-# v0.9
-
-- [x] add current deposit/filtering for GR
-- [x] add moments for GR
-- [x] add Maxwellian for GR
-
-# v1.0.0
-
-- [x] particle output
-- [x] BUG in MPI particles/currents
-
-# v1.1.0
-
-- [ ] custom boundary conditions for particles and fields
-- [ ] transfer GR from v0.9
-
 ### Performance improvements to try
 
 - [ ] removing temporary variables in interpolation

cmake/report.cmake

@@ -276,10 +276,11 @@ endif()
 
 message("  ${PRECISION_REPORT}")
 message("  ${OUTPUT_REPORT}")
-message("${DASHED_LINE_SYMBOL}
-Compile configurations")
+message("${DASHED_LINE_SYMBOL}\nCompile configurations")
 
-message("  ${ARCH_REPORT}")
+if(NOT "${ARCH_REPORT}" STREQUAL "")
+  message("  ${ARCH_REPORT}")
+endif()
 message("  ${CUDA_REPORT}")
 message("  ${HIP_REPORT}")
 message("  ${OPENMP_REPORT}")

cmake/tests.cmake

@@ -10,12 +10,14 @@ add_subdirectory(${SRC_DIR}/archetypes ${CMAKE_CURRENT_BINARY_DIR}/archetypes)
 add_subdirectory(${SRC_DIR}/framework ${CMAKE_CURRENT_BINARY_DIR}/framework)
 if (${output})
   add_subdirectory(${SRC_DIR}/output ${CMAKE_CURRENT_BINARY_DIR}/output)
+  add_subdirectory(${SRC_DIR}/checkpoint ${CMAKE_CURRENT_BINARY_DIR}/checkpoint)
 endif()
 
 if (${mpi})
   # tests with mpi
   if (${output})
     add_subdirectory(${SRC_DIR}/output/tests ${CMAKE_CURRENT_BINARY_DIR}/output/tests)
+    add_subdirectory(${SRC_DIR}/framework/tests ${CMAKE_CURRENT_BINARY_DIR}/checkpoint/tests)
     add_subdirectory(${SRC_DIR}/framework/tests ${CMAKE_CURRENT_BINARY_DIR}/framework/tests)
   endif()
 else()
@@ -27,5 +29,6 @@ else()
   add_subdirectory(${SRC_DIR}/framework/tests ${CMAKE_CURRENT_BINARY_DIR}/framework/tests)
   if (${output})
     add_subdirectory(${SRC_DIR}/output/tests ${CMAKE_CURRENT_BINARY_DIR}/output/tests)
+    add_subdirectory(${SRC_DIR}/checkpoint/tests ${CMAKE_CURRENT_BINARY_DIR}/checkpoint/tests)
   endif()
 endif()

dev/runners/Dockerfile.runner.cuda

@@ -59,14 +59,17 @@ ARG HOME=/home/$USER
 WORKDIR $HOME
 
 # gh runner
+ARG RUNNER_VERSION=2.317.0
 RUN mkdir actions-runner
 WORKDIR $HOME/actions-runner
 
-RUN --mount=type=secret,id=ghtoken \
-    curl -o actions-runner-linux-x64-2.317.0.tar.gz \
-         -L https://github.com/actions/runner/releases/download/v2.317.0/actions-runner-linux-x64-2.317.0.tar.gz && \
-    tar xzf ./actions-runner-linux-x64-2.317.0.tar.gz && \
-    sudo ./bin/installdependencies.sh && \
-    ./config.sh --url https://github.com/entity-toolkit/entity --token "$(sudo cat /run/secrets/ghtoken)" --labels nvidia-gpu
+RUN curl -o actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz \
+         -L https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz && \
+    tar xzf ./actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz && \
+    sudo ./bin/installdependencies.sh
 
-ENTRYPOINT ["./run.sh"]
+ADD start.sh start.sh
+RUN sudo chown $USER:$USER start.sh && \
+    sudo chmod +x start.sh
+
+ENTRYPOINT ["./start.sh"]

input.example.toml

@@ -232,7 +232,7 @@
     #   @default: 0.9
     e_ovr_b_max = ""
     # Maximum Larmor radius allowed for GCA particles (in physical units):
-    #   @type: float: > 0
+    #   @type: float
     #   @default: 0.0
     #   @note: When `larmor_max` == 0, the limit is disabled
     larmor_max = ""
@@ -253,7 +253,7 @@
   #   @default: false
   use_weights = ""
   # Timesteps between particle re-sorting:
-  #   @type: unsigned int: >= 0
+  #   @type: unsigned int
   #   @default: 100
   #   @note: When MPI is enable, particles are sorted every step.
   #   @note: When `sort_interval` == 0, the sorting is disabled.
@@ -293,7 +293,7 @@
     #   @valid: "Boris", "Vay", "Boris,GCA", "Vay,GCA", "Photon", "None"
     pusher = ""
     # Number of additional (payload) variables for each particle of the given species:
-    #   @type: unsigned short: >= 0
+    #   @type: unsigned short
     #   @default: 0
     n_payloads = ""
     # Radiation reaction to use for the species:
@@ -316,7 +316,7 @@
   #   @default: 1
   interval = ""
   # Physical (code) time interval between all outputs (overriden by specific output intervals below):
-  #   @type: float: > 0
+  #   @type: float
   #   @default: -1.0 (disabled)
   #   @note: When `interval_time` < 0, the output is controlled by `interval`, otherwise by `interval_time`
   interval_time = ""
@@ -345,15 +345,15 @@
     #   @default: 1
     stride = ""
     # Smoothing window for the output of moments (e.g., "Rho", "Charge", "T", etc.):
-    #   @type: unsigned short: >= 0
+    #   @type: unsigned short
     #   @default: 0
     mom_smooth = ""
     # Number of timesteps between field outputs (overrides `output.interval`):
-    #   @type: unsigned int: > 0
+    #   @type: unsigned int
     #   @default: 0 (use `output.interval`)
     interval = ""
     # Physical (code) time interval between field outputs (overrides `output.interval_time`):
-    #   @type: float: > 0
+    #   @type: float
     #   @default: -1.0 (use `output.interval_time`)
     #   @note: When `interval_time` < 0, the output is controlled by `interval`, otherwise by `interval_time`
     interval_time = ""
@@ -376,7 +376,7 @@
     #   @default: 0 (use `output.interval`)
     interval = ""
     # Physical (code) time interval between field outputs (overrides `output.interval_time`):
-    #   @type: float: > 0
+    #   @type: float
     #   @default: -1.0 (use `output.interval_time`)
     #   @note: When `interval_time` < 0, the output is controlled by `interval`, otherwise by `interval_time`
     interval_time = ""
@@ -407,7 +407,7 @@
     #   @default: 0 (use `output.interval`)
     interval = ""
     # Physical (code) time interval between spectra outputs (overrides `output.interval_time`):
-    #   @type: float: > 0
+    #   @type: float
     #   @default: -1.0 (use `output.interval_time`)
     #   @note: When `interval_time` < 0, the output is controlled by `interval`, otherwise by `interval_time`
     interval_time = ""
@@ -422,6 +422,37 @@
     #   @default: false
     ghosts = ""
 
+[checkpoint]
+  # Number of timesteps between checkpoints:
+  #   @type: unsigned int: > 0
+  #   @default: 1000
+  interval = ""
+  # Physical (code) time interval between checkpoints:
+  #   @type: float: > 0
+  #   @default: -1.0 (disabled)
+  #   @note: When `interval_time` < 0, the output is controlled by `interval`, otherwise by `interval_time`
+  interval_time = ""
+  # Number of checkpoints to keep:
+  #   @type: int
+  #   @default: 2
+  #   @note: 0 = disable checkpointing
+  #   @note: -1 = keep all checkpoints
+  keep = ""
+
+  # @inferred:
+  # - is_resuming
+  #     @brief: Whether the simulation is resuming from a checkpoint
+  #     @type: bool
+  #     @from: command-line flag
+  # - start_step
+  #     @brief: Timestep of the checkpoint used to resume
+  #     @type: unsigned int
+  #     @from: automatically determined during restart
+  # - start_time
+  #     @brief: Time of the checkpoint used to resume
+  #     @type: float
+  #     @from: automatically determined during restart
+
 [diagnostics]
   # Number of timesteps between diagnostic logs:
   #   @type: int: > 0

legacy/src/framework/utils/particle_injectors.hpp

@@ -165,7 +165,7 @@ namespace ntt {
    * @brief Volumetrically uniform particle injector parallelized over particles.
    * @tparam D dimension.
    * @tparam S simulation engine.
-   * @tparam EnDist energy distribution [default = ColdDist].
+   * @tparam EnDist energy distribution [default = Cold].
    *
    * @param params simulation parameters.
    * @param mblock meshblock.
@@ -174,7 +174,7 @@ namespace ntt {
    * @param region region to inject particles as a list of coordinates [optional].
    * @param time current time [optional].
    */
-  template <Dimension D, SimulationEngine S, template <Dimension, SimulationEngine> class EnDist = ColdDist>
+  template <Dimension D, SimulationEngine S, template <Dimension, SimulationEngine> class EnDist = Cold>
   inline void InjectUniform(const SimulationParams& params,
                             Meshblock<D, S>&        mblock,
                             const std::vector<int>& species,
@@ -613,8 +613,8 @@ namespace ntt {
    * @brief Particle injector parallelized by cells in a volume.
    * @tparam D dimension.
    * @tparam S simulation engine.
-   * @tparam EnDist energy distribution [default = ColdDist].
-   * @tparam SpDist spatial distribution [default = UniformDist].
+   * @tparam EnDist energy distribution [default = Cold].
+   * @tparam SpDist spatial distribution [default = Uniform].
    * @tparam InjCrit injection criterion [default = NoCriterion].
    *
    * @param params simulation parameters.
@@ -626,8 +626,8 @@ namespace ntt {
    */
   template <Dimension        D,
             SimulationEngine S,
-            template <Dimension, SimulationEngine> class EnDist  = ColdDist,
-            template <Dimension, SimulationEngine> class SpDist  = UniformDist,
+            template <Dimension, SimulationEngine> class EnDist  = Cold,
+            template <Dimension, SimulationEngine> class SpDist  = Uniform,
             template <Dimension, SimulationEngine> class InjCrit = NoCriterion>
   inline void InjectInVolume(const SimulationParams& params,
                              Meshblock<D, S>&        mblock,
@@ -928,7 +928,7 @@ namespace ntt {
    * @brief ... up to certain number density.
    * @tparam D dimension.
    * @tparam S simulation engine.
-   * @tparam EnDist energy distribution [default = ColdDist].
+   * @tparam EnDist energy distribution [default = Cold].
    * @tparam InjCrit injection criterion [default = NoCriterion].
    *
    * @param params simulation parameters.
@@ -940,7 +940,7 @@ namespace ntt {
    */
   template <Dimension        D,
             SimulationEngine S,
-            template <Dimension, SimulationEngine> class EnDist  = ColdDist,
+            template <Dimension, SimulationEngine> class EnDist  = Cold,
             template <Dimension, SimulationEngine> class InjCrit = NoCriterion>
   inline void InjectNonUniform(const SimulationParams&      params,
                                Meshblock<D, S>&             mblock,

setups/srpic/turbulence/pgen.hpp

@@ -185,30 +185,25 @@ namespace user {
         const auto injector = arch::UniformInjector<S, M, arch::Maxwellian>(
           energy_dist,
           { 1, 2 });
-        const real_t ndens = 1.0;
+        const real_t ndens = 0.9;
         arch::InjectUniform<S, M, decltype(injector)>(params,
                                                       local_domain,
                                                       injector,
                                                       ndens);
       }
 
       {
-        // const auto energy_dist = arch::Maxwellian<S, M>(local_domain.mesh.metric,
-        //                                                 local_domain.random_pool,
-        //                                                 temperature*10);        
-        // // const auto energy_dist = arch::Maxwellian<S, M>(local_domain.mesh.metric,
-        // //                                                 local_domain.random_pool,
-        // //                                                 temperature * 2,
-        // //                                                 10.0,
-        // //                                                 1);
-        // const auto injector = arch::UniformInjector<S, M, arch::Maxwellian>(
-        //   energy_dist,
-        //   { 1, 2 });
-        // const real_t ndens = 0.01;
-        // arch::InjectUniform<S, M, decltype(injector)>(params,
-        //                                               local_domain,
-        //                                               injector,
-        //                                               ndens);
+        const auto energy_dist = arch::PowerlawDist<S, M>(local_domain.mesh.metric,
+                                                        local_domain.random_pool,
+                                                        0.1, 100.0, -3.0);
+        const auto injector = arch::UniformInjector<S, M, arch::PowerlawDist>(
+          energy_dist,
+          { 1, 2 });
+        const real_t ndens = 0.1;
+        arch::InjectUniform<S, M, decltype(injector)>(params,
+                                                      local_domain,
+                                                      injector,
+                                                      ndens);
       }
     }
 

src/CMakeLists.txt

@@ -7,6 +7,7 @@
 # - ntt_framework [required]
 # - ntt_metrics [required]
 # - ntt_engine [required]
+# - ntt_pgen [required]
 # @uses:
 # - kokkos [required]
 # - plog [required]
@@ -30,9 +31,12 @@ add_subdirectory(${SRC_DIR}/kernels ${CMAKE_CURRENT_BINARY_DIR}/kernels)
 add_subdirectory(${SRC_DIR}/archetypes ${CMAKE_CURRENT_BINARY_DIR}/archetypes)
 add_subdirectory(${SRC_DIR}/framework ${CMAKE_CURRENT_BINARY_DIR}/framework)
 add_subdirectory(${SRC_DIR}/engines ${CMAKE_CURRENT_BINARY_DIR}/engines)
-if (${output} STREQUAL "ON")
-add_subdirectory(${SRC_DIR}/output ${CMAKE_CURRENT_BINARY_DIR}/output)
+
+if (${output})
+  add_subdirectory(${SRC_DIR}/output ${CMAKE_CURRENT_BINARY_DIR}/output)
+  add_subdirectory(${SRC_DIR}/checkpoint ${CMAKE_CURRENT_BINARY_DIR}/checkpoint)
 endif()
+
 add_subdirectory(${SRC_DIR}/../setups ${CMAKE_CURRENT_BINARY_DIR}/setups)
 
 set(libs ntt_global ntt_framework ntt_metrics ntt_engines ntt_pgen)