From a8836ac7ae3a00b3ccbef447479715b80b924435 Mon Sep 17 00:00:00 2001 From: hayk Date: Tue, 27 May 2025 15:05:27 -0400 Subject: [PATCH 01/13] walltime checkpoint added --- input.example.toml | 9 +++++++++ src/engines/engine.hpp | 5 ++++- src/engines/engine_init.cpp | 16 ++++++++++++++++ src/engines/engine_run.cpp | 10 ++++++++++ src/framework/parameters.cpp | 10 ++++++---- src/global/global.h | 12 +++++++----- 6 files changed, 52 insertions(+), 10 deletions(-) diff --git a/input.example.toml b/input.example.toml index 4a671fab6..5faacf2fc 100644 --- a/input.example.toml +++ b/input.example.toml @@ -470,6 +470,15 @@ # @note: 0 = disable checkpointing # @note: -1 = keep all checkpoints keep = "" + # Write a checkpoint once after a fixed walltime + # @type: string + # @default: "" (disabled) + # @note: The format is "HH:MM:SS" + # @note: Useful for long-running simulations to ensure that ... + # @note: ... the simulation can be resumed after allocation expires + # @note: Empty string means disables this (still writes checkpoints on interval) + # @note: This does not exit the simulation + walltime = "" # @inferred: # - is_resuming diff --git a/src/engines/engine.hpp b/src/engines/engine.hpp index 9525874a6..261b1d6ea 100644 --- a/src/engines/engine.hpp +++ b/src/engines/engine.hpp @@ -23,7 +23,6 @@ #include "arch/traits.h" #include "utils/error.h" -#include "utils/progressbar.h" #include "utils/timer.h" #include "utils/toml.h" @@ -44,6 +43,7 @@ #include #endif // MPI_ENABLED +#include #include #include @@ -61,6 +61,9 @@ namespace ntt { #else adios2::ADIOS m_adios; #endif + const timestamp_t start_walltime { std::chrono::system_clock::now() }; + timestamp_t end_walltime {}; + bool walltime_checkpoint_pending { false }; #endif SimulationParams m_params; diff --git a/src/engines/engine_init.cpp b/src/engines/engine_init.cpp index 833a7771f..0e99451e9 100644 --- a/src/engines/engine_init.cpp +++ b/src/engines/engine_init.cpp @@ -16,6 +16,8 @@ #include +#include + namespace ntt { template @@ -25,6 +27,20 @@ namespace ntt { #if defined(OUTPUT_ENABLED) m_metadomain.InitWriter(&m_adios, m_params, is_resuming); m_metadomain.InitCheckpointWriter(&m_adios, m_params); + + const auto checkpoint_walltime = m_params.template get( + "checkpoint.walltime"); + if (not checkpoint_walltime.empty()) { + walltime_checkpoint_pending = true; + raise::ErrorIf(checkpoint_walltime.size() != 8, + "invalid checkpoint walltime format, expected HH:MM:SS", + HERE); + end_walltime = + start_walltime + + std::chrono::hours(std::stoi(checkpoint_walltime.substr(0, 2))) + + std::chrono::minutes(std::stoi(checkpoint_walltime.substr(3, 2))) + + std::chrono::seconds(std::stoi(checkpoint_walltime.substr(6, 2))); + } #endif logger::Checkpoint("Initializing Engine", HERE); if (not is_resuming) { diff --git a/src/engines/engine_run.cpp b/src/engines/engine_run.cpp index 472acae46..256aac8e3 100644 --- a/src/engines/engine_run.cpp +++ b/src/engines/engine_run.cpp @@ -97,6 +97,15 @@ namespace ntt { step - 1, time, time - dt); + if ((not print_checkpoint) and (walltime_checkpoint_pending) and + (std::chrono::system_clock::now() >= end_walltime)) { + print_checkpoint = m_metadomain.WriteCheckpoint(m_params, + step, + step - 1, + time, + time - dt); + walltime_checkpoint_pending = false; + } timers.stop("Checkpoint"); #endif @@ -133,4 +142,5 @@ namespace ntt { template void Engine>::run(); template void Engine>::run(); template void Engine>::run(); + } // namespace ntt diff --git a/src/framework/parameters.cpp b/src/framework/parameters.cpp index 168640e1f..3c0b334c7 100644 --- a/src/framework/parameters.cpp +++ b/src/framework/parameters.cpp @@ -31,10 +31,10 @@ namespace ntt { template - auto get_dx0_V0( - const std::vector& resolution, - const boundaries_t& extent, - const std::map& params) -> std::pair { + auto get_dx0_V0(const std::vector& resolution, + const boundaries_t& extent, + const std::map& params) + -> std::pair { const auto metric = M(resolution, extent, params); const auto dx0 = metric.dxMin(); coord_t x_corner { ZERO }; @@ -603,6 +603,8 @@ namespace ntt { toml::find_or(toml_data, "checkpoint", "interval_time", -1.0)); set("checkpoint.keep", toml::find_or(toml_data, "checkpoint", "keep", defaults::checkpoint::keep)); + set("checkpoint.walltime", + toml::find_or(toml_data, "checkpoint", "walltime", std::string(""))); /* [diagnostics] -------------------------------------------------------- */ set("diagnostics.interval", diff --git a/src/global/global.h b/src/global/global.h index 42114a1da..1f7987929 100644 --- a/src/global/global.h +++ b/src/global/global.h @@ -342,12 +342,14 @@ template using vec_t = tuple_t; // time/duration +using duration_t = double; +using simtime_t = double; +using timestep_t = std::size_t; +using ncells_t = std::size_t; +using npart_t = unsigned long int; + +// walltime using timestamp_t = std::chrono::time_point; -using duration_t = double; -using simtime_t = double; -using timestep_t = std::size_t; -using ncells_t = std::size_t; -using npart_t = unsigned long int; // index/number using index_t = const std::size_t; From f3cb42ba308ab131defdbb1913d18dc024b923ea Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 30 May 2025 02:44:41 -0400 Subject: [PATCH 02/13] additional range checks (in compl. with gcc-15) --- src/kernels/tests/deposit.cpp | 95 +++++++++++++--------------- src/kernels/tests/digital_filter.cpp | 93 +++++++++++++-------------- src/kernels/tests/prtl_bc.cpp | 90 +++++++++++++++----------- src/metrics/kerr_schild.h | 4 +- src/metrics/kerr_schild_0.h | 4 +- src/metrics/metric_base.h | 46 +++++++++++--- src/metrics/minkowski.h | 23 ++++--- src/metrics/qkerr_schild.h | 4 +- src/metrics/qspherical.h | 7 +- src/metrics/spherical.h | 7 +- 10 files changed, 205 insertions(+), 168 deletions(-) diff --git a/src/kernels/tests/deposit.cpp b/src/kernels/tests/deposit.cpp index e6967eb14..1570937b8 100644 --- a/src/kernels/tests/deposit.cpp +++ b/src/kernels/tests/deposit.cpp @@ -55,7 +55,12 @@ void testDeposit(const std::vector& res, errorIf(res.size() != M::Dim, "res.size() != M::Dim"); using namespace ntt; - M metric { res, ext, params }; + auto extents = ext; + if constexpr (M::CoordType != Coord::Cart) { + extents.emplace_back(ZERO, (real_t)(constant::PI)); + } + + M metric { res, extents, params }; const auto nx1 = res[0]; const auto nx2 = res[1]; @@ -170,59 +175,49 @@ auto main(int argc, char* argv[]) -> int { using namespace ntt; using namespace metric; - testDeposit, SimEngine::SRPIC>( - { - 10, - 10 - }, - { { 0.0, 55.0 }, { 0.0, 55.0 } }, - {}, - 500); - - testDeposit, SimEngine::SRPIC>( - { - 10, - 10 - }, - { { 1.0, 100.0 } }, - {}, - 500); - - testDeposit, SimEngine::SRPIC>( - { - 10, - 10 + const auto res = std::vector { 10, 10 }; + const auto r_extent = boundaries_t { + { 0.0, 100.0 } + }; + const auto xy_extent = boundaries_t { + { 0.0, 55.0 }, + { 0.0, 55.0 } + }; + + testDeposit, SimEngine::SRPIC>(res, xy_extent, {}, 500); + + testDeposit, SimEngine::SRPIC>(res, r_extent, {}, 500); + + testDeposit, SimEngine::SRPIC>(res, + r_extent, + { + { "r0", 0.0 }, + { "h", 0.25 } }, - { { 1.0, 100.0 } }, - { { "r0", 0.0 }, { "h", 0.25 } }, - 500); - - testDeposit, SimEngine::GRPIC>( - { - 10, - 10 + 500); + + testDeposit, SimEngine::GRPIC>(res, + r_extent, + { + { "a", 0.9 } }, - { { 1.0, 100.0 } }, - { { "a", 0.9 } }, - 500); - - testDeposit, SimEngine::GRPIC>( - { - 10, - 10 + 500); + + testDeposit, SimEngine::GRPIC>(res, + r_extent, + { + { "r0", 0.0 }, + { "h", 0.25 }, + { "a", 0.9 } }, - { { 1.0, 100.0 } }, - { { "r0", 0.0 }, { "h", 0.25 }, { "a", 0.9 } }, - 500); - - testDeposit, SimEngine::GRPIC>( - { - 10, - 10 + 500); + + testDeposit, SimEngine::GRPIC>(res, + r_extent, + { + { "a", 0.9 } }, - { { 1.0, 100.0 } }, - { { "a", 0.9 } }, - 500); + 500); } catch (std::exception& e) { std::cerr << e.what() << std::endl; diff --git a/src/kernels/tests/digital_filter.cpp b/src/kernels/tests/digital_filter.cpp index 03a63753b..30059e471 100644 --- a/src/kernels/tests/digital_filter.cpp +++ b/src/kernels/tests/digital_filter.cpp @@ -20,7 +20,6 @@ #include #include #include -#include #include void errorIf(bool condition, const std::string& message) { @@ -34,15 +33,17 @@ void testFilter(const std::vector& res, const boundaries_t& ext, const std::map& params = {}) { static_assert(M::Dim == 2); - errorIf(res.size() != M::Dim, "res.size() != M::Dim"); + errorIf(res.size() != static_cast(M::Dim), "res.size() != M::Dim"); using namespace ntt; auto boundaries = boundaries_t {}; + auto extents = ext; if constexpr (M::CoordType != Coord::Cart) { boundaries = { { FldsBC::CUSTOM, FldsBC::CUSTOM }, { FldsBC::AXIS, FldsBC::AXIS } }; + extents.emplace_back(ZERO, (real_t)(constant::PI)); } else { boundaries = { { FldsBC::PERIODIC, FldsBC::PERIODIC }, @@ -50,7 +51,7 @@ void testFilter(const std::vector& res, }; } - M metric { res, ext, params }; + M metric { res, extents, params }; const auto nx1 = res[0]; const auto nx2 = res[1]; @@ -133,53 +134,45 @@ auto main(int argc, char* argv[]) -> int { using namespace ntt; using namespace metric; - testFilter>( - { - 10, - 10 - }, - { { 0.0, 55.0 }, { 0.0, 55.0 } }, - {}); - - testFilter>( - { - 10, - 10 - }, - { { 1.0, 100.0 } }, - {}); - - testFilter>( - { - 10, - 10 - }, - { { 1.0, 100.0 } }, - { { "r0", 0.0 }, { "h", 0.25 } }); - - testFilter>( - { - 10, - 10 - }, - { { 1.0, 100.0 } }, - { { "a", 0.9 } }); - - testFilter>( - { - 10, - 10 - }, - { { 1.0, 100.0 } }, - { { "r0", 0.0 }, { "h", 0.25 }, { "a", 0.9 } }); - - testFilter>( - { - 10, - 10 - }, - { { 1.0, 100.0 } }, - { { "a", 0.9 } }); + const auto res = std::vector { 10, 10 }; + const auto r_extent = boundaries_t { + { 0.0, 100.0 } + }; + const auto xy_extent = boundaries_t { + { 0.0, 55.0 }, + { 0.0, 55.0 } + }; + + testFilter>(res, xy_extent, {}); + + testFilter>(res, r_extent, {}); + + testFilter>(res, + r_extent, + { + { "r0", 0.0 }, + { "h", 0.25 } + }); + + testFilter>(res, + r_extent, + { + { "a", 0.9 } + }); + + testFilter>(res, + r_extent, + { + { "r0", 0.0 }, + { "h", 0.25 }, + { "a", 0.9 } + }); + + testFilter>(res, + r_extent, + { + { "a", 0.9 } + }); } catch (std::exception& e) { std::cerr << e.what() << std::endl; diff --git a/src/kernels/tests/prtl_bc.cpp b/src/kernels/tests/prtl_bc.cpp index 14c1a9f54..80e4ec7ed 100644 --- a/src/kernels/tests/prtl_bc.cpp +++ b/src/kernels/tests/prtl_bc.cpp @@ -53,35 +53,45 @@ void testPeriodicBC(const std::vector& res, const auto NoGCA = false; const auto NoExtForce = false; - boundaries_t extent; - extent = ext; - const auto sx = static_cast(extent[0].second - extent[0].first); - const auto sy = static_cast( - extent.size() > 1 ? extent[1].second - extent[1].first : 0); - const auto sz = static_cast( - extent.size() > 2 ? extent[2].second - extent[2].first : 0); + real_t sx = ZERO, sy = ZERO, sz = ZERO; + if (ext.size() > 0) { + sx = static_cast(ext.at(0).second - ext.at(0).first); + } + if (ext.size() > 1) { + sy = static_cast(ext.at(1).second - ext.at(1).first); + } + if (ext.size() > 2) { + sz = static_cast(ext.at(2).second - ext.at(2).first); + } - M metric { res, extent, params }; + M metric { res, ext, params }; - const int nx1 = res[0]; - const int nx2 = res[1]; - const int nx3 = res[2]; + int nx1 = 0, nx2 = 0, nx3 = 0; + if (res.size() > 0) { + nx1 = static_cast(res.at(0)); + } + if (res.size() > 1) { + nx2 = static_cast(res.at(1)); + } + if (res.size() > 2) { + nx3 = static_cast(res.at(2)); + } - const real_t dt = 0.1 * (extent[0].second - extent[0].first) / sx; + const real_t dt = 0.1 * (ext.at(0).second - ext.at(0).first) / sx; const real_t coeff = HALF * dt; ndfield_t emfield; if constexpr (M::Dim == Dim::_1D) { - emfield = ndfield_t { "emfield", res[0] + 2 * N_GHOSTS }; + emfield = ndfield_t { "emfield", res.at(0) + 2 * N_GHOSTS }; } else if constexpr (M::Dim == Dim::_2D) { emfield = ndfield_t { "emfield", - res[0] + 2 * N_GHOSTS, - res[1] + 2 * N_GHOSTS }; + res.at(0) + 2 * N_GHOSTS, + res.at(1) + 2 * N_GHOSTS }; } else { emfield = ndfield_t { "emfield", - res[0] + 2 * N_GHOSTS, - res[1] + 2 * N_GHOSTS, - res[2] + 2 * N_GHOSTS }; + res.at(0) + 2 * N_GHOSTS, + res.at(1) + 2 * N_GHOSTS, + res.at(2) + 2 * N_GHOSTS }; } const short sp_idx = 1; @@ -106,18 +116,26 @@ void testPeriodicBC(const std::vector& res, array_t phi; // init parameters of prtl #1 - real_t xi_1 = 0.515460 * sx + extent[0].first; - real_t yi_1 = 0.340680 * sy + extent[1].first; - real_t zi_1 = 0.940722 * sz + extent[2].first; + real_t xi_1 = ZERO, yi_1 = ZERO, zi_1 = ZERO; + real_t xi_2 = ZERO, yi_2 = ZERO, zi_2 = ZERO; + if constexpr (M::Dim == Dim::_1D or M::Dim == Dim::_2D or M::Dim == Dim::_3D) { + xi_1 = 0.515460 * sx + ext.at(0).first; + xi_2 = 0.149088 * sx + ext.at(0).first; + } + if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { + yi_1 = 0.340680 * sy + ext.at(1).first; + yi_2 = 0.997063 * sy + ext.at(1).first; + } + if constexpr (M::Dim == Dim::_3D) { + zi_1 = 0.940722 * sz + ext.at(2).first; + zi_2 = 0.607354 * sz + ext.at(2).first; + } real_t ux_1 = 0.569197; real_t uy_1 = 0.716085; real_t uz_1 = 0.760101; real_t gamma_1 = math::sqrt(1.0 + SQR(ux_1) + SQR(uy_1) + SQR(uz_1)); // init parameters of prtl #2 - real_t xi_2 = 0.149088 * sx + extent[0].first; - real_t yi_2 = 0.997063 * sy + extent[1].first; - real_t zi_2 = 0.607354 * sz + extent[2].first; real_t ux_2 = -0.872069; real_t uy_2 = 0.0484461; real_t uz_2 = -0.613575; @@ -268,16 +286,16 @@ void testPeriodicBC(const std::vector& res, if constexpr (M::Dim == Dim::_1D or M::Dim == Dim::_2D or M::Dim == Dim::_3D) { xi_1 += dt * ux_1 / gamma_1; xi_2 += dt * ux_2 / gamma_2; - if (xi_1 >= extent[0].second) { + if (xi_1 >= ext.at(0).second) { xi_1 -= sx; } - if (xi_1 < extent[0].first) { + if (xi_1 < ext.at(0).first) { xi_1 += sx; } - if (xi_2 >= extent[0].second) { + if (xi_2 >= ext.at(0).second) { xi_2 -= sx; } - if (xi_2 < extent[0].first) { + if (xi_2 < ext.at(0).first) { xi_2 += sx; } errorIf(not equal(xPh_1[0] / sx, @@ -290,16 +308,16 @@ void testPeriodicBC(const std::vector& res, if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { yi_1 += dt * uy_1 / gamma_1; yi_2 += dt * uy_2 / gamma_2; - if (yi_1 >= extent[1].second) { + if (yi_1 >= ext.at(1).second) { yi_1 -= sy; } - if (yi_1 < extent[1].first) { + if (yi_1 < ext.at(1).first) { yi_1 += sy; } - if (yi_2 >= extent[1].second) { + if (yi_2 >= ext.at(1).second) { yi_2 -= sy; } - if (yi_2 < extent[1].first) { + if (yi_2 < ext.at(1).first) { yi_2 += sy; } errorIf(not equal(xPh_1[1] / sy, @@ -312,16 +330,16 @@ void testPeriodicBC(const std::vector& res, if constexpr (M::Dim == Dim::_3D) { zi_1 += dt * uz_1 / gamma_1; zi_2 += dt * uz_2 / gamma_2; - if (zi_1 >= extent[2].second) { + if (zi_1 >= ext.at(2).second) { zi_1 -= sz; } - if (zi_1 < extent[2].first) { + if (zi_1 < ext.at(2).first) { zi_1 += sz; } - if (zi_2 >= extent[2].second) { + if (zi_2 >= ext.at(2).second) { zi_2 -= sz; } - if (zi_2 < extent[2].first) { + if (zi_2 < ext.at(2).first) { zi_2 += sz; } errorIf(not equal(xPh_1[2] / sz, diff --git a/src/metrics/kerr_schild.h b/src/metrics/kerr_schild.h index 13294dd7d..2ef702cce 100644 --- a/src/metrics/kerr_schild.h +++ b/src/metrics/kerr_schild.h @@ -72,8 +72,8 @@ namespace metric { using MetricBase::nx3; using MetricBase::set_dxMin; - KerrSchild(std::vector res, - boundaries_t ext, + KerrSchild(const std::vector& res, + const boundaries_t& ext, const std::map& params) : MetricBase { res, ext } , a { params.at("a") } diff --git a/src/metrics/kerr_schild_0.h b/src/metrics/kerr_schild_0.h index fdd2fd847..ccacbf6d9 100644 --- a/src/metrics/kerr_schild_0.h +++ b/src/metrics/kerr_schild_0.h @@ -53,8 +53,8 @@ namespace metric { using MetricBase::nx3; using MetricBase::set_dxMin; - KerrSchild0(std::vector res, - boundaries_t ext, + KerrSchild0(const std::vector& res, + const boundaries_t& ext, const std::map& = {}) : MetricBase { res, ext } , dr { (x1_max - x1_min) / nx1 } diff --git a/src/metrics/metric_base.h b/src/metrics/metric_base.h index 66cc28918..a418361a3 100644 --- a/src/metrics/metric_base.h +++ b/src/metrics/metric_base.h @@ -46,12 +46,38 @@ #include "global.h" +#include "utils/error.h" #include "utils/numeric.h" #include namespace metric { + namespace { + template + auto getNXi(const std::vector& res) -> real_t { + if constexpr (i >= static_cast(D)) { + return ONE; + } else { + raise::ErrorIf(res.size() <= i, "Invalid res size provided to metric", HERE); + return static_cast(res.at(i)); + } + }; + + template + auto getExtent(const boundaries_t& ext) -> real_t { + if constexpr (i >= static_cast(D)) { + return ZERO; + } else { + raise::ErrorIf(ext.size() <= i, "Invalid ext size provided to metric", HERE); + return min ? ext.at(i).first : ext.at(i).second; + } + }; + + constexpr bool XMin = true; + constexpr bool XMax = false; + }; // namespace + /** * Virtual parent metric class template: h_ij * Coordinates vary from `0` to `nx1` ... (code units) @@ -61,16 +87,16 @@ namespace metric { static constexpr bool is_metric { true }; static constexpr Dimension Dim { D }; - MetricBase(std::vector res, boundaries_t ext) - : nx1 { res.size() > 0 ? (real_t)(res[0]) : ONE } - , nx2 { res.size() > 1 ? (real_t)(res[1]) : ONE } - , nx3 { res.size() > 2 ? (real_t)(res[2]) : ONE } - , x1_min { res.size() > 0 ? ext[0].first : ZERO } - , x1_max { res.size() > 0 ? ext[0].second : ZERO } - , x2_min { res.size() > 1 ? ext[1].first : ZERO } - , x2_max { res.size() > 1 ? ext[1].second : ZERO } - , x3_min { res.size() > 2 ? ext[2].first : ZERO } - , x3_max { res.size() > 2 ? ext[2].second : ZERO } {} + MetricBase(const std::vector& res, const boundaries_t& ext) + : nx1 { getNXi(res) } + , nx2 { getNXi(res) } + , nx3 { getNXi(res) } + , x1_min { getExtent(ext) } + , x1_max { getExtent(ext) } + , x2_min { getExtent(ext) } + , x2_max { getExtent(ext) } + , x3_min { getExtent(ext) } + , x3_max { getExtent(ext) } {} ~MetricBase() = default; diff --git a/src/metrics/minkowski.h b/src/metrics/minkowski.h index 244ce9f25..e9ee76afa 100644 --- a/src/metrics/minkowski.h +++ b/src/metrics/minkowski.h @@ -47,24 +47,26 @@ namespace metric { using MetricBase::nx3; using MetricBase::set_dxMin; - Minkowski(std::vector res, - boundaries_t ext, + Minkowski(const std::vector& res, + const boundaries_t& ext, const std::map& = {}) : MetricBase { res, ext } , dx { (x1_max - x1_min) / nx1 } , dx_inv { ONE / dx } { set_dxMin(find_dxMin()); const auto epsilon = std::numeric_limits::epsilon() * - static_cast(100.0); + static_cast(100.0); if constexpr (D != Dim::_1D) { - raise::ErrorIf(not cmp::AlmostEqual((x2_max - x2_min) / (real_t)(nx2), dx, epsilon), - "dx2 must be equal to dx1 in 2D", - HERE); + raise::ErrorIf( + not cmp::AlmostEqual((x2_max - x2_min) / (real_t)(nx2), dx, epsilon), + "dx2 must be equal to dx1 in 2D", + HERE); } if constexpr (D == Dim::_3D) { - raise::ErrorIf(not cmp::AlmostEqual((x3_max - x3_min) / (real_t)(nx3), dx, epsilon), - "dx3 must be equal to dx1 in 3D", - HERE); + raise::ErrorIf( + not cmp::AlmostEqual((x3_max - x3_min) / (real_t)(nx3), dx, epsilon), + "dx3 must be equal to dx1 in 3D", + HERE); } } @@ -242,7 +244,8 @@ namespace metric { * @note tetrad/cart <-> cntrv <-> cov */ template - Inline auto transform(const coord_t& xi, const real_t& v_in) const -> real_t { + Inline auto transform(const coord_t& xi, const real_t& v_in) const + -> real_t { static_assert(i > 0 && i <= 3, "Invalid index i"); static_assert(in != out, "Invalid vector transformation"); if constexpr (i > static_cast(D)) { diff --git a/src/metrics/qkerr_schild.h b/src/metrics/qkerr_schild.h index d64fe05da..836646377 100644 --- a/src/metrics/qkerr_schild.h +++ b/src/metrics/qkerr_schild.h @@ -72,8 +72,8 @@ namespace metric { using MetricBase::nx3; using MetricBase::set_dxMin; - QKerrSchild(std::vector res, - boundaries_t ext, + QKerrSchild(const std::vector& res, + const boundaries_t& ext, const std::map& params) : MetricBase { res, ext } , a { params.at("a") } diff --git a/src/metrics/qspherical.h b/src/metrics/qspherical.h index 4df5db866..77c28dbff 100644 --- a/src/metrics/qspherical.h +++ b/src/metrics/qspherical.h @@ -55,8 +55,8 @@ namespace metric { using MetricBase::nx3; using MetricBase::set_dxMin; - QSpherical(std::vector res, - boundaries_t ext, + QSpherical(const std::vector& res, + const boundaries_t& ext, const std::map& params) : MetricBase { res, ext } , r0 { params.at("r0") } @@ -284,7 +284,8 @@ namespace metric { * @note tetrad/sph <-> cntrv <-> cov */ template - Inline auto transform(const coord_t& xi, const real_t& v_in) const -> real_t { + Inline auto transform(const coord_t& xi, const real_t& v_in) const + -> real_t { static_assert(i > 0 && i <= 3, "Invalid index i"); static_assert(in != out, "Invalid vector transformation"); if constexpr ((in == Idx::T && out == Idx::Sph) || diff --git a/src/metrics/spherical.h b/src/metrics/spherical.h index 54e12367d..5f510d5e1 100644 --- a/src/metrics/spherical.h +++ b/src/metrics/spherical.h @@ -50,8 +50,8 @@ namespace metric { using MetricBase::nx3; using MetricBase::set_dxMin; - Spherical(std::vector res, - boundaries_t ext, + Spherical(const std::vector& res, + const boundaries_t& ext, const std::map& = {}) : MetricBase { res, ext } , dr((x1_max - x1_min) / nx1) @@ -252,7 +252,8 @@ namespace metric { * @note tetrad/sph <-> cntrv <-> cov */ template - Inline auto transform(const coord_t& xi, const real_t& v_in) const -> real_t { + Inline auto transform(const coord_t& xi, const real_t& v_in) const + -> real_t { static_assert(i > 0 && i <= 3, "Invalid index i"); static_assert(in != out, "Invalid vector transformation"); if constexpr ((in == Idx::T && out == Idx::Sph) || From 54b9a2c71c2d789ef3e370f703b75580b3ec5412 Mon Sep 17 00:00:00 2001 From: haykh Date: Fri, 6 Jun 2025 21:03:29 -0500 Subject: [PATCH 03/13] LICENSE upd --- LICENSE | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/LICENSE b/LICENSE index 554820782..639d2b41f 100644 --- a/LICENSE +++ b/LICENSE @@ -1,19 +1,8 @@ BSD 3-Clause License -Copyright (c) 2024, Princeton University (PU), and -Princeton Plasma Physics Lab (PPPL). +Copyright (c) 2021-present, Entity development team. All rights reserved. -Core developers (alphabetical order): -* Alexander Chernoglazov -* Benjamin Crinquand -* Alisa Galishnikova -* Hayk Hakobyan -* Jens Mahlmann -* Sasha Philippov -* Arno Vanthieghem -* Muni Zhou - Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: From b42022a87f7ed828763e5f59d096f057a69dfe03 Mon Sep 17 00:00:00 2001 From: haykh Date: Fri, 6 Jun 2025 21:03:50 -0500 Subject: [PATCH 04/13] minor nix changes --- dev/nix/adios2.nix | 16 ++++++++-------- dev/nix/kokkos.nix | 14 +++++++------- dev/nix/shell.nix | 3 ++- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/dev/nix/adios2.nix b/dev/nix/adios2.nix index 7218f894f..0418b71cd 100644 --- a/dev/nix/adios2.nix +++ b/dev/nix/adios2.nix @@ -19,10 +19,12 @@ let BUILD_TESTING = "OFF"; ADIOS2_BUILD_EXAMPLES = "OFF"; ADIOS2_USE_MPI = if mpi then "ON" else "OFF"; + ADIOS2_HAVE_HDF5_VOL = if mpi then "ON" else "OFF"; CMAKE_BUILD_TYPE = "Release"; - } // (if !mpi then { ADIOS2_HAVE_HDF5_VOL = "OFF"; } else { }); + }; + stdenv = pkgs.gcc13Stdenv; in -pkgs.stdenv.mkDerivation { +stdenv.mkDerivation { pname = "${name}${if hdf5 then "-hdf5" else ""}${if mpi then "-mpi" else ""}"; version = "${version}"; src = pkgs.fetchgit { @@ -36,12 +38,10 @@ pkgs.stdenv.mkDerivation { perl ]; - propagatedBuildInputs = - [ - pkgs.gcc13 - ] - ++ (if hdf5 then (if mpi then [ pkgs.hdf5-mpi ] else [ pkgs.hdf5 ]) else [ ]) - ++ (if mpi then [ pkgs.openmpi ] else [ ]); + propagatedBuildInputs = [ + pkgs.gcc13 + ] ++ (if hdf5 then (if mpi then [ pkgs.hdf5-mpi ] else [ pkgs.hdf5-cpp ]) else [ ]); + # ++ (if mpi then [ pkgs.openmpi ] else [ ]); configurePhase = '' cmake -B build $src ${ diff --git a/dev/nix/kokkos.nix b/dev/nix/kokkos.nix index c83a489e5..8dfe1f380 100644 --- a/dev/nix/kokkos.nix +++ b/dev/nix/kokkos.nix @@ -20,11 +20,18 @@ let "CUDA" = with pkgs.cudaPackages; [ cudatoolkit cuda_cudart + pkgs.gcc13 ]; "NONE" = [ pkgs.gcc13 ]; }; + getArch = + _: + if gpu != "NONE" && arch == "NATIVE" then + throw "Please specify an architecture when the GPU support is enabled. Available architectures: https://kokkos.org/kokkos-core-wiki/keywords.html#architectures" + else + arch; cmakeExtraFlags = { "HIP" = [ "-D Kokkos_ENABLE_HIP=ON" @@ -39,13 +46,6 @@ let ]; "NONE" = [ ]; }; - getArch = - _: - if gpu != "NONE" && arch == "NATIVE" then - throw "Please specify an architecture when the GPU support is enabled. Available architectures: https://kokkos.org/kokkos-core-wiki/keywords.html#architectures" - else - arch; - in pkgs.stdenv.mkDerivation rec { pname = "${name}"; diff --git a/dev/nix/shell.nix b/dev/nix/shell.nix index be4b87bcb..ae5451ec8 100644 --- a/dev/nix/shell.nix +++ b/dev/nix/shell.nix @@ -42,7 +42,8 @@ pkgs.mkShell { zlib cmake - clang-tools + llvmPackages_18.clang-tools + libgcc adios2Pkg kokkosPkg From 50b6a6ed68816d1d6792e93ddee5bff48f0abf70 Mon Sep 17 00:00:00 2001 From: haykh Date: Fri, 6 Jun 2025 21:06:36 -0500 Subject: [PATCH 05/13] cosmetic changes --- src/checkpoint/reader.cpp | 86 ++++++++---------------- src/checkpoint/writer.cpp | 128 +++++++++++++++++------------------- src/output/writer.cpp | 133 ++++++++++++-------------------------- 3 files changed, 128 insertions(+), 219 deletions(-) diff --git a/src/checkpoint/reader.cpp b/src/checkpoint/reader.cpp index 24aa0f079..d973b0ddd 100644 --- a/src/checkpoint/reader.cpp +++ b/src/checkpoint/reader.cpp @@ -110,7 +110,7 @@ namespace checkpoint { void ReadParticlePayloads(adios2::IO& io, adios2::Engine& reader, - spidx_t s, + spidx_t s, array_t& array, std::size_t nplds, npart_t count, @@ -130,64 +130,32 @@ namespace checkpoint { } } - template void ReadFields(adios2::IO&, - adios2::Engine&, - const std::string&, - const adios2::Box&, - ndfield_t&); - template void ReadFields(adios2::IO&, - adios2::Engine&, - const std::string&, - const adios2::Box&, - ndfield_t&); - template void ReadFields(adios2::IO&, - adios2::Engine&, - const std::string&, - const adios2::Box&, - ndfield_t&); - template void ReadFields(adios2::IO&, - adios2::Engine&, - const std::string&, - const adios2::Box&, - ndfield_t&); - template void ReadFields(adios2::IO&, - adios2::Engine&, - const std::string&, - const adios2::Box&, - ndfield_t&); - template void ReadFields(adios2::IO&, - adios2::Engine&, - const std::string&, - const adios2::Box&, - ndfield_t&); +#define CHECKPOINT_FIELDS(D, N) \ + template void ReadFields(adios2::IO&, \ + adios2::Engine&, \ + const std::string&, \ + const adios2::Box&, \ + ndfield_t&); + CHECKPOINT_FIELDS(Dim::_1D, 3) + CHECKPOINT_FIELDS(Dim::_2D, 3) + CHECKPOINT_FIELDS(Dim::_3D, 3) + CHECKPOINT_FIELDS(Dim::_1D, 6) + CHECKPOINT_FIELDS(Dim::_2D, 6) + CHECKPOINT_FIELDS(Dim::_3D, 6) +#undef CHECKPOINT_FIELDS - template void ReadParticleData(adios2::IO&, - adios2::Engine&, - const std::string&, - spidx_t, - array_t&, - npart_t, - npart_t); - template void ReadParticleData(adios2::IO&, - adios2::Engine&, - const std::string&, - spidx_t, - array_t&, - npart_t, - npart_t); - template void ReadParticleData(adios2::IO&, - adios2::Engine&, - const std::string&, - spidx_t, - array_t&, - npart_t, - npart_t); - template void ReadParticleData(adios2::IO&, - adios2::Engine&, - const std::string&, - spidx_t, - array_t&, - npart_t, - npart_t); +#define CHECKPOINT_PARTICLE_DATA(T) \ + template void ReadParticleData(adios2::IO&, \ + adios2::Engine&, \ + const std::string&, \ + spidx_t, \ + array_t&, \ + npart_t, \ + npart_t); + CHECKPOINT_PARTICLE_DATA(int) + CHECKPOINT_PARTICLE_DATA(float) + CHECKPOINT_PARTICLE_DATA(double) + CHECKPOINT_PARTICLE_DATA(short) +#undef CHECKPOINT_PARTICLE_DATA } // namespace checkpoint diff --git a/src/checkpoint/writer.cpp b/src/checkpoint/writer.cpp index b8571c246..b766ddfbd 100644 --- a/src/checkpoint/writer.cpp +++ b/src/checkpoint/writer.cpp @@ -15,21 +15,23 @@ #include #include #include -#include #include namespace checkpoint { - void Writer::init(adios2::ADIOS* ptr_adios, - timestep_t interval, - simtime_t interval_time, - int keep) { - m_keep = keep; - m_enabled = keep != 0; + void Writer::init(adios2::ADIOS* ptr_adios, + const path_t& checkpoint_root, + timestep_t interval, + simtime_t interval_time, + int keep, + const std::string& walltime) { + m_keep = keep; + m_checkpoint_root = checkpoint_root; + m_enabled = keep != 0; if (not m_enabled) { return; } - m_tracker.init("checkpoint", interval, interval_time); + m_tracker.init("checkpoint", interval, interval_time, walltime); p_adios = ptr_adios; raise::ErrorIf(p_adios == nullptr, "ADIOS pointer is null", HERE); @@ -40,12 +42,13 @@ namespace checkpoint { m_io.DefineVariable("Time"); m_io.DefineAttribute("NGhosts", ntt::N_GHOSTS); - CallOnce([]() { - const std::filesystem::path save_path { "checkpoints" }; - if (!std::filesystem::exists(save_path)) { - std::filesystem::create_directory(save_path); - } - }); + CallOnce( + [](auto&& checkpoint_root) { + if (!std::filesystem::exists(checkpoint_root)) { + std::filesystem::create_directory(checkpoint_root); + } + }, + m_checkpoint_root); } void Writer::defineFieldVariables(const ntt::SimEngine& S, @@ -147,13 +150,14 @@ namespace checkpoint { } m_writing_mode = true; try { - auto fname = fmt::format("checkpoints/step-%08lu.bp", step); - m_writer = m_io.Open(fname, adios2::Mode::Write); - auto meta_fname = fmt::format("checkpoints/meta-%08lu.toml", step); - m_written.push_back({ fname, meta_fname }); + const auto filename = m_checkpoint_root / fmt::format("step-%08lu.bp", step); + const auto metafilename = m_checkpoint_root / + fmt::format("meta-%08lu.toml", step); + m_writer = m_io.Open(filename, adios2::Mode::Write); + m_written.push_back({ filename, metafilename }); logger::Checkpoint(fmt::format("Writing checkpoint to %s and %s", - fname.c_str(), - meta_fname.c_str()), + filename.c_str(), + metafilename.c_str()), HERE); } catch (std::exception& e) { raise::Fatal(e.what(), HERE); @@ -261,54 +265,38 @@ namespace checkpoint { m_writer.Put(var, data_sub.data(), adios2::Mode::Sync); } - template void Writer::savePerDomainVariable(const std::string&, - std::size_t, - std::size_t, - int); - template void Writer::savePerDomainVariable(const std::string&, - std::size_t, - std::size_t, - float); - template void Writer::savePerDomainVariable(const std::string&, - std::size_t, - std::size_t, - double); - template void Writer::savePerDomainVariable(const std::string&, - std::size_t, - std::size_t, - npart_t); - - template void Writer::saveField(const std::string&, - const ndfield_t&); - template void Writer::saveField(const std::string&, - const ndfield_t&); - template void Writer::saveField(const std::string&, - const ndfield_t&); - template void Writer::saveField(const std::string&, - const ndfield_t&); - template void Writer::saveField(const std::string&, - const ndfield_t&); - template void Writer::saveField(const std::string&, - const ndfield_t&); - - template void Writer::saveParticleQuantity(const std::string&, - npart_t, - npart_t, - npart_t, - const array_t&); - template void Writer::saveParticleQuantity(const std::string&, - npart_t, - npart_t, - npart_t, - const array_t&); - template void Writer::saveParticleQuantity(const std::string&, - npart_t, - npart_t, - npart_t, - const array_t&); - template void Writer::saveParticleQuantity(const std::string&, - npart_t, - npart_t, - npart_t, - const array_t&); +#define CHECKPOINT_PERDOMAIN_VARIABLE(T) \ + template void Writer::savePerDomainVariable(const std::string&, \ + std::size_t, \ + std::size_t, \ + T); + CHECKPOINT_PERDOMAIN_VARIABLE(int) + CHECKPOINT_PERDOMAIN_VARIABLE(float) + CHECKPOINT_PERDOMAIN_VARIABLE(double) + CHECKPOINT_PERDOMAIN_VARIABLE(npart_t) +#undef CHECKPOINT_PERDOMAIN_VARIABLE + +#define CHECKPOINT_FIELD(D, N) \ + template void Writer::saveField(const std::string&, \ + const ndfield_t&); + CHECKPOINT_FIELD(Dim::_1D, 3) + CHECKPOINT_FIELD(Dim::_1D, 6) + CHECKPOINT_FIELD(Dim::_2D, 3) + CHECKPOINT_FIELD(Dim::_2D, 6) + CHECKPOINT_FIELD(Dim::_3D, 3) + CHECKPOINT_FIELD(Dim::_3D, 6) +#undef CHECKPOINT_FIELD + +#define CHECKPOINT_PARTICLE_QUANTITY(T) \ + template void Writer::saveParticleQuantity(const std::string&, \ + npart_t, \ + npart_t, \ + npart_t, \ + const array_t&); + CHECKPOINT_PARTICLE_QUANTITY(int) + CHECKPOINT_PARTICLE_QUANTITY(float) + CHECKPOINT_PARTICLE_QUANTITY(double) + CHECKPOINT_PARTICLE_QUANTITY(short) +#undef CHECKPOINT_PARTICLE_QUANTITY + } // namespace checkpoint diff --git a/src/output/writer.cpp b/src/output/writer.cpp index a259ba889..b073295e3 100644 --- a/src/output/writer.cpp +++ b/src/output/writer.cpp @@ -39,7 +39,7 @@ namespace out { m_io.DefineVariable("Step"); m_io.DefineVariable("Time"); - m_fname = title; + m_fname = path_t(title); } void Writer::addTracker(const std::string& type, @@ -48,8 +48,9 @@ namespace out { m_trackers.insert({ type, tools::Tracker(type, interval, interval_time) }); } - auto Writer::shouldWrite(const std::string& type, timestep_t step, simtime_t time) - -> bool { + auto Writer::shouldWrite(const std::string& type, + timestep_t step, + simtime_t time) -> bool { if (m_trackers.find(type) != m_trackers.end()) { return m_trackers.at(type).shouldWrite(step, time); } else { @@ -427,53 +428,53 @@ namespace out { if (m_active_mode != WriteMode::None) { raise::Fatal("Already writing", HERE); } - m_active_mode = write_mode; try { - std::string filename; + path_t filename; + const std::string ext = (m_engine == "hdf5") ? "h5" : "bp"; if (m_separate_files) { std::string mode_str; - if (m_active_mode == WriteMode::Fields) { + if (write_mode == WriteMode::Fields) { mode_str = "fields"; - } else if (m_active_mode == WriteMode::Particles) { + } else if (write_mode == WriteMode::Particles) { mode_str = "particles"; - } else if (m_active_mode == WriteMode::Spectra) { + } else if (write_mode == WriteMode::Spectra) { mode_str = "spectra"; } else { raise::Fatal("Unknown write mode", HERE); } CallOnce( - [](auto& main_path, auto& mode_path) { - const std::filesystem::path main { main_path }; - const std::filesystem::path mode { mode_path }; + [](auto&& main_path, auto&& mode_path) { + const path_t main { main_path }; + const path_t mode { mode_path }; if (!std::filesystem::exists(main_path)) { std::filesystem::create_directory(main_path); } - if (!std::filesystem::exists(main / mode)) { - std::filesystem::create_directory(main / mode); + if (!std::filesystem::exists(main_path / mode_path)) { + std::filesystem::create_directory(main_path / mode_path); } }, m_fname, mode_str); - filename = fmt::format("%s/%s/%s.%08lu.%s", - m_fname.c_str(), - mode_str.c_str(), - mode_str.c_str(), - tstep, - ext.c_str()); - m_mode = adios2::Mode::Write; +#if defined(MPI_ENABLED) + MPI_Barrier(MPI_COMM_WORLD); +#endif + filename = m_fname / path_t(mode_str) / + fmt::format("%s.%08lu.%s", mode_str.c_str(), tstep, ext.c_str()); + m_mode = adios2::Mode::Write; } else { filename = fmt::format("%s.%s", m_fname.c_str(), ext.c_str()); m_mode = std::filesystem::exists(filename) ? adios2::Mode::Append : adios2::Mode::Write; } m_writer = m_io.Open(filename, m_mode); + m_writer.BeginStep(); + m_writer.Put(m_io.InquireVariable("Step"), &tstep); + m_writer.Put(m_io.InquireVariable("Time"), &time); + m_active_mode = write_mode; } catch (std::exception& e) { raise::Fatal(e.what(), HERE); } - m_writer.BeginStep(); - m_writer.Put(m_io.InquireVariable("Step"), &tstep); - m_writer.Put(m_io.InquireVariable("Time"), &time); } void Writer::endWriting(WriteModeTags write_mode) { @@ -490,72 +491,24 @@ namespace out { m_writer.Close(); } - template void Writer::writeField(const std::vector&, - const ndfield_t&, - const std::vector&); - template void Writer::writeField(const std::vector&, - const ndfield_t&, - const std::vector&); - template void Writer::writeField(const std::vector&, - const ndfield_t&, - const std::vector&); - template void Writer::writeField(const std::vector&, - const ndfield_t&, - const std::vector&); - template void Writer::writeField(const std::vector&, - const ndfield_t&, - const std::vector&); - template void Writer::writeField(const std::vector&, - const ndfield_t&, - const std::vector&); - - template void WriteField(adios2::IO&, - adios2::Engine&, - const std::string&, - const ndfield_t&, - std::size_t, - std::vector, - std::vector, - bool); - template void WriteField(adios2::IO&, - adios2::Engine&, - const std::string&, - const ndfield_t&, - std::size_t, - std::vector, - std::vector, - bool); - template void WriteField(adios2::IO&, - adios2::Engine&, - const std::string&, - const ndfield_t&, - std::size_t, - std::vector, - std::vector, - bool); - template void WriteField(adios2::IO&, - adios2::Engine&, - const std::string&, - const ndfield_t&, - std::size_t, - std::vector, - std::vector, - bool); - template void WriteField(adios2::IO&, - adios2::Engine&, - const std::string&, - const ndfield_t&, - std::size_t, - std::vector, - std::vector, - bool); - template void WriteField(adios2::IO&, - adios2::Engine&, - const std::string&, - const ndfield_t&, - std::size_t, - std::vector, - std::vector, - bool); +#define WRITE_FIELD(D, N) \ + template void Writer::writeField(const std::vector&, \ + const ndfield_t&, \ + const std::vector&); \ + template void WriteField(adios2::IO&, \ + adios2::Engine&, \ + const std::string&, \ + const ndfield_t&, \ + std::size_t, \ + std::vector, \ + std::vector, \ + bool); + WRITE_FIELD(Dim::_1D, 3) + WRITE_FIELD(Dim::_1D, 6) + WRITE_FIELD(Dim::_2D, 3) + WRITE_FIELD(Dim::_2D, 6) + WRITE_FIELD(Dim::_3D, 3) + WRITE_FIELD(Dim::_3D, 6) +#undef WRITE_FIELD } // namespace out From 0599bcc71a11871e1249542bf71eee8d4e3fdd58 Mon Sep 17 00:00:00 2001 From: haykh Date: Fri, 6 Jun 2025 21:12:04 -0500 Subject: [PATCH 06/13] minor warnings + bugs --- src/archetypes/energy_dist.h | 2 +- src/checkpoint/tests/checkpoint-nompi.cpp | 35 ++++++++++++----------- src/engines/engine_printer.cpp | 10 +++---- src/framework/containers/particles.cpp | 5 ---- src/framework/simulation.h | 3 -- src/global/global.h | 4 +++ src/kernels/fields_bcs.hpp | 2 +- src/output/writer.h | 4 +-- 8 files changed, 32 insertions(+), 33 deletions(-) diff --git a/src/archetypes/energy_dist.h b/src/archetypes/energy_dist.h index e9ef8b0b7..901dbce3a 100644 --- a/src/archetypes/energy_dist.h +++ b/src/archetypes/energy_dist.h @@ -392,7 +392,7 @@ namespace arch { Inline void operator()(const coord_t& x_Code, vec_t& v, - spidx_t sp = 0) const override { + spidx_t = 0) const override { if (cmp::AlmostZero(temperature)) { v[0] = ZERO; v[1] = ZERO; diff --git a/src/checkpoint/tests/checkpoint-nompi.cpp b/src/checkpoint/tests/checkpoint-nompi.cpp index be5e97e24..85871183d 100644 --- a/src/checkpoint/tests/checkpoint-nompi.cpp +++ b/src/checkpoint/tests/checkpoint-nompi.cpp @@ -12,14 +12,13 @@ #include #include -#include using namespace ntt; using namespace checkpoint; void cleanup() { namespace fs = std::filesystem; - fs::path temp_path { "checkpoints" }; + fs::path temp_path { "chck" }; fs::remove_all(temp_path); } @@ -58,18 +57,21 @@ auto main(int argc, char* argv[]) -> int { CreateRangePolicy({ i1min, i2min, i3min }, { i1max, i2max, i3max }), Lambda(index_t i1, index_t i2, index_t i3) { - field1(i1, i2, i3, 0) = i1 + i2 + i3; - field1(i1, i2, i3, 1) = i1 * i2 / i3; - field1(i1, i2, i3, 2) = i1 / i2 * i3; - field1(i1, i2, i3, 3) = i1 + i2 - i3; - field1(i1, i2, i3, 4) = i1 * i2 + i3; - field1(i1, i2, i3, 5) = i1 / i2 - i3; - field2(i1, i2, i3, 0) = -(i1 + i2 + i3); - field2(i1, i2, i3, 1) = -(i1 * i2 / i3); - field2(i1, i2, i3, 2) = -(i1 / i2 * i3); - field2(i1, i2, i3, 3) = -(i1 + i2 - i3); - field2(i1, i2, i3, 4) = -(i1 * i2 + i3); - field2(i1, i2, i3, 5) = -(i1 / i2 - i3); + const auto i1_ = static_cast(i1); + const auto i2_ = static_cast(i2); + const auto i3_ = static_cast(i3); + field1(i1, i2, i3, 0) = i1_ + i2_ + i3; + field1(i1, i2, i3, 1) = i1_ * i2_ / i3; + field1(i1, i2, i3, 2) = i1_ / i2_ * i3; + field1(i1, i2, i3, 3) = i1_ + i2_ - i3; + field1(i1, i2, i3, 4) = i1_ * i2_ + i3; + field1(i1, i2, i3, 5) = i1_ / i2_ - i3; + field2(i1, i2, i3, 0) = -(i1_ + i2_ + i3); + field2(i1, i2, i3, 1) = -(i1_ * i2_ / i3); + field2(i1, i2, i3, 2) = -(i1_ / i2_ * i3); + field2(i1, i2, i3, 3) = -(i1_ + i2_ - i3); + field2(i1, i2, i3, 4) = -(i1_ * i2_ + i3); + field2(i1, i2, i3, 5) = -(i1_ / i2_ - i3); }); Kokkos::parallel_for( "fillPrtl1", @@ -88,10 +90,11 @@ auto main(int argc, char* argv[]) -> int { } adios2::ADIOS adios; + const path_t checkpoint_path { "chck" }; { // write checkpoint - Writer writer; + Writer writer { checkpoint_path }; writer.init(&adios, 0, 0.0, 1); writer.defineFieldVariables(SimEngine::GRPIC, @@ -127,7 +130,7 @@ auto main(int argc, char* argv[]) -> int { array_t u2_read { "u_2", npart2 }; adios2::IO io = adios.DeclareIO("checkpointRead"); - adios2::Engine reader = io.Open("checkpoints/step-00000000.bp", + adios2::Engine reader = io.Open(checkpoint_path / "step-00000000.bp", adios2::Mode::Read); reader.BeginStep(); diff --git a/src/engines/engine_printer.cpp b/src/engines/engine_printer.cpp index 9d1f74a74..20f5e81ba 100644 --- a/src/engines/engine_printer.cpp +++ b/src/engines/engine_printer.cpp @@ -106,13 +106,13 @@ namespace ntt { color::RESET); } - auto bytes_to_human_readable(std::size_t bytes) - -> std::pair { + auto bytes_to_human_readable( + std::size_t bytes) -> std::pair { const std::vector units { "B", "KB", "MB", "GB", "TB" }; idx_t unit_idx = 0; - auto size = static_cast(bytes); - while ((size >= 1024) && (unit_idx < units.size() - 1)) { - size /= 1024; + auto size = static_cast(bytes); + while ((size >= 1024.0) and (unit_idx < units.size() - 1)) { + size /= 1024.0; ++unit_idx; } return { size, units[unit_idx] }; diff --git a/src/framework/containers/particles.cpp b/src/framework/containers/particles.cpp index b69e48de4..d2db9c491 100644 --- a/src/framework/containers/particles.cpp +++ b/src/framework/containers/particles.cpp @@ -4,8 +4,6 @@ #include "global.h" #include "arch/kokkos_aliases.h" -#include "utils/numeric.h" -#include "utils/sorting.h" #include "framework/containers/species.h" @@ -13,9 +11,6 @@ #include #include -#include -#include -#include #include #include diff --git a/src/framework/simulation.h b/src/framework/simulation.h index 3cc664995..33750030f 100644 --- a/src/framework/simulation.h +++ b/src/framework/simulation.h @@ -22,9 +22,6 @@ #include "framework/parameters.h" -#include -#include - namespace ntt { class Simulation { diff --git a/src/global/global.h b/src/global/global.h index 1f7987929..04552db7f 100644 --- a/src/global/global.h +++ b/src/global/global.h @@ -93,6 +93,7 @@ #define GLOBAL_GLOBAL_H #include +#include #include #include #include @@ -357,6 +358,9 @@ using idx_t = unsigned short; using spidx_t = unsigned short; using dim_t = unsigned short; +// utility +using path_t = std::filesystem::path; + using range_tuple_t = std::pair; template diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 0a5dc6168..a520616bf 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -491,8 +491,8 @@ namespace kernel::bc { "Invalid component index"); ndfield_t Fld; - const BCTags tags; const std::size_t i_edge; + const BCTags tags; ConductorBoundaries_kernel(ndfield_t Fld, std::size_t i_edge, BCTags tags) : Fld { Fld } diff --git a/src/output/writer.h b/src/output/writer.h index 2e8d20de5..66ee4e597 100644 --- a/src/output/writer.h +++ b/src/output/writer.h @@ -57,7 +57,7 @@ namespace out { bool m_flds_ghosts; std::string m_engine; - std::string m_fname; + path_t m_fname; std::map m_trackers; @@ -116,7 +116,7 @@ namespace out { void endWriting(WriteModeTags); /* getters -------------------------------------------------------------- */ - auto fname() const -> const std::string& { + auto fname() const -> const path_t& { return m_fname; } From ca2e1a3253124ffccb408d3ccd77aa8dcbd15264 Mon Sep 17 00:00:00 2001 From: haykh Date: Fri, 6 Jun 2025 21:12:54 -0500 Subject: [PATCH 07/13] checkpoint: read/write path + walltime --- input.example.toml | 9 +++++ src/checkpoint/writer.h | 12 ++++-- src/engines/engine.hpp | 5 --- src/engines/engine_init.cpp | 16 +------- src/engines/engine_run.cpp | 9 ----- src/framework/domain/checkpoint.cpp | 18 ++++++--- src/framework/domain/stats.cpp | 25 ++++++------- src/framework/parameters.cpp | 17 +++++++-- src/framework/parameters.h | 2 + src/framework/simulation.cpp | 28 +++++++++----- src/global/utils/tools.h | 57 ++++++++++++++++++++++------- 11 files changed, 120 insertions(+), 78 deletions(-) diff --git a/input.example.toml b/input.example.toml index bef6ab237..aa532273b 100644 --- a/input.example.toml +++ b/input.example.toml @@ -483,6 +483,15 @@ # @note: Empty string means disables this (still writes checkpoints on interval) # @note: This does not exit the simulation walltime = "" + # Parent directory to write checkpoints to + # @type: string + # @default: "checkpoints" + # @note: The directory is created if it does not exist + write_path = "" + # Parent directory to use when resuming from a checkpoint + # @type: string + # @default: inherits from `write_path` + read_path = "" # @inferred: # - is_resuming diff --git a/src/checkpoint/writer.h b/src/checkpoint/writer.h index 91c9d7a41..3aa30d04d 100644 --- a/src/checkpoint/writer.h +++ b/src/checkpoint/writer.h @@ -39,15 +39,21 @@ namespace checkpoint { std::vector> m_written; - int m_keep; - bool m_enabled; + int m_keep; + bool m_enabled; + path_t m_checkpoint_root; public: Writer() {} ~Writer() = default; - void init(adios2::ADIOS*, timestep_t, simtime_t, int); + void init(adios2::ADIOS*, + const path_t&, + timestep_t, + simtime_t, + int, + const std::string&); auto shouldSave(timestep_t, simtime_t) -> bool; diff --git a/src/engines/engine.hpp b/src/engines/engine.hpp index 261b1d6ea..17103f1de 100644 --- a/src/engines/engine.hpp +++ b/src/engines/engine.hpp @@ -43,7 +43,6 @@ #include #endif // MPI_ENABLED -#include #include #include @@ -61,9 +60,6 @@ namespace ntt { #else adios2::ADIOS m_adios; #endif - const timestamp_t start_walltime { std::chrono::system_clock::now() }; - timestamp_t end_walltime {}; - bool walltime_checkpoint_pending { false }; #endif SimulationParams m_params; @@ -114,7 +110,6 @@ namespace ntt { , time { start_time } , step { start_step } { raise::ErrorIf(not pgen_is_ok, "Problem generator is not compatible with the picked engine/metric/dimension", HERE); - print_report(); } ~Engine() = default; diff --git a/src/engines/engine_init.cpp b/src/engines/engine_init.cpp index 0e99451e9..8953b3da8 100644 --- a/src/engines/engine_init.cpp +++ b/src/engines/engine_init.cpp @@ -27,20 +27,6 @@ namespace ntt { #if defined(OUTPUT_ENABLED) m_metadomain.InitWriter(&m_adios, m_params, is_resuming); m_metadomain.InitCheckpointWriter(&m_adios, m_params); - - const auto checkpoint_walltime = m_params.template get( - "checkpoint.walltime"); - if (not checkpoint_walltime.empty()) { - walltime_checkpoint_pending = true; - raise::ErrorIf(checkpoint_walltime.size() != 8, - "invalid checkpoint walltime format, expected HH:MM:SS", - HERE); - end_walltime = - start_walltime + - std::chrono::hours(std::stoi(checkpoint_walltime.substr(0, 2))) + - std::chrono::minutes(std::stoi(checkpoint_walltime.substr(3, 2))) + - std::chrono::seconds(std::stoi(checkpoint_walltime.substr(6, 2))); - } #endif logger::Checkpoint("Initializing Engine", HERE); if (not is_resuming) { @@ -82,6 +68,7 @@ namespace ntt { #endif } } + print_report(); } template class Engine>; @@ -92,4 +79,5 @@ namespace ntt { template class Engine>; template class Engine>; template class Engine>; + } // namespace ntt diff --git a/src/engines/engine_run.cpp b/src/engines/engine_run.cpp index 80e9cb62d..65c87e939 100644 --- a/src/engines/engine_run.cpp +++ b/src/engines/engine_run.cpp @@ -115,15 +115,6 @@ namespace ntt { step - 1, time, time - dt); - if ((not print_checkpoint) and (walltime_checkpoint_pending) and - (std::chrono::system_clock::now() >= end_walltime)) { - print_checkpoint = m_metadomain.WriteCheckpoint(m_params, - step, - step - 1, - time, - time - dt); - walltime_checkpoint_pending = false; - } timers.stop("Checkpoint"); #endif diff --git a/src/framework/domain/checkpoint.cpp b/src/framework/domain/checkpoint.cpp index a39f18e66..e0e34f993 100644 --- a/src/framework/domain/checkpoint.cpp +++ b/src/framework/domain/checkpoint.cpp @@ -47,11 +47,16 @@ namespace ntt { nplds.push_back(local_domain->species[s].npld()); } + const path_t checkpoint_root = params.template get( + "checkpoint.write_path"); + g_checkpoint_writer.init( ptr_adios, + checkpoint_root, params.template get("checkpoint.interval"), params.template get("checkpoint.interval_time"), - params.template get("checkpoint.keep")); + params.template get("checkpoint.keep"), + params.template get("checkpoint.walltime")); if (g_checkpoint_writer.enabled()) { g_checkpoint_writer.defineFieldVariables(S, glob_shape_with_ghosts, @@ -261,9 +266,12 @@ namespace ntt { void Metadomain::ContinueFromCheckpoint(adios2::ADIOS* ptr_adios, const SimulationParams& params) { raise::ErrorIf(ptr_adios == nullptr, "adios == nullptr", HERE); - auto fname = fmt::format( - "checkpoints/step-%08lu.bp", - params.template get("checkpoint.start_step")); + const path_t checkpoint_root = params.template get( + "checkpoint.read_path"); + const auto fname = checkpoint_root / + fmt::format("step-%08lu.bp", + params.template get( + "checkpoint.start_step")); logger::Checkpoint(fmt::format("Reading checkpoint from %s", fname.c_str()), HERE); @@ -484,7 +492,6 @@ namespace ntt { simtime_t) -> bool; \ template void Metadomain::ContinueFromCheckpoint(adios2::ADIOS*, \ const SimulationParams&); - METADOMAIN_CHECKPOINTS(SimEngine::SRPIC, metric::Minkowski) METADOMAIN_CHECKPOINTS(SimEngine::SRPIC, metric::Minkowski) METADOMAIN_CHECKPOINTS(SimEngine::SRPIC, metric::Minkowski) @@ -493,7 +500,6 @@ namespace ntt { METADOMAIN_CHECKPOINTS(SimEngine::GRPIC, metric::KerrSchild) METADOMAIN_CHECKPOINTS(SimEngine::GRPIC, metric::QKerrSchild) METADOMAIN_CHECKPOINTS(SimEngine::GRPIC, metric::KerrSchild0) - #undef METADOMAIN_CHECKPOINTS } // namespace ntt diff --git a/src/framework/domain/stats.cpp b/src/framework/domain/stats.cpp index a252e975d..6c5bb0ffa 100644 --- a/src/framework/domain/stats.cpp +++ b/src/framework/domain/stats.cpp @@ -89,7 +89,6 @@ namespace ntt { } // some parameters const auto use_weights = params.template get("particles.use_weights"); - const auto inv_n0 = ONE / params.template get("scales.n0"); real_t buffer = static_cast(0); for (const auto& sp : specs) { @@ -186,8 +185,8 @@ namespace ntt { timestep_t finished_step, simtime_t current_time, simtime_t finished_time, - std::function&)> CustomStat) - -> bool { + std::function&)> + CustomStat) -> bool { if (not(params.template get("output.stats.enable") and g_stats_writer.shouldWrite(finished_step, finished_time))) { return false; @@ -272,16 +271,16 @@ namespace ntt { return true; } -#define METADOMAIN_STATS(S, M) \ - template void Metadomain::InitStatsWriter(const SimulationParams&, bool); \ - template auto Metadomain::WriteStats( \ - const SimulationParams&, \ - timestep_t, \ - timestep_t, \ - simtime_t, \ - simtime_t, \ - std::function&)>) \ - -> bool; +#define METADOMAIN_STATS(S, M) \ + template void Metadomain::InitStatsWriter(const SimulationParams&, bool); \ + template auto Metadomain::WriteStats( \ + const SimulationParams&, \ + timestep_t, \ + timestep_t, \ + simtime_t, \ + simtime_t, \ + std::function< \ + real_t(const std::string&, timestep_t, simtime_t, const Domain&)>) -> bool; METADOMAIN_STATS(SimEngine::SRPIC, metric::Minkowski) METADOMAIN_STATS(SimEngine::SRPIC, metric::Minkowski) diff --git a/src/framework/parameters.cpp b/src/framework/parameters.cpp index ac0a5e0dd..586a01941 100644 --- a/src/framework/parameters.cpp +++ b/src/framework/parameters.cpp @@ -31,10 +31,10 @@ namespace ntt { template - auto get_dx0_V0(const std::vector& resolution, - const boundaries_t& extent, - const std::map& params) - -> std::pair { + auto get_dx0_V0( + const std::vector& resolution, + const boundaries_t& extent, + const std::map& params) -> std::pair { const auto metric = M(resolution, extent, params); const auto dx0 = metric.dxMin(); coord_t x_corner { ZERO }; @@ -612,6 +612,15 @@ namespace ntt { set("checkpoint.walltime", toml::find_or(toml_data, "checkpoint", "walltime", std::string(""))); + const auto checkpoint_write_path = toml::find_or( + toml_data, + "checkpoint", + "write_path", + std::string("checkpoints")); + set("checkpoint.write_path", checkpoint_write_path); + set("checkpoint.read_path", + toml::find_or(toml_data, "checkpoint", "read_path", checkpoint_write_path)); + /* [diagnostics] -------------------------------------------------------- */ set("diagnostics.interval", toml::find_or(toml_data, "diagnostics", "interval", defaults::diag::interval)); diff --git a/src/framework/parameters.h b/src/framework/parameters.h index 83192f447..723e860db 100644 --- a/src/framework/parameters.h +++ b/src/framework/parameters.h @@ -26,6 +26,8 @@ namespace ntt { SimulationParams() {} + SimulationParams(const SimulationParams&) = default; + SimulationParams& operator=(const SimulationParams& other) { vars = std::move(other.vars); promises = std::move(other.promises); diff --git a/src/framework/simulation.cpp b/src/framework/simulation.cpp index 6865140c9..b130d10a7 100644 --- a/src/framework/simulation.cpp +++ b/src/framework/simulation.cpp @@ -11,8 +11,6 @@ #include "utils/plog.h" #include "utils/toml.h" -#include "framework/parameters.h" - #include #include @@ -55,13 +53,24 @@ namespace ntt { m_params.setRawData(raw_params); timestep_t checkpoint_step = 0; + if (is_resuming) { logger::Checkpoint("Reading params from a checkpoint", HERE); - if (not std::filesystem::exists("checkpoints")) { + const auto checkpoint_write_path = toml::find_or( + raw_params, + "checkpoint", + "write_path", + "checkpoints"); + const path_t checkpoint_read_path = toml::find_or( + raw_params, + "checkpoint", + "read_path", + checkpoint_write_path); + if (not std::filesystem::exists(checkpoint_read_path)) { raise::Fatal("No checkpoints found", HERE); } for (const auto& entry : - std::filesystem::directory_iterator("checkpoints")) { + std::filesystem::directory_iterator(checkpoint_read_path)) { const auto fname = entry.path().filename().string(); if (fname.find("step-") == 0) { const timestep_t step = std::stoi(fname.substr(5, fname.size() - 5 - 3)); @@ -70,17 +79,16 @@ namespace ntt { } } } - std::string checkpoint_inputfname = fmt::format( - "checkpoints/meta-%08lu.toml", - checkpoint_step); - if (not std::filesystem::exists(checkpoint_inputfname)) { + path_t checkpoint_metafname = checkpoint_read_path / + fmt::format("meta-%08lu.toml", checkpoint_step); + if (not std::filesystem::exists(checkpoint_metafname)) { raise::Fatal( fmt::format("metainformation for %lu not found", checkpoint_step), HERE); - checkpoint_inputfname = inputfname; + checkpoint_metafname = inputfname; } logger::Checkpoint(fmt::format("Using %08lu", checkpoint_step), HERE); - const auto raw_checkpoint_params = toml::parse(checkpoint_inputfname); + const auto raw_checkpoint_params = toml::parse(checkpoint_metafname); const auto start_time = toml::find(raw_checkpoint_params, "metadata", "time"); diff --git a/src/global/utils/tools.h b/src/global/utils/tools.h index 95765e2de..0f2ab0bec 100644 --- a/src/global/utils/tools.h +++ b/src/global/utils/tools.h @@ -18,11 +18,11 @@ #include "global.h" -#include "arch/kokkos_aliases.h" #include "utils/comparators.h" #include "utils/error.h" #include "utils/numeric.h" +#include #include #include #include @@ -275,6 +275,11 @@ namespace tools { } } + /** + * Class for tracking the passage of time either in steps, physical time units, or walltime + * + * @note Primarily used for writing checkpoints and all types of outputs at specified intervals + */ class Tracker { bool m_initialized { false }; @@ -283,31 +288,55 @@ namespace tools { simtime_t m_interval_time; bool m_use_time; + timestamp_t m_start_walltime; + timestamp_t m_end_walltime; + bool m_walltime_pending { false }; + simtime_t m_last_output_time { -1.0 }; public: Tracker() = default; - Tracker(const std::string& type, timestep_t interval, simtime_t interval_time) - : m_initialized { true } - , m_type { type } - , m_interval { interval } - , m_interval_time { interval_time } - , m_use_time { interval_time > 0.0 } {} + Tracker(const std::string& type, + timestep_t interval, + simtime_t interval_time, + const std::string& end_walltime = "", + const timestamp_t& start_walltime = std::chrono::system_clock::now()) { + init(type, interval, interval_time, end_walltime, start_walltime); + } ~Tracker() = default; - void init(const std::string& type, timestep_t interval, simtime_t interval_time) { - m_type = type; - m_interval = interval; - m_interval_time = interval_time; - m_use_time = interval_time > 0.0; - m_initialized = true; + void init(const std::string& type, + timestep_t interval, + simtime_t interval_time, + const std::string& end_walltime = "", + const timestamp_t& start_walltime = std::chrono::system_clock::now()) { + m_initialized = true; + m_type = type; + m_interval = interval; + m_interval_time = interval_time; + m_use_time = interval_time > 0.0; + m_start_walltime = start_walltime; + if (not end_walltime.empty()) { + m_walltime_pending = true; + raise::ErrorIf(end_walltime.size() != 8, + "invalid end walltime format, expected HH:MM:SS", + HERE); + m_end_walltime = m_start_walltime + + std::chrono::hours(std::stoi(end_walltime.substr(0, 2))) + + std::chrono::minutes(std::stoi(end_walltime.substr(3, 2))) + + std::chrono::seconds(std::stoi(end_walltime.substr(6, 2))); + } } auto shouldWrite(timestep_t step, simtime_t time) -> bool { raise::ErrorIf(!m_initialized, "Tracker not initialized", HERE); - if (m_use_time) { + if (m_walltime_pending and + (std::chrono::system_clock::now() > m_end_walltime)) { + m_walltime_pending = false; + return true; + } else if (m_use_time) { if ((m_last_output_time < 0) or (time - m_last_output_time >= m_interval_time)) { m_last_output_time = time; From b122d489fdf06e5575d11890ce6908989e48e7ae Mon Sep 17 00:00:00 2001 From: haykh Date: Fri, 6 Jun 2025 21:24:34 -0500 Subject: [PATCH 08/13] warning on failed write + fixed logging --- src/global/utils/param_container.cpp | 5 +++++ src/global/utils/plog.h | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/global/utils/param_container.cpp b/src/global/utils/param_container.cpp index e1dd57c9e..3c19b48a9 100644 --- a/src/global/utils/param_container.cpp +++ b/src/global/utils/param_container.cpp @@ -331,6 +331,11 @@ namespace prm { try { write_any(io, key, value); } catch (const std::exception& e) { + raise::Warning( + fmt::format("Failed to write parameter '%s', skipping. Error msg: %s", + key.c_str(), + e.what()), + HERE); continue; } } diff --git a/src/global/utils/plog.h b/src/global/utils/plog.h index 2422c7cf9..7713a3728 100644 --- a/src/global/utils/plog.h +++ b/src/global/utils/plog.h @@ -80,9 +80,9 @@ namespace logger { static plog::RollingFileAppender errfileAppender( errfile_name.c_str()); auto log_severity = plog::verbose; - if (fmt::toLower(log_level) == "WARNING") { + if (fmt::toLower(log_level) == "warning") { log_severity = plog::warning; - } else if (fmt::toLower(log_level) == "ERROR") { + } else if (fmt::toLower(log_level) == "error") { log_severity = plog::error; } plog::init(log_severity, &logfileAppender); From 224bdec00ef318cc994d52cb62b02da931197328 Mon Sep 17 00:00:00 2001 From: haykh Date: Fri, 6 Jun 2025 21:45:21 -0500 Subject: [PATCH 09/13] fixed meta writing after restart --- src/engines/engine_init.cpp | 2 +- src/framework/domain/metadomain.h | 2 +- src/framework/domain/output.cpp | 12 +++--------- src/output/writer.cpp | 24 ++++++++++++------------ src/output/writer.h | 6 +++--- 5 files changed, 20 insertions(+), 26 deletions(-) diff --git a/src/engines/engine_init.cpp b/src/engines/engine_init.cpp index 8953b3da8..7a963b615 100644 --- a/src/engines/engine_init.cpp +++ b/src/engines/engine_init.cpp @@ -25,7 +25,7 @@ namespace ntt { if constexpr (pgen_is_ok) { m_metadomain.InitStatsWriter(m_params, is_resuming); #if defined(OUTPUT_ENABLED) - m_metadomain.InitWriter(&m_adios, m_params, is_resuming); + m_metadomain.InitWriter(&m_adios, m_params); m_metadomain.InitCheckpointWriter(&m_adios, m_params); #endif logger::Checkpoint("Initializing Engine", HERE); diff --git a/src/framework/domain/metadomain.h b/src/framework/domain/metadomain.h index 0cb670965..efc639a86 100644 --- a/src/framework/domain/metadomain.h +++ b/src/framework/domain/metadomain.h @@ -116,7 +116,7 @@ namespace ntt { ~Metadomain() = default; #if defined(OUTPUT_ENABLED) - void InitWriter(adios2::ADIOS*, const SimulationParams&, bool); + void InitWriter(adios2::ADIOS*, const SimulationParams&); auto Write(const SimulationParams&, timestep_t, timestep_t, diff --git a/src/framework/domain/output.cpp b/src/framework/domain/output.cpp index ce5a13168..63f4b6e72 100644 --- a/src/framework/domain/output.cpp +++ b/src/framework/domain/output.cpp @@ -40,8 +40,7 @@ namespace ntt { template void Metadomain::InitWriter(adios2::ADIOS* ptr_adios, - const SimulationParams& params, - bool is_resuming) { + const SimulationParams& params) { raise::ErrorIf( l_subdomain_indices().size() != 1, "Output for now is only supported for one subdomain per rank", @@ -103,11 +102,7 @@ namespace ntt { params.template get( "output." + std::string(type) + ".interval_time")); } - if (is_resuming and std::filesystem::exists(g_writer.fname())) { - g_writer.setMode(adios2::Mode::Append); - } else { - g_writer.writeAttrs(params); - } + g_writer.writeAttrs(params); } template @@ -715,8 +710,7 @@ namespace ntt { #define METADOMAIN_OUTPUT(S, M) \ template void Metadomain::InitWriter(adios2::ADIOS*, \ - const SimulationParams&, \ - bool); \ + const SimulationParams&); \ template auto Metadomain::Write( \ const SimulationParams&, \ timestep_t, \ diff --git a/src/output/writer.cpp b/src/output/writer.cpp index b073295e3..65fa50d4c 100644 --- a/src/output/writer.cpp +++ b/src/output/writer.cpp @@ -39,7 +39,7 @@ namespace out { m_io.DefineVariable("Step"); m_io.DefineVariable("Time"); - m_fname = path_t(title); + m_root = path_t(title); } void Writer::addTracker(const std::string& type, @@ -321,7 +321,7 @@ namespace out { } auto output_field_h = Kokkos::create_mirror_view(output_field); Kokkos::deep_copy(output_field_h, output_field); - writer.Put(var, output_field_h); + writer.Put(var, output_field_h, adios2::Mode::Sync); } template @@ -356,7 +356,7 @@ namespace out { adios2::Box({ loc_offset }, { array.extent(0) })); auto array_h = Kokkos::create_mirror_view(array); Kokkos::deep_copy(array_h, array); - m_writer.Put(var, array_h); + m_writer.Put(var, array_h, adios2::Mode::Sync); } void Writer::writeSpectrum(const array_t& counts, @@ -378,12 +378,12 @@ namespace out { if (rank == MPI_ROOT_RANK) { auto var = m_io.InquireVariable(varname); var.SetSelection(adios2::Box({}, { counts.extent(0) })); - m_writer.Put(var, counts_h_all); + m_writer.Put(var, counts_h_all, adios2::Mode::Sync); } #else auto var = m_io.InquireVariable(varname); var.SetSelection(adios2::Box({}, { counts.extent(0) })); - m_writer.Put(var, counts_h); + m_writer.Put(var, counts_h, adios2::Mode::Sync); #endif } @@ -400,7 +400,7 @@ namespace out { var.SetSelection(adios2::Box({}, { e_bins.extent(0) })); auto e_bins_h = Kokkos::create_mirror_view(e_bins); Kokkos::deep_copy(e_bins_h, e_bins); - m_writer.Put(var, e_bins_h); + m_writer.Put(var, e_bins_h, adios2::Mode::Sync); } void Writer::writeMesh(unsigned short dim, @@ -413,11 +413,11 @@ namespace out { auto xe_h = Kokkos::create_mirror_view(xe); Kokkos::deep_copy(xc_h, xc); Kokkos::deep_copy(xe_h, xe); - m_writer.Put(varc, xc_h); - m_writer.Put(vare, xe_h); + m_writer.Put(varc, xc_h, adios2::Mode::Sync); + m_writer.Put(vare, xe_h, adios2::Mode::Sync); auto vard = m_io.InquireVariable( "N" + std::to_string(dim + 1) + "l"); - m_writer.Put(vard, loc_off_sz.data()); + m_writer.Put(vard, loc_off_sz.data(), adios2::Mode::Sync); } void Writer::beginWriting(WriteModeTags write_mode, @@ -454,16 +454,16 @@ namespace out { std::filesystem::create_directory(main_path / mode_path); } }, - m_fname, + m_root, mode_str); #if defined(MPI_ENABLED) MPI_Barrier(MPI_COMM_WORLD); #endif - filename = m_fname / path_t(mode_str) / + filename = m_root / path_t(mode_str) / fmt::format("%s.%08lu.%s", mode_str.c_str(), tstep, ext.c_str()); m_mode = adios2::Mode::Write; } else { - filename = fmt::format("%s.%s", m_fname.c_str(), ext.c_str()); + filename = fmt::format("%s.%s", m_root.c_str(), ext.c_str()); m_mode = std::filesystem::exists(filename) ? adios2::Mode::Append : adios2::Mode::Write; } diff --git a/src/output/writer.h b/src/output/writer.h index 66ee4e597..cc3edc733 100644 --- a/src/output/writer.h +++ b/src/output/writer.h @@ -57,7 +57,7 @@ namespace out { bool m_flds_ghosts; std::string m_engine; - path_t m_fname; + path_t m_root; std::map m_trackers; @@ -116,8 +116,8 @@ namespace out { void endWriting(WriteModeTags); /* getters -------------------------------------------------------------- */ - auto fname() const -> const path_t& { - return m_fname; + auto root() const -> const path_t& { + return m_root; } auto fieldWriters() const -> const std::vector& { From 30afad12e997ae1673678b912d9f6a72cff9a06e Mon Sep 17 00:00:00 2001 From: haykh Date: Fri, 6 Jun 2025 21:59:59 -0500 Subject: [PATCH 10/13] minor changes in tests --- src/checkpoint/tests/checkpoint-nompi.cpp | 28 ++++----- src/checkpoint/writer.h | 2 +- src/kernels/tests/prtl_bc.cpp | 74 ++++++++++++++--------- src/kernels/tests/reduced_stats.cpp | 37 ++++-------- 4 files changed, 70 insertions(+), 71 deletions(-) diff --git a/src/checkpoint/tests/checkpoint-nompi.cpp b/src/checkpoint/tests/checkpoint-nompi.cpp index 85871183d..132a3679a 100644 --- a/src/checkpoint/tests/checkpoint-nompi.cpp +++ b/src/checkpoint/tests/checkpoint-nompi.cpp @@ -60,18 +60,18 @@ auto main(int argc, char* argv[]) -> int { const auto i1_ = static_cast(i1); const auto i2_ = static_cast(i2); const auto i3_ = static_cast(i3); - field1(i1, i2, i3, 0) = i1_ + i2_ + i3; - field1(i1, i2, i3, 1) = i1_ * i2_ / i3; - field1(i1, i2, i3, 2) = i1_ / i2_ * i3; - field1(i1, i2, i3, 3) = i1_ + i2_ - i3; - field1(i1, i2, i3, 4) = i1_ * i2_ + i3; - field1(i1, i2, i3, 5) = i1_ / i2_ - i3; - field2(i1, i2, i3, 0) = -(i1_ + i2_ + i3); - field2(i1, i2, i3, 1) = -(i1_ * i2_ / i3); - field2(i1, i2, i3, 2) = -(i1_ / i2_ * i3); - field2(i1, i2, i3, 3) = -(i1_ + i2_ - i3); - field2(i1, i2, i3, 4) = -(i1_ * i2_ + i3); - field2(i1, i2, i3, 5) = -(i1_ / i2_ - i3); + field1(i1, i2, i3, 0) = i1_ + i2_ + i3_; + field1(i1, i2, i3, 1) = i1_ * i2_ / i3_; + field1(i1, i2, i3, 2) = i1_ / i2_ * i3_; + field1(i1, i2, i3, 3) = i1_ + i2_ - i3_; + field1(i1, i2, i3, 4) = i1_ * i2_ + i3_; + field1(i1, i2, i3, 5) = i1_ / i2_ - i3_; + field2(i1, i2, i3, 0) = -(i1_ + i2_ + i3_); + field2(i1, i2, i3, 1) = -(i1_ * i2_ / i3_); + field2(i1, i2, i3, 2) = -(i1_ / i2_ * i3_); + field2(i1, i2, i3, 3) = -(i1_ + i2_ - i3_); + field2(i1, i2, i3, 4) = -(i1_ * i2_ + i3_); + field2(i1, i2, i3, 5) = -(i1_ / i2_ - i3_); }); Kokkos::parallel_for( "fillPrtl1", @@ -94,8 +94,8 @@ auto main(int argc, char* argv[]) -> int { { // write checkpoint - Writer writer { checkpoint_path }; - writer.init(&adios, 0, 0.0, 1); + Writer writer {}; + writer.init(&adios, checkpoint_path, 0, 0.0, 1); writer.defineFieldVariables(SimEngine::GRPIC, { nx1_gh, nx2_gh, nx3_gh }, diff --git a/src/checkpoint/writer.h b/src/checkpoint/writer.h index 3aa30d04d..6f8bc8cb5 100644 --- a/src/checkpoint/writer.h +++ b/src/checkpoint/writer.h @@ -53,7 +53,7 @@ namespace checkpoint { timestep_t, simtime_t, int, - const std::string&); + const std::string& = ""); auto shouldSave(timestep_t, simtime_t) -> bool; diff --git a/src/kernels/tests/prtl_bc.cpp b/src/kernels/tests/prtl_bc.cpp index 80e4ec7ed..6e49f8ba5 100644 --- a/src/kernels/tests/prtl_bc.cpp +++ b/src/kernels/tests/prtl_bc.cpp @@ -160,21 +160,28 @@ void testPeriodicBC(const std::vector& res, xi[2] = zi_1; } metric.template convert_xyz(xi, xCd); - put_value(i1, static_cast(xCd[0]), prtl_idx); - put_value(i2, static_cast(xCd[1]), prtl_idx); - put_value(i3, static_cast(xCd[2]), prtl_idx); - put_value(dx1, - static_cast(xCd[0]) - - static_cast(static_cast(xCd[0])), - prtl_idx); - put_value(dx2, - static_cast(xCd[1]) - - static_cast(static_cast(xCd[1])), - prtl_idx); - put_value(dx3, - static_cast(xCd[2]) - - static_cast(static_cast(xCd[2])), - prtl_idx); + if constexpr (M::PrtlDim == Dim::_1D or M::PrtlDim == Dim::_2D or + M::PrtlDim == Dim::_3D) { + put_value(i1, static_cast(xCd[0]), prtl_idx); + put_value(dx1, + static_cast(xCd[0]) - + static_cast(static_cast(xCd[0])), + prtl_idx); + } + if constexpr (M::PrtlDim == Dim::_2D or M::PrtlDim == Dim::_3D) { + put_value(i2, static_cast(xCd[1]), prtl_idx); + put_value(dx2, + static_cast(xCd[1]) - + static_cast(static_cast(xCd[1])), + prtl_idx); + } + if constexpr (M::PrtlDim == Dim::_3D) { + put_value(i3, static_cast(xCd[2]), prtl_idx); + put_value(dx3, + static_cast(xCd[2]) - + static_cast(static_cast(xCd[2])), + prtl_idx); + } put_value(ux1, ux_1, prtl_idx); put_value(ux2, uy_1, prtl_idx); put_value(ux3, uz_1, prtl_idx); @@ -195,21 +202,28 @@ void testPeriodicBC(const std::vector& res, xi[2] = zi_2; } metric.template convert_xyz(xi, xCd); - put_value(i1, static_cast(xCd[0]), prtl_idx); - put_value(i2, static_cast(xCd[1]), prtl_idx); - put_value(i3, static_cast(xCd[2]), prtl_idx); - put_value(dx1, - static_cast(xCd[0]) - - static_cast(static_cast(xCd[0])), - prtl_idx); - put_value(dx2, - static_cast(xCd[1]) - - static_cast(static_cast(xCd[1])), - prtl_idx); - put_value(dx3, - static_cast(xCd[2]) - - static_cast(static_cast(xCd[2])), - prtl_idx); + if constexpr (M::PrtlDim == Dim::_1D or M::PrtlDim == Dim::_2D or + M::PrtlDim == Dim::_3D) { + put_value(i1, static_cast(xCd[0]), prtl_idx); + put_value(dx1, + static_cast(xCd[0]) - + static_cast(static_cast(xCd[0])), + prtl_idx); + } + if constexpr (M::PrtlDim == Dim::_2D or M::PrtlDim == Dim::_3D) { + put_value(i2, static_cast(xCd[1]), prtl_idx); + put_value(dx2, + static_cast(xCd[1]) - + static_cast(static_cast(xCd[1])), + prtl_idx); + } + if constexpr (M::PrtlDim == Dim::_3D) { + put_value(i3, static_cast(xCd[2]), prtl_idx); + put_value(dx3, + static_cast(xCd[2]) - + static_cast(static_cast(xCd[2])), + prtl_idx); + } put_value(ux1, ux_2, prtl_idx); put_value(ux2, uy_2, prtl_idx); put_value(ux3, uz_2, prtl_idx); diff --git a/src/kernels/tests/reduced_stats.cpp b/src/kernels/tests/reduced_stats.cpp index caae51ccd..499c24aa9 100644 --- a/src/kernels/tests/reduced_stats.cpp +++ b/src/kernels/tests/reduced_stats.cpp @@ -66,8 +66,7 @@ template auto compute_field_stat(const M& metric, const ndfield_t& em, const ndfield_t& j, - const range_t& range, - ncells_t num_cells) -> real_t { + const range_t& range) -> real_t { real_t buff = ZERO; Kokkos::parallel_reduce("ReduceFields", range, @@ -113,13 +112,11 @@ void testReducedStats(const std::vector& res, ndfield_t EM; ndfield_t J; range_t cell_range; - ncells_t num_cells; if constexpr (M::Dim == Dim::_1D) { EM = ndfield_t { "EM", res[0] + 2 * N_GHOSTS }; J = ndfield_t { "J", res[0] + 2 * N_GHOSTS }; cell_range = { N_GHOSTS, res[0] + N_GHOSTS }; - num_cells = res[0]; put_value(EM, values[0], em::ex1); put_value(EM, values[1], em::ex2); @@ -140,7 +137,6 @@ void testReducedStats(const std::vector& res, { N_GHOSTS, N_GHOSTS }, { res[0] + N_GHOSTS, res[1] + N_GHOSTS } }; - num_cells = res[0] * res[1]; put_value(EM, values[0], em::ex1); put_value(EM, values[1], em::ex2); @@ -167,7 +163,6 @@ void testReducedStats(const std::vector& res, { N_GHOSTS, N_GHOSTS, N_GHOSTS }, { res[0] + N_GHOSTS, res[1] + N_GHOSTS, res[2] + N_GHOSTS } }; - num_cells = res[0] * res[1] * res[2]; put_value(EM, values[0], em::ex1); put_value(EM, values[1], em::ex2); @@ -186,8 +181,7 @@ void testReducedStats(const std::vector& res, const auto Ex_Sq = compute_field_stat(metric, EM, J, - cell_range, - num_cells); + cell_range); raise::ErrorIf(not almost_equal(Ex_Sq, (real_t)(1), acc), "Ex_Sq does not match expected value", HERE); @@ -197,8 +191,7 @@ void testReducedStats(const std::vector& res, const auto Ey_Sq = compute_field_stat(metric, EM, J, - cell_range, - num_cells); + cell_range); raise::ErrorIf(not almost_equal(Ey_Sq, (real_t)(4), acc), "Ey_Sq does not match expected value", HERE); @@ -208,8 +201,7 @@ void testReducedStats(const std::vector& res, const auto Ez_Sq = compute_field_stat(metric, EM, J, - cell_range, - num_cells); + cell_range); raise::ErrorIf(not almost_equal(Ez_Sq, (real_t)(9), acc), "Ez_Sq does not match expected value", HERE); @@ -219,8 +211,7 @@ void testReducedStats(const std::vector& res, const auto Bx_Sq = compute_field_stat(metric, EM, J, - cell_range, - num_cells); + cell_range); raise::ErrorIf(not almost_equal(Bx_Sq, (real_t)(1), acc), "Bx_Sq does not match expected value", HERE); @@ -230,8 +221,7 @@ void testReducedStats(const std::vector& res, const auto By_Sq = compute_field_stat(metric, EM, J, - cell_range, - num_cells); + cell_range); raise::ErrorIf(not almost_equal(By_Sq, (real_t)(4), acc), "By_Sq does not match expected value", HERE); @@ -241,8 +231,7 @@ void testReducedStats(const std::vector& res, const auto Bz_Sq = compute_field_stat(metric, EM, J, - cell_range, - num_cells); + cell_range); raise::ErrorIf(not almost_equal(Bz_Sq, (real_t)(9), acc), "Bz_Sq does not match expected value", HERE); @@ -252,8 +241,7 @@ void testReducedStats(const std::vector& res, const auto ExB_x = compute_field_stat(metric, EM, J, - cell_range, - num_cells); + cell_range); raise::ErrorIf(not almost_equal(ExB_x, (real_t)(12), acc), "ExB_x does not match expected value", HERE); @@ -263,8 +251,7 @@ void testReducedStats(const std::vector& res, const auto ExB_y = compute_field_stat(metric, EM, J, - cell_range, - num_cells); + cell_range); raise::ErrorIf(not almost_equal(ExB_y, (real_t)(-6), acc), "ExB_y does not match expected value", HERE); @@ -274,8 +261,7 @@ void testReducedStats(const std::vector& res, const auto ExB_z = compute_field_stat(metric, EM, J, - cell_range, - num_cells); + cell_range); raise::ErrorIf(not almost_equal(ExB_z, (real_t)(0), acc), "ExB_z does not match expected value", HERE); @@ -285,8 +271,7 @@ void testReducedStats(const std::vector& res, const auto JdotE = compute_field_stat(metric, EM, J, - cell_range, - num_cells); + cell_range); raise::ErrorIf(not almost_equal(JdotE, (real_t)(11), acc), "JdotE does not match expected value", HERE); From bebf5d020bd78febe13a15db838b0a2ba6a66967 Mon Sep 17 00:00:00 2001 From: hayk Date: Sun, 8 Jun 2025 15:23:23 -0400 Subject: [PATCH 11/13] walltime defaults to 00:00:00 to prevent error in attr writing --- input.example.toml | 2 +- src/framework/parameters.cpp | 20 ++++++++++------ src/global/defaults.h | 6 +++-- src/global/utils/param_container.cpp | 33 ++++++++++++++------------- src/global/utils/tools.h | 34 +++++++++++++--------------- src/output/writer.cpp | 5 ++-- 6 files changed, 53 insertions(+), 47 deletions(-) diff --git a/input.example.toml b/input.example.toml index aa532273b..d3e8e5d81 100644 --- a/input.example.toml +++ b/input.example.toml @@ -476,7 +476,7 @@ keep = "" # Write a checkpoint once after a fixed walltime # @type: string - # @default: "" (disabled) + # @default: "00:00:00" (disabled) # @note: The format is "HH:MM:SS" # @note: Useful for long-running simulations to ensure that ... # @note: ... the simulation can be resumed after allocation expires diff --git a/src/framework/parameters.cpp b/src/framework/parameters.cpp index 586a01941..d901ce1ec 100644 --- a/src/framework/parameters.cpp +++ b/src/framework/parameters.cpp @@ -31,10 +31,10 @@ namespace ntt { template - auto get_dx0_V0( - const std::vector& resolution, - const boundaries_t& extent, - const std::map& params) -> std::pair { + auto get_dx0_V0(const std::vector& resolution, + const boundaries_t& extent, + const std::map& params) + -> std::pair { const auto metric = M(resolution, extent, params); const auto dx0 = metric.dxMin(); coord_t x_corner { ZERO }; @@ -609,14 +609,20 @@ namespace ntt { toml::find_or(toml_data, "checkpoint", "interval_time", -1.0)); set("checkpoint.keep", toml::find_or(toml_data, "checkpoint", "keep", defaults::checkpoint::keep)); - set("checkpoint.walltime", - toml::find_or(toml_data, "checkpoint", "walltime", std::string(""))); + auto walltime_str = toml::find_or(toml_data, + "checkpoint", + "walltime", + defaults::checkpoint::walltime); + if (walltime_str.empty()) { + walltime_str = defaults::checkpoint::walltime; + } + set("checkpoint.walltime", walltime_str); const auto checkpoint_write_path = toml::find_or( toml_data, "checkpoint", "write_path", - std::string("checkpoints")); + defaults::checkpoint::write_path); set("checkpoint.write_path", checkpoint_write_path); set("checkpoint.read_path", toml::find_or(toml_data, "checkpoint", "read_path", checkpoint_write_path)); diff --git a/src/global/defaults.h b/src/global/defaults.h index d673a503c..8ddbc0ce0 100644 --- a/src/global/defaults.h +++ b/src/global/defaults.h @@ -67,8 +67,10 @@ namespace ntt::defaults { } // namespace output namespace checkpoint { - const timestep_t interval = 1000; - const int keep = 2; + const timestep_t interval = 1000; + const int keep = 2; + const std::string walltime = "00:00:00"; + const std::string write_path = "checkpoints"; } // namespace checkpoint namespace diag { diff --git a/src/global/utils/param_container.cpp b/src/global/utils/param_container.cpp index 3c19b48a9..54d8591b3 100644 --- a/src/global/utils/param_container.cpp +++ b/src/global/utils/param_container.cpp @@ -13,7 +13,6 @@ #include #include #include - #include #include #include @@ -32,8 +31,8 @@ namespace prm { } template - auto write(adios2::IO& io, const std::string& name, T var) -> decltype(void(T()), - void()) { + auto write(adios2::IO& io, const std::string& name, T var) + -> decltype(void(T()), void()) { io.DefineAttribute(name, var); } @@ -48,8 +47,8 @@ namespace prm { } template - auto write_pair(adios2::IO& io, const std::string& name, std::pair var) -> - typename std::enable_if::value, void>::type { + auto write_pair(adios2::IO& io, const std::string& name, std::pair var) + -> typename std::enable_if::value, void>::type { std::vector var_str; var_str.push_back(var.first.to_string()); var_str.push_back(var.second.to_string()); @@ -57,9 +56,8 @@ namespace prm { } template - auto write_pair(adios2::IO& io, - const std::string& name, - std::pair var) -> decltype(void(T()), void()) { + auto write_pair(adios2::IO& io, const std::string& name, std::pair var) + -> decltype(void(T()), void()) { std::vector var_vec; var_vec.push_back(var.first); var_vec.push_back(var.second); @@ -77,9 +75,8 @@ namespace prm { } template - auto write_vec(adios2::IO& io, - const std::string& name, - std::vector var) -> decltype(void(T()), void()) { + auto write_vec(adios2::IO& io, const std::string& name, std::vector var) + -> decltype(void(T()), void()) { io.DefineAttribute(name, var.data(), var.size()); } @@ -99,8 +96,8 @@ namespace prm { template auto write_vec_pair(adios2::IO& io, const std::string& name, - std::vector> var) -> decltype(void(T()), - void()) { + std::vector> var) + -> decltype(void(T()), void()) { std::vector var_vec; for (const auto& v : var) { var_vec.push_back(v.first); @@ -126,8 +123,8 @@ namespace prm { template auto write_vec_vec(adios2::IO& io, const std::string& name, - std::vector> var) -> decltype(void(T()), - void()) { + std::vector> var) + -> decltype(void(T()), void()) { std::vector var_vec; for (const auto& vec : var) { for (const auto& v : vec) { @@ -328,6 +325,10 @@ namespace prm { register_write_function_for_dict(); for (auto& [key, value] : allVars()) { + // @TODO: add particles.species support in attrs + if (key == "particles.species") { + continue; + } try { write_any(io, key, value); } catch (const std::exception& e) { @@ -343,4 +344,4 @@ namespace prm { } // namespace prm -#endif +#endif // OUTPUT_ENABLED diff --git a/src/global/utils/tools.h b/src/global/utils/tools.h index 0f2ab0bec..27c7e22c8 100644 --- a/src/global/utils/tools.h +++ b/src/global/utils/tools.h @@ -60,8 +60,8 @@ namespace tools { * @return Tensor product of list */ template - inline auto TensorProduct( - const std::vector>& list) -> std::vector> { + inline auto TensorProduct(const std::vector>& list) + -> std::vector> { std::vector> result = { {} }; for (const auto& sublist : list) { std::vector> temp; @@ -81,8 +81,8 @@ namespace tools { * @param ndomains Number of domains * @param ncells Number of cells */ - inline auto decompose1D(unsigned int ndomains, - ncells_t ncells) -> std::vector { + inline auto decompose1D(unsigned int ndomains, ncells_t ncells) + -> std::vector { auto size = (ncells_t)((double)ncells / (double)ndomains); auto ncells_domain = std::vector(ndomains, size); for (auto i { 0u }; i < ncells - size * ndomains; ++i) { @@ -107,10 +107,8 @@ namespace tools { * @param s1 Proportion of the first dimension * @param s2 Proportion of the second dimension */ - inline auto divideInProportions2D( - unsigned int ntot, - unsigned int s1, - unsigned int s2) -> std::tuple { + inline auto divideInProportions2D(unsigned int ntot, unsigned int s1, unsigned int s2) + -> std::tuple { auto n1 = (unsigned int)(std::sqrt((double)ntot * (double)s1 / (double)s2)); if (n1 == 0) { return { 1, ntot }; @@ -132,11 +130,11 @@ namespace tools { * @param s2 Proportion of the second dimension * @param s3 Proportion of the third dimension */ - inline auto divideInProportions3D( - unsigned int ntot, - unsigned int s1, - unsigned int s2, - unsigned int s3) -> std::tuple { + inline auto divideInProportions3D(unsigned int ntot, + unsigned int s1, + unsigned int s2, + unsigned int s3) + -> std::tuple { auto n1 = (unsigned int)(std::cbrt( (double)ntot * (double)(SQR(s1)) / (double)(s2 * s3))); if (n1 > ntot) { @@ -165,10 +163,10 @@ namespace tools { * * @note If decomposition has -1, it will be calculated automatically */ - inline auto Decompose( - unsigned int ndomains, - const std::vector& ncells, - const std::vector& decomposition) -> std::vector> { + inline auto Decompose(unsigned int ndomains, + const std::vector& ncells, + const std::vector& decomposition) + -> std::vector> { const auto dimension = ncells.size(); raise::ErrorIf(dimension != decomposition.size(), "Decomposition error: dimension != decomposition.size", @@ -318,7 +316,7 @@ namespace tools { m_interval_time = interval_time; m_use_time = interval_time > 0.0; m_start_walltime = start_walltime; - if (not end_walltime.empty()) { + if (not(end_walltime.empty() or end_walltime == "00:00:00")) { m_walltime_pending = true; raise::ErrorIf(end_walltime.size() != 8, "invalid end walltime format, expected HH:MM:SS", diff --git a/src/output/writer.cpp b/src/output/writer.cpp index 65fa50d4c..9a96f9f56 100644 --- a/src/output/writer.cpp +++ b/src/output/writer.cpp @@ -48,9 +48,8 @@ namespace out { m_trackers.insert({ type, tools::Tracker(type, interval, interval_time) }); } - auto Writer::shouldWrite(const std::string& type, - timestep_t step, - simtime_t time) -> bool { + auto Writer::shouldWrite(const std::string& type, timestep_t step, simtime_t time) + -> bool { if (m_trackers.find(type) != m_trackers.end()) { return m_trackers.at(type).shouldWrite(step, time); } else { From ff75f581a2193a0d9b11a6db764e6249c2732fc8 Mon Sep 17 00:00:00 2001 From: hayk Date: Mon, 9 Jun 2025 13:47:24 -0400 Subject: [PATCH 12/13] default name for checkpoints -> %s.ckpt --- src/framework/parameters.cpp | 3 ++- src/framework/simulation.cpp | 2 +- src/global/defaults.h | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/framework/parameters.cpp b/src/framework/parameters.cpp index d901ce1ec..c9f612cec 100644 --- a/src/framework/parameters.cpp +++ b/src/framework/parameters.cpp @@ -622,7 +622,8 @@ namespace ntt { toml_data, "checkpoint", "write_path", - defaults::checkpoint::write_path); + fmt::format(defaults::checkpoint::write_path.c_str(), + get("simulation.name").c_str())); set("checkpoint.write_path", checkpoint_write_path); set("checkpoint.read_path", toml::find_or(toml_data, "checkpoint", "read_path", checkpoint_write_path)); diff --git a/src/framework/simulation.cpp b/src/framework/simulation.cpp index b130d10a7..6735eda79 100644 --- a/src/framework/simulation.cpp +++ b/src/framework/simulation.cpp @@ -60,7 +60,7 @@ namespace ntt { raw_params, "checkpoint", "write_path", - "checkpoints"); + fmt::format(defaults::checkpoint::write_path.c_str(), sim_name.c_str())); const path_t checkpoint_read_path = toml::find_or( raw_params, "checkpoint", diff --git a/src/global/defaults.h b/src/global/defaults.h index 8ddbc0ce0..9513493b1 100644 --- a/src/global/defaults.h +++ b/src/global/defaults.h @@ -70,7 +70,7 @@ namespace ntt::defaults { const timestep_t interval = 1000; const int keep = 2; const std::string walltime = "00:00:00"; - const std::string write_path = "checkpoints"; + const std::string write_path = "%s.ckpt"; } // namespace checkpoint namespace diag { From 79a0a6941b90693c9cc30f93013b48e2541ddb99 Mon Sep 17 00:00:00 2001 From: hayk Date: Mon, 9 Jun 2025 13:50:12 -0400 Subject: [PATCH 13/13] default checkpoint.write_path in input --- input.example.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/input.example.toml b/input.example.toml index d3e8e5d81..6bb1537a4 100644 --- a/input.example.toml +++ b/input.example.toml @@ -485,7 +485,7 @@ walltime = "" # Parent directory to write checkpoints to # @type: string - # @default: "checkpoints" + # @default: ".ckpt" # @note: The directory is created if it does not exist write_path = "" # Parent directory to use when resuming from a checkpoint