diff --git a/LICENSE b/LICENSE index 55482078..639d2b41 100644 --- a/LICENSE +++ b/LICENSE @@ -1,19 +1,8 @@ BSD 3-Clause License -Copyright (c) 2024, Princeton University (PU), and -Princeton Plasma Physics Lab (PPPL). +Copyright (c) 2021-present, Entity development team. All rights reserved. -Core developers (alphabetical order): -* Alexander Chernoglazov -* Benjamin Crinquand -* Alisa Galishnikova -* Hayk Hakobyan -* Jens Mahlmann -* Sasha Philippov -* Arno Vanthieghem -* Muni Zhou - Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/dev/nix/adios2.nix b/dev/nix/adios2.nix index 7218f894..0418b71c 100644 --- a/dev/nix/adios2.nix +++ b/dev/nix/adios2.nix @@ -19,10 +19,12 @@ let BUILD_TESTING = "OFF"; ADIOS2_BUILD_EXAMPLES = "OFF"; ADIOS2_USE_MPI = if mpi then "ON" else "OFF"; + ADIOS2_HAVE_HDF5_VOL = if mpi then "ON" else "OFF"; CMAKE_BUILD_TYPE = "Release"; - } // (if !mpi then { ADIOS2_HAVE_HDF5_VOL = "OFF"; } else { }); + }; + stdenv = pkgs.gcc13Stdenv; in -pkgs.stdenv.mkDerivation { +stdenv.mkDerivation { pname = "${name}${if hdf5 then "-hdf5" else ""}${if mpi then "-mpi" else ""}"; version = "${version}"; src = pkgs.fetchgit { @@ -36,12 +38,10 @@ pkgs.stdenv.mkDerivation { perl ]; - propagatedBuildInputs = - [ - pkgs.gcc13 - ] - ++ (if hdf5 then (if mpi then [ pkgs.hdf5-mpi ] else [ pkgs.hdf5 ]) else [ ]) - ++ (if mpi then [ pkgs.openmpi ] else [ ]); + propagatedBuildInputs = [ + pkgs.gcc13 + ] ++ (if hdf5 then (if mpi then [ pkgs.hdf5-mpi ] else [ pkgs.hdf5-cpp ]) else [ ]); + # ++ (if mpi then [ pkgs.openmpi ] else [ ]); configurePhase = '' cmake -B build $src ${ diff --git a/dev/nix/kokkos.nix b/dev/nix/kokkos.nix index c83a489e..8dfe1f38 100644 --- a/dev/nix/kokkos.nix +++ b/dev/nix/kokkos.nix @@ -20,11 +20,18 @@ let "CUDA" = with pkgs.cudaPackages; [ cudatoolkit cuda_cudart + pkgs.gcc13 ]; "NONE" = [ pkgs.gcc13 ]; }; + getArch = + _: + if gpu != "NONE" && arch == "NATIVE" then + throw "Please specify an architecture when the GPU support is enabled. Available architectures: https://kokkos.org/kokkos-core-wiki/keywords.html#architectures" + else + arch; cmakeExtraFlags = { "HIP" = [ "-D Kokkos_ENABLE_HIP=ON" @@ -39,13 +46,6 @@ let ]; "NONE" = [ ]; }; - getArch = - _: - if gpu != "NONE" && arch == "NATIVE" then - throw "Please specify an architecture when the GPU support is enabled. Available architectures: https://kokkos.org/kokkos-core-wiki/keywords.html#architectures" - else - arch; - in pkgs.stdenv.mkDerivation rec { pname = "${name}"; diff --git a/dev/nix/shell.nix b/dev/nix/shell.nix index be4b87bc..ae5451ec 100644 --- a/dev/nix/shell.nix +++ b/dev/nix/shell.nix @@ -42,7 +42,8 @@ pkgs.mkShell { zlib cmake - clang-tools + llvmPackages_18.clang-tools + libgcc adios2Pkg kokkosPkg diff --git a/input.example.toml b/input.example.toml index 26a5d1c9..6bb1537a 100644 --- a/input.example.toml +++ b/input.example.toml @@ -474,6 +474,24 @@ # @note: 0 = disable checkpointing # @note: -1 = keep all checkpoints keep = "" + # Write a checkpoint once after a fixed walltime + # @type: string + # @default: "00:00:00" (disabled) + # @note: The format is "HH:MM:SS" + # @note: Useful for long-running simulations to ensure that ... + # @note: ... the simulation can be resumed after allocation expires + # @note: Empty string means disables this (still writes checkpoints on interval) + # @note: This does not exit the simulation + walltime = "" + # Parent directory to write checkpoints to + # @type: string + # @default: ".ckpt" + # @note: The directory is created if it does not exist + write_path = "" + # Parent directory to use when resuming from a checkpoint + # @type: string + # @default: inherits from `write_path` + read_path = "" # @inferred: # - is_resuming diff --git a/src/archetypes/energy_dist.h b/src/archetypes/energy_dist.h index e9ef8b0b..901dbce3 100644 --- a/src/archetypes/energy_dist.h +++ b/src/archetypes/energy_dist.h @@ -392,7 +392,7 @@ namespace arch { Inline void operator()(const coord_t& x_Code, vec_t& v, - spidx_t sp = 0) const override { + spidx_t = 0) const override { if (cmp::AlmostZero(temperature)) { v[0] = ZERO; v[1] = ZERO; diff --git a/src/checkpoint/reader.cpp b/src/checkpoint/reader.cpp index 24aa0f07..d973b0dd 100644 --- a/src/checkpoint/reader.cpp +++ b/src/checkpoint/reader.cpp @@ -110,7 +110,7 @@ namespace checkpoint { void ReadParticlePayloads(adios2::IO& io, adios2::Engine& reader, - spidx_t s, + spidx_t s, array_t& array, std::size_t nplds, npart_t count, @@ -130,64 +130,32 @@ namespace checkpoint { } } - template void ReadFields(adios2::IO&, - adios2::Engine&, - const std::string&, - const adios2::Box&, - ndfield_t&); - template void ReadFields(adios2::IO&, - adios2::Engine&, - const std::string&, - const adios2::Box&, - ndfield_t&); - template void ReadFields(adios2::IO&, - adios2::Engine&, - const std::string&, - const adios2::Box&, - ndfield_t&); - template void ReadFields(adios2::IO&, - adios2::Engine&, - const std::string&, - const adios2::Box&, - ndfield_t&); - template void ReadFields(adios2::IO&, - adios2::Engine&, - const std::string&, - const adios2::Box&, - ndfield_t&); - template void ReadFields(adios2::IO&, - adios2::Engine&, - const std::string&, - const adios2::Box&, - ndfield_t&); +#define CHECKPOINT_FIELDS(D, N) \ + template void ReadFields(adios2::IO&, \ + adios2::Engine&, \ + const std::string&, \ + const adios2::Box&, \ + ndfield_t&); + CHECKPOINT_FIELDS(Dim::_1D, 3) + CHECKPOINT_FIELDS(Dim::_2D, 3) + CHECKPOINT_FIELDS(Dim::_3D, 3) + CHECKPOINT_FIELDS(Dim::_1D, 6) + CHECKPOINT_FIELDS(Dim::_2D, 6) + CHECKPOINT_FIELDS(Dim::_3D, 6) +#undef CHECKPOINT_FIELDS - template void ReadParticleData(adios2::IO&, - adios2::Engine&, - const std::string&, - spidx_t, - array_t&, - npart_t, - npart_t); - template void ReadParticleData(adios2::IO&, - adios2::Engine&, - const std::string&, - spidx_t, - array_t&, - npart_t, - npart_t); - template void ReadParticleData(adios2::IO&, - adios2::Engine&, - const std::string&, - spidx_t, - array_t&, - npart_t, - npart_t); - template void ReadParticleData(adios2::IO&, - adios2::Engine&, - const std::string&, - spidx_t, - array_t&, - npart_t, - npart_t); +#define CHECKPOINT_PARTICLE_DATA(T) \ + template void ReadParticleData(adios2::IO&, \ + adios2::Engine&, \ + const std::string&, \ + spidx_t, \ + array_t&, \ + npart_t, \ + npart_t); + CHECKPOINT_PARTICLE_DATA(int) + CHECKPOINT_PARTICLE_DATA(float) + CHECKPOINT_PARTICLE_DATA(double) + CHECKPOINT_PARTICLE_DATA(short) +#undef CHECKPOINT_PARTICLE_DATA } // namespace checkpoint diff --git a/src/checkpoint/tests/checkpoint-nompi.cpp b/src/checkpoint/tests/checkpoint-nompi.cpp index be5e97e2..132a3679 100644 --- a/src/checkpoint/tests/checkpoint-nompi.cpp +++ b/src/checkpoint/tests/checkpoint-nompi.cpp @@ -12,14 +12,13 @@ #include #include -#include using namespace ntt; using namespace checkpoint; void cleanup() { namespace fs = std::filesystem; - fs::path temp_path { "checkpoints" }; + fs::path temp_path { "chck" }; fs::remove_all(temp_path); } @@ -58,18 +57,21 @@ auto main(int argc, char* argv[]) -> int { CreateRangePolicy({ i1min, i2min, i3min }, { i1max, i2max, i3max }), Lambda(index_t i1, index_t i2, index_t i3) { - field1(i1, i2, i3, 0) = i1 + i2 + i3; - field1(i1, i2, i3, 1) = i1 * i2 / i3; - field1(i1, i2, i3, 2) = i1 / i2 * i3; - field1(i1, i2, i3, 3) = i1 + i2 - i3; - field1(i1, i2, i3, 4) = i1 * i2 + i3; - field1(i1, i2, i3, 5) = i1 / i2 - i3; - field2(i1, i2, i3, 0) = -(i1 + i2 + i3); - field2(i1, i2, i3, 1) = -(i1 * i2 / i3); - field2(i1, i2, i3, 2) = -(i1 / i2 * i3); - field2(i1, i2, i3, 3) = -(i1 + i2 - i3); - field2(i1, i2, i3, 4) = -(i1 * i2 + i3); - field2(i1, i2, i3, 5) = -(i1 / i2 - i3); + const auto i1_ = static_cast(i1); + const auto i2_ = static_cast(i2); + const auto i3_ = static_cast(i3); + field1(i1, i2, i3, 0) = i1_ + i2_ + i3_; + field1(i1, i2, i3, 1) = i1_ * i2_ / i3_; + field1(i1, i2, i3, 2) = i1_ / i2_ * i3_; + field1(i1, i2, i3, 3) = i1_ + i2_ - i3_; + field1(i1, i2, i3, 4) = i1_ * i2_ + i3_; + field1(i1, i2, i3, 5) = i1_ / i2_ - i3_; + field2(i1, i2, i3, 0) = -(i1_ + i2_ + i3_); + field2(i1, i2, i3, 1) = -(i1_ * i2_ / i3_); + field2(i1, i2, i3, 2) = -(i1_ / i2_ * i3_); + field2(i1, i2, i3, 3) = -(i1_ + i2_ - i3_); + field2(i1, i2, i3, 4) = -(i1_ * i2_ + i3_); + field2(i1, i2, i3, 5) = -(i1_ / i2_ - i3_); }); Kokkos::parallel_for( "fillPrtl1", @@ -88,11 +90,12 @@ auto main(int argc, char* argv[]) -> int { } adios2::ADIOS adios; + const path_t checkpoint_path { "chck" }; { // write checkpoint - Writer writer; - writer.init(&adios, 0, 0.0, 1); + Writer writer {}; + writer.init(&adios, checkpoint_path, 0, 0.0, 1); writer.defineFieldVariables(SimEngine::GRPIC, { nx1_gh, nx2_gh, nx3_gh }, @@ -127,7 +130,7 @@ auto main(int argc, char* argv[]) -> int { array_t u2_read { "u_2", npart2 }; adios2::IO io = adios.DeclareIO("checkpointRead"); - adios2::Engine reader = io.Open("checkpoints/step-00000000.bp", + adios2::Engine reader = io.Open(checkpoint_path / "step-00000000.bp", adios2::Mode::Read); reader.BeginStep(); diff --git a/src/checkpoint/writer.cpp b/src/checkpoint/writer.cpp index b8571c24..b766ddfb 100644 --- a/src/checkpoint/writer.cpp +++ b/src/checkpoint/writer.cpp @@ -15,21 +15,23 @@ #include #include #include -#include #include namespace checkpoint { - void Writer::init(adios2::ADIOS* ptr_adios, - timestep_t interval, - simtime_t interval_time, - int keep) { - m_keep = keep; - m_enabled = keep != 0; + void Writer::init(adios2::ADIOS* ptr_adios, + const path_t& checkpoint_root, + timestep_t interval, + simtime_t interval_time, + int keep, + const std::string& walltime) { + m_keep = keep; + m_checkpoint_root = checkpoint_root; + m_enabled = keep != 0; if (not m_enabled) { return; } - m_tracker.init("checkpoint", interval, interval_time); + m_tracker.init("checkpoint", interval, interval_time, walltime); p_adios = ptr_adios; raise::ErrorIf(p_adios == nullptr, "ADIOS pointer is null", HERE); @@ -40,12 +42,13 @@ namespace checkpoint { m_io.DefineVariable("Time"); m_io.DefineAttribute("NGhosts", ntt::N_GHOSTS); - CallOnce([]() { - const std::filesystem::path save_path { "checkpoints" }; - if (!std::filesystem::exists(save_path)) { - std::filesystem::create_directory(save_path); - } - }); + CallOnce( + [](auto&& checkpoint_root) { + if (!std::filesystem::exists(checkpoint_root)) { + std::filesystem::create_directory(checkpoint_root); + } + }, + m_checkpoint_root); } void Writer::defineFieldVariables(const ntt::SimEngine& S, @@ -147,13 +150,14 @@ namespace checkpoint { } m_writing_mode = true; try { - auto fname = fmt::format("checkpoints/step-%08lu.bp", step); - m_writer = m_io.Open(fname, adios2::Mode::Write); - auto meta_fname = fmt::format("checkpoints/meta-%08lu.toml", step); - m_written.push_back({ fname, meta_fname }); + const auto filename = m_checkpoint_root / fmt::format("step-%08lu.bp", step); + const auto metafilename = m_checkpoint_root / + fmt::format("meta-%08lu.toml", step); + m_writer = m_io.Open(filename, adios2::Mode::Write); + m_written.push_back({ filename, metafilename }); logger::Checkpoint(fmt::format("Writing checkpoint to %s and %s", - fname.c_str(), - meta_fname.c_str()), + filename.c_str(), + metafilename.c_str()), HERE); } catch (std::exception& e) { raise::Fatal(e.what(), HERE); @@ -261,54 +265,38 @@ namespace checkpoint { m_writer.Put(var, data_sub.data(), adios2::Mode::Sync); } - template void Writer::savePerDomainVariable(const std::string&, - std::size_t, - std::size_t, - int); - template void Writer::savePerDomainVariable(const std::string&, - std::size_t, - std::size_t, - float); - template void Writer::savePerDomainVariable(const std::string&, - std::size_t, - std::size_t, - double); - template void Writer::savePerDomainVariable(const std::string&, - std::size_t, - std::size_t, - npart_t); - - template void Writer::saveField(const std::string&, - const ndfield_t&); - template void Writer::saveField(const std::string&, - const ndfield_t&); - template void Writer::saveField(const std::string&, - const ndfield_t&); - template void Writer::saveField(const std::string&, - const ndfield_t&); - template void Writer::saveField(const std::string&, - const ndfield_t&); - template void Writer::saveField(const std::string&, - const ndfield_t&); - - template void Writer::saveParticleQuantity(const std::string&, - npart_t, - npart_t, - npart_t, - const array_t&); - template void Writer::saveParticleQuantity(const std::string&, - npart_t, - npart_t, - npart_t, - const array_t&); - template void Writer::saveParticleQuantity(const std::string&, - npart_t, - npart_t, - npart_t, - const array_t&); - template void Writer::saveParticleQuantity(const std::string&, - npart_t, - npart_t, - npart_t, - const array_t&); +#define CHECKPOINT_PERDOMAIN_VARIABLE(T) \ + template void Writer::savePerDomainVariable(const std::string&, \ + std::size_t, \ + std::size_t, \ + T); + CHECKPOINT_PERDOMAIN_VARIABLE(int) + CHECKPOINT_PERDOMAIN_VARIABLE(float) + CHECKPOINT_PERDOMAIN_VARIABLE(double) + CHECKPOINT_PERDOMAIN_VARIABLE(npart_t) +#undef CHECKPOINT_PERDOMAIN_VARIABLE + +#define CHECKPOINT_FIELD(D, N) \ + template void Writer::saveField(const std::string&, \ + const ndfield_t&); + CHECKPOINT_FIELD(Dim::_1D, 3) + CHECKPOINT_FIELD(Dim::_1D, 6) + CHECKPOINT_FIELD(Dim::_2D, 3) + CHECKPOINT_FIELD(Dim::_2D, 6) + CHECKPOINT_FIELD(Dim::_3D, 3) + CHECKPOINT_FIELD(Dim::_3D, 6) +#undef CHECKPOINT_FIELD + +#define CHECKPOINT_PARTICLE_QUANTITY(T) \ + template void Writer::saveParticleQuantity(const std::string&, \ + npart_t, \ + npart_t, \ + npart_t, \ + const array_t&); + CHECKPOINT_PARTICLE_QUANTITY(int) + CHECKPOINT_PARTICLE_QUANTITY(float) + CHECKPOINT_PARTICLE_QUANTITY(double) + CHECKPOINT_PARTICLE_QUANTITY(short) +#undef CHECKPOINT_PARTICLE_QUANTITY + } // namespace checkpoint diff --git a/src/checkpoint/writer.h b/src/checkpoint/writer.h index 91c9d7a4..6f8bc8cb 100644 --- a/src/checkpoint/writer.h +++ b/src/checkpoint/writer.h @@ -39,15 +39,21 @@ namespace checkpoint { std::vector> m_written; - int m_keep; - bool m_enabled; + int m_keep; + bool m_enabled; + path_t m_checkpoint_root; public: Writer() {} ~Writer() = default; - void init(adios2::ADIOS*, timestep_t, simtime_t, int); + void init(adios2::ADIOS*, + const path_t&, + timestep_t, + simtime_t, + int, + const std::string& = ""); auto shouldSave(timestep_t, simtime_t) -> bool; diff --git a/src/engines/engine.hpp b/src/engines/engine.hpp index 5956ea5d..17103f1d 100644 --- a/src/engines/engine.hpp +++ b/src/engines/engine.hpp @@ -110,7 +110,6 @@ namespace ntt { , time { start_time } , step { start_step } { raise::ErrorIf(not pgen_is_ok, "Problem generator is not compatible with the picked engine/metric/dimension", HERE); - print_report(); } ~Engine() = default; diff --git a/src/engines/engine_init.cpp b/src/engines/engine_init.cpp index 833a7771..7a963b61 100644 --- a/src/engines/engine_init.cpp +++ b/src/engines/engine_init.cpp @@ -16,6 +16,8 @@ #include +#include + namespace ntt { template @@ -23,7 +25,7 @@ namespace ntt { if constexpr (pgen_is_ok) { m_metadomain.InitStatsWriter(m_params, is_resuming); #if defined(OUTPUT_ENABLED) - m_metadomain.InitWriter(&m_adios, m_params, is_resuming); + m_metadomain.InitWriter(&m_adios, m_params); m_metadomain.InitCheckpointWriter(&m_adios, m_params); #endif logger::Checkpoint("Initializing Engine", HERE); @@ -66,6 +68,7 @@ namespace ntt { #endif } } + print_report(); } template class Engine>; @@ -76,4 +79,5 @@ namespace ntt { template class Engine>; template class Engine>; template class Engine>; + } // namespace ntt diff --git a/src/engines/engine_printer.cpp b/src/engines/engine_printer.cpp index 9d1f74a7..20f5e81b 100644 --- a/src/engines/engine_printer.cpp +++ b/src/engines/engine_printer.cpp @@ -106,13 +106,13 @@ namespace ntt { color::RESET); } - auto bytes_to_human_readable(std::size_t bytes) - -> std::pair { + auto bytes_to_human_readable( + std::size_t bytes) -> std::pair { const std::vector units { "B", "KB", "MB", "GB", "TB" }; idx_t unit_idx = 0; - auto size = static_cast(bytes); - while ((size >= 1024) && (unit_idx < units.size() - 1)) { - size /= 1024; + auto size = static_cast(bytes); + while ((size >= 1024.0) and (unit_idx < units.size() - 1)) { + size /= 1024.0; ++unit_idx; } return { size, units[unit_idx] }; diff --git a/src/framework/containers/particles.cpp b/src/framework/containers/particles.cpp index b69e48de..d2db9c49 100644 --- a/src/framework/containers/particles.cpp +++ b/src/framework/containers/particles.cpp @@ -4,8 +4,6 @@ #include "global.h" #include "arch/kokkos_aliases.h" -#include "utils/numeric.h" -#include "utils/sorting.h" #include "framework/containers/species.h" @@ -13,9 +11,6 @@ #include #include -#include -#include -#include #include #include diff --git a/src/framework/domain/checkpoint.cpp b/src/framework/domain/checkpoint.cpp index a39f18e6..e0e34f99 100644 --- a/src/framework/domain/checkpoint.cpp +++ b/src/framework/domain/checkpoint.cpp @@ -47,11 +47,16 @@ namespace ntt { nplds.push_back(local_domain->species[s].npld()); } + const path_t checkpoint_root = params.template get( + "checkpoint.write_path"); + g_checkpoint_writer.init( ptr_adios, + checkpoint_root, params.template get("checkpoint.interval"), params.template get("checkpoint.interval_time"), - params.template get("checkpoint.keep")); + params.template get("checkpoint.keep"), + params.template get("checkpoint.walltime")); if (g_checkpoint_writer.enabled()) { g_checkpoint_writer.defineFieldVariables(S, glob_shape_with_ghosts, @@ -261,9 +266,12 @@ namespace ntt { void Metadomain::ContinueFromCheckpoint(adios2::ADIOS* ptr_adios, const SimulationParams& params) { raise::ErrorIf(ptr_adios == nullptr, "adios == nullptr", HERE); - auto fname = fmt::format( - "checkpoints/step-%08lu.bp", - params.template get("checkpoint.start_step")); + const path_t checkpoint_root = params.template get( + "checkpoint.read_path"); + const auto fname = checkpoint_root / + fmt::format("step-%08lu.bp", + params.template get( + "checkpoint.start_step")); logger::Checkpoint(fmt::format("Reading checkpoint from %s", fname.c_str()), HERE); @@ -484,7 +492,6 @@ namespace ntt { simtime_t) -> bool; \ template void Metadomain::ContinueFromCheckpoint(adios2::ADIOS*, \ const SimulationParams&); - METADOMAIN_CHECKPOINTS(SimEngine::SRPIC, metric::Minkowski) METADOMAIN_CHECKPOINTS(SimEngine::SRPIC, metric::Minkowski) METADOMAIN_CHECKPOINTS(SimEngine::SRPIC, metric::Minkowski) @@ -493,7 +500,6 @@ namespace ntt { METADOMAIN_CHECKPOINTS(SimEngine::GRPIC, metric::KerrSchild) METADOMAIN_CHECKPOINTS(SimEngine::GRPIC, metric::QKerrSchild) METADOMAIN_CHECKPOINTS(SimEngine::GRPIC, metric::KerrSchild0) - #undef METADOMAIN_CHECKPOINTS } // namespace ntt diff --git a/src/framework/domain/metadomain.h b/src/framework/domain/metadomain.h index 0cb67096..efc639a8 100644 --- a/src/framework/domain/metadomain.h +++ b/src/framework/domain/metadomain.h @@ -116,7 +116,7 @@ namespace ntt { ~Metadomain() = default; #if defined(OUTPUT_ENABLED) - void InitWriter(adios2::ADIOS*, const SimulationParams&, bool); + void InitWriter(adios2::ADIOS*, const SimulationParams&); auto Write(const SimulationParams&, timestep_t, timestep_t, diff --git a/src/framework/domain/output.cpp b/src/framework/domain/output.cpp index ce5a1316..63f4b6e7 100644 --- a/src/framework/domain/output.cpp +++ b/src/framework/domain/output.cpp @@ -40,8 +40,7 @@ namespace ntt { template void Metadomain::InitWriter(adios2::ADIOS* ptr_adios, - const SimulationParams& params, - bool is_resuming) { + const SimulationParams& params) { raise::ErrorIf( l_subdomain_indices().size() != 1, "Output for now is only supported for one subdomain per rank", @@ -103,11 +102,7 @@ namespace ntt { params.template get( "output." + std::string(type) + ".interval_time")); } - if (is_resuming and std::filesystem::exists(g_writer.fname())) { - g_writer.setMode(adios2::Mode::Append); - } else { - g_writer.writeAttrs(params); - } + g_writer.writeAttrs(params); } template @@ -715,8 +710,7 @@ namespace ntt { #define METADOMAIN_OUTPUT(S, M) \ template void Metadomain::InitWriter(adios2::ADIOS*, \ - const SimulationParams&, \ - bool); \ + const SimulationParams&); \ template auto Metadomain::Write( \ const SimulationParams&, \ timestep_t, \ diff --git a/src/framework/domain/stats.cpp b/src/framework/domain/stats.cpp index a252e975..6c5bb0ff 100644 --- a/src/framework/domain/stats.cpp +++ b/src/framework/domain/stats.cpp @@ -89,7 +89,6 @@ namespace ntt { } // some parameters const auto use_weights = params.template get("particles.use_weights"); - const auto inv_n0 = ONE / params.template get("scales.n0"); real_t buffer = static_cast(0); for (const auto& sp : specs) { @@ -186,8 +185,8 @@ namespace ntt { timestep_t finished_step, simtime_t current_time, simtime_t finished_time, - std::function&)> CustomStat) - -> bool { + std::function&)> + CustomStat) -> bool { if (not(params.template get("output.stats.enable") and g_stats_writer.shouldWrite(finished_step, finished_time))) { return false; @@ -272,16 +271,16 @@ namespace ntt { return true; } -#define METADOMAIN_STATS(S, M) \ - template void Metadomain::InitStatsWriter(const SimulationParams&, bool); \ - template auto Metadomain::WriteStats( \ - const SimulationParams&, \ - timestep_t, \ - timestep_t, \ - simtime_t, \ - simtime_t, \ - std::function&)>) \ - -> bool; +#define METADOMAIN_STATS(S, M) \ + template void Metadomain::InitStatsWriter(const SimulationParams&, bool); \ + template auto Metadomain::WriteStats( \ + const SimulationParams&, \ + timestep_t, \ + timestep_t, \ + simtime_t, \ + simtime_t, \ + std::function< \ + real_t(const std::string&, timestep_t, simtime_t, const Domain&)>) -> bool; METADOMAIN_STATS(SimEngine::SRPIC, metric::Minkowski) METADOMAIN_STATS(SimEngine::SRPIC, metric::Minkowski) diff --git a/src/framework/parameters.cpp b/src/framework/parameters.cpp index 40779d73..c9f612ce 100644 --- a/src/framework/parameters.cpp +++ b/src/framework/parameters.cpp @@ -609,6 +609,24 @@ namespace ntt { toml::find_or(toml_data, "checkpoint", "interval_time", -1.0)); set("checkpoint.keep", toml::find_or(toml_data, "checkpoint", "keep", defaults::checkpoint::keep)); + auto walltime_str = toml::find_or(toml_data, + "checkpoint", + "walltime", + defaults::checkpoint::walltime); + if (walltime_str.empty()) { + walltime_str = defaults::checkpoint::walltime; + } + set("checkpoint.walltime", walltime_str); + + const auto checkpoint_write_path = toml::find_or( + toml_data, + "checkpoint", + "write_path", + fmt::format(defaults::checkpoint::write_path.c_str(), + get("simulation.name").c_str())); + set("checkpoint.write_path", checkpoint_write_path); + set("checkpoint.read_path", + toml::find_or(toml_data, "checkpoint", "read_path", checkpoint_write_path)); /* [diagnostics] -------------------------------------------------------- */ set("diagnostics.interval", diff --git a/src/framework/parameters.h b/src/framework/parameters.h index 83192f44..723e860d 100644 --- a/src/framework/parameters.h +++ b/src/framework/parameters.h @@ -26,6 +26,8 @@ namespace ntt { SimulationParams() {} + SimulationParams(const SimulationParams&) = default; + SimulationParams& operator=(const SimulationParams& other) { vars = std::move(other.vars); promises = std::move(other.promises); diff --git a/src/framework/simulation.cpp b/src/framework/simulation.cpp index 6865140c..6735eda7 100644 --- a/src/framework/simulation.cpp +++ b/src/framework/simulation.cpp @@ -11,8 +11,6 @@ #include "utils/plog.h" #include "utils/toml.h" -#include "framework/parameters.h" - #include #include @@ -55,13 +53,24 @@ namespace ntt { m_params.setRawData(raw_params); timestep_t checkpoint_step = 0; + if (is_resuming) { logger::Checkpoint("Reading params from a checkpoint", HERE); - if (not std::filesystem::exists("checkpoints")) { + const auto checkpoint_write_path = toml::find_or( + raw_params, + "checkpoint", + "write_path", + fmt::format(defaults::checkpoint::write_path.c_str(), sim_name.c_str())); + const path_t checkpoint_read_path = toml::find_or( + raw_params, + "checkpoint", + "read_path", + checkpoint_write_path); + if (not std::filesystem::exists(checkpoint_read_path)) { raise::Fatal("No checkpoints found", HERE); } for (const auto& entry : - std::filesystem::directory_iterator("checkpoints")) { + std::filesystem::directory_iterator(checkpoint_read_path)) { const auto fname = entry.path().filename().string(); if (fname.find("step-") == 0) { const timestep_t step = std::stoi(fname.substr(5, fname.size() - 5 - 3)); @@ -70,17 +79,16 @@ namespace ntt { } } } - std::string checkpoint_inputfname = fmt::format( - "checkpoints/meta-%08lu.toml", - checkpoint_step); - if (not std::filesystem::exists(checkpoint_inputfname)) { + path_t checkpoint_metafname = checkpoint_read_path / + fmt::format("meta-%08lu.toml", checkpoint_step); + if (not std::filesystem::exists(checkpoint_metafname)) { raise::Fatal( fmt::format("metainformation for %lu not found", checkpoint_step), HERE); - checkpoint_inputfname = inputfname; + checkpoint_metafname = inputfname; } logger::Checkpoint(fmt::format("Using %08lu", checkpoint_step), HERE); - const auto raw_checkpoint_params = toml::parse(checkpoint_inputfname); + const auto raw_checkpoint_params = toml::parse(checkpoint_metafname); const auto start_time = toml::find(raw_checkpoint_params, "metadata", "time"); diff --git a/src/framework/simulation.h b/src/framework/simulation.h index 3cc66499..33750030 100644 --- a/src/framework/simulation.h +++ b/src/framework/simulation.h @@ -22,9 +22,6 @@ #include "framework/parameters.h" -#include -#include - namespace ntt { class Simulation { diff --git a/src/global/defaults.h b/src/global/defaults.h index d673a503..9513493b 100644 --- a/src/global/defaults.h +++ b/src/global/defaults.h @@ -67,8 +67,10 @@ namespace ntt::defaults { } // namespace output namespace checkpoint { - const timestep_t interval = 1000; - const int keep = 2; + const timestep_t interval = 1000; + const int keep = 2; + const std::string walltime = "00:00:00"; + const std::string write_path = "%s.ckpt"; } // namespace checkpoint namespace diag { diff --git a/src/global/global.h b/src/global/global.h index 42114a1d..04552db7 100644 --- a/src/global/global.h +++ b/src/global/global.h @@ -93,6 +93,7 @@ #define GLOBAL_GLOBAL_H #include +#include #include #include #include @@ -342,12 +343,14 @@ template using vec_t = tuple_t; // time/duration +using duration_t = double; +using simtime_t = double; +using timestep_t = std::size_t; +using ncells_t = std::size_t; +using npart_t = unsigned long int; + +// walltime using timestamp_t = std::chrono::time_point; -using duration_t = double; -using simtime_t = double; -using timestep_t = std::size_t; -using ncells_t = std::size_t; -using npart_t = unsigned long int; // index/number using index_t = const std::size_t; @@ -355,6 +358,9 @@ using idx_t = unsigned short; using spidx_t = unsigned short; using dim_t = unsigned short; +// utility +using path_t = std::filesystem::path; + using range_tuple_t = std::pair; template diff --git a/src/global/utils/param_container.cpp b/src/global/utils/param_container.cpp index e1dd57c9..54d8591b 100644 --- a/src/global/utils/param_container.cpp +++ b/src/global/utils/param_container.cpp @@ -13,7 +13,6 @@ #include #include #include - #include #include #include @@ -32,8 +31,8 @@ namespace prm { } template - auto write(adios2::IO& io, const std::string& name, T var) -> decltype(void(T()), - void()) { + auto write(adios2::IO& io, const std::string& name, T var) + -> decltype(void(T()), void()) { io.DefineAttribute(name, var); } @@ -48,8 +47,8 @@ namespace prm { } template - auto write_pair(adios2::IO& io, const std::string& name, std::pair var) -> - typename std::enable_if::value, void>::type { + auto write_pair(adios2::IO& io, const std::string& name, std::pair var) + -> typename std::enable_if::value, void>::type { std::vector var_str; var_str.push_back(var.first.to_string()); var_str.push_back(var.second.to_string()); @@ -57,9 +56,8 @@ namespace prm { } template - auto write_pair(adios2::IO& io, - const std::string& name, - std::pair var) -> decltype(void(T()), void()) { + auto write_pair(adios2::IO& io, const std::string& name, std::pair var) + -> decltype(void(T()), void()) { std::vector var_vec; var_vec.push_back(var.first); var_vec.push_back(var.second); @@ -77,9 +75,8 @@ namespace prm { } template - auto write_vec(adios2::IO& io, - const std::string& name, - std::vector var) -> decltype(void(T()), void()) { + auto write_vec(adios2::IO& io, const std::string& name, std::vector var) + -> decltype(void(T()), void()) { io.DefineAttribute(name, var.data(), var.size()); } @@ -99,8 +96,8 @@ namespace prm { template auto write_vec_pair(adios2::IO& io, const std::string& name, - std::vector> var) -> decltype(void(T()), - void()) { + std::vector> var) + -> decltype(void(T()), void()) { std::vector var_vec; for (const auto& v : var) { var_vec.push_back(v.first); @@ -126,8 +123,8 @@ namespace prm { template auto write_vec_vec(adios2::IO& io, const std::string& name, - std::vector> var) -> decltype(void(T()), - void()) { + std::vector> var) + -> decltype(void(T()), void()) { std::vector var_vec; for (const auto& vec : var) { for (const auto& v : vec) { @@ -328,9 +325,18 @@ namespace prm { register_write_function_for_dict(); for (auto& [key, value] : allVars()) { + // @TODO: add particles.species support in attrs + if (key == "particles.species") { + continue; + } try { write_any(io, key, value); } catch (const std::exception& e) { + raise::Warning( + fmt::format("Failed to write parameter '%s', skipping. Error msg: %s", + key.c_str(), + e.what()), + HERE); continue; } } @@ -338,4 +344,4 @@ namespace prm { } // namespace prm -#endif +#endif // OUTPUT_ENABLED diff --git a/src/global/utils/plog.h b/src/global/utils/plog.h index 2422c7cf..7713a372 100644 --- a/src/global/utils/plog.h +++ b/src/global/utils/plog.h @@ -80,9 +80,9 @@ namespace logger { static plog::RollingFileAppender errfileAppender( errfile_name.c_str()); auto log_severity = plog::verbose; - if (fmt::toLower(log_level) == "WARNING") { + if (fmt::toLower(log_level) == "warning") { log_severity = plog::warning; - } else if (fmt::toLower(log_level) == "ERROR") { + } else if (fmt::toLower(log_level) == "error") { log_severity = plog::error; } plog::init(log_severity, &logfileAppender); diff --git a/src/global/utils/tools.h b/src/global/utils/tools.h index 95765e2d..27c7e22c 100644 --- a/src/global/utils/tools.h +++ b/src/global/utils/tools.h @@ -18,11 +18,11 @@ #include "global.h" -#include "arch/kokkos_aliases.h" #include "utils/comparators.h" #include "utils/error.h" #include "utils/numeric.h" +#include #include #include #include @@ -60,8 +60,8 @@ namespace tools { * @return Tensor product of list */ template - inline auto TensorProduct( - const std::vector>& list) -> std::vector> { + inline auto TensorProduct(const std::vector>& list) + -> std::vector> { std::vector> result = { {} }; for (const auto& sublist : list) { std::vector> temp; @@ -81,8 +81,8 @@ namespace tools { * @param ndomains Number of domains * @param ncells Number of cells */ - inline auto decompose1D(unsigned int ndomains, - ncells_t ncells) -> std::vector { + inline auto decompose1D(unsigned int ndomains, ncells_t ncells) + -> std::vector { auto size = (ncells_t)((double)ncells / (double)ndomains); auto ncells_domain = std::vector(ndomains, size); for (auto i { 0u }; i < ncells - size * ndomains; ++i) { @@ -107,10 +107,8 @@ namespace tools { * @param s1 Proportion of the first dimension * @param s2 Proportion of the second dimension */ - inline auto divideInProportions2D( - unsigned int ntot, - unsigned int s1, - unsigned int s2) -> std::tuple { + inline auto divideInProportions2D(unsigned int ntot, unsigned int s1, unsigned int s2) + -> std::tuple { auto n1 = (unsigned int)(std::sqrt((double)ntot * (double)s1 / (double)s2)); if (n1 == 0) { return { 1, ntot }; @@ -132,11 +130,11 @@ namespace tools { * @param s2 Proportion of the second dimension * @param s3 Proportion of the third dimension */ - inline auto divideInProportions3D( - unsigned int ntot, - unsigned int s1, - unsigned int s2, - unsigned int s3) -> std::tuple { + inline auto divideInProportions3D(unsigned int ntot, + unsigned int s1, + unsigned int s2, + unsigned int s3) + -> std::tuple { auto n1 = (unsigned int)(std::cbrt( (double)ntot * (double)(SQR(s1)) / (double)(s2 * s3))); if (n1 > ntot) { @@ -165,10 +163,10 @@ namespace tools { * * @note If decomposition has -1, it will be calculated automatically */ - inline auto Decompose( - unsigned int ndomains, - const std::vector& ncells, - const std::vector& decomposition) -> std::vector> { + inline auto Decompose(unsigned int ndomains, + const std::vector& ncells, + const std::vector& decomposition) + -> std::vector> { const auto dimension = ncells.size(); raise::ErrorIf(dimension != decomposition.size(), "Decomposition error: dimension != decomposition.size", @@ -275,6 +273,11 @@ namespace tools { } } + /** + * Class for tracking the passage of time either in steps, physical time units, or walltime + * + * @note Primarily used for writing checkpoints and all types of outputs at specified intervals + */ class Tracker { bool m_initialized { false }; @@ -283,31 +286,55 @@ namespace tools { simtime_t m_interval_time; bool m_use_time; + timestamp_t m_start_walltime; + timestamp_t m_end_walltime; + bool m_walltime_pending { false }; + simtime_t m_last_output_time { -1.0 }; public: Tracker() = default; - Tracker(const std::string& type, timestep_t interval, simtime_t interval_time) - : m_initialized { true } - , m_type { type } - , m_interval { interval } - , m_interval_time { interval_time } - , m_use_time { interval_time > 0.0 } {} + Tracker(const std::string& type, + timestep_t interval, + simtime_t interval_time, + const std::string& end_walltime = "", + const timestamp_t& start_walltime = std::chrono::system_clock::now()) { + init(type, interval, interval_time, end_walltime, start_walltime); + } ~Tracker() = default; - void init(const std::string& type, timestep_t interval, simtime_t interval_time) { - m_type = type; - m_interval = interval; - m_interval_time = interval_time; - m_use_time = interval_time > 0.0; - m_initialized = true; + void init(const std::string& type, + timestep_t interval, + simtime_t interval_time, + const std::string& end_walltime = "", + const timestamp_t& start_walltime = std::chrono::system_clock::now()) { + m_initialized = true; + m_type = type; + m_interval = interval; + m_interval_time = interval_time; + m_use_time = interval_time > 0.0; + m_start_walltime = start_walltime; + if (not(end_walltime.empty() or end_walltime == "00:00:00")) { + m_walltime_pending = true; + raise::ErrorIf(end_walltime.size() != 8, + "invalid end walltime format, expected HH:MM:SS", + HERE); + m_end_walltime = m_start_walltime + + std::chrono::hours(std::stoi(end_walltime.substr(0, 2))) + + std::chrono::minutes(std::stoi(end_walltime.substr(3, 2))) + + std::chrono::seconds(std::stoi(end_walltime.substr(6, 2))); + } } auto shouldWrite(timestep_t step, simtime_t time) -> bool { raise::ErrorIf(!m_initialized, "Tracker not initialized", HERE); - if (m_use_time) { + if (m_walltime_pending and + (std::chrono::system_clock::now() > m_end_walltime)) { + m_walltime_pending = false; + return true; + } else if (m_use_time) { if ((m_last_output_time < 0) or (time - m_last_output_time >= m_interval_time)) { m_last_output_time = time; diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 0a5dc616..a520616b 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -491,8 +491,8 @@ namespace kernel::bc { "Invalid component index"); ndfield_t Fld; - const BCTags tags; const std::size_t i_edge; + const BCTags tags; ConductorBoundaries_kernel(ndfield_t Fld, std::size_t i_edge, BCTags tags) : Fld { Fld } diff --git a/src/kernels/tests/deposit.cpp b/src/kernels/tests/deposit.cpp index e6967eb1..1570937b 100644 --- a/src/kernels/tests/deposit.cpp +++ b/src/kernels/tests/deposit.cpp @@ -55,7 +55,12 @@ void testDeposit(const std::vector& res, errorIf(res.size() != M::Dim, "res.size() != M::Dim"); using namespace ntt; - M metric { res, ext, params }; + auto extents = ext; + if constexpr (M::CoordType != Coord::Cart) { + extents.emplace_back(ZERO, (real_t)(constant::PI)); + } + + M metric { res, extents, params }; const auto nx1 = res[0]; const auto nx2 = res[1]; @@ -170,59 +175,49 @@ auto main(int argc, char* argv[]) -> int { using namespace ntt; using namespace metric; - testDeposit, SimEngine::SRPIC>( - { - 10, - 10 - }, - { { 0.0, 55.0 }, { 0.0, 55.0 } }, - {}, - 500); - - testDeposit, SimEngine::SRPIC>( - { - 10, - 10 - }, - { { 1.0, 100.0 } }, - {}, - 500); - - testDeposit, SimEngine::SRPIC>( - { - 10, - 10 + const auto res = std::vector { 10, 10 }; + const auto r_extent = boundaries_t { + { 0.0, 100.0 } + }; + const auto xy_extent = boundaries_t { + { 0.0, 55.0 }, + { 0.0, 55.0 } + }; + + testDeposit, SimEngine::SRPIC>(res, xy_extent, {}, 500); + + testDeposit, SimEngine::SRPIC>(res, r_extent, {}, 500); + + testDeposit, SimEngine::SRPIC>(res, + r_extent, + { + { "r0", 0.0 }, + { "h", 0.25 } }, - { { 1.0, 100.0 } }, - { { "r0", 0.0 }, { "h", 0.25 } }, - 500); - - testDeposit, SimEngine::GRPIC>( - { - 10, - 10 + 500); + + testDeposit, SimEngine::GRPIC>(res, + r_extent, + { + { "a", 0.9 } }, - { { 1.0, 100.0 } }, - { { "a", 0.9 } }, - 500); - - testDeposit, SimEngine::GRPIC>( - { - 10, - 10 + 500); + + testDeposit, SimEngine::GRPIC>(res, + r_extent, + { + { "r0", 0.0 }, + { "h", 0.25 }, + { "a", 0.9 } }, - { { 1.0, 100.0 } }, - { { "r0", 0.0 }, { "h", 0.25 }, { "a", 0.9 } }, - 500); - - testDeposit, SimEngine::GRPIC>( - { - 10, - 10 + 500); + + testDeposit, SimEngine::GRPIC>(res, + r_extent, + { + { "a", 0.9 } }, - { { 1.0, 100.0 } }, - { { "a", 0.9 } }, - 500); + 500); } catch (std::exception& e) { std::cerr << e.what() << std::endl; diff --git a/src/kernels/tests/digital_filter.cpp b/src/kernels/tests/digital_filter.cpp index 03a63753..30059e47 100644 --- a/src/kernels/tests/digital_filter.cpp +++ b/src/kernels/tests/digital_filter.cpp @@ -20,7 +20,6 @@ #include #include #include -#include #include void errorIf(bool condition, const std::string& message) { @@ -34,15 +33,17 @@ void testFilter(const std::vector& res, const boundaries_t& ext, const std::map& params = {}) { static_assert(M::Dim == 2); - errorIf(res.size() != M::Dim, "res.size() != M::Dim"); + errorIf(res.size() != static_cast(M::Dim), "res.size() != M::Dim"); using namespace ntt; auto boundaries = boundaries_t {}; + auto extents = ext; if constexpr (M::CoordType != Coord::Cart) { boundaries = { { FldsBC::CUSTOM, FldsBC::CUSTOM }, { FldsBC::AXIS, FldsBC::AXIS } }; + extents.emplace_back(ZERO, (real_t)(constant::PI)); } else { boundaries = { { FldsBC::PERIODIC, FldsBC::PERIODIC }, @@ -50,7 +51,7 @@ void testFilter(const std::vector& res, }; } - M metric { res, ext, params }; + M metric { res, extents, params }; const auto nx1 = res[0]; const auto nx2 = res[1]; @@ -133,53 +134,45 @@ auto main(int argc, char* argv[]) -> int { using namespace ntt; using namespace metric; - testFilter>( - { - 10, - 10 - }, - { { 0.0, 55.0 }, { 0.0, 55.0 } }, - {}); - - testFilter>( - { - 10, - 10 - }, - { { 1.0, 100.0 } }, - {}); - - testFilter>( - { - 10, - 10 - }, - { { 1.0, 100.0 } }, - { { "r0", 0.0 }, { "h", 0.25 } }); - - testFilter>( - { - 10, - 10 - }, - { { 1.0, 100.0 } }, - { { "a", 0.9 } }); - - testFilter>( - { - 10, - 10 - }, - { { 1.0, 100.0 } }, - { { "r0", 0.0 }, { "h", 0.25 }, { "a", 0.9 } }); - - testFilter>( - { - 10, - 10 - }, - { { 1.0, 100.0 } }, - { { "a", 0.9 } }); + const auto res = std::vector { 10, 10 }; + const auto r_extent = boundaries_t { + { 0.0, 100.0 } + }; + const auto xy_extent = boundaries_t { + { 0.0, 55.0 }, + { 0.0, 55.0 } + }; + + testFilter>(res, xy_extent, {}); + + testFilter>(res, r_extent, {}); + + testFilter>(res, + r_extent, + { + { "r0", 0.0 }, + { "h", 0.25 } + }); + + testFilter>(res, + r_extent, + { + { "a", 0.9 } + }); + + testFilter>(res, + r_extent, + { + { "r0", 0.0 }, + { "h", 0.25 }, + { "a", 0.9 } + }); + + testFilter>(res, + r_extent, + { + { "a", 0.9 } + }); } catch (std::exception& e) { std::cerr << e.what() << std::endl; diff --git a/src/kernels/tests/prtl_bc.cpp b/src/kernels/tests/prtl_bc.cpp index 14c1a9f5..6e49f8ba 100644 --- a/src/kernels/tests/prtl_bc.cpp +++ b/src/kernels/tests/prtl_bc.cpp @@ -53,35 +53,45 @@ void testPeriodicBC(const std::vector& res, const auto NoGCA = false; const auto NoExtForce = false; - boundaries_t extent; - extent = ext; - const auto sx = static_cast(extent[0].second - extent[0].first); - const auto sy = static_cast( - extent.size() > 1 ? extent[1].second - extent[1].first : 0); - const auto sz = static_cast( - extent.size() > 2 ? extent[2].second - extent[2].first : 0); + real_t sx = ZERO, sy = ZERO, sz = ZERO; + if (ext.size() > 0) { + sx = static_cast(ext.at(0).second - ext.at(0).first); + } + if (ext.size() > 1) { + sy = static_cast(ext.at(1).second - ext.at(1).first); + } + if (ext.size() > 2) { + sz = static_cast(ext.at(2).second - ext.at(2).first); + } - M metric { res, extent, params }; + M metric { res, ext, params }; - const int nx1 = res[0]; - const int nx2 = res[1]; - const int nx3 = res[2]; + int nx1 = 0, nx2 = 0, nx3 = 0; + if (res.size() > 0) { + nx1 = static_cast(res.at(0)); + } + if (res.size() > 1) { + nx2 = static_cast(res.at(1)); + } + if (res.size() > 2) { + nx3 = static_cast(res.at(2)); + } - const real_t dt = 0.1 * (extent[0].second - extent[0].first) / sx; + const real_t dt = 0.1 * (ext.at(0).second - ext.at(0).first) / sx; const real_t coeff = HALF * dt; ndfield_t emfield; if constexpr (M::Dim == Dim::_1D) { - emfield = ndfield_t { "emfield", res[0] + 2 * N_GHOSTS }; + emfield = ndfield_t { "emfield", res.at(0) + 2 * N_GHOSTS }; } else if constexpr (M::Dim == Dim::_2D) { emfield = ndfield_t { "emfield", - res[0] + 2 * N_GHOSTS, - res[1] + 2 * N_GHOSTS }; + res.at(0) + 2 * N_GHOSTS, + res.at(1) + 2 * N_GHOSTS }; } else { emfield = ndfield_t { "emfield", - res[0] + 2 * N_GHOSTS, - res[1] + 2 * N_GHOSTS, - res[2] + 2 * N_GHOSTS }; + res.at(0) + 2 * N_GHOSTS, + res.at(1) + 2 * N_GHOSTS, + res.at(2) + 2 * N_GHOSTS }; } const short sp_idx = 1; @@ -106,18 +116,26 @@ void testPeriodicBC(const std::vector& res, array_t phi; // init parameters of prtl #1 - real_t xi_1 = 0.515460 * sx + extent[0].first; - real_t yi_1 = 0.340680 * sy + extent[1].first; - real_t zi_1 = 0.940722 * sz + extent[2].first; + real_t xi_1 = ZERO, yi_1 = ZERO, zi_1 = ZERO; + real_t xi_2 = ZERO, yi_2 = ZERO, zi_2 = ZERO; + if constexpr (M::Dim == Dim::_1D or M::Dim == Dim::_2D or M::Dim == Dim::_3D) { + xi_1 = 0.515460 * sx + ext.at(0).first; + xi_2 = 0.149088 * sx + ext.at(0).first; + } + if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { + yi_1 = 0.340680 * sy + ext.at(1).first; + yi_2 = 0.997063 * sy + ext.at(1).first; + } + if constexpr (M::Dim == Dim::_3D) { + zi_1 = 0.940722 * sz + ext.at(2).first; + zi_2 = 0.607354 * sz + ext.at(2).first; + } real_t ux_1 = 0.569197; real_t uy_1 = 0.716085; real_t uz_1 = 0.760101; real_t gamma_1 = math::sqrt(1.0 + SQR(ux_1) + SQR(uy_1) + SQR(uz_1)); // init parameters of prtl #2 - real_t xi_2 = 0.149088 * sx + extent[0].first; - real_t yi_2 = 0.997063 * sy + extent[1].first; - real_t zi_2 = 0.607354 * sz + extent[2].first; real_t ux_2 = -0.872069; real_t uy_2 = 0.0484461; real_t uz_2 = -0.613575; @@ -142,21 +160,28 @@ void testPeriodicBC(const std::vector& res, xi[2] = zi_1; } metric.template convert_xyz(xi, xCd); - put_value(i1, static_cast(xCd[0]), prtl_idx); - put_value(i2, static_cast(xCd[1]), prtl_idx); - put_value(i3, static_cast(xCd[2]), prtl_idx); - put_value(dx1, - static_cast(xCd[0]) - - static_cast(static_cast(xCd[0])), - prtl_idx); - put_value(dx2, - static_cast(xCd[1]) - - static_cast(static_cast(xCd[1])), - prtl_idx); - put_value(dx3, - static_cast(xCd[2]) - - static_cast(static_cast(xCd[2])), - prtl_idx); + if constexpr (M::PrtlDim == Dim::_1D or M::PrtlDim == Dim::_2D or + M::PrtlDim == Dim::_3D) { + put_value(i1, static_cast(xCd[0]), prtl_idx); + put_value(dx1, + static_cast(xCd[0]) - + static_cast(static_cast(xCd[0])), + prtl_idx); + } + if constexpr (M::PrtlDim == Dim::_2D or M::PrtlDim == Dim::_3D) { + put_value(i2, static_cast(xCd[1]), prtl_idx); + put_value(dx2, + static_cast(xCd[1]) - + static_cast(static_cast(xCd[1])), + prtl_idx); + } + if constexpr (M::PrtlDim == Dim::_3D) { + put_value(i3, static_cast(xCd[2]), prtl_idx); + put_value(dx3, + static_cast(xCd[2]) - + static_cast(static_cast(xCd[2])), + prtl_idx); + } put_value(ux1, ux_1, prtl_idx); put_value(ux2, uy_1, prtl_idx); put_value(ux3, uz_1, prtl_idx); @@ -177,21 +202,28 @@ void testPeriodicBC(const std::vector& res, xi[2] = zi_2; } metric.template convert_xyz(xi, xCd); - put_value(i1, static_cast(xCd[0]), prtl_idx); - put_value(i2, static_cast(xCd[1]), prtl_idx); - put_value(i3, static_cast(xCd[2]), prtl_idx); - put_value(dx1, - static_cast(xCd[0]) - - static_cast(static_cast(xCd[0])), - prtl_idx); - put_value(dx2, - static_cast(xCd[1]) - - static_cast(static_cast(xCd[1])), - prtl_idx); - put_value(dx3, - static_cast(xCd[2]) - - static_cast(static_cast(xCd[2])), - prtl_idx); + if constexpr (M::PrtlDim == Dim::_1D or M::PrtlDim == Dim::_2D or + M::PrtlDim == Dim::_3D) { + put_value(i1, static_cast(xCd[0]), prtl_idx); + put_value(dx1, + static_cast(xCd[0]) - + static_cast(static_cast(xCd[0])), + prtl_idx); + } + if constexpr (M::PrtlDim == Dim::_2D or M::PrtlDim == Dim::_3D) { + put_value(i2, static_cast(xCd[1]), prtl_idx); + put_value(dx2, + static_cast(xCd[1]) - + static_cast(static_cast(xCd[1])), + prtl_idx); + } + if constexpr (M::PrtlDim == Dim::_3D) { + put_value(i3, static_cast(xCd[2]), prtl_idx); + put_value(dx3, + static_cast(xCd[2]) - + static_cast(static_cast(xCd[2])), + prtl_idx); + } put_value(ux1, ux_2, prtl_idx); put_value(ux2, uy_2, prtl_idx); put_value(ux3, uz_2, prtl_idx); @@ -268,16 +300,16 @@ void testPeriodicBC(const std::vector& res, if constexpr (M::Dim == Dim::_1D or M::Dim == Dim::_2D or M::Dim == Dim::_3D) { xi_1 += dt * ux_1 / gamma_1; xi_2 += dt * ux_2 / gamma_2; - if (xi_1 >= extent[0].second) { + if (xi_1 >= ext.at(0).second) { xi_1 -= sx; } - if (xi_1 < extent[0].first) { + if (xi_1 < ext.at(0).first) { xi_1 += sx; } - if (xi_2 >= extent[0].second) { + if (xi_2 >= ext.at(0).second) { xi_2 -= sx; } - if (xi_2 < extent[0].first) { + if (xi_2 < ext.at(0).first) { xi_2 += sx; } errorIf(not equal(xPh_1[0] / sx, @@ -290,16 +322,16 @@ void testPeriodicBC(const std::vector& res, if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { yi_1 += dt * uy_1 / gamma_1; yi_2 += dt * uy_2 / gamma_2; - if (yi_1 >= extent[1].second) { + if (yi_1 >= ext.at(1).second) { yi_1 -= sy; } - if (yi_1 < extent[1].first) { + if (yi_1 < ext.at(1).first) { yi_1 += sy; } - if (yi_2 >= extent[1].second) { + if (yi_2 >= ext.at(1).second) { yi_2 -= sy; } - if (yi_2 < extent[1].first) { + if (yi_2 < ext.at(1).first) { yi_2 += sy; } errorIf(not equal(xPh_1[1] / sy, @@ -312,16 +344,16 @@ void testPeriodicBC(const std::vector& res, if constexpr (M::Dim == Dim::_3D) { zi_1 += dt * uz_1 / gamma_1; zi_2 += dt * uz_2 / gamma_2; - if (zi_1 >= extent[2].second) { + if (zi_1 >= ext.at(2).second) { zi_1 -= sz; } - if (zi_1 < extent[2].first) { + if (zi_1 < ext.at(2).first) { zi_1 += sz; } - if (zi_2 >= extent[2].second) { + if (zi_2 >= ext.at(2).second) { zi_2 -= sz; } - if (zi_2 < extent[2].first) { + if (zi_2 < ext.at(2).first) { zi_2 += sz; } errorIf(not equal(xPh_1[2] / sz, diff --git a/src/kernels/tests/reduced_stats.cpp b/src/kernels/tests/reduced_stats.cpp index caae51cc..499c24aa 100644 --- a/src/kernels/tests/reduced_stats.cpp +++ b/src/kernels/tests/reduced_stats.cpp @@ -66,8 +66,7 @@ template auto compute_field_stat(const M& metric, const ndfield_t& em, const ndfield_t& j, - const range_t& range, - ncells_t num_cells) -> real_t { + const range_t& range) -> real_t { real_t buff = ZERO; Kokkos::parallel_reduce("ReduceFields", range, @@ -113,13 +112,11 @@ void testReducedStats(const std::vector& res, ndfield_t EM; ndfield_t J; range_t cell_range; - ncells_t num_cells; if constexpr (M::Dim == Dim::_1D) { EM = ndfield_t { "EM", res[0] + 2 * N_GHOSTS }; J = ndfield_t { "J", res[0] + 2 * N_GHOSTS }; cell_range = { N_GHOSTS, res[0] + N_GHOSTS }; - num_cells = res[0]; put_value(EM, values[0], em::ex1); put_value(EM, values[1], em::ex2); @@ -140,7 +137,6 @@ void testReducedStats(const std::vector& res, { N_GHOSTS, N_GHOSTS }, { res[0] + N_GHOSTS, res[1] + N_GHOSTS } }; - num_cells = res[0] * res[1]; put_value(EM, values[0], em::ex1); put_value(EM, values[1], em::ex2); @@ -167,7 +163,6 @@ void testReducedStats(const std::vector& res, { N_GHOSTS, N_GHOSTS, N_GHOSTS }, { res[0] + N_GHOSTS, res[1] + N_GHOSTS, res[2] + N_GHOSTS } }; - num_cells = res[0] * res[1] * res[2]; put_value(EM, values[0], em::ex1); put_value(EM, values[1], em::ex2); @@ -186,8 +181,7 @@ void testReducedStats(const std::vector& res, const auto Ex_Sq = compute_field_stat(metric, EM, J, - cell_range, - num_cells); + cell_range); raise::ErrorIf(not almost_equal(Ex_Sq, (real_t)(1), acc), "Ex_Sq does not match expected value", HERE); @@ -197,8 +191,7 @@ void testReducedStats(const std::vector& res, const auto Ey_Sq = compute_field_stat(metric, EM, J, - cell_range, - num_cells); + cell_range); raise::ErrorIf(not almost_equal(Ey_Sq, (real_t)(4), acc), "Ey_Sq does not match expected value", HERE); @@ -208,8 +201,7 @@ void testReducedStats(const std::vector& res, const auto Ez_Sq = compute_field_stat(metric, EM, J, - cell_range, - num_cells); + cell_range); raise::ErrorIf(not almost_equal(Ez_Sq, (real_t)(9), acc), "Ez_Sq does not match expected value", HERE); @@ -219,8 +211,7 @@ void testReducedStats(const std::vector& res, const auto Bx_Sq = compute_field_stat(metric, EM, J, - cell_range, - num_cells); + cell_range); raise::ErrorIf(not almost_equal(Bx_Sq, (real_t)(1), acc), "Bx_Sq does not match expected value", HERE); @@ -230,8 +221,7 @@ void testReducedStats(const std::vector& res, const auto By_Sq = compute_field_stat(metric, EM, J, - cell_range, - num_cells); + cell_range); raise::ErrorIf(not almost_equal(By_Sq, (real_t)(4), acc), "By_Sq does not match expected value", HERE); @@ -241,8 +231,7 @@ void testReducedStats(const std::vector& res, const auto Bz_Sq = compute_field_stat(metric, EM, J, - cell_range, - num_cells); + cell_range); raise::ErrorIf(not almost_equal(Bz_Sq, (real_t)(9), acc), "Bz_Sq does not match expected value", HERE); @@ -252,8 +241,7 @@ void testReducedStats(const std::vector& res, const auto ExB_x = compute_field_stat(metric, EM, J, - cell_range, - num_cells); + cell_range); raise::ErrorIf(not almost_equal(ExB_x, (real_t)(12), acc), "ExB_x does not match expected value", HERE); @@ -263,8 +251,7 @@ void testReducedStats(const std::vector& res, const auto ExB_y = compute_field_stat(metric, EM, J, - cell_range, - num_cells); + cell_range); raise::ErrorIf(not almost_equal(ExB_y, (real_t)(-6), acc), "ExB_y does not match expected value", HERE); @@ -274,8 +261,7 @@ void testReducedStats(const std::vector& res, const auto ExB_z = compute_field_stat(metric, EM, J, - cell_range, - num_cells); + cell_range); raise::ErrorIf(not almost_equal(ExB_z, (real_t)(0), acc), "ExB_z does not match expected value", HERE); @@ -285,8 +271,7 @@ void testReducedStats(const std::vector& res, const auto JdotE = compute_field_stat(metric, EM, J, - cell_range, - num_cells); + cell_range); raise::ErrorIf(not almost_equal(JdotE, (real_t)(11), acc), "JdotE does not match expected value", HERE); diff --git a/src/metrics/kerr_schild.h b/src/metrics/kerr_schild.h index 5edd43f2..74f5f1e4 100644 --- a/src/metrics/kerr_schild.h +++ b/src/metrics/kerr_schild.h @@ -72,8 +72,8 @@ namespace metric { using MetricBase::nx3; using MetricBase::set_dxMin; - KerrSchild(std::vector res, - boundaries_t ext, + KerrSchild(const std::vector& res, + const boundaries_t& ext, const std::map& params) : MetricBase { res, ext } , a { params.at("a") } diff --git a/src/metrics/kerr_schild_0.h b/src/metrics/kerr_schild_0.h index 6e4f86c1..e7cfc825 100644 --- a/src/metrics/kerr_schild_0.h +++ b/src/metrics/kerr_schild_0.h @@ -53,8 +53,8 @@ namespace metric { using MetricBase::nx3; using MetricBase::set_dxMin; - KerrSchild0(std::vector res, - boundaries_t ext, + KerrSchild0(const std::vector& res, + const boundaries_t& ext, const std::map& = {}) : MetricBase { res, ext } , dr { (x1_max - x1_min) / nx1 } diff --git a/src/metrics/metric_base.h b/src/metrics/metric_base.h index b386d46a..8b0e5cae 100644 --- a/src/metrics/metric_base.h +++ b/src/metrics/metric_base.h @@ -46,12 +46,38 @@ #include "global.h" +#include "utils/error.h" #include "utils/numeric.h" #include namespace metric { + namespace { + template + auto getNXi(const std::vector& res) -> real_t { + if constexpr (i >= static_cast(D)) { + return ONE; + } else { + raise::ErrorIf(res.size() <= i, "Invalid res size provided to metric", HERE); + return static_cast(res.at(i)); + } + }; + + template + auto getExtent(const boundaries_t& ext) -> real_t { + if constexpr (i >= static_cast(D)) { + return ZERO; + } else { + raise::ErrorIf(ext.size() <= i, "Invalid ext size provided to metric", HERE); + return min ? ext.at(i).first : ext.at(i).second; + } + }; + + constexpr bool XMin = true; + constexpr bool XMax = false; + }; // namespace + /** * Virtual parent metric class template: h_ij * Coordinates vary from `0` to `nx1` ... (code units) @@ -61,16 +87,16 @@ namespace metric { static constexpr bool is_metric { true }; static constexpr Dimension Dim { D }; - MetricBase(std::vector res, boundaries_t ext) - : nx1 { res.size() > 0 ? (real_t)(res[0]) : ONE } - , nx2 { res.size() > 1 ? (real_t)(res[1]) : ONE } - , nx3 { res.size() > 2 ? (real_t)(res[2]) : ONE } - , x1_min { res.size() > 0 ? ext[0].first : ZERO } - , x1_max { res.size() > 0 ? ext[0].second : ZERO } - , x2_min { res.size() > 1 ? ext[1].first : ZERO } - , x2_max { res.size() > 1 ? ext[1].second : ZERO } - , x3_min { res.size() > 2 ? ext[2].first : ZERO } - , x3_max { res.size() > 2 ? ext[2].second : ZERO } {} + MetricBase(const std::vector& res, const boundaries_t& ext) + : nx1 { getNXi(res) } + , nx2 { getNXi(res) } + , nx3 { getNXi(res) } + , x1_min { getExtent(ext) } + , x1_max { getExtent(ext) } + , x2_min { getExtent(ext) } + , x2_max { getExtent(ext) } + , x3_min { getExtent(ext) } + , x3_max { getExtent(ext) } {} ~MetricBase() = default; diff --git a/src/metrics/minkowski.h b/src/metrics/minkowski.h index c1eda49d..96c665a3 100644 --- a/src/metrics/minkowski.h +++ b/src/metrics/minkowski.h @@ -47,8 +47,8 @@ namespace metric { using MetricBase::nx3; using MetricBase::set_dxMin; - Minkowski(std::vector res, - boundaries_t ext, + Minkowski(const std::vector& res, + const boundaries_t& ext, const std::map& = {}) : MetricBase { res, ext } , dx { (x1_max - x1_min) / nx1 } diff --git a/src/metrics/qkerr_schild.h b/src/metrics/qkerr_schild.h index 7d327b89..d39ddd72 100644 --- a/src/metrics/qkerr_schild.h +++ b/src/metrics/qkerr_schild.h @@ -72,8 +72,8 @@ namespace metric { using MetricBase::nx3; using MetricBase::set_dxMin; - QKerrSchild(std::vector res, - boundaries_t ext, + QKerrSchild(const std::vector& res, + const boundaries_t& ext, const std::map& params) : MetricBase { res, ext } , a { params.at("a") } diff --git a/src/metrics/qspherical.h b/src/metrics/qspherical.h index 4a5a9d38..534b1f76 100644 --- a/src/metrics/qspherical.h +++ b/src/metrics/qspherical.h @@ -56,8 +56,8 @@ namespace metric { using MetricBase::nx3; using MetricBase::set_dxMin; - QSpherical(std::vector res, - boundaries_t ext, + QSpherical(const std::vector& res, + const boundaries_t& ext, const std::map& params) : MetricBase { res, ext } , r0 { params.at("r0") } diff --git a/src/metrics/spherical.h b/src/metrics/spherical.h index 05ec6b01..5b5f8705 100644 --- a/src/metrics/spherical.h +++ b/src/metrics/spherical.h @@ -51,8 +51,8 @@ namespace metric { using MetricBase::nx3; using MetricBase::set_dxMin; - Spherical(std::vector res, - boundaries_t ext, + Spherical(const std::vector& res, + const boundaries_t& ext, const std::map& = {}) : MetricBase { res, ext } , dr((x1_max - x1_min) / nx1) diff --git a/src/output/writer.cpp b/src/output/writer.cpp index a259ba88..9a96f9f5 100644 --- a/src/output/writer.cpp +++ b/src/output/writer.cpp @@ -39,7 +39,7 @@ namespace out { m_io.DefineVariable("Step"); m_io.DefineVariable("Time"); - m_fname = title; + m_root = path_t(title); } void Writer::addTracker(const std::string& type, @@ -320,7 +320,7 @@ namespace out { } auto output_field_h = Kokkos::create_mirror_view(output_field); Kokkos::deep_copy(output_field_h, output_field); - writer.Put(var, output_field_h); + writer.Put(var, output_field_h, adios2::Mode::Sync); } template @@ -355,7 +355,7 @@ namespace out { adios2::Box({ loc_offset }, { array.extent(0) })); auto array_h = Kokkos::create_mirror_view(array); Kokkos::deep_copy(array_h, array); - m_writer.Put(var, array_h); + m_writer.Put(var, array_h, adios2::Mode::Sync); } void Writer::writeSpectrum(const array_t& counts, @@ -377,12 +377,12 @@ namespace out { if (rank == MPI_ROOT_RANK) { auto var = m_io.InquireVariable(varname); var.SetSelection(adios2::Box({}, { counts.extent(0) })); - m_writer.Put(var, counts_h_all); + m_writer.Put(var, counts_h_all, adios2::Mode::Sync); } #else auto var = m_io.InquireVariable(varname); var.SetSelection(adios2::Box({}, { counts.extent(0) })); - m_writer.Put(var, counts_h); + m_writer.Put(var, counts_h, adios2::Mode::Sync); #endif } @@ -399,7 +399,7 @@ namespace out { var.SetSelection(adios2::Box({}, { e_bins.extent(0) })); auto e_bins_h = Kokkos::create_mirror_view(e_bins); Kokkos::deep_copy(e_bins_h, e_bins); - m_writer.Put(var, e_bins_h); + m_writer.Put(var, e_bins_h, adios2::Mode::Sync); } void Writer::writeMesh(unsigned short dim, @@ -412,11 +412,11 @@ namespace out { auto xe_h = Kokkos::create_mirror_view(xe); Kokkos::deep_copy(xc_h, xc); Kokkos::deep_copy(xe_h, xe); - m_writer.Put(varc, xc_h); - m_writer.Put(vare, xe_h); + m_writer.Put(varc, xc_h, adios2::Mode::Sync); + m_writer.Put(vare, xe_h, adios2::Mode::Sync); auto vard = m_io.InquireVariable( "N" + std::to_string(dim + 1) + "l"); - m_writer.Put(vard, loc_off_sz.data()); + m_writer.Put(vard, loc_off_sz.data(), adios2::Mode::Sync); } void Writer::beginWriting(WriteModeTags write_mode, @@ -427,53 +427,53 @@ namespace out { if (m_active_mode != WriteMode::None) { raise::Fatal("Already writing", HERE); } - m_active_mode = write_mode; try { - std::string filename; + path_t filename; + const std::string ext = (m_engine == "hdf5") ? "h5" : "bp"; if (m_separate_files) { std::string mode_str; - if (m_active_mode == WriteMode::Fields) { + if (write_mode == WriteMode::Fields) { mode_str = "fields"; - } else if (m_active_mode == WriteMode::Particles) { + } else if (write_mode == WriteMode::Particles) { mode_str = "particles"; - } else if (m_active_mode == WriteMode::Spectra) { + } else if (write_mode == WriteMode::Spectra) { mode_str = "spectra"; } else { raise::Fatal("Unknown write mode", HERE); } CallOnce( - [](auto& main_path, auto& mode_path) { - const std::filesystem::path main { main_path }; - const std::filesystem::path mode { mode_path }; + [](auto&& main_path, auto&& mode_path) { + const path_t main { main_path }; + const path_t mode { mode_path }; if (!std::filesystem::exists(main_path)) { std::filesystem::create_directory(main_path); } - if (!std::filesystem::exists(main / mode)) { - std::filesystem::create_directory(main / mode); + if (!std::filesystem::exists(main_path / mode_path)) { + std::filesystem::create_directory(main_path / mode_path); } }, - m_fname, + m_root, mode_str); - filename = fmt::format("%s/%s/%s.%08lu.%s", - m_fname.c_str(), - mode_str.c_str(), - mode_str.c_str(), - tstep, - ext.c_str()); - m_mode = adios2::Mode::Write; +#if defined(MPI_ENABLED) + MPI_Barrier(MPI_COMM_WORLD); +#endif + filename = m_root / path_t(mode_str) / + fmt::format("%s.%08lu.%s", mode_str.c_str(), tstep, ext.c_str()); + m_mode = adios2::Mode::Write; } else { - filename = fmt::format("%s.%s", m_fname.c_str(), ext.c_str()); + filename = fmt::format("%s.%s", m_root.c_str(), ext.c_str()); m_mode = std::filesystem::exists(filename) ? adios2::Mode::Append : adios2::Mode::Write; } m_writer = m_io.Open(filename, m_mode); + m_writer.BeginStep(); + m_writer.Put(m_io.InquireVariable("Step"), &tstep); + m_writer.Put(m_io.InquireVariable("Time"), &time); + m_active_mode = write_mode; } catch (std::exception& e) { raise::Fatal(e.what(), HERE); } - m_writer.BeginStep(); - m_writer.Put(m_io.InquireVariable("Step"), &tstep); - m_writer.Put(m_io.InquireVariable("Time"), &time); } void Writer::endWriting(WriteModeTags write_mode) { @@ -490,72 +490,24 @@ namespace out { m_writer.Close(); } - template void Writer::writeField(const std::vector&, - const ndfield_t&, - const std::vector&); - template void Writer::writeField(const std::vector&, - const ndfield_t&, - const std::vector&); - template void Writer::writeField(const std::vector&, - const ndfield_t&, - const std::vector&); - template void Writer::writeField(const std::vector&, - const ndfield_t&, - const std::vector&); - template void Writer::writeField(const std::vector&, - const ndfield_t&, - const std::vector&); - template void Writer::writeField(const std::vector&, - const ndfield_t&, - const std::vector&); - - template void WriteField(adios2::IO&, - adios2::Engine&, - const std::string&, - const ndfield_t&, - std::size_t, - std::vector, - std::vector, - bool); - template void WriteField(adios2::IO&, - adios2::Engine&, - const std::string&, - const ndfield_t&, - std::size_t, - std::vector, - std::vector, - bool); - template void WriteField(adios2::IO&, - adios2::Engine&, - const std::string&, - const ndfield_t&, - std::size_t, - std::vector, - std::vector, - bool); - template void WriteField(adios2::IO&, - adios2::Engine&, - const std::string&, - const ndfield_t&, - std::size_t, - std::vector, - std::vector, - bool); - template void WriteField(adios2::IO&, - adios2::Engine&, - const std::string&, - const ndfield_t&, - std::size_t, - std::vector, - std::vector, - bool); - template void WriteField(adios2::IO&, - adios2::Engine&, - const std::string&, - const ndfield_t&, - std::size_t, - std::vector, - std::vector, - bool); +#define WRITE_FIELD(D, N) \ + template void Writer::writeField(const std::vector&, \ + const ndfield_t&, \ + const std::vector&); \ + template void WriteField(adios2::IO&, \ + adios2::Engine&, \ + const std::string&, \ + const ndfield_t&, \ + std::size_t, \ + std::vector, \ + std::vector, \ + bool); + WRITE_FIELD(Dim::_1D, 3) + WRITE_FIELD(Dim::_1D, 6) + WRITE_FIELD(Dim::_2D, 3) + WRITE_FIELD(Dim::_2D, 6) + WRITE_FIELD(Dim::_3D, 3) + WRITE_FIELD(Dim::_3D, 6) +#undef WRITE_FIELD } // namespace out diff --git a/src/output/writer.h b/src/output/writer.h index 2e8d20de..cc3edc73 100644 --- a/src/output/writer.h +++ b/src/output/writer.h @@ -57,7 +57,7 @@ namespace out { bool m_flds_ghosts; std::string m_engine; - std::string m_fname; + path_t m_root; std::map m_trackers; @@ -116,8 +116,8 @@ namespace out { void endWriting(WriteModeTags); /* getters -------------------------------------------------------------- */ - auto fname() const -> const std::string& { - return m_fname; + auto root() const -> const path_t& { + return m_root; } auto fieldWriters() const -> const std::vector& {