From cbeca51456c0717e08cfb00106d053db7a79fe15 Mon Sep 17 00:00:00 2001 From: hayk Date: Tue, 15 Oct 2024 23:19:47 -0400 Subject: [PATCH 01/22] proper output readwrite test --- src/output/tests/writer-nompi.cpp | 170 +++++++++++++++++++----------- 1 file changed, 107 insertions(+), 63 deletions(-) diff --git a/src/output/tests/writer-nompi.cpp b/src/output/tests/writer-nompi.cpp index 25a9a2c51..c087d2895 100644 --- a/src/output/tests/writer-nompi.cpp +++ b/src/output/tests/writer-nompi.cpp @@ -16,6 +16,8 @@ #include #include +using namespace ntt; + void cleanup() { namespace fs = std::filesystem; fs::path tempfile_path { "test.h5" }; @@ -26,86 +28,128 @@ auto main(int argc, char* argv[]) -> int { Kokkos::initialize(argc, argv); try { - adios2::ADIOS adios; + constexpr auto nx1 = 10; + constexpr auto nx1_gh = nx1 + 2 * N_GHOSTS; + constexpr auto nx2 = 10; + constexpr auto nx2_gh = nx2 + 2 * N_GHOSTS; + constexpr auto nx3 = 10; + constexpr auto nx3_gh = nx3 + 2 * N_GHOSTS; + constexpr auto i1min = N_GHOSTS; + constexpr auto i2min = N_GHOSTS; + constexpr auto i3min = N_GHOSTS; + constexpr auto i1max = nx1 + N_GHOSTS; + constexpr auto i2max = nx2 + N_GHOSTS; + constexpr auto i3max = nx3 + N_GHOSTS; + + ndfield_t field { "fld", nx1_gh, nx2_gh, nx3_gh }; + std::vector field_names; - using namespace ntt; - auto writer = out::Writer(); - writer.init(&adios, "hdf5", "test"); - writer.defineMeshLayout({ 10, 10, 10 }, - { 0, 0, 0 }, - { 10, 10, 10 }, - false, - Coord::Cart); - writer.defineFieldOutputs(SimEngine::SRPIC, { "E", "B", "Rho_1_3", "N_2" }); - - ndfield_t field { "fld", - 10 + 2 * N_GHOSTS, - 10 + 2 * N_GHOSTS, - 10 + 2 * N_GHOSTS }; - Kokkos::parallel_for( - "fill", - CreateRangePolicy({ N_GHOSTS, N_GHOSTS, N_GHOSTS }, - { 10 + N_GHOSTS, 10 + N_GHOSTS, 10 + N_GHOSTS }), - Lambda(index_t i1, index_t i2, index_t i3) { - field(i1, i2, i3, 0) = i1 + i2 + i3; - field(i1, i2, i3, 1) = i1 * i2 / i3; - field(i1, i2, i3, 2) = i1 / i2 * i3; - }); - std::vector names; - std::vector addresses; - for (auto i = 0; i < 3; ++i) { - names.push_back(writer.fieldWriters()[0].name(i)); - addresses.push_back(i); + { + // fill data + Kokkos::parallel_for( + "fill", + CreateRangePolicy({ i1min, i2min, i3min }, + { i1max, i2max, i3max }), + Lambda(index_t i1, index_t i2, index_t i3) { + field(i1, i2, i3, 0) = i1 + i2 + i3; + field(i1, i2, i3, 1) = i1 * i2 / i3; + field(i1, i2, i3, 2) = i1 / i2 * i3; + }); } - writer.beginWriting(0, 0.0); - writer.writeField(names, field, addresses); - writer.endWriting(); - writer.beginWriting(1, 0.1); - writer.writeField(names, field, addresses); - writer.endWriting(); + adios2::ADIOS adios; + + { + // write + auto writer = out::Writer(); + writer.init(&adios, "hdf5", "test"); + writer.defineMeshLayout({ nx1, nx2, nx3 }, + { 0, 0, 0 }, + { nx1, nx3, nx3 }, + false, + Coord::Cart); + writer.defineFieldOutputs(SimEngine::SRPIC, { "E", "B", "Rho_1_3", "N_2" }); + + std::vector addresses; + for (auto i = 0; i < 3; ++i) { + field_names.push_back(writer.fieldWriters()[0].name(i)); + addresses.push_back(i); + } + writer.beginWriting(10, 123.0); + writer.writeField(field_names, field, addresses); + writer.endWriting(); + + writer.beginWriting(20, 123.4); + writer.writeField(field_names, field, addresses); + writer.endWriting(); + } { // read - adios2::ADIOS adios; - adios2::IO io = adios.DeclareIO("read-test"); + adios2::IO io = adios.DeclareIO("read-test"); io.SetEngine("hdf5"); adios2::Engine reader = io.Open("test.h5", adios2::Mode::Read); - std::size_t step { 0 }; - long double time { 0.0 }; - reader.Get(io.InquireVariable("Step"), step); - reader.Get(io.InquireVariable("Time"), time); - raise::ErrorIf(step != 0, "Step is not 0", HERE); - raise::ErrorIf(time != 0.0, "Time is not 0.0", HERE); + raise::ErrorIf(io.InquireAttribute("NGhosts").Data()[0] != 0, + "NGhosts is not correct", + HERE); + raise::ErrorIf(io.InquireAttribute("Dimension").Data()[0] != 3, + "Dimension is not correct", + HERE); for (std::size_t step = 0; reader.BeginStep() == adios2::StepStatus::OK; ++step) { - std::size_t step_read; - adios2::Variable stepVar = io.InquireVariable( - "Step"); - reader.Get(stepVar, step_read); - + std::size_t step_read; long double time_read; + + reader.Get(io.InquireVariable("Step"), step_read); reader.Get(io.InquireVariable("Time"), time_read); - raise::ErrorIf(step_read != step, "Step is not correct", HERE); - raise::ErrorIf((float)time_read != (float)step / 10.0f, + raise::ErrorIf(step_read != (step + 1) * 10, "Step is not correct", HERE); + raise::ErrorIf((float)time_read != 123 + (float)step * 0.4f, "Time is not correct", HERE); - for (const auto& name : names) { - auto data = io.InquireVariable(name); - raise::ErrorIf(data.Shape().size() != 3, - fmt::format("%s is not 3D", name.c_str()), - HERE); - - auto dims = data.Shape(); - std::size_t nx1 = dims[0]; - std::size_t nx2 = dims[1]; - std::size_t nx3 = dims[2]; - raise::ErrorIf((nx1 != 10) || (nx2 != 10) || (nx3 != 10), - fmt::format("%s is not 10x10x10", name.c_str()), - HERE); + array_t field_read { "fld_read", nx1, nx2, nx3 }; + auto field_read_h = Kokkos::create_mirror_view(field_read); + + int cntr = 0; + for (const auto& name : field_names) { + auto fieldVar = io.InquireVariable(name); + if (fieldVar) { + raise::ErrorIf(fieldVar.Shape().size() != 3, + fmt::format("%s is not 3D", name.c_str()), + HERE); + + auto dims = fieldVar.Shape(); + std::size_t nx1_r = dims[0]; + std::size_t nx2_r = dims[1]; + std::size_t nx3_r = dims[2]; + raise::ErrorIf( + (nx1_r != 10) || (nx2_r != 10) || (nx3_r != 10), + fmt::format("%s is not %dx%dx%d", name.c_str(), nx1_r, nx2_r, nx3_r), + HERE); + fieldVar.SetSelection( + adios2::Box({ 0, 0, 0 }, { nx1_r, nx2_r, nx3_r })); + reader.Get(fieldVar, field_read_h.data(), adios2::Mode::Sync); + Kokkos::deep_copy(field_read, field_read_h); + + Kokkos::parallel_for( + "check", + CreateRangePolicy({ 0, 0, 0 }, { nx1_r, nx2_r, nx3_r }), + Lambda(index_t i1, index_t i2, index_t i3) { + if (not cmp::AlmostEqual( + field_read(i1, i2, i3), + field(i1 + i1min, i2 + i2min, i3 + i3min, cntr))) { + printf("%e %e\n", + field_read(i1, i2, i3), + field(i1 + i1min, i2 + i2min, i3 + i3min, cntr)); + raise::KernelError(HERE, "Field is not read correctly"); + } + }); + ++cntr; + } else { + raise::Error("Field not found", HERE); + } } reader.EndStep(); } From de597cfa3d1745fa488bab56a9da0c2af5ce11d0 Mon Sep 17 00:00:00 2001 From: hayk Date: Tue, 15 Oct 2024 23:20:13 -0400 Subject: [PATCH 02/22] TODO marked --- src/output/tests/CMakeLists.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/output/tests/CMakeLists.txt b/src/output/tests/CMakeLists.txt index d33cc6c54..37af95fac 100644 --- a/src/output/tests/CMakeLists.txt +++ b/src/output/tests/CMakeLists.txt @@ -4,8 +4,6 @@ # - kokkos [required] # - mpi [optional] # - adios2 [optional] -# !TODO: -# - add more proper write tests for ADIOS2 # ------------------------------ set(SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../) @@ -27,4 +25,4 @@ if (NOT ${mpi}) gen_test(writer-nompi) else() gen_test(writer-mpi) -endif() \ No newline at end of file +endif() From f91ca40039eced9db228964c50cac8af8e0e965e Mon Sep 17 00:00:00 2001 From: hayk Date: Wed, 16 Oct 2024 00:10:56 -0400 Subject: [PATCH 03/22] added downsampling input param --- input.example.toml | 9 +++++---- src/framework/parameters.cpp | 16 ++++++++++------ src/framework/tests/parameters.cpp | 24 ++++++++++++++++++------ src/global/defaults.h | 1 - 4 files changed, 33 insertions(+), 17 deletions(-) diff --git a/input.example.toml b/input.example.toml index 06225024a..5ee34d65d 100644 --- a/input.example.toml +++ b/input.example.toml @@ -340,10 +340,6 @@ # @type: array of strings # @default: [] custom = "" - # @NOT_IMPLEMENTED: Stride for the output of fields: - # @type: unsigned short: > 1 - # @default: 1 - stride = "" # Smoothing window for the output of moments (e.g., "Rho", "Charge", "T", etc.): # @type: unsigned short # @default: 0 @@ -357,6 +353,11 @@ # @default: -1.0 (use `output.interval_time`) # @note: When `interval_time` < 0, the output is controlled by `interval`, otherwise by `interval_time` interval_time = "" + # Downsample factor for the output of fields: + # @type: array of unsigned int >= 1 + # @default: [1, 1, 1] + # @note: The output is downsampled by the given factors in each direction + downsampling = "" [output.particles] # Toggle for the particles output: diff --git a/src/framework/parameters.cpp b/src/framework/parameters.cpp index 1d4672212..3e2f1f9d7 100644 --- a/src/framework/parameters.cpp +++ b/src/framework/parameters.cpp @@ -494,12 +494,16 @@ namespace ntt { "fields", "mom_smooth", defaults::output::mom_smooth)); - set("output.fields.stride", - toml::find_or(toml_data, - "output", - "fields", - "stride", - defaults::output::flds_stride)); + auto field_dwn = toml::find_or(toml_data, + "output", + "fields", + "downsampling", + std::vector { 1, 1, 1 }); + raise::ErrorIf(field_dwn.size() > 3, "invalid `output.fields.downsampling`", HERE); + if (field_dwn.size() > dim) { + field_dwn.erase(field_dwn.begin() + (std::size_t)(dim), field_dwn.end()); + } + set("output.fields.downsampling", field_dwn); // particles const auto prtl_out = toml::find_or(toml_data, diff --git a/src/framework/tests/parameters.cpp b/src/framework/tests/parameters.cpp index 8d30355b9..393cd2409 100644 --- a/src/framework/tests/parameters.cpp +++ b/src/framework/tests/parameters.cpp @@ -73,13 +73,18 @@ const auto mink_1d = u8R"( mystr = "hi" [output] - fields = ["Rho", "J", "B"] - particles = ["X", "U"] format = "hdf5" - mom_smooth = 2 - fields_stride = 1 - prtl_stride = 100 - interval_time = 0.01 + + [output.fields] + quantities = ["Rho", "J", "B"] + mom_smooth = 2 + downsampling = [4, 5] + interval = 100 + + [output.particles] + species = [1, 2] + stride = 100 + interval_time = 0.01 )"_toml; const auto sph_2d = u8R"( @@ -315,6 +320,13 @@ auto main(int argc, char* argv[]) -> int { assert_equal(params_mink_1d.get("setup.mystr"), "hi", "setup.mystr"); + + const auto output_stride = params_mink_1d.get>( + "output.fields.downsampling"); + assert_equal(output_stride.size(), + 1, + "output.fields.downsampling.size()"); + assert_equal(output_stride[0], 4, "output.fields.downsampling[0]"); } { diff --git a/src/global/defaults.h b/src/global/defaults.h index ee9a65af5..be92acbf9 100644 --- a/src/global/defaults.h +++ b/src/global/defaults.h @@ -51,7 +51,6 @@ namespace ntt::defaults { const std::string format = "hdf5"; const std::size_t interval = 100; const unsigned short mom_smooth = 0; - const unsigned short flds_stride = 1; const std::size_t prtl_stride = 100; const real_t spec_emin = 1e-3; const real_t spec_emax = 1e3; From 97cccfaf260568e4aa5945f76e12589211625368 Mon Sep 17 00:00:00 2001 From: hayk Date: Wed, 16 Oct 2024 04:05:50 -0400 Subject: [PATCH 04/22] added field downsampling + test --- extern/adios2 | 2 +- src/checkpoint/tests/checkpoint-nompi.cpp | 4 +- src/framework/domain/output.cpp | 53 ++++-- src/framework/parameters.cpp | 19 +- src/output/tests/writer-nompi.cpp | 66 +++++-- src/output/writer.cpp | 213 ++++++++++++++++------ src/output/writer.h | 26 ++- 7 files changed, 275 insertions(+), 108 deletions(-) diff --git a/extern/adios2 b/extern/adios2 index e524dce1b..a6e8314cc 160000 --- a/extern/adios2 +++ b/extern/adios2 @@ -1 +1 @@ -Subproject commit e524dce1b72ccf75422cea6342ee2d64a6a87964 +Subproject commit a6e8314cc3c0b28d496b44dcd4f15685013b887b diff --git a/src/checkpoint/tests/checkpoint-nompi.cpp b/src/checkpoint/tests/checkpoint-nompi.cpp index 8f7a522fd..23dbd8871 100644 --- a/src/checkpoint/tests/checkpoint-nompi.cpp +++ b/src/checkpoint/tests/checkpoint-nompi.cpp @@ -29,9 +29,9 @@ auto main(int argc, char* argv[]) -> int { try { constexpr auto nx1 = 10; constexpr auto nx1_gh = nx1 + 2 * N_GHOSTS; - constexpr auto nx2 = 10; + constexpr auto nx2 = 13; constexpr auto nx2_gh = nx2 + 2 * N_GHOSTS; - constexpr auto nx3 = 10; + constexpr auto nx3 = 9; constexpr auto nx3_gh = nx3 + 2 * N_GHOSTS; constexpr auto i1min = N_GHOSTS; constexpr auto i2min = N_GHOSTS; diff --git a/src/framework/domain/output.cpp b/src/framework/domain/output.cpp index 0918eb2d3..46ca95ede 100644 --- a/src/framework/domain/output.cpp +++ b/src/framework/domain/output.cpp @@ -69,6 +69,8 @@ namespace ntt { g_writer.defineMeshLayout(glob_shape_with_ghosts, off_ncells_with_ghosts, loc_shape_with_ghosts, + params.template get>( + "output.fields.downsampling"), incl_ghosts, M::CoordType); const auto fields_to_write = params.template get>( @@ -216,37 +218,50 @@ namespace ntt { g_writer.beginWriting(current_step, current_time); if (write_fields) { const auto incl_ghosts = params.template get("output.debug.ghosts"); + const auto dwn = params.template get>( + "output.fields.downsampling"); for (unsigned short dim = 0; dim < M::Dim; ++dim) { - const auto is_last = local_domain->offset_ncells()[dim] + - local_domain->mesh.n_active()[dim] == - mesh().n_active()[dim]; - array_t xc { "Xc", - local_domain->mesh.n_active()[dim] + - (incl_ghosts ? 2 * N_GHOSTS : 0) }; - array_t xe { "Xe", - local_domain->mesh.n_active()[dim] + - (incl_ghosts ? 2 * N_GHOSTS : 0) + - (is_last ? 1 : 0) }; - const auto offset = (incl_ghosts ? N_GHOSTS : 0); - const auto ncells = local_domain->mesh.n_active()[dim]; - const auto& metric = local_domain->mesh.metric; + const auto l_size = local_domain->mesh.n_active()[dim]; + const auto l_offset = local_domain->offset_ncells()[dim]; + const auto g_size = mesh().n_active()[dim]; + + const auto dwn_in_dim = dwn[dim]; + const auto l_size_dwn = static_cast(l_size / dwn_in_dim); + + const auto is_last = l_offset + l_size == g_size; + + const auto add_ghost = (incl_ghosts ? 2 * N_GHOSTS : 0); + const auto add_last = (is_last ? 1 : 0); + + array_t xc { "Xc", l_size_dwn + add_ghost }; + array_t xe { "Xe", l_size_dwn + add_ghost + add_last }; + + const auto offset = (incl_ghosts ? N_GHOSTS : 0); + const auto ncells = l_size_dwn; + const auto first_cell = static_cast(l_offset / dwn_in_dim) * + dwn_in_dim - + l_offset; + + const auto& metric = local_domain->mesh.metric; + Kokkos::parallel_for( "GenerateMesh", ncells, - Lambda(index_t i) { + Lambda(index_t i_dwn) { + const auto i = first_cell + i_dwn * dwn_in_dim; const auto i_ = static_cast(i); coord_t x_Cd { ZERO }, x_Ph { ZERO }; x_Cd[dim] = i_ + HALF; metric.template convert(x_Cd, x_Ph); - xc(offset + i) = x_Ph[dim]; - x_Cd[dim] = i_; + xc(offset + i_dwn) = x_Ph[dim]; + x_Cd[dim] = i_; metric.template convert(x_Cd, x_Ph); - xe(offset + i) = x_Ph[dim]; - if (is_last && i == ncells - 1) { + xe(offset + i_dwn) = x_Ph[dim]; + if (is_last && i_dwn == ncells - 1) { x_Cd[dim] = i_ + ONE; metric.template convert(x_Cd, x_Ph); - xe(offset + i + 1) = x_Ph[dim]; + xe(offset + i_dwn + 1) = x_Ph[dim]; } }); g_writer.writeMesh(dim, xc, xe); diff --git a/src/framework/parameters.cpp b/src/framework/parameters.cpp index 3e2f1f9d7..b667b5ac9 100644 --- a/src/framework/parameters.cpp +++ b/src/framework/parameters.cpp @@ -503,6 +503,9 @@ namespace ntt { if (field_dwn.size() > dim) { field_dwn.erase(field_dwn.begin() + (std::size_t)(dim), field_dwn.end()); } + for (const auto& dwn : field_dwn) { + raise::ErrorIf(dwn == 0, "downsampling factor must be nonzero", HERE); + } set("output.fields.downsampling", field_dwn); // particles @@ -565,8 +568,20 @@ namespace ntt { /* [output.debug] ------------------------------------------------------- */ set("output.debug.as_is", toml::find_or(toml_data, "output", "debug", "as_is", false)); - set("output.debug.ghosts", - toml::find_or(toml_data, "output", "debug", "ghosts", false)); + const auto output_ghosts = toml::find_or(toml_data, + "output", + "debug", + "ghosts", + false); + set("output.debug.ghosts", output_ghosts); + if (output_ghosts) { + for (const auto& dwn : field_dwn) { + raise::ErrorIf( + dwn != 1, + "full resolution required when outputting with ghost cells", + HERE); + } + } /* [checkpoint] --------------------------------------------------------- */ set("checkpoint.interval", diff --git a/src/output/tests/writer-nompi.cpp b/src/output/tests/writer-nompi.cpp index c087d2895..4c032094b 100644 --- a/src/output/tests/writer-nompi.cpp +++ b/src/output/tests/writer-nompi.cpp @@ -30,9 +30,9 @@ auto main(int argc, char* argv[]) -> int { try { constexpr auto nx1 = 10; constexpr auto nx1_gh = nx1 + 2 * N_GHOSTS; - constexpr auto nx2 = 10; + constexpr auto nx2 = 14; constexpr auto nx2_gh = nx2 + 2 * N_GHOSTS; - constexpr auto nx3 = 10; + constexpr auto nx3 = 17; constexpr auto nx3_gh = nx3 + 2 * N_GHOSTS; constexpr auto i1min = N_GHOSTS; constexpr auto i2min = N_GHOSTS; @@ -41,6 +41,10 @@ auto main(int argc, char* argv[]) -> int { constexpr auto i2max = nx2 + N_GHOSTS; constexpr auto i3max = nx3 + N_GHOSTS; + constexpr auto dwn1 = 2; + constexpr auto dwn2 = 1; + constexpr auto dwn3 = 5; + ndfield_t field { "fld", nx1_gh, nx2_gh, nx3_gh }; std::vector field_names; @@ -51,9 +55,12 @@ auto main(int argc, char* argv[]) -> int { CreateRangePolicy({ i1min, i2min, i3min }, { i1max, i2max, i3max }), Lambda(index_t i1, index_t i2, index_t i3) { - field(i1, i2, i3, 0) = i1 + i2 + i3; - field(i1, i2, i3, 1) = i1 * i2 / i3; - field(i1, i2, i3, 2) = i1 / i2 * i3; + const auto i1_ = static_cast(i1); + const auto i2_ = static_cast(i2); + const auto i3_ = static_cast(i3); + field(i1, i2, i3, 0) = i1_; + field(i1, i2, i3, 1) = i2_; + field(i1, i2, i3, 2) = i3_; }); } @@ -65,7 +72,8 @@ auto main(int argc, char* argv[]) -> int { writer.init(&adios, "hdf5", "test"); writer.defineMeshLayout({ nx1, nx2, nx3 }, { 0, 0, 0 }, - { nx1, nx3, nx3 }, + { nx1, nx2, nx3 }, + { dwn1, dwn2, dwn3 }, false, Coord::Cart); writer.defineFieldOutputs(SimEngine::SRPIC, { "E", "B", "Rho_1_3", "N_2" }); @@ -109,8 +117,7 @@ auto main(int argc, char* argv[]) -> int { "Time is not correct", HERE); - array_t field_read { "fld_read", nx1, nx2, nx3 }; - auto field_read_h = Kokkos::create_mirror_view(field_read); + array_t field_read {}; int cntr = 0; for (const auto& name : field_names) { @@ -124,12 +131,22 @@ auto main(int argc, char* argv[]) -> int { std::size_t nx1_r = dims[0]; std::size_t nx2_r = dims[1]; std::size_t nx3_r = dims[2]; - raise::ErrorIf( - (nx1_r != 10) || (nx2_r != 10) || (nx3_r != 10), - fmt::format("%s is not %dx%dx%d", name.c_str(), nx1_r, nx2_r, nx3_r), - HERE); + raise::ErrorIf((nx1_r != nx1 / dwn1) || (nx2_r != nx2 / dwn2) || + (nx3_r != nx3 / dwn3), + fmt::format("%s = %ldx%ldx%ld is not %dx%dx%d", + name.c_str(), + nx1_r, + nx2_r, + nx3_r, + nx1 / dwn1, + nx2 / dwn2, + nx3 / dwn3), + HERE); + fieldVar.SetSelection( adios2::Box({ 0, 0, 0 }, { nx1_r, nx2_r, nx3_r })); + field_read = array_t(name, nx1_r, nx2_r, nx3_r); + auto field_read_h = Kokkos::create_mirror_view(field_read); reader.Get(fieldVar, field_read_h.data(), adios2::Mode::Sync); Kokkos::deep_copy(field_read, field_read_h); @@ -137,19 +154,32 @@ auto main(int argc, char* argv[]) -> int { "check", CreateRangePolicy({ 0, 0, 0 }, { nx1_r, nx2_r, nx3_r }), Lambda(index_t i1, index_t i2, index_t i3) { - if (not cmp::AlmostEqual( - field_read(i1, i2, i3), - field(i1 + i1min, i2 + i2min, i3 + i3min, cntr))) { - printf("%e %e\n", + if (not cmp::AlmostEqual(field_read(i1, i2, i3), + field(i1 * dwn1 + i1min, + i2 * dwn2 + i2min, + i3 * dwn3 + i3min, + cntr))) { + printf("\n:::::::::::::::\nfield_read(%ld, %ld, %ld) = %f != " + "field(%ld, %ld, %ld, %d) = %f\n:::::::::::::::\n", + i1, + i2, + i3, field_read(i1, i2, i3), - field(i1 + i1min, i2 + i2min, i3 + i3min, cntr)); + i1 * dwn1 + i1min, + i2 * dwn2 + i2min, + i3 * dwn3 + i3min, + cntr, + field(i1 * dwn1 + i1min, + i2 * dwn2 + i2min, + i3 * dwn3 + i3min, + cntr)); raise::KernelError(HERE, "Field is not read correctly"); } }); - ++cntr; } else { raise::Error("Field not found", HERE); } + ++cntr; } reader.EndStep(); } diff --git a/src/output/writer.cpp b/src/output/writer.cpp index 3d526b306..5c6dfe6d6 100644 --- a/src/output/writer.cpp +++ b/src/output/writer.cpp @@ -60,25 +60,44 @@ namespace out { m_mode = mode; } - void Writer::defineMeshLayout(const std::vector& glob_shape, - const std::vector& loc_corner, - const std::vector& loc_shape, - bool incl_ghosts, - Coord coords) { - m_flds_ghosts = incl_ghosts; + void Writer::defineMeshLayout(const std::vector& glob_shape, + const std::vector& loc_corner, + const std::vector& loc_shape, + const std::vector& dwn, + bool incl_ghosts, + Coord coords) { + m_flds_ghosts = incl_ghosts; + m_dwn = dwn; + m_flds_g_shape = glob_shape; m_flds_l_corner = loc_corner; m_flds_l_shape = loc_shape; + for (std::size_t i { 0 }; i < glob_shape.size(); ++i) { + raise::ErrorIf(dwn[i] != 1 && incl_ghosts, + "Downsampling with ghosts not supported", + HERE); + m_flds_g_shape_dwn.push_back( + static_cast(glob_shape[i] / m_dwn[i])); + m_flds_l_corner_dwn.push_back( + static_cast(loc_corner[i] / m_dwn[i])); + m_flds_l_shape_dwn.push_back( + static_cast((loc_corner[i] + loc_shape[i]) / m_dwn[i]) - + static_cast(loc_corner[i] / m_dwn[i])); + m_flds_l_first.push_back( + static_cast(loc_corner[i] / m_dwn[i]) * m_dwn[i] - + loc_corner[i]); + } + m_io.DefineAttribute("NGhosts", incl_ghosts ? N_GHOSTS : 0); m_io.DefineAttribute("Dimension", m_flds_g_shape.size()); m_io.DefineAttribute("Coordinates", std::string(coords.to_string())); for (std::size_t i { 0 }; i < m_flds_g_shape.size(); ++i) { // cell-centers - adios2::Dims g_shape = { m_flds_g_shape[i] }; - adios2::Dims l_corner = { m_flds_l_corner[i] }; - adios2::Dims l_shape = { m_flds_l_shape[i] }; + adios2::Dims g_shape = { m_flds_g_shape_dwn[i] }; + adios2::Dims l_corner = { m_flds_l_corner_dwn[i] }; + adios2::Dims l_shape = { m_flds_l_shape_dwn[i] }; m_io.DefineVariable("X" + std::to_string(i + 1), g_shape, l_corner, @@ -87,8 +106,8 @@ namespace out { // cell-edges const auto is_last = (m_flds_l_corner[i] + m_flds_l_shape[i] == m_flds_g_shape[i]); - adios2::Dims g_shape1 = { m_flds_g_shape[i] + 1 }; - adios2::Dims l_shape1 = { m_flds_l_shape[i] + (is_last ? 1 : 0) }; + adios2::Dims g_shape1 = { m_flds_g_shape_dwn[i] + 1 }; + adios2::Dims l_shape1 = { m_flds_l_shape_dwn[i] + (is_last ? 1 : 0) }; m_io.DefineVariable("X" + std::to_string(i + 1) + "e", g_shape1, l_corner, @@ -100,9 +119,6 @@ namespace out { Kokkos::LayoutRight>::value) { m_io.DefineAttribute("LayoutRight", 1); } else { - std::reverse(m_flds_g_shape.begin(), m_flds_g_shape.end()); - std::reverse(m_flds_l_corner.begin(), m_flds_l_corner.end()); - std::reverse(m_flds_l_shape.begin(), m_flds_l_shape.end()); m_io.DefineAttribute("LayoutRight", 0); } } @@ -110,8 +126,9 @@ namespace out { void Writer::defineFieldOutputs(const SimEngine& S, const std::vector& flds_out) { m_flds_writers.clear(); - raise::ErrorIf((m_flds_g_shape.size() == 0) || (m_flds_l_corner.size() == 0) || - (m_flds_l_shape.size() == 0), + raise::ErrorIf((m_flds_g_shape_dwn.size() == 0) || + (m_flds_l_corner_dwn.size() == 0) || + (m_flds_l_shape_dwn.size() == 0), "Mesh layout must be defined before field output", HERE); for (const auto& fld : flds_out) { @@ -119,17 +136,19 @@ namespace out { } for (const auto& fld : m_flds_writers) { if (fld.comp.size() == 0) { + // scalar m_io.DefineVariable(fld.name(), - m_flds_g_shape, - m_flds_l_corner, - m_flds_l_shape, + m_flds_g_shape_dwn, + m_flds_l_corner_dwn, + m_flds_l_shape_dwn, adios2::ConstantDims); } else { + // vector or tensor for (std::size_t i { 0 }; i < fld.comp.size(); ++i) { m_io.DefineVariable(fld.name(i), - m_flds_g_shape, - m_flds_l_corner, - m_flds_l_shape, + m_flds_g_shape_dwn, + m_flds_l_corner_dwn, + m_flds_l_shape_dwn, adios2::ConstantDims); } } @@ -178,48 +197,105 @@ namespace out { } template - void WriteField(adios2::IO& io, - adios2::Engine& writer, - const std::string& varname, - const ndfield_t& field, - std::size_t comp, - bool ghosts) { - auto var = io.InquireVariable(varname); - const auto gh_zones = ghosts ? 0 : N_GHOSTS; + void WriteField(adios2::IO& io, + adios2::Engine& writer, + const std::string& varname, + const ndfield_t& field, + std::size_t comp, + std::vector dwn, + std::vector first_cell, + bool ghosts) { + // when dwn != 1 in any direction, it is assumed that ghosts == false + auto var = io.InquireVariable(varname); + const auto gh_zones = ghosts ? 0 : N_GHOSTS; + ndarray_t output_field {}; if constexpr (D == Dim::_1D) { - auto slice_i1 = range_tuple_t(gh_zones, field.extent(0) - gh_zones); - auto slice = Kokkos::subview(field, slice_i1, comp); - auto output_field = array_t("output_field", slice.extent(0)); - Kokkos::deep_copy(output_field, slice); - auto output_field_host = Kokkos::create_mirror_view(output_field); - Kokkos::deep_copy(output_field_host, output_field); - writer.Put(var, output_field_host); + if (ghosts || dwn[0] == 1) { + auto slice_i1 = range_tuple_t(gh_zones, field.extent(0) - gh_zones); + auto slice = Kokkos::subview(field, slice_i1, comp); + output_field = array_t { "output_field", slice.extent(0) }; + Kokkos::deep_copy(output_field, slice); + } else { + const auto dwn1 = dwn[0]; + const auto nx1_dwn = static_cast( + (field.extent(0) - 2 * N_GHOSTS) / dwn1); + const auto first_cell1 = first_cell[0]; + output_field = array_t { "output_field", nx1_dwn }; + Kokkos::parallel_for( + "outputField", + nx1_dwn, + Lambda(index_t i1) { + output_field(i1) = field(first_cell1 + i1 * dwn1 + N_GHOSTS, comp); + }); + } } else if constexpr (D == Dim::_2D) { - auto slice_i1 = range_tuple_t(gh_zones, field.extent(0) - gh_zones); - auto slice_i2 = range_tuple_t(gh_zones, field.extent(1) - gh_zones); - auto slice = Kokkos::subview(field, slice_i1, slice_i2, comp); - auto output_field = array_t("output_field", + if (ghosts || (dwn[0] == 1 && dwn[1] == 1)) { + auto slice_i1 = range_tuple_t(gh_zones, field.extent(0) - gh_zones); + auto slice_i2 = range_tuple_t(gh_zones, field.extent(1) - gh_zones); + auto slice = Kokkos::subview(field, slice_i1, slice_i2, comp); + output_field = array_t { "output_field", slice.extent(0), - slice.extent(1)); - Kokkos::deep_copy(output_field, slice); - auto output_field_host = Kokkos::create_mirror_view(output_field); - Kokkos::deep_copy(output_field_host, output_field); - writer.Put(var, output_field_host); + slice.extent(1) }; + Kokkos::deep_copy(output_field, slice); + } else { + const auto dwn1 = dwn[0]; + const auto dwn2 = dwn[1]; + const auto nx1_dwn = static_cast( + (field.extent(0) - 2 * N_GHOSTS) / dwn1); + const auto nx2_dwn = static_cast( + (field.extent(1) - 2 * N_GHOSTS) / dwn2); + const auto first_cell1 = first_cell[0]; + const auto first_cell2 = first_cell[1]; + output_field = array_t { "output_field", nx1_dwn, nx2_dwn }; + Kokkos::parallel_for( + "outputField", + CreateRangePolicy({ 0, 0 }, { nx1_dwn, nx2_dwn }), + Lambda(index_t i1, index_t i2) { + output_field(i1, i2) = field(first_cell1 + i1 * dwn1 + N_GHOSTS, + first_cell2 + i2 * dwn2 + N_GHOSTS, + comp); + }); + } } else if constexpr (D == Dim::_3D) { - auto slice_i1 = range_tuple_t(gh_zones, field.extent(0) - gh_zones); - auto slice_i2 = range_tuple_t(gh_zones, field.extent(1) - gh_zones); - auto slice_i3 = range_tuple_t(gh_zones, field.extent(2) - gh_zones); - auto slice = Kokkos::subview(field, slice_i1, slice_i2, slice_i3, comp); - auto output_field = array_t("output_field", - slice.extent(0), - slice.extent(1), - slice.extent(2)); - Kokkos::deep_copy(output_field, slice); - auto output_field_host = Kokkos::create_mirror_view(output_field); - Kokkos::deep_copy(output_field_host, output_field); - writer.Put(var, output_field_host); + if (ghosts || (dwn[0] == 1 && dwn[1] == 1 && dwn[2] == 1)) { + auto slice_i1 = range_tuple_t(gh_zones, field.extent(0) - gh_zones); + auto slice_i2 = range_tuple_t(gh_zones, field.extent(1) - gh_zones); + auto slice_i3 = range_tuple_t(gh_zones, field.extent(2) - gh_zones); + auto slice = Kokkos::subview(field, slice_i1, slice_i2, slice_i3, comp); + output_field = array_t { "output_field", + slice.extent(0), + slice.extent(1), + slice.extent(2) }; + Kokkos::deep_copy(output_field, slice); + } else { + const auto dwn1 = dwn[0]; + const auto dwn2 = dwn[1]; + const auto dwn3 = dwn[2]; + const auto nx1_dwn = static_cast( + (field.extent(0) - 2 * N_GHOSTS) / dwn1); + const auto nx2_dwn = static_cast( + (field.extent(1) - 2 * N_GHOSTS) / dwn2); + const auto nx3_dwn = static_cast( + (field.extent(2) - 2 * N_GHOSTS) / dwn3); + const auto first_cell1 = first_cell[0]; + const auto first_cell2 = first_cell[1]; + const auto first_cell3 = first_cell[2]; + output_field = array_t { "output_field", nx1_dwn, nx2_dwn, nx3_dwn }; + Kokkos::parallel_for( + "outputField", + CreateRangePolicy({ 0, 0, 0 }, { nx1_dwn, nx2_dwn, nx3_dwn }), + Lambda(index_t i1, index_t i2, index_t i3) { + output_field(i1, i2, i3) = field(first_cell1 + i1 * dwn1 + N_GHOSTS, + first_cell2 + i2 * dwn2 + N_GHOSTS, + first_cell3 + i3 * dwn3 + N_GHOSTS, + comp); + }); + } } + auto output_field_h = Kokkos::create_mirror_view(output_field); + Kokkos::deep_copy(output_field_h, output_field); + writer.Put(var, output_field_h); } template @@ -233,7 +309,14 @@ namespace out { "# of names != # of addresses ", HERE); for (std::size_t i { 0 }; i < addresses.size(); ++i) { - WriteField(m_io, m_writer, names[i], fld, addresses[i], m_flds_ghosts); + WriteField(m_io, + m_writer, + names[i], + fld, + addresses[i], + m_dwn, + m_flds_l_first, + m_flds_ghosts); } } @@ -360,36 +443,48 @@ namespace out { const std::string&, const ndfield_t&, std::size_t, + std::vector, + std::vector, bool); template void WriteField(adios2::IO&, adios2::Engine&, const std::string&, const ndfield_t&, std::size_t, + std::vector, + std::vector, bool); template void WriteField(adios2::IO&, adios2::Engine&, const std::string&, const ndfield_t&, std::size_t, + std::vector, + std::vector, bool); template void WriteField(adios2::IO&, adios2::Engine&, const std::string&, const ndfield_t&, std::size_t, + std::vector, + std::vector, bool); template void WriteField(adios2::IO&, adios2::Engine&, const std::string&, const ndfield_t&, std::size_t, + std::vector, + std::vector, bool); template void WriteField(adios2::IO&, adios2::Engine&, const std::string&, const ndfield_t&, std::size_t, + std::vector, + std::vector, bool); } // namespace out diff --git a/src/output/writer.h b/src/output/writer.h index ba24a3d65..566da44b2 100644 --- a/src/output/writer.h +++ b/src/output/writer.h @@ -37,14 +37,25 @@ namespace out { adios2::Mode m_mode { adios2::Mode::Write }; // global shape of the fields array to output - adios2::Dims m_flds_g_shape; + std::vector m_flds_g_shape; // local corner of the fields array to output - adios2::Dims m_flds_l_corner; + std::vector m_flds_l_corner; // local shape of the fields array to output - adios2::Dims m_flds_l_shape; - bool m_flds_ghosts; - std::string m_engine; - std::string m_fname; + std::vector m_flds_l_shape; + + // downsampling factors for each dimension + std::vector m_dwn; + // starting cell in each dimension (not including ghosts) + std::vector m_flds_l_first; + + // same but downsampled + adios2::Dims m_flds_g_shape_dwn; + adios2::Dims m_flds_l_corner_dwn; + adios2::Dims m_flds_l_shape_dwn; + + bool m_flds_ghosts; + std::string m_engine; + std::string m_fname; std::map m_trackers; @@ -73,7 +84,8 @@ namespace out { void defineMeshLayout(const std::vector&, const std::vector&, const std::vector&, - bool incl_ghosts, + const std::vector&, + bool, Coord); void defineFieldOutputs(const SimEngine&, const std::vector&); From 24691ea2fe8a8ae6ca48e2a9e082e8b099274b73 Mon Sep 17 00:00:00 2001 From: hayk Date: Wed, 16 Oct 2024 04:28:17 -0400 Subject: [PATCH 05/22] added a cpu runner for actions (totest) --- .github/workflows/actions.yml | 4 +- dev/runners/Dockerfile.runner.cpu | 73 +++++++++++++++++++++++++++++++ dev/runners/README.md | 6 +++ 3 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 dev/runners/Dockerfile.runner.cpu diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index cd7119789..331d6526c 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - device: [amd-gpu, nvidia-gpu] + device: [cpu, amd-gpu, nvidia-gpu] precision: [double, single] exclude: - device: amd-gpu @@ -35,6 +35,8 @@ jobs: fi elif [ "${{ matrix.device }}" = "amd-gpu" ]; then FLAGS="-D Kokkos_ENABLE_HIP=ON -D Kokkos_ARCH_AMD_GFX1100=ON" + elif [ "${{ matrix.device }}" = "cpu" ]; then + FLAGS="" fi cmake -B build -D TESTS=ON -D output=ON -D precision=${{ matrix.precision }} $FLAGS - name: Compile diff --git a/dev/runners/Dockerfile.runner.cpu b/dev/runners/Dockerfile.runner.cpu new file mode 100644 index 000000000..fc13ec9b5 --- /dev/null +++ b/dev/runners/Dockerfile.runner.cpu @@ -0,0 +1,73 @@ +FROM ubuntu:22.04 + +ARG DEBIAN_FRONTEND=noninteractive + +# upgrade +RUN apt-get update && apt-get upgrade -y + +# cmake & build tools +RUN apt-get remove -y --purge cmake && \ + apt-get install -y sudo wget curl build-essential && \ + wget "https://github.com/Kitware/CMake/releases/download/v3.29.6/cmake-3.29.6-linux-x86_64.tar.gz" -P /opt && \ + tar xvf /opt/cmake-3.29.6-linux-x86_64.tar.gz -C /opt && \ + rm /opt/cmake-3.29.6-linux-x86_64.tar.gz +ENV PATH=/opt/cmake-3.29.6-linux-x86_64/bin:$PATH + +# adios2 +RUN apt-get update && apt-get install -y git libhdf5-dev && \ + git clone https://github.com/ornladios/ADIOS2.git /opt/adios2-src && \ + cd /opt/adios2-src && \ + cmake -B build \ + -D CMAKE_CXX_STANDARD=17 \ + -D CMAKE_CXX_EXTENSIONS=OFF \ + -D CMAKE_POSITION_INDEPENDENT_CODE=TRUE \ + -D BUILD_SHARED_LIBS=ON \ + -D ADIOS2_USE_HDF5=ON \ + -D ADIOS2_USE_Python=OFF \ + -D ADIOS2_USE_Fortran=OFF \ + -D ADIOS2_USE_ZeroMQ=OFF \ + -D BUILD_TESTING=OFF \ + -D ADIOS2_BUILD_EXAMPLES=OFF \ + -D ADIOS2_USE_MPI=OFF \ + -D ADIOS2_HAVE_HDF5_VOL=OFF \ + -D CMAKE_INSTALL_PREFIX=/opt/adios2 && \ + cmake --build build -j && \ + cmake --install build && \ + rm -rf /opt/adios2-src + +ENV HDF5_ROOT=/usr +ENV ADIOS2_DIR=/opt/adios2 +ENV PATH=/opt/adios2/bin:$PATH + +# cleanup +RUN apt-get clean && \ + apt-get autoclean && \ + apt-get autoremove -y && \ + rm -rf /var/lib/cache/* && \ + rm -rf /var/lib/log/* && \ + rm -rf /var/lib/apt/lists/* + +ARG USER=runner +RUN useradd -ms /usr/bin/zsh $USER && \ + usermod -aG sudo $USER && \ + echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +USER $USER +ARG HOME=/home/$USER +WORKDIR $HOME + +# gh runner +ARG RUNNER_VERSION=2.317.0 +RUN mkdir actions-runner +WORKDIR $HOME/actions-runner + +RUN curl -o actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz \ + -L https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz && \ + tar xzf ./actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz && \ + sudo ./bin/installdependencies.sh + +ADD start.sh start.sh +RUN sudo chown $USER:$USER start.sh && \ + sudo chmod +x start.sh + +ENTRYPOINT ["./start.sh"] diff --git a/dev/runners/README.md b/dev/runners/README.md index 08d0cd176..957898fa7 100644 --- a/dev/runners/README.md +++ b/dev/runners/README.md @@ -19,3 +19,9 @@ docker run -e TOKEN= -e LABEL=nvidia-gpu --runtime=nvidia --gpus=all -dt docker build -t ghrunner:amd -f Dockerfile.runner.rocm . docker run -e TOKEN= -e LABEL=amd-gpu --device=/dev/kfd --device=/dev/dri --security-opt seccomp=unconfined --group-add video -dt ghrunner:amd ``` + +### CPU + +```sh +docker build -t ghrunner:cpu -f Dockerfile.runner.cpu . +``` From e3d40c3b10d2088d95acd215713ac934283a4488 Mon Sep 17 00:00:00 2001 From: hayk Date: Wed, 16 Oct 2024 04:30:59 -0400 Subject: [PATCH 06/22] RUNTEST --- .github/workflows/actions.yml | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 331d6526c..4db596243 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -1,27 +1,36 @@ name: Unit tests on: - pull_request: - branches: - - '**rc' - - 'master' + push: jobs: + check-commit: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Check commit message + id: check_message + run: | + if git log -1 --pretty=%B | grep -q "RUNTEST"; then + echo "::set-output name=run_tests::true" + else + echo "::set-output name=run_tests::false" + exit 1 tests: + needs: check-commit + if: steps.check_message.outputs.run_tests == 'true' strategy: fail-fast: false matrix: device: [cpu, amd-gpu, nvidia-gpu] precision: [double, single] - exclude: + exclude: # my AMD GPU doesn't support fp64 atomics : ( - device: amd-gpu precision: double - # my AMD GPU doesn't support fp64 atomics : ( - runs-on: [self-hosted, "${{ matrix.device }}"] - if: contains(github.event.head_commit.message, 'totest') steps: - name: Checkout - uses: actions/checkout@v3.3.0 + uses: actions/checkout@v4 - name: Configure run: | if [ "${{ matrix.device }}" = "nvidia-gpu" ]; then From 81f5df0baee2eac88d7220f9d29b6abf4ac2ffa2 Mon Sep 17 00:00:00 2001 From: hayk Date: Wed, 16 Oct 2024 04:33:16 -0400 Subject: [PATCH 07/22] RUNTEST --- .github/workflows/actions.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 4db596243..703f97a6a 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -28,6 +28,7 @@ jobs: exclude: # my AMD GPU doesn't support fp64 atomics : ( - device: amd-gpu precision: double + runs-on: [self-hosted, "${{ matrix.device }}"] steps: - name: Checkout uses: actions/checkout@v4 From 7b7382bc90427654c1baf0b23e56adfdab969027 Mon Sep 17 00:00:00 2001 From: hayk Date: Wed, 16 Oct 2024 04:35:26 -0400 Subject: [PATCH 08/22] RUNTEST --- .github/workflows/actions.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 703f97a6a..3af457b6e 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -6,6 +6,8 @@ on: jobs: check-commit: runs-on: ubuntu-latest + outputs: + run_tests: ${{ steps.check_message.outputs.run_tests }} steps: - name: Checkout repository uses: actions/checkout@v4 @@ -19,7 +21,7 @@ jobs: exit 1 tests: needs: check-commit - if: steps.check_message.outputs.run_tests == 'true' + if: needs.check-commit.outputs.run_tests == 'true' strategy: fail-fast: false matrix: From 0a465ad47accef51a4767eb3302182d1fcf1c2ed Mon Sep 17 00:00:00 2001 From: hayk Date: Wed, 16 Oct 2024 04:38:12 -0400 Subject: [PATCH 09/22] RUNTEST --- .github/workflows/actions.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 3af457b6e..22f55d8be 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -19,6 +19,7 @@ jobs: else echo "::set-output name=run_tests::false" exit 1 + fi tests: needs: check-commit if: needs.check-commit.outputs.run_tests == 'true' From 0caddf9f24103cc5bd1539527921d684261cc06e Mon Sep 17 00:00:00 2001 From: hayk Date: Thu, 17 Oct 2024 07:42:12 -0400 Subject: [PATCH 10/22] WIP proper mpi test for checkpoints --- cmake/tests.cmake | 2 +- src/checkpoint/tests/checkpoint-mpi.cpp | 223 ++++++++++++++++++++++++ src/framework/CMakeLists.txt | 1 - src/framework/domain/metadomain.h | 2 +- src/output/tests/writer-mpi.cpp | 7 +- 5 files changed, 229 insertions(+), 6 deletions(-) create mode 100644 src/checkpoint/tests/checkpoint-mpi.cpp diff --git a/cmake/tests.cmake b/cmake/tests.cmake index f1342f679..643ac3d29 100644 --- a/cmake/tests.cmake +++ b/cmake/tests.cmake @@ -17,7 +17,7 @@ if (${mpi}) # tests with mpi if (${output}) add_subdirectory(${SRC_DIR}/output/tests ${CMAKE_CURRENT_BINARY_DIR}/output/tests) - add_subdirectory(${SRC_DIR}/framework/tests ${CMAKE_CURRENT_BINARY_DIR}/checkpoint/tests) + add_subdirectory(${SRC_DIR}/checkpoint/tests ${CMAKE_CURRENT_BINARY_DIR}/checkpoint/tests) add_subdirectory(${SRC_DIR}/framework/tests ${CMAKE_CURRENT_BINARY_DIR}/framework/tests) endif() else() diff --git a/src/checkpoint/tests/checkpoint-mpi.cpp b/src/checkpoint/tests/checkpoint-mpi.cpp new file mode 100644 index 000000000..3ce4bab14 --- /dev/null +++ b/src/checkpoint/tests/checkpoint-mpi.cpp @@ -0,0 +1,223 @@ +#include "enums.h" +#include "global.h" + +#include "utils/comparators.h" + +#include "checkpoint/reader.h" +#include "checkpoint/writer.h" + +#include +#include +#include +#include + +#include +#include +#include + +using namespace ntt; +using namespace checkpoint; + +void cleanup() { + namespace fs = std::filesystem; + fs::path temp_path { "checkpoints" }; + fs::remove_all(temp_path); +} + +auto main(int argc, char* argv[]) -> int { + Kokkos::initialize(argc, argv); + MPI_Init(&argc, &argv); + int rank, size; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + + try { + // assuming 4 ranks + // |------|------| + // | 2 | 3 | + // |------|------| + // | | | + // | 0 | 1 | + // |------|------| + constexpr auto g_nx1 = 20; + constexpr auto g_nx2 = 15; + constexpr auto g_nx1_gh = g_nx1 + 4 * N_GHOSTS; + constexpr auto g_nx2_gh = g_nx2 + 4 * N_GHOSTS; + + constexpr auto l_nx1 = 10; + constexpr auto l_nx2 = (rank < 2) ? 10 : 5; + + constexpr auto l_nx1_gh = l_nx1 + 2 * N_GHOSTS; + constexpr auto l_nx2_gh = l_nx2 + 2 * N_GHOSTS; + + constexpr auto l_corner_x1 = (rank % 2) * l_nx1; + constexpr auto l_corner_x2 = (rank / 2) * l_nx2; + + constexpr auto i1min = N_GHOSTS; + constexpr auto i2min = N_GHOSTS; + constexpr auto i1max = l_nx1 + N_GHOSTS; + constexpr auto i2max = l_nx2 + N_GHOSTS; + + constexpr auto npart1 = (rank % 2 + rank) * 23 + 100; + constexpr auto npart2 = (rank % 2 + rank) * 37 + 100; + + // init data + ndfield_t field1 { "fld1", l_nx1_gh, l_nx2_gh }; + ndfield_t field2 { "fld2", l_nx1_gh, l_nx2_gh }; + + array_t i1 { "i_1", npart1 }; + array_t u1 { "u_1", npart1 }; + array_t i2 { "i_2", npart2 }; + array_t u2 { "u_2", npart2 }; + + { + // fill data + Kokkos::parallel_for( + "fillFlds", + CreateRangePolicy({ i1min, i2min }, { i1max, i2max }), + Lambda(index_t i1, index_t i2) { + field1(i1, i2, 0) = static_cast(i1 + i2); + field1(i1, i2, 1) = static_cast(i1 * i2); + field1(i1, i2, 2) = static_cast(i1 / i2); + field1(i1, i2, 3) = static_cast(i1 - i2); + field1(i1, i2, 4) = static_cast(i2 / i1); + field1(i1, i2, 5) = static_cast(i1); + field2(i1, i2, 0) = static_cast(-(i1 + i2)); + field2(i1, i2, 1) = static_cast(-(i1 * i2)); + field2(i1, i2, 2) = static_cast(-(i1 / i2)); + field2(i1, i2, 3) = static_cast(-(i1 - i2)); + field2(i1, i2, 4) = static_cast(-(i2 / i1)); + field2(i1, i2, 5) = static_cast(-i1); + }); + Kokkos::parallel_for( + "fillPrtl1", + npart1, + Lambda(index_t p) { + u1(p) = static_cast(p); + i1(p) = static_cast(p); + }); + Kokkos::parallel_for( + "fillPrtl2", + npart2, + Lambda(index_t p) { + u2(p) = -static_cast(p); + i2(p) = -static_cast(p); + }); + } + + adios2::ADIOS adios; + + { + // write checkpoint + Writer writer; + writer.init(&adios, 0, 0.0, 1); + + writer.defineFieldVariables(SimEngine::GRPIC, + { g_nx1_gh, g_nx2_gh }, + { l_corner_x1, l_corner_x2 }, + { l_nx1, l_nx2 }); + writer.defineParticleVariables(Coord::Sph, Dim::_2D, 2, { 0, 0 }); + + writer.beginSaving(0, 0.0); + + writer.saveField("em", field1); + writer.saveField("em0", field2); + + writer.savePerDomainVariable("s1_npart", 1, 0, npart1); + writer.savePerDomainVariable("s2_npart", 1, 0, npart2); + + writer.saveParticleQuantity("s1_i1", npart1, 0, npart1, i1); + writer.saveParticleQuantity("s1_ux1", npart1, 0, npart1, u1); + writer.saveParticleQuantity("s2_i1", npart2, 0, npart2, i2); + writer.saveParticleQuantity("s2_ux1", npart2, 0, npart2, u2); + + writer.endSaving(); + } + + { + // read checkpoint + ndfield_t field1_read { "fld1_read", nx1_gh, nx2_gh, nx3_gh }; + ndfield_t field2_read { "fld2_read", nx1_gh, nx2_gh, nx3_gh }; + + array_t i1_read { "i_1", npart1 }; + array_t u1_read { "u_1", npart1 }; + array_t i2_read { "i_2", npart2 }; + array_t u2_read { "u_2", npart2 }; + + adios2::IO io = adios.DeclareIO("checkpointRead"); + adios2::Engine reader = io.Open("checkpoints/step-00000000.bp", + adios2::Mode::Read); + reader.BeginStep(); + + auto fieldRange = adios2::Box({ 0, 0, 0, 0 }, + { nx1_gh, nx2_gh, nx3_gh, 6 }); + ReadFields(io, reader, "em", fieldRange, field1_read); + ReadFields(io, reader, "em0", fieldRange, field2_read); + + auto [nprtl1, noff1] = ReadParticleCount(io, reader, 0, 0, 1); + auto [nprtl2, noff2] = ReadParticleCount(io, reader, 1, 0, 1); + + ReadParticleData(io, reader, "ux1", 0, u1_read, nprtl1, noff1); + ReadParticleData(io, reader, "ux1", 1, u2_read, nprtl2, noff2); + ReadParticleData(io, reader, "i1", 0, i1_read, nprtl1, noff1); + ReadParticleData(io, reader, "i1", 1, i2_read, nprtl2, noff2); + + reader.EndStep(); + reader.Close(); + + // check the validity + Kokkos::parallel_for( + "checkFields", + CreateRangePolicy({ 0, 0, 0 }, { nx1_gh, nx2_gh, nx3_gh }), + Lambda(index_t i1, index_t i2, index_t i3) { + for (int i = 0; i < 6; ++i) { + if (not cmp::AlmostEqual(field1(i1, i2, i3, i), + field1_read(i1, i2, i3, i))) { + raise::KernelError(HERE, "Field1 read failed"); + } + if (not cmp::AlmostEqual(field2(i1, i2, i3, i), + field2_read(i1, i2, i3, i))) { + raise::KernelError(HERE, "Field2 read failed"); + } + } + }); + + raise::ErrorIf(npart1 != nprtl1, "Particle count 1 mismatch", HERE); + raise::ErrorIf(npart2 != nprtl2, "Particle count 2 mismatch", HERE); + raise::ErrorIf(noff1 != 0, "Particle offset 1 mismatch", HERE); + raise::ErrorIf(noff2 != 0, "Particle offset 2 mismatch", HERE); + + Kokkos::parallel_for( + "checkPrtl1", + npart1, + Lambda(index_t p) { + if (not cmp::AlmostEqual(u1(p), u1_read(p))) { + raise::KernelError(HERE, "u1 read failed"); + } + if (i1(p) != i1_read(p)) { + raise::KernelError(HERE, "i1 read failed"); + } + }); + Kokkos::parallel_for( + "checkPrtl2", + npart2, + Lambda(index_t p) { + if (not cmp::AlmostEqual(u2(p), u2_read(p))) { + raise::KernelError(HERE, "u2 read failed"); + } + if (i2(p) != i2_read(p)) { + raise::KernelError(HERE, "i2 read failed"); + } + }); + } + + } catch (std::exception& e) { + std::cerr << e.what() << std::endl; + cleanup(); + Kokkos::finalize(); + return 1; + } + cleanup(); + Kokkos::finalize(); + return 0; +} diff --git a/src/framework/CMakeLists.txt b/src/framework/CMakeLists.txt index 241780575..e01759d14 100644 --- a/src/framework/CMakeLists.txt +++ b/src/framework/CMakeLists.txt @@ -32,7 +32,6 @@ set(SOURCES ${SRC_DIR}/domain/grid.cpp ${SRC_DIR}/domain/metadomain.cpp ${SRC_DIR}/domain/communications.cpp - ${SRC_DIR}/domain/checkpoint.cpp ${SRC_DIR}/containers/particles.cpp ${SRC_DIR}/containers/fields.cpp ) diff --git a/src/framework/domain/metadomain.h b/src/framework/domain/metadomain.h index 027a2982d..7b3042b5b 100644 --- a/src/framework/domain/metadomain.h +++ b/src/framework/domain/metadomain.h @@ -21,7 +21,6 @@ #include "arch/kokkos_aliases.h" #include "utils/timer.h" -#include "checkpoint/writer.h" #include "framework/containers/species.h" #include "framework/domain/domain.h" #include "framework/domain/mesh.h" @@ -32,6 +31,7 @@ #endif // MPI_ENABLED #if defined(OUTPUT_ENABLED) + #include "checkpoint/writer.h" #include "output/writer.h" #include diff --git a/src/output/tests/writer-mpi.cpp b/src/output/tests/writer-mpi.cpp index 6b810fa22..074b9acf0 100644 --- a/src/output/tests/writer-mpi.cpp +++ b/src/output/tests/writer-mpi.cpp @@ -36,10 +36,11 @@ auto main(int argc, char* argv[]) -> int { adios2::ADIOS adios { MPI_COMM_WORLD }; auto writer = out::Writer(); - writer.init(&adios, "hdf5"); + writer.init(&adios, "hdf5", "test"); writer.defineMeshLayout({ static_cast(size) * 10 }, { static_cast(rank) * 10 }, { 10 }, + { 1 }, false, Coord::Cart); writer.defineFieldOutputs(SimEngine::SRPIC, { "E" }); @@ -59,11 +60,11 @@ auto main(int argc, char* argv[]) -> int { names.push_back(writer.fieldWriters()[0].name(i)); addresses.push_back(i); } - writer.beginWriting("test", 0, 0.0); + writer.beginWriting(0, 0.0); writer.writeField(names, field, addresses); writer.endWriting(); - writer.beginWriting("test", 1, 0.1); + writer.beginWriting(1, 0.1); writer.writeField(names, field, addresses); writer.endWriting(); From 0cf9baf37667726f8d2e1a1711d5a7c2ae4980c3 Mon Sep 17 00:00:00 2001 From: Sasha Chernoglazov Date: Wed, 23 Oct 2024 14:18:31 -0400 Subject: [PATCH 11/22] bookkeeping for downsampling --- src/framework/domain/output.cpp | 21 +++++++++++++++++---- src/output/writer.cpp | 27 ++++++++++++++++++++------- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/src/framework/domain/output.cpp b/src/framework/domain/output.cpp index 46ca95ede..4a8871324 100644 --- a/src/framework/domain/output.cpp +++ b/src/framework/domain/output.cpp @@ -224,10 +224,16 @@ namespace ntt { for (unsigned short dim = 0; dim < M::Dim; ++dim) { const auto l_size = local_domain->mesh.n_active()[dim]; const auto l_offset = local_domain->offset_ncells()[dim]; + //std::cout << "offset " << l_offset << " " << dim << std::endl; const auto g_size = mesh().n_active()[dim]; const auto dwn_in_dim = dwn[dim]; - const auto l_size_dwn = static_cast(l_size / dwn_in_dim); + double n {l_size}; + double d {dwn_in_dim}; + double l {l_offset}; + double f = math::ceil(l/d)*d-l; + const auto first_cell = static_cast(f); + const auto l_size_dwn = static_cast(math::ceil((n-f) / d)); const auto is_last = l_offset + l_size == g_size; @@ -239,9 +245,12 @@ namespace ntt { const auto offset = (incl_ghosts ? N_GHOSTS : 0); const auto ncells = l_size_dwn; - const auto first_cell = static_cast(l_offset / dwn_in_dim) * - dwn_in_dim - - l_offset; + //const auto first_cell = ((static_cast(l_offset / dwn_in_dim) + 1) * dwn_in_dim - l_offset) % dwn_in_dim; + //const auto first_cell = static_cast(l_offset / dwn_in_dim) * + // dwn_in_dim - + // l_offset; + std::cout << "first cell " << first_cell << " dim " << dim << " l_offset " << l_offset << " dwn_in_dim " << dwn_in_dim << std::endl; + const auto& metric = local_domain->mesh.metric; @@ -251,8 +260,12 @@ namespace ntt { Lambda(index_t i_dwn) { const auto i = first_cell + i_dwn * dwn_in_dim; const auto i_ = static_cast(i); + //if (dim == 1){ + // printf(" i %lu and %f \n", i, i_ ); + // } coord_t x_Cd { ZERO }, x_Ph { ZERO }; x_Cd[dim] = i_ + HALF; + // TODO : change to convert by component metric.template convert(x_Cd, x_Ph); xc(offset + i_dwn) = x_Ph[dim]; x_Cd[dim] = i_; diff --git a/src/output/writer.cpp b/src/output/writer.cpp index 5c6dfe6d6..a63fafa51 100644 --- a/src/output/writer.cpp +++ b/src/output/writer.cpp @@ -77,17 +77,26 @@ namespace out { raise::ErrorIf(dwn[i] != 1 && incl_ghosts, "Downsampling with ghosts not supported", HERE); + + double g = glob_shape[i]; + double d = m_dwn[i]; + double l = loc_corner[i]; + double n = loc_shape[i]; + double f = math::ceil(l/d)*d-l; m_flds_g_shape_dwn.push_back( - static_cast(glob_shape[i] / m_dwn[i])); + static_cast(math::ceil(g / d))); m_flds_l_corner_dwn.push_back( - static_cast(loc_corner[i] / m_dwn[i])); - m_flds_l_shape_dwn.push_back( - static_cast((loc_corner[i] + loc_shape[i]) / m_dwn[i]) - - static_cast(loc_corner[i] / m_dwn[i])); + static_cast(math::ceil(l /d))); m_flds_l_first.push_back( - static_cast(loc_corner[i] / m_dwn[i]) * m_dwn[i] - - loc_corner[i]); + static_cast(f)); + + m_flds_l_shape_dwn.push_back( + static_cast(math::ceil((n - f) / d))); + //m_flds_l_first.push_back( + // static_cast(loc_corner[i] / m_dwn[i]) * m_dwn[i] - + // loc_corner[i]); } + m_io.DefineAttribute("NGhosts", incl_ghosts ? N_GHOSTS : 0); m_io.DefineAttribute("Dimension", m_flds_g_shape.size()); @@ -119,6 +128,9 @@ namespace out { Kokkos::LayoutRight>::value) { m_io.DefineAttribute("LayoutRight", 1); } else { + std::reverse(m_flds_g_shape_dwn.begin(), m_flds_g_shape_dwn.end()); + std::reverse(m_flds_l_corner_dwn.begin(), m_flds_l_corner_dwn.end()); + std::reverse(m_flds_l_shape_dwn.begin(), m_flds_l_shape_dwn.end()); m_io.DefineAttribute("LayoutRight", 0); } } @@ -247,6 +259,7 @@ namespace out { (field.extent(1) - 2 * N_GHOSTS) / dwn2); const auto first_cell1 = first_cell[0]; const auto first_cell2 = first_cell[1]; + printf("%ld %ld : %ld %ld \n", nx1_dwn, nx2_dwn, first_cell1, first_cell2); output_field = array_t { "output_field", nx1_dwn, nx2_dwn }; Kokkos::parallel_for( "outputField", From a0b1ff244c0448e8434c5931847ff3b76d54e0de Mon Sep 17 00:00:00 2001 From: Sasha Chernoglazov Date: Wed, 23 Oct 2024 22:05:07 -0400 Subject: [PATCH 12/22] correct downsampling of field output --- src/framework/domain/output.cpp | 16 ++------ src/output/writer.cpp | 68 +++++++++++++++++---------------- 2 files changed, 39 insertions(+), 45 deletions(-) diff --git a/src/framework/domain/output.cpp b/src/framework/domain/output.cpp index 4a8871324..ac2108a75 100644 --- a/src/framework/domain/output.cpp +++ b/src/framework/domain/output.cpp @@ -224,13 +224,12 @@ namespace ntt { for (unsigned short dim = 0; dim < M::Dim; ++dim) { const auto l_size = local_domain->mesh.n_active()[dim]; const auto l_offset = local_domain->offset_ncells()[dim]; - //std::cout << "offset " << l_offset << " " << dim << std::endl; const auto g_size = mesh().n_active()[dim]; const auto dwn_in_dim = dwn[dim]; - double n {l_size}; - double d {dwn_in_dim}; - double l {l_offset}; + double n = l_size; + double d = dwn_in_dim; + double l = l_offset; double f = math::ceil(l/d)*d-l; const auto first_cell = static_cast(f); const auto l_size_dwn = static_cast(math::ceil((n-f) / d)); @@ -245,13 +244,7 @@ namespace ntt { const auto offset = (incl_ghosts ? N_GHOSTS : 0); const auto ncells = l_size_dwn; - //const auto first_cell = ((static_cast(l_offset / dwn_in_dim) + 1) * dwn_in_dim - l_offset) % dwn_in_dim; - //const auto first_cell = static_cast(l_offset / dwn_in_dim) * - // dwn_in_dim - - // l_offset; - std::cout << "first cell " << first_cell << " dim " << dim << " l_offset " << l_offset << " dwn_in_dim " << dwn_in_dim << std::endl; - const auto& metric = local_domain->mesh.metric; Kokkos::parallel_for( @@ -260,9 +253,6 @@ namespace ntt { Lambda(index_t i_dwn) { const auto i = first_cell + i_dwn * dwn_in_dim; const auto i_ = static_cast(i); - //if (dim == 1){ - // printf(" i %lu and %f \n", i, i_ ); - // } coord_t x_Cd { ZERO }, x_Ph { ZERO }; x_Cd[dim] = i_ + HALF; // TODO : change to convert by component diff --git a/src/output/writer.cpp b/src/output/writer.cpp index a63fafa51..b35e3ab17 100644 --- a/src/output/writer.cpp +++ b/src/output/writer.cpp @@ -83,18 +83,10 @@ namespace out { double l = loc_corner[i]; double n = loc_shape[i]; double f = math::ceil(l/d)*d-l; - m_flds_g_shape_dwn.push_back( - static_cast(math::ceil(g / d))); - m_flds_l_corner_dwn.push_back( - static_cast(math::ceil(l /d))); - m_flds_l_first.push_back( - static_cast(f)); - - m_flds_l_shape_dwn.push_back( - static_cast(math::ceil((n - f) / d))); - //m_flds_l_first.push_back( - // static_cast(loc_corner[i] / m_dwn[i]) * m_dwn[i] - - // loc_corner[i]); + m_flds_g_shape_dwn.push_back(static_cast(math::ceil(g / d))); + m_flds_l_corner_dwn.push_back(static_cast(math::ceil(l /d))); + m_flds_l_first.push_back(static_cast(f)); + m_flds_l_shape_dwn.push_back(static_cast(math::ceil((n - f) / d))); } @@ -229,10 +221,14 @@ namespace out { output_field = array_t { "output_field", slice.extent(0) }; Kokkos::deep_copy(output_field, slice); } else { - const auto dwn1 = dwn[0]; - const auto nx1_dwn = static_cast( - (field.extent(0) - 2 * N_GHOSTS) / dwn1); + + const auto dwn1 = dwn[0]; + const double first_cell1_d = first_cell[0]; + const double nx1_full = field.extent(0) - 2 * N_GHOSTS; const auto first_cell1 = first_cell[0]; + + const auto nx1_dwn = static_cast(math::ceil((nx1_full - first_cell1_d) / dwn1)); + output_field = array_t { "output_field", nx1_dwn }; Kokkos::parallel_for( "outputField", @@ -251,15 +247,17 @@ namespace out { slice.extent(1) }; Kokkos::deep_copy(output_field, slice); } else { - const auto dwn1 = dwn[0]; - const auto dwn2 = dwn[1]; - const auto nx1_dwn = static_cast( - (field.extent(0) - 2 * N_GHOSTS) / dwn1); - const auto nx2_dwn = static_cast( - (field.extent(1) - 2 * N_GHOSTS) / dwn2); + const auto dwn1 = dwn[0]; + const auto dwn2 = dwn[1]; + const double first_cell1_d = first_cell[0]; + const double first_cell2_d = first_cell[1]; + const double nx1_full = field.extent(0) - 2 * N_GHOSTS; + const double nx2_full = field.extent(1) - 2 * N_GHOSTS; const auto first_cell1 = first_cell[0]; const auto first_cell2 = first_cell[1]; - printf("%ld %ld : %ld %ld \n", nx1_dwn, nx2_dwn, first_cell1, first_cell2); + + const auto nx1_dwn = static_cast(math::ceil((nx1_full - first_cell1_d) / dwn1)); + const auto nx2_dwn = static_cast(math::ceil((nx2_full - first_cell2_d) / dwn2)); output_field = array_t { "output_field", nx1_dwn, nx2_dwn }; Kokkos::parallel_for( "outputField", @@ -282,18 +280,24 @@ namespace out { slice.extent(2) }; Kokkos::deep_copy(output_field, slice); } else { - const auto dwn1 = dwn[0]; - const auto dwn2 = dwn[1]; - const auto dwn3 = dwn[2]; - const auto nx1_dwn = static_cast( - (field.extent(0) - 2 * N_GHOSTS) / dwn1); - const auto nx2_dwn = static_cast( - (field.extent(1) - 2 * N_GHOSTS) / dwn2); - const auto nx3_dwn = static_cast( - (field.extent(2) - 2 * N_GHOSTS) / dwn3); + const auto dwn1 = dwn[0]; + const auto dwn2 = dwn[1]; + const auto dwn3 = dwn[2]; + const double first_cell1_d = first_cell[0]; + const double first_cell2_d = first_cell[1]; + const double first_cell3_d = first_cell[2]; + const double nx1_full = field.extent(0) - 2 * N_GHOSTS; + const double nx2_full = field.extent(1) - 2 * N_GHOSTS; + const double nx3_full = field.extent(2) - 2 * N_GHOSTS; const auto first_cell1 = first_cell[0]; const auto first_cell2 = first_cell[1]; - const auto first_cell3 = first_cell[2]; + const auto first_cell3 = first_cell[2]; + + const auto nx1_dwn = static_cast(math::ceil((nx1_full - first_cell1_d) / dwn1)); + const auto nx2_dwn = static_cast(math::ceil((nx2_full - first_cell2_d) / dwn2)); + const auto nx3_dwn = static_cast(math::ceil((nx3_full - first_cell3_d) / dwn3)); + + output_field = array_t { "output_field", nx1_dwn, nx2_dwn, nx3_dwn }; Kokkos::parallel_for( "outputField", From 905fb98167d5ffe304ad8d65372ea84deb82a1b3 Mon Sep 17 00:00:00 2001 From: hayk Date: Mon, 4 Nov 2024 13:24:33 -0500 Subject: [PATCH 13/22] MPI write test --- .gitignore | 1 + src/checkpoint/reader.cpp | 2 +- src/engines/engine_init.cpp | 6 + src/framework/domain/output.cpp | 18 +-- src/output/tests/writer-mpi.cpp | 176 ++++++++++++++++++++++++------ src/output/tests/writer-nompi.cpp | 18 +-- src/output/writer.cpp | 93 ++++++++-------- 7 files changed, 218 insertions(+), 96 deletions(-) diff --git a/.gitignore b/.gitignore index 53d09b648..a1b05e751 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,7 @@ venv/ # CMake testing files Testing/ +.clangd .schema.json *_old/ action-token diff --git a/src/checkpoint/reader.cpp b/src/checkpoint/reader.cpp index 66fcd6757..e89b7d384 100644 --- a/src/checkpoint/reader.cpp +++ b/src/checkpoint/reader.cpp @@ -35,7 +35,7 @@ namespace checkpoint { reader.Get(field_var, array_h.data(), adios2::Mode::Sync); Kokkos::deep_copy(array, array_h); } else { - raise::Error(fmt::format("Field variable: %s not found", field), HERE); + raise::Error(fmt::format("Field variable: %s not found", field.c_str()), HERE); } } diff --git a/src/engines/engine_init.cpp b/src/engines/engine_init.cpp index e4ce9fa5f..0239724e1 100644 --- a/src/engines/engine_init.cpp +++ b/src/engines/engine_init.cpp @@ -50,6 +50,7 @@ namespace ntt { }); } } else { +#if defined(OUTPUT_ENABLED) // read simulation data from the checkpoint raise::ErrorIf( m_params.template get("checkpoint.start_step") == 0, @@ -57,6 +58,11 @@ namespace ntt { HERE); logger::Checkpoint("Resuming simulation from a checkpoint", HERE); m_metadomain.ContinueFromCheckpoint(&m_adios, m_params); +#else + raise::Error( + "Resuming simulation from a checkpoint requires -D output=ON", + HERE); +#endif } } } diff --git a/src/framework/domain/output.cpp b/src/framework/domain/output.cpp index ac2108a75..c7cb6bb65 100644 --- a/src/framework/domain/output.cpp +++ b/src/framework/domain/output.cpp @@ -227,12 +227,14 @@ namespace ntt { const auto g_size = mesh().n_active()[dim]; const auto dwn_in_dim = dwn[dim]; - double n = l_size; - double d = dwn_in_dim; - double l = l_offset; - double f = math::ceil(l/d)*d-l; - const auto first_cell = static_cast(f); - const auto l_size_dwn = static_cast(math::ceil((n-f) / d)); + + const double n = l_size; + const double d = dwn_in_dim; + const double l = l_offset; + const double f = math::ceil(l / d) * d - l; + + const auto first_cell = static_cast(f); + const auto l_size_dwn = static_cast(math::ceil((n - f) / d)); const auto is_last = l_offset + l_size == g_size; @@ -244,7 +246,7 @@ namespace ntt { const auto offset = (incl_ghosts ? N_GHOSTS : 0); const auto ncells = l_size_dwn; - + const auto& metric = local_domain->mesh.metric; Kokkos::parallel_for( @@ -255,7 +257,7 @@ namespace ntt { const auto i_ = static_cast(i); coord_t x_Cd { ZERO }, x_Ph { ZERO }; x_Cd[dim] = i_ + HALF; - // TODO : change to convert by component + // TODO : change to convert by component metric.template convert(x_Cd, x_Ph); xc(offset + i_dwn) = x_Ph[dim]; x_Cd[dim] = i_; diff --git a/src/output/tests/writer-mpi.cpp b/src/output/tests/writer-mpi.cpp index 074b9acf0..c2729f658 100644 --- a/src/output/tests/writer-mpi.cpp +++ b/src/output/tests/writer-mpi.cpp @@ -2,9 +2,7 @@ #include "global.h" #include "arch/mpi_aliases.h" -#include "utils/formatting.h" -#include "output/fields.h" #include "output/writer.h" #include @@ -14,7 +12,6 @@ #include #include -#include #include #include @@ -24,49 +21,154 @@ void cleanup() { fs::remove(tempfile_path); } +#define CEILDIV(a, b) \ + (static_cast(math::ceil(static_cast(a) / static_cast(b)))) + auto main(int argc, char* argv[]) -> int { Kokkos::initialize(argc, argv); MPI_Init(&argc, &argv); - int rank, size; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &size); + int mpi_rank, mpi_size; + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); try { using namespace ntt; + constexpr auto nx1 = 10; + constexpr auto nx1_gh = nx1 + 2 * N_GHOSTS; + constexpr auto i1min = N_GHOSTS; + constexpr auto i1max = nx1 + N_GHOSTS; + constexpr auto dwn1 = 3; + + ndfield_t field { "fld", nx1_gh }; + std::vector field_names; + + { + // fill data + Kokkos::parallel_for( + "fill", + CreateRangePolicy({ i1min }, { i1max }), + Lambda(index_t i1) { + const auto i1_ = static_cast(i1); + field(i1, 0) = i1_; + field(i1, 1) = -i1_; + field(i1, 2) = SQR(i1_); + }); + } adios2::ADIOS adios { MPI_COMM_WORLD }; - auto writer = out::Writer(); - writer.init(&adios, "hdf5", "test"); - writer.defineMeshLayout({ static_cast(size) * 10 }, - { static_cast(rank) * 10 }, - { 10 }, - { 1 }, - false, - Coord::Cart); - writer.defineFieldOutputs(SimEngine::SRPIC, { "E" }); - - ndfield_t field { "fld", 10 + 2 * N_GHOSTS }; - Kokkos::parallel_for( - "fill", - CreateRangePolicy({ N_GHOSTS }, { 10 + N_GHOSTS }), - Lambda(index_t i1) { - field(i1, 0) = i1; - field(i1, 1) = -(real_t)(i1); - field(i1, 2) = i1 / 2; - }); - std::vector names; - std::vector addresses; - for (auto i = 0; i < 3; ++i) { - names.push_back(writer.fieldWriters()[0].name(i)); - addresses.push_back(i); + + { + // write + auto writer = out::Writer(); + writer.init(&adios, "hdf5", "test"); + writer.defineMeshLayout({ static_cast(mpi_size) * nx1 }, + { static_cast(mpi_rank) * nx1 }, + { nx1 }, + { dwn1 }, + false, + Coord::Cart); + writer.defineFieldOutputs(SimEngine::SRPIC, { "E" }); + + std::vector addresses; + for (auto i = 0; i < 3; ++i) { + field_names.push_back(writer.fieldWriters()[0].name(i)); + addresses.push_back(i); + } + writer.beginWriting(0, 0.0); + writer.writeField(field_names, field, addresses); + writer.endWriting(); + + writer.beginWriting(1, 0.1); + writer.writeField(field_names, field, addresses); + writer.endWriting(); + adios.ExitComputationBlock(); } - writer.beginWriting(0, 0.0); - writer.writeField(names, field, addresses); - writer.endWriting(); - writer.beginWriting(1, 0.1); - writer.writeField(names, field, addresses); - writer.endWriting(); + { + // read + adios2::IO io = adios.DeclareIO("read-test"); + io.SetEngine("hdf5"); + adios2::Engine reader = io.Open("test.h5", adios2::Mode::Read, MPI_COMM_SELF); + raise::ErrorIf(io.InquireAttribute("NGhosts").Data()[0] != 0, + "NGhosts is not correct", + HERE); + raise::ErrorIf(io.InquireAttribute("Dimension").Data()[0] != 1, + "Dimension is not correct", + HERE); + for (std::size_t step = 0; reader.BeginStep() == adios2::StepStatus::OK; + ++step) { + std::size_t step_read; + long double time_read; + + reader.Get(io.InquireVariable("Step"), step_read); + reader.Get(io.InquireVariable("Time"), time_read); + raise::ErrorIf(step_read != step, "Step is not correct", HERE); + raise::ErrorIf((float)time_read != (float)step * 0.1f, + "Time is not correct", + HERE); + + const auto l_size = nx1; + const auto l_offset = nx1 * mpi_rank; + const auto g_size = nx1 * mpi_size; + + const double n = l_size; + const double d = dwn1; + const double l = l_offset; + const double f = math::ceil(l / d) * d - l; + + const auto first_cell = static_cast(f); + const auto l_size_dwn = static_cast(math::ceil((n - f) / d)); + const auto l_corner_dwn = static_cast(math::ceil(l / d)); + + array_t field_read {}; + int cntr = 0; + for (const auto& name : field_names) { + auto fieldVar = io.InquireVariable(name); + if (fieldVar) { + raise::ErrorIf(fieldVar.Shape().size() != 1, + fmt::format("%s is not 1D", name.c_str()), + HERE); + auto dims = fieldVar.Shape(); + std::size_t nx1_r = dims[0]; + raise::ErrorIf((nx1_r != CEILDIV(nx1 * mpi_size, dwn1)), + fmt::format("%s = %ld is not %d", + name.c_str(), + nx1_r, + CEILDIV(nx1 * mpi_size, dwn1)), + HERE); + + fieldVar.SetSelection( + adios2::Box({ l_corner_dwn }, { l_size_dwn })); + field_read = array_t(name, l_size_dwn); + auto field_read_h = Kokkos::create_mirror_view(field_read); + reader.Get(fieldVar, field_read_h.data(), adios2::Mode::Sync); + Kokkos::deep_copy(field_read, field_read_h); + + Kokkos::parallel_for( + "check", + CreateRangePolicy({ 0 }, { l_size_dwn }), + Lambda(index_t i1) { + if (not cmp::AlmostEqual( + field_read(i1), + field(i1 * dwn1 + first_cell + i1min, cntr))) { + printf("\n:::::::::::::::\nfield_read(%ld) = %f != " + "field(%ld, %d) = %f\n:::::::::::::::\n", + i1, + field_read(i1), + i1 * dwn1 + first_cell + i1min, + cntr, + field(i1 * dwn1 + first_cell + i1min, cntr)); + raise::KernelError(HERE, "Field is not read correctly"); + } + }); + } else { + raise::Error("Field not found", HERE); + } + ++cntr; + } + } + reader.Close(); + } } catch (std::exception& e) { std::cerr << e.what() << std::endl; @@ -82,3 +184,5 @@ auto main(int argc, char* argv[]) -> int { Kokkos::finalize(); return 0; } + +#undef CEILDIV diff --git a/src/output/tests/writer-nompi.cpp b/src/output/tests/writer-nompi.cpp index 4c032094b..3fe42bf1b 100644 --- a/src/output/tests/writer-nompi.cpp +++ b/src/output/tests/writer-nompi.cpp @@ -3,7 +3,6 @@ #include "utils/formatting.h" -#include "output/fields.h" #include "output/writer.h" #include @@ -12,7 +11,6 @@ #include #include -#include #include #include @@ -24,6 +22,9 @@ void cleanup() { fs::remove(tempfile_path); } +#define CEILDIV(a, b) \ + (static_cast(math::ceil(static_cast(a) / static_cast(b)))) + auto main(int argc, char* argv[]) -> int { Kokkos::initialize(argc, argv); @@ -131,16 +132,17 @@ auto main(int argc, char* argv[]) -> int { std::size_t nx1_r = dims[0]; std::size_t nx2_r = dims[1]; std::size_t nx3_r = dims[2]; - raise::ErrorIf((nx1_r != nx1 / dwn1) || (nx2_r != nx2 / dwn2) || - (nx3_r != nx3 / dwn3), + raise::ErrorIf((nx1_r != CEILDIV(nx1, dwn1)) || + (nx2_r != CEILDIV(nx2, dwn2)) || + (nx3_r != CEILDIV(nx3, dwn3)), fmt::format("%s = %ldx%ldx%ld is not %dx%dx%d", name.c_str(), nx1_r, nx2_r, nx3_r, - nx1 / dwn1, - nx2 / dwn2, - nx3 / dwn3), + CEILDIV(nx1, dwn1), + CEILDIV(nx2, dwn2), + CEILDIV(nx3, dwn3)), HERE); fieldVar.SetSelection( @@ -195,3 +197,5 @@ auto main(int argc, char* argv[]) -> int { Kokkos::finalize(); return 0; } + +#undef CEILDIV diff --git a/src/output/writer.cpp b/src/output/writer.cpp index b35e3ab17..4ba0ea14c 100644 --- a/src/output/writer.cpp +++ b/src/output/writer.cpp @@ -78,17 +78,17 @@ namespace out { "Downsampling with ghosts not supported", HERE); - double g = glob_shape[i]; - double d = m_dwn[i]; - double l = loc_corner[i]; - double n = loc_shape[i]; - double f = math::ceil(l/d)*d-l; + const double g = glob_shape[i]; + const double d = m_dwn[i]; + const double l = loc_corner[i]; + const double n = loc_shape[i]; + const double f = math::ceil(l / d) * d - l; m_flds_g_shape_dwn.push_back(static_cast(math::ceil(g / d))); - m_flds_l_corner_dwn.push_back(static_cast(math::ceil(l /d))); + m_flds_l_corner_dwn.push_back(static_cast(math::ceil(l / d))); m_flds_l_first.push_back(static_cast(f)); - m_flds_l_shape_dwn.push_back(static_cast(math::ceil((n - f) / d))); + m_flds_l_shape_dwn.push_back( + static_cast(math::ceil((n - f) / d))); } - m_io.DefineAttribute("NGhosts", incl_ghosts ? N_GHOSTS : 0); m_io.DefineAttribute("Dimension", m_flds_g_shape.size()); @@ -222,14 +222,15 @@ namespace out { Kokkos::deep_copy(output_field, slice); } else { - const auto dwn1 = dwn[0]; - const double first_cell1_d = first_cell[0]; - const double nx1_full = field.extent(0) - 2 * N_GHOSTS; - const auto first_cell1 = first_cell[0]; + const auto dwn1 = dwn[0]; + const double first_cell1_d = first_cell[0]; + const double nx1_full = field.extent(0) - 2 * N_GHOSTS; + const auto first_cell1 = first_cell[0]; - const auto nx1_dwn = static_cast(math::ceil((nx1_full - first_cell1_d) / dwn1)); - - output_field = array_t { "output_field", nx1_dwn }; + const auto nx1_dwn = static_cast( + math::ceil((nx1_full - first_cell1_d) / dwn1)); + + output_field = array_t { "output_field", nx1_dwn }; Kokkos::parallel_for( "outputField", nx1_dwn, @@ -247,17 +248,19 @@ namespace out { slice.extent(1) }; Kokkos::deep_copy(output_field, slice); } else { - const auto dwn1 = dwn[0]; - const auto dwn2 = dwn[1]; - const double first_cell1_d = first_cell[0]; - const double first_cell2_d = first_cell[1]; - const double nx1_full = field.extent(0) - 2 * N_GHOSTS; - const double nx2_full = field.extent(1) - 2 * N_GHOSTS; - const auto first_cell1 = first_cell[0]; - const auto first_cell2 = first_cell[1]; - - const auto nx1_dwn = static_cast(math::ceil((nx1_full - first_cell1_d) / dwn1)); - const auto nx2_dwn = static_cast(math::ceil((nx2_full - first_cell2_d) / dwn2)); + const auto dwn1 = dwn[0]; + const auto dwn2 = dwn[1]; + const double first_cell1_d = first_cell[0]; + const double first_cell2_d = first_cell[1]; + const double nx1_full = field.extent(0) - 2 * N_GHOSTS; + const double nx2_full = field.extent(1) - 2 * N_GHOSTS; + const auto first_cell1 = first_cell[0]; + const auto first_cell2 = first_cell[1]; + + const auto nx1_dwn = static_cast( + math::ceil((nx1_full - first_cell1_d) / dwn1)); + const auto nx2_dwn = static_cast( + math::ceil((nx2_full - first_cell2_d) / dwn2)); output_field = array_t { "output_field", nx1_dwn, nx2_dwn }; Kokkos::parallel_for( "outputField", @@ -280,24 +283,26 @@ namespace out { slice.extent(2) }; Kokkos::deep_copy(output_field, slice); } else { - const auto dwn1 = dwn[0]; - const auto dwn2 = dwn[1]; - const auto dwn3 = dwn[2]; - const double first_cell1_d = first_cell[0]; - const double first_cell2_d = first_cell[1]; - const double first_cell3_d = first_cell[2]; - const double nx1_full = field.extent(0) - 2 * N_GHOSTS; - const double nx2_full = field.extent(1) - 2 * N_GHOSTS; - const double nx3_full = field.extent(2) - 2 * N_GHOSTS; - const auto first_cell1 = first_cell[0]; - const auto first_cell2 = first_cell[1]; - const auto first_cell3 = first_cell[2]; - - const auto nx1_dwn = static_cast(math::ceil((nx1_full - first_cell1_d) / dwn1)); - const auto nx2_dwn = static_cast(math::ceil((nx2_full - first_cell2_d) / dwn2)); - const auto nx3_dwn = static_cast(math::ceil((nx3_full - first_cell3_d) / dwn3)); - - + const auto dwn1 = dwn[0]; + const auto dwn2 = dwn[1]; + const auto dwn3 = dwn[2]; + const double first_cell1_d = first_cell[0]; + const double first_cell2_d = first_cell[1]; + const double first_cell3_d = first_cell[2]; + const double nx1_full = field.extent(0) - 2 * N_GHOSTS; + const double nx2_full = field.extent(1) - 2 * N_GHOSTS; + const double nx3_full = field.extent(2) - 2 * N_GHOSTS; + const auto first_cell1 = first_cell[0]; + const auto first_cell2 = first_cell[1]; + const auto first_cell3 = first_cell[2]; + + const auto nx1_dwn = static_cast( + math::ceil((nx1_full - first_cell1_d) / dwn1)); + const auto nx2_dwn = static_cast( + math::ceil((nx2_full - first_cell2_d) / dwn2)); + const auto nx3_dwn = static_cast( + math::ceil((nx3_full - first_cell3_d) / dwn3)); + output_field = array_t { "output_field", nx1_dwn, nx2_dwn, nx3_dwn }; Kokkos::parallel_for( "outputField", From af5ac780f96043ce277b1e2df0c76d771ade6e93 Mon Sep 17 00:00:00 2001 From: hayk Date: Mon, 4 Nov 2024 14:22:13 -0500 Subject: [PATCH 14/22] cmake formatting --- CMakeLists.txt | 67 ++++++------ cmake/MPIConfig.cmake | 3 +- cmake/adios2Config.cmake | 24 +++-- cmake/config.cmake | 31 ++++-- cmake/defaults.cmake | 76 ++++++++++---- cmake/dependencies.cmake | 106 ++++++++++++------- cmake/kokkosConfig.cmake | 52 +++++++--- cmake/report.cmake | 155 +++++++++++++++++----------- cmake/styling.cmake | 27 +++-- cmake/tests.cmake | 37 ++++--- setups/CMakeLists.txt | 28 ++--- src/CMakeLists.txt | 35 ++++--- src/archetypes/CMakeLists.txt | 20 ++-- src/archetypes/tests/CMakeLists.txt | 8 +- src/checkpoint/CMakeLists.txt | 33 +++--- src/checkpoint/tests/CMakeLists.txt | 12 ++- src/engines/CMakeLists.txt | 58 ++++++----- src/framework/CMakeLists.txt | 71 +++++++------ src/framework/tests/CMakeLists.txt | 50 +++++---- src/global/CMakeLists.txt | 48 ++++----- src/global/tests/CMakeLists.txt | 12 ++- src/kernels/CMakeLists.txt | 20 ++-- src/kernels/tests/CMakeLists.txt | 8 +- src/metrics/CMakeLists.txt | 16 ++- src/metrics/tests/CMakeLists.txt | 13 ++- src/output/CMakeLists.txt | 37 ++++--- src/output/tests/CMakeLists.txt | 28 +++-- 27 files changed, 662 insertions(+), 413 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 62319559b..4ee00d1b4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,12 +8,13 @@ project( VERSION 1.2.0 LANGUAGES CXX C) add_compile_options("-D ENTITY_VERSION=\"${PROJECT_VERSION}\"") -execute_process(COMMAND - bash -c "git diff --quiet src/ && echo $(git rev-parse HEAD) || echo $(git rev-parse HEAD)-mod" +execute_process( + COMMAND + bash -c + "git diff --quiet src/ && echo $(git rev-parse HEAD) || echo $(git rev-parse HEAD)-mod" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" OUTPUT_VARIABLE GIT_HASH - ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE -) + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) message(STATUS "Git hash: ${GIT_HASH}") add_compile_options("-D ENTITY_GIT_HASH=\"${GIT_HASH}\"") @@ -25,56 +26,57 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/defaults.cmake) # defaults set(DEBUG - ${default_debug} - CACHE BOOL "Debug mode") + ${default_debug} + CACHE BOOL "Debug mode") set(precision - ${default_precision} - CACHE STRING "Precision") + ${default_precision} + CACHE STRING "Precision") set(pgen - ${default_pgen} - CACHE STRING "Problem generator") + ${default_pgen} + CACHE STRING "Problem generator") set(gui - ${default_gui} - CACHE BOOL "Use GUI [nttiny]") + ${default_gui} + CACHE BOOL "Use GUI [nttiny]") set(output - ${default_output} - CACHE BOOL "Enable output") + ${default_output} + CACHE BOOL "Enable output") set(mpi - ${default_mpi} - CACHE BOOL "Use MPI") + ${default_mpi} + CACHE BOOL "Use MPI") # -------------------------- Compilation settings -------------------------- # set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) if(${DEBUG} STREQUAL "OFF") set(CMAKE_BUILD_TYPE - Release - CACHE STRING "CMake build type") + Release + CACHE STRING "CMake build type") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DNDEBUG") else() set(CMAKE_BUILD_TYPE - Debug - CACHE STRING "CMake build type") + Debug + CACHE STRING "CMake build type") set(CMAKE_CXX_FLAGS - "${CMAKE_CXX_FLAGS} -DDEBUG -Wall -Wextra -Wno-unknown-pragmas") + "${CMAKE_CXX_FLAGS} -DDEBUG -Wall -Wextra -Wno-unknown-pragmas") endif() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-local-typedefs") # options set(precisions - "single" "double" - CACHE STRING "Precisions") + "single" "double" + CACHE STRING "Precisions") include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/config.cmake) # ------------------------- Third-Party Tests ------------------------------ # set(BUILD_TESTING - OFF - CACHE BOOL "Build tests") + OFF + CACHE BOOL "Build tests") # ------------------------ Third-party dependencies ------------------------ # include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/kokkosConfig.cmake) @@ -98,12 +100,12 @@ endif() if(${output}) include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/adios2Config.cmake) find_or_fetch_dependency(adios2 FALSE) - if (NOT DEFINED ENV{HDF5_ROOT}) + if(NOT DEFINED ENV{HDF5_ROOT}) set(USE_CUSTOM_HDF5 OFF) - if (DEFINED ENV{CONDA_PREFIX}) + if(DEFINED ENV{CONDA_PREFIX}) execute_process(COMMAND bash -c "conda list | grep \"hdf5\" -q" - RESULT_VARIABLE HDF5_INSTALLED) - if (HDF5_INSTALLED EQUAL 0) + RESULT_VARIABLE HDF5_INSTALLED) + if(HDF5_INSTALLED EQUAL 0) set(HDF5_ROOT $ENV{CONDA_PREFIX}) else() set(USE_CUSTOM_HDF5 ON) @@ -111,8 +113,11 @@ if(${output}) else() set(USE_CUSTOM_HDF5 ON) endif() - if (USE_CUSTOM_HDF5) - message(FATAL_ERROR "HDF5_ROOT is not set. Please set it to the root of the HDF5 installation") + if(USE_CUSTOM_HDF5) + message( + FATAL_ERROR + "HDF5_ROOT is not set. Please set it to the root of the HDF5 installation" + ) endif() endif() find_package(HDF5 REQUIRED) diff --git a/cmake/MPIConfig.cmake b/cmake/MPIConfig.cmake index b426641ec..d1bfeaab2 100644 --- a/cmake/MPIConfig.cmake +++ b/cmake/MPIConfig.cmake @@ -1,3 +1,4 @@ find_package(MPI REQUIRED) include_directories(${MPI_CXX_INCLUDE_PATH}) -add_compile_options("-D MPI_ENABLED") \ No newline at end of file +add_compile_options("-D MPI_ENABLED") + diff --git a/cmake/adios2Config.cmake b/cmake/adios2Config.cmake index 16c0c30c7..5c480f3d8 100644 --- a/cmake/adios2Config.cmake +++ b/cmake/adios2Config.cmake @@ -1,15 +1,27 @@ # ----------------------------- Adios2 settings ---------------------------- # -set(ADIOS2_BUILD_EXAMPLES OFF CACHE BOOL "Build ADIOS2 examples") +set(ADIOS2_BUILD_EXAMPLES + OFF + CACHE BOOL "Build ADIOS2 examples") # Language support -set(ADIOS2_USE_Python OFF CACHE BOOL "Use Python for ADIOS2") -set(ADIOS2_USE_Fortran OFF CACHE BOOL "Use Fortran for ADIOS2") +set(ADIOS2_USE_Python + OFF + CACHE BOOL "Use Python for ADIOS2") +set(ADIOS2_USE_Fortran + OFF + CACHE BOOL "Use Fortran for ADIOS2") # Format/compression support -set(ADIOS2_USE_ZeroMQ OFF CACHE BOOL "Use ZeroMQ for ADIOS2") +set(ADIOS2_USE_ZeroMQ + OFF + CACHE BOOL "Use ZeroMQ for ADIOS2") -set(ADIOS2_USE_MPI ${mpi} CACHE BOOL "Use MPI for ADIOS2") +set(ADIOS2_USE_MPI + ${mpi} + CACHE BOOL "Use MPI for ADIOS2") -set(ADIOS2_USE_CUDA OFF CACHE BOOL "Use CUDA for ADIOS2") +set(ADIOS2_USE_CUDA + OFF + CACHE BOOL "Use CUDA for ADIOS2") add_compile_options("-D OUTPUT_ENABLED") diff --git a/cmake/config.cmake b/cmake/config.cmake index fa18a87eb..58dd467e9 100644 --- a/cmake/config.cmake +++ b/cmake/config.cmake @@ -3,7 +3,10 @@ function(set_precision precision_name) list(FIND precisions ${precision_name} PRECISION_FOUND) if(${PRECISION_FOUND} EQUAL -1) - message(FATAL_ERROR "Invalid precision: ${precision_name}\nValid options are: ${precisions}") + message( + FATAL_ERROR + "Invalid precision: ${precision_name}\nValid options are: ${precisions}" + ) endif() if(${precision_name} STREQUAL "single") @@ -13,19 +16,31 @@ endfunction() # ---------------------------- Problem generator --------------------------- # function(set_problem_generator pgen_name) - file(GLOB_RECURSE PGENS "${CMAKE_CURRENT_SOURCE_DIR}/setups/**/pgen.hpp" "${CMAKE_CURRENT_SOURCE_DIR}/setups/pgen.hpp") + file(GLOB_RECURSE PGENS "${CMAKE_CURRENT_SOURCE_DIR}/setups/**/pgen.hpp" + "${CMAKE_CURRENT_SOURCE_DIR}/setups/pgen.hpp") foreach(PGEN ${PGENS}) get_filename_component(PGEN_NAME ${PGEN} DIRECTORY) - string(REPLACE "${CMAKE_CURRENT_SOURCE_DIR}/setups/" "" PGEN_NAME ${PGEN_NAME}) - string(REPLACE "${CMAKE_CURRENT_SOURCE_DIR}/setups" "" PGEN_NAME ${PGEN_NAME}) + string(REPLACE "${CMAKE_CURRENT_SOURCE_DIR}/setups/" "" PGEN_NAME + ${PGEN_NAME}) + string(REPLACE "${CMAKE_CURRENT_SOURCE_DIR}/setups" "" PGEN_NAME + ${PGEN_NAME}) list(APPEND PGEN_NAMES ${PGEN_NAME}) endforeach() list(FIND PGEN_NAMES ${pgen_name} PGEN_FOUND) if(NOT ${pgen_name} STREQUAL "." AND ${PGEN_FOUND} EQUAL -1) - message(FATAL_ERROR "Invalid problem generator: ${pgen_name}\nValid options are: ${PGEN_NAMES}") + message( + FATAL_ERROR + "Invalid problem generator: ${pgen_name}\nValid options are: ${PGEN_NAMES}" + ) endif() - set(PGEN ${pgen_name} PARENT_SCOPE) + set(PGEN + ${pgen_name} + PARENT_SCOPE) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/setups/${pgen_name}) - set(PGEN_FOUND TRUE PARENT_SCOPE) - set(problem_generators ${PGEN_NAMES} PARENT_SCOPE) + set(PGEN_FOUND + TRUE + PARENT_SCOPE) + set(problem_generators + ${PGEN_NAMES} + PARENT_SCOPE) endfunction() diff --git a/cmake/defaults.cmake b/cmake/defaults.cmake index f70120e0d..46b4609c5 100644 --- a/cmake/defaults.cmake +++ b/cmake/defaults.cmake @@ -1,62 +1,100 @@ # ----------------------------- Defaults ---------------------------------- # if(DEFINED ENV{Entity_ENABLE_DEBUG}) - set(default_debug $ENV{Entity_ENABLE_DEBUG} CACHE INTERNAL "Default flag for debug mode") + set(default_debug + $ENV{Entity_ENABLE_DEBUG} + CACHE INTERNAL "Default flag for debug mode") else() - set(default_debug OFF CACHE INTERNAL "Default flag for debug mode") + set(default_debug + OFF + CACHE INTERNAL "Default flag for debug mode") endif() set_property(CACHE default_debug PROPERTY TYPE BOOL) -set(default_engine "pic" CACHE INTERNAL "Default engine") -set(default_precision "single" CACHE INTERNAL "Default precision") -set(default_pgen "." CACHE INTERNAL "Default problem generator") -set(default_sr_metric "minkowski" CACHE INTERNAL "Default SR metric") -set(default_gr_metric "kerr_schild" CACHE INTERNAL "Default GR metric") +set(default_engine + "pic" + CACHE INTERNAL "Default engine") +set(default_precision + "single" + CACHE INTERNAL "Default precision") +set(default_pgen + "." + CACHE INTERNAL "Default problem generator") +set(default_sr_metric + "minkowski" + CACHE INTERNAL "Default SR metric") +set(default_gr_metric + "kerr_schild" + CACHE INTERNAL "Default GR metric") if(DEFINED ENV{Entity_ENABLE_OUTPUT}) - set(default_output $ENV{Entity_ENABLE_OUTPUT} CACHE INTERNAL "Default flag for output") + set(default_output + $ENV{Entity_ENABLE_OUTPUT} + CACHE INTERNAL "Default flag for output") else() - set(default_output OFF CACHE INTERNAL "Default flag for output") + set(default_output + OFF + CACHE INTERNAL "Default flag for output") endif() set_property(CACHE default_output PROPERTY TYPE BOOL) if(DEFINED ENV{Entity_ENABLE_GUI}) - set(default_gui $ENV{Entity_ENABLE_GUI} CACHE INTERNAL "Default flag for GUI") + set(default_gui + $ENV{Entity_ENABLE_GUI} + CACHE INTERNAL "Default flag for GUI") else() - set(default_gui OFF CACHE INTERNAL "Default flag for GUI") + set(default_gui + OFF + CACHE INTERNAL "Default flag for GUI") endif() set_property(CACHE default_gui PROPERTY TYPE BOOL) if(DEFINED ENV{Kokkos_ENABLE_CUDA}) - set(default_KOKKOS_ENABLE_CUDA $ENV{Kokkos_ENABLE_CUDA} CACHE INTERNAL "Default flag for CUDA") + set(default_KOKKOS_ENABLE_CUDA + $ENV{Kokkos_ENABLE_CUDA} + CACHE INTERNAL "Default flag for CUDA") else() - set(default_KOKKOS_ENABLE_CUDA OFF CACHE INTERNAL "Default flag for CUDA") + set(default_KOKKOS_ENABLE_CUDA + OFF + CACHE INTERNAL "Default flag for CUDA") endif() set_property(CACHE default_KOKKOS_ENABLE_CUDA PROPERTY TYPE BOOL) if(DEFINED ENV{Kokkos_ENABLE_HIP}) - set(default_KOKKOS_ENABLE_HIP $ENV{Kokkos_ENABLE_HIP} CACHE INTERNAL "Default flag for HIP") + set(default_KOKKOS_ENABLE_HIP + $ENV{Kokkos_ENABLE_HIP} + CACHE INTERNAL "Default flag for HIP") else() - set(default_KOKKOS_ENABLE_HIP OFF CACHE INTERNAL "Default flag for HIP") + set(default_KOKKOS_ENABLE_HIP + OFF + CACHE INTERNAL "Default flag for HIP") endif() set_property(CACHE default_KOKKOS_ENABLE_HIP PROPERTY TYPE BOOL) if(DEFINED ENV{Kokkos_ENABLE_OPENMP}) - set(default_KOKKOS_ENABLE_OPENMP $ENV{Kokkos_ENABLE_OPENMP} CACHE INTERNAL "Default flag for OpenMP") + set(default_KOKKOS_ENABLE_OPENMP + $ENV{Kokkos_ENABLE_OPENMP} + CACHE INTERNAL "Default flag for OpenMP") else() - set(default_KOKKOS_ENABLE_OPENMP OFF CACHE INTERNAL "Default flag for OpenMP") + set(default_KOKKOS_ENABLE_OPENMP + OFF + CACHE INTERNAL "Default flag for OpenMP") endif() set_property(CACHE default_KOKKOS_ENABLE_OPENMP PROPERTY TYPE BOOL) if(DEFINED ENV{Entity_ENABLE_MPI}) - set(default_mpi $ENV{Entity_ENABLE_MPI} CACHE INTERNAL "Default flag for MPI") + set(default_mpi + $ENV{Entity_ENABLE_MPI} + CACHE INTERNAL "Default flag for MPI") else() - set(default_mpi OFF CACHE INTERNAL "Default flag for MPI") + set(default_mpi + OFF + CACHE INTERNAL "Default flag for MPI") endif() set_property(CACHE default_mpi PROPERTY TYPE BOOL) diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index b143befdf..06a3e6a1f 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -1,24 +1,34 @@ -set(Kokkos_REPOSITORY https://github.com/kokkos/kokkos.git CACHE STRING "Kokkos repository") -set(plog_REPOSITORY https://github.com/SergiusTheBest/plog.git CACHE STRING "plog repository") -set(toml11_REPOSITORY https://github.com/ToruNiina/toml11 CACHE STRING "toml11 repository") +set(Kokkos_REPOSITORY + https://github.com/kokkos/kokkos.git + CACHE STRING "Kokkos repository") +set(plog_REPOSITORY + https://github.com/SergiusTheBest/plog.git + CACHE STRING "plog repository") -# set (adios2_REPOSITORY https://github.com/ornladios/ADIOS2.git CACHE STRING "ADIOS2 repository") +# set (adios2_REPOSITORY https://github.com/ornladios/ADIOS2.git CACHE STRING +# "ADIOS2 repository") function(check_internet_connection) if(OFFLINE STREQUAL "ON") - set(FETCHCONTENT_FULLY_DISCONNECTED ON CACHE BOOL "Connection status") + set(FETCHCONTENT_FULLY_DISCONNECTED + ON + CACHE BOOL "Connection status") message(STATUS "${Blue}Offline mode.${ColorReset}") else() execute_process( COMMAND ping 8.8.8.8 -c 2 RESULT_VARIABLE NO_CONNECTION - OUTPUT_QUIET - ) + OUTPUT_QUIET) if(NO_CONNECTION GREATER 0) - set(FETCHCONTENT_FULLY_DISCONNECTED ON CACHE BOOL "Connection status") - message(STATUS "${Red}No internet connection. Fetching disabled.${ColorReset}") + set(FETCHCONTENT_FULLY_DISCONNECTED + ON + CACHE BOOL "Connection status") + message( + STATUS "${Red}No internet connection. Fetching disabled.${ColorReset}") else() - set(FETCHCONTENT_FULLY_DISCONNECTED OFF CACHE BOOL "Connection status") + set(FETCHCONTENT_FULLY_DISCONNECTED + OFF + CACHE BOOL "Connection status") message(STATUS "${Green}Internet connection established.${ColorReset}") endif() endif() @@ -30,66 +40,92 @@ function(find_or_fetch_dependency package_name header_only) endif() if(NOT ${package_name}_FOUND) - if(DEFINED ${package_name}_REPOSITORY AND NOT FETCHCONTENT_FULLY_DISCONNECTED) + if(DEFINED ${package_name}_REPOSITORY AND NOT + FETCHCONTENT_FULLY_DISCONNECTED) # fetching package - message(STATUS "${Blue}${package_name} not found. Fetching from ${${package_name}_REPOSITORY}${ColorReset}") + message( + STATUS + "${Blue}${package_name} not found. Fetching from ${${package_name}_REPOSITORY}${ColorReset}" + ) include(FetchContent) if(${package_name} STREQUAL "Kokkos") FetchContent_Declare( ${package_name} GIT_REPOSITORY ${${package_name}_REPOSITORY} - GIT_TAG 4.3.00 - ) + GIT_TAG 4.3.00) else() - FetchContent_Declare( - ${package_name} - GIT_REPOSITORY ${${package_name}_REPOSITORY} - ) + FetchContent_Declare(${package_name} + GIT_REPOSITORY ${${package_name}_REPOSITORY}) endif() FetchContent_MakeAvailable(${package_name}) set(lower_pckg_name ${package_name}) string(TOLOWER ${lower_pckg_name} lower_pckg_name) - set(${package_name}_SRC ${CMAKE_CURRENT_BINARY_DIR}/_deps/${lower_pckg_name}-src CACHE PATH "Path to ${package_name} src") - set(${package_name}_FETCHED TRUE CACHE BOOL "Whether ${package_name} was fetched") + set(${package_name}_SRC + ${CMAKE_CURRENT_BINARY_DIR}/_deps/${lower_pckg_name}-src + CACHE PATH "Path to ${package_name} src") + set(${package_name}_FETCHED + TRUE + CACHE BOOL "Whether ${package_name} was fetched") message(STATUS "${Green}${package_name} fetched.${ColorReset}") else() # get as submodule - message(STATUS "${Yellow}${package_name} not found. Using as submodule.${ColorReset}") + message( + STATUS + "${Yellow}${package_name} not found. Using as submodule.${ColorReset}" + ) - set(${package_name}_FETCHED FALSE CACHE BOOL "Whether ${package_name} was fetched") + set(${package_name}_FETCHED + FALSE + CACHE BOOL "Whether ${package_name} was fetched") if(NOT FETCHCONTENT_FULLY_DISCONNECTED) - message(STATUS "${GREEN}Updating ${package_name} submodule.${ColorReset}") + message( + STATUS "${GREEN}Updating ${package_name} submodule.${ColorReset}") execute_process( - COMMAND git submodule update --init --remote ${CMAKE_CURRENT_SOURCE_DIR}/extern/${package_name} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - ) + COMMAND git submodule update --init --remote + ${CMAKE_CURRENT_SOURCE_DIR}/extern/${package_name} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) endif() - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extern/${package_name} extern/${package_name}) - set(${package_name}_SRC ${CMAKE_CURRENT_SOURCE_DIR}/extern/${package_name} CACHE PATH "Path to ${package_name} src") - set(${package_name}_BUILD_DIR ${CMAKE_CURRENT_SOURCE_DIR}/build/extern/${package_name} CACHE PATH "Path to ${package_name} build") + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extern/${package_name} + extern/${package_name}) + set(${package_name}_SRC + ${CMAKE_CURRENT_SOURCE_DIR}/extern/${package_name} + CACHE PATH "Path to ${package_name} src") + set(${package_name}_BUILD_DIR + ${CMAKE_CURRENT_SOURCE_DIR}/build/extern/${package_name} + CACHE PATH "Path to ${package_name} build") endif() else() message(STATUS "${Green}${package_name} found.${ColorReset}") - set(${package_name}_FETCHED FALSE CACHE BOOL "Whether ${package_name} was fetched") - set(${package_name}_VERSION ${${package_name}_VERSION} CACHE INTERNAL "${package_name} version") + set(${package_name}_FETCHED + FALSE + CACHE BOOL "Whether ${package_name} was fetched") + set(${package_name}_VERSION + ${${package_name}_VERSION} + CACHE INTERNAL "${package_name} version") endif() if(${package_name} STREQUAL "adios2") if(NOT DEFINED adios2_VERSION OR adios2_VERSION STREQUAL "") - get_directory_property(adios2_VERSION DIRECTORY ${adios2_BUILD_DIR} DEFINITION ADIOS2_VERSION) - set(adios2_VERSION ${adios2_VERSION} CACHE INTERNAL "ADIOS2 version") + get_directory_property(adios2_VERSION DIRECTORY ${adios2_BUILD_DIR} + DEFINITION ADIOS2_VERSION) + set(adios2_VERSION + ${adios2_VERSION} + CACHE INTERNAL "ADIOS2 version") endif() endif() if(${package_name} STREQUAL "Kokkos") if(NOT DEFINED Kokkos_VERSION OR Kokkos_VERSION STREQUAL "") - get_directory_property(Kokkos_VERSION DIRECTORY ${Kokkos_SRC} DEFINITION Kokkos_VERSION) - set(Kokkos_VERSION ${Kokkos_VERSION} CACHE INTERNAL "Kokkos version") + get_directory_property(Kokkos_VERSION DIRECTORY ${Kokkos_SRC} DEFINITION + Kokkos_VERSION) + set(Kokkos_VERSION + ${Kokkos_VERSION} + CACHE INTERNAL "Kokkos version") endif() endif() endfunction() diff --git a/cmake/kokkosConfig.cmake b/cmake/kokkosConfig.cmake index 8928253ae..63c32622d 100644 --- a/cmake/kokkosConfig.cmake +++ b/cmake/kokkosConfig.cmake @@ -1,19 +1,41 @@ # ----------------------------- Kokkos settings ---------------------------- # if(${DEBUG} STREQUAL "OFF") - set(Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION ON CACHE BOOL "Kokkos aggressive vectorization") - set(Kokkos_ENABLE_COMPILER_WARNINGS OFF CACHE BOOL "Kokkos compiler warnings") - set(Kokkos_ENABLE_DEBUG OFF CACHE BOOL "Kokkos debug") - set(Kokkos_ENABLE_DEBUG_BOUNDS_CHECK OFF CACHE BOOL "Kokkos debug bounds check") + set(Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION + ON + CACHE BOOL "Kokkos aggressive vectorization") + set(Kokkos_ENABLE_COMPILER_WARNINGS + OFF + CACHE BOOL "Kokkos compiler warnings") + set(Kokkos_ENABLE_DEBUG + OFF + CACHE BOOL "Kokkos debug") + set(Kokkos_ENABLE_DEBUG_BOUNDS_CHECK + OFF + CACHE BOOL "Kokkos debug bounds check") else() - set(Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION OFF CACHE BOOL "Kokkos aggressive vectorization") - set(Kokkos_ENABLE_COMPILER_WARNINGS ON CACHE BOOL "Kokkos compiler warnings") - set(Kokkos_ENABLE_DEBUG ON CACHE BOOL "Kokkos debug") - set(Kokkos_ENABLE_DEBUG_BOUNDS_CHECK ON CACHE BOOL "Kokkos debug bounds check") + set(Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION + OFF + CACHE BOOL "Kokkos aggressive vectorization") + set(Kokkos_ENABLE_COMPILER_WARNINGS + ON + CACHE BOOL "Kokkos compiler warnings") + set(Kokkos_ENABLE_DEBUG + ON + CACHE BOOL "Kokkos debug") + set(Kokkos_ENABLE_DEBUG_BOUNDS_CHECK + ON + CACHE BOOL "Kokkos debug bounds check") endif() -set(Kokkos_ENABLE_HIP ${default_KOKKOS_ENABLE_HIP} CACHE BOOL "Enable HIP") -set(Kokkos_ENABLE_CUDA ${default_KOKKOS_ENABLE_CUDA} CACHE BOOL "Enable CUDA") -set(Kokkos_ENABLE_OPENMP ${default_KOKKOS_ENABLE_OPENMP} CACHE BOOL "Enable OpenMP") +set(Kokkos_ENABLE_HIP + ${default_KOKKOS_ENABLE_HIP} + CACHE BOOL "Enable HIP") +set(Kokkos_ENABLE_CUDA + ${default_KOKKOS_ENABLE_CUDA} + CACHE BOOL "Enable CUDA") +set(Kokkos_ENABLE_OPENMP + ${default_KOKKOS_ENABLE_OPENMP} + CACHE BOOL "Enable OpenMP") # set memory space if(${Kokkos_ENABLE_CUDA}) @@ -51,7 +73,11 @@ add_compile_options("-D HostExeSpace=${HOST_EXE_SPACE}") add_compile_options("-D HostMemSpace=${HOST_MEM_SPACE}") if(${BUILD_TESTING} STREQUAL "OFF") - set(Kokkos_ENABLE_TESTS OFF CACHE BOOL "Kokkos tests") + set(Kokkos_ENABLE_TESTS + OFF + CACHE BOOL "Kokkos tests") else() - set(Kokkos_ENABLE_TESTS ON CACHE BOOL "Kokkos tests") + set(Kokkos_ENABLE_TESTS + ON + CACHE BOOL "Kokkos tests") endif() diff --git a/cmake/report.cmake b/cmake/report.cmake index 6733dbcd4..13dde63f7 100644 --- a/cmake/report.cmake +++ b/cmake/report.cmake @@ -18,10 +18,22 @@ function(PadTo Text Padding Target Result) set(${rt} "${rt}") endif() - set(${Result} "${rt}" PARENT_SCOPE) + set(${Result} + "${rt}" + PARENT_SCOPE) endfunction() -function(PrintChoices Label Flag Choices Value Default Color OutputString Multiline Padding) +function( + PrintChoices + Label + Flag + Choices + Value + Default + Color + OutputString + Multiline + Padding) list(LENGTH "${Choices}" nchoices) set(rstring "") set(counter 0) @@ -35,14 +47,14 @@ function(PrintChoices Label Flag Choices Value Default Color OutputString Multil endif() set(rstring_i "${rstring_i}:") - PadTo("${rstring_i}" " " ${Padding} rstring_i) + padto("${rstring_i}" " " ${Padding} rstring_i) else() set(rstring_i "") if(NOT ${counter} EQUAL ${nchoices}) if(${Multiline} EQUAL 1) set(rstring_i "${rstring_i}\n") - PadTo("${rstring_i}" " " ${Padding} rstring_i) + padto("${rstring_i}" " " ${Padding} rstring_i) else() set(rstring_i "${rstring_i}/") endif() @@ -71,13 +83,16 @@ function(PrintChoices Label Flag Choices Value Default Color OutputString Multil set(rstring_i "") endforeach() - set(${OutputString} "${rstring}" PARENT_SCOPE) + set(${OutputString} + "${rstring}" + PARENT_SCOPE) endfunction() set(ON_OFF_VALUES "ON" "OFF") if(${PGEN_FOUND}) - PrintChoices("Problem generator" + printchoices( + "Problem generator" "pgen" "${problem_generators}" ${PGEN} @@ -85,11 +100,11 @@ if(${PGEN_FOUND}) "${Blue}" PGEN_REPORT 1 - 36 - ) + 36) endif() -PrintChoices("Precision" +printchoices( + "Precision" "precision" "${precisions}" ${precision} @@ -97,9 +112,9 @@ PrintChoices("Precision" "${Blue}" PRECISION_REPORT 1 - 36 -) -PrintChoices("Output" + 36) +printchoices( + "Output" "output" "${ON_OFF_VALUES}" ${output} @@ -107,9 +122,9 @@ PrintChoices("Output" "${Green}" OUTPUT_REPORT 0 - 36 -) -PrintChoices("GUI" + 36) +printchoices( + "GUI" "gui" "${ON_OFF_VALUES}" ${gui} @@ -117,9 +132,9 @@ PrintChoices("GUI" "${Green}" GUI_REPORT 0 - 36 -) -PrintChoices("MPI" + 36) +printchoices( + "MPI" "mpi" "${ON_OFF_VALUES}" ${mpi} @@ -127,9 +142,9 @@ PrintChoices("MPI" "${Green}" MPI_REPORT 0 - 42 -) -PrintChoices("Debug mode" + 42) +printchoices( + "Debug mode" "DEBUG" "${ON_OFF_VALUES}" ${DEBUG} @@ -137,10 +152,10 @@ PrintChoices("Debug mode" "${Green}" DEBUG_REPORT 0 - 42 -) + 42) -PrintChoices("CUDA" +printchoices( + "CUDA" "Kokkos_ENABLE_CUDA" "${ON_OFF_VALUES}" ${Kokkos_ENABLE_CUDA} @@ -148,9 +163,9 @@ PrintChoices("CUDA" "${Green}" CUDA_REPORT 0 - 42 -) -PrintChoices("HIP" + 42) +printchoices( + "HIP" "Kokkos_ENABLE_HIP" "${ON_OFF_VALUES}" ${Kokkos_ENABLE_HIP} @@ -158,9 +173,9 @@ PrintChoices("HIP" "${Green}" HIP_REPORT 0 - 42 -) -PrintChoices("OpenMP" + 42) +printchoices( + "OpenMP" "Kokkos_ENABLE_OPENMP" "${ON_OFF_VALUES}" ${Kokkos_ENABLE_OPENMP} @@ -168,10 +183,10 @@ PrintChoices("OpenMP" "${Green}" OPENMP_REPORT 0 - 42 -) + 42) -PrintChoices("C++ compiler" +printchoices( + "C++ compiler" "CMAKE_CXX_COMPILER" "${CMAKE_CXX_COMPILER} v${CMAKE_CXX_COMPILER_VERSION}" "${CMAKE_CXX_COMPILER} v${CMAKE_CXX_COMPILER_VERSION}" @@ -179,10 +194,10 @@ PrintChoices("C++ compiler" "${ColorReset}" CXX_COMPILER_REPORT 0 - 42 -) + 42) -PrintChoices("C compiler" +printchoices( + "C compiler" "CMAKE_C_COMPILER" "${CMAKE_C_COMPILER} v${CMAKE_C_COMPILER_VERSION}" "${CMAKE_C_COMPILER} v${CMAKE_C_COMPILER_VERSION}" @@ -190,21 +205,24 @@ PrintChoices("C compiler" "${ColorReset}" C_COMPILER_REPORT 0 - 42 -) + 42) get_cmake_property(_variableNames VARIABLES) -foreach (_variableName ${_variableNames}) - string(REGEX MATCH "Kokkos_ARCH_*" _isMatched ${_variableName}) - if(_isMatched) - get_property(isSet CACHE ${_variableName} PROPERTY VALUE) - if(isSet STREQUAL "ON") - string(REGEX REPLACE "Kokkos_ARCH_" "" ARCH ${_variableName}) - break() - endif() +foreach(_variableName ${_variableNames}) + string(REGEX MATCH "Kokkos_ARCH_*" _isMatched ${_variableName}) + if(_isMatched) + get_property( + isSet + CACHE ${_variableName} + PROPERTY VALUE) + if(isSet STREQUAL "ON") + string(REGEX REPLACE "Kokkos_ARCH_" "" ARCH ${_variableName}) + break() endif() + endif() endforeach() -PrintChoices("Architecture" +printchoices( + "Architecture" "Kokkos_ARCH_*" "${ARCH}" "${ARCH}" @@ -212,8 +230,7 @@ PrintChoices("Architecture" "${ColorReset}" ARCH_REPORT 0 - 42 -) + 42) if(${Kokkos_ENABLE_CUDA}) if("${CMAKE_CUDA_COMPILER}" STREQUAL "") @@ -225,11 +242,15 @@ if(${Kokkos_ENABLE_CUDA}) string(STRIP ${CUDACOMP} CUDACOMP) message(STATUS "CUDA compiler: ${CUDACOMP}") - execute_process(COMMAND bash -c "${CUDACOMP} --version | grep release | sed -e 's/.*release //' -e 's/,.*//'" + execute_process( + COMMAND + bash -c + "${CUDACOMP} --version | grep release | sed -e 's/.*release //' -e 's/,.*//'" OUTPUT_VARIABLE CUDACOMP_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) - PrintChoices("CUDA compiler" + printchoices( + "CUDA compiler" "CMAKE_CUDA_COMPILER" "${CUDACOMP}" "${CUDACOMP}" @@ -237,28 +258,37 @@ if(${Kokkos_ENABLE_CUDA}) "${ColorReset}" CUDA_COMPILER_REPORT 0 - 42 - ) + 42) endif() -if (${Kokkos_ENABLE_HIP}) - execute_process(COMMAND bash -c "hipcc --version | grep HIP | cut -d ':' -f 2 | tr -d ' '" +if(${Kokkos_ENABLE_HIP}) + execute_process( + COMMAND bash -c "hipcc --version | grep HIP | cut -d ':' -f 2 | tr -d ' '" OUTPUT_VARIABLE ROCM_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) endif() set(DOT_SYMBOL "${ColorReset}.") -set(DOTTED_LINE_SYMBOL "${ColorReset}. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . ") +set(DOTTED_LINE_SYMBOL + "${ColorReset}. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " +) -set(DASHED_LINE_SYMBOL "${ColorReset}....................................................................... ") +set(DASHED_LINE_SYMBOL + "${ColorReset}....................................................................... " +) if(NOT ${PROJECT_VERSION_TWEAK} EQUAL 0) - set(VERSION_SYMBOL "v${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}-rc${PROJECT_VERSION_TWEAK}") + set(VERSION_SYMBOL + "v${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}-rc${PROJECT_VERSION_TWEAK}" + ) else() - set(VERSION_SYMBOL "v${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH} ") + set(VERSION_SYMBOL + "v${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH} " + ) endif() -message("${Blue} __ __ +message( + "${Blue} __ __ /\\ \\__ __/\\ \\__ __ ___\\ \\ _\\/\\_\\ \\ _\\ __ __ / __ \\ / __ \\ \\ \\/\\/\\ \\ \\ \\/ /\\ \\/\\ \\ @@ -299,7 +329,7 @@ message(" ${DEBUG_REPORT}") message("${DASHED_LINE_SYMBOL}\nDependencies") -if (NOT "${CUDACOMP_VERSION}" STREQUAL "") +if(NOT "${CUDACOMP_VERSION}" STREQUAL "") message(" - CUDA:\tv${CUDACOMP_VERSION}") elseif(NOT "${ROCM_VERSION}" STREQUAL "") message(" - ROCm:\tv${ROCM_VERSION}") @@ -312,7 +342,8 @@ if(${HDF5_FOUND}) message(" - HDF5:\tv${HDF5_VERSION}") endif() -message("${DASHED_LINE_SYMBOL} +message( + "${DASHED_LINE_SYMBOL} Notes ${Dim}: Set flags with `cmake ... -D ${Magenta}${ColorReset}${Dim}=`, the ${Underline}default${ColorReset}${Dim} value : will be used unless the variable is explicitly set.${ColorReset} diff --git a/cmake/styling.cmake b/cmake/styling.cmake index fb9cfcc87..70c448fff 100644 --- a/cmake/styling.cmake +++ b/cmake/styling.cmake @@ -23,20 +23,17 @@ if(NOT WIN32) set(StrikeEnd "${Esc}[0m") endif() -# message("This is normal") -# message("${Red}This is Red${ColorReset}") -# message("${Green}This is Green${ColorReset}") -# message("${Yellow}This is Yellow${ColorReset}") -# message("${Blue}This is Blue${ColorReset}") -# message("${Magenta}This is Magenta${ColorReset}") -# message("${Cyan}This is Cyan${ColorReset}") -# message("${White}This is White${ColorReset}") -# message("${BoldRed}This is BoldRed${ColorReset}") -# message("${BoldGreen}This is BoldGreen${ColorReset}") -# message("${BoldYellow}This is BoldYellow${ColorReset}") -# message("${BoldBlue}This is BoldBlue${ColorReset}") +# message("This is normal") message("${Red}This is Red${ColorReset}") +# message("${Green}This is Green${ColorReset}") message("${Yellow}This is +# Yellow${ColorReset}") message("${Blue}This is Blue${ColorReset}") +# message("${Magenta}This is Magenta${ColorReset}") message("${Cyan}This is +# Cyan${ColorReset}") message("${White}This is White${ColorReset}") +# message("${BoldRed}This is BoldRed${ColorReset}") message("${BoldGreen}This is +# BoldGreen${ColorReset}") message("${BoldYellow}This is +# BoldYellow${ColorReset}") message("${BoldBlue}This is BoldBlue${ColorReset}") # message("${BoldMagenta}This is BoldMagenta${ColorReset}") -# message("${BoldCyan}This is BoldCyan${ColorReset}") -# message("${BoldWhite}This is BoldWhite\n\n${ColorReset}") +# message("${BoldCyan}This is BoldCyan${ColorReset}") message("${BoldWhite}This +# is BoldWhite\n\n${ColorReset}") + +# message() -# message() \ No newline at end of file diff --git a/cmake/tests.cmake b/cmake/tests.cmake index 643ac3d29..7820a5192 100644 --- a/cmake/tests.cmake +++ b/cmake/tests.cmake @@ -8,27 +8,36 @@ add_subdirectory(${SRC_DIR}/metrics ${CMAKE_CURRENT_BINARY_DIR}/metrics) add_subdirectory(${SRC_DIR}/kernels ${CMAKE_CURRENT_BINARY_DIR}/kernels) add_subdirectory(${SRC_DIR}/archetypes ${CMAKE_CURRENT_BINARY_DIR}/archetypes) add_subdirectory(${SRC_DIR}/framework ${CMAKE_CURRENT_BINARY_DIR}/framework) -if (${output}) +if(${output}) add_subdirectory(${SRC_DIR}/output ${CMAKE_CURRENT_BINARY_DIR}/output) add_subdirectory(${SRC_DIR}/checkpoint ${CMAKE_CURRENT_BINARY_DIR}/checkpoint) endif() -if (${mpi}) +if(${mpi}) # tests with mpi - if (${output}) - add_subdirectory(${SRC_DIR}/output/tests ${CMAKE_CURRENT_BINARY_DIR}/output/tests) - add_subdirectory(${SRC_DIR}/checkpoint/tests ${CMAKE_CURRENT_BINARY_DIR}/checkpoint/tests) - add_subdirectory(${SRC_DIR}/framework/tests ${CMAKE_CURRENT_BINARY_DIR}/framework/tests) + if(${output}) + add_subdirectory(${SRC_DIR}/output/tests + ${CMAKE_CURRENT_BINARY_DIR}/output/tests) + add_subdirectory(${SRC_DIR}/checkpoint/tests + ${CMAKE_CURRENT_BINARY_DIR}/checkpoint/tests) + add_subdirectory(${SRC_DIR}/framework/tests + ${CMAKE_CURRENT_BINARY_DIR}/framework/tests) endif() else() # tests without mpi - add_subdirectory(${SRC_DIR}/global/tests ${CMAKE_CURRENT_BINARY_DIR}/global/tests) - add_subdirectory(${SRC_DIR}/metrics/tests ${CMAKE_CURRENT_BINARY_DIR}/metrics/tests) - add_subdirectory(${SRC_DIR}/kernels/tests ${CMAKE_CURRENT_BINARY_DIR}/kernels/tests) - add_subdirectory(${SRC_DIR}/archetypes/tests ${CMAKE_CURRENT_BINARY_DIR}/archetypes/tests) - add_subdirectory(${SRC_DIR}/framework/tests ${CMAKE_CURRENT_BINARY_DIR}/framework/tests) - if (${output}) - add_subdirectory(${SRC_DIR}/output/tests ${CMAKE_CURRENT_BINARY_DIR}/output/tests) - add_subdirectory(${SRC_DIR}/checkpoint/tests ${CMAKE_CURRENT_BINARY_DIR}/checkpoint/tests) + add_subdirectory(${SRC_DIR}/global/ ${CMAKE_CURRENT_BINARY_DIR}/global/tests) + add_subdirectory(${SRC_DIR}/metrics/tests + ${CMAKE_CURRENT_BINARY_DIR}/metrics/tests) + add_subdirectory(${SRC_DIR}/kernels/tests + ${CMAKE_CURRENT_BINARY_DIR}/kernels/tests) + add_subdirectory(${SRC_DIR}/archetypes/tests + ${CMAKE_CURRENT_BINARY_DIR}/archetypes/tests) + add_subdirectory(${SRC_DIR}/framework/tests + ${CMAKE_CURRENT_BINARY_DIR}/framework/tests) + if(${output}) + add_subdirectory(${SRC_DIR}/output/tests + ${CMAKE_CURRENT_BINARY_DIR}/output/tests) + add_subdirectory(${SRC_DIR}/checkpoint/tests + ${CMAKE_CURRENT_BINARY_DIR}/checkpoint/tests) endif() endif() diff --git a/setups/CMakeLists.txt b/setups/CMakeLists.txt index b1753d7b8..c92c1d345 100644 --- a/setups/CMakeLists.txt +++ b/setups/CMakeLists.txt @@ -1,23 +1,25 @@ # ------------------------------ # @defines: ntt_pgen [INTERFACE] +# # @includes: -# - ../ +# +# * ../ +# # @depends: -# - ntt_pgen [required] +# +# * ntt_pgen [required] +# # @uses: -# - kokkos [required] -# - plog [required] -# - mpi [optional] +# +# * kokkos [required] +# * plog [required] +# * mpi [optional] # ------------------------------ add_library(ntt_pgen INTERFACE) -target_link_libraries(ntt_pgen INTERFACE - ntt_global - ntt_framework - ntt_archetypes - ntt_kernels -) +target_link_libraries(ntt_pgen INTERFACE ntt_global ntt_framework + ntt_archetypes ntt_kernels) target_include_directories(ntt_pgen - INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/${PGEN} -) \ No newline at end of file + INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/${PGEN}) + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d75094c2b..a41b84900 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,27 +1,30 @@ # ------------------------------ # @defines: entity [STATIC/SHARED] +# # @sources: -# - entity.cpp +# +# * entity.cpp +# # @depends: -# - ntt_global [required] -# - ntt_framework [required] -# - ntt_metrics [required] -# - ntt_engine [required] -# - ntt_pgen [required] +# +# * ntt_global [required] +# * ntt_framework [required] +# * ntt_metrics [required] +# * ntt_engine [required] +# * ntt_pgen [required] +# # @uses: -# - kokkos [required] -# - plog [required] -# - toml11 [required] -# - ADIOS2 [optional] -# - mpi [optional] +# +# * kokkos [required] +# * plog [required] +# * toml11 [required] +# * ADIOS2 [optional] +# * mpi [optional] # ------------------------------ - set(ENTITY ${PROJECT_NAME}.xc) set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) -set(SOURCES - ${SRC_DIR}/entity.cpp -) +set(SOURCES ${SRC_DIR}/entity.cpp) add_executable(${ENTITY} entity.cpp) # dependencies @@ -32,7 +35,7 @@ add_subdirectory(${SRC_DIR}/archetypes ${CMAKE_CURRENT_BINARY_DIR}/archetypes) add_subdirectory(${SRC_DIR}/framework ${CMAKE_CURRENT_BINARY_DIR}/framework) add_subdirectory(${SRC_DIR}/engines ${CMAKE_CURRENT_BINARY_DIR}/engines) -if (${output}) +if(${output}) add_subdirectory(${SRC_DIR}/output ${CMAKE_CURRENT_BINARY_DIR}/output) add_subdirectory(${SRC_DIR}/checkpoint ${CMAKE_CURRENT_BINARY_DIR}/checkpoint) endif() diff --git a/src/archetypes/CMakeLists.txt b/src/archetypes/CMakeLists.txt index 7883ba6a5..8e2f325af 100644 --- a/src/archetypes/CMakeLists.txt +++ b/src/archetypes/CMakeLists.txt @@ -1,13 +1,19 @@ # ------------------------------ # @defines: ntt_archetypes [INTERFACE] +# # @includes: -# - ../ +# +# * ../ +# # @depends: -# - ntt_global [required] -# - ntt_kernels [required] +# +# * ntt_global [required] +# * ntt_kernels [required] +# # @uses: -# - kokkos [required] -# - mpi [optional] +# +# * kokkos [required] +# * mpi [optional] # ------------------------------ add_library(ntt_archetypes INTERFACE) @@ -17,5 +23,5 @@ add_dependencies(ntt_archetypes ${libs}) target_link_libraries(ntt_archetypes INTERFACE ${libs}) target_include_directories(ntt_archetypes - INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/../ -) \ No newline at end of file + INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/../) + diff --git a/src/archetypes/tests/CMakeLists.txt b/src/archetypes/tests/CMakeLists.txt index 4ffc35322..694a6b4f9 100644 --- a/src/archetypes/tests/CMakeLists.txt +++ b/src/archetypes/tests/CMakeLists.txt @@ -1,9 +1,11 @@ # ------------------------------ # @brief: Generates tests for the `ntt_archetypes` module +# # @uses: -# - kokkos [required] -# - plog [required] -# - mpi [optional] +# +# * kokkos [required] +# * plog [required] +# * mpi [optional] # ------------------------------ set(SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../) diff --git a/src/checkpoint/CMakeLists.txt b/src/checkpoint/CMakeLists.txt index d97bd4a34..fa641bfb5 100644 --- a/src/checkpoint/CMakeLists.txt +++ b/src/checkpoint/CMakeLists.txt @@ -1,23 +1,28 @@ # ------------------------------ # @defines: ntt_checkpoint [STATIC/SHARED] +# # @sources: -# - writer.cpp -# - reader.cpp +# +# * writer.cpp +# * reader.cpp +# # @includes: -# - ../ +# +# * ../ +# # @depends: -# - ntt_global [required] +# +# * ntt_global [required] +# # @uses: -# - kokkos [required] -# - ADIOS2 [required] -# - mpi [optional] +# +# * kokkos [required] +# * ADIOS2 [required] +# * mpi [optional] # ------------------------------ set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) -set(SOURCES - ${SRC_DIR}/writer.cpp - ${SRC_DIR}/reader.cpp -) +set(SOURCES ${SRC_DIR}/writer.cpp ${SRC_DIR}/reader.cpp) add_library(ntt_checkpoint ${SOURCES}) set(libs ntt_global) @@ -25,7 +30,7 @@ add_dependencies(ntt_checkpoint ${libs}) target_link_libraries(ntt_checkpoint PUBLIC ${libs}) target_link_libraries(ntt_checkpoint PRIVATE stdc++fs) -target_include_directories(ntt_checkpoint +target_include_directories( + ntt_checkpoint PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../ - INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/../ -) + INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/../) diff --git a/src/checkpoint/tests/CMakeLists.txt b/src/checkpoint/tests/CMakeLists.txt index 3d7475a52..10836554b 100644 --- a/src/checkpoint/tests/CMakeLists.txt +++ b/src/checkpoint/tests/CMakeLists.txt @@ -1,9 +1,11 @@ # ------------------------------ # @brief: Generates tests for the `ntt_checkpoint` module +# # @uses: -# - kokkos [required] -# - adios2 [required] -# - mpi [optional] +# +# * kokkos [required] +# * adios2 [required] +# * mpi [optional] # ------------------------------ set(SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../) @@ -13,14 +15,14 @@ function(gen_test title) set(src ${title}.cpp) add_executable(${exec} ${src}) - set (libs ntt_checkpoint ntt_global) + set(libs ntt_checkpoint ntt_global) add_dependencies(${exec} ${libs}) target_link_libraries(${exec} PRIVATE ${libs} stdc++fs) add_test(NAME "CHECKPOINT::${title}" COMMAND "${exec}") endfunction() -if (NOT ${mpi}) +if(NOT ${mpi}) gen_test(checkpoint-nompi) else() # gen_test(checkpoint-mpi) diff --git a/src/engines/CMakeLists.txt b/src/engines/CMakeLists.txt index 2ab7289b2..6da2f4efd 100644 --- a/src/engines/CMakeLists.txt +++ b/src/engines/CMakeLists.txt @@ -1,37 +1,43 @@ # ------------------------------ # @defines: ntt_engines [STATIC/SHARED] +# # @sources: -# - engine_printer.cpp -# - engine_init.cpp -# - engine_run.cpp +# +# * engine_printer.cpp +# * engine_init.cpp +# * engine_run.cpp +# # @includes: -# - ../ +# +# * ../ +# # @depends: -# - ntt_global [required] -# - ntt_framework [required] -# - ntt_metrics [required] -# - ntt_kernels [required] -# - ntt_archetypes [required] -# - ntt_pgen [required] -# - ntt_output [optional] +# +# * ntt_global [required] +# * ntt_framework [required] +# * ntt_metrics [required] +# * ntt_kernels [required] +# * ntt_archetypes [required] +# * ntt_pgen [required] +# * ntt_output [optional] +# # @uses: -# - kokkos [required] -# - plog [required] -# - toml11 [required] -# - adios2 [optional] -# - hdf5 [optional] -# - mpi [optional] +# +# * kokkos [required] +# * plog [required] +# * toml11 [required] +# * adios2 [optional] +# * hdf5 [optional] +# * mpi [optional] # ------------------------------ set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) -set(SOURCES - ${SRC_DIR}/engine_printer.cpp - ${SRC_DIR}/engine_init.cpp - ${SRC_DIR}/engine_run.cpp -) +set(SOURCES ${SRC_DIR}/engine_printer.cpp ${SRC_DIR}/engine_init.cpp + ${SRC_DIR}/engine_run.cpp) add_library(ntt_engines ${SOURCES}) -set(libs ntt_global ntt_framework ntt_metrics ntt_archetypes ntt_kernels ntt_pgen) +set(libs ntt_global ntt_framework ntt_metrics ntt_archetypes ntt_kernels + ntt_pgen) if(${output}) list(APPEND libs ntt_output hdf5::hdf5) endif() @@ -39,7 +45,7 @@ add_dependencies(ntt_engines ${libs}) target_link_libraries(ntt_engines PUBLIC ${libs}) target_compile_definitions(ntt_engines PRIVATE PGEN=\"${PGEN}\") -target_include_directories(ntt_engines +target_include_directories( + ntt_engines PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../ - INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/../ -) + INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/../) diff --git a/src/framework/CMakeLists.txt b/src/framework/CMakeLists.txt index e01759d14..8802f696b 100644 --- a/src/framework/CMakeLists.txt +++ b/src/framework/CMakeLists.txt @@ -1,41 +1,48 @@ # ------------------------------ # @defines: ntt_framework [STATIC/SHARED] +# # @sources: -# - parameters.cpp -# - simulation.cpp -# - domain/grid.cpp -# - domain/metadomain.cpp -# - domain/communications.cpp -# - domain/checkpoint.cpp -# - containers/particles.cpp -# - containers/fields.cpp -# - domain/output.cpp +# +# * parameters.cpp +# * simulation.cpp +# * domain/grid.cpp +# * domain/metadomain.cpp +# * domain/communications.cpp +# * domain/checkpoint.cpp +# * containers/particles.cpp +# * containers/fields.cpp +# * domain/output.cpp +# # @includes: -# - ../ +# +# * ../ +# # @depends: -# - ntt_global [required] -# - ntt_metrics [required] -# - ntt_kernels [required] -# - ntt_output [optional] +# +# * ntt_global [required] +# * ntt_metrics [required] +# * ntt_kernels [required] +# * ntt_output [optional] +# # @uses: -# - kokkos [required] -# - plog [required] -# - toml11 [required] -# - ADIOS2 [optional] -# - mpi [optional] +# +# * kokkos [required] +# * plog [required] +# * toml11 [required] +# * ADIOS2 [optional] +# * mpi [optional] # ------------------------------ set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) -set(SOURCES - ${SRC_DIR}/parameters.cpp - ${SRC_DIR}/simulation.cpp - ${SRC_DIR}/domain/grid.cpp - ${SRC_DIR}/domain/metadomain.cpp - ${SRC_DIR}/domain/communications.cpp - ${SRC_DIR}/containers/particles.cpp - ${SRC_DIR}/containers/fields.cpp -) -if (${output}) +set(SOURCES + ${SRC_DIR}/parameters.cpp + ${SRC_DIR}/simulation.cpp + ${SRC_DIR}/domain/grid.cpp + ${SRC_DIR}/domain/metadomain.cpp + ${SRC_DIR}/domain/communications.cpp + ${SRC_DIR}/containers/particles.cpp + ${SRC_DIR}/containers/fields.cpp) +if(${output}) list(APPEND SOURCES ${SRC_DIR}/domain/output.cpp) list(APPEND SOURCES ${SRC_DIR}/domain/checkpoint.cpp) endif() @@ -50,7 +57,7 @@ add_dependencies(ntt_framework ${libs}) target_link_libraries(ntt_framework PUBLIC ${libs}) target_link_libraries(ntt_framework PRIVATE stdc++fs) -target_include_directories(ntt_framework +target_include_directories( + ntt_framework PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../ - INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/../ -) + INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/../) diff --git a/src/framework/tests/CMakeLists.txt b/src/framework/tests/CMakeLists.txt index 56ad0783b..ce188e9f1 100644 --- a/src/framework/tests/CMakeLists.txt +++ b/src/framework/tests/CMakeLists.txt @@ -1,19 +1,23 @@ # ------------------------------ # @brief: Generates tests for the `ntt_framework` module +# # @uses: -# - kokkos [required] -# - plog [required] -# - toml11 [required] -# - mpi [optional] -# - adios2 [optional] +# +# * kokkos [required] +# * plog [required] +# * toml11 [required] +# * mpi [optional] +# * adios2 [optional] +# # !TODO: -# - add tests for mesh separately -# - add test for 3D metadomain +# +# * add tests for mesh separately +# * add test for 3D metadomain # ------------------------------ set(SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../) -function(gen_test title) +function(gen_test title is_parallel) set(exec test-framework-${title}.xc) set(src ${title}.cpp) add_executable(${exec} ${src}) @@ -22,24 +26,30 @@ function(gen_test title) add_dependencies(${exec} ${libs}) target_link_libraries(${exec} PRIVATE ${libs}) - add_test(NAME "FRAMEWORK::${title}" COMMAND "${exec}") + if(${is_parallel}) + add_test(NAME "FRAMEWORK::${title}" + COMMAND "${MPIEXEC_EXECUTABLE}" "${MPIEXEC_NUMPROC_FLAG}" "4" + "${exec}") + else() + add_test(NAME "FRAMEWORK::${title}" COMMAND "${exec}") + endif() endfunction() -if (${mpi}) - gen_test(comm_mpi) +if(${mpi}) + gen_test(comm_mpi true) else() - gen_test(parameters) - gen_test(particles) - gen_test(fields) - gen_test(grid_mesh) - if (${DEBUG}) - gen_test(metadomain) + gen_test(parameters false) + gen_test(particles false) + gen_test(fields false) + gen_test(grid_mesh false) + if(${DEBUG}) + gen_test(metadomain false) endif() - gen_test(comm_nompi) + gen_test(comm_nompi false) endif() - # this test is only run manually to ensure ... # ... command line args are working properly ... # ... and that the logging is done correctly -# gen_test(simulation) +# +# gen_test(simulation) diff --git a/src/global/CMakeLists.txt b/src/global/CMakeLists.txt index 334ce078d..97946f059 100644 --- a/src/global/CMakeLists.txt +++ b/src/global/CMakeLists.txt @@ -1,36 +1,38 @@ # ------------------------------ # @defines: ntt_global [STATIC/SHARED] +# # @sources: -# - global.cpp -# - arch/kokkos_aliases.cpp -# - utils/cargs.cpp -# - utils/param_container.cpp -# - utils/timer.cpp -# - utils/diag.cpp -# - utils/progressbar.cpp +# +# * global.cpp +# * arch/kokkos_aliases.cpp +# * utils/cargs.cpp +# * utils/param_container.cpp +# * utils/timer.cpp +# * utils/diag.cpp +# * utils/progressbar.cpp +# # @includes: -# - ./ +# +# * ./ +# # @uses: -# - kokkos [required] -# - plog [required] -# - mpi [optional] +# +# * kokkos [required] +# * plog [required] +# * mpi [optional] # ------------------------------ set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) -set(SOURCES - ${SRC_DIR}/global.cpp - ${SRC_DIR}/arch/kokkos_aliases.cpp - ${SRC_DIR}/utils/cargs.cpp - ${SRC_DIR}/utils/timer.cpp - ${SRC_DIR}/utils/diag.cpp - ${SRC_DIR}/utils/progressbar.cpp -) -if (${output}) +set(SOURCES + ${SRC_DIR}/global.cpp ${SRC_DIR}/arch/kokkos_aliases.cpp + ${SRC_DIR}/utils/cargs.cpp ${SRC_DIR}/utils/timer.cpp + ${SRC_DIR}/utils/diag.cpp ${SRC_DIR}/utils/progressbar.cpp) +if(${output}) list(APPEND SOURCES ${SRC_DIR}/utils/param_container.cpp) endif() add_library(ntt_global ${SOURCES}) -target_include_directories(ntt_global +target_include_directories( + ntt_global PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} - INTERFACE ${CMAKE_CURRENT_SOURCE_DIR} -) + INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(ntt_global PRIVATE stdc++fs) diff --git a/src/global/tests/CMakeLists.txt b/src/global/tests/CMakeLists.txt index e9e5de687..e30da20a0 100644 --- a/src/global/tests/CMakeLists.txt +++ b/src/global/tests/CMakeLists.txt @@ -1,11 +1,15 @@ # ------------------------------ # @brief: Generates tests for the `ntt_global` module +# # @uses: -# - kokkos [required] -# - plog [required] -# - mpi [optional] +# +# * kokkos [required] +# * plog [required] +# * mpi [optional] +# # !TODO: -# - add optional tests for the `mpi_aliases.h` +# +# * add optional tests for the `mpi_aliases.h` # ------------------------------ set(SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../) diff --git a/src/kernels/CMakeLists.txt b/src/kernels/CMakeLists.txt index d24dff0a4..c8a1f409f 100644 --- a/src/kernels/CMakeLists.txt +++ b/src/kernels/CMakeLists.txt @@ -1,13 +1,19 @@ # ------------------------------ # @defines: ntt_kernels [INTERFACE] +# # @includes: -# - ../ +# +# * ../ +# # @depends: -# - ntt_global [required] +# +# * ntt_global [required] +# # @uses: -# - kokkos [required] -# - plog [required] -# - mpi [optional] +# +# * kokkos [required] +# * plog [required] +# * mpi [optional] # ------------------------------ add_library(ntt_kernels INTERFACE) @@ -17,5 +23,5 @@ add_dependencies(ntt_kernels ${libs}) target_link_libraries(ntt_kernels INTERFACE ${libs}) target_include_directories(ntt_kernels - INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/../ -) \ No newline at end of file + INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/../) + diff --git a/src/kernels/tests/CMakeLists.txt b/src/kernels/tests/CMakeLists.txt index e55dbc111..10e8bb944 100644 --- a/src/kernels/tests/CMakeLists.txt +++ b/src/kernels/tests/CMakeLists.txt @@ -1,9 +1,11 @@ # ------------------------------ # @brief: Generates tests for the `ntt_kernels` module +# # @uses: -# - kokkos [required] -# - plog [required] -# - mpi [optional] +# +# * kokkos [required] +# * plog [required] +# * mpi [optional] # ------------------------------ set(SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../) diff --git a/src/metrics/CMakeLists.txt b/src/metrics/CMakeLists.txt index 0f303fcfc..e053bb61c 100644 --- a/src/metrics/CMakeLists.txt +++ b/src/metrics/CMakeLists.txt @@ -1,11 +1,17 @@ # ------------------------------ # @defines: ntt_metrics [INTERFACE] +# # @includes: -# - ../ +# +# * ../ +# # @depends: -# - ntt_global [required] +# +# * ntt_global [required] +# # @uses: -# - kokkos [required] +# +# * kokkos [required] # ------------------------------ add_library(ntt_metrics INTERFACE) @@ -15,5 +21,5 @@ add_dependencies(ntt_metrics ${libs}) target_link_libraries(ntt_metrics INTERFACE ${libs}) target_include_directories(ntt_metrics - INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/../ -) \ No newline at end of file + INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/../) + diff --git a/src/metrics/tests/CMakeLists.txt b/src/metrics/tests/CMakeLists.txt index 117cb3295..c997ab079 100644 --- a/src/metrics/tests/CMakeLists.txt +++ b/src/metrics/tests/CMakeLists.txt @@ -1,9 +1,11 @@ # ------------------------------ # @brief: Generates tests for the `ntt_metrics` module +# # @uses: -# - kokkos [required] -# - plog [required] -# - mpi [optional] +# +# * kokkos [required] +# * plog [required] +# * mpi [optional] # ------------------------------ set(SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../) @@ -13,7 +15,7 @@ function(gen_test title) set(src ${title}.cpp) add_executable(${exec} ${src}) - set (libs ntt_metrics) + set(libs ntt_metrics) add_dependencies(${exec} ${libs}) target_link_libraries(${exec} PRIVATE ${libs}) @@ -25,4 +27,5 @@ gen_test(vec_trans) gen_test(coord_trans) gen_test(sph-qsph) gen_test(ks-qks) -gen_test(sr-cart-sph) \ No newline at end of file +gen_test(sr-cart-sph) + diff --git a/src/output/CMakeLists.txt b/src/output/CMakeLists.txt index 2c25631ec..e6dbcc03a 100644 --- a/src/output/CMakeLists.txt +++ b/src/output/CMakeLists.txt @@ -1,32 +1,37 @@ # ------------------------------ # @defines: ntt_output [STATIC/SHARED] +# # @sources: -# - writer.cpp -# - fields.cpp -# - utils/interpret_prompt.cpp +# +# * writer.cpp +# * fields.cpp +# * utils/interpret_prompt.cpp +# # @includes: -# - ../ +# +# * ../ +# # @depends: -# - ntt_global [required] +# +# * ntt_global [required] +# # @uses: -# - kokkos [required] -# - ADIOS2 [required] -# - mpi [optional] +# +# * kokkos [required] +# * ADIOS2 [required] +# * mpi [optional] # ------------------------------ set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) -set(SOURCES - ${SRC_DIR}/writer.cpp - ${SRC_DIR}/fields.cpp - ${SRC_DIR}/utils/interpret_prompt.cpp -) +set(SOURCES ${SRC_DIR}/writer.cpp ${SRC_DIR}/fields.cpp + ${SRC_DIR}/utils/interpret_prompt.cpp) add_library(ntt_output ${SOURCES}) set(libs ntt_global) add_dependencies(ntt_output ${libs}) target_link_libraries(ntt_output PUBLIC ${libs}) -target_include_directories(ntt_output +target_include_directories( + ntt_output PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../ - INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/../ -) + INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/../) diff --git a/src/output/tests/CMakeLists.txt b/src/output/tests/CMakeLists.txt index 37af95fac..afc7950c4 100644 --- a/src/output/tests/CMakeLists.txt +++ b/src/output/tests/CMakeLists.txt @@ -1,28 +1,36 @@ # ------------------------------ # @brief: Generates tests for the `ntt_output` module +# # @uses: -# - kokkos [required] -# - mpi [optional] -# - adios2 [optional] +# +# * kokkos [required] +# * mpi [optional] +# * adios2 [optional] # ------------------------------ set(SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../) -function(gen_test title) +function(gen_test title is_parallel) set(exec test-output-${title}.xc) set(src ${title}.cpp) add_executable(${exec} ${src}) - set (libs ntt_output ntt_global ntt_metrics ntt_framework) + set(libs ntt_output ntt_global ntt_metrics ntt_framework) add_dependencies(${exec} ${libs}) target_link_libraries(${exec} PRIVATE ${libs} stdc++fs) - add_test(NAME "OUTPUT::${title}" COMMAND "${exec}") + if(${is_parallel}) + add_test(NAME "OUTPUT::${title}" + COMMAND "${MPIEXEC_EXECUTABLE}" "${MPIEXEC_NUMPROC_FLAG}" "4" + "${exec}") + else() + add_test(NAME "OUTPUT::${title}" COMMAND "${exec}") + endif() endfunction() -if (NOT ${mpi}) - gen_test(fields) - gen_test(writer-nompi) +if(NOT ${mpi}) + gen_test(fields false) + gen_test(writer-nompi false) else() - gen_test(writer-mpi) + gen_test(writer-mpi true) endif() From 8a6321b5309cce50bbe9b56c83736645fd24807c Mon Sep 17 00:00:00 2001 From: hayk Date: Mon, 4 Nov 2024 14:30:45 -0500 Subject: [PATCH 15/22] proper gh action (hopefully) --- .github/workflows/actions.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 22f55d8be..3e0de8808 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -15,10 +15,9 @@ jobs: id: check_message run: | if git log -1 --pretty=%B | grep -q "RUNTEST"; then - echo "::set-output name=run_tests::true" + echo "run_tests=true" >> "$GITHUB_OUTPUT" else - echo "::set-output name=run_tests::false" - exit 1 + echo "run_tests=false" >> "$GITHUB_OUTPUT" fi tests: needs: check-commit From 50c7b9be966dd83183038fd14840fee2246f289f Mon Sep 17 00:00:00 2001 From: hayk Date: Mon, 4 Nov 2024 14:36:10 -0500 Subject: [PATCH 16/22] readme (RUNTEST) --- README.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index d6f4597f5..26ca92072 100644 --- a/README.md +++ b/README.md @@ -10,21 +10,23 @@ Our [detailed documentation](https://entity-toolkit.github.io/) includes everyth [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) -## Core developers (alphabetical) +## Lead developers -👀 __Yangyang Cai__ {[@StaticObserver](https://github.com/StaticObserver): GRPIC} +☕ __Hayk Hakobyan__ {[@haykh](https://github.com/haykh)} -💁‍♂️ __Alexander Chernoglazov__ {[@SChernoglazov](https://github.com/SChernoglazov): PIC} +🥔 __Jens Mahlmann__ {[@jmahlmann](https://github.com/jmahlmann)} -🍵 __Benjamin Crinquand__ {[@bcrinquand](https://github.com/bcrinquand): GRPIC, cubed-sphere} +💁‍♂️ __Alexander Chernoglazov__ {[@SChernoglazov](https://github.com/SChernoglazov)} -🧋 __Alisa Galishnikova__ {[@alisagk](https://github.com/alisagk): GRPIC} +🧋 __Alisa Galishnikova__ {[@alisagk](https://github.com/alisagk)} -☕ __Hayk Hakobyan__ {[@haykh](https://github.com/haykh): framework, PIC, GRPIC, cubed-sphere} +🐬 __Sasha Philippov__ {[@sashaph](https://github.com/sashaph)} -🥔 __Jens Mahlmann__ {[@jmahlmann](https://github.com/jmahlmann): framework, MPI, cubed-sphere} +## Contributors (alphabetical) -🐬 __Sasha Philippov__ {[@sashaph](https://github.com/sashaph): all-around} +👀 __Yangyang Cai__ {[@StaticObserver](https://github.com/StaticObserver): GRPIC} + +🍵 __Benjamin Crinquand__ {[@bcrinquand](https://github.com/bcrinquand): GRPIC, cubed-sphere} 🤷 __Arno Vanthieghem__ {[@vanthieg](https://github.com/vanthieg): framework, PIC} From 6cd75c105b19f64ee317daab60125ff4ea44ce30 Mon Sep 17 00:00:00 2001 From: hayk Date: Mon, 4 Nov 2024 14:43:30 -0500 Subject: [PATCH 17/22] minor issue in cmake (RUNTEST) --- cmake/tests.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/tests.cmake b/cmake/tests.cmake index 7820a5192..ca8ee69c4 100644 --- a/cmake/tests.cmake +++ b/cmake/tests.cmake @@ -25,7 +25,8 @@ if(${mpi}) endif() else() # tests without mpi - add_subdirectory(${SRC_DIR}/global/ ${CMAKE_CURRENT_BINARY_DIR}/global/tests) + add_subdirectory(${SRC_DIR}/global/tests + ${CMAKE_CURRENT_BINARY_DIR}/global/tests) add_subdirectory(${SRC_DIR}/metrics/tests ${CMAKE_CURRENT_BINARY_DIR}/metrics/tests) add_subdirectory(${SRC_DIR}/kernels/tests From 727eb1eff09315129acb0e72e81a2d1969554faa Mon Sep 17 00:00:00 2001 From: hayk Date: Mon, 4 Nov 2024 14:54:43 -0500 Subject: [PATCH 18/22] test fixed (RUNTEST) --- src/metrics/tests/sr-cart-sph.cpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/metrics/tests/sr-cart-sph.cpp b/src/metrics/tests/sr-cart-sph.cpp index ec2f6ddc0..42aa5d639 100644 --- a/src/metrics/tests/sr-cart-sph.cpp +++ b/src/metrics/tests/sr-cart-sph.cpp @@ -123,30 +123,30 @@ auto main(int argc, char* argv[]) -> int { const auto res2d = std::vector { 64, 32 }; const auto res3d = std::vector { 64, 32, 16 }; const auto ext1dcart = boundaries_t { - {10.0, 20.0} + { 10.0, 20.0 } }; const auto ext2dcart = boundaries_t { - {0.0, 20.0}, - {0.0, 10.0} + { 0.0, 20.0 }, + { 0.0, 10.0 } }; const auto ext3dcart = boundaries_t { - {-2.0, 2.0}, - {-1.0, 1.0}, - {-0.5, 0.5} + { -2.0, 2.0 }, + { -1.0, 1.0 }, + { -0.5, 0.5 } }; const auto extsph = boundaries_t { - {1.0, 10.0}, - {0.0, constant::PI} + { 1.0, 10.0 }, + { 0.0, constant::PI } }; const auto params = std::map { - {"r0", -ONE}, - { "h", (real_t)0.25} + { "r0", -ONE }, + { "h", (real_t)0.25 } }; testMetric>({ 128 }, ext1dcart); testMetric>(res2d, ext2dcart, 200); testMetric>(res3d, ext3dcart, 500); - testMetric>(res2d, extsph, 10); + testMetric>(res2d, extsph, 100); testMetric>(res2d, extsph, 200, params); } catch (std::exception& e) { From 6bdbeb93806f21a8ca8eede8a70312a988f4dc4c Mon Sep 17 00:00:00 2001 From: hayk Date: Mon, 4 Nov 2024 16:56:12 -0500 Subject: [PATCH 19/22] issues in writer tests (RUNTEST) --- .github/workflows/actions.yml | 2 +- dev/runners/Dockerfile.runner.cpu | 6 +++--- src/output/tests/writer-mpi.cpp | 4 ++-- src/output/tests/writer-nompi.cpp | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 3e0de8808..f60ee9061 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -48,7 +48,7 @@ jobs: elif [ "${{ matrix.device }}" = "amd-gpu" ]; then FLAGS="-D Kokkos_ENABLE_HIP=ON -D Kokkos_ARCH_AMD_GFX1100=ON" elif [ "${{ matrix.device }}" = "cpu" ]; then - FLAGS="" + FLAGS="-D mpi=ON" fi cmake -B build -D TESTS=ON -D output=ON -D precision=${{ matrix.precision }} $FLAGS - name: Compile diff --git a/dev/runners/Dockerfile.runner.cpu b/dev/runners/Dockerfile.runner.cpu index fc13ec9b5..3c2cf4926 100644 --- a/dev/runners/Dockerfile.runner.cpu +++ b/dev/runners/Dockerfile.runner.cpu @@ -7,14 +7,14 @@ RUN apt-get update && apt-get upgrade -y # cmake & build tools RUN apt-get remove -y --purge cmake && \ - apt-get install -y sudo wget curl build-essential && \ + apt-get install -y sudo wget curl build-essential openmpi-bin openmpi-common libopenmpi-dev && \ wget "https://github.com/Kitware/CMake/releases/download/v3.29.6/cmake-3.29.6-linux-x86_64.tar.gz" -P /opt && \ tar xvf /opt/cmake-3.29.6-linux-x86_64.tar.gz -C /opt && \ rm /opt/cmake-3.29.6-linux-x86_64.tar.gz ENV PATH=/opt/cmake-3.29.6-linux-x86_64/bin:$PATH # adios2 -RUN apt-get update && apt-get install -y git libhdf5-dev && \ +RUN apt-get update && apt-get install -y git libhdf5-openmpi-dev && \ git clone https://github.com/ornladios/ADIOS2.git /opt/adios2-src && \ cd /opt/adios2-src && \ cmake -B build \ @@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y git libhdf5-dev && \ -D ADIOS2_USE_ZeroMQ=OFF \ -D BUILD_TESTING=OFF \ -D ADIOS2_BUILD_EXAMPLES=OFF \ - -D ADIOS2_USE_MPI=OFF \ + -D ADIOS2_USE_MPI=ON \ -D ADIOS2_HAVE_HDF5_VOL=OFF \ -D CMAKE_INSTALL_PREFIX=/opt/adios2 && \ cmake --build build -j && \ diff --git a/src/output/tests/writer-mpi.cpp b/src/output/tests/writer-mpi.cpp index c2729f658..5a5ae8007 100644 --- a/src/output/tests/writer-mpi.cpp +++ b/src/output/tests/writer-mpi.cpp @@ -100,8 +100,8 @@ auto main(int argc, char* argv[]) -> int { std::size_t step_read; long double time_read; - reader.Get(io.InquireVariable("Step"), step_read); - reader.Get(io.InquireVariable("Time"), time_read); + reader.Get(io.InquireVariable("Step"), &step_read); + reader.Get(io.InquireVariable("Time"), &time_read); raise::ErrorIf(step_read != step, "Step is not correct", HERE); raise::ErrorIf((float)time_read != (float)step * 0.1f, "Time is not correct", diff --git a/src/output/tests/writer-nompi.cpp b/src/output/tests/writer-nompi.cpp index 3fe42bf1b..803f907e8 100644 --- a/src/output/tests/writer-nompi.cpp +++ b/src/output/tests/writer-nompi.cpp @@ -111,8 +111,8 @@ auto main(int argc, char* argv[]) -> int { std::size_t step_read; long double time_read; - reader.Get(io.InquireVariable("Step"), step_read); - reader.Get(io.InquireVariable("Time"), time_read); + reader.Get(io.InquireVariable("Step"), &step_read); + reader.Get(io.InquireVariable("Time"), &time_read); raise::ErrorIf(step_read != (step + 1) * 10, "Step is not correct", HERE); raise::ErrorIf((float)time_read != 123 + (float)step * 0.4f, "Time is not correct", From cdd3e8717735c43a34ecae4227886693a0ae69a1 Mon Sep 17 00:00:00 2001 From: hayk Date: Mon, 4 Nov 2024 17:10:28 -0500 Subject: [PATCH 20/22] writer test -> Sync (RUNTEST) --- src/output/tests/writer-mpi.cpp | 8 ++++++-- src/output/tests/writer-nompi.cpp | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/output/tests/writer-mpi.cpp b/src/output/tests/writer-mpi.cpp index 5a5ae8007..c6f5e5a09 100644 --- a/src/output/tests/writer-mpi.cpp +++ b/src/output/tests/writer-mpi.cpp @@ -100,8 +100,12 @@ auto main(int argc, char* argv[]) -> int { std::size_t step_read; long double time_read; - reader.Get(io.InquireVariable("Step"), &step_read); - reader.Get(io.InquireVariable("Time"), &time_read); + reader.Get(io.InquireVariable("Step"), + &step_read, + adios2::Mode::Sync); + reader.Get(io.InquireVariable("Time"), + &time_read, + adios2::Mode::Sync); raise::ErrorIf(step_read != step, "Step is not correct", HERE); raise::ErrorIf((float)time_read != (float)step * 0.1f, "Time is not correct", diff --git a/src/output/tests/writer-nompi.cpp b/src/output/tests/writer-nompi.cpp index 803f907e8..ee93202b9 100644 --- a/src/output/tests/writer-nompi.cpp +++ b/src/output/tests/writer-nompi.cpp @@ -111,8 +111,12 @@ auto main(int argc, char* argv[]) -> int { std::size_t step_read; long double time_read; - reader.Get(io.InquireVariable("Step"), &step_read); - reader.Get(io.InquireVariable("Time"), &time_read); + reader.Get(io.InquireVariable("Step"), + &step_read, + adios2::Mode::Sync); + reader.Get(io.InquireVariable("Time"), + &time_read, + adios2::Mode::Sync); raise::ErrorIf(step_read != (step + 1) * 10, "Step is not correct", HERE); raise::ErrorIf((float)time_read != 123 + (float)step * 0.4f, "Time is not correct", From 3c793cdd31c04035be3a20106a98a7d49a77be53 Mon Sep 17 00:00:00 2001 From: hayk Date: Mon, 4 Nov 2024 17:19:06 -0500 Subject: [PATCH 21/22] added flushall to tests (RUNTEST) --- src/output/tests/writer-mpi.cpp | 2 ++ src/output/tests/writer-nompi.cpp | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/output/tests/writer-mpi.cpp b/src/output/tests/writer-mpi.cpp index c6f5e5a09..6ab16305f 100644 --- a/src/output/tests/writer-mpi.cpp +++ b/src/output/tests/writer-mpi.cpp @@ -84,6 +84,8 @@ auto main(int argc, char* argv[]) -> int { adios.ExitComputationBlock(); } + adios.FlushAll(); + { // read adios2::IO io = adios.DeclareIO("read-test"); diff --git a/src/output/tests/writer-nompi.cpp b/src/output/tests/writer-nompi.cpp index ee93202b9..d22881741 100644 --- a/src/output/tests/writer-nompi.cpp +++ b/src/output/tests/writer-nompi.cpp @@ -93,6 +93,8 @@ auto main(int argc, char* argv[]) -> int { writer.endWriting(); } + adios.FlushAll(); + { // read adios2::IO io = adios.DeclareIO("read-test"); From c18162d9e04ed366895da4905653b45d0e3473f7 Mon Sep 17 00:00:00 2001 From: haykh Date: Mon, 4 Nov 2024 18:09:55 -0500 Subject: [PATCH 22/22] correct layout in tests (RUNTEST) --- src/output/tests/writer-nompi.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/output/tests/writer-nompi.cpp b/src/output/tests/writer-nompi.cpp index d22881741..08200d804 100644 --- a/src/output/tests/writer-nompi.cpp +++ b/src/output/tests/writer-nompi.cpp @@ -100,6 +100,8 @@ auto main(int argc, char* argv[]) -> int { adios2::IO io = adios.DeclareIO("read-test"); io.SetEngine("hdf5"); adios2::Engine reader = io.Open("test.h5", adios2::Mode::Read); + const auto layoutRight = io.InquireAttribute("LayoutRight").Data()[0] == + 1; raise::ErrorIf(io.InquireAttribute("NGhosts").Data()[0] != 0, "NGhosts is not correct", @@ -138,6 +140,9 @@ auto main(int argc, char* argv[]) -> int { std::size_t nx1_r = dims[0]; std::size_t nx2_r = dims[1]; std::size_t nx3_r = dims[2]; + if (!layoutRight) { + std::swap(nx1_r, nx3_r); + } raise::ErrorIf((nx1_r != CEILDIV(nx1, dwn1)) || (nx2_r != CEILDIV(nx2, dwn2)) || (nx3_r != CEILDIV(nx3, dwn3)), @@ -151,8 +156,14 @@ auto main(int argc, char* argv[]) -> int { CEILDIV(nx3, dwn3)), HERE); + if (!layoutRight) { + std::swap(nx1_r, nx3_r); + } fieldVar.SetSelection( adios2::Box({ 0, 0, 0 }, { nx1_r, nx2_r, nx3_r })); + if (!layoutRight) { + std::swap(nx1_r, nx3_r); + } field_read = array_t(name, nx1_r, nx2_r, nx3_r); auto field_read_h = Kokkos::create_mirror_view(field_read); reader.Get(fieldVar, field_read_h.data(), adios2::Mode::Sync);