diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e68751f4..2152ebcf2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,7 @@ set(PROJECT_NAME entity) project( ${PROJECT_NAME} - VERSION 1.1.0 + VERSION 1.1.1 LANGUAGES CXX C) add_compile_options("-D ENTITY_VERSION=\"${PROJECT_VERSION}\"") execute_process(COMMAND diff --git a/README.md b/README.md index 41a5fe47e..d6f4597f5 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ Our [detailed documentation](https://entity-toolkit.github.io/) includes everyth ## Core developers (alphabetical) +👀 __Yangyang Cai__ {[@StaticObserver](https://github.com/StaticObserver): GRPIC} + 💁‍♂️ __Alexander Chernoglazov__ {[@SChernoglazov](https://github.com/SChernoglazov): PIC} 🍵 __Benjamin Crinquand__ {[@bcrinquand](https://github.com/bcrinquand): GRPIC, cubed-sphere} diff --git a/src/framework/domain/output.cpp b/src/framework/domain/output.cpp index cabb37093..be154ce16 100644 --- a/src/framework/domain/output.cpp +++ b/src/framework/domain/output.cpp @@ -26,6 +26,10 @@ #include #include +#if defined(MPI_ENABLED) + #include +#endif // MPI_ENABLED + #include #include #include @@ -458,34 +462,55 @@ namespace ntt { ((D == Dim::_2D) and (M::CoordType != Coord::Cart))) { buff_x3 = array_t { "x3", nout }; } - // clang-format off - Kokkos::parallel_for( - "PrtlToPhys", - nout, - kernel::PrtlToPhys_kernel(prtl_stride, - buff_x1, buff_x2, buff_x3, - buff_ux1, buff_ux2, buff_ux3, - buff_wei, - species.i1, species.i2, species.i3, - species.dx1, species.dx2, species.dx3, - species.ux1, species.ux2, species.ux3, - species.phi, species.weight, - local_domain->mesh.metric)); - // clang-format on - g_writer.writeParticleQuantity(buff_wei, prtl.name("W", 0)); - g_writer.writeParticleQuantity(buff_ux1, prtl.name("U", 1)); - g_writer.writeParticleQuantity(buff_ux2, prtl.name("U", 2)); - g_writer.writeParticleQuantity(buff_ux3, prtl.name("U", 3)); + if (nout > 0) { + // clang-format off + Kokkos::parallel_for( + "PrtlToPhys", + nout, + kernel::PrtlToPhys_kernel(prtl_stride, + buff_x1, buff_x2, buff_x3, + buff_ux1, buff_ux2, buff_ux3, + buff_wei, + species.i1, species.i2, species.i3, + species.dx1, species.dx2, species.dx3, + species.ux1, species.ux2, species.ux3, + species.phi, species.weight, + local_domain->mesh.metric)); + // clang-format on + } + std::size_t offset = 0; + std::size_t glob_tot = nout; +#if defined(MPI_ENABLED) + auto glob_nout = std::vector(g_ndomains); + MPI_Allgather(&nout, + 1, + mpi::get_type(), + glob_nout.data(), + 1, + mpi::get_type(), + MPI_COMM_WORLD); + glob_tot = 0; + for (auto r = 0; r < g_mpi_size; ++r) { + if (r < g_mpi_rank) { + offset += glob_nout[r]; + } + glob_tot += glob_nout[r]; + } +#endif // MPI_ENABLED + g_writer.writeParticleQuantity(buff_wei, glob_tot, offset, prtl.name("W", 0)); + g_writer.writeParticleQuantity(buff_ux1, glob_tot, offset, prtl.name("U", 1)); + g_writer.writeParticleQuantity(buff_ux2, glob_tot, offset, prtl.name("U", 2)); + g_writer.writeParticleQuantity(buff_ux3, glob_tot, offset, prtl.name("U", 3)); if constexpr (M::Dim == Dim::_1D or M::Dim == Dim::_2D or M::Dim == Dim::_3D) { - g_writer.writeParticleQuantity(buff_x1, prtl.name("X", 1)); + g_writer.writeParticleQuantity(buff_x1, glob_tot, offset, prtl.name("X", 1)); } if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { - g_writer.writeParticleQuantity(buff_x2, prtl.name("X", 2)); + g_writer.writeParticleQuantity(buff_x2, glob_tot, offset, prtl.name("X", 2)); } if constexpr (M::Dim == Dim::_3D or ((D == Dim::_2D) and (M::CoordType != Coord::Cart))) { - g_writer.writeParticleQuantity(buff_x3, prtl.name("X", 3)); + g_writer.writeParticleQuantity(buff_x3, glob_tot, offset, prtl.name("X", 3)); } } } // end shouldWrite("particles", step, time) diff --git a/src/framework/parameters.cpp b/src/framework/parameters.cpp index 6e581a6b7..20c7e83d8 100644 --- a/src/framework/parameters.cpp +++ b/src/framework/parameters.cpp @@ -408,11 +408,16 @@ namespace ntt { toml::find_or(raw_data, "output", "fields", "stride", defaults::output::flds_stride)); // particles - const auto prtl_out = toml::find_or(raw_data, - "output", - "particles", - "species", - std::vector {}); + auto prtl_out = toml::find_or(raw_data, + "output", + "particles", + "species", + std::vector {}); + if (prtl_out.size() == 0) { + for (unsigned short i = 0; i < species.size(); ++i) { + prtl_out.push_back(i + 1); + } + } set("output.particles.species", prtl_out); set("output.particles.stride", toml::find_or(raw_data, diff --git a/src/output/writer.cpp b/src/output/writer.cpp index 182705efa..91bf596a8 100644 --- a/src/output/writer.cpp +++ b/src/output/writer.cpp @@ -130,17 +130,20 @@ namespace out { for (const auto& prtl : m_prtl_writers) { for (auto d { 0u }; d < dim; ++d) { m_io.DefineVariable(prtl.name("X", d + 1), - {}, - {}, + { adios2::UnknownDim }, + { adios2::UnknownDim }, { adios2::UnknownDim }); } for (auto d { 0u }; d < Dim::_3D; ++d) { m_io.DefineVariable(prtl.name("U", d + 1), - {}, - {}, + { adios2::UnknownDim }, + { adios2::UnknownDim }, { adios2::UnknownDim }); } - m_io.DefineVariable(prtl.name("W", 0), {}, {}, { adios2::UnknownDim }); + m_io.DefineVariable(prtl.name("W", 0), + { adios2::UnknownDim }, + { adios2::UnknownDim }, + { adios2::UnknownDim }); } } @@ -216,9 +219,13 @@ namespace out { } void Writer::writeParticleQuantity(const array_t& array, + std::size_t glob_total, + std::size_t loc_offset, const std::string& varname) { auto var = m_io.InquireVariable(varname); - var.SetSelection(adios2::Box({}, { array.extent(0) })); + var.SetShape({ glob_total }); + var.SetSelection( + adios2::Box({ loc_offset }, { array.extent(0) })); auto array_h = Kokkos::create_mirror_view(array); Kokkos::deep_copy(array_h, array); m_writer.Put(var, array_h); diff --git a/src/output/writer.h b/src/output/writer.h index d6a089095..517a1655d 100644 --- a/src/output/writer.h +++ b/src/output/writer.h @@ -114,7 +114,10 @@ namespace out { const ndfield_t&, const std::vector&); - void writeParticleQuantity(const array_t&, const std::string&); + void writeParticleQuantity(const array_t&, + std::size_t, + std::size_t, + const std::string&); void writeSpectrum(const array_t&, const std::string&); void writeSpectrumBins(const array_t&, const std::string&);