Skip to content

Bugs in particle output #68

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
haykh merged 9 commits into from
Aug 17, 2024
Merged

Bugs in particle output #68

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
46c35f2
version bump
haykh Jul 29, 2024
07f5931
fixes some issues with prtl output
haykh Aug 9, 2024
a5af284
added report on arch & rocm+cuda versions
haykh Jul 29, 2024
e87ebdf
Update magnetar setup.
Jul 29, 2024
9a88010
version bump
haykh Jul 29, 2024
dfd6d50
fixes some issues with prtl output
haykh Aug 9, 2024
5f38b86
Merge branch 'bug/mpiprtlout' of github.com:entity-toolkit/entity int…
haykh Aug 9, 2024
dc2611e
rebase
haykh Aug 9, 2024
51ad430
skip prtl2phys kernel when nout == 0
haykh Aug 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ set(PROJECT_NAME entity)

project(
${PROJECT_NAME}
VERSION 1.1.0
VERSION 1.1.1
LANGUAGES CXX C)
add_compile_options("-D ENTITY_VERSION=\"${PROJECT_VERSION}\"")
execute_process(COMMAND
Expand Down
67 changes: 46 additions & 21 deletions src/framework/domain/output.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,10 @@
#include <Kokkos_ScatterView.hpp>
#include <Kokkos_StdAlgorithms.hpp>

#if defined(MPI_ENABLED)
#include <mpi.h>
#endif // MPI_ENABLED

#include <algorithm>
#include <iterator>
#include <vector>
Expand Down Expand Up @@ -458,34 +462,55 @@ namespace ntt {
((D == Dim::_2D) and (M::CoordType != Coord::Cart))) {
buff_x3 = array_t<real_t*> { "x3", nout };
}
// clang-format off
Kokkos::parallel_for(
"PrtlToPhys",
nout,
kernel::PrtlToPhys_kernel<S, M>(prtl_stride,
buff_x1, buff_x2, buff_x3,
buff_ux1, buff_ux2, buff_ux3,
buff_wei,
species.i1, species.i2, species.i3,
species.dx1, species.dx2, species.dx3,
species.ux1, species.ux2, species.ux3,
species.phi, species.weight,
local_domain->mesh.metric));
// clang-format on
g_writer.writeParticleQuantity(buff_wei, prtl.name("W", 0));
g_writer.writeParticleQuantity(buff_ux1, prtl.name("U", 1));
g_writer.writeParticleQuantity(buff_ux2, prtl.name("U", 2));
g_writer.writeParticleQuantity(buff_ux3, prtl.name("U", 3));
if (nout > 0) {
// clang-format off
Kokkos::parallel_for(
"PrtlToPhys",
nout,
kernel::PrtlToPhys_kernel<S, M>(prtl_stride,
buff_x1, buff_x2, buff_x3,
buff_ux1, buff_ux2, buff_ux3,
buff_wei,
species.i1, species.i2, species.i3,
species.dx1, species.dx2, species.dx3,
species.ux1, species.ux2, species.ux3,
species.phi, species.weight,
local_domain->mesh.metric));
// clang-format on
}
std::size_t offset = 0;
std::size_t glob_tot = nout;
#if defined(MPI_ENABLED)
auto glob_nout = std::vector<std::size_t>(g_ndomains);
MPI_Allgather(&nout,
1,
mpi::get_type<std::size_t>(),
glob_nout.data(),
1,
mpi::get_type<std::size_t>(),
MPI_COMM_WORLD);
glob_tot = 0;
for (auto r = 0; r < g_mpi_size; ++r) {
if (r < g_mpi_rank) {
offset += glob_nout[r];
}
glob_tot += glob_nout[r];
}
#endif // MPI_ENABLED
g_writer.writeParticleQuantity(buff_wei, glob_tot, offset, prtl.name("W", 0));
g_writer.writeParticleQuantity(buff_ux1, glob_tot, offset, prtl.name("U", 1));
g_writer.writeParticleQuantity(buff_ux2, glob_tot, offset, prtl.name("U", 2));
g_writer.writeParticleQuantity(buff_ux3, glob_tot, offset, prtl.name("U", 3));
if constexpr (M::Dim == Dim::_1D or M::Dim == Dim::_2D or
M::Dim == Dim::_3D) {
g_writer.writeParticleQuantity(buff_x1, prtl.name("X", 1));
g_writer.writeParticleQuantity(buff_x1, glob_tot, offset, prtl.name("X", 1));
}
if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) {
g_writer.writeParticleQuantity(buff_x2, prtl.name("X", 2));
g_writer.writeParticleQuantity(buff_x2, glob_tot, offset, prtl.name("X", 2));
}
if constexpr (M::Dim == Dim::_3D or
((D == Dim::_2D) and (M::CoordType != Coord::Cart))) {
g_writer.writeParticleQuantity(buff_x3, prtl.name("X", 3));
g_writer.writeParticleQuantity(buff_x3, glob_tot, offset, prtl.name("X", 3));
}
}
} // end shouldWrite("particles", step, time)
Expand Down
15 changes: 10 additions & 5 deletions src/framework/parameters.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -408,11 +408,16 @@ namespace ntt {
toml::find_or(raw_data, "output", "fields", "stride", defaults::output::flds_stride));

// particles
const auto prtl_out = toml::find_or(raw_data,
"output",
"particles",
"species",
std::vector<unsigned short> {});
auto prtl_out = toml::find_or(raw_data,
"output",
"particles",
"species",
std::vector<unsigned short> {});
if (prtl_out.size() == 0) {
for (unsigned short i = 0; i < species.size(); ++i) {
prtl_out.push_back(i + 1);
}
}
set("output.particles.species", prtl_out);
set("output.particles.stride",
toml::find_or(raw_data,
Expand Down
19 changes: 13 additions & 6 deletions src/output/writer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,17 +130,20 @@ namespace out {
for (const auto& prtl : m_prtl_writers) {
for (auto d { 0u }; d < dim; ++d) {
m_io.DefineVariable<real_t>(prtl.name("X", d + 1),
{},
{},
{ adios2::UnknownDim },
{ adios2::UnknownDim },
{ adios2::UnknownDim });
}
for (auto d { 0u }; d < Dim::_3D; ++d) {
m_io.DefineVariable<real_t>(prtl.name("U", d + 1),
{},
{},
{ adios2::UnknownDim },
{ adios2::UnknownDim },
{ adios2::UnknownDim });
}
m_io.DefineVariable<real_t>(prtl.name("W", 0), {}, {}, { adios2::UnknownDim });
m_io.DefineVariable<real_t>(prtl.name("W", 0),
{ adios2::UnknownDim },
{ adios2::UnknownDim },
{ adios2::UnknownDim });
}
}

Expand Down Expand Up @@ -216,9 +219,13 @@ namespace out {
}

void Writer::writeParticleQuantity(const array_t<real_t*>& array,
std::size_t glob_total,
std::size_t loc_offset,
const std::string& varname) {
auto var = m_io.InquireVariable<real_t>(varname);
var.SetSelection(adios2::Box<adios2::Dims>({}, { array.extent(0) }));
var.SetShape({ glob_total });
var.SetSelection(
adios2::Box<adios2::Dims>({ loc_offset }, { array.extent(0) }));
auto array_h = Kokkos::create_mirror_view(array);
Kokkos::deep_copy(array_h, array);
m_writer.Put<real_t>(var, array_h);
Expand Down
5 changes: 4 additions & 1 deletion src/output/writer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,10 @@ namespace out {
const ndfield_t<D, N>&,
const std::vector<std::size_t>&);

void writeParticleQuantity(const array_t<real_t*>&, const std::string&);
void writeParticleQuantity(const array_t<real_t*>&,
std::size_t,
std::size_t,
const std::string&);
void writeSpectrum(const array_t<real_t*>&, const std::string&);
void writeSpectrumBins(const array_t<real_t*>&, const std::string&);

Expand Down