From 46c35f21e44d00428a2c8a32ef9e293906fe44d5 Mon Sep 17 00:00:00 2001 From: haykh Date: Sun, 28 Jul 2024 23:44:56 -0400 Subject: [PATCH 1/9] version bump --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e68751f4..2260323a3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,7 @@ set(PROJECT_NAME entity) project( ${PROJECT_NAME} - VERSION 1.1.0 + VERSION 1.2.0 LANGUAGES CXX C) add_compile_options("-D ENTITY_VERSION=\"${PROJECT_VERSION}\"") execute_process(COMMAND From 236390c00ce2ae68690987685e18e56ca4060058 Mon Sep 17 00:00:00 2001 From: haykh Date: Mon, 29 Jul 2024 09:10:41 -0400 Subject: [PATCH 2/9] patch version + upd readme --- CMakeLists.txt | 2 +- README.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e68751f4..2152ebcf2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,7 @@ set(PROJECT_NAME entity) project( ${PROJECT_NAME} - VERSION 1.1.0 + VERSION 1.1.1 LANGUAGES CXX C) add_compile_options("-D ENTITY_VERSION=\"${PROJECT_VERSION}\"") execute_process(COMMAND diff --git a/README.md b/README.md index 41a5fe47e..d6f4597f5 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ Our [detailed documentation](https://entity-toolkit.github.io/) includes everyth ## Core developers (alphabetical) +👀 __Yangyang Cai__ {[@StaticObserver](https://github.com/StaticObserver): GRPIC} + 💁‍♂️ __Alexander Chernoglazov__ {[@SChernoglazov](https://github.com/SChernoglazov): PIC} 🍵 __Benjamin Crinquand__ {[@bcrinquand](https://github.com/bcrinquand): GRPIC, cubed-sphere} From 07f5931f3c788a87ebaa2f13b51489740ed0a892 Mon Sep 17 00:00:00 2001 From: haykh Date: Thu, 8 Aug 2024 21:20:24 -0400 Subject: [PATCH 3/9] fixes some issues with prtl output --- src/framework/domain/output.cpp | 38 +++++++++++++++++++++++++++------ src/framework/parameters.cpp | 15 ++++++++----- src/output/writer.cpp | 19 +++++++++++------ src/output/writer.h | 5 ++++- 4 files changed, 58 insertions(+), 19 deletions(-) diff --git a/src/framework/domain/output.cpp b/src/framework/domain/output.cpp index cabb37093..165ddb8b6 100644 --- a/src/framework/domain/output.cpp +++ b/src/framework/domain/output.cpp @@ -26,6 +26,10 @@ #include #include +#if defined(MPI_ENABLED) + #include +#endif // MPI_ENABLED + #include #include #include @@ -471,21 +475,41 @@ namespace ntt { species.ux1, species.ux2, species.ux3, species.phi, species.weight, local_domain->mesh.metric)); + // clang-format on - g_writer.writeParticleQuantity(buff_wei, prtl.name("W", 0)); - g_writer.writeParticleQuantity(buff_ux1, prtl.name("U", 1)); - g_writer.writeParticleQuantity(buff_ux2, prtl.name("U", 2)); - g_writer.writeParticleQuantity(buff_ux3, prtl.name("U", 3)); + std::size_t offset = 0; + std::size_t glob_tot = nout; +#if defined(MPI_ENABLED) + auto glob_nout = std::vector(g_ndomains); + MPI_Allgather(&nout, + 1, + mpi::get_type(), + glob_nout.data(), + 1, + mpi::get_type(), + MPI_COMM_WORLD); + glob_tot = 0; + for (auto r = 0; r < g_mpi_size; ++r) { + if (r < g_mpi_rank) { + offset += glob_nout[r]; + } + glob_tot += glob_nout[r]; + } +#endif // MPI_ENABLED + g_writer.writeParticleQuantity(buff_wei, glob_tot, offset, prtl.name("W", 0)); + g_writer.writeParticleQuantity(buff_ux1, glob_tot, offset, prtl.name("U", 1)); + g_writer.writeParticleQuantity(buff_ux2, glob_tot, offset, prtl.name("U", 2)); + g_writer.writeParticleQuantity(buff_ux3, glob_tot, offset, prtl.name("U", 3)); if constexpr (M::Dim == Dim::_1D or M::Dim == Dim::_2D or M::Dim == Dim::_3D) { - g_writer.writeParticleQuantity(buff_x1, prtl.name("X", 1)); + g_writer.writeParticleQuantity(buff_x1, glob_tot, offset, prtl.name("X", 1)); } if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { - g_writer.writeParticleQuantity(buff_x2, prtl.name("X", 2)); + g_writer.writeParticleQuantity(buff_x2, glob_tot, offset, prtl.name("X", 2)); } if constexpr (M::Dim == Dim::_3D or ((D == Dim::_2D) and (M::CoordType != Coord::Cart))) { - g_writer.writeParticleQuantity(buff_x3, prtl.name("X", 3)); + g_writer.writeParticleQuantity(buff_x3, glob_tot, offset, prtl.name("X", 3)); } } } // end shouldWrite("particles", step, time) diff --git a/src/framework/parameters.cpp b/src/framework/parameters.cpp index 6e581a6b7..20c7e83d8 100644 --- a/src/framework/parameters.cpp +++ b/src/framework/parameters.cpp @@ -408,11 +408,16 @@ namespace ntt { toml::find_or(raw_data, "output", "fields", "stride", defaults::output::flds_stride)); // particles - const auto prtl_out = toml::find_or(raw_data, - "output", - "particles", - "species", - std::vector {}); + auto prtl_out = toml::find_or(raw_data, + "output", + "particles", + "species", + std::vector {}); + if (prtl_out.size() == 0) { + for (unsigned short i = 0; i < species.size(); ++i) { + prtl_out.push_back(i + 1); + } + } set("output.particles.species", prtl_out); set("output.particles.stride", toml::find_or(raw_data, diff --git a/src/output/writer.cpp b/src/output/writer.cpp index 182705efa..91bf596a8 100644 --- a/src/output/writer.cpp +++ b/src/output/writer.cpp @@ -130,17 +130,20 @@ namespace out { for (const auto& prtl : m_prtl_writers) { for (auto d { 0u }; d < dim; ++d) { m_io.DefineVariable(prtl.name("X", d + 1), - {}, - {}, + { adios2::UnknownDim }, + { adios2::UnknownDim }, { adios2::UnknownDim }); } for (auto d { 0u }; d < Dim::_3D; ++d) { m_io.DefineVariable(prtl.name("U", d + 1), - {}, - {}, + { adios2::UnknownDim }, + { adios2::UnknownDim }, { adios2::UnknownDim }); } - m_io.DefineVariable(prtl.name("W", 0), {}, {}, { adios2::UnknownDim }); + m_io.DefineVariable(prtl.name("W", 0), + { adios2::UnknownDim }, + { adios2::UnknownDim }, + { adios2::UnknownDim }); } } @@ -216,9 +219,13 @@ namespace out { } void Writer::writeParticleQuantity(const array_t& array, + std::size_t glob_total, + std::size_t loc_offset, const std::string& varname) { auto var = m_io.InquireVariable(varname); - var.SetSelection(adios2::Box({}, { array.extent(0) })); + var.SetShape({ glob_total }); + var.SetSelection( + adios2::Box({ loc_offset }, { array.extent(0) })); auto array_h = Kokkos::create_mirror_view(array); Kokkos::deep_copy(array_h, array); m_writer.Put(var, array_h); diff --git a/src/output/writer.h b/src/output/writer.h index d6a089095..517a1655d 100644 --- a/src/output/writer.h +++ b/src/output/writer.h @@ -114,7 +114,10 @@ namespace out { const ndfield_t&, const std::vector&); - void writeParticleQuantity(const array_t&, const std::string&); + void writeParticleQuantity(const array_t&, + std::size_t, + std::size_t, + const std::string&); void writeSpectrum(const array_t&, const std::string&); void writeSpectrumBins(const array_t&, const std::string&); From a5af284688e438472d7b5adc87f4a82412777f81 Mon Sep 17 00:00:00 2001 From: haykh Date: Sun, 28 Jul 2024 20:15:35 -0400 Subject: [PATCH 4/9] added report on arch & rocm+cuda versions --- cmake/report.cmake | 50 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/cmake/report.cmake b/cmake/report.cmake index e15074943..fe914baa8 100644 --- a/cmake/report.cmake +++ b/cmake/report.cmake @@ -150,6 +150,16 @@ PrintChoices("CUDA" 0 42 ) +PrintChoices("HIP" + "Kokkos_ENABLE_HIP" + "${ON_OFF_VALUES}" + ${Kokkos_ENABLE_HIP} + "OFF" + "${Green}" + HIP_REPORT + 0 + 42 +) PrintChoices("OpenMP" "Kokkos_ENABLE_OPENMP" "${ON_OFF_VALUES}" @@ -183,6 +193,28 @@ PrintChoices("C compiler" 42 ) +get_cmake_property(_variableNames VARIABLES) +foreach (_variableName ${_variableNames}) + string(REGEX MATCH "Kokkos_ARCH_*" _isMatched ${_variableName}) + if(_isMatched) + get_property(isSet CACHE ${_variableName} PROPERTY VALUE) + if(isSet STREQUAL "ON") + string(REGEX REPLACE "Kokkos_ARCH_" "" ARCH ${_variableName}) + break() + endif() + endif() +endforeach() +PrintChoices("Architecture" + "Kokkos_ARCH_*" + "${ARCH}" + "${ARCH}" + "N/A" + "${ColorReset}" + ARCH_REPORT + 0 + 42 +) + if(${Kokkos_ENABLE_CUDA}) if("${CMAKE_CUDA_COMPILER}" STREQUAL "") execute_process(COMMAND which nvcc OUTPUT_VARIABLE CUDACOMP) @@ -194,14 +226,13 @@ if(${Kokkos_ENABLE_CUDA}) message(STATUS "CUDA compiler: ${CUDACOMP}") execute_process(COMMAND bash -c "${CUDACOMP} --version | grep release | sed -e 's/.*release //' -e 's/,.*//'" - OUTPUT_VARIABLE CUDACOMP_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) PrintChoices("CUDA compiler" "CMAKE_CUDA_COMPILER" - "${CUDACOMP} v${CUDACOMP_VERSION}" - "${CUDACOMP} v${CUDACOMP_VERSION}" + "${CUDACOMP}" + "${CUDACOMP}" "N/A" "${ColorReset}" CUDA_COMPILER_REPORT @@ -210,6 +241,12 @@ if(${Kokkos_ENABLE_CUDA}) ) endif() +if (${Kokkos_ENABLE_HIP}) + execute_process(COMMAND bash -c "hipcc --version | grep HIP | cut -d ':' -f 2 | tr -d ' '" + OUTPUT_VARIABLE ROCM_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE) +endif() + set(DOT_SYMBOL "${ColorReset}.") set(DOTTED_LINE_SYMBOL "${ColorReset}. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . ") @@ -242,7 +279,9 @@ message(" ${OUTPUT_REPORT}") message("${DASHED_LINE_SYMBOL} Compile configurations") +message(" ${ARCH_REPORT}") message(" ${CUDA_REPORT}") +message(" ${HIP_REPORT}") message(" ${OPENMP_REPORT}") message(" ${C_COMPILER_REPORT}") @@ -259,6 +298,11 @@ message(" ${DEBUG_REPORT}") message("${DASHED_LINE_SYMBOL}\nDependencies") +if (NOT "${CUDACOMP_VERSION}" STREQUAL "") + message(" - CUDA:\tv${CUDACOMP_VERSION}") +elseif(NOT "${ROCM_VERSION}" STREQUAL "") + message(" - ROCm:\tv${ROCM_VERSION}") +endif() message(" - Kokkos:\tv${Kokkos_VERSION}") if(${output}) message(" - ADIOS2:\tv${adios2_VERSION}") From e87ebdfe1d7aa8f5424805b1e3c2e0312e33b79e Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sun, 28 Jul 2024 23:15:22 -0400 Subject: [PATCH 5/9] Update magnetar setup. --- setups/srpic/magnetar/magnetar.py | 8 +- setups/srpic/magnetar/magnetar.toml | 28 +-- setups/srpic/magnetar/pgen.hpp | 329 +++++++++++++++------------- 3 files changed, 190 insertions(+), 175 deletions(-) diff --git a/setups/srpic/magnetar/magnetar.py b/setups/srpic/magnetar/magnetar.py index b9b72d73d..0bbf790e5 100644 --- a/setups/srpic/magnetar/magnetar.py +++ b/setups/srpic/magnetar/magnetar.py @@ -1,9 +1,9 @@ import nt2.read as nt2r import matplotlib as mpl -data = nt2r.Data("magnetosphere.h5") +data = nt2r.Data("magnetar.h5") def plot (ti, data): - (data.N_2 - data.N_1).isel(t=ti).polar.pcolor( - norm=mpl.colors.SymLogNorm(vmin=-5, vmax=5, linthresh=1e-2, linscale=1), - cmap="RdBu_r") \ No newline at end of file + (data.Bph*(data.r*np.sin(data.th))).isel(t=ti).polar.pcolor( + norm=mpl.colors.Normalize(vmin=-0.075, vmax=0.075), + cmap="PuOr") \ No newline at end of file diff --git a/setups/srpic/magnetar/magnetar.toml b/setups/srpic/magnetar/magnetar.toml index 76bb3af8e..2a2260af5 100644 --- a/setups/srpic/magnetar/magnetar.toml +++ b/setups/srpic/magnetar/magnetar.toml @@ -1,10 +1,10 @@ [simulation] name = "magnetar" engine = "srpic" - runtime = 200.0 + runtime = 50.0 [grid] - resolution = [4096,2048] + resolution = [2048,1024] extent = [[1.0, 400.0]] [grid.metric] @@ -25,7 +25,7 @@ ds = 0.5 [scales] - larmor0 = 5e-6 + larmor0 = 1e-5 skindepth0 = 0.01 [algorithms] @@ -47,54 +47,56 @@ label = "e-" mass = 1.0 charge = -1.0 - maxnpart = 1e8 + maxnpart = 5e7 pusher = "Boris,GCA" [[particles.species]] label = "e+" mass = 1.0 charge = 1.0 - maxnpart = 1e8 + maxnpart = 5e7 pusher = "Boris,GCA" - [[particles.species]] + [[particles.species]] label = "e-" mass = 1.0 charge = -1.0 - maxnpart = 1e4 + maxnpart = 5e7 pusher = "Boris,GCA" [[particles.species]] label = "e+" mass = 1.0 charge = 1.0 - maxnpart = 1e4 + maxnpart = 5e7 pusher = "Boris,GCA" - [[particles.species]] + [[particles.species]] label = "e-" mass = 1.0 charge = -1.0 - maxnpart = 2e8 + maxnpart = 5e7 pusher = "Boris,GCA" [[particles.species]] label = "e+" mass = 1.0 charge = 1.0 - maxnpart = 2e8 + maxnpart = 5e7 pusher = "Boris,GCA" [setup] Bsurf = 1.0 omega = 0.0125 + pp_thres = 10.0 + gamma_pairs = 1.75 [output] format = "hdf5" [output.fields] interval_time = 0.5 - quantities = ["N_1", "N_2", "N_5", "N_6", "B", "E", "J"] + quantities = ["N_1", "N_2", "N_3", "N_4", "N_5", "N_6", "B", "E", "J"] [output.particles] enable = false @@ -103,4 +105,4 @@ enable = false [diagnostics] - interval = 100 + interval = 1 diff --git a/setups/srpic/magnetar/pgen.hpp b/setups/srpic/magnetar/pgen.hpp index 24dcb9d3a..cacbb7c9a 100644 --- a/setups/srpic/magnetar/pgen.hpp +++ b/setups/srpic/magnetar/pgen.hpp @@ -53,7 +53,7 @@ namespace user { } Inline auto ex2(const coord_t& x_Ph) const -> real_t { - auto sigma = (x_Ph[1] - 0.5 * constant::PI) / + auto sigma = (x_Ph[1] - HALF * constant::PI) / (static_cast(0.2) * constant::PI); return -Omega * bx1(x_Ph) * x_Ph[0] * math::sin(x_Ph[1]) * sigma * math::exp((ONE - SQR(SQR(sigma))) * INV_4); @@ -83,52 +83,22 @@ namespace user { const Metadomain& global_domain; - const real_t Bsurf, Rstar, Omega; + const real_t Bsurf, Rstar, Omega, gamma_pairs, pp_thres; InitFields init_flds; - + inline PGen(const SimulationParams& p, const Metadomain& m) : arch::ProblemGenerator(p) , global_domain { m } , Bsurf { p.template get("setup.Bsurf", ONE) } , Rstar { m.mesh().extent(in::x1).first } , Omega { p.template get("setup.omega") } - , init_flds { Bsurf, Rstar } {} + , pp_thres { p.template get("setup.pp_thres") } + , gamma_pairs { p.template get("setup.gamma_pairs") } + , init_flds { Bsurf, Rstar } { + } inline PGen() {} - // inline void InitPrtls(Domain& local_domain) { - // Kokkos::deep_copy(local_domain.fields.bckp, ZERO); - // // @HACK - // const auto x_surf = 1.1f; - // const auto ds = params.template get( - // "grid.boundaries.atmosphere.ds"); - // const auto temp = params.template get( - // "grid.boundaries.atmosphere.temperature"); - // const auto height = params.template get( - // "grid.boundaries.atmosphere.height"); - // const auto species = params.template get>( - // "grid.boundaries.atmosphere.species"); - // const auto nmax = params.template get( - // "grid.boundaries.atmosphere.density"); - // const auto atm_injector = - // arch::AtmosphereInjector { - // local_domain.mesh.metric, - // local_domain.fields.bckp, - // nmax, - // height, - // x_surf, - // ds, - // temp, - // local_domain.random_pool, - // species - // }; - // arch::InjectNonUniform(params, - // local_domain, - // atm_injector, - // nmax, - // true); - // } - auto FieldDriver(real_t time) const -> DriveFields { const real_t omega_t = Omega * @@ -139,127 +109,170 @@ namespace user { return DriveFields { time, Bsurf, Rstar, omega_t }; } - // void CustomPostStep(std::size_t time, long double, Domain& domain) { - - // const auto pp_thres = 1*10.0; - // const auto gamma_pairs = 1*0.5 * 3.5; - - // auto& species_e = domain.species[4]; - // auto& species_p = domain.species[5]; - // auto metric = domain.mesh.metric; - - // for (std::size_t s { 0 }; s < 6; ++s) { - // if ((s == 1) || (s == 2) || (s == 3)) { - // continue; - // } - - // array_t elec_ind("elec_ind"); - // array_t pos_ind("pos_ind"); - // auto offset_e = species_e.npart(); - // auto offset_p = species_p.npart(); - - // auto ux1_e = species_e.ux1; - // auto ux2_e = species_e.ux2; - // auto ux3_e = species_e.ux3; - // auto i1_e = species_e.i1; - // auto i2_e = species_e.i2; - // auto dx1_e = species_e.dx1; - // auto dx2_e = species_e.dx2; - // auto phi_e = species_e.phi; - // auto weight_e = species_e.weight; - // auto tag_e = species_e.tag; - - // auto ux1_p = species_p.ux1; - // auto ux2_p = species_p.ux2; - // auto ux3_p = species_p.ux3; - // auto i1_p = species_p.i1; - // auto i2_p = species_p.i2; - // auto dx1_p = species_p.dx1; - // auto dx2_p = species_p.dx2; - // auto phi_p = species_p.phi; - // auto weight_p = species_p.weight; - // auto tag_p = species_p.tag; - - // auto& species = domain.species[s]; - // auto ux1 = species.ux1; - // auto ux2 = species.ux2; - // auto ux3 = species.ux3; - // auto i1 = species.i1; - // auto i2 = species.i2; - // auto dx1 = species.dx1; - // auto dx2 = species.dx2; - // auto phi = species.phi; - // auto weight = species.weight; - // auto tag = species.tag; - - // Kokkos::parallel_for( - // "InjectPairs", species.rangeActiveParticles(), Lambda(index_t p) { - // if (tag(p) == ParticleTag::dead) { - // return; - // } - - // auto px = ux1(p); - // auto py = ux2(p); - // auto pz = ux3(p); - // auto gamma = math::sqrt(ONE + SQR(px) + SQR(py) + SQR(pz)); - - // const coord_t xCd{ - // static_cast(i1(p)) + dx1(p), - // static_cast(i2(p)) + dx2(p)}; - - // coord_t xPh { ZERO }; - // metric.template convert(xCd, xPh); - - // if ((gamma > pp_thres) && (math::sin(xPh[1]) > 0.1)) { - - // auto new_gamma = gamma - 2.0 * gamma_pairs; - // auto new_fac = math::sqrt(SQR(new_gamma) - 1.0) / math::sqrt(SQR(gamma) - 1.0); - // auto pair_fac = math::sqrt(SQR(gamma_pairs) - 1.0) / math::sqrt(SQR(gamma) - 1.0); - - // auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); - // auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); - - // i1_e(elec_p + offset_e) = i1(p); - // dx1_e(elec_p + offset_e) = dx1(p); - // i2_e(elec_p + offset_e) = i2(p); - // dx2_e(elec_p + offset_e) = dx2(p); - // phi_e(elec_p + offset_e) = phi(p); - // ux1_e(elec_p + offset_e) = px * pair_fac; - // ux2_e(elec_p + offset_e) = py * pair_fac; - // ux3_e(elec_p + offset_e) = pz * pair_fac; - // weight_e(elec_p + offset_e) = weight(p); - // tag_e(elec_p + offset_e) = ParticleTag::alive; - - // i1_p(pos_p + offset_p) = i1(p); - // dx1_p(pos_p + offset_p) = dx1(p); - // i2_p(pos_p + offset_p) = i2(p); - // dx2_p(pos_p + offset_p) = dx2(p); - // phi_p(pos_p + offset_p) = phi(p); - // ux1_p(pos_p + offset_p) = px * pair_fac; - // ux2_p(pos_p + offset_p) = py * pair_fac; - // ux3_p(pos_p + offset_p) = pz * pair_fac; - // weight_p(pos_p + offset_p) = weight(p); - // tag_p(pos_p + offset_p) = ParticleTag::alive; - - // ux1(p) *= new_fac; - // ux2(p) *= new_fac; - // ux3(p) *= new_fac; - - // } - - // }); - - // auto elec_ind_h = Kokkos::create_mirror(elec_ind); - // Kokkos::deep_copy(elec_ind_h, elec_ind); - // species_e.set_npart(offset_e + elec_ind_h()); - - // auto pos_ind_h = Kokkos::create_mirror(pos_ind); - // Kokkos::deep_copy(pos_ind_h, pos_ind); - // species_p.set_npart(offset_p + pos_ind_h()); - - // } - - // } + void CustomPostStep(std::size_t , long double, Domain& domain) { + + // Ad-hoc PP kernel + { + + auto& species2_e = domain.species[2]; + auto& species2_p = domain.species[3]; + auto& species3_e = domain.species[4]; + auto& species3_p = domain.species[5]; + auto metric = domain.mesh.metric; + auto pp_thres_ = this->pp_thres; + auto gamma_pairs_ = this->gamma_pairs; + + for (std::size_t s { 0 }; s < 6; ++s) { + if (s == 1) { + continue; + } + + array_t elec_ind("elec_ind"); + array_t pos_ind("pos_ind"); + + auto offset_e = species3_e.npart(); + auto offset_p = species3_p.npart(); + + auto ux1_e = species3_e.ux1; + auto ux2_e = species3_e.ux2; + auto ux3_e = species3_e.ux3; + auto i1_e = species3_e.i1; + auto i2_e = species3_e.i2; + auto dx1_e = species3_e.dx1; + auto dx2_e = species3_e.dx2; + auto phi_e = species3_e.phi; + auto weight_e = species3_e.weight; + auto tag_e = species3_e.tag; + + auto ux1_p = species3_p.ux1; + auto ux2_p = species3_p.ux2; + auto ux3_p = species3_p.ux3; + auto i1_p = species3_p.i1; + auto i2_p = species3_p.i2; + auto dx1_p = species3_p.dx1; + auto dx2_p = species3_p.dx2; + auto phi_p = species3_p.phi; + auto weight_p = species3_p.weight; + auto tag_p = species3_p.tag; + + if (s == 0) { + + offset_e = species2_e.npart(); + offset_p = species2_p.npart(); + + ux1_e = species2_e.ux1; + ux2_e = species2_e.ux2; + ux3_e = species2_e.ux3; + i1_e = species2_e.i1; + i2_e = species2_e.i2; + dx1_e = species2_e.dx1; + dx2_e = species2_e.dx2; + phi_e = species2_e.phi; + weight_e = species2_e.weight; + tag_e = species2_e.tag; + + ux1_p = species2_p.ux1; + ux2_p = species2_p.ux2; + ux3_p = species2_p.ux3; + i1_p = species2_p.i1; + i2_p = species2_p.i2; + dx1_p = species2_p.dx1; + dx2_p = species2_p.dx2; + phi_p = species2_p.phi; + weight_p = species2_p.weight; + tag_p = species2_p.tag; + + } + + auto& species = domain.species[s]; + auto ux1 = species.ux1; + auto ux2 = species.ux2; + auto ux3 = species.ux3; + auto i1 = species.i1; + auto i2 = species.i2; + auto dx1 = species.dx1; + auto dx2 = species.dx2; + auto phi = species.phi; + auto weight = species.weight; + auto tag = species.tag; + + Kokkos::parallel_for( + "InjectPairs", species.rangeActiveParticles(), Lambda(index_t p) { + if (tag(p) == ParticleTag::dead) { + return; + } + + auto px = ux1(p); + auto py = ux2(p); + auto pz = ux3(p); + auto gamma = math::sqrt(ONE + SQR(px) + SQR(py) + SQR(pz)); + + const coord_t xCd{ + static_cast(i1(p)) + dx1(p), + static_cast(i2(p)) + dx2(p)}; + + coord_t xPh { ZERO }; + metric.template convert(xCd, xPh); + + if ((gamma > pp_thres_) && (math::sin(xPh[1]) > 0.1)) { + + auto new_gamma = gamma - 2.0 * gamma_pairs_; + auto new_fac = math::sqrt(SQR(new_gamma) - 1.0) / math::sqrt(SQR(gamma) - 1.0); + auto pair_fac = math::sqrt(SQR(gamma_pairs_) - 1.0) / math::sqrt(SQR(gamma) - 1.0); + + auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); + auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); + + i1_e(elec_p + offset_e) = i1(p); + dx1_e(elec_p + offset_e) = dx1(p); + i2_e(elec_p + offset_e) = i2(p); + dx2_e(elec_p + offset_e) = dx2(p); + phi_e(elec_p + offset_e) = phi(p); + ux1_e(elec_p + offset_e) = px * pair_fac; + ux2_e(elec_p + offset_e) = py * pair_fac; + ux3_e(elec_p + offset_e) = pz * pair_fac; + weight_e(elec_p + offset_e) = weight(p); + tag_e(elec_p + offset_e) = ParticleTag::alive; + + i1_p(pos_p + offset_p) = i1(p); + dx1_p(pos_p + offset_p) = dx1(p); + i2_p(pos_p + offset_p) = i2(p); + dx2_p(pos_p + offset_p) = dx2(p); + phi_p(pos_p + offset_p) = phi(p); + ux1_p(pos_p + offset_p) = px * pair_fac; + ux2_p(pos_p + offset_p) = py * pair_fac; + ux3_p(pos_p + offset_p) = pz * pair_fac; + weight_p(pos_p + offset_p) = weight(p); + tag_p(pos_p + offset_p) = ParticleTag::alive; + + ux1(p) *= new_fac; + ux2(p) *= new_fac; + ux3(p) *= new_fac; + } + + }); + + auto elec_ind_h = Kokkos::create_mirror(elec_ind); + Kokkos::deep_copy(elec_ind_h, elec_ind); + if (s == 0) { + species2_e.set_npart(offset_e + elec_ind_h()); + } else { + species3_e.set_npart(offset_e + elec_ind_h()); + } + + auto pos_ind_h = Kokkos::create_mirror(pos_ind); + Kokkos::deep_copy(pos_ind_h, pos_ind); + if (s == 0) { + species2_p.set_npart(offset_p + pos_ind_h()); + } else { + species3_p.set_npart(offset_p + pos_ind_h()); + } + + + } + } // Ad-hoc PP kernel + } + }; } // namespace user From 9a88010ac5d12d195cc63192eef43ee50ca7e387 Mon Sep 17 00:00:00 2001 From: haykh Date: Sun, 28 Jul 2024 23:44:56 -0400 Subject: [PATCH 6/9] version bump --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e68751f4..2260323a3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,7 @@ set(PROJECT_NAME entity) project( ${PROJECT_NAME} - VERSION 1.1.0 + VERSION 1.2.0 LANGUAGES CXX C) add_compile_options("-D ENTITY_VERSION=\"${PROJECT_VERSION}\"") execute_process(COMMAND From dfd6d50a661f6287f75990147ba5911a2082fc45 Mon Sep 17 00:00:00 2001 From: haykh Date: Thu, 8 Aug 2024 21:20:24 -0400 Subject: [PATCH 7/9] fixes some issues with prtl output --- src/framework/domain/output.cpp | 38 +++++++++++++++++++++++++++------ src/framework/parameters.cpp | 15 ++++++++----- src/output/writer.cpp | 19 +++++++++++------ src/output/writer.h | 5 ++++- 4 files changed, 58 insertions(+), 19 deletions(-) diff --git a/src/framework/domain/output.cpp b/src/framework/domain/output.cpp index cabb37093..165ddb8b6 100644 --- a/src/framework/domain/output.cpp +++ b/src/framework/domain/output.cpp @@ -26,6 +26,10 @@ #include #include +#if defined(MPI_ENABLED) + #include +#endif // MPI_ENABLED + #include #include #include @@ -471,21 +475,41 @@ namespace ntt { species.ux1, species.ux2, species.ux3, species.phi, species.weight, local_domain->mesh.metric)); + // clang-format on - g_writer.writeParticleQuantity(buff_wei, prtl.name("W", 0)); - g_writer.writeParticleQuantity(buff_ux1, prtl.name("U", 1)); - g_writer.writeParticleQuantity(buff_ux2, prtl.name("U", 2)); - g_writer.writeParticleQuantity(buff_ux3, prtl.name("U", 3)); + std::size_t offset = 0; + std::size_t glob_tot = nout; +#if defined(MPI_ENABLED) + auto glob_nout = std::vector(g_ndomains); + MPI_Allgather(&nout, + 1, + mpi::get_type(), + glob_nout.data(), + 1, + mpi::get_type(), + MPI_COMM_WORLD); + glob_tot = 0; + for (auto r = 0; r < g_mpi_size; ++r) { + if (r < g_mpi_rank) { + offset += glob_nout[r]; + } + glob_tot += glob_nout[r]; + } +#endif // MPI_ENABLED + g_writer.writeParticleQuantity(buff_wei, glob_tot, offset, prtl.name("W", 0)); + g_writer.writeParticleQuantity(buff_ux1, glob_tot, offset, prtl.name("U", 1)); + g_writer.writeParticleQuantity(buff_ux2, glob_tot, offset, prtl.name("U", 2)); + g_writer.writeParticleQuantity(buff_ux3, glob_tot, offset, prtl.name("U", 3)); if constexpr (M::Dim == Dim::_1D or M::Dim == Dim::_2D or M::Dim == Dim::_3D) { - g_writer.writeParticleQuantity(buff_x1, prtl.name("X", 1)); + g_writer.writeParticleQuantity(buff_x1, glob_tot, offset, prtl.name("X", 1)); } if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { - g_writer.writeParticleQuantity(buff_x2, prtl.name("X", 2)); + g_writer.writeParticleQuantity(buff_x2, glob_tot, offset, prtl.name("X", 2)); } if constexpr (M::Dim == Dim::_3D or ((D == Dim::_2D) and (M::CoordType != Coord::Cart))) { - g_writer.writeParticleQuantity(buff_x3, prtl.name("X", 3)); + g_writer.writeParticleQuantity(buff_x3, glob_tot, offset, prtl.name("X", 3)); } } } // end shouldWrite("particles", step, time) diff --git a/src/framework/parameters.cpp b/src/framework/parameters.cpp index 6e581a6b7..20c7e83d8 100644 --- a/src/framework/parameters.cpp +++ b/src/framework/parameters.cpp @@ -408,11 +408,16 @@ namespace ntt { toml::find_or(raw_data, "output", "fields", "stride", defaults::output::flds_stride)); // particles - const auto prtl_out = toml::find_or(raw_data, - "output", - "particles", - "species", - std::vector {}); + auto prtl_out = toml::find_or(raw_data, + "output", + "particles", + "species", + std::vector {}); + if (prtl_out.size() == 0) { + for (unsigned short i = 0; i < species.size(); ++i) { + prtl_out.push_back(i + 1); + } + } set("output.particles.species", prtl_out); set("output.particles.stride", toml::find_or(raw_data, diff --git a/src/output/writer.cpp b/src/output/writer.cpp index 182705efa..91bf596a8 100644 --- a/src/output/writer.cpp +++ b/src/output/writer.cpp @@ -130,17 +130,20 @@ namespace out { for (const auto& prtl : m_prtl_writers) { for (auto d { 0u }; d < dim; ++d) { m_io.DefineVariable(prtl.name("X", d + 1), - {}, - {}, + { adios2::UnknownDim }, + { adios2::UnknownDim }, { adios2::UnknownDim }); } for (auto d { 0u }; d < Dim::_3D; ++d) { m_io.DefineVariable(prtl.name("U", d + 1), - {}, - {}, + { adios2::UnknownDim }, + { adios2::UnknownDim }, { adios2::UnknownDim }); } - m_io.DefineVariable(prtl.name("W", 0), {}, {}, { adios2::UnknownDim }); + m_io.DefineVariable(prtl.name("W", 0), + { adios2::UnknownDim }, + { adios2::UnknownDim }, + { adios2::UnknownDim }); } } @@ -216,9 +219,13 @@ namespace out { } void Writer::writeParticleQuantity(const array_t& array, + std::size_t glob_total, + std::size_t loc_offset, const std::string& varname) { auto var = m_io.InquireVariable(varname); - var.SetSelection(adios2::Box({}, { array.extent(0) })); + var.SetShape({ glob_total }); + var.SetSelection( + adios2::Box({ loc_offset }, { array.extent(0) })); auto array_h = Kokkos::create_mirror_view(array); Kokkos::deep_copy(array_h, array); m_writer.Put(var, array_h); diff --git a/src/output/writer.h b/src/output/writer.h index d6a089095..517a1655d 100644 --- a/src/output/writer.h +++ b/src/output/writer.h @@ -114,7 +114,10 @@ namespace out { const ndfield_t&, const std::vector&); - void writeParticleQuantity(const array_t&, const std::string&); + void writeParticleQuantity(const array_t&, + std::size_t, + std::size_t, + const std::string&); void writeSpectrum(const array_t&, const std::string&); void writeSpectrumBins(const array_t&, const std::string&); From dc2611ebcc3548fd0ea3a47d24bacb7b3707f2b9 Mon Sep 17 00:00:00 2001 From: haykh Date: Thu, 8 Aug 2024 21:30:41 -0400 Subject: [PATCH 8/9] rebase --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2260323a3..2152ebcf2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,7 @@ set(PROJECT_NAME entity) project( ${PROJECT_NAME} - VERSION 1.2.0 + VERSION 1.1.1 LANGUAGES CXX C) add_compile_options("-D ENTITY_VERSION=\"${PROJECT_VERSION}\"") execute_process(COMMAND From 51ad430ae8161bf7b993ad3e7ba6d55ee09cfb30 Mon Sep 17 00:00:00 2001 From: haykh Date: Fri, 9 Aug 2024 00:09:52 -0400 Subject: [PATCH 9/9] skip prtl2phys kernel when nout == 0 --- src/framework/domain/output.cpp | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/framework/domain/output.cpp b/src/framework/domain/output.cpp index 165ddb8b6..be154ce16 100644 --- a/src/framework/domain/output.cpp +++ b/src/framework/domain/output.cpp @@ -462,21 +462,22 @@ namespace ntt { ((D == Dim::_2D) and (M::CoordType != Coord::Cart))) { buff_x3 = array_t { "x3", nout }; } - // clang-format off - Kokkos::parallel_for( - "PrtlToPhys", - nout, - kernel::PrtlToPhys_kernel(prtl_stride, - buff_x1, buff_x2, buff_x3, - buff_ux1, buff_ux2, buff_ux3, - buff_wei, - species.i1, species.i2, species.i3, - species.dx1, species.dx2, species.dx3, - species.ux1, species.ux2, species.ux3, - species.phi, species.weight, - local_domain->mesh.metric)); - - // clang-format on + if (nout > 0) { + // clang-format off + Kokkos::parallel_for( + "PrtlToPhys", + nout, + kernel::PrtlToPhys_kernel(prtl_stride, + buff_x1, buff_x2, buff_x3, + buff_ux1, buff_ux2, buff_ux3, + buff_wei, + species.i1, species.i2, species.i3, + species.dx1, species.dx2, species.dx3, + species.ux1, species.ux2, species.ux3, + species.phi, species.weight, + local_domain->mesh.metric)); + // clang-format on + } std::size_t offset = 0; std::size_t glob_tot = nout; #if defined(MPI_ENABLED)