From 1920f4415ec6d78c5f880a2afd73a2deac805213 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Wed, 21 Aug 2024 20:15:11 -0400 Subject: [PATCH 001/183] Add bulk velocity as moment output. --- src/framework/domain/output.cpp | 8 ++++ src/global/enums.h | 5 ++- src/global/tests/enums.cpp | 2 +- src/kernels/particle_moments.hpp | 65 +++++++++++++++++++++++++++++++- src/output/fields.cpp | 3 ++ src/output/fields.h | 4 +- 6 files changed, 81 insertions(+), 6 deletions(-) diff --git a/src/framework/domain/output.cpp b/src/framework/domain/output.cpp index be154ce16..39c44192d 100644 --- a/src/framework/domain/output.cpp +++ b/src/framework/domain/output.cpp @@ -294,6 +294,14 @@ namespace ntt { {}, local_domain->fields.bckp, c); + } else if (fld.id() == FldsID::V) { + ComputeMoments(params, + local_domain->mesh, + local_domain->species, + fld.species, + fld.comp[0], + local_domain->fields.bckp, + c); } else { raise::Error("Wrong moment requested for output", HERE); } diff --git a/src/global/enums.h b/src/global/enums.h index 57822dec4..f7b1278c6 100644 --- a/src/global/enums.h +++ b/src/global/enums.h @@ -289,16 +289,17 @@ namespace ntt { N = 12, Nppc = 13, Custom = 14, + V = 15, }; constexpr FldsID(uint8_t c) : enums_hidden::BaseEnum { c } {} static constexpr type variants[] = { E, divE, D, divD, B, H, J, - A, T, Rho, Charge, N, Nppc, Custom }; + A, T, Rho, Charge, N, Nppc, Custom , V}; static constexpr const char* lookup[] = { "e", "dive", "d", "divd", "b", "h", "j", "a", "t", "rho", "charge", "n", - "nppc", "custom" }; + "nppc", "custom", "v" }; static constexpr std::size_t total = sizeof(variants) / sizeof(variants[0]); }; diff --git a/src/global/tests/enums.cpp b/src/global/tests/enums.cpp index 1fc57398f..8f814e9df 100644 --- a/src/global/tests/enums.cpp +++ b/src/global/tests/enums.cpp @@ -68,7 +68,7 @@ auto main() -> int { enum_str_t all_out_flds = { "e", "dive", "d", "divd", "b", "h", "j", "a", "t", "rho", - "charge", "n", "nppc", "custom" }; + "charge", "n", "nppc", "custom" , "v"}; checkEnum(all_coords); checkEnum(all_metrics); diff --git a/src/kernels/particle_moments.hpp b/src/kernels/particle_moments.hpp index 8b668a036..83caea563 100644 --- a/src/kernels/particle_moments.hpp +++ b/src/kernels/particle_moments.hpp @@ -41,7 +41,7 @@ namespace kernel { static constexpr auto D = M::Dim; static_assert((F == FldsID::Rho) || (F == FldsID::Charge) || - (F == FldsID::N) || (F == FldsID::Nppc) || (F == FldsID::T), + (F == FldsID::N) || (F == FldsID::Nppc) || (F == FldsID::T) || (F == FldsID::V), "Invalid field ID"); const unsigned short c1, c2; @@ -89,7 +89,7 @@ namespace kernel { std::size_t ni2, real_t inv_n0, unsigned short window) - : c1 { (components.size() == 2) ? components[0] + : c1 { (components.size() > 0) ? components[0] : static_cast(0) } , c2 { (components.size() == 2) ? components[1] : static_cast(0) } @@ -205,6 +205,67 @@ namespace kernel { coeff = contrib; } + if constexpr (F == FldsID::V) { + real_t gamma { ZERO }; + // for stress-energy tensor + vec_t u_Phys { ZERO }; + if constexpr (S == SimEngine::SRPIC) { + // SR + // stress-energy tensor for SR is computed in the tetrad (hatted) basis + if constexpr (M::CoordType == Coord::Cart) { + u_Phys[0] = ux1(p); + u_Phys[1] = ux2(p); + u_Phys[2] = ux3(p); + } else { + static_assert(D != Dim::_1D, "non-Cartesian SRPIC 1D"); + coord_t x_Code { ZERO }; + x_Code[0] = static_cast(i1(p)) + static_cast(dx1(p)); + x_Code[1] = static_cast(i2(p)) + static_cast(dx2(p)); + if constexpr (D == Dim::_3D) { + x_Code[2] = static_cast(i3(p)) + static_cast(dx3(p)); + } else { + x_Code[2] = phi(p); + } + metric.template transform_xyz( + x_Code, + { ux1(p), ux2(p), ux3(p) }, + u_Phys); + } + if (mass == ZERO) { + gamma = NORM(u_Phys[0], u_Phys[1], u_Phys[2]); + } else { + gamma = math::sqrt(ONE + NORM_SQR(u_Phys[0], u_Phys[1], u_Phys[2])); + } + } else { + // GR + // stress-energy tensor for GR is computed in contravariant basis + static_assert(D != Dim::_1D, "GRPIC 1D"); + coord_t x_Code { ZERO }; + x_Code[0] = static_cast(i1(p)) + static_cast(dx1(p)); + x_Code[1] = static_cast(i2(p)) + static_cast(dx2(p)); + if constexpr (D == Dim::_3D) { + x_Code[2] = static_cast(i3(p)) + static_cast(dx3(p)); + } + vec_t u_Cntrv { ZERO }; + // compute u_i u^i for energy + metric.template transform(x_Code, + { ux1(p), ux2(p), ux3(p) }, + u_Cntrv); + gamma = u_Cntrv[0] * ux1(p) + u_Cntrv[1] * ux2(p) + u_Cntrv[2] * ux3(p); + if (mass == ZERO) { + gamma = math::sqrt(gamma); + } else { + gamma = math::sqrt(ONE + gamma); + } + metric.template transform(x_Code, u_Cntrv, u_Phys); + } + // compute the corresponding moment + coeff = u_Phys[c1 - 1] / gamma; + } else { + // for other cases, use the `contrib` defined above + coeff = contrib; + } + if constexpr (F != FldsID::Nppc) { // for nppc calculation ... // ... do not take volume, weights or smoothing into account diff --git a/src/output/fields.cpp b/src/output/fields.cpp index aa5a752d4..0c2ea5e50 100644 --- a/src/output/fields.cpp +++ b/src/output/fields.cpp @@ -44,6 +44,9 @@ namespace out { } else if (id() == FldsID::T) { // energy-momentum tensor comp = InterpretComponents({ name.substr(1, 1), name.substr(2, 1) }); + } else if (id() == FldsID::V) { + // energy-momentum tensor + comp = InterpretComponents({ name.substr(1, 1) }); } else { // scalar (Rho, divE, Custom, etc.) comp = {}; diff --git a/src/output/fields.h b/src/output/fields.h index a520a246d..4fde18ed2 100644 --- a/src/output/fields.h +++ b/src/output/fields.h @@ -43,7 +43,7 @@ namespace out { [[nodiscard]] auto is_moment() const -> bool { return (id() == FldsID::T || id() == FldsID::Rho || id() == FldsID::Nppc || - id() == FldsID::N || id() == FldsID::Charge); + id() == FldsID::N || id() == FldsID::Charge || id() == FldsID::V); } [[nodiscard]] @@ -94,6 +94,8 @@ namespace out { tmp += m_name.substr(1, 2); } else if (id() == FldsID::A) { tmp += "3"; + } else if (id() == FldsID::V) { + tmp += m_name.substr(1, 1); } else if (is_field()) { tmp += "i"; } From bdd96be69727ba0b39ec02b4469a7dca56ed6f40 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Mon, 26 Aug 2024 11:46:57 -0400 Subject: [PATCH 002/183] Update input. --- input.example.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/input.example.toml b/input.example.toml index 88589495c..1c4e0466d 100644 --- a/input.example.toml +++ b/input.example.toml @@ -329,7 +329,7 @@ # Field quantities to output: # @type: array of strings # @valid: fields: "E", "B", "J", "divE" - # @valid: moments: "Rho", "Charge", "N", "Nppc", "T0i", "Tij" + # @valid: moments: "Rho", "Charge", "N", "Nppc", "T0i", "Tij", "Vi" # @valid: for GR: "D", "H", "divD", "A" # @default: [] # @note: For T, you can use unspecified indices, e.g., Tij, T0i, or specific ones, e.g., Ttt, T00, T02, T23 From a52d07d3aad40f53500eb052836d5016141fc8d1 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Wed, 16 Oct 2024 08:53:39 -0400 Subject: [PATCH 003/183] Bugfix in moment calculation. --- src/kernels/particle_moments.hpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/kernels/particle_moments.hpp b/src/kernels/particle_moments.hpp index 83caea563..904dc3b08 100644 --- a/src/kernels/particle_moments.hpp +++ b/src/kernels/particle_moments.hpp @@ -261,10 +261,7 @@ namespace kernel { } // compute the corresponding moment coeff = u_Phys[c1 - 1] / gamma; - } else { - // for other cases, use the `contrib` defined above - coeff = contrib; - } + } if constexpr (F != FldsID::Nppc) { // for nppc calculation ... From 3e1848aaba12da9b4c2844f0c2aba95d2102a47b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Feb 2025 10:43:05 -0600 Subject: [PATCH 004/183] prep for conductor boundaries --- src/global/arch/traits.h | 12 ++++++++++++ src/global/enums.h | 7 ++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/global/arch/traits.h b/src/global/arch/traits.h index 4cde4fca5..65cc63cf8 100644 --- a/src/global/arch/traits.h +++ b/src/global/arch/traits.h @@ -112,6 +112,18 @@ namespace traits { template using fix_fields_const_t = decltype(&T::FixFieldsConst); + template + using perfect_conductor_fields_t = decltype(&T::PerfectConductorFields); + + template + using perfect_conductor_fields_const_t = decltype(&T::PerfectConductorFieldsConst); + + template + using perfect_conductor_currents_t = decltype(&T::PerfectConductorCurrents); + + template + using perfect_conductor_currents_const_t = decltype(&T::PerfectConductorCurrentsConst); + template using custom_fields_t = decltype(&T::CustomFields); diff --git a/src/global/enums.h b/src/global/enums.h index 8f2495c13..2b3bf5936 100644 --- a/src/global/enums.h +++ b/src/global/enums.h @@ -222,15 +222,16 @@ namespace ntt { HORIZON = 6, AXIS = 7, SYNC = 8, // <- SYNC means synchronization with other domains + CONDUCTOR = 9 }; constexpr FldsBC(uint8_t c) : enums_hidden::BaseEnum { c } {} - static constexpr type variants[] = { PERIODIC, MATCH, FIXED, ATMOSPHERE, - CUSTOM, HORIZON, AXIS, SYNC }; + static constexpr type variants[] = { PERIODIC, MATCH, FIXED, ATMOSPHERE, + CUSTOM, HORIZON, AXIS, SYNC, CONDUCTOR }; static constexpr const char* lookup[] = { "periodic", "match", "fixed", "atmosphere", "custom", "horizon", - "axis", "sync" }; + "axis", "sync", "conductor"}; static constexpr std::size_t total = sizeof(variants) / sizeof(variants[0]); }; From bacbe24117dff5ebb832465aa9c539c9e0231900 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Feb 2025 10:43:32 -0600 Subject: [PATCH 005/183] first stubborn attempt at conductor boundaries (broken) --- setups/srpic/shock/pgen.hpp | 46 +++++++++++++ src/engines/srpic.hpp | 133 ++++++++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index b8f169521..59c5590c9 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -106,6 +106,52 @@ namespace user { } } + + auto PerfectConductorFieldsConst(const bc_in&, const em& comp) const -> real_t{ + + // electric field components + if (comp == em::ex1) { + return ONE; + } else if (comp == em::ex2) { + return -ONE; + } else if (comp == em::ex3) { + return -ONE; } + // magentic field components + else if (comp == em::bx1) { + return -ONE; + } else if (comp == em::bx2) { + return ONE; + } else if (comp == em::bx3) { + return ONE;} + // should never be the case + else + { + return ZERO; + } + } + + // auto PerfectConductorCurrentsConst(const bc_in &, const cur &comp) const + // -> std::pair + // { + // // ToDo + // if (comp == cur::jx1) + // { + // return ZERO; + // } + // else if (comp == cur::jx2) + // { + // return ZERO; + // } + // else if (comp == cur::jx3) + // { + // return ZERO; + // } + // else + // { + // return ZERO; + // } + // } + auto MatchFields(real_t time) const -> InitFields { return init_flds; } diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 9f5e4551f..4fca96674 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -596,6 +596,10 @@ namespace ntt { if (domain.mesh.flds_bc_in(direction) == FldsBC::FIXED) { FixedFieldsIn(direction, domain, tags); } + } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::CONDUCTOR) { + if (domain.mesh.flds_bc_in(direction) == FldsBC::CONDUCTOR) { + PerfectConductorFieldsIn(direction, domain, tags); + } } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::CUSTOM) { if (domain.mesh.flds_bc_in(direction) == FldsBC::CUSTOM) { CustomFieldsIn(direction, domain, tags); @@ -834,6 +838,135 @@ namespace ntt { } } + void PerfectConductorFieldsIn(dir::direction_t direction, + domain_t& domain, + BCTags tags) { + /** + * perfect conductor field boundaries + */ + const auto sign = direction.get_sign(); + const auto dim = direction.get_dim(); + raise::ErrorIf(dim != in::x1 and M::CoordType != Coord::Cart, + "Perfect conductor BCs only implemented for x1 in " + "non-cartesian coordinates", + HERE); + + // magnetic and electron field components + em normal_b_comp, tang_b_comp1, tang_b_comp2, + normal_e_comp, tang_e_comp1, tang_e_comp2; + + // current components + // cur normal_j_comp, tang_j_comp1, tang_j_comp2; + + if (dim == in::x1) { + normal_b_comp = em::bx1; + tang_b_comp1 = em::bx2; + tang_b_comp2 = em::bx3; + + normal_e_comp = em::ex1; + tang_e_comp1 = em::ex2; + tang_e_comp2 = em::ex3; + } else if (dim == in::x2) { + normal_b_comp = em::bx2; + tang_b_comp1 = em::bx1; + tang_b_comp2 = em::bx3; + + normal_e_comp = em::ex2; + tang_e_comp1 = em::ex1; + tang_e_comp2 = em::ex3; + } else if (dim == in::x3) { + normal_b_comp = em::bx3; + tang_b_comp1 = em::bx1; + tang_b_comp2 = em::bx2; + + normal_e_comp = em::ex3; + tang_e_comp1 = em::ex1; + tang_e_comp2 = em::ex2; + } else { + raise::Error("Invalid dimension", HERE); + } + + std::vector origin_xi_min, origin_xi_max, + target_xi_min, target_xi_max; + const std::vector all_dirs { in::x1, in::x2, in::x3 }; + + for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { + const auto dd = all_dirs[d]; + if (dim == dd) { + // origin: right side of boundary + origin_xi_min.push_back(N_GHOSTS+1); + origin_xi_max.push_back(2*N_GHOSTS); + // target: left side of boundary + target_xi_min.push_back(0); + target_xi_max.push_back(N_GHOSTS); + + } else { + origin_xi_min.push_back(0); + origin_xi_max.push_back(domain.mesh.n_all(dd)); + + target_xi_min.push_back(0); + target_xi_max.push_back(domain.mesh.n_all(dd)); + } + } + raise::ErrorIf(target_xi_min.size() != origin_xi_min.size() or + origin_xi_min.size() != static_cast(M::Dim), + "Invalid range size", + HERE); + + std::vector comps; + if (tags & BC::E) { + comps.push_back(normal_e_comp); + comps.push_back(tang_e_comp1); + comps.push_back(tang_e_comp2); + } + if (tags & BC::B) { + comps.push_back(normal_b_comp); + comps.push_back(tang_b_comp1); + comps.push_back(tang_b_comp2); + } + + // ToDo: smarter loop/views + auto EB = domain.fields.em; + + // loop over all components + for (const auto& comp : comps) { + + // store sign of component behind boundary + auto new_sign = m_pgen.PerfectConductorFieldsConst( + (bc_in)(sign * ((short)dim + 1)), + (em)comp); + // to do: Kokkos::parallel_for + for (int i = 0; i < N_GHOSTS; i++) + { + if constexpr (M::Dim == Dim::_1D) { + // multiply with correct sign + EB(target_xi_min[0]+i, comp) = new_sign * EB(origin_xi_max[0]-i, comp); + + } else if constexpr (M::Dim == Dim::_2D) { + for (int j = 0; j < domain.mesh.n_all(in::x2); j++) + { + EB(target_xi_min[0]+i, j, comp) = + new_sign * EB(origin_xi_max[0]-i, j, comp); + } + } else if constexpr (M::Dim == Dim::_3D) { + for (int j = 0; j < domain.mesh.n_all(in::x2); j++) + { + for (int k = 0; k < domain.mesh.n_all(in::x3); k++) + { + EB(target_xi_min[0]+i, j, k, comp) = + new_sign * EB(origin_xi_max[0]-i, j, k, comp); + } + } + } else { + raise::Error("Invalid dimension", HERE); + } + } + + // ToDo: set zero at boundary + } + } + + void AtmosphereFieldsIn(dir::direction_t direction, domain_t& domain, BCTags tags) { From 2081a39b3aea9b4c9fc5d9c122bd168879781aca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Feb 2025 16:41:20 -0600 Subject: [PATCH 006/183] first attempt at ConductorBoundaries_kernel --- src/kernels/fields_bcs.hpp | 107 +++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 363ff3ad2..fa0b7ec0d 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -485,6 +485,113 @@ namespace kernel::bc { } }; + template + struct ConductorBoundaries_kernel { + static_assert(M::is_metric, "M must be a metric class"); + static_assert(static_cast(o) < + static_cast(M::Dim), + "Invalid component index"); + static constexpr idx_t i = static_cast(o) + 1u; + + ndfield_t Fld; + const I fset; + const M metric; + const BCTags tags; + + ConductorBoundaries_kernel(ndfield_t Fld, + BCTags tags) + : Fld { Fld } + , tags { tags } {} + + Inline void operator()(index_t i1) const { + if constexpr (M::Dim == Dim::_1D) { + + if constexpr (S == SimEngine::SRPIC) { + + if (tags & BC::E) { + Fld((N_GHOSTS-1)-i1, em::ex1) = Fld(N_GHOSTS+i1, em::ex1); + Fld((N_GHOSTS-1)-i1, em::ex2) = -Fld(N_GHOSTS+i1, em::ex2); + Fld((N_GHOSTS-1)-i1, em::ex3) = -Fld(N_GHOSTS+i1, em::ex3); + } + + if (tags & BC::B) + { + Fld((N_GHOSTS-1)-i1, em::bx1) = -Fld(N_GHOSTS+i1, em::bx1); + Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); + Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); + } + + } else { + // GRPIC + raise::KernelError(HERE, "1D GRPIC not implemented"); + } + } else { + raise::KernelError( + HERE, + "ConductorBoundaries_kernel: 1D implementation called for D != 1"); + } + } + + Inline void operator()(index_t i1, index_t i2) const { + if constexpr (M::Dim == Dim::_2D) { + + if constexpr (S == SimEngine::SRPIC) { + // SRPIC + if (tags & BC::E) { + Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); + Fld((N_GHOSTS-1)-i1, i2, em::ex2) = -Fld(N_GHOSTS+i1, i2, em::ex2); + Fld((N_GHOSTS-1)-i1, i2, em::ex3) = -Fld(N_GHOSTS+i1, i2, em::ex3); + } + + if (tags & BC::B) + { + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = -Fld(N_GHOSTS+i1, i2, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); + } + } else { + // GRPIC + raise::KernelError(HERE, "2D GRPIC not implemented"); + } + } else { + raise::KernelError( + HERE, + "ConductorBoundaries_kernel: 2D implementation called for D != 2"); + } + } + + Inline void operator()(index_t i1, index_t i2, index_t i3) const { + if constexpr (M::Dim == Dim::_3D) { + const auto i1_ = COORD(i1); + const auto i2_ = COORD(i2); + const auto i3_ = COORD(i3); + + if constexpr (S == SimEngine::SRPIC) { + // SRPIC + if (tags & BC::E) { + Fld((N_GHOSTS-1)-i1, i2, i3, em::ex1) = Fld(N_GHOSTS+i1, i2, i3, em::ex1); + Fld((N_GHOSTS-1)-i1, i2, i3, em::ex2) = -Fld(N_GHOSTS+i1, i2, i3, em::ex2); + Fld((N_GHOSTS-1)-i1, i2, i3, em::ex3) = -Fld(N_GHOSTS+i1, i2, i3, em::ex3); + } + + if (tags & BC::B) + { + Fld((N_GHOSTS-1)-i1, i2, i3, em::bx1) = -Fld(N_GHOSTS+i1, i2, i3, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, i3, em::bx2) = Fld(N_GHOSTS+i1, i2, i3, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, i3, em::bx3) = Fld(N_GHOSTS+i1, i2, i3, em::bx3); + } + } else { + // GRPIC + raise::KernelError(HERE, "3D GRPIC not implemented"); + } + } else { + raise::KernelError( + HERE, + "ConductorBoundaries_kernel: 3D implementation called for D != 3"); + } + } + }; + /* * @tparam D: Dimension * @tparam P: Positive/Negative direction From b7a863c6de52fe5e32bdc0fa89010e07237f5be7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Feb 2025 16:55:21 -0600 Subject: [PATCH 007/183] bugfix --- src/kernels/fields_bcs.hpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index fa0b7ec0d..3a253820a 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -485,7 +485,7 @@ namespace kernel::bc { } }; - template + template struct ConductorBoundaries_kernel { static_assert(M::is_metric, "M must be a metric class"); static_assert(static_cast(o) < @@ -494,7 +494,6 @@ namespace kernel::bc { static constexpr idx_t i = static_cast(o) + 1u; ndfield_t Fld; - const I fset; const M metric; const BCTags tags; @@ -562,13 +561,10 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2, index_t i3) const { if constexpr (M::Dim == Dim::_3D) { - const auto i1_ = COORD(i1); - const auto i2_ = COORD(i2); - const auto i3_ = COORD(i3); if constexpr (S == SimEngine::SRPIC) { // SRPIC - if (tags & BC::E) { + if (tags & BC::E) { Fld((N_GHOSTS-1)-i1, i2, i3, em::ex1) = Fld(N_GHOSTS+i1, i2, i3, em::ex1); Fld((N_GHOSTS-1)-i1, i2, i3, em::ex2) = -Fld(N_GHOSTS+i1, i2, i3, em::ex2); Fld((N_GHOSTS-1)-i1, i2, i3, em::ex3) = -Fld(N_GHOSTS+i1, i2, i3, em::ex3); From f383e492633417d53f7ddd266b29fe07004f03d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Feb 2025 17:02:04 -0600 Subject: [PATCH 008/183] first attempt at Kokkos::parallel_for loop (broken) --- src/engines/srpic.hpp | 117 ++++++------------------------------------ 1 file changed, 17 insertions(+), 100 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 4fca96674..4198c9531 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -851,122 +851,39 @@ namespace ntt { "non-cartesian coordinates", HERE); - // magnetic and electron field components - em normal_b_comp, tang_b_comp1, tang_b_comp2, - normal_e_comp, tang_e_comp1, tang_e_comp2; - // current components - // cur normal_j_comp, tang_j_comp1, tang_j_comp2; - - if (dim == in::x1) { - normal_b_comp = em::bx1; - tang_b_comp1 = em::bx2; - tang_b_comp2 = em::bx3; - - normal_e_comp = em::ex1; - tang_e_comp1 = em::ex2; - tang_e_comp2 = em::ex3; - } else if (dim == in::x2) { - normal_b_comp = em::bx2; - tang_b_comp1 = em::bx1; - tang_b_comp2 = em::bx3; - - normal_e_comp = em::ex2; - tang_e_comp1 = em::ex1; - tang_e_comp2 = em::ex3; - } else if (dim == in::x3) { - normal_b_comp = em::bx3; - tang_b_comp1 = em::bx1; - tang_b_comp2 = em::bx2; - - normal_e_comp = em::ex3; - tang_e_comp1 = em::ex1; - tang_e_comp2 = em::ex2; - } else { - raise::Error("Invalid dimension", HERE); - } + std::vector xi_min, xi_max; - std::vector origin_xi_min, origin_xi_max, - target_xi_min, target_xi_max; const std::vector all_dirs { in::x1, in::x2, in::x3 }; for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { const auto dd = all_dirs[d]; if (dim == dd) { - // origin: right side of boundary - origin_xi_min.push_back(N_GHOSTS+1); - origin_xi_max.push_back(2*N_GHOSTS); - // target: left side of boundary - target_xi_min.push_back(0); - target_xi_max.push_back(N_GHOSTS); - + xi_min.push_back(0); + xi_max.push_back(N_GHOSTS); } else { - origin_xi_min.push_back(0); - origin_xi_max.push_back(domain.mesh.n_all(dd)); - - target_xi_min.push_back(0); - target_xi_max.push_back(domain.mesh.n_all(dd)); + xi_min.push_back(0); + xi_max.push_back(domain.mesh.n_all(dd)); } } - raise::ErrorIf(target_xi_min.size() != origin_xi_min.size() or - origin_xi_min.size() != static_cast(M::Dim), + raise::ErrorIf(xi_min.size() != xi_max.size() or + xi_min.size() != static_cast(M::Dim), "Invalid range size", HERE); - std::vector comps; - if (tags & BC::E) { - comps.push_back(normal_e_comp); - comps.push_back(tang_e_comp1); - comps.push_back(tang_e_comp2); - } - if (tags & BC::B) { - comps.push_back(normal_b_comp); - comps.push_back(tang_b_comp1); - comps.push_back(tang_b_comp2); - } - - // ToDo: smarter loop/views - auto EB = domain.fields.em; - - // loop over all components - for (const auto& comp : comps) { - - // store sign of component behind boundary - auto new_sign = m_pgen.PerfectConductorFieldsConst( - (bc_in)(sign * ((short)dim + 1)), - (em)comp); - // to do: Kokkos::parallel_for - for (int i = 0; i < N_GHOSTS; i++) - { - if constexpr (M::Dim == Dim::_1D) { - // multiply with correct sign - EB(target_xi_min[0]+i, comp) = new_sign * EB(origin_xi_max[0]-i, comp); - - } else if constexpr (M::Dim == Dim::_2D) { - for (int j = 0; j < domain.mesh.n_all(in::x2); j++) - { - EB(target_xi_min[0]+i, j, comp) = - new_sign * EB(origin_xi_max[0]-i, j, comp); - } - } else if constexpr (M::Dim == Dim::_3D) { - for (int j = 0; j < domain.mesh.n_all(in::x2); j++) - { - for (int k = 0; k < domain.mesh.n_all(in::x3); k++) - { - EB(target_xi_min[0]+i, j, k, comp) = - new_sign * EB(origin_xi_max[0]-i, j, k, comp); - } - } - } else { - raise::Error("Invalid dimension", HERE); - } - } - - // ToDo: set zero at boundary + if (dim == in::x1) + { + Kokkos::parallel_for( + "MatchFields", + CreateRangePolicy(xi_min, xi_max), + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); } + + } - void AtmosphereFieldsIn(dir::direction_t direction, domain_t& domain, BCTags tags) { From 7666918b4117ab726422a6440fcf0135aa39cb05 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 21 Feb 2025 22:12:56 -0500 Subject: [PATCH 009/183] Ongoing. --- src/engines/srpic.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 4198c9531..fadb55087 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -871,11 +871,11 @@ namespace ntt { "Invalid range size", HERE); - if (dim == in::x1) + if constexpr (M::Dim == Dim::_1D) { { Kokkos::parallel_for( "MatchFields", - CreateRangePolicy(xi_min, xi_max), + CreateRangePolicy(xi_min[0], xi_max[0]), kernel::bc::ConductorBoundaries_kernel( domain.fields.em, tags)); From af065442ded1c0fa3ada6ab01398ec522cde1e16 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 21 Feb 2025 22:16:54 -0500 Subject: [PATCH 010/183] Ongoing. --- src/engines/srpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index fadb55087..da111256a 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -871,7 +871,7 @@ namespace ntt { "Invalid range size", HERE); - if constexpr (M::Dim == Dim::_1D) { + if constexpr (M::Dim == Dim::_1D) { Kokkos::parallel_for( "MatchFields", From 0d3cc1cf90ce489d23f918c1b657b98d8b44e0fe Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 21 Feb 2025 22:19:01 -0500 Subject: [PATCH 011/183] Ongoing. --- src/engines/srpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index da111256a..7f38d567a 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -875,7 +875,7 @@ namespace ntt { { Kokkos::parallel_for( "MatchFields", - CreateRangePolicy(xi_min[0], xi_max[0]), + CreateRangePolicy( { xi_min[0] } , { xi_max[0] } ), kernel::bc::ConductorBoundaries_kernel( domain.fields.em, tags)); From 4a57c38538916967860bdbbbfe34e452ed785942 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 21 Feb 2025 22:35:06 -0500 Subject: [PATCH 012/183] Ongoing. --- src/engines/srpic.hpp | 2 +- src/kernels/fields_bcs.hpp | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 7f38d567a..6e4f51945 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -876,7 +876,7 @@ namespace ntt { Kokkos::parallel_for( "MatchFields", CreateRangePolicy( { xi_min[0] } , { xi_max[0] } ), - kernel::bc::ConductorBoundaries_kernel( + kernel::bc::ConductorBoundaries_kernel( domain.fields.em, tags)); } diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 3a253820a..48f2aa7d3 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -494,7 +494,6 @@ namespace kernel::bc { static constexpr idx_t i = static_cast(o) + 1u; ndfield_t Fld; - const M metric; const BCTags tags; ConductorBoundaries_kernel(ndfield_t Fld, From ff5a3ee8dbb2ca2bc60279f8885918497a1acb5a Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 21 Feb 2025 22:35:54 -0500 Subject: [PATCH 013/183] Ongoing. --- src/engines/srpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 6e4f51945..7f38d567a 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -876,7 +876,7 @@ namespace ntt { Kokkos::parallel_for( "MatchFields", CreateRangePolicy( { xi_min[0] } , { xi_max[0] } ), - kernel::bc::ConductorBoundaries_kernel( + kernel::bc::ConductorBoundaries_kernel( domain.fields.em, tags)); } From d6b2b1c266a127763a60ee6706f59f64214f7795 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 11:16:58 -0500 Subject: [PATCH 014/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 179 ++++++++++++++++++++++++++++++ setups/srpic/shocktest/shock.py | 75 +++++++++++++ setups/srpic/shocktest/shock.toml | 56 ++++++++++ 3 files changed, 310 insertions(+) create mode 100644 setups/srpic/shocktest/pgen.hpp create mode 100644 setups/srpic/shocktest/shock.py create mode 100644 setups/srpic/shocktest/shock.toml diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp new file mode 100644 index 000000000..83f19df52 --- /dev/null +++ b/setups/srpic/shocktest/pgen.hpp @@ -0,0 +1,179 @@ +#ifndef PROBLEM_GENERATOR_H +#define PROBLEM_GENERATOR_H + +#include "enums.h" +#include "global.h" + +#include "arch/traits.h" +#include "utils/error.h" +#include "utils/numeric.h" + +#include "archetypes/energy_dist.h" +#include "archetypes/particle_injector.h" +#include "archetypes/problem_generator.h" +#include "framework/domain/metadomain.h" + +#include + +namespace user { + using namespace ntt; + + template + struct InitFields { + /* + Sets up magnetic and electric field components for the simulation. + Must satisfy E = -v x B for Lorentz Force to be zero. + + @param bmag: magnetic field scaling + @param btheta: magnetic field polar angle + @param bphi: magnetic field azimuthal angle + @param drift_ux: drift velocity in the x direction + */ + InitFields(real_t bmag, real_t btheta, real_t bphi, real_t drift_ux) + : Bmag { bmag } + , Btheta { btheta * static_cast(convert::deg2rad) } + , Bphi { bphi * static_cast(convert::deg2rad) } + , Vx { drift_ux } {} + + // magnetic field components + Inline auto bx1(const coord_t& x_Ph) const -> real_t { + return x_Ph[0]; + } + + Inline auto bx2(const coord_t&) const -> real_t { + return x_Ph[0]; + } + + Inline auto bx3(const coord_t&) const -> real_t { + return x_Ph[0]; + } + + // electric field components + Inline auto ex1(const coord_t&) const -> real_t { + return x_Ph[0]; + } + + Inline auto ex2(const coord_t&) const -> real_t { + return x_Ph[0]; + } + + Inline auto ex3(const coord_t&) const -> real_t { + return x_Ph[0]; + } + + private: + const real_t Btheta, Bphi, Vx, Bmag; + }; + + template + struct PGen : public arch::ProblemGenerator { + // compatibility traits for the problem generator + static constexpr auto engines { traits::compatible_with::value }; + static constexpr auto metrics { traits::compatible_with::value }; + static constexpr auto dimensions { + traits::compatible_with::value + }; + + // for easy access to variables in the child class + using arch::ProblemGenerator::D; + using arch::ProblemGenerator::C; + using arch::ProblemGenerator::params; + + const real_t drift_ux, temperature; + + const real_t Btheta, Bphi, Bmag; + InitFields init_flds; + + inline PGen(const SimulationParams& p, const Metadomain& m) + : arch::ProblemGenerator { p } + , drift_ux { p.template get("setup.drift_ux") } + , temperature { p.template get("setup.temperature") } + , Bmag { p.template get("setup.Bmag", ZERO) } + , Btheta { p.template get("setup.Btheta", ZERO) } + , Bphi { p.template get("setup.Bphi", ZERO) } + , init_flds { Bmag, Btheta, Bphi, drift_ux } {} + + inline PGen() {} + + auto FixFieldsConst(const bc_in&, const em& comp) const + -> std::pair { + if (comp == em::ex2) { + return { init_flds.ex2({ ZERO }), true }; + } else if (comp == em::ex3) { + return { init_flds.ex3({ ZERO }), true }; + } else { + return { ZERO, false }; + } + } + + + auto PerfectConductorFieldsConst(const bc_in&, const em& comp) const -> real_t{ + + // electric field components + if (comp == em::ex1) { + return ONE; + } else if (comp == em::ex2) { + return -ONE; + } else if (comp == em::ex3) { + return -ONE; } + // magentic field components + else if (comp == em::bx1) { + return -ONE; + } else if (comp == em::bx2) { + return ONE; + } else if (comp == em::bx3) { + return ONE;} + // should never be the case + else + { + return ZERO; + } + } + + // auto PerfectConductorCurrentsConst(const bc_in &, const cur &comp) const + // -> std::pair + // { + // // ToDo + // if (comp == cur::jx1) + // { + // return ZERO; + // } + // else if (comp == cur::jx2) + // { + // return ZERO; + // } + // else if (comp == cur::jx3) + // { + // return ZERO; + // } + // else + // { + // return ZERO; + // } + // } + + auto MatchFields(real_t time) const -> InitFields { + return init_flds; + } + + inline void InitPrtls(Domain& local_domain) { + const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, + local_domain.random_pool, + temperature, + -drift_ux, + in::x1); + + const auto injector = arch::UniformInjector( + energy_dist, + { 1, 2 }); + // arch::InjectUniform>( + // params, + // local_domain, + // injector, + // 1.0); + } + }; + +} // namespace user + +#endif diff --git a/setups/srpic/shocktest/shock.py b/setups/srpic/shocktest/shock.py new file mode 100644 index 000000000..dc1565572 --- /dev/null +++ b/setups/srpic/shocktest/shock.py @@ -0,0 +1,75 @@ +import nt2.read as nt2r +import matplotlib.pyplot as plt +import matplotlib as mpl + +data = nt2r.Data("shock.h5") + + +def frame(ti, f): + quantities = [ + { + "name": "density", + "compute": lambda f: f.N_2 + f.N_1, + "cmap": "inferno", + "norm": mpl.colors.Normalize(0, 5), + }, + { + "name": r"$E_x$", + "compute": lambda f: f.Ex, + "cmap": "RdBu_r", + "norm": mpl.colors.Normalize(-0.05, 0.05), + }, + { + "name": r"$E_y$", + "compute": lambda f: f.Ey, + "cmap": "RdBu_r", + "norm": mpl.colors.Normalize(-0.05, 0.05), + }, + { + "name": r"$E_z$", + "compute": lambda f: f.Ez, + "cmap": "RdBu_r", + "norm": mpl.colors.Normalize(-0.05, 0.05), + }, + { + "name": r"$B_x$", + "compute": lambda f: f.Bx, + "cmap": "BrBG", + "norm": mpl.colors.Normalize(-0.05, 0.05), + }, + { + "name": r"$B_y$", + "compute": lambda f: f.By, + "cmap": "BrBG", + "norm": mpl.colors.Normalize(-0.05, 0.05), + }, + { + "name": r"$B_z$", + "compute": lambda f: f.Bz, + "cmap": "BrBG", + "norm": mpl.colors.Normalize(-0.05, 0.05), + }, + ] + fig = plt.figure(figsize=(12, 5.5), dpi=300) + gs = fig.add_gridspec(len(quantities), 1, hspace=0.02) + axs = [fig.add_subplot(gs[i]) for i in range(len(quantities))] + + for ax, q in zip(axs, quantities): + q["compute"](f.isel(t=ti)).plot( + ax=ax, + cmap=q["cmap"], + norm=q["norm"], + cbar_kwargs={"label": q["name"], "shrink": 0.8, "aspect": 10, "pad": 0.005}, + ) + for i, ax in enumerate(axs): + ax.set(aspect=1) + if i != 0: + ax.set(title=None) + if i != len(axs) - 1: + ax.set( + xticks=[], + xticklabels=[], + xlabel=None, + title=ax.get_title().split(",")[0], + ) + return fig diff --git a/setups/srpic/shocktest/shock.toml b/setups/srpic/shocktest/shock.toml new file mode 100644 index 000000000..fdbc44465 --- /dev/null +++ b/setups/srpic/shocktest/shock.toml @@ -0,0 +1,56 @@ +[simulation] + name = "shock" + engine = "srpic" + runtime = 50.0 + +[grid] + resolution = [2048, 128] + extent = [[0.0, 10.0], [-0.3125, 0.3125]] + + [grid.metric] + metric = "minkowski" + + [grid.boundaries] + fields = [["CONDUCTOR", "FIXED"], ["PERIODIC"]] + particles = [["REFLECT", "ABSORB"], ["PERIODIC"]] + +[scales] + larmor0 = 1e-2 + skindepth0 = 1e-2 + +[algorithms] + current_filters = 8 + fieldsolver = "false" + deposit = "false" + + [algorithms.timestep] + CFL = 0.5 + +[particles] + ppc0 = 16.0 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = -1.0 + maxnpart = 1e8 + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = 1.0 + maxnpart = 1e8 + +[setup] + drift_ux = 0.1 + temperature = 1e-3 + Bmag = 1.0 + Btheta = 0.0 + Bphi = 0.0 + +[output] + interval = 1 + format = "hdf5" + + [output.fields] + quantities = ["N_1", "N_2", "E", "B", "T0i_1", "T0i_2", "J"] From f0ed6c7ba59988ecb52de96760643a3d34d513bf Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 11:20:28 -0500 Subject: [PATCH 015/183] Ongoing. --- setups/srpic/shocktest/shock.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setups/srpic/shocktest/shock.toml b/setups/srpic/shocktest/shock.toml index fdbc44465..a77f5c2e9 100644 --- a/setups/srpic/shocktest/shock.toml +++ b/setups/srpic/shocktest/shock.toml @@ -54,3 +54,6 @@ [output.fields] quantities = ["N_1", "N_2", "E", "B", "T0i_1", "T0i_2", "J"] + + [output.debug] + ghosts = true From 79d323741ef5eeeaf939ee502a56629842baeac1 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 14:01:06 -0500 Subject: [PATCH 016/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 83f19df52..7724893a1 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -40,24 +40,24 @@ namespace user { return x_Ph[0]; } - Inline auto bx2(const coord_t&) const -> real_t { + Inline auto bx2(const coord_t& x_Ph) const -> real_t { return x_Ph[0]; } - Inline auto bx3(const coord_t&) const -> real_t { + Inline auto bx3(const coord_t& x_Ph) const -> real_t { return x_Ph[0]; } // electric field components - Inline auto ex1(const coord_t&) const -> real_t { + Inline auto ex1(const coord_t& x_Ph) const -> real_t { return x_Ph[0]; } - Inline auto ex2(const coord_t&) const -> real_t { + Inline auto ex2(const coord_t& x_Ph) const -> real_t { return x_Ph[0]; } - Inline auto ex3(const coord_t&) const -> real_t { + Inline auto ex3(const coord_t& x_Ph) const -> real_t { return x_Ph[0]; } From 17b469bfa3137317145a50853271773e3ec0c8b2 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 17:15:38 -0500 Subject: [PATCH 017/183] Ongoing. --- src/kernels/fields_bcs.hpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 48f2aa7d3..8eda1b34a 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,9 +543,12 @@ namespace kernel::bc { if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = -Fld(N_GHOSTS+i1, i2, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); + // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = -Fld(N_GHOSTS+i1, i2, em::bx1); + // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); + // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; } } else { // GRPIC From e234173bd8688c8585f4ef6d51c1cc721ff49e38 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 17:21:07 -0500 Subject: [PATCH 018/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 7724893a1..0dc91d3e2 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -41,7 +41,7 @@ namespace user { } Inline auto bx2(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return -x_Ph[0]; } Inline auto bx3(const coord_t& x_Ph) const -> real_t { From de510ea1abdc4e7381f5705e38fbe33510e3bc9a Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 18:50:58 -0500 Subject: [PATCH 019/183] Ongoing. --- src/engines/srpic.hpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 7f38d567a..90aa271a3 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -880,6 +880,16 @@ namespace ntt { domain.fields.em, tags)); } + + if constexpr (M::Dim == Dim::_2D) + { + Kokkos::parallel_for( + "MatchFields", + CreateRangePolicy( { xi_min[0], xi_min[1] } , { xi_max[0], xi_max[1] } ), + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } } From 0524d782381c66c6fcf291027e85009ccdfa8cd6 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 19:05:19 -0500 Subject: [PATCH 020/183] Ongoing. --- src/kernels/fields_bcs.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 8eda1b34a..490042304 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,12 +543,12 @@ namespace kernel::bc { if (tags & BC::B) { - // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = -Fld(N_GHOSTS+i1, i2, em::bx1); - // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); - // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = Fld(N_GHOSTS+i1, i2, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); + // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; + // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; + // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; } } else { // GRPIC From 9d78b9e2a43740cc1a1fb7d2faffc6871c8ba0b1 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 19:10:18 -0500 Subject: [PATCH 021/183] Ongoing. --- src/kernels/fields_bcs.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 490042304..33f2813ca 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,9 +543,9 @@ namespace kernel::bc { if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = Fld(N_GHOSTS+i1, i2, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; From cec22b04960cad3889473c956a17b4e270ef83c0 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 20:01:27 -0500 Subject: [PATCH 022/183] Ongoing. --- src/kernels/fields_bcs.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 33f2813ca..aebe35794 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,9 +543,9 @@ namespace kernel::bc { if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); + Fld((N_GHOSTS)-i1, i2, em::bx1) = - Fld(N_GHOSTS+1+i1, i2, em::bx1); + Fld((N_GHOSTS)-i1, i2, em::bx2) = - Fld(N_GHOSTS+1+i1, i2, em::bx2); + Fld((N_GHOSTS)-i1, i2, em::bx3) = - Fld(N_GHOSTS+1+i1, i2, em::bx3); // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; From b3754e29a4bf20ea04a11e494866dd7c5323d59c Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 20:54:37 -0500 Subject: [PATCH 023/183] Ongoing. --- src/kernels/fields_bcs.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index aebe35794..33f2813ca 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,9 +543,9 @@ namespace kernel::bc { if (tags & BC::B) { - Fld((N_GHOSTS)-i1, i2, em::bx1) = - Fld(N_GHOSTS+1+i1, i2, em::bx1); - Fld((N_GHOSTS)-i1, i2, em::bx2) = - Fld(N_GHOSTS+1+i1, i2, em::bx2); - Fld((N_GHOSTS)-i1, i2, em::bx3) = - Fld(N_GHOSTS+1+i1, i2, em::bx3); + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; From 83dba373e226994a77d5845b574b12c43e69e947 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 21:02:54 -0500 Subject: [PATCH 024/183] Ongoing. --- src/kernels/fields_bcs.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 33f2813ca..3e6458877 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,12 +543,12 @@ namespace kernel::bc { if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); - // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; - // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; - // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; + // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); + // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); + // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; } } else { // GRPIC From a7ef051440b5695e3c30bc88dc63bba7865abf2c Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 21:13:50 -0500 Subject: [PATCH 025/183] Ongoing. --- src/kernels/fields_bcs.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 3e6458877..082d06329 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,12 +543,12 @@ namespace kernel::bc { if (tags & BC::B) { - // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); - // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); - // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+1+i1, i2, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); + // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; + // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; + // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; } } else { // GRPIC From d219fc43854fcaafc099d3889587403f1b2d1a38 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 21:43:02 -0500 Subject: [PATCH 026/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 0dc91d3e2..685521f09 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -45,7 +45,7 @@ namespace user { } Inline auto bx3(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return ONE; } // electric field components From 044ccea51284764cc110069d64d6702f2c028913 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 22:06:25 -0500 Subject: [PATCH 027/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 685521f09..0dc91d3e2 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -45,7 +45,7 @@ namespace user { } Inline auto bx3(const coord_t& x_Ph) const -> real_t { - return ONE; + return x_Ph[0]; } // electric field components From 2af41d83e01527f3bdbb2548e95ab7d098648b25 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 22:16:17 -0500 Subject: [PATCH 028/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 0dc91d3e2..f8bf67dab 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -37,7 +37,7 @@ namespace user { // magnetic field components Inline auto bx1(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return ZERO; } Inline auto bx2(const coord_t& x_Ph) const -> real_t { @@ -50,15 +50,15 @@ namespace user { // electric field components Inline auto ex1(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return ZERO; } Inline auto ex2(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return ZERO; } Inline auto ex3(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return ZERO; } private: From 056c6294df2c23d568805e402eb28a1ae9952fb7 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 22:34:56 -0500 Subject: [PATCH 029/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index f8bf67dab..39ce8ea03 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -45,7 +45,7 @@ namespace user { } Inline auto bx3(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return -x_Ph[0]; } // electric field components From 7d9a1a39d75d239bb8ff387bc47cee4c5305f333 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 22:58:19 -0500 Subject: [PATCH 030/183] Ongoing. --- src/kernels/fields_bcs.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 082d06329..fe5ae4122 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -536,9 +536,9 @@ namespace kernel::bc { if constexpr (S == SimEngine::SRPIC) { // SRPIC if (tags & BC::E) { - Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); - Fld((N_GHOSTS-1)-i1, i2, em::ex2) = -Fld(N_GHOSTS+i1, i2, em::ex2); - Fld((N_GHOSTS-1)-i1, i2, em::ex3) = -Fld(N_GHOSTS+i1, i2, em::ex3); + Fld((N_GHOSTS-1)-i1, i2, em::ex1) = - Fld(N_GHOSTS+i1, i2, em::ex1); + Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+1+i1, i2, em::ex2); + Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+1+i1, i2, em::ex3); } if (tags & BC::B) From d8abf1b2688f61636fb604b017e5c7a85073c0df Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Tue, 25 Feb 2025 23:51:36 -0500 Subject: [PATCH 031/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 39ce8ea03..2cb66539f 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -36,16 +36,16 @@ namespace user { , Vx { drift_ux } {} // magnetic field components - Inline auto bx1(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto bx1(const coord_t&) const -> real_t { + return Bmag * math::cos(Btheta); } - Inline auto bx2(const coord_t& x_Ph) const -> real_t { - return -x_Ph[0]; + Inline auto bx2(const coord_t&) const -> real_t { + return Bmag * math::sin(Btheta) * math::sin(Bphi); } - Inline auto bx3(const coord_t& x_Ph) const -> real_t { - return -x_Ph[0]; + Inline auto bx3(const coord_t&) const -> real_t { + return Bmag * math::sin(Btheta) * math::cos(Bphi); } // electric field components From a17db6280545241cd28c85936d772314369606e6 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Tue, 25 Feb 2025 23:59:44 -0500 Subject: [PATCH 032/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 2cb66539f..b77583ccb 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -49,16 +49,16 @@ namespace user { } // electric field components - Inline auto ex1(const coord_t& x_Ph) const -> real_t { + Inline auto ex1(const coord_t&) const -> real_t { return ZERO; } - Inline auto ex2(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto ex2(const coord_t&) const -> real_t { + return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); } - Inline auto ex3(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto ex3(const coord_t&) const -> real_t { + return Vx * Bmag * math::sin(Btheta) * math::sin(Bphi); } private: From 1d109a062753f09885239f289c4c0cf2280320a7 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Thu, 27 Feb 2025 11:42:46 -0500 Subject: [PATCH 033/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index b77583ccb..f92086528 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -36,29 +36,34 @@ namespace user { , Vx { drift_ux } {} // magnetic field components - Inline auto bx1(const coord_t&) const -> real_t { - return Bmag * math::cos(Btheta); + Inline auto bx1(const coord_t& x_ph) const -> real_t { + // return Bmag * math::cos(Btheta); + return ZERO; } - Inline auto bx2(const coord_t&) const -> real_t { - return Bmag * math::sin(Btheta) * math::sin(Bphi); + Inline auto bx2(const coord_t& x_ph) const -> real_t { + // return Bmag * math::sin(Btheta) * math::sin(Bphi); + return ZERO; } - Inline auto bx3(const coord_t&) const -> real_t { - return Bmag * math::sin(Btheta) * math::cos(Bphi); + Inline auto bx3(const coord_t& x_ph) const -> real_t { + // return Bmag * math::sin(Btheta) * math::cos(Bphi); + return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x))*math::tanh(20.*(-0.5 + x)))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); } // electric field components - Inline auto ex1(const coord_t&) const -> real_t { + Inline auto ex1(const coord_t& x_ph) const -> real_t { return ZERO; } - Inline auto ex2(const coord_t&) const -> real_t { - return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); + Inline auto ex2(const coord_t& x_ph) const -> real_t { + // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); + return ZERO + 0.01 * (ONE - math::tanh(20.*(-1.5 + x))*math::tanh(20.*(-0.5 + x)))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); } - Inline auto ex3(const coord_t&) const -> real_t { - return Vx * Bmag * math::sin(Btheta) * math::sin(Bphi); + Inline auto ex3(const coord_t& x_ph) const -> real_t { + // return Vx * Bmag * math::sin(Btheta) * math::sin(Bphi); + return ZERO; } private: From bee7530bf59a8386b14799114a393a015105f7f7 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Thu, 27 Feb 2025 12:01:42 -0500 Subject: [PATCH 034/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index f92086528..c43688d02 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -48,7 +48,7 @@ namespace user { Inline auto bx3(const coord_t& x_ph) const -> real_t { // return Bmag * math::sin(Btheta) * math::cos(Bphi); - return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x))*math::tanh(20.*(-0.5 + x)))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); } // electric field components @@ -58,7 +58,7 @@ namespace user { Inline auto ex2(const coord_t& x_ph) const -> real_t { // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - return ZERO + 0.01 * (ONE - math::tanh(20.*(-1.5 + x))*math::tanh(20.*(-0.5 + x)))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ZERO + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); } Inline auto ex3(const coord_t& x_ph) const -> real_t { From 2a269fde2d70d3e2b05b7627b30eaa4484ac83f6 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Thu, 27 Feb 2025 13:13:12 -0500 Subject: [PATCH 035/183] Ongoing. --- src/kernels/fields_bcs.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index fe5ae4122..f201d7cc8 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -536,7 +536,7 @@ namespace kernel::bc { if constexpr (S == SimEngine::SRPIC) { // SRPIC if (tags & BC::E) { - Fld((N_GHOSTS-1)-i1, i2, em::ex1) = - Fld(N_GHOSTS+i1, i2, em::ex1); + Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+1+i1, i2, em::ex2); Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+1+i1, i2, em::ex3); } @@ -544,8 +544,8 @@ namespace kernel::bc { if (tags & BC::B) { Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+1+i1, i2, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; From 49820ff2a0d67ea68b7b3cbee687e1531df96641 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Thu, 27 Feb 2025 13:18:43 -0500 Subject: [PATCH 036/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index c43688d02..10d4c6b59 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -58,7 +58,7 @@ namespace user { Inline auto ex2(const coord_t& x_ph) const -> real_t { // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - return ZERO + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); } Inline auto ex3(const coord_t& x_ph) const -> real_t { From d2167da1bc2c1c0a88ba94f2dfb4b80ca363447f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Thu, 27 Feb 2025 18:04:22 -0600 Subject: [PATCH 037/183] first attempt at single particle injection --- setups/srpic/shocktest/pgen.hpp | 164 ++++++++++++++++++++------------ src/kernels/fields_bcs.hpp | 27 +++--- 2 files changed, 116 insertions(+), 75 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 10d4c6b59..4d4b1cb37 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -111,72 +111,116 @@ namespace user { } } - - auto PerfectConductorFieldsConst(const bc_in&, const em& comp) const -> real_t{ - - // electric field components - if (comp == em::ex1) { - return ONE; - } else if (comp == em::ex2) { - return -ONE; - } else if (comp == em::ex3) { - return -ONE; } - // magentic field components - else if (comp == em::bx1) { - return -ONE; - } else if (comp == em::bx2) { - return ONE; - } else if (comp == em::bx3) { - return ONE;} - // should never be the case - else - { - return ZERO; - } - } - - // auto PerfectConductorCurrentsConst(const bc_in &, const cur &comp) const - // -> std::pair - // { - // // ToDo - // if (comp == cur::jx1) - // { - // return ZERO; - // } - // else if (comp == cur::jx2) - // { - // return ZERO; - // } - // else if (comp == cur::jx3) - // { - // return ZERO; - // } - // else - // { - // return ZERO; - // } - // } auto MatchFields(real_t time) const -> InitFields { return init_flds; } - inline void InitPrtls(Domain& local_domain) { - const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, - local_domain.random_pool, - temperature, - -drift_ux, - in::x1); - - const auto injector = arch::UniformInjector( - energy_dist, - { 1, 2 }); - // arch::InjectUniform>( - // params, - // local_domain, - // injector, - // 1.0); + inline void InitPrtls(Domain& domain) { + + auto& species_e = domain.species[0]; + auto& species_p = domain.species[1]; + auto metric = domain.mesh.metric; + auto m = domain.mesh.metric; + + array_t elec_ind("elec_ind"); + array_t pos_ind("pos_ind"); + + auto offset_e = species_e.npart(); + auto offset_p = species_p.npart(); + + auto ux1_e = species_e.ux1; + auto ux2_e = species_e.ux2; + auto ux3_e = species_e.ux3; + auto i1_e = species_e.i1; + auto i2_e = species_e.i2; + auto dx1_e = species_e.dx1; + auto dx2_e = species_e.dx2; + auto phi_e = species_e.phi; + auto weight_e = species_e.weight; + auto tag_e = species_e.tag; + + auto ux1_p = species_p.ux1; + auto ux2_p = species_p.ux2; + auto ux3_p = species_p.ux3; + auto i1_p = species_p.i1; + auto i2_p = species_p.i2; + auto dx1_p = species_p.dx1; + auto dx2_p = species_p.dx2; + auto phi_p = species_p.phi; + auto weight_p = species_p.weight; + auto tag_p = species_p.tag; + + int nseed = 1; + auto dseed = HALF * constant::PI / static_cast(nseed); + + Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { + + // ToDo: fix this + auto i1_ = ONE; + auto i2_ = ONE; + auto dx1_ = ONE - HALF; + auto dx2_ = ONE - HALF; + + + auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); + auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); + + i1_e(elec_p + offset_e) = i1_; + dx1_e(elec_p + offset_e) = dx1_; + i2_e(elec_p + offset_e) = i2_; + dx2_e(elec_p + offset_e) = dx2_; + phi_e(elec_p + offset_e) = ZERO; + ux1_e(elec_p + offset_e) = -drift_ux; + ux2_e(elec_p + offset_e) = -drift_ux; + ux3_e(elec_p + offset_e) = ZERO; + weight_e(elec_p + offset_e) = ONE; + tag_e(elec_p + offset_e) = ParticleTag::alive; + + i1_p(pos_p + offset_p) = i1_; + dx1_p(pos_p + offset_p) = dx1_; + i2_p(pos_p + offset_p) = i2_; + dx2_p(pos_p + offset_p) = dx2_; + phi_p(pos_p + offset_p) = ZERO; + ux1_p(pos_p + offset_p) = -drift_ux; + ux2_p(pos_p + offset_p) = drift_ux; + ux3_p(pos_p + offset_p) = ZERO; + weight_p(pos_p + offset_p) = ONE; + tag_p(pos_p + offset_p) = ParticleTag::alive; + + + }); + + + auto elec_ind_h = Kokkos::create_mirror(elec_ind); + Kokkos::deep_copy(elec_ind_h, elec_ind); + species_e.set_npart(offset_e + elec_ind_h()); + + auto pos_ind_h = Kokkos::create_mirror(pos_ind); + Kokkos::deep_copy(pos_ind_h, pos_ind); + species_p.set_npart(offset_p + pos_ind_h()); + + } + + + // inline void InitPrtls(Domain& local_domain) { + // const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, + // local_domain.random_pool, + // temperature, + // -drift_ux, + // in::x1); + + // const auto injector = arch::UniformInjector( + // energy_dist, + // { 1, 2 }); + // arch::InjectUniform>( + // params, + // local_domain, + // injector, + // 1.0); + // } + }; } // namespace user diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index f201d7cc8..39dfaf5dc 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -506,19 +506,19 @@ namespace kernel::bc { if constexpr (S == SimEngine::SRPIC) { - if (tags & BC::E) { - Fld((N_GHOSTS-1)-i1, em::ex1) = Fld(N_GHOSTS+i1, em::ex1); - Fld((N_GHOSTS-1)-i1, em::ex2) = -Fld(N_GHOSTS+i1, em::ex2); - Fld((N_GHOSTS-1)-i1, em::ex3) = -Fld(N_GHOSTS+i1, em::ex3); - } - - if (tags & BC::B) - { - Fld((N_GHOSTS-1)-i1, em::bx1) = -Fld(N_GHOSTS+i1, em::bx1); - Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); - Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); - } + if (tags & BC::E) + { + Fld((N_GHOSTS - 1) - i1, em::ex1) = Fld(N_GHOSTS + i1, em::ex1); + Fld((N_GHOSTS - 1) - i1, em::ex2) = -Fld(N_GHOSTS + 1 + i1, em::ex2); + Fld((N_GHOSTS - 1) - i1, em::ex3) = -Fld(N_GHOSTS + 1 + i1, em::ex3); + } + if (tags & BC::B) + { + Fld((N_GHOSTS - 1) - i1, em::bx1) = -Fld(N_GHOSTS + 1 + i1, em::bx1); + Fld((N_GHOSTS - 1) - i1, em::bx2) = Fld(N_GHOSTS + i1, em::bx2); + Fld((N_GHOSTS - 1) - i1, em::bx3) = Fld(N_GHOSTS + i1, em::bx3); + } } else { // GRPIC raise::KernelError(HERE, "1D GRPIC not implemented"); @@ -546,9 +546,6 @@ namespace kernel::bc { Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+1+i1, i2, em::bx1); Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); - // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; - // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; - // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; } } else { // GRPIC From fc3e647f62b596501107776847a9f6a9019bfb65 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 07:44:22 -0500 Subject: [PATCH 038/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 4d4b1cb37..14c2006e9 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -48,7 +48,8 @@ namespace user { Inline auto bx3(const coord_t& x_ph) const -> real_t { // return Bmag * math::sin(Btheta) * math::cos(Bphi); - return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + // return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ZERO; } // electric field components @@ -58,7 +59,8 @@ namespace user { Inline auto ex2(const coord_t& x_ph) const -> real_t { // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + // return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ZERO; } Inline auto ex3(const coord_t& x_ph) const -> real_t { @@ -120,8 +122,6 @@ namespace user { auto& species_e = domain.species[0]; auto& species_p = domain.species[1]; - auto metric = domain.mesh.metric; - auto m = domain.mesh.metric; array_t elec_ind("elec_ind"); array_t pos_ind("pos_ind"); @@ -152,15 +152,14 @@ namespace user { auto tag_p = species_p.tag; int nseed = 1; - auto dseed = HALF * constant::PI / static_cast(nseed); Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { // ToDo: fix this auto i1_ = ONE; auto i2_ = ONE; - auto dx1_ = ONE - HALF; - auto dx2_ = ONE - HALF; + auto dx1_ = HALF; + auto dx2_ = HALF; auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); From 83c144f4f99960c3c381e2945d1b319ecf15ef04 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 07:47:32 -0500 Subject: [PATCH 039/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 14c2006e9..85d08214a 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -156,8 +156,8 @@ namespace user { Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { // ToDo: fix this - auto i1_ = ONE; - auto i2_ = ONE; + auto i1_ = 100; + auto i2_ = 100; auto dx1_ = HALF; auto dx2_ = HALF; From 7c7c3f9cecd887948c9b42e73f01cfb83da17056 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 09:42:48 -0500 Subject: [PATCH 040/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 90 ++++++++++++++++----------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 85d08214a..b682cd621 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -153,51 +153,51 @@ namespace user { int nseed = 1; - Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { - - // ToDo: fix this - auto i1_ = 100; - auto i2_ = 100; - auto dx1_ = HALF; - auto dx2_ = HALF; - - - auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); - auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); - - i1_e(elec_p + offset_e) = i1_; - dx1_e(elec_p + offset_e) = dx1_; - i2_e(elec_p + offset_e) = i2_; - dx2_e(elec_p + offset_e) = dx2_; - phi_e(elec_p + offset_e) = ZERO; - ux1_e(elec_p + offset_e) = -drift_ux; - ux2_e(elec_p + offset_e) = -drift_ux; - ux3_e(elec_p + offset_e) = ZERO; - weight_e(elec_p + offset_e) = ONE; - tag_e(elec_p + offset_e) = ParticleTag::alive; - - i1_p(pos_p + offset_p) = i1_; - dx1_p(pos_p + offset_p) = dx1_; - i2_p(pos_p + offset_p) = i2_; - dx2_p(pos_p + offset_p) = dx2_; - phi_p(pos_p + offset_p) = ZERO; - ux1_p(pos_p + offset_p) = -drift_ux; - ux2_p(pos_p + offset_p) = drift_ux; - ux3_p(pos_p + offset_p) = ZERO; - weight_p(pos_p + offset_p) = ONE; - tag_p(pos_p + offset_p) = ParticleTag::alive; - - - }); - - - auto elec_ind_h = Kokkos::create_mirror(elec_ind); - Kokkos::deep_copy(elec_ind_h, elec_ind); - species_e.set_npart(offset_e + elec_ind_h()); - - auto pos_ind_h = Kokkos::create_mirror(pos_ind); - Kokkos::deep_copy(pos_ind_h, pos_ind); - species_p.set_npart(offset_p + pos_ind_h()); + // Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { + + // // ToDo: fix this + // auto i1_ = 100; + // auto i2_ = 100; + // auto dx1_ = HALF; + // auto dx2_ = HALF; + + + // auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); + // auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); + + // i1_e(elec_p + offset_e) = i1_; + // dx1_e(elec_p + offset_e) = dx1_; + // i2_e(elec_p + offset_e) = i2_; + // dx2_e(elec_p + offset_e) = dx2_; + // phi_e(elec_p + offset_e) = ZERO; + // ux1_e(elec_p + offset_e) = -drift_ux; + // ux2_e(elec_p + offset_e) = -drift_ux; + // ux3_e(elec_p + offset_e) = ZERO; + // weight_e(elec_p + offset_e) = ONE; + // tag_e(elec_p + offset_e) = ParticleTag::alive; + + // i1_p(pos_p + offset_p) = i1_; + // dx1_p(pos_p + offset_p) = dx1_; + // i2_p(pos_p + offset_p) = i2_; + // dx2_p(pos_p + offset_p) = dx2_; + // phi_p(pos_p + offset_p) = ZERO; + // ux1_p(pos_p + offset_p) = -drift_ux; + // ux2_p(pos_p + offset_p) = drift_ux; + // ux3_p(pos_p + offset_p) = ZERO; + // weight_p(pos_p + offset_p) = ONE; + // tag_p(pos_p + offset_p) = ParticleTag::alive; + + + // }); + + + // auto elec_ind_h = Kokkos::create_mirror(elec_ind); + // Kokkos::deep_copy(elec_ind_h, elec_ind); + // species_e.set_npart(offset_e + elec_ind_h()); + + // auto pos_ind_h = Kokkos::create_mirror(pos_ind); + // Kokkos::deep_copy(pos_ind_h, pos_ind); + // species_p.set_npart(offset_p + pos_ind_h()); } From 939369eb65a4aa66e6335054451d816a09ac63f6 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 09:51:22 -0500 Subject: [PATCH 041/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 70 ++++++++++++++++----------------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index b682cd621..f74e45080 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -151,53 +151,51 @@ namespace user { auto weight_p = species_p.weight; auto tag_p = species_p.tag; - int nseed = 1; + int nseed = 10; - // Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { + Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { - // // ToDo: fix this - // auto i1_ = 100; - // auto i2_ = 100; - // auto dx1_ = HALF; - // auto dx2_ = HALF; + // ToDo: fix this + auto i1_ = math::floor(100); + auto i2_ = math::floor(64); + auto dx1_ = HALF; + auto dx2_ = HALF; - // auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); - // auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); + auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); + auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); - // i1_e(elec_p + offset_e) = i1_; - // dx1_e(elec_p + offset_e) = dx1_; - // i2_e(elec_p + offset_e) = i2_; - // dx2_e(elec_p + offset_e) = dx2_; - // phi_e(elec_p + offset_e) = ZERO; - // ux1_e(elec_p + offset_e) = -drift_ux; - // ux2_e(elec_p + offset_e) = -drift_ux; - // ux3_e(elec_p + offset_e) = ZERO; - // weight_e(elec_p + offset_e) = ONE; - // tag_e(elec_p + offset_e) = ParticleTag::alive; + i1_e(elec_p + offset_e) = i1_; + dx1_e(elec_p + offset_e) = dx1_; + i2_e(elec_p + offset_e) = i2_; + dx2_e(elec_p + offset_e) = dx2_; + ux1_e(elec_p + offset_e) = ZERO; + ux2_e(elec_p + offset_e) = ZERO; + ux3_e(elec_p + offset_e) = ZERO; + weight_e(elec_p + offset_e) = ONE; + tag_e(elec_p + offset_e) = ParticleTag::alive; - // i1_p(pos_p + offset_p) = i1_; - // dx1_p(pos_p + offset_p) = dx1_; - // i2_p(pos_p + offset_p) = i2_; - // dx2_p(pos_p + offset_p) = dx2_; - // phi_p(pos_p + offset_p) = ZERO; - // ux1_p(pos_p + offset_p) = -drift_ux; - // ux2_p(pos_p + offset_p) = drift_ux; - // ux3_p(pos_p + offset_p) = ZERO; - // weight_p(pos_p + offset_p) = ONE; - // tag_p(pos_p + offset_p) = ParticleTag::alive; + i1_p(pos_p + offset_p) = i1_; + dx1_p(pos_p + offset_p) = dx1_; + i2_p(pos_p + offset_p) = i2_; + dx2_p(pos_p + offset_p) = dx2_; + ux1_p(pos_p + offset_p) = ZERO; + ux2_p(pos_p + offset_p) = ZERO; + ux3_p(pos_p + offset_p) = ZERO; + weight_p(pos_p + offset_p) = ONE; + tag_p(pos_p + offset_p) = ParticleTag::alive; - // }); + }); - // auto elec_ind_h = Kokkos::create_mirror(elec_ind); - // Kokkos::deep_copy(elec_ind_h, elec_ind); - // species_e.set_npart(offset_e + elec_ind_h()); + auto elec_ind_h = Kokkos::create_mirror(elec_ind); + Kokkos::deep_copy(elec_ind_h, elec_ind); + species_e.set_npart(offset_e + elec_ind_h()); - // auto pos_ind_h = Kokkos::create_mirror(pos_ind); - // Kokkos::deep_copy(pos_ind_h, pos_ind); - // species_p.set_npart(offset_p + pos_ind_h()); + auto pos_ind_h = Kokkos::create_mirror(pos_ind); + Kokkos::deep_copy(pos_ind_h, pos_ind); + species_p.set_npart(offset_p + pos_ind_h()); } From fa5a38fe127df07335793ea0433e9e4e95d9ae76 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 09:56:29 -0500 Subject: [PATCH 042/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index f74e45080..a835b7990 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -151,7 +151,7 @@ namespace user { auto weight_p = species_p.weight; auto tag_p = species_p.tag; - int nseed = 10; + int nseed = 1; Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { @@ -169,8 +169,8 @@ namespace user { dx1_e(elec_p + offset_e) = dx1_; i2_e(elec_p + offset_e) = i2_; dx2_e(elec_p + offset_e) = dx2_; - ux1_e(elec_p + offset_e) = ZERO; - ux2_e(elec_p + offset_e) = ZERO; + ux1_e(elec_p + offset_e) = -drift_ux; + ux2_e(elec_p + offset_e) = drift_ux; ux3_e(elec_p + offset_e) = ZERO; weight_e(elec_p + offset_e) = ONE; tag_e(elec_p + offset_e) = ParticleTag::alive; @@ -179,8 +179,8 @@ namespace user { dx1_p(pos_p + offset_p) = dx1_; i2_p(pos_p + offset_p) = i2_; dx2_p(pos_p + offset_p) = dx2_; - ux1_p(pos_p + offset_p) = ZERO; - ux2_p(pos_p + offset_p) = ZERO; + ux1_p(pos_p + offset_p) = -drift_ux; + ux2_p(pos_p + offset_p) = drift_ux; ux3_p(pos_p + offset_p) = ZERO; weight_p(pos_p + offset_p) = ONE; tag_p(pos_p + offset_p) = ParticleTag::alive; From eeca02d46f7d63575fa923b23b16a003d2fac936 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 09:58:00 -0500 Subject: [PATCH 043/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index a835b7990..9711646e3 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -169,8 +169,8 @@ namespace user { dx1_e(elec_p + offset_e) = dx1_; i2_e(elec_p + offset_e) = i2_; dx2_e(elec_p + offset_e) = dx2_; - ux1_e(elec_p + offset_e) = -drift_ux; - ux2_e(elec_p + offset_e) = drift_ux; + ux1_e(elec_p + offset_e) = -0.5; + ux2_e(elec_p + offset_e) = 0.5; ux3_e(elec_p + offset_e) = ZERO; weight_e(elec_p + offset_e) = ONE; tag_e(elec_p + offset_e) = ParticleTag::alive; @@ -179,8 +179,8 @@ namespace user { dx1_p(pos_p + offset_p) = dx1_; i2_p(pos_p + offset_p) = i2_; dx2_p(pos_p + offset_p) = dx2_; - ux1_p(pos_p + offset_p) = -drift_ux; - ux2_p(pos_p + offset_p) = drift_ux; + ux1_p(pos_p + offset_p) = -0.5; + ux2_p(pos_p + offset_p) = 0.5; ux3_p(pos_p + offset_p) = ZERO; weight_p(pos_p + offset_p) = ONE; tag_p(pos_p + offset_p) = ParticleTag::alive; From c18840ca3a1f4ccab2992c320ff1f2a0d61fce03 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 11:05:00 -0500 Subject: [PATCH 044/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 9711646e3..4cf89ac69 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -156,7 +156,7 @@ namespace user { Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { // ToDo: fix this - auto i1_ = math::floor(100); + auto i1_ = math::floor(10); auto i2_ = math::floor(64); auto dx1_ = HALF; auto dx2_ = HALF; From 9085afd497f0535126f75ecf353d289c6a248682 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 11:18:31 -0500 Subject: [PATCH 045/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 4cf89ac69..a1ce9a1a9 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -179,8 +179,8 @@ namespace user { dx1_p(pos_p + offset_p) = dx1_; i2_p(pos_p + offset_p) = i2_; dx2_p(pos_p + offset_p) = dx2_; - ux1_p(pos_p + offset_p) = -0.5; - ux2_p(pos_p + offset_p) = 0.5; + ux1_p(pos_p + offset_p) = 0.5; + ux2_p(pos_p + offset_p) = -0.5; ux3_p(pos_p + offset_p) = ZERO; weight_p(pos_p + offset_p) = ONE; tag_p(pos_p + offset_p) = ParticleTag::alive; From 87b37beb08e476081bcc4e601f16d2281549d62c Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 13:20:21 -0500 Subject: [PATCH 046/183] Ongoing. --- src/kernels/fields_bcs.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 39dfaf5dc..e477c4eb1 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -537,13 +537,13 @@ namespace kernel::bc { // SRPIC if (tags & BC::E) { Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); - Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+1+i1, i2, em::ex2); - Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+1+i1, i2, em::ex3); + Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+i1, i2, em::ex2); + Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+i1, i2, em::ex3); } if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+1+i1, i2, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); } From 5712e3bd4bb628fa9df2480e2e31e9a0f885995c Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 13:34:59 -0500 Subject: [PATCH 047/183] Ongoing. --- src/kernels/fields_bcs.hpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index e477c4eb1..115aa3415 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -536,16 +536,19 @@ namespace kernel::bc { if constexpr (S == SimEngine::SRPIC) { // SRPIC if (tags & BC::E) { - Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); - Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+i1, i2, em::ex2); - Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+i1, i2, em::ex3); + // Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); + // Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+1+i1, i2, em::ex2); + // Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+1+i1, i2, em::ex3); + Fld((N_GHOSTS-1)-i1, i2, em::ex1) = ZERO; + Fld((N_GHOSTS-1)-i1, i2, em::ex2) = ZERO; + Fld((N_GHOSTS-1)-i1, i2, em::ex3) = ZERO; } if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = ZERO; + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = ZERO; + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = ZERO; } } else { // GRPIC From af258c6c91d800977523f6d26ad25c423be84b5b Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 13:36:55 -0500 Subject: [PATCH 048/183] Ongoing. --- src/kernels/fields_bcs.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 115aa3415..b15a16546 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -542,6 +542,8 @@ namespace kernel::bc { Fld((N_GHOSTS-1)-i1, i2, em::ex1) = ZERO; Fld((N_GHOSTS-1)-i1, i2, em::ex2) = ZERO; Fld((N_GHOSTS-1)-i1, i2, em::ex3) = ZERO; + Fld((N_GHOSTS), i2, em::ex2) = ZERO; + Fld((N_GHOSTS), i2, em::ex3) = ZERO; } if (tags & BC::B) From 86f05972d2623612b13e9d51b814b0b715e5f566 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 14:53:48 -0500 Subject: [PATCH 049/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 156 ++++++++++++++++---------------- 1 file changed, 78 insertions(+), 78 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index a1ce9a1a9..1b8926c67 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -48,8 +48,8 @@ namespace user { Inline auto bx3(const coord_t& x_ph) const -> real_t { // return Bmag * math::sin(Btheta) * math::cos(Bphi); - // return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); - return ZERO; + return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + // return ZERO; } // electric field components @@ -59,8 +59,8 @@ namespace user { Inline auto ex2(const coord_t& x_ph) const -> real_t { // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - // return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); - return ZERO; + return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + // return ZERO; } Inline auto ex3(const coord_t& x_ph) const -> real_t { @@ -120,82 +120,82 @@ namespace user { inline void InitPrtls(Domain& domain) { - auto& species_e = domain.species[0]; - auto& species_p = domain.species[1]; + // auto& species_e = domain.species[0]; + // auto& species_p = domain.species[1]; - array_t elec_ind("elec_ind"); - array_t pos_ind("pos_ind"); + // array_t elec_ind("elec_ind"); + // array_t pos_ind("pos_ind"); - auto offset_e = species_e.npart(); - auto offset_p = species_p.npart(); - - auto ux1_e = species_e.ux1; - auto ux2_e = species_e.ux2; - auto ux3_e = species_e.ux3; - auto i1_e = species_e.i1; - auto i2_e = species_e.i2; - auto dx1_e = species_e.dx1; - auto dx2_e = species_e.dx2; - auto phi_e = species_e.phi; - auto weight_e = species_e.weight; - auto tag_e = species_e.tag; - - auto ux1_p = species_p.ux1; - auto ux2_p = species_p.ux2; - auto ux3_p = species_p.ux3; - auto i1_p = species_p.i1; - auto i2_p = species_p.i2; - auto dx1_p = species_p.dx1; - auto dx2_p = species_p.dx2; - auto phi_p = species_p.phi; - auto weight_p = species_p.weight; - auto tag_p = species_p.tag; - - int nseed = 1; - - Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { - - // ToDo: fix this - auto i1_ = math::floor(10); - auto i2_ = math::floor(64); - auto dx1_ = HALF; - auto dx2_ = HALF; - - - auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); - auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); - - i1_e(elec_p + offset_e) = i1_; - dx1_e(elec_p + offset_e) = dx1_; - i2_e(elec_p + offset_e) = i2_; - dx2_e(elec_p + offset_e) = dx2_; - ux1_e(elec_p + offset_e) = -0.5; - ux2_e(elec_p + offset_e) = 0.5; - ux3_e(elec_p + offset_e) = ZERO; - weight_e(elec_p + offset_e) = ONE; - tag_e(elec_p + offset_e) = ParticleTag::alive; - - i1_p(pos_p + offset_p) = i1_; - dx1_p(pos_p + offset_p) = dx1_; - i2_p(pos_p + offset_p) = i2_; - dx2_p(pos_p + offset_p) = dx2_; - ux1_p(pos_p + offset_p) = 0.5; - ux2_p(pos_p + offset_p) = -0.5; - ux3_p(pos_p + offset_p) = ZERO; - weight_p(pos_p + offset_p) = ONE; - tag_p(pos_p + offset_p) = ParticleTag::alive; - - - }); - - - auto elec_ind_h = Kokkos::create_mirror(elec_ind); - Kokkos::deep_copy(elec_ind_h, elec_ind); - species_e.set_npart(offset_e + elec_ind_h()); - - auto pos_ind_h = Kokkos::create_mirror(pos_ind); - Kokkos::deep_copy(pos_ind_h, pos_ind); - species_p.set_npart(offset_p + pos_ind_h()); + // auto offset_e = species_e.npart(); + // auto offset_p = species_p.npart(); + + // auto ux1_e = species_e.ux1; + // auto ux2_e = species_e.ux2; + // auto ux3_e = species_e.ux3; + // auto i1_e = species_e.i1; + // auto i2_e = species_e.i2; + // auto dx1_e = species_e.dx1; + // auto dx2_e = species_e.dx2; + // auto phi_e = species_e.phi; + // auto weight_e = species_e.weight; + // auto tag_e = species_e.tag; + + // auto ux1_p = species_p.ux1; + // auto ux2_p = species_p.ux2; + // auto ux3_p = species_p.ux3; + // auto i1_p = species_p.i1; + // auto i2_p = species_p.i2; + // auto dx1_p = species_p.dx1; + // auto dx2_p = species_p.dx2; + // auto phi_p = species_p.phi; + // auto weight_p = species_p.weight; + // auto tag_p = species_p.tag; + + // int nseed = 1; + + // Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { + + // // ToDo: fix this + // auto i1_ = math::floor(10); + // auto i2_ = math::floor(64); + // auto dx1_ = HALF; + // auto dx2_ = HALF; + + + // auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); + // auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); + + // i1_e(elec_p + offset_e) = i1_; + // dx1_e(elec_p + offset_e) = dx1_; + // i2_e(elec_p + offset_e) = i2_; + // dx2_e(elec_p + offset_e) = dx2_; + // ux1_e(elec_p + offset_e) = -0.5; + // ux2_e(elec_p + offset_e) = 0.5; + // ux3_e(elec_p + offset_e) = ZERO; + // weight_e(elec_p + offset_e) = ONE; + // tag_e(elec_p + offset_e) = ParticleTag::alive; + + // i1_p(pos_p + offset_p) = i1_; + // dx1_p(pos_p + offset_p) = dx1_; + // i2_p(pos_p + offset_p) = i2_; + // dx2_p(pos_p + offset_p) = dx2_; + // ux1_p(pos_p + offset_p) = 0.5; + // ux2_p(pos_p + offset_p) = -0.5; + // ux3_p(pos_p + offset_p) = ZERO; + // weight_p(pos_p + offset_p) = ONE; + // tag_p(pos_p + offset_p) = ParticleTag::alive; + + + // }); + + + // auto elec_ind_h = Kokkos::create_mirror(elec_ind); + // Kokkos::deep_copy(elec_ind_h, elec_ind); + // species_e.set_npart(offset_e + elec_ind_h()); + + // auto pos_ind_h = Kokkos::create_mirror(pos_ind); + // Kokkos::deep_copy(pos_ind_h, pos_ind); + // species_p.set_npart(offset_p + pos_ind_h()); } From d0ce7c8e64caab5fe200a85b4479dd07d21b5657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 3 Mar 2025 13:11:03 -0600 Subject: [PATCH 050/183] cleanup of conductor BCs --- src/kernels/fields_bcs.hpp | 101 ++++++++++++++++++++----------------- 1 file changed, 56 insertions(+), 45 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index b15a16546..720408d05 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -506,19 +506,19 @@ namespace kernel::bc { if constexpr (S == SimEngine::SRPIC) { - if (tags & BC::E) - { - Fld((N_GHOSTS - 1) - i1, em::ex1) = Fld(N_GHOSTS + i1, em::ex1); - Fld((N_GHOSTS - 1) - i1, em::ex2) = -Fld(N_GHOSTS + 1 + i1, em::ex2); - Fld((N_GHOSTS - 1) - i1, em::ex3) = -Fld(N_GHOSTS + 1 + i1, em::ex3); - } + if (tags & BC::E) { + Fld((N_GHOSTS-1)-i1, em::ex1) = Fld(N_GHOSTS+i1, em::ex1); + Fld((N_GHOSTS-1)-i1, em::ex2) = -Fld(N_GHOSTS+i1+1, em::ex2); + Fld((N_GHOSTS-1)-i1, em::ex3) = -Fld(N_GHOSTS+i1+1, em::ex3); + } + + if (tags & BC::B) + { + Fld((N_GHOSTS-1)-i1, em::bx1) = -Fld(N_GHOSTS+i1+1, em::bx1); + Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); + Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); + } - if (tags & BC::B) - { - Fld((N_GHOSTS - 1) - i1, em::bx1) = -Fld(N_GHOSTS + 1 + i1, em::bx1); - Fld((N_GHOSTS - 1) - i1, em::bx2) = Fld(N_GHOSTS + i1, em::bx2); - Fld((N_GHOSTS - 1) - i1, em::bx3) = Fld(N_GHOSTS + i1, em::bx3); - } } else { // GRPIC raise::KernelError(HERE, "1D GRPIC not implemented"); @@ -530,64 +530,75 @@ namespace kernel::bc { } } - Inline void operator()(index_t i1, index_t i2) const { - if constexpr (M::Dim == Dim::_2D) { + Inline void operator()(index_t i1, index_t i2) const + { + if constexpr (M::Dim == Dim::_2D) + { - if constexpr (S == SimEngine::SRPIC) { + if constexpr (S == SimEngine::SRPIC) + { // SRPIC - if (tags & BC::E) { - // Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); - // Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+1+i1, i2, em::ex2); - // Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+1+i1, i2, em::ex3); - Fld((N_GHOSTS-1)-i1, i2, em::ex1) = ZERO; - Fld((N_GHOSTS-1)-i1, i2, em::ex2) = ZERO; - Fld((N_GHOSTS-1)-i1, i2, em::ex3) = ZERO; - Fld((N_GHOSTS), i2, em::ex2) = ZERO; - Fld((N_GHOSTS), i2, em::ex3) = ZERO; + if (tags & BC::E) + { + Fld((N_GHOSTS - 1) - i1, i2, em::ex1) = Fld(N_GHOSTS + i1, i2, em::ex1); + Fld((N_GHOSTS - 1) - i1, i2, em::ex2) = -Fld(N_GHOSTS + 1 + i1, i2, em::ex2); + Fld((N_GHOSTS - 1) - i1, i2, em::ex3) = -Fld(N_GHOSTS + 1 + i1, i2, em::ex3); } if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = ZERO; - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = ZERO; - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = ZERO; + Fld((N_GHOSTS - 1) - i1, i2, em::bx1) = -Fld(N_GHOSTS + 1 + i1, i2, em::bx1); + Fld((N_GHOSTS - 1) - i1, i2, em::bx2) = Fld(N_GHOSTS + i1, i2, em::bx2); + Fld((N_GHOSTS - 1) - i1, i2, em::bx3) = Fld(N_GHOSTS + i1, i2, em::bx3); } - } else { + } + else + { // GRPIC raise::KernelError(HERE, "2D GRPIC not implemented"); } - } else { + } + else + { raise::KernelError( - HERE, - "ConductorBoundaries_kernel: 2D implementation called for D != 2"); + HERE, + "ConductorBoundaries_kernel: 2D implementation called for D != 2"); } } - Inline void operator()(index_t i1, index_t i2, index_t i3) const { - if constexpr (M::Dim == Dim::_3D) { + Inline void operator()(index_t i1, index_t i2, index_t i3) const + { + if constexpr (M::Dim == Dim::_3D) + { - if constexpr (S == SimEngine::SRPIC) { + if constexpr (S == SimEngine::SRPIC) + { // SRPIC - if (tags & BC::E) { - Fld((N_GHOSTS-1)-i1, i2, i3, em::ex1) = Fld(N_GHOSTS+i1, i2, i3, em::ex1); - Fld((N_GHOSTS-1)-i1, i2, i3, em::ex2) = -Fld(N_GHOSTS+i1, i2, i3, em::ex2); - Fld((N_GHOSTS-1)-i1, i2, i3, em::ex3) = -Fld(N_GHOSTS+i1, i2, i3, em::ex3); + if (tags & BC::E) + { + Fld((N_GHOSTS - 1) - i1, i2, i3, em::ex1) = Fld(N_GHOSTS + i1, i2, i3, em::ex1); + Fld((N_GHOSTS - 1) - i1, i2, i3, em::ex2) = -Fld(N_GHOSTS + i1 + 1, i2, i3, em::ex2); + Fld((N_GHOSTS - 1) - i1, i2, i3, em::ex3) = -Fld(N_GHOSTS + i1 + 1, i2, i3, em::ex3); } if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, i3, em::bx1) = -Fld(N_GHOSTS+i1, i2, i3, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, i3, em::bx2) = Fld(N_GHOSTS+i1, i2, i3, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, i3, em::bx3) = Fld(N_GHOSTS+i1, i2, i3, em::bx3); + Fld((N_GHOSTS - 1) - i1, i2, i3, em::bx1) = -Fld(N_GHOSTS + i1 + 1, i2, i3, em::bx1); + Fld((N_GHOSTS - 1) - i1, i2, i3, em::bx2) = Fld(N_GHOSTS + i1, i2, i3, em::bx2); + Fld((N_GHOSTS - 1) - i1, i2, i3, em::bx3) = Fld(N_GHOSTS + i1, i2, i3, em::bx3); } - } else { + } + else + { // GRPIC raise::KernelError(HERE, "3D GRPIC not implemented"); } - } else { + } + else + { raise::KernelError( - HERE, - "ConductorBoundaries_kernel: 3D implementation called for D != 3"); + HERE, + "ConductorBoundaries_kernel: 3D implementation called for D != 3"); } } }; From 0fd05b9ea515024beb64ed7ddf9b92d941d09949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 3 Mar 2025 13:11:50 -0600 Subject: [PATCH 051/183] add 3D case and set correct boundary cells to zero --- src/engines/srpic.hpp | 963 ++++++++++++++++++++++++------------------ 1 file changed, 547 insertions(+), 416 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 90aa271a3..1c7ffa869 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -871,225 +871,321 @@ namespace ntt { "Invalid range size", HERE); + if constexpr (M::Dim == Dim::_1D) { - Kokkos::parallel_for( - "MatchFields", - CreateRangePolicy( { xi_min[0] } , { xi_max[0] } ), - kernel::bc::ConductorBoundaries_kernel( - domain.fields.em, - tags)); + Kokkos::parallel_for( + "MatchFields", + CreateRangePolicy({xi_min[0]}, {xi_max[0]}), + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); } if constexpr (M::Dim == Dim::_2D) { - Kokkos::parallel_for( - "MatchFields", - CreateRangePolicy( { xi_min[0], xi_min[1] } , { xi_max[0], xi_max[1] } ), - kernel::bc::ConductorBoundaries_kernel( - domain.fields.em, - tags)); + Kokkos::parallel_for( + "MatchFields", + CreateRangePolicy({xi_min[0], xi_min[1]}, {xi_max[0], xi_max[1]}), + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); } - - - } - void AtmosphereFieldsIn(dir::direction_t direction, - domain_t& domain, - BCTags tags) { - /** - * atmosphere field boundaries - */ - if constexpr (traits::has_member::value) { - const auto [sign, dim, xg_min, xg_max] = get_atm_extent(direction); - const auto dd = static_cast(dim); - boundaries_t box; - boundaries_t incl_ghosts; - for (unsigned short d { 0 }; d < M::Dim; ++d) { - if (d == dd) { - box.push_back({ xg_min, xg_max }); - if (sign > 0) { - incl_ghosts.push_back({ false, true }); - } else { - incl_ghosts.push_back({ true, false }); - } - } else { - box.push_back(Range::All); - incl_ghosts.push_back({ true, true }); - } - } - if (not domain.mesh.Intersects(box)) { - return; - } - const auto intersect_range = domain.mesh.ExtentToRange(box, incl_ghosts); - tuple_t range_min { 0 }; - tuple_t range_max { 0 }; + if constexpr (M::Dim == Dim::_3D) + { + Kokkos::parallel_for( + "MatchFields", + CreateRangePolicy({xi_min[0], xi_min[1], xi_min[2]}, {xi_max[0], xi_max[1], xi_max[2]}), + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } - for (unsigned short d { 0 }; d < M::Dim; ++d) { - range_min[d] = intersect_range[d].first; - range_max[d] = intersect_range[d].second; + // these components need to be set to zero at the boundary + std::vector comps = {em::ex2, em::ex3, em::bx1}; + for (const auto &comp : comps) + { + if constexpr (M::Dim == Dim::_1D) + { + Kokkos::deep_copy( + Kokkos::subview(domain.fields.em, + std::make_pair(N_GHOSTS, N_GHOSTS), + comp), + ZERO); } - auto atm_fields = m_pgen.AtmFields(time); - std::size_t il_edge; - if (sign > 0) { - il_edge = range_min[dd] - N_GHOSTS; - } else { - il_edge = range_max[dd] - 1 - N_GHOSTS; + else if constexpr (M::Dim == Dim::_2D) + { + Kokkos::deep_copy( + Kokkos::subview(domain.fields.em, + std::make_pair(N_GHOSTS, N_GHOSTS), + std::make_pair(xi_min[1], xi_max[1]), + comp), + ZERO); } - const auto range = CreateRangePolicy(range_min, range_max); - if (dim == in::x1) { - if (sign > 0) { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } else { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } - } else if (dim == in::x2) { - if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { - if (sign > 0) { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } else { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } - } else { - raise::Error("Invalid dimension", HERE); - } - } else if (dim == in::x3) { - if constexpr (M::Dim == Dim::_3D) { - if (sign > 0) { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } else { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } - } else { - raise::Error("Invalid dimension", HERE); - } - } else { + else if constexpr (M::Dim == Dim::_3D) + { + Kokkos::deep_copy( + Kokkos::subview(domain.fields.em, + std::make_pair(N_GHOSTS, N_GHOSTS), + std::make_pair(xi_min[1], xi_max[1]), + std::make_pair(xi_min[2], xi_max[2]), + comp), + ZERO); + } + else + { raise::Error("Invalid dimension", HERE); } - } else { - raise::Error("Atm fields not implemented in PGEN for atmosphere BCs", HERE); } } - void CustomFieldsIn(dir::direction_t direction, - domain_t& domain, - BCTags tags) { - (void)direction; - (void)domain; - (void)tags; - raise::Error("Custom boundaries not implemented", HERE); - // if constexpr ( - // traits::has_member::value) { - // const auto [box, custom_fields] = m_pgen.CustomFields(time); - // if (domain.mesh.Intersects(box)) { - // } - // - // } else { - // raise::Error("Custom boundaries not implemented", HERE); - // } - } + void AtmosphereFieldsIn(dir::direction_t direction, + domain_t & domain, + BCTags tags) + { + /** + * atmosphere field boundaries + */ + if constexpr (traits::has_member::value) + { + const auto [sign, dim, xg_min, xg_max] = get_atm_extent(direction); + const auto dd = static_cast(dim); + boundaries_t box; + boundaries_t incl_ghosts; + for (unsigned short d{0}; d < M::Dim; ++d) + { + if (d == dd) + { + box.push_back({xg_min, xg_max}); + if (sign > 0) + { + incl_ghosts.push_back({false, true}); + } + else + { + incl_ghosts.push_back({true, false}); + } + } + else + { + box.push_back(Range::All); + incl_ghosts.push_back({true, true}); + } + } + if (not domain.mesh.Intersects(box)) + { + return; + } + const auto intersect_range = domain.mesh.ExtentToRange(box, incl_ghosts); + tuple_t range_min{0}; + tuple_t range_max{0}; + + for (unsigned short d{0}; d < M::Dim; ++d) + { + range_min[d] = intersect_range[d].first; + range_max[d] = intersect_range[d].second; + } + auto atm_fields = m_pgen.AtmFields(time); + std::size_t il_edge; + if (sign > 0) + { + il_edge = range_min[dd] - N_GHOSTS; + } + else + { + il_edge = range_max[dd] - 1 - N_GHOSTS; + } + const auto range = CreateRangePolicy(range_min, range_max); + if (dim == in::x1) + { + if (sign > 0) + { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } + else + { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } + } + else if (dim == in::x2) + { + if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) + { + if (sign > 0) + { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } + else + { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } + } + else + { + raise::Error("Invalid dimension", HERE); + } + } + else if (dim == in::x3) + { + if constexpr (M::Dim == Dim::_3D) + { + if (sign > 0) + { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } + else + { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } + } + else + { + raise::Error("Invalid dimension", HERE); + } + } + else + { + raise::Error("Invalid dimension", HERE); + } + } + else + { + raise::Error("Atm fields not implemented in PGEN for atmosphere BCs", HERE); + } + } - void AtmosphereParticlesIn(const dir::direction_t& direction, - domain_t& domain, - InjTags tags) { - const auto [sign, dim, xg_min, xg_max] = get_atm_extent(direction); - - const auto x_surf = sign > 0 ? xg_min : xg_max; - const auto ds = m_params.template get( - "grid.boundaries.atmosphere.ds"); - const auto temp = m_params.template get( - "grid.boundaries.atmosphere.temperature"); - const auto height = m_params.template get( - "grid.boundaries.atmosphere.height"); - const auto species = - m_params.template get>( - "grid.boundaries.atmosphere.species"); - const auto nmax = m_params.template get( - "grid.boundaries.atmosphere.density"); - - Kokkos::deep_copy(domain.fields.bckp, ZERO); - auto scatter_bckp = Kokkos::Experimental::create_scatter_view( - domain.fields.bckp); - const auto use_weights = M::CoordType != Coord::Cart; - const auto ni2 = domain.mesh.n_active(in::x2); - const auto inv_n0 = ONE / m_params.template get("scales.n0"); - - // compute the density of the two species - if (tags & Inj::AssumeEmpty) { - if constexpr (M::Dim == Dim::_1D) { - Kokkos::deep_copy( - Kokkos::subview(domain.fields.bckp, Kokkos::ALL, std::make_pair(0, 1)), - ZERO); - } else if constexpr (M::Dim == Dim::_2D) { - Kokkos::deep_copy(Kokkos::subview(domain.fields.bckp, - Kokkos::ALL, - Kokkos::ALL, - std::make_pair(0, 1)), - ZERO); - } else if constexpr (M::Dim == Dim::_3D) { - Kokkos::deep_copy(Kokkos::subview(domain.fields.bckp, - Kokkos::ALL, - Kokkos::ALL, - Kokkos::ALL, - std::make_pair(0, 1)), - ZERO); + void CustomFieldsIn(dir::direction_t direction, + domain_t & domain, + BCTags tags) + { + (void)direction; + (void)domain; + (void)tags; + raise::Error("Custom boundaries not implemented", HERE); + // if constexpr ( + // traits::has_member::value) { + // const auto [box, custom_fields] = m_pgen.CustomFields(time); + // if (domain.mesh.Intersects(box)) { + // } + // + // } else { + // raise::Error("Custom boundaries not implemented", HERE); + // } } - } else { - for (const auto& sp : - std::vector({ species.first, species.second })) { - auto& prtl_spec = domain.species[sp - 1]; - if (prtl_spec.npart() == 0) { - continue; + + void AtmosphereParticlesIn(const dir::direction_t &direction, + domain_t &domain, + InjTags tags) + { + const auto [sign, dim, xg_min, xg_max] = get_atm_extent(direction); + + const auto x_surf = sign > 0 ? xg_min : xg_max; + const auto ds = m_params.template get( + "grid.boundaries.atmosphere.ds"); + const auto temp = m_params.template get( + "grid.boundaries.atmosphere.temperature"); + const auto height = m_params.template get( + "grid.boundaries.atmosphere.height"); + const auto species = + m_params.template get>( + "grid.boundaries.atmosphere.species"); + const auto nmax = m_params.template get( + "grid.boundaries.atmosphere.density"); + + Kokkos::deep_copy(domain.fields.bckp, ZERO); + auto scatter_bckp = Kokkos::Experimental::create_scatter_view( + domain.fields.bckp); + const auto use_weights = M::CoordType != Coord::Cart; + const auto ni2 = domain.mesh.n_active(in::x2); + const auto inv_n0 = ONE / m_params.template get("scales.n0"); + + // compute the density of the two species + if (tags & Inj::AssumeEmpty) + { + if constexpr (M::Dim == Dim::_1D) + { + Kokkos::deep_copy( + Kokkos::subview(domain.fields.bckp, Kokkos::ALL, std::make_pair(0, 1)), + ZERO); + } + else if constexpr (M::Dim == Dim::_2D) + { + Kokkos::deep_copy(Kokkos::subview(domain.fields.bckp, + Kokkos::ALL, + Kokkos::ALL, + std::make_pair(0, 1)), + ZERO); + } + else if constexpr (M::Dim == Dim::_3D) + { + Kokkos::deep_copy(Kokkos::subview(domain.fields.bckp, + Kokkos::ALL, + Kokkos::ALL, + Kokkos::ALL, + std::make_pair(0, 1)), + ZERO); + } } - // clang-format off + else + { + for (const auto &sp : + std::vector({species.first, species.second})) + { + auto &prtl_spec = domain.species[sp - 1]; + if (prtl_spec.npart() == 0) + { + continue; + } + // clang-format off Kokkos::parallel_for( "ComputeMoments", prtl_spec.rangeActiveParticles(), @@ -1103,230 +1199,265 @@ namespace ntt { use_weights, domain.mesh.metric, domain.mesh.flds_bc(), ni2, inv_n0, 0)); - // clang-format on - prtl_spec.set_unsorted(); - } - Kokkos::Experimental::contribute(domain.fields.bckp, scatter_bckp); - m_metadomain.SynchronizeFields(domain, Comm::Bckp, { 0, 1 }); - } - - if (dim == in::x1) { - if (sign > 0) { - const auto atm_injector = - arch::AtmosphereInjector { - domain.mesh.metric, - domain.fields.bckp, - nmax, - height, - x_surf, - ds, - temp, - domain.random_pool, - species - }; - arch::InjectNonUniform(m_params, - domain, - atm_injector, - nmax, - use_weights); - } else { - const auto atm_injector = - arch::AtmosphereInjector { - domain.mesh.metric, - domain.fields.bckp, - nmax, - height, - x_surf, - ds, - temp, - domain.random_pool, - species - }; - arch::InjectNonUniform(m_params, - domain, - atm_injector, - nmax, - use_weights); - } - } else if (dim == in::x2) { - if (sign > 0) { - const auto atm_injector = - arch::AtmosphereInjector { - domain.mesh.metric, - domain.fields.bckp, - nmax, - height, - x_surf, - ds, - temp, - domain.random_pool, - species - }; - arch::InjectNonUniform(m_params, - domain, - atm_injector, - nmax, - use_weights); - } else { - const auto atm_injector = - arch::AtmosphereInjector { - domain.mesh.metric, - domain.fields.bckp, - nmax, - height, - x_surf, - ds, - temp, - domain.random_pool, - species - }; - arch::InjectNonUniform(m_params, - domain, - atm_injector, - nmax, - use_weights); - } - } else if (dim == in::x3) { - if (sign > 0) { - const auto atm_injector = - arch::AtmosphereInjector { - domain.mesh.metric, - domain.fields.bckp, - nmax, - height, - x_surf, - ds, - temp, - domain.random_pool, - species - }; - arch::InjectNonUniform(m_params, - domain, - atm_injector, - nmax, - use_weights); - } else { - const auto atm_injector = - arch::AtmosphereInjector { - domain.mesh.metric, - domain.fields.bckp, - nmax, - height, - x_surf, - ds, - temp, - domain.random_pool, - species - }; - arch::InjectNonUniform(m_params, - domain, - atm_injector, - nmax, - use_weights); - } - } else { - raise::Error("Invalid dimension", HERE); - } - return; - } - - private: - /** - * @brief Get the buffer region of the atmosphere and the direction - * @param direction direction in which the atmosphere is applied - * @return tuple: [sign of the direction, the direction (as in::), the min and max extent - * @note xg_min and xg_max are the extents where the fields are set, not the atmosphere itself - * @note i.e. - * - * fields set particles injected - * ghost zone | | - * v v v - * |....|...........|*******************..... -> x1 - * ^ ^ - * xg_min xg_max - * | | | - * |<-- buffer -->|<-- atmosphere -->| - * - * in this case the function returns { -1, in::x1, xg_min, xg_max } - */ - auto get_atm_extent(dir::direction_t direction) const - -> std::tuple { - const auto sign = direction.get_sign(); - const auto dim = direction.get_dim(); - const auto min_buff = m_params.template get( - "algorithms.current_filters") + - 2; - const auto buffer_ncells = min_buff > 5 ? min_buff : 5; - if (M::CoordType != Coord::Cart and (dim != in::x1 or sign > 0)) { - raise::Error("For non-cartesian coordinates atmosphere BCs is " - "possible only in -x1 (@ rmin)", - HERE); - } - real_t xg_min { ZERO }, xg_max { ZERO }; - std::size_t ig_min, ig_max; - if (sign > 0) { // + direction - ig_min = m_metadomain.mesh().n_active(dim) - buffer_ncells; - ig_max = m_metadomain.mesh().n_active(dim); - } else { // - direction - ig_min = 0; - ig_max = buffer_ncells; - } + // clang-format on + prtl_spec.set_unsorted(); + } + Kokkos::Experimental::contribute(domain.fields.bckp, scatter_bckp); + m_metadomain.SynchronizeFields(domain, Comm::Bckp, {0, 1}); + } - if (dim == in::x1) { - xg_min = m_metadomain.mesh().metric.template convert<1, Crd::Cd, Crd::Ph>( - static_cast(ig_min)); - xg_max = m_metadomain.mesh().metric.template convert<1, Crd::Cd, Crd::Ph>( - static_cast(ig_max)); - } else if (dim == in::x2) { - if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { - xg_min = m_metadomain.mesh().metric.template convert<2, Crd::Cd, Crd::Ph>( - static_cast(ig_min)); - xg_max = m_metadomain.mesh().metric.template convert<2, Crd::Cd, Crd::Ph>( - static_cast(ig_max)); - } else { - raise::Error("Invalid dimension", HERE); - } - } else if (dim == in::x3) { - if constexpr (M::Dim == Dim::_3D) { - xg_min = m_metadomain.mesh().metric.template convert<3, Crd::Cd, Crd::Ph>( - static_cast(ig_min)); - xg_max = m_metadomain.mesh().metric.template convert<3, Crd::Cd, Crd::Ph>( - static_cast(ig_max)); - } else { - raise::Error("Invalid dimension", HERE); + if (dim == in::x1) + { + if (sign > 0) + { + const auto atm_injector = + arch::AtmosphereInjector{ + domain.mesh.metric, + domain.fields.bckp, + nmax, + height, + x_surf, + ds, + temp, + domain.random_pool, + species}; + arch::InjectNonUniform(m_params, + domain, + atm_injector, + nmax, + use_weights); + } + else + { + const auto atm_injector = + arch::AtmosphereInjector{ + domain.mesh.metric, + domain.fields.bckp, + nmax, + height, + x_surf, + ds, + temp, + domain.random_pool, + species}; + arch::InjectNonUniform(m_params, + domain, + atm_injector, + nmax, + use_weights); + } + } + else if (dim == in::x2) + { + if (sign > 0) + { + const auto atm_injector = + arch::AtmosphereInjector{ + domain.mesh.metric, + domain.fields.bckp, + nmax, + height, + x_surf, + ds, + temp, + domain.random_pool, + species}; + arch::InjectNonUniform(m_params, + domain, + atm_injector, + nmax, + use_weights); + } + else + { + const auto atm_injector = + arch::AtmosphereInjector{ + domain.mesh.metric, + domain.fields.bckp, + nmax, + height, + x_surf, + ds, + temp, + domain.random_pool, + species}; + arch::InjectNonUniform(m_params, + domain, + atm_injector, + nmax, + use_weights); + } + } + else if (dim == in::x3) + { + if (sign > 0) + { + const auto atm_injector = + arch::AtmosphereInjector{ + domain.mesh.metric, + domain.fields.bckp, + nmax, + height, + x_surf, + ds, + temp, + domain.random_pool, + species}; + arch::InjectNonUniform(m_params, + domain, + atm_injector, + nmax, + use_weights); + } + else + { + const auto atm_injector = + arch::AtmosphereInjector{ + domain.mesh.metric, + domain.fields.bckp, + nmax, + height, + x_surf, + ds, + temp, + domain.random_pool, + species}; + arch::InjectNonUniform(m_params, + domain, + atm_injector, + nmax, + use_weights); + } + } + else + { + raise::Error("Invalid dimension", HERE); + } + return; } - } else { - raise::Error("Invalid dimension", HERE); - } - return { sign, dim, xg_min, xg_max }; - } - auto range_with_axis_BCs(const domain_t& domain) -> range_t { - auto range = domain.mesh.rangeActiveCells(); - if constexpr (M::CoordType != Coord::Cart) { + private: /** - * @brief taking one extra cell in the x2 direction if AXIS BCs + * @brief Get the buffer region of the atmosphere and the direction + * @param direction direction in which the atmosphere is applied + * @return tuple: [sign of the direction, the direction (as in::), the min and max extent + * @note xg_min and xg_max are the extents where the fields are set, not the atmosphere itself + * @note i.e. + * + * fields set particles injected + * ghost zone | | + * v v v + * |....|...........|*******************..... -> x1 + * ^ ^ + * xg_min xg_max + * | | | + * |<-- buffer -->|<-- atmosphere -->| + * + * in this case the function returns { -1, in::x1, xg_min, xg_max } */ - if constexpr (M::Dim == Dim::_2D) { - if (domain.mesh.flds_bc_in({ 0, +1 }) == FldsBC::AXIS) { - range = CreateRangePolicy( - { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) }, - { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1 }); + auto get_atm_extent(dir::direction_t direction) const + -> std::tuple + { + const auto sign = direction.get_sign(); + const auto dim = direction.get_dim(); + const auto min_buff = m_params.template get( + "algorithms.current_filters") + + 2; + const auto buffer_ncells = min_buff > 5 ? min_buff : 5; + if (M::CoordType != Coord::Cart and (dim != in::x1 or sign > 0)) + { + raise::Error("For non-cartesian coordinates atmosphere BCs is " + "possible only in -x1 (@ rmin)", + HERE); + } + real_t xg_min{ZERO}, xg_max{ZERO}; + std::size_t ig_min, ig_max; + if (sign > 0) + { // + direction + ig_min = m_metadomain.mesh().n_active(dim) - buffer_ncells; + ig_max = m_metadomain.mesh().n_active(dim); + } + else + { // - direction + ig_min = 0; + ig_max = buffer_ncells; + } + + if (dim == in::x1) + { + xg_min = m_metadomain.mesh().metric.template convert<1, Crd::Cd, Crd::Ph>( + static_cast(ig_min)); + xg_max = m_metadomain.mesh().metric.template convert<1, Crd::Cd, Crd::Ph>( + static_cast(ig_max)); } - } else if constexpr (M::Dim == Dim::_3D) { - if (domain.mesh.flds_bc_in({ 0, +1, 0 }) == FldsBC::AXIS) { - range = CreateRangePolicy({ domain.mesh.i_min(in::x1), - domain.mesh.i_min(in::x2), - domain.mesh.i_min(in::x3) }, - { domain.mesh.i_max(in::x1), - domain.mesh.i_max(in::x2) + 1, - domain.mesh.i_max(in::x3) }); + else if (dim == in::x2) + { + if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) + { + xg_min = m_metadomain.mesh().metric.template convert<2, Crd::Cd, Crd::Ph>( + static_cast(ig_min)); + xg_max = m_metadomain.mesh().metric.template convert<2, Crd::Cd, Crd::Ph>( + static_cast(ig_max)); + } + else + { + raise::Error("Invalid dimension", HERE); + } } + else if (dim == in::x3) + { + if constexpr (M::Dim == Dim::_3D) + { + xg_min = m_metadomain.mesh().metric.template convert<3, Crd::Cd, Crd::Ph>( + static_cast(ig_min)); + xg_max = m_metadomain.mesh().metric.template convert<3, Crd::Cd, Crd::Ph>( + static_cast(ig_max)); + } + else + { + raise::Error("Invalid dimension", HERE); + } + } + else + { + raise::Error("Invalid dimension", HERE); + } + return {sign, dim, xg_min, xg_max}; } - } - return range; - } - }; + + auto range_with_axis_BCs(const domain_t &domain) -> range_t + { + auto range = domain.mesh.rangeActiveCells(); + if constexpr (M::CoordType != Coord::Cart) + { + /** + * @brief taking one extra cell in the x2 direction if AXIS BCs + */ + if constexpr (M::Dim == Dim::_2D) + { + if (domain.mesh.flds_bc_in({0, +1}) == FldsBC::AXIS) + { + range = CreateRangePolicy( + {domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2)}, + {domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1}); + } + } + else if constexpr (M::Dim == Dim::_3D) + { + if (domain.mesh.flds_bc_in({0, +1, 0}) == FldsBC::AXIS) + { + range = CreateRangePolicy({domain.mesh.i_min(in::x1), + domain.mesh.i_min(in::x2), + domain.mesh.i_min(in::x3)}, + {domain.mesh.i_max(in::x1), + domain.mesh.i_max(in::x2) + 1, + domain.mesh.i_max(in::x3)}); + } + } + } + return range; + } + }; } // namespace ntt From 425abe5fcfbb040127e531c7e9d47efd6a6a7657 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Tue, 4 Mar 2025 13:22:38 -0600 Subject: [PATCH 052/183] Ongoing --- setups/srpic/shocktest/pgen.hpp | 156 ++++++++++++++++---------------- 1 file changed, 78 insertions(+), 78 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 1b8926c67..a1ce9a1a9 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -48,8 +48,8 @@ namespace user { Inline auto bx3(const coord_t& x_ph) const -> real_t { // return Bmag * math::sin(Btheta) * math::cos(Bphi); - return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); - // return ZERO; + // return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ZERO; } // electric field components @@ -59,8 +59,8 @@ namespace user { Inline auto ex2(const coord_t& x_ph) const -> real_t { // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); - // return ZERO; + // return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ZERO; } Inline auto ex3(const coord_t& x_ph) const -> real_t { @@ -120,82 +120,82 @@ namespace user { inline void InitPrtls(Domain& domain) { - // auto& species_e = domain.species[0]; - // auto& species_p = domain.species[1]; + auto& species_e = domain.species[0]; + auto& species_p = domain.species[1]; - // array_t elec_ind("elec_ind"); - // array_t pos_ind("pos_ind"); + array_t elec_ind("elec_ind"); + array_t pos_ind("pos_ind"); - // auto offset_e = species_e.npart(); - // auto offset_p = species_p.npart(); - - // auto ux1_e = species_e.ux1; - // auto ux2_e = species_e.ux2; - // auto ux3_e = species_e.ux3; - // auto i1_e = species_e.i1; - // auto i2_e = species_e.i2; - // auto dx1_e = species_e.dx1; - // auto dx2_e = species_e.dx2; - // auto phi_e = species_e.phi; - // auto weight_e = species_e.weight; - // auto tag_e = species_e.tag; - - // auto ux1_p = species_p.ux1; - // auto ux2_p = species_p.ux2; - // auto ux3_p = species_p.ux3; - // auto i1_p = species_p.i1; - // auto i2_p = species_p.i2; - // auto dx1_p = species_p.dx1; - // auto dx2_p = species_p.dx2; - // auto phi_p = species_p.phi; - // auto weight_p = species_p.weight; - // auto tag_p = species_p.tag; - - // int nseed = 1; - - // Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { - - // // ToDo: fix this - // auto i1_ = math::floor(10); - // auto i2_ = math::floor(64); - // auto dx1_ = HALF; - // auto dx2_ = HALF; - - - // auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); - // auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); - - // i1_e(elec_p + offset_e) = i1_; - // dx1_e(elec_p + offset_e) = dx1_; - // i2_e(elec_p + offset_e) = i2_; - // dx2_e(elec_p + offset_e) = dx2_; - // ux1_e(elec_p + offset_e) = -0.5; - // ux2_e(elec_p + offset_e) = 0.5; - // ux3_e(elec_p + offset_e) = ZERO; - // weight_e(elec_p + offset_e) = ONE; - // tag_e(elec_p + offset_e) = ParticleTag::alive; - - // i1_p(pos_p + offset_p) = i1_; - // dx1_p(pos_p + offset_p) = dx1_; - // i2_p(pos_p + offset_p) = i2_; - // dx2_p(pos_p + offset_p) = dx2_; - // ux1_p(pos_p + offset_p) = 0.5; - // ux2_p(pos_p + offset_p) = -0.5; - // ux3_p(pos_p + offset_p) = ZERO; - // weight_p(pos_p + offset_p) = ONE; - // tag_p(pos_p + offset_p) = ParticleTag::alive; - - - // }); - - - // auto elec_ind_h = Kokkos::create_mirror(elec_ind); - // Kokkos::deep_copy(elec_ind_h, elec_ind); - // species_e.set_npart(offset_e + elec_ind_h()); - - // auto pos_ind_h = Kokkos::create_mirror(pos_ind); - // Kokkos::deep_copy(pos_ind_h, pos_ind); - // species_p.set_npart(offset_p + pos_ind_h()); + auto offset_e = species_e.npart(); + auto offset_p = species_p.npart(); + + auto ux1_e = species_e.ux1; + auto ux2_e = species_e.ux2; + auto ux3_e = species_e.ux3; + auto i1_e = species_e.i1; + auto i2_e = species_e.i2; + auto dx1_e = species_e.dx1; + auto dx2_e = species_e.dx2; + auto phi_e = species_e.phi; + auto weight_e = species_e.weight; + auto tag_e = species_e.tag; + + auto ux1_p = species_p.ux1; + auto ux2_p = species_p.ux2; + auto ux3_p = species_p.ux3; + auto i1_p = species_p.i1; + auto i2_p = species_p.i2; + auto dx1_p = species_p.dx1; + auto dx2_p = species_p.dx2; + auto phi_p = species_p.phi; + auto weight_p = species_p.weight; + auto tag_p = species_p.tag; + + int nseed = 1; + + Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { + + // ToDo: fix this + auto i1_ = math::floor(10); + auto i2_ = math::floor(64); + auto dx1_ = HALF; + auto dx2_ = HALF; + + + auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); + auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); + + i1_e(elec_p + offset_e) = i1_; + dx1_e(elec_p + offset_e) = dx1_; + i2_e(elec_p + offset_e) = i2_; + dx2_e(elec_p + offset_e) = dx2_; + ux1_e(elec_p + offset_e) = -0.5; + ux2_e(elec_p + offset_e) = 0.5; + ux3_e(elec_p + offset_e) = ZERO; + weight_e(elec_p + offset_e) = ONE; + tag_e(elec_p + offset_e) = ParticleTag::alive; + + i1_p(pos_p + offset_p) = i1_; + dx1_p(pos_p + offset_p) = dx1_; + i2_p(pos_p + offset_p) = i2_; + dx2_p(pos_p + offset_p) = dx2_; + ux1_p(pos_p + offset_p) = 0.5; + ux2_p(pos_p + offset_p) = -0.5; + ux3_p(pos_p + offset_p) = ZERO; + weight_p(pos_p + offset_p) = ONE; + tag_p(pos_p + offset_p) = ParticleTag::alive; + + + }); + + + auto elec_ind_h = Kokkos::create_mirror(elec_ind); + Kokkos::deep_copy(elec_ind_h, elec_ind); + species_e.set_npart(offset_e + elec_ind_h()); + + auto pos_ind_h = Kokkos::create_mirror(pos_ind); + Kokkos::deep_copy(pos_ind_h, pos_ind); + species_p.set_npart(offset_p + pos_ind_h()); } From cfee51ecac9150e9c6c43668a83ed097f98a64ff Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Tue, 4 Mar 2025 15:06:18 -0600 Subject: [PATCH 053/183] Ongoing --- src/engines/srpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 1c7ffa869..21a44373d 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -918,7 +918,7 @@ namespace ntt { { Kokkos::deep_copy( Kokkos::subview(domain.fields.em, - std::make_pair(N_GHOSTS, N_GHOSTS), + std::make_pair(N_GHOSTS+1, N_GHOSTS+1), std::make_pair(xi_min[1], xi_max[1]), comp), ZERO); From 7d317e6bf1db35098a153bc8bccd5a5f90630500 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Tue, 4 Mar 2025 15:40:21 -0600 Subject: [PATCH 054/183] Ongoing --- src/engines/srpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 21a44373d..1c7ffa869 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -918,7 +918,7 @@ namespace ntt { { Kokkos::deep_copy( Kokkos::subview(domain.fields.em, - std::make_pair(N_GHOSTS+1, N_GHOSTS+1), + std::make_pair(N_GHOSTS, N_GHOSTS), std::make_pair(xi_min[1], xi_max[1]), comp), ZERO); From 21d99374744e333e5a6400058acab22347e0ecdd Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Tue, 4 Mar 2025 15:53:24 -0600 Subject: [PATCH 055/183] Ongoing --- src/engines/srpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 1c7ffa869..12872bf4e 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -918,7 +918,7 @@ namespace ntt { { Kokkos::deep_copy( Kokkos::subview(domain.fields.em, - std::make_pair(N_GHOSTS, N_GHOSTS), + std::make_pair(N_GHOSTS, N_GHOSTS+1), std::make_pair(xi_min[1], xi_max[1]), comp), ZERO); From bd723f3905bd3356b30faf6a8380106a1a890a9e Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 7 Mar 2025 11:30:47 -0500 Subject: [PATCH 056/183] minor reformatting of conductor BC --- input.example.toml | 2 +- setups/srpic/shocktest/pgen.hpp | 149 ++-- setups/srpic/shocktest/shock.toml | 26 +- src/engines/srpic.hpp | 1044 +++++++++++++---------------- src/global/enums.h | 19 +- src/global/tests/enums.cpp | 5 +- src/kernels/fields_bcs.hpp | 157 +++-- 7 files changed, 622 insertions(+), 780 deletions(-) diff --git a/input.example.toml b/input.example.toml index 788c30685..b327df54d 100644 --- a/input.example.toml +++ b/input.example.toml @@ -90,7 +90,7 @@ # Boundary conditions for fields: # @required # @type: 1/2/3-size array of string tuples, each of size 1 or 2 - # @valid: "PERIODIC", "MATCH", "FIXED", "ATMOSPHERE", "CUSTOM", "HORIZON" + # @valid: "PERIODIC", "MATCH", "FIXED", "ATMOSPHERE", "CUSTOM", "HORIZON", "CONDUCTOR" # @example: [["CUSTOM", "MATCH"]] (for 2D spherical [[rmin, rmax]]) # @note: When periodic in any of the directions, you should only set one value: [..., ["PERIODIC"], ...] # @note: In spherical, bondaries in theta/phi are set automatically (only specify bc @ [rmin, rmax]): [["ATMOSPHERE", "MATCH"]] diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index a1ce9a1a9..4f6decc76 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -14,6 +14,7 @@ #include "framework/domain/metadomain.h" #include +#include namespace user { using namespace ntt; @@ -48,7 +49,8 @@ namespace user { Inline auto bx3(const coord_t& x_ph) const -> real_t { // return Bmag * math::sin(Btheta) * math::cos(Bphi); - // return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + // return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + // + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); return ZERO; } @@ -59,7 +61,8 @@ namespace user { Inline auto ex2(const coord_t& x_ph) const -> real_t { // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - // return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + // return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + // + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); return ZERO; } @@ -88,17 +91,32 @@ namespace user { const real_t drift_ux, temperature; - const real_t Btheta, Bphi, Bmag; - InitFields init_flds; + const std::vector x1arr_e, x2arr_e, ux1arr_e, ux2arr_e, ux3arr_e; + const std::vector x1arr_i, x2arr_i, ux1arr_i, ux2arr_i, ux3arr_i; + + const real_t Btheta, Bphi, Bmag; + InitFields init_flds; + const Metadomain* metadomain; inline PGen(const SimulationParams& p, const Metadomain& m) : arch::ProblemGenerator { p } , drift_ux { p.template get("setup.drift_ux") } , temperature { p.template get("setup.temperature") } - , Bmag { p.template get("setup.Bmag", ZERO) } + , x1arr_e { p.template get>("setup.x_e") } + , x2arr_e { p.template get>("setup.y_e") } + , ux1arr_e { p.template get>("setup.ux_e") } + , ux2arr_e { p.template get>("setup.uy_e") } + , ux3arr_e { p.template get>("setup.uz_e") } + , x1arr_i { p.template get>("setup.x_i") } + , x2arr_i { p.template get>("setup.y_i") } + , ux1arr_i { p.template get>("setup.ux_i") } + , ux2arr_i { p.template get>("setup.uy_i") } + , ux3arr_i { p.template get>("setup.uz_i") } , Btheta { p.template get("setup.Btheta", ZERO) } + , Bmag { p.template get("setup.Bmag", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } - , init_flds { Bmag, Btheta, Bphi, drift_ux } {} + , init_flds { Bmag, Btheta, Bphi, drift_ux } + , metadomain { &m } {} inline PGen() {} @@ -113,111 +131,32 @@ namespace user { } } - auto MatchFields(real_t time) const -> InitFields { return init_flds; } inline void InitPrtls(Domain& domain) { - - auto& species_e = domain.species[0]; - auto& species_p = domain.species[1]; - - array_t elec_ind("elec_ind"); - array_t pos_ind("pos_ind"); - - auto offset_e = species_e.npart(); - auto offset_p = species_p.npart(); - - auto ux1_e = species_e.ux1; - auto ux2_e = species_e.ux2; - auto ux3_e = species_e.ux3; - auto i1_e = species_e.i1; - auto i2_e = species_e.i2; - auto dx1_e = species_e.dx1; - auto dx2_e = species_e.dx2; - auto phi_e = species_e.phi; - auto weight_e = species_e.weight; - auto tag_e = species_e.tag; - - auto ux1_p = species_p.ux1; - auto ux2_p = species_p.ux2; - auto ux3_p = species_p.ux3; - auto i1_p = species_p.i1; - auto i2_p = species_p.i2; - auto dx1_p = species_p.dx1; - auto dx2_p = species_p.dx2; - auto phi_p = species_p.phi; - auto weight_p = species_p.weight; - auto tag_p = species_p.tag; - - int nseed = 1; - - Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { - - // ToDo: fix this - auto i1_ = math::floor(10); - auto i2_ = math::floor(64); - auto dx1_ = HALF; - auto dx2_ = HALF; - - - auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); - auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); - - i1_e(elec_p + offset_e) = i1_; - dx1_e(elec_p + offset_e) = dx1_; - i2_e(elec_p + offset_e) = i2_; - dx2_e(elec_p + offset_e) = dx2_; - ux1_e(elec_p + offset_e) = -0.5; - ux2_e(elec_p + offset_e) = 0.5; - ux3_e(elec_p + offset_e) = ZERO; - weight_e(elec_p + offset_e) = ONE; - tag_e(elec_p + offset_e) = ParticleTag::alive; - - i1_p(pos_p + offset_p) = i1_; - dx1_p(pos_p + offset_p) = dx1_; - i2_p(pos_p + offset_p) = i2_; - dx2_p(pos_p + offset_p) = dx2_; - ux1_p(pos_p + offset_p) = 0.5; - ux2_p(pos_p + offset_p) = -0.5; - ux3_p(pos_p + offset_p) = ZERO; - weight_p(pos_p + offset_p) = ONE; - tag_p(pos_p + offset_p) = ParticleTag::alive; - - - }); - - - auto elec_ind_h = Kokkos::create_mirror(elec_ind); - Kokkos::deep_copy(elec_ind_h, elec_ind); - species_e.set_npart(offset_e + elec_ind_h()); - - auto pos_ind_h = Kokkos::create_mirror(pos_ind); - Kokkos::deep_copy(pos_ind_h, pos_ind); - species_p.set_npart(offset_p + pos_ind_h()); - - + arch::InjectGlobally(*metadomain, + domain, + 1, + { + { "x1", x1arr_e }, + { "x2", x2arr_e }, + { "ux1", ux1arr_e }, + { "ux2", ux1arr_e }, + { "ux3", ux3arr_e } + }); + arch::InjectGlobally(*metadomain, + domain, + 2, + { + { "x1", x1arr_i }, + { "x2", x2arr_i }, + { "ux1", ux1arr_i }, + { "ux2", ux1arr_i }, + { "ux3", ux3arr_i } + }); } - - - // inline void InitPrtls(Domain& local_domain) { - // const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, - // local_domain.random_pool, - // temperature, - // -drift_ux, - // in::x1); - - // const auto injector = arch::UniformInjector( - // energy_dist, - // { 1, 2 }); - // arch::InjectUniform>( - // params, - // local_domain, - // injector, - // 1.0); - // } - }; } // namespace user diff --git a/setups/srpic/shocktest/shock.toml b/setups/srpic/shocktest/shock.toml index a77f5c2e9..6c0c9a3a0 100644 --- a/setups/srpic/shocktest/shock.toml +++ b/setups/srpic/shocktest/shock.toml @@ -11,7 +11,7 @@ metric = "minkowski" [grid.boundaries] - fields = [["CONDUCTOR", "FIXED"], ["PERIODIC"]] + fields = [["CONDUCTOR", "FIXED"], ["PERIODIC"]] particles = [["REFLECT", "ABSORB"], ["PERIODIC"]] [scales] @@ -20,8 +20,8 @@ [algorithms] current_filters = 8 - fieldsolver = "false" - deposit = "false" + fieldsolver = "false" + deposit = "false" [algorithms.timestep] CFL = 0.5 @@ -44,13 +44,23 @@ [setup] drift_ux = 0.1 temperature = 1e-3 - Bmag = 1.0 - Btheta = 0.0 - Bphi = 0.0 + Bmag = 1.0 + Btheta = 0.0 + Bphi = 0.0 + x_e = [0.05] + y_e = [0.0] + ux_e = [-0.01] + uy_e = [0.01] + uz_e = [0.001] + x_i = [0.05] + y_i = [0.0] + ux_i = [0.01] + uy_i = [-0.01] + uz_i = [-0.001] [output] - interval = 1 - format = "hdf5" + interval = 1 + format = "hdf5" [output.fields] quantities = ["N_1", "N_2", "E", "B", "T0i_1", "T0i_2", "J"] diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 12872bf4e..dd2f5ccec 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -839,353 +839,284 @@ namespace ntt { } void PerfectConductorFieldsIn(dir::direction_t direction, - domain_t& domain, - BCTags tags) { + domain_t& domain, + BCTags tags) { /** * perfect conductor field boundaries */ - const auto sign = direction.get_sign(); - const auto dim = direction.get_dim(); - raise::ErrorIf(dim != in::x1 and M::CoordType != Coord::Cart, - "Perfect conductor BCs only implemented for x1 in " - "non-cartesian coordinates", - HERE); + if constexpr (M::CoordType != Coord::Cart) { + (void)direction; + (void)domain; + (void)tags; + raise::Error( + "Perfect conductor BCs only applicable to cartesian coordinates", + HERE); + } else { + const auto sign = direction.get_sign(); + const auto dim = direction.get_dim(); + raise::ErrorIf(dim != in::x1, + "Perfect conductor BCs only implemented for x1", + HERE); + std::vector xi_min, xi_max; - std::vector xi_min, xi_max; + const std::vector all_dirs { in::x1, in::x2, in::x3 }; - const std::vector all_dirs { in::x1, in::x2, in::x3 }; - - for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { - const auto dd = all_dirs[d]; - if (dim == dd) { + for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { + const auto dd = all_dirs[d]; + if (dim == dd) { xi_min.push_back(0); - xi_max.push_back(N_GHOSTS); + xi_max.push_back(N_GHOSTS + 1); + } else { + xi_min.push_back(0); + xi_max.push_back(domain.mesh.n_all(dd)); + } + } + raise::ErrorIf(xi_min.size() != xi_max.size() or + xi_min.size() != static_cast(M::Dim), + "Invalid range size", + HERE); + + range_t range; + if constexpr (M::Dim == Dim::_1D) { + range = CreateRangePolicy({ xi_min[0] }, { xi_max[0] }); + } else if constexpr (M::Dim == Dim::_2D) { + range = CreateRangePolicy({ xi_min[0], xi_min[1] }, + { xi_max[0], xi_max[1] }); + } else if constexpr (M::Dim == Dim::_3D) { + range = CreateRangePolicy({ xi_min[0], xi_min[1], xi_min[2] }, + { xi_max[0], xi_max[1], xi_max[2] }); } else { - xi_min.push_back(0); - xi_max.push_back(domain.mesh.n_all(dd)); + raise::Error("Invalid dimension", HERE); } - } - raise::ErrorIf(xi_min.size() != xi_max.size() or - xi_min.size() != static_cast(M::Dim), - "Invalid range size", - HERE); - - - if constexpr (M::Dim == Dim::_1D) - { Kokkos::parallel_for( - "MatchFields", - CreateRangePolicy({xi_min[0]}, {xi_max[0]}), - kernel::bc::ConductorBoundaries_kernel( - domain.fields.em, - tags)); + "MatchFields", + range, + kernel::bc::ConductorBoundaries_kernel(domain.fields.em, + tags)); + + // if constexpr (M::Dim == Dim::_1D) { + // Kokkos::parallel_for( + // "MatchFields", + // CreateRangePolicy({ xi_min[0] }, { xi_max[0] }), + // kernel::bc::ConductorBoundaries_kernel(domain.fields.em, + // tags)); + // } else if constexpr (M::Dim == Dim::_2D) { + // Kokkos::parallel_for( + // "MatchFields", + // CreateRangePolicy({ xi_min[0], xi_min[1] }, + // { xi_max[0], xi_max[1] }), + // kernel::bc::ConductorBoundaries_kernel(domain.fields.em, + // tags)); + // } else if constexpr (M::Dim == Dim::_3D) { + // Kokkos::parallel_for( + // "MatchFields", + // CreateRangePolicy({ xi_min[0], xi_min[1], xi_min[2] }, + // { xi_max[0], xi_max[1], xi_max[2] }), + // kernel::bc::ConductorBoundaries_kernel(domain.fields.em, + // tags)); + // } else { + // raise::Error("Invalid dimension", HERE); + // } } + } - if constexpr (M::Dim == Dim::_2D) - { - Kokkos::parallel_for( - "MatchFields", - CreateRangePolicy({xi_min[0], xi_min[1]}, {xi_max[0], xi_max[1]}), - kernel::bc::ConductorBoundaries_kernel( + void AtmosphereFieldsIn(dir::direction_t direction, + domain_t& domain, + BCTags tags) { + /** + * atmosphere field boundaries + */ + if constexpr (traits::has_member::value) { + const auto [sign, dim, xg_min, xg_max] = get_atm_extent(direction); + const auto dd = static_cast(dim); + boundaries_t box; + boundaries_t incl_ghosts; + for (unsigned short d { 0 }; d < M::Dim; ++d) { + if (d == dd) { + box.push_back({ xg_min, xg_max }); + if (sign > 0) { + incl_ghosts.push_back({ false, true }); + } else { + incl_ghosts.push_back({ true, false }); + } + } else { + box.push_back(Range::All); + incl_ghosts.push_back({ true, true }); + } + } + if (not domain.mesh.Intersects(box)) { + return; + } + const auto intersect_range = domain.mesh.ExtentToRange(box, incl_ghosts); + tuple_t range_min { 0 }; + tuple_t range_max { 0 }; + + for (unsigned short d { 0 }; d < M::Dim; ++d) { + range_min[d] = intersect_range[d].first; + range_max[d] = intersect_range[d].second; + } + auto atm_fields = m_pgen.AtmFields(time); + std::size_t il_edge; + if (sign > 0) { + il_edge = range_min[dd] - N_GHOSTS; + } else { + il_edge = range_max[dd] - 1 - N_GHOSTS; + } + const auto range = CreateRangePolicy(range_min, range_max); + if (dim == in::x1) { + if (sign > 0) { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, tags)); - } - - if constexpr (M::Dim == Dim::_3D) - { - Kokkos::parallel_for( - "MatchFields", - CreateRangePolicy({xi_min[0], xi_min[1], xi_min[2]}, {xi_max[0], xi_max[1], xi_max[2]}), - kernel::bc::ConductorBoundaries_kernel( + } else { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, tags)); - } - - // these components need to be set to zero at the boundary - std::vector comps = {em::ex2, em::ex3, em::bx1}; - for (const auto &comp : comps) - { - if constexpr (M::Dim == Dim::_1D) - { - Kokkos::deep_copy( - Kokkos::subview(domain.fields.em, - std::make_pair(N_GHOSTS, N_GHOSTS), - comp), - ZERO); - } - else if constexpr (M::Dim == Dim::_2D) - { - Kokkos::deep_copy( - Kokkos::subview(domain.fields.em, - std::make_pair(N_GHOSTS, N_GHOSTS+1), - std::make_pair(xi_min[1], xi_max[1]), - comp), - ZERO); - } - else if constexpr (M::Dim == Dim::_3D) - { - Kokkos::deep_copy( - Kokkos::subview(domain.fields.em, - std::make_pair(N_GHOSTS, N_GHOSTS), - std::make_pair(xi_min[1], xi_max[1]), - std::make_pair(xi_min[2], xi_max[2]), - comp), - ZERO); - } - else - { + } + } else if (dim == in::x2) { + if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { + if (sign > 0) { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } else { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } + } else { + raise::Error("Invalid dimension", HERE); + } + } else if (dim == in::x3) { + if constexpr (M::Dim == Dim::_3D) { + if (sign > 0) { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } else { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } + } else { + raise::Error("Invalid dimension", HERE); + } + } else { raise::Error("Invalid dimension", HERE); } + } else { + raise::Error("Atm fields not implemented in PGEN for atmosphere BCs", HERE); } } - void AtmosphereFieldsIn(dir::direction_t direction, - domain_t & domain, - BCTags tags) - { - /** - * atmosphere field boundaries - */ - if constexpr (traits::has_member::value) - { - const auto [sign, dim, xg_min, xg_max] = get_atm_extent(direction); - const auto dd = static_cast(dim); - boundaries_t box; - boundaries_t incl_ghosts; - for (unsigned short d{0}; d < M::Dim; ++d) - { - if (d == dd) - { - box.push_back({xg_min, xg_max}); - if (sign > 0) - { - incl_ghosts.push_back({false, true}); - } - else - { - incl_ghosts.push_back({true, false}); - } - } - else - { - box.push_back(Range::All); - incl_ghosts.push_back({true, true}); - } - } - if (not domain.mesh.Intersects(box)) - { - return; - } - const auto intersect_range = domain.mesh.ExtentToRange(box, incl_ghosts); - tuple_t range_min{0}; - tuple_t range_max{0}; - - for (unsigned short d{0}; d < M::Dim; ++d) - { - range_min[d] = intersect_range[d].first; - range_max[d] = intersect_range[d].second; - } - auto atm_fields = m_pgen.AtmFields(time); - std::size_t il_edge; - if (sign > 0) - { - il_edge = range_min[dd] - N_GHOSTS; - } - else - { - il_edge = range_max[dd] - 1 - N_GHOSTS; - } - const auto range = CreateRangePolicy(range_min, range_max); - if (dim == in::x1) - { - if (sign > 0) - { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } - else - { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } - } - else if (dim == in::x2) - { - if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) - { - if (sign > 0) - { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } - else - { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } - } - else - { - raise::Error("Invalid dimension", HERE); - } - } - else if (dim == in::x3) - { - if constexpr (M::Dim == Dim::_3D) - { - if (sign > 0) - { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } - else - { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } - } - else - { - raise::Error("Invalid dimension", HERE); - } - } - else - { - raise::Error("Invalid dimension", HERE); - } - } - else - { - raise::Error("Atm fields not implemented in PGEN for atmosphere BCs", HERE); - } - } + void CustomFieldsIn(dir::direction_t direction, + domain_t& domain, + BCTags tags) { + (void)direction; + (void)domain; + (void)tags; + raise::Error("Custom boundaries not implemented", HERE); + // if constexpr ( + // traits::has_member::value) { + // const auto [box, custom_fields] = m_pgen.CustomFields(time); + // if (domain.mesh.Intersects(box)) { + // } + // + // } else { + // raise::Error("Custom boundaries not implemented", HERE); + // } + } - void CustomFieldsIn(dir::direction_t direction, - domain_t & domain, - BCTags tags) - { - (void)direction; - (void)domain; - (void)tags; - raise::Error("Custom boundaries not implemented", HERE); - // if constexpr ( - // traits::has_member::value) { - // const auto [box, custom_fields] = m_pgen.CustomFields(time); - // if (domain.mesh.Intersects(box)) { - // } - // - // } else { - // raise::Error("Custom boundaries not implemented", HERE); - // } + void AtmosphereParticlesIn(const dir::direction_t& direction, + domain_t& domain, + InjTags tags) { + const auto [sign, dim, xg_min, xg_max] = get_atm_extent(direction); + + const auto x_surf = sign > 0 ? xg_min : xg_max; + const auto ds = m_params.template get( + "grid.boundaries.atmosphere.ds"); + const auto temp = m_params.template get( + "grid.boundaries.atmosphere.temperature"); + const auto height = m_params.template get( + "grid.boundaries.atmosphere.height"); + const auto species = + m_params.template get>( + "grid.boundaries.atmosphere.species"); + const auto nmax = m_params.template get( + "grid.boundaries.atmosphere.density"); + + Kokkos::deep_copy(domain.fields.bckp, ZERO); + auto scatter_bckp = Kokkos::Experimental::create_scatter_view( + domain.fields.bckp); + const auto use_weights = M::CoordType != Coord::Cart; + const auto ni2 = domain.mesh.n_active(in::x2); + const auto inv_n0 = ONE / m_params.template get("scales.n0"); + + // compute the density of the two species + if (tags & Inj::AssumeEmpty) { + if constexpr (M::Dim == Dim::_1D) { + Kokkos::deep_copy( + Kokkos::subview(domain.fields.bckp, Kokkos::ALL, std::make_pair(0, 1)), + ZERO); + } else if constexpr (M::Dim == Dim::_2D) { + Kokkos::deep_copy(Kokkos::subview(domain.fields.bckp, + Kokkos::ALL, + Kokkos::ALL, + std::make_pair(0, 1)), + ZERO); + } else if constexpr (M::Dim == Dim::_3D) { + Kokkos::deep_copy(Kokkos::subview(domain.fields.bckp, + Kokkos::ALL, + Kokkos::ALL, + Kokkos::ALL, + std::make_pair(0, 1)), + ZERO); } - - void AtmosphereParticlesIn(const dir::direction_t &direction, - domain_t &domain, - InjTags tags) - { - const auto [sign, dim, xg_min, xg_max] = get_atm_extent(direction); - - const auto x_surf = sign > 0 ? xg_min : xg_max; - const auto ds = m_params.template get( - "grid.boundaries.atmosphere.ds"); - const auto temp = m_params.template get( - "grid.boundaries.atmosphere.temperature"); - const auto height = m_params.template get( - "grid.boundaries.atmosphere.height"); - const auto species = - m_params.template get>( - "grid.boundaries.atmosphere.species"); - const auto nmax = m_params.template get( - "grid.boundaries.atmosphere.density"); - - Kokkos::deep_copy(domain.fields.bckp, ZERO); - auto scatter_bckp = Kokkos::Experimental::create_scatter_view( - domain.fields.bckp); - const auto use_weights = M::CoordType != Coord::Cart; - const auto ni2 = domain.mesh.n_active(in::x2); - const auto inv_n0 = ONE / m_params.template get("scales.n0"); - - // compute the density of the two species - if (tags & Inj::AssumeEmpty) - { - if constexpr (M::Dim == Dim::_1D) - { - Kokkos::deep_copy( - Kokkos::subview(domain.fields.bckp, Kokkos::ALL, std::make_pair(0, 1)), - ZERO); - } - else if constexpr (M::Dim == Dim::_2D) - { - Kokkos::deep_copy(Kokkos::subview(domain.fields.bckp, - Kokkos::ALL, - Kokkos::ALL, - std::make_pair(0, 1)), - ZERO); - } - else if constexpr (M::Dim == Dim::_3D) - { - Kokkos::deep_copy(Kokkos::subview(domain.fields.bckp, - Kokkos::ALL, - Kokkos::ALL, - Kokkos::ALL, - std::make_pair(0, 1)), - ZERO); - } + } else { + for (const auto& sp : + std::vector({ species.first, species.second })) { + auto& prtl_spec = domain.species[sp - 1]; + if (prtl_spec.npart() == 0) { + continue; } - else - { - for (const auto &sp : - std::vector({species.first, species.second})) - { - auto &prtl_spec = domain.species[sp - 1]; - if (prtl_spec.npart() == 0) - { - continue; - } - // clang-format off + // clang-format off Kokkos::parallel_for( "ComputeMoments", prtl_spec.rangeActiveParticles(), @@ -1199,265 +1130,230 @@ namespace ntt { use_weights, domain.mesh.metric, domain.mesh.flds_bc(), ni2, inv_n0, 0)); - // clang-format on - prtl_spec.set_unsorted(); - } - Kokkos::Experimental::contribute(domain.fields.bckp, scatter_bckp); - m_metadomain.SynchronizeFields(domain, Comm::Bckp, {0, 1}); - } + // clang-format on + prtl_spec.set_unsorted(); + } + Kokkos::Experimental::contribute(domain.fields.bckp, scatter_bckp); + m_metadomain.SynchronizeFields(domain, Comm::Bckp, { 0, 1 }); + } - if (dim == in::x1) - { - if (sign > 0) - { - const auto atm_injector = - arch::AtmosphereInjector{ - domain.mesh.metric, - domain.fields.bckp, - nmax, - height, - x_surf, - ds, - temp, - domain.random_pool, - species}; - arch::InjectNonUniform(m_params, - domain, - atm_injector, - nmax, - use_weights); - } - else - { - const auto atm_injector = - arch::AtmosphereInjector{ - domain.mesh.metric, - domain.fields.bckp, - nmax, - height, - x_surf, - ds, - temp, - domain.random_pool, - species}; - arch::InjectNonUniform(m_params, - domain, - atm_injector, - nmax, - use_weights); - } - } - else if (dim == in::x2) - { - if (sign > 0) - { - const auto atm_injector = - arch::AtmosphereInjector{ - domain.mesh.metric, - domain.fields.bckp, - nmax, - height, - x_surf, - ds, - temp, - domain.random_pool, - species}; - arch::InjectNonUniform(m_params, - domain, - atm_injector, - nmax, - use_weights); - } - else - { - const auto atm_injector = - arch::AtmosphereInjector{ - domain.mesh.metric, - domain.fields.bckp, - nmax, - height, - x_surf, - ds, - temp, - domain.random_pool, - species}; - arch::InjectNonUniform(m_params, - domain, - atm_injector, - nmax, - use_weights); - } - } - else if (dim == in::x3) - { - if (sign > 0) - { - const auto atm_injector = - arch::AtmosphereInjector{ - domain.mesh.metric, - domain.fields.bckp, - nmax, - height, - x_surf, - ds, - temp, - domain.random_pool, - species}; - arch::InjectNonUniform(m_params, - domain, - atm_injector, - nmax, - use_weights); - } - else - { - const auto atm_injector = - arch::AtmosphereInjector{ - domain.mesh.metric, - domain.fields.bckp, - nmax, - height, - x_surf, - ds, - temp, - domain.random_pool, - species}; - arch::InjectNonUniform(m_params, - domain, - atm_injector, - nmax, - use_weights); - } - } - else - { - raise::Error("Invalid dimension", HERE); - } - return; + if (dim == in::x1) { + if (sign > 0) { + const auto atm_injector = + arch::AtmosphereInjector { + domain.mesh.metric, + domain.fields.bckp, + nmax, + height, + x_surf, + ds, + temp, + domain.random_pool, + species + }; + arch::InjectNonUniform(m_params, + domain, + atm_injector, + nmax, + use_weights); + } else { + const auto atm_injector = + arch::AtmosphereInjector { + domain.mesh.metric, + domain.fields.bckp, + nmax, + height, + x_surf, + ds, + temp, + domain.random_pool, + species + }; + arch::InjectNonUniform(m_params, + domain, + atm_injector, + nmax, + use_weights); } + } else if (dim == in::x2) { + if (sign > 0) { + const auto atm_injector = + arch::AtmosphereInjector { + domain.mesh.metric, + domain.fields.bckp, + nmax, + height, + x_surf, + ds, + temp, + domain.random_pool, + species + }; + arch::InjectNonUniform(m_params, + domain, + atm_injector, + nmax, + use_weights); + } else { + const auto atm_injector = + arch::AtmosphereInjector { + domain.mesh.metric, + domain.fields.bckp, + nmax, + height, + x_surf, + ds, + temp, + domain.random_pool, + species + }; + arch::InjectNonUniform(m_params, + domain, + atm_injector, + nmax, + use_weights); + } + } else if (dim == in::x3) { + if (sign > 0) { + const auto atm_injector = + arch::AtmosphereInjector { + domain.mesh.metric, + domain.fields.bckp, + nmax, + height, + x_surf, + ds, + temp, + domain.random_pool, + species + }; + arch::InjectNonUniform(m_params, + domain, + atm_injector, + nmax, + use_weights); + } else { + const auto atm_injector = + arch::AtmosphereInjector { + domain.mesh.metric, + domain.fields.bckp, + nmax, + height, + x_surf, + ds, + temp, + domain.random_pool, + species + }; + arch::InjectNonUniform(m_params, + domain, + atm_injector, + nmax, + use_weights); + } + } else { + raise::Error("Invalid dimension", HERE); + } + return; + } - private: - /** - * @brief Get the buffer region of the atmosphere and the direction - * @param direction direction in which the atmosphere is applied - * @return tuple: [sign of the direction, the direction (as in::), the min and max extent - * @note xg_min and xg_max are the extents where the fields are set, not the atmosphere itself - * @note i.e. - * - * fields set particles injected - * ghost zone | | - * v v v - * |....|...........|*******************..... -> x1 - * ^ ^ - * xg_min xg_max - * | | | - * |<-- buffer -->|<-- atmosphere -->| - * - * in this case the function returns { -1, in::x1, xg_min, xg_max } - */ - auto get_atm_extent(dir::direction_t direction) const - -> std::tuple - { - const auto sign = direction.get_sign(); - const auto dim = direction.get_dim(); - const auto min_buff = m_params.template get( - "algorithms.current_filters") + - 2; - const auto buffer_ncells = min_buff > 5 ? min_buff : 5; - if (M::CoordType != Coord::Cart and (dim != in::x1 or sign > 0)) - { - raise::Error("For non-cartesian coordinates atmosphere BCs is " - "possible only in -x1 (@ rmin)", - HERE); - } - real_t xg_min{ZERO}, xg_max{ZERO}; - std::size_t ig_min, ig_max; - if (sign > 0) - { // + direction - ig_min = m_metadomain.mesh().n_active(dim) - buffer_ncells; - ig_max = m_metadomain.mesh().n_active(dim); - } - else - { // - direction - ig_min = 0; - ig_max = buffer_ncells; - } + private: + /** + * @brief Get the buffer region of the atmosphere and the direction + * @param direction direction in which the atmosphere is applied + * @return tuple: [sign of the direction, the direction (as in::), the min and max extent + * @note xg_min and xg_max are the extents where the fields are set, not the atmosphere itself + * @note i.e. + * + * fields set particles injected + * ghost zone | | + * v v v + * |....|...........|*******************..... -> x1 + * ^ ^ + * xg_min xg_max + * | | | + * |<-- buffer -->|<-- atmosphere -->| + * + * in this case the function returns { -1, in::x1, xg_min, xg_max } + */ + auto get_atm_extent(dir::direction_t direction) const + -> std::tuple { + const auto sign = direction.get_sign(); + const auto dim = direction.get_dim(); + const auto min_buff = m_params.template get( + "algorithms.current_filters") + + 2; + const auto buffer_ncells = min_buff > 5 ? min_buff : 5; + if (M::CoordType != Coord::Cart and (dim != in::x1 or sign > 0)) { + raise::Error("For non-cartesian coordinates atmosphere BCs is " + "possible only in -x1 (@ rmin)", + HERE); + } + real_t xg_min { ZERO }, xg_max { ZERO }; + std::size_t ig_min, ig_max; + if (sign > 0) { // + direction + ig_min = m_metadomain.mesh().n_active(dim) - buffer_ncells; + ig_max = m_metadomain.mesh().n_active(dim); + } else { // - direction + ig_min = 0; + ig_max = buffer_ncells; + } - if (dim == in::x1) - { - xg_min = m_metadomain.mesh().metric.template convert<1, Crd::Cd, Crd::Ph>( - static_cast(ig_min)); - xg_max = m_metadomain.mesh().metric.template convert<1, Crd::Cd, Crd::Ph>( - static_cast(ig_max)); - } - else if (dim == in::x2) - { - if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) - { - xg_min = m_metadomain.mesh().metric.template convert<2, Crd::Cd, Crd::Ph>( - static_cast(ig_min)); - xg_max = m_metadomain.mesh().metric.template convert<2, Crd::Cd, Crd::Ph>( - static_cast(ig_max)); - } - else - { - raise::Error("Invalid dimension", HERE); - } - } - else if (dim == in::x3) - { - if constexpr (M::Dim == Dim::_3D) - { - xg_min = m_metadomain.mesh().metric.template convert<3, Crd::Cd, Crd::Ph>( - static_cast(ig_min)); - xg_max = m_metadomain.mesh().metric.template convert<3, Crd::Cd, Crd::Ph>( - static_cast(ig_max)); - } - else - { - raise::Error("Invalid dimension", HERE); - } - } - else - { - raise::Error("Invalid dimension", HERE); - } - return {sign, dim, xg_min, xg_max}; + if (dim == in::x1) { + xg_min = m_metadomain.mesh().metric.template convert<1, Crd::Cd, Crd::Ph>( + static_cast(ig_min)); + xg_max = m_metadomain.mesh().metric.template convert<1, Crd::Cd, Crd::Ph>( + static_cast(ig_max)); + } else if (dim == in::x2) { + if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { + xg_min = m_metadomain.mesh().metric.template convert<2, Crd::Cd, Crd::Ph>( + static_cast(ig_min)); + xg_max = m_metadomain.mesh().metric.template convert<2, Crd::Cd, Crd::Ph>( + static_cast(ig_max)); + } else { + raise::Error("Invalid dimension", HERE); + } + } else if (dim == in::x3) { + if constexpr (M::Dim == Dim::_3D) { + xg_min = m_metadomain.mesh().metric.template convert<3, Crd::Cd, Crd::Ph>( + static_cast(ig_min)); + xg_max = m_metadomain.mesh().metric.template convert<3, Crd::Cd, Crd::Ph>( + static_cast(ig_max)); + } else { + raise::Error("Invalid dimension", HERE); } + } else { + raise::Error("Invalid dimension", HERE); + } + return { sign, dim, xg_min, xg_max }; + } - auto range_with_axis_BCs(const domain_t &domain) -> range_t - { - auto range = domain.mesh.rangeActiveCells(); - if constexpr (M::CoordType != Coord::Cart) - { - /** - * @brief taking one extra cell in the x2 direction if AXIS BCs - */ - if constexpr (M::Dim == Dim::_2D) - { - if (domain.mesh.flds_bc_in({0, +1}) == FldsBC::AXIS) - { - range = CreateRangePolicy( - {domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2)}, - {domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1}); - } - } - else if constexpr (M::Dim == Dim::_3D) - { - if (domain.mesh.flds_bc_in({0, +1, 0}) == FldsBC::AXIS) - { - range = CreateRangePolicy({domain.mesh.i_min(in::x1), - domain.mesh.i_min(in::x2), - domain.mesh.i_min(in::x3)}, - {domain.mesh.i_max(in::x1), - domain.mesh.i_max(in::x2) + 1, - domain.mesh.i_max(in::x3)}); - } - } + auto range_with_axis_BCs(const domain_t& domain) -> range_t { + auto range = domain.mesh.rangeActiveCells(); + if constexpr (M::CoordType != Coord::Cart) { + /** + * @brief taking one extra cell in the x2 direction if AXIS BCs + */ + if constexpr (M::Dim == Dim::_2D) { + if (domain.mesh.flds_bc_in({ 0, +1 }) == FldsBC::AXIS) { + range = CreateRangePolicy( + { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) }, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1 }); + } + } else if constexpr (M::Dim == Dim::_3D) { + if (domain.mesh.flds_bc_in({ 0, +1, 0 }) == FldsBC::AXIS) { + range = CreateRangePolicy({ domain.mesh.i_min(in::x1), + domain.mesh.i_min(in::x2), + domain.mesh.i_min(in::x3) }, + { domain.mesh.i_max(in::x1), + domain.mesh.i_max(in::x2) + 1, + domain.mesh.i_max(in::x3) }); } - return range; } - }; + } + return range; + } + }; } // namespace ntt diff --git a/src/global/enums.h b/src/global/enums.h index 2b3bf5936..32ef17157 100644 --- a/src/global/enums.h +++ b/src/global/enums.h @@ -9,7 +9,7 @@ * - enum ntt::PrtlBC // periodic, absorb, atmosphere, custom, * reflect, horizon, axis, sync * - enum ntt::FldsBC // periodic, match, fixed, atmosphere, - * custom, horizon, axis, sync + * custom, horizon, axis, conductor, sync * - enum ntt::PrtlPusher // boris, vay, photon, none * - enum ntt::Cooling // synchrotron, none * - enum ntt::FldsID // e, dive, d, divd, b, h, j, @@ -221,17 +221,20 @@ namespace ntt { CUSTOM = 5, HORIZON = 6, AXIS = 7, - SYNC = 8, // <- SYNC means synchronization with other domains - CONDUCTOR = 9 + CONDUCTOR = 8, + SYNC = 9 // <- SYNC means synchronization with other domains }; constexpr FldsBC(uint8_t c) : enums_hidden::BaseEnum { c } {} - static constexpr type variants[] = { PERIODIC, MATCH, FIXED, ATMOSPHERE, - CUSTOM, HORIZON, AXIS, SYNC, CONDUCTOR }; - static constexpr const char* lookup[] = { "periodic", "match", "fixed", - "atmosphere", "custom", "horizon", - "axis", "sync", "conductor"}; + static constexpr type variants[] = { + PERIODIC, MATCH, FIXED, ATMOSPHERE, CUSTOM, + HORIZON, AXIS, CONDUCTOR, SYNC, + }; + static constexpr const char* lookup[] = { + "periodic", "match", "fixed", "atmosphere", "custom", + "horizon", "axis", "conductor", "sync" + }; static constexpr std::size_t total = sizeof(variants) / sizeof(variants[0]); }; diff --git a/src/global/tests/enums.cpp b/src/global/tests/enums.cpp index 7785ec1a3..b26a2050f 100644 --- a/src/global/tests/enums.cpp +++ b/src/global/tests/enums.cpp @@ -61,8 +61,9 @@ auto main() -> int { enum_str_t all_simulation_engines = { "srpic", "grpic" }; enum_str_t all_particle_bcs = { "periodic", "absorb", "atmosphere", "custom", "reflect", "horizon", "axis", "sync" }; - enum_str_t all_fields_bcs = { "periodic", "match", "fixed", "atmosphere", - "custom", "horizon", "axis", "sync" }; + enum_str_t all_fields_bcs = { "periodic", "match", "fixed", + "atmosphere", "custom", "horizon", + "axis", "conductor", "sync" }; enum_str_t all_particle_pushers = { "boris", "vay", "photon", "none" }; enum_str_t all_coolings = { "synchrotron", "none" }; diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 720408d05..9d8e9c74e 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -5,6 +5,7 @@ * - kernel::bc::MatchBoundaries_kernel<> * - kernel::bc::AxisBoundaries_kernel<> * - kernel::bc::EnforcedBoundaries_kernel<> + * - kernel::bc::ConductorBoundaries_kernel<> * @namespaces: * - kernel::bc:: */ @@ -485,43 +486,40 @@ namespace kernel::bc { } }; - template + template struct ConductorBoundaries_kernel { - static_assert(M::is_metric, "M must be a metric class"); - static_assert(static_cast(o) < - static_cast(M::Dim), + static_assert(static_cast(o) < static_cast(D), "Invalid component index"); - static constexpr idx_t i = static_cast(o) + 1u; + // static constexpr idx_t i = static_cast(o) + 1u; - ndfield_t Fld; - const BCTags tags; + ndfield_t Fld; + const BCTags tags; - ConductorBoundaries_kernel(ndfield_t Fld, - BCTags tags) + ConductorBoundaries_kernel(ndfield_t Fld, BCTags tags) : Fld { Fld } , tags { tags } {} Inline void operator()(index_t i1) const { - if constexpr (M::Dim == Dim::_1D) { - - if constexpr (S == SimEngine::SRPIC) { - + if constexpr (D == Dim::_1D) { if (tags & BC::E) { - Fld((N_GHOSTS-1)-i1, em::ex1) = Fld(N_GHOSTS+i1, em::ex1); - Fld((N_GHOSTS-1)-i1, em::ex2) = -Fld(N_GHOSTS+i1+1, em::ex2); - Fld((N_GHOSTS-1)-i1, em::ex3) = -Fld(N_GHOSTS+i1+1, em::ex3); - } - - if (tags & BC::B) - { - Fld((N_GHOSTS-1)-i1, em::bx1) = -Fld(N_GHOSTS+i1+1, em::bx1); - Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); - Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); + if (i1 == 0) { + Fld(N_GHOSTS, em::ex2) = ZERO; + Fld(N_GHOSTS, em::ex3) = ZERO; + } else { + Fld(N_GHOSTS - i1, em::ex1) = Fld(N_GHOSTS + i1 - 1, em::ex1); + Fld(N_GHOSTS - i1, em::ex2) = -Fld(N_GHOSTS + i1, em::ex2); + Fld(N_GHOSTS - i1, em::ex3) = -Fld(N_GHOSTS + i1, em::ex3); + } } - } else { - // GRPIC - raise::KernelError(HERE, "1D GRPIC not implemented"); + if (tags & BC::B) { + if (i1 == 0) { + Fld(N_GHOSTS, em::bx1) = ZERO; + } else { + Fld(N_GHOSTS - i1, em::bx1) = -Fld(N_GHOSTS + i1, em::bx1); + Fld(N_GHOSTS - i1, em::bx2) = Fld(N_GHOSTS + i1 - 1, em::bx2); + Fld(N_GHOSTS - i1, em::bx3) = Fld(N_GHOSTS + i1 - 1, em::bx3); + } } } else { raise::KernelError( @@ -530,75 +528,70 @@ namespace kernel::bc { } } - Inline void operator()(index_t i1, index_t i2) const - { - if constexpr (M::Dim == Dim::_2D) - { - - if constexpr (S == SimEngine::SRPIC) - { - // SRPIC - if (tags & BC::E) - { - Fld((N_GHOSTS - 1) - i1, i2, em::ex1) = Fld(N_GHOSTS + i1, i2, em::ex1); - Fld((N_GHOSTS - 1) - i1, i2, em::ex2) = -Fld(N_GHOSTS + 1 + i1, i2, em::ex2); - Fld((N_GHOSTS - 1) - i1, i2, em::ex3) = -Fld(N_GHOSTS + 1 + i1, i2, em::ex3); + Inline void operator()(index_t i1, index_t i2) const { + if constexpr (D == Dim::_2D) { + if (tags & BC::E) { + if (i1 == 0) { + Fld(N_GHOSTS, i2, em::ex2) = ZERO; + Fld(N_GHOSTS, i2, em::ex3) = ZERO; + } else { + Fld(N_GHOSTS - i1, i2, em::ex1) = Fld(N_GHOSTS + i1 - 1, i2, em::ex1); + Fld(N_GHOSTS - i1, i2, em::ex2) = -Fld(N_GHOSTS + i1, i2, em::ex2); + Fld(N_GHOSTS - i1, i2, em::ex3) = -Fld(N_GHOSTS + i1, i2, em::ex3); } + } - if (tags & BC::B) - { - Fld((N_GHOSTS - 1) - i1, i2, em::bx1) = -Fld(N_GHOSTS + 1 + i1, i2, em::bx1); - Fld((N_GHOSTS - 1) - i1, i2, em::bx2) = Fld(N_GHOSTS + i1, i2, em::bx2); - Fld((N_GHOSTS - 1) - i1, i2, em::bx3) = Fld(N_GHOSTS + i1, i2, em::bx3); + if (tags & BC::B) { + if (i1 == 0) { + Fld(N_GHOSTS, i2, em::bx1) = ZERO; + } else { + Fld(N_GHOSTS - i1, i2, em::bx1) = -Fld(N_GHOSTS + i1, i2, em::bx1); + Fld(N_GHOSTS - i1, i2, em::bx2) = Fld(N_GHOSTS + i1 - 1, i2, em::bx2); + Fld(N_GHOSTS - i1, i2, em::bx3) = Fld(N_GHOSTS + i1 - 1, i2, em::bx3); } } - else - { - // GRPIC - raise::KernelError(HERE, "2D GRPIC not implemented"); - } - } - else - { + } else { raise::KernelError( - HERE, - "ConductorBoundaries_kernel: 2D implementation called for D != 2"); + HERE, + "ConductorBoundaries_kernel: 2D implementation called for D != 2"); } } - Inline void operator()(index_t i1, index_t i2, index_t i3) const - { - if constexpr (M::Dim == Dim::_3D) - { - - if constexpr (S == SimEngine::SRPIC) - { - // SRPIC - if (tags & BC::E) - { - Fld((N_GHOSTS - 1) - i1, i2, i3, em::ex1) = Fld(N_GHOSTS + i1, i2, i3, em::ex1); - Fld((N_GHOSTS - 1) - i1, i2, i3, em::ex2) = -Fld(N_GHOSTS + i1 + 1, i2, i3, em::ex2); - Fld((N_GHOSTS - 1) - i1, i2, i3, em::ex3) = -Fld(N_GHOSTS + i1 + 1, i2, i3, em::ex3); + Inline void operator()(index_t i1, index_t i2, index_t i3) const { + if constexpr (D == Dim::_3D) { + if (tags & BC::E) { + if (i1 == 0) { + Fld(N_GHOSTS, i2, i3, em::ex2) = ZERO; + Fld(N_GHOSTS, i2, i3, em::ex3) = ZERO; + } else { + Fld(N_GHOSTS - i1, i2, i3, em::ex1) = Fld(N_GHOSTS + i1 - 1, + i2, + i3, + em::ex1); + Fld(N_GHOSTS - i1, i2, i3, em::ex2) = -Fld(N_GHOSTS + i1, i2, i3, em::ex2); + Fld(N_GHOSTS - i1, i2, i3, em::ex3) = -Fld(N_GHOSTS + i1, i2, i3, em::ex3); } + } - if (tags & BC::B) - { - Fld((N_GHOSTS - 1) - i1, i2, i3, em::bx1) = -Fld(N_GHOSTS + i1 + 1, i2, i3, em::bx1); - Fld((N_GHOSTS - 1) - i1, i2, i3, em::bx2) = Fld(N_GHOSTS + i1, i2, i3, em::bx2); - Fld((N_GHOSTS - 1) - i1, i2, i3, em::bx3) = Fld(N_GHOSTS + i1, i2, i3, em::bx3); + if (tags & BC::B) { + if (i1 == 0) { + Fld(N_GHOSTS, i2, i3, em::bx1) = ZERO; + } else { + Fld(N_GHOSTS - i1, i2, i3, em::bx1) = -Fld(N_GHOSTS + i1, i2, i3, em::bx1); + Fld(N_GHOSTS - i1, i2, i3, em::bx2) = Fld(N_GHOSTS + i1 - 1, + i2, + i3, + em::bx2); + Fld(N_GHOSTS - i1, i2, i3, em::bx3) = Fld(N_GHOSTS + i1 - 1, + i2, + i3, + em::bx3); } } - else - { - // GRPIC - raise::KernelError(HERE, "3D GRPIC not implemented"); - } - } - else - { + } else { raise::KernelError( - HERE, - "ConductorBoundaries_kernel: 3D implementation called for D != 3"); + HERE, + "ConductorBoundaries_kernel: 3D implementation called for D != 3"); } } }; From 28a02f13191464de82b09fc6959390a08511e71f Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 7 Mar 2025 12:11:31 -0500 Subject: [PATCH 057/183] filters adapted for conducting BCs --- src/kernels/digital_filter.hpp | 132 +++++++++++++++++++++------------ 1 file changed, 86 insertions(+), 46 deletions(-) diff --git a/src/kernels/digital_filter.hpp b/src/kernels/digital_filter.hpp index 5d05fad2d..6a4599d35 100644 --- a/src/kernels/digital_filter.hpp +++ b/src/kernels/digital_filter.hpp @@ -17,9 +17,13 @@ #include "utils/error.h" #include "utils/numeric.h" -#define FILTER_IN_I1(ARR, COMP, I, J) \ +#define FILTER2D_IN_I1(ARR, COMP, I, J) \ INV_2*(ARR)((I), (J), (COMP)) + \ - INV_4*((ARR)((I)-1, (J), (COMP)) + (ARR)((I) + 1, (J), (COMP))) + INV_4*((ARR)((I) - 1, (J), (COMP)) + (ARR)((I) + 1, (J), (COMP))) + +#define FILTER2D_IN_I2(ARR, COMP, I, J) \ + INV_2*(ARR)((I), (J), (COMP)) + \ + INV_4*((ARR)((I), (J) - 1, (COMP)) + (ARR)((I), (J) + 1, (COMP))) namespace kernel { using namespace ntt; @@ -28,8 +32,9 @@ namespace kernel { class DigitalFilter_kernel { ndfield_t array; const ndfield_t buffer; - bool is_axis_i2min { false }, is_axis_i2max { false }; - static constexpr auto i2_min = N_GHOSTS; + const bool is_axis_i2min, is_axis_i2max; + const bool is_conductor_i1min; + static constexpr auto i1_min = N_GHOSTS, i2_min = N_GHOSTS; const std::size_t i2_max; public: @@ -39,20 +44,31 @@ namespace kernel { const boundaries_t& boundaries) : array { array } , buffer { buffer } - , i2_max { (short)D > 1 ? size_[1] + N_GHOSTS : 0 } { - if constexpr ((C != Coord::Cart) && (D != Dim::_1D)) { - raise::ErrorIf(boundaries.size() < 2, "boundaries defined incorrectly", HERE); - is_axis_i2min = (boundaries[1].first == FldsBC::AXIS); - is_axis_i2max = (boundaries[1].second == FldsBC::AXIS); - } - } + , is_axis_i2min { (D == Dim::_2D) and (boundaries[1].first == FldsBC::AXIS) } + , is_axis_i2max { (D == Dim::_2D) and (boundaries[1].second == FldsBC::AXIS) } + , is_conductor_i1min { boundaries[0].first == FldsBC::CONDUCTOR } + , i2_max { (short)D > 1 ? size_[1] + N_GHOSTS : 0 } {} Inline void operator()(index_t i1) const { if constexpr ((D == Dim::_1D) && (C == Coord::Cart)) { + if (is_conductor_i1min and i1 == i1_min) { + array(i1, cur::jx1) = (THREE * INV_4) * buffer(i1, cur::jx1) + + (INV_4)*buffer(i1 + 1, cur::jx1); + } else if (is_conductor_i1min and i1 == i1_min + 1) { + array(i1, cur::jx1) = INV_2 * buffer(i1, cur::jx1) + + INV_4 * (buffer(i1 - 1, cur::jx1) + + buffer(i1 + 1, cur::jx1)); + array(i1, cur::jx2) = (INV_2)*buffer(i1, cur::jx2) + + (INV_4)*buffer(i1 + 1, cur::jx2); + array(i1, cur::jx3) = (INV_2)*buffer(i1, cur::jx3) + + (INV_4)*buffer(i1 + 1, cur::jx3); + } else { #pragma unroll - for (const auto& comp : { cur::jx1, cur::jx2, cur::jx3 }) { - array(i1, comp) = INV_2 * buffer(i1, comp) + - INV_4 * (buffer(i1 - 1, comp) + buffer(i1 + 1, comp)); + for (const auto& comp : { cur::jx1, cur::jx2, cur::jx3 }) { + array(i1, comp) = INV_2 * buffer(i1, comp) + + INV_4 * + (buffer(i1 - 1, comp) + buffer(i1 + 1, comp)); + } } } else { raise::KernelError(HERE, "DigitalFilter_kernel: 1D implementation called for D != 1 or for non-Cartesian metric"); @@ -62,25 +78,49 @@ namespace kernel { Inline void operator()(index_t i1, index_t i2) const { if constexpr (D == Dim::_2D) { if constexpr (C == Coord::Cart) { + if (is_conductor_i1min and i1 == i1_min) { + array(i1, i2, cur::jx1) = + (THREE * INV_4) * (FILTER2D_IN_I2(buffer, cur::jx1, i1, i2)) + + (INV_4) * (FILTER2D_IN_I2(buffer, cur::jx1, i1 + 1, i2)); + } else if (is_conductor_i1min and i1 == i1_min + 1) { + array(i1, + i2, + cur::jx1) = INV_2 * (FILTER2D_IN_I2(buffer, cur::jx1, i1, i2)) + + INV_4 * + ((FILTER2D_IN_I2(buffer, cur::jx1, i1 - 1, i2)) + + (FILTER2D_IN_I2(buffer, cur::jx1, i1 + 1, i2))); + array(i1, + i2, + cur::jx2) = INV_2 * (FILTER2D_IN_I2(buffer, cur::jx2, i1, i2)) + + INV_4 * + (FILTER2D_IN_I2(buffer, cur::jx2, i1 + 1, i2)); + array(i1, + i2, + cur::jx3) = INV_2 * (FILTER2D_IN_I2(buffer, cur::jx3, i1, i2)) + + INV_4 * + (FILTER2D_IN_I2(buffer, cur::jx3, i1 + 1, i2)); + } else { #pragma unroll - for (const auto comp : { cur::jx1, cur::jx2, cur::jx3 }) { - array(i1, i2, comp) = INV_4 * buffer(i1, i2, comp) + - INV_8 * (buffer(i1 - 1, i2, comp) + - buffer(i1 + 1, i2, comp) + - buffer(i1, i2 - 1, comp) + - buffer(i1, i2 + 1, comp)) + - INV_16 * (buffer(i1 - 1, i2 - 1, comp) + - buffer(i1 + 1, i2 + 1, comp) + - buffer(i1 - 1, i2 + 1, comp) + - buffer(i1 + 1, i2 - 1, comp)); + for (const auto comp : { cur::jx1, cur::jx2, cur::jx3 }) { + array(i1, i2, comp) = INV_4 * buffer(i1, i2, comp) + + INV_8 * (buffer(i1 - 1, i2, comp) + + buffer(i1 + 1, i2, comp) + + buffer(i1, i2 - 1, comp) + + buffer(i1, i2 + 1, comp)) + + INV_16 * (buffer(i1 - 1, i2 - 1, comp) + + buffer(i1 + 1, i2 + 1, comp) + + buffer(i1 - 1, i2 + 1, comp) + + buffer(i1 + 1, i2 - 1, comp)); + } } } else { // spherical + // @TODO: get rid of temporary variables real_t cur_00, cur_0p1, cur_0m1; if (is_axis_i2min && (i2 == i2_min)) { /* --------------------------------- r, phi --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx1, i1, i2); - cur_0p1 = FILTER_IN_I1(buffer, cur::jx1, i1, i2 + 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2); + cur_0p1 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2 + 1); // ... filter in theta array(i1, i2, cur::jx1) = INV_2 * cur_00 + INV_2 * cur_0p1; @@ -88,58 +128,58 @@ namespace kernel { /* ---------------------------------- theta --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx2, i1, i2); - cur_0p1 = FILTER_IN_I1(buffer, cur::jx2, i1, i2 + 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2); + cur_0p1 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2 + 1); // ... filter in theta array(i1, i2, cur::jx2) = INV_4 * (cur_00 + cur_0p1); } else if (is_axis_i2min && (i2 == i2_min + 1)) { /* --------------------------------- r, phi --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx1, i1, i2); - cur_0p1 = FILTER_IN_I1(buffer, cur::jx1, i1, i2 + 1); - cur_0m1 = FILTER_IN_I1(buffer, cur::jx1, i1, i2 - 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2); + cur_0p1 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2 + 1); + cur_0m1 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2 - 1); // ... filter in theta array(i1, i2, cur::jx1) = INV_2 * cur_00 + INV_4 * (cur_0p1 + cur_0m1); // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx3, i1, i2); - cur_0p1 = FILTER_IN_I1(buffer, cur::jx3, i1, i2 + 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx3, i1, i2); + cur_0p1 = FILTER2D_IN_I1(buffer, cur::jx3, i1, i2 + 1); // ... filter in theta array(i1, i2, cur::jx3) = INV_2 * cur_00 + INV_4 * cur_0p1; /* ---------------------------------- theta --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx2, i1, i2); - cur_0p1 = FILTER_IN_I1(buffer, cur::jx2, i1, i2 + 1); - cur_0m1 = FILTER_IN_I1(buffer, cur::jx2, i1, i2 - 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2); + cur_0p1 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2 + 1); + cur_0m1 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2 - 1); // ... filter in theta array(i1, i2, cur::jx2) = INV_2 * cur_00 + INV_4 * (cur_0m1 + cur_0p1); } else if (is_axis_i2max && (i2 == i2_max - 1)) { /* --------------------------------- r, phi --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx1, i1, i2); - cur_0p1 = FILTER_IN_I1(buffer, cur::jx1, i1, i2 + 1); - cur_0m1 = FILTER_IN_I1(buffer, cur::jx1, i1, i2 - 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2); + cur_0p1 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2 + 1); + cur_0m1 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2 - 1); // ... filter in theta array(i1, i2, cur::jx1) = INV_2 * cur_00 + INV_4 * (cur_0m1 + cur_0p1); // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx3, i1, i2); - cur_0m1 = FILTER_IN_I1(buffer, cur::jx3, i1, i2 - 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx3, i1, i2); + cur_0m1 = FILTER2D_IN_I1(buffer, cur::jx3, i1, i2 - 1); // ... filter in theta array(i1, i2, cur::jx3) = INV_2 * cur_00 + INV_4 * cur_0m1; /* ---------------------------------- theta --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx2, i1, i2); - cur_0m1 = FILTER_IN_I1(buffer, cur::jx2, i1, i2 - 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2); + cur_0m1 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2 - 1); // ... filter in theta array(i1, i2, cur::jx2) = INV_4 * (cur_00 + cur_0m1); } else if (is_axis_i2max && (i2 == i2_max)) { /* --------------------------------- r, phi --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx1, i1, i2); - cur_0m1 = FILTER_IN_I1(buffer, cur::jx1, i1, i2 - 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2); + cur_0m1 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2 - 1); // ... filter in theta array(i1, i2, cur::jx1) = INV_2 * cur_00 + INV_2 * cur_0m1; @@ -210,6 +250,6 @@ namespace kernel { } // namespace kernel -#undef FILTER_IN_I1 +#undef FILTER2D_IN_I1 #endif // DIGITAL_FILTER_HPP From b800b615ef1e0c1388ae629856a7e5a86e9531ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 7 Mar 2025 20:26:49 -0600 Subject: [PATCH 058/183] revert to old pgen for testing --- setups/srpic/shocktest/pgen.hpp | 171 +++++++++++++++++++++++--------- 1 file changed, 125 insertions(+), 46 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 4f6decc76..ff54923d1 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -38,20 +38,15 @@ namespace user { // magnetic field components Inline auto bx1(const coord_t& x_ph) const -> real_t { - // return Bmag * math::cos(Btheta); - return ZERO; + return Bmag * math::cos(Btheta); } Inline auto bx2(const coord_t& x_ph) const -> real_t { - // return Bmag * math::sin(Btheta) * math::sin(Bphi); - return ZERO; + return Bmag * math::sin(Btheta) * math::sin(Bphi); } Inline auto bx3(const coord_t& x_ph) const -> real_t { - // return Bmag * math::sin(Btheta) * math::cos(Bphi); - // return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 - // + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); - return ZERO; + return Bmag * math::sin(Btheta) * math::cos(Bphi); } // electric field components @@ -60,15 +55,11 @@ namespace user { } Inline auto ex2(const coord_t& x_ph) const -> real_t { - // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - // return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 - // + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); - return ZERO; + return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); } Inline auto ex3(const coord_t& x_ph) const -> real_t { - // return Vx * Bmag * math::sin(Btheta) * math::sin(Bphi); - return ZERO; + return Vx * Bmag * math::sin(Btheta) * math::sin(Bphi); } private: @@ -102,16 +93,16 @@ namespace user { : arch::ProblemGenerator { p } , drift_ux { p.template get("setup.drift_ux") } , temperature { p.template get("setup.temperature") } - , x1arr_e { p.template get>("setup.x_e") } - , x2arr_e { p.template get>("setup.y_e") } - , ux1arr_e { p.template get>("setup.ux_e") } - , ux2arr_e { p.template get>("setup.uy_e") } - , ux3arr_e { p.template get>("setup.uz_e") } - , x1arr_i { p.template get>("setup.x_i") } - , x2arr_i { p.template get>("setup.y_i") } - , ux1arr_i { p.template get>("setup.ux_i") } - , ux2arr_i { p.template get>("setup.uy_i") } - , ux3arr_i { p.template get>("setup.uz_i") } + // , x1arr_e { p.template get>("setup.x_e") } + // , x2arr_e { p.template get>("setup.y_e") } + // , ux1arr_e { p.template get>("setup.ux_e") } + // , ux2arr_e { p.template get>("setup.uy_e") } + // , ux3arr_e { p.template get>("setup.uz_e") } + // , x1arr_i { p.template get>("setup.x_i") } + // , x2arr_i { p.template get>("setup.y_i") } + // , ux1arr_i { p.template get>("setup.ux_i") } + // , ux2arr_i { p.template get>("setup.uy_i") } + // , ux3arr_i { p.template get>("setup.uz_i") } , Btheta { p.template get("setup.Btheta", ZERO) } , Bmag { p.template get("setup.Bmag", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } @@ -135,28 +126,116 @@ namespace user { return init_flds; } - inline void InitPrtls(Domain& domain) { - arch::InjectGlobally(*metadomain, - domain, - 1, - { - { "x1", x1arr_e }, - { "x2", x2arr_e }, - { "ux1", ux1arr_e }, - { "ux2", ux1arr_e }, - { "ux3", ux3arr_e } - }); - arch::InjectGlobally(*metadomain, - domain, - 2, - { - { "x1", x1arr_i }, - { "x2", x2arr_i }, - { "ux1", ux1arr_i }, - { "ux2", ux1arr_i }, - { "ux3", ux3arr_i } - }); - } + // inline void InitPrtls(Domain& domain) { + // arch::InjectGlobally(*metadomain, + // domain, + // 1, + // { + // { "x1", x1arr_e }, + // { "x2", x2arr_e }, + // { "ux1", ux1arr_e }, + // { "ux2", ux1arr_e }, + // { "ux3", ux3arr_e } + // }); + // arch::InjectGlobally(*metadomain, + // domain, + // 2, + // { + // { "x1", x1arr_i }, + // { "x2", x2arr_i }, + // { "ux1", ux1arr_i }, + // { "ux2", ux1arr_i }, + // { "ux3", ux3arr_i } + // }); + // } + + inline void InitPrtls(Domain& domain) { + + // auto& species_e = domain.species[0]; + // auto& species_p = domain.species[1]; + auto& species_e = domain.species[0]; + auto& species_p = domain.species[1]; + + // array_t elec_ind("elec_ind"); + // array_t pos_ind("pos_ind"); + array_t elec_ind("elec_ind"); + array_t pos_ind("pos_ind"); + + auto offset_e = species_e.npart(); + auto offset_p = species_p.npart(); + + auto ux1_e = species_e.ux1; + auto ux2_e = species_e.ux2; + auto ux3_e = species_e.ux3; + auto i1_e = species_e.i1; + auto i2_e = species_e.i2; + auto dx1_e = species_e.dx1; + auto dx2_e = species_e.dx2; + auto phi_e = species_e.phi; + auto weight_e = species_e.weight; + auto tag_e = species_e.tag; + + auto ux1_p = species_p.ux1; + auto ux2_p = species_p.ux2; + auto ux3_p = species_p.ux3; + auto i1_p = species_p.i1; + auto i2_p = species_p.i2; + auto dx1_p = species_p.dx1; + auto dx2_p = species_p.dx2; + auto phi_p = species_p.phi; + auto weight_p = species_p.weight; + auto tag_p = species_p.tag; + + int nseed = 1; + + Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { + + // ToDo: fix this + auto i1_ = math::floor(10); + auto i2_ = math::floor(64); + auto dx1_ = HALF; + auto dx2_ = HALF; + + + auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); + auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); + + i1_e(elec_p + offset_e) = i1_; + dx1_e(elec_p + offset_e) = dx1_; + i2_e(elec_p + offset_e) = i2_; + dx2_e(elec_p + offset_e) = 2*dx2_; + // ux1_e(elec_p + offset_e) = -0.5; + // ux2_e(elec_p + offset_e) = 0.5; + ux1_e(elec_p + offset_e) = -drift_ux; + ux2_e(elec_p + offset_e) = ZERO; + ux3_e(elec_p + offset_e) = ZERO; + weight_e(elec_p + offset_e) = ONE; + tag_e(elec_p + offset_e) = ParticleTag::alive; + + i1_p(pos_p + offset_p) = i1_; + dx1_p(pos_p + offset_p) = dx1_; + i2_p(pos_p + offset_p) = i2_; + dx2_p(pos_p + offset_p) = -2*dx2_; + // ux1_p(pos_p + offset_p) = 0.5; + // ux2_p(pos_p + offset_p) = -0.5; + ux1_p(pos_p + offset_p) = -drift_ux; + ux2_p(pos_p + offset_p) = ZERO; + ux3_p(pos_p + offset_p) = ZERO; + weight_p(pos_p + offset_p) = ONE; + tag_p(pos_p + offset_p) = ParticleTag::alive; + + + }); + + + auto elec_ind_h = Kokkos::create_mirror(elec_ind); + Kokkos::deep_copy(elec_ind_h, elec_ind); + species_e.set_npart(offset_e + elec_ind_h()); + + auto pos_ind_h = Kokkos::create_mirror(pos_ind); + Kokkos::deep_copy(pos_ind_h, pos_ind); + species_p.set_npart(offset_p + pos_ind_h()); + } }; } // namespace user From 2b269f399782c7fab69c67ee589e4b274f9265f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 7 Mar 2025 21:26:16 -0600 Subject: [PATCH 059/183] enforce B0/E0 at boundary --- src/engines/srpic.hpp | 8 +++++--- src/kernels/fields_bcs.hpp | 33 +++++++++++++++++++++------------ 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index dd2f5ccec..891336309 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -889,11 +889,13 @@ namespace ntt { } else { raise::Error("Invalid dimension", HERE); } + + auto match_fields = m_pgen.MatchFields(time); Kokkos::parallel_for( - "MatchFields", + "ConductorFields", range, - kernel::bc::ConductorBoundaries_kernel(domain.fields.em, - tags)); + kernel::bc::ConductorBoundaries_kernel(domain.fields.em, + match_fields, tags)); // if constexpr (M::Dim == Dim::_1D) { // Kokkos::parallel_for( diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 9d8e9c74e..4d6be9380 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -486,25 +486,32 @@ namespace kernel::bc { } }; - template + template struct ConductorBoundaries_kernel { static_assert(static_cast(o) < static_cast(D), "Invalid component index"); - // static constexpr idx_t i = static_cast(o) + 1u; + static constexpr bool defines_ex2 = traits::has_method::value; + static constexpr bool defines_ex3 = traits::has_method::value; + static constexpr bool defines_bx1 = traits::has_method::value; + static_assert(( defines_ex2 or defines_ex3 or defines_bx1), + "none of the components of E or B are specified in PGEN"); ndfield_t Fld; + const I fset; const BCTags tags; - ConductorBoundaries_kernel(ndfield_t Fld, BCTags tags) + ConductorBoundaries_kernel(ndfield_t Fld, const I& fset, BCTags tags) : Fld { Fld } + , fset { fset } , tags { tags } {} Inline void operator()(index_t i1) const { if constexpr (D == Dim::_1D) { + coord_t x_Ph_H { ZERO }; if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, em::ex2) = ZERO; - Fld(N_GHOSTS, em::ex3) = ZERO; + Fld(N_GHOSTS, em::ex2) = fset.ex2(x_Ph_H); + Fld(N_GHOSTS, em::ex3) = fset.ex3(x_Ph_H); } else { Fld(N_GHOSTS - i1, em::ex1) = Fld(N_GHOSTS + i1 - 1, em::ex1); Fld(N_GHOSTS - i1, em::ex2) = -Fld(N_GHOSTS + i1, em::ex2); @@ -514,7 +521,7 @@ namespace kernel::bc { if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, em::bx1) = ZERO; + Fld(N_GHOSTS, em::bx1) = fset.bx1(x_Ph_H); } else { Fld(N_GHOSTS - i1, em::bx1) = -Fld(N_GHOSTS + i1, em::bx1); Fld(N_GHOSTS - i1, em::bx2) = Fld(N_GHOSTS + i1 - 1, em::bx2); @@ -530,10 +537,11 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2) const { if constexpr (D == Dim::_2D) { + coord_t x_Ph_H { ZERO }; if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, i2, em::ex2) = ZERO; - Fld(N_GHOSTS, i2, em::ex3) = ZERO; + Fld(N_GHOSTS, i2, em::ex2) = fset.ex2(x_Ph_H); + Fld(N_GHOSTS, i2, em::ex3) = fset.ex3(x_Ph_H); } else { Fld(N_GHOSTS - i1, i2, em::ex1) = Fld(N_GHOSTS + i1 - 1, i2, em::ex1); Fld(N_GHOSTS - i1, i2, em::ex2) = -Fld(N_GHOSTS + i1, i2, em::ex2); @@ -543,7 +551,7 @@ namespace kernel::bc { if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, i2, em::bx1) = ZERO; + Fld(N_GHOSTS, i2, em::bx1) = fset.bx1(x_Ph_H); } else { Fld(N_GHOSTS - i1, i2, em::bx1) = -Fld(N_GHOSTS + i1, i2, em::bx1); Fld(N_GHOSTS - i1, i2, em::bx2) = Fld(N_GHOSTS + i1 - 1, i2, em::bx2); @@ -559,10 +567,11 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2, index_t i3) const { if constexpr (D == Dim::_3D) { + coord_t x_Ph_H { ZERO }; if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, i2, i3, em::ex2) = ZERO; - Fld(N_GHOSTS, i2, i3, em::ex3) = ZERO; + Fld(N_GHOSTS, i2, i3, em::ex2) = fset.ex2(x_Ph_H); + Fld(N_GHOSTS, i2, i3, em::ex3) = fset.ex3(x_Ph_H); } else { Fld(N_GHOSTS - i1, i2, i3, em::ex1) = Fld(N_GHOSTS + i1 - 1, i2, @@ -575,7 +584,7 @@ namespace kernel::bc { if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, i2, i3, em::bx1) = ZERO; + Fld(N_GHOSTS, i2, i3, em::bx1) = fset.bx1(x_Ph_H); } else { Fld(N_GHOSTS - i1, i2, i3, em::bx1) = -Fld(N_GHOSTS + i1, i2, i3, em::bx1); Fld(N_GHOSTS - i1, i2, i3, em::bx2) = Fld(N_GHOSTS + i1 - 1, From 9b237323a2f43fea4f1ea1c25fe3d1466f8d3415 Mon Sep 17 00:00:00 2001 From: hayk Date: Sat, 8 Mar 2025 16:16:31 -0500 Subject: [PATCH 060/183] filter test fixed --- src/kernels/tests/digital_filter.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/kernels/tests/digital_filter.cpp b/src/kernels/tests/digital_filter.cpp index e0cc352f5..03a63753b 100644 --- a/src/kernels/tests/digital_filter.cpp +++ b/src/kernels/tests/digital_filter.cpp @@ -40,8 +40,13 @@ void testFilter(const std::vector& res, auto boundaries = boundaries_t {}; if constexpr (M::CoordType != Coord::Cart) { boundaries = { - {FldsBC::CUSTOM, FldsBC::CUSTOM}, - { FldsBC::AXIS, FldsBC::AXIS} + { FldsBC::CUSTOM, FldsBC::CUSTOM }, + { FldsBC::AXIS, FldsBC::AXIS } + }; + } else { + boundaries = { + { FldsBC::PERIODIC, FldsBC::PERIODIC }, + { FldsBC::PERIODIC, FldsBC::PERIODIC } }; } @@ -183,4 +188,4 @@ auto main(int argc, char* argv[]) -> int { } Kokkos::finalize(); return 0; -} \ No newline at end of file +} From 179fafee756f0a921e84f62f53e2c0ef27aaff75 Mon Sep 17 00:00:00 2001 From: hayk Date: Sun, 9 Mar 2025 01:59:57 -0500 Subject: [PATCH 061/183] removed extra kokkos flags --- cmake/kokkosConfig.cmake | 35 ----------------- legacy/tests/kernels-sr.cpp | 10 ++--- src/framework/containers/particles.cpp | 4 +- src/framework/domain/comm_mpi.hpp | 12 +++--- src/framework/domain/comm_nompi.hpp | 6 +-- src/global/arch/kokkos_aliases.cpp | 54 +++++++++++++------------- src/global/arch/kokkos_aliases.h | 18 ++++----- src/kernels/tests/prtls_to_phys.cpp | 45 ++++++++++----------- 8 files changed, 76 insertions(+), 108 deletions(-) diff --git a/cmake/kokkosConfig.cmake b/cmake/kokkosConfig.cmake index 63c32622d..8800f21a0 100644 --- a/cmake/kokkosConfig.cmake +++ b/cmake/kokkosConfig.cmake @@ -37,41 +37,6 @@ set(Kokkos_ENABLE_OPENMP ${default_KOKKOS_ENABLE_OPENMP} CACHE BOOL "Enable OpenMP") -# set memory space -if(${Kokkos_ENABLE_CUDA}) - add_compile_definitions(CUDA_ENABLED) - set(ACC_MEM_SPACE Kokkos::CudaSpace) -elseif(${Kokkos_ENABLE_HIP}) - add_compile_definitions(HIP_ENABLED) - set(ACC_MEM_SPACE Kokkos::HIPSpace) -else() - set(ACC_MEM_SPACE Kokkos::HostSpace) -endif() - -set(HOST_MEM_SPACE Kokkos::HostSpace) - -# set execution space -if(${Kokkos_ENABLE_CUDA}) - set(ACC_EXE_SPACE Kokkos::Cuda) -elseif(${Kokkos_ENABLE_HIP}) - set(ACC_EXE_SPACE Kokkos::HIP) -elseif(${Kokkos_ENABLE_OPENMP}) - set(ACC_EXE_SPACE Kokkos::OpenMP) -else() - set(ACC_EXE_SPACE Kokkos::Serial) -endif() - -if(${Kokkos_ENABLE_OPENMP}) - set(HOST_EXE_SPACE Kokkos::OpenMP) -else() - set(HOST_EXE_SPACE Kokkos::Serial) -endif() - -add_compile_options("-D AccelExeSpace=${ACC_EXE_SPACE}") -add_compile_options("-D AccelMemSpace=${ACC_MEM_SPACE}") -add_compile_options("-D HostExeSpace=${HOST_EXE_SPACE}") -add_compile_options("-D HostMemSpace=${HOST_MEM_SPACE}") - if(${BUILD_TESTING} STREQUAL "OFF") set(Kokkos_ENABLE_TESTS OFF diff --git a/legacy/tests/kernels-sr.cpp b/legacy/tests/kernels-sr.cpp index 59ce0646b..d765799e3 100644 --- a/legacy/tests/kernels-sr.cpp +++ b/legacy/tests/kernels-sr.cpp @@ -1,17 +1,17 @@ -#include "wrapper.h" - #include #include #include +#include "wrapper.h" + #include METRIC_HEADER #include PGEN_HEADER -#include "particle_macros.h" - #include "kernels/particle_pusher_sr.hpp" +#include "particle_macros.h" + template void put_value(ntt::array_t& arr, T value, int i) { auto arr_h = Kokkos::create_mirror_view(arr); @@ -221,4 +221,4 @@ auto main(int argc, char* argv[]) -> int { ntt::GlobalFinalize(); return 0; -} \ No newline at end of file +} diff --git a/src/framework/containers/particles.cpp b/src/framework/containers/particles.cpp index d78055824..d165f6233 100644 --- a/src/framework/containers/particles.cpp +++ b/src/framework/containers/particles.cpp @@ -220,14 +220,14 @@ namespace ntt { Kokkos::Experimental::fill( "TagAliveParticles", - AccelExeSpace(), + Kokkos::DefaultExecutionSpace(), Kokkos::subview(this_tag, std::make_pair(static_cast(0), n_alive)), ParticleTag::alive); Kokkos::Experimental::fill( "TagDeadParticles", - AccelExeSpace(), + Kokkos::DefaultExecutionSpace(), Kokkos::subview(this_tag, std::make_pair(n_alive, n_alive + n_dead)), ParticleTag::dead); diff --git a/src/framework/domain/comm_mpi.hpp b/src/framework/domain/comm_mpi.hpp index e5bc2d21e..d4f8b5651 100644 --- a/src/framework/domain/comm_mpi.hpp +++ b/src/framework/domain/comm_mpi.hpp @@ -83,7 +83,7 @@ namespace comm { (long int)(send_slice[0].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, comps.first }, { recv_slice[0].second, comps.second }), Lambda(index_t i1, index_t ci) { @@ -96,7 +96,7 @@ namespace comm { (long int)(send_slice[1].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, comps.first }, { recv_slice[0].second, recv_slice[1].second, comps.second }), Lambda(index_t i1, index_t i2, index_t ci) { @@ -111,7 +111,7 @@ namespace comm { (long int)(send_slice[2].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, recv_slice[2].first, @@ -239,7 +239,7 @@ namespace comm { const auto offset_c = comps.first; Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, comps.first }, { recv_slice[0].second, comps.second }), Lambda(index_t i1, index_t ci) { @@ -251,7 +251,7 @@ namespace comm { const auto offset_c = comps.first; Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, comps.first }, { recv_slice[0].second, recv_slice[1].second, comps.second }), Lambda(index_t i1, index_t i2, index_t ci) { @@ -266,7 +266,7 @@ namespace comm { const auto offset_c = comps.first; Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, recv_slice[2].first, diff --git a/src/framework/domain/comm_nompi.hpp b/src/framework/domain/comm_nompi.hpp index 197d336fa..b477ac176 100644 --- a/src/framework/domain/comm_nompi.hpp +++ b/src/framework/domain/comm_nompi.hpp @@ -70,7 +70,7 @@ namespace comm { (long int)(send_slice[0].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, comps.first }, { recv_slice[0].second, comps.second }), Lambda(index_t i1, index_t ci) { @@ -83,7 +83,7 @@ namespace comm { (long int)(send_slice[1].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, comps.first }, { recv_slice[0].second, recv_slice[1].second, comps.second }), Lambda(index_t i1, index_t i2, index_t ci) { @@ -98,7 +98,7 @@ namespace comm { (long int)(send_slice[2].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, recv_slice[2].first, diff --git a/src/global/arch/kokkos_aliases.cpp b/src/global/arch/kokkos_aliases.cpp index 6c15e3d52..73bc364d1 100644 --- a/src/global/arch/kokkos_aliases.cpp +++ b/src/global/arch/kokkos_aliases.cpp @@ -5,73 +5,75 @@ #include template <> -auto CreateRangePolicy( - const tuple_t& i1, - const tuple_t& i2) -> range_t { +auto CreateRangePolicy(const tuple_t& i1, + const tuple_t& i2) + -> range_t { index_t i1min = i1[0]; index_t i1max = i2[0]; - return Kokkos::RangePolicy(i1min, i1max); + return Kokkos::RangePolicy(i1min, i1max); } template <> -auto CreateRangePolicy( - const tuple_t& i1, - const tuple_t& i2) -> range_t { +auto CreateRangePolicy(const tuple_t& i1, + const tuple_t& i2) + -> range_t { index_t i1min = i1[0]; index_t i1max = i2[0]; index_t i2min = i1[1]; index_t i2max = i2[1]; - return Kokkos::MDRangePolicy, AccelExeSpace>({ i1min, i2min }, - { i1max, i2max }); + return Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( + { i1min, i2min }, + { i1max, i2max }); } template <> -auto CreateRangePolicy( - const tuple_t& i1, - const tuple_t& i2) -> range_t { +auto CreateRangePolicy(const tuple_t& i1, + const tuple_t& i2) + -> range_t { index_t i1min = i1[0]; index_t i1max = i2[0]; index_t i2min = i1[1]; index_t i2max = i2[1]; index_t i3min = i1[2]; index_t i3max = i2[2]; - return Kokkos::MDRangePolicy, AccelExeSpace>( + return Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { i1min, i2min, i3min }, { i1max, i2max, i3max }); } template <> -auto CreateRangePolicyOnHost( - const tuple_t& i1, - const tuple_t& i2) -> range_h_t { +auto CreateRangePolicyOnHost(const tuple_t& i1, + const tuple_t& i2) + -> range_h_t { index_t i1min = i1[0]; index_t i1max = i2[0]; - return Kokkos::RangePolicy(i1min, i1max); + return Kokkos::RangePolicy(i1min, i1max); } template <> -auto CreateRangePolicyOnHost( - const tuple_t& i1, - const tuple_t& i2) -> range_h_t { +auto CreateRangePolicyOnHost(const tuple_t& i1, + const tuple_t& i2) + -> range_h_t { index_t i1min = i1[0]; index_t i1max = i2[0]; index_t i2min = i1[1]; index_t i2max = i2[1]; - return Kokkos::MDRangePolicy, HostExeSpace>({ i1min, i2min }, - { i1max, i2max }); + return Kokkos::MDRangePolicy, Kokkos::DefaultHostExecutionSpace>( + { i1min, i2min }, + { i1max, i2max }); } template <> -auto CreateRangePolicyOnHost( - const tuple_t& i1, - const tuple_t& i2) -> range_h_t { +auto CreateRangePolicyOnHost(const tuple_t& i1, + const tuple_t& i2) + -> range_h_t { index_t i1min = i1[0]; index_t i1max = i2[0]; index_t i2min = i1[1]; index_t i2max = i2[1]; index_t i3min = i1[2]; index_t i3max = i2[2]; - return Kokkos::MDRangePolicy, HostExeSpace>( + return Kokkos::MDRangePolicy, Kokkos::DefaultHostExecutionSpace>( { i1min, i2min, i3min }, { i1max, i2max, i3max }); } diff --git a/src/global/arch/kokkos_aliases.h b/src/global/arch/kokkos_aliases.h index f9aac9685..c20738f11 100644 --- a/src/global/arch/kokkos_aliases.h +++ b/src/global/arch/kokkos_aliases.h @@ -34,7 +34,7 @@ namespace math = Kokkos; template -using array_t = Kokkos::View; +using array_t = Kokkos::View; // Array mirror alias of arbitrary type template @@ -174,17 +174,17 @@ namespace kokkos_aliases_hidden { template <> struct range_impl { - using type = Kokkos::RangePolicy; + using type = Kokkos::RangePolicy; }; template <> struct range_impl { - using type = Kokkos::MDRangePolicy, AccelExeSpace>; + using type = Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>; }; template <> struct range_impl { - using type = Kokkos::MDRangePolicy, AccelExeSpace>; + using type = Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>; }; } // namespace kokkos_aliases_hidden @@ -201,17 +201,17 @@ namespace kokkos_aliases_hidden { template <> struct range_h_impl { - using type = Kokkos::RangePolicy; + using type = Kokkos::RangePolicy; }; template <> struct range_h_impl { - using type = Kokkos::MDRangePolicy, HostExeSpace>; + using type = Kokkos::MDRangePolicy, Kokkos::DefaultHostExecutionSpace>; }; template <> struct range_h_impl { - using type = Kokkos::MDRangePolicy, HostExeSpace>; + using type = Kokkos::MDRangePolicy, Kokkos::DefaultHostExecutionSpace>; }; } // namespace kokkos_aliases_hidden @@ -242,8 +242,8 @@ auto CreateRangePolicyOnHost(const tuple_t&, const tuple_t&) -> range_h_t; // Random number pool/generator type alias -using random_number_pool_t = Kokkos::Random_XorShift1024_Pool; -using random_generator_t = typename random_number_pool_t::generator_type; +using random_number_pool_t = Kokkos::Random_XorShift1024_Pool; +using random_generator_t = typename random_number_pool_t::generator_type; // Random number generator functions template diff --git a/src/kernels/tests/prtls_to_phys.cpp b/src/kernels/tests/prtls_to_phys.cpp index 4719fe6a1..0ceb88cd1 100644 --- a/src/kernels/tests/prtls_to_phys.cpp +++ b/src/kernels/tests/prtls_to_phys.cpp @@ -177,28 +177,29 @@ void testPrtl2PhysSR(const std::vector& res, array_t buff_ux3 { "buff_ux3", nprtl / stride }; array_t buff_wei { "buff_wei", nprtl / stride }; - Kokkos::parallel_for("Init", - Kokkos::RangePolicy(0, nprtl / stride), - kernel::PrtlToPhys_kernel(stride, - buff_x1, - buff_x2, - buff_x3, - buff_ux1, - buff_ux2, - buff_ux3, - buff_wei, - i1, - i2, - i3, - dx1, - dx2, - dx3, - ux1, - ux2, - ux3, - phi, - weight, - metric)); + Kokkos::parallel_for( + "Init", + Kokkos::RangePolicy(0, nprtl / stride), + kernel::PrtlToPhys_kernel(stride, + buff_x1, + buff_x2, + buff_x3, + buff_ux1, + buff_ux2, + buff_ux3, + buff_wei, + i1, + i2, + i3, + dx1, + dx2, + dx3, + ux1, + ux2, + ux3, + phi, + weight, + metric)); Kokkos::parallel_for("Check", nprtl / stride, Checker(metric, From e644b65600747323d108e88d8ed488ad5f74ad2b Mon Sep 17 00:00:00 2001 From: haykh Date: Sun, 9 Mar 2025 10:24:03 -0400 Subject: [PATCH 062/183] nix cfgs --- dev/nix/adios2.nix | 4 +-- dev/nix/kokkos.nix | 17 ++++++------ dev/nix/shell.nix | 66 +++++++++++++++++++++++++++++++++++++++------- 3 files changed, 66 insertions(+), 21 deletions(-) diff --git a/dev/nix/adios2.nix b/dev/nix/adios2.nix index 8ec1fd36c..7218f894f 100644 --- a/dev/nix/adios2.nix +++ b/dev/nix/adios2.nix @@ -1,7 +1,7 @@ { pkgs ? import { }, - hdf5 ? false, - mpi ? false, + hdf5, + mpi, }: let diff --git a/dev/nix/kokkos.nix b/dev/nix/kokkos.nix index cfe583c7a..6271604c5 100644 --- a/dev/nix/kokkos.nix +++ b/dev/nix/kokkos.nix @@ -1,11 +1,10 @@ { pkgs ? import { }, - arch ? "native", - gpu ? "none", + arch, + gpu, }: let - gpuUpper = pkgs.lib.toUpper gpu; name = "kokkos"; version = "4.5.01"; compilerPkgs = { @@ -30,10 +29,10 @@ let }; getArch = _: - if gpu != "none" && arch == "native" then + if gpu != "NONE" && arch == "NATIVE" then throw "Please specify an architecture when the GPU support is enabled. Available architectures: https://kokkos.org/kokkos-core-wiki/keywords.html#architectures" else - pkgs.lib.toUpper arch; + arch; in pkgs.stdenv.mkDerivation { @@ -41,7 +40,7 @@ pkgs.stdenv.mkDerivation { version = "${version}"; src = pkgs.fetchgit { url = "https://github.com/kokkos/kokkos/"; - rev = "v${version}"; + rev = "${version}"; sha256 = "sha256-cI2p+6J+8BRV5fXTDxxHTfh6P5PeeLUiF73o5zVysHQ="; }; @@ -49,16 +48,16 @@ pkgs.stdenv.mkDerivation { cmake ]; - propagatedBuildInputs = compilerPkgs.${gpuUpper}; + propagatedBuildInputs = compilerPkgs.${gpu}; cmakeFlags = [ "-D CMAKE_CXX_STANDARD=17" "-D CMAKE_CXX_EXTENSIONS=OFF" "-D CMAKE_POSITION_INDEPENDENT_CODE=TRUE" "-D Kokkos_ARCH_${getArch { }}=ON" - (if gpu != "none" then "-D Kokkos_ENABLE_${gpuUpper}=ON" else "") + (if gpu != "none" then "-D Kokkos_ENABLE_${gpu}=ON" else "") "-D CMAKE_BUILD_TYPE=Release" - ] ++ cmakeFlags.${gpuUpper}; + ] ++ cmakeFlags.${gpu}; enableParallelBuilding = true; } diff --git a/dev/nix/shell.nix b/dev/nix/shell.nix index 1f21e82b0..01d80298b 100644 --- a/dev/nix/shell.nix +++ b/dev/nix/shell.nix @@ -1,15 +1,45 @@ { pkgs ? import { }, + gpu ? "NONE", + arch ? "NATIVE", + hdf5 ? true, mpi ? false, - hdf5 ? false, - gpu ? "none", - arch ? "native", }: let + gpuUpper = pkgs.lib.toUpper gpu; + archUpper = pkgs.lib.toUpper arch; name = "entity-dev"; adios2Pkg = (pkgs.callPackage ./adios2.nix { inherit pkgs mpi hdf5; }); - kokkosPkg = (pkgs.callPackage ./kokkos.nix { inherit pkgs arch gpu; }); + kokkosPkg = ( + pkgs.callPackage ./kokkos.nix { + inherit pkgs; + arch = archUpper; + gpu = gpuUpper; + } + ); + envVars = { + compiler = rec { + NONE = { + CXX = "g++"; + CC = "gcc"; + }; + HIP = { + CXX = "hipcc"; + CC = "hipcc"; + }; + CUDA = NONE; + }; + kokkos = { + HIP = { + Kokkos_ENABLE_HIP = "ON"; + }; + CUDA = { + Kokkos_ENABLE_CUDA = "ON"; + }; + NONE = { }; + }; + }; in pkgs.mkShell { name = "${name}-env"; @@ -39,11 +69,27 @@ pkgs.mkShell { pkgs.zlib ]); - shellHook = '' - BLUE='\033[0;34m' - NC='\033[0m' + shellHook = + '' + BLUE='\033[0;34m' + NC='\033[0m' + + echo "following environment variables are set:" + '' + + pkgs.lib.concatStringsSep "" ( + pkgs.lib.mapAttrsToList ( + category: vars: + pkgs.lib.concatStringsSep "" ( + pkgs.lib.mapAttrsToList (name: value: '' + export ${name}=${value} + echo -e " ''\${BLUE}${name}''\${NC}=${value}" + '') vars.${gpuUpper} + ) + ) envVars + ) + + '' + echo "" + echo -e "${name} nix-shell activated" + ''; - echo "" - echo -e "${name} nix-shell activated" - ''; } From 158c385c2e69351ad228eace4862aca425061ec6 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Wed, 21 Aug 2024 20:15:11 -0400 Subject: [PATCH 063/183] Add bulk velocity as moment output. --- src/framework/domain/output.cpp | 8 ++++ src/global/enums.h | 5 ++- src/global/tests/enums.cpp | 2 +- src/kernels/particle_moments.hpp | 65 +++++++++++++++++++++++++++++++- src/output/fields.cpp | 3 ++ src/output/fields.h | 4 +- 6 files changed, 81 insertions(+), 6 deletions(-) diff --git a/src/framework/domain/output.cpp b/src/framework/domain/output.cpp index 6961d2826..8283be825 100644 --- a/src/framework/domain/output.cpp +++ b/src/framework/domain/output.cpp @@ -328,6 +328,14 @@ namespace ntt { {}, local_domain->fields.bckp, c); + } else if (fld.id() == FldsID::V) { + ComputeMoments(params, + local_domain->mesh, + local_domain->species, + fld.species, + fld.comp[0], + local_domain->fields.bckp, + c); } else { raise::Error("Wrong moment requested for output", HERE); } diff --git a/src/global/enums.h b/src/global/enums.h index 8f2495c13..4d2348244 100644 --- a/src/global/enums.h +++ b/src/global/enums.h @@ -288,16 +288,17 @@ namespace ntt { N = 12, Nppc = 13, Custom = 14, + V = 15, }; constexpr FldsID(uint8_t c) : enums_hidden::BaseEnum { c } {} static constexpr type variants[] = { E, divE, D, divD, B, H, J, - A, T, Rho, Charge, N, Nppc, Custom }; + A, T, Rho, Charge, N, Nppc, Custom , V}; static constexpr const char* lookup[] = { "e", "dive", "d", "divd", "b", "h", "j", "a", "t", "rho", "charge", "n", - "nppc", "custom" }; + "nppc", "custom", "v" }; static constexpr std::size_t total = sizeof(variants) / sizeof(variants[0]); }; diff --git a/src/global/tests/enums.cpp b/src/global/tests/enums.cpp index 7785ec1a3..d5eeb76e2 100644 --- a/src/global/tests/enums.cpp +++ b/src/global/tests/enums.cpp @@ -68,7 +68,7 @@ auto main() -> int { enum_str_t all_out_flds = { "e", "dive", "d", "divd", "b", "h", "j", "a", "t", "rho", - "charge", "n", "nppc", "custom" }; + "charge", "n", "nppc", "custom" , "v"}; checkEnum(all_coords); checkEnum(all_metrics); diff --git a/src/kernels/particle_moments.hpp b/src/kernels/particle_moments.hpp index 0621646ad..3c6cd37c9 100644 --- a/src/kernels/particle_moments.hpp +++ b/src/kernels/particle_moments.hpp @@ -41,7 +41,7 @@ namespace kernel { static constexpr auto D = M::Dim; static_assert((F == FldsID::Rho) || (F == FldsID::Charge) || - (F == FldsID::N) || (F == FldsID::Nppc) || (F == FldsID::T), + (F == FldsID::N) || (F == FldsID::Nppc) || (F == FldsID::T) || (F == FldsID::V), "Invalid field ID"); const unsigned short c1, c2; @@ -89,7 +89,7 @@ namespace kernel { std::size_t ni2, real_t inv_n0, unsigned short window) - : c1 { (components.size() == 2) ? components[0] + : c1 { (components.size() > 0) ? components[0] : static_cast(0) } , c2 { (components.size() == 2) ? components[1] : static_cast(0) } @@ -205,6 +205,67 @@ namespace kernel { coeff = contrib; } + if constexpr (F == FldsID::V) { + real_t gamma { ZERO }; + // for stress-energy tensor + vec_t u_Phys { ZERO }; + if constexpr (S == SimEngine::SRPIC) { + // SR + // stress-energy tensor for SR is computed in the tetrad (hatted) basis + if constexpr (M::CoordType == Coord::Cart) { + u_Phys[0] = ux1(p); + u_Phys[1] = ux2(p); + u_Phys[2] = ux3(p); + } else { + static_assert(D != Dim::_1D, "non-Cartesian SRPIC 1D"); + coord_t x_Code { ZERO }; + x_Code[0] = static_cast(i1(p)) + static_cast(dx1(p)); + x_Code[1] = static_cast(i2(p)) + static_cast(dx2(p)); + if constexpr (D == Dim::_3D) { + x_Code[2] = static_cast(i3(p)) + static_cast(dx3(p)); + } else { + x_Code[2] = phi(p); + } + metric.template transform_xyz( + x_Code, + { ux1(p), ux2(p), ux3(p) }, + u_Phys); + } + if (mass == ZERO) { + gamma = NORM(u_Phys[0], u_Phys[1], u_Phys[2]); + } else { + gamma = math::sqrt(ONE + NORM_SQR(u_Phys[0], u_Phys[1], u_Phys[2])); + } + } else { + // GR + // stress-energy tensor for GR is computed in contravariant basis + static_assert(D != Dim::_1D, "GRPIC 1D"); + coord_t x_Code { ZERO }; + x_Code[0] = static_cast(i1(p)) + static_cast(dx1(p)); + x_Code[1] = static_cast(i2(p)) + static_cast(dx2(p)); + if constexpr (D == Dim::_3D) { + x_Code[2] = static_cast(i3(p)) + static_cast(dx3(p)); + } + vec_t u_Cntrv { ZERO }; + // compute u_i u^i for energy + metric.template transform(x_Code, + { ux1(p), ux2(p), ux3(p) }, + u_Cntrv); + gamma = u_Cntrv[0] * ux1(p) + u_Cntrv[1] * ux2(p) + u_Cntrv[2] * ux3(p); + if (mass == ZERO) { + gamma = math::sqrt(gamma); + } else { + gamma = math::sqrt(ONE + gamma); + } + metric.template transform(x_Code, u_Cntrv, u_Phys); + } + // compute the corresponding moment + coeff = u_Phys[c1 - 1] / gamma; + } else { + // for other cases, use the `contrib` defined above + coeff = contrib; + } + if constexpr (F != FldsID::Nppc) { // for nppc calculation ... // ... do not take volume, weights or smoothing into account diff --git a/src/output/fields.cpp b/src/output/fields.cpp index aa5a752d4..0c2ea5e50 100644 --- a/src/output/fields.cpp +++ b/src/output/fields.cpp @@ -44,6 +44,9 @@ namespace out { } else if (id() == FldsID::T) { // energy-momentum tensor comp = InterpretComponents({ name.substr(1, 1), name.substr(2, 1) }); + } else if (id() == FldsID::V) { + // energy-momentum tensor + comp = InterpretComponents({ name.substr(1, 1) }); } else { // scalar (Rho, divE, Custom, etc.) comp = {}; diff --git a/src/output/fields.h b/src/output/fields.h index a520a246d..4fde18ed2 100644 --- a/src/output/fields.h +++ b/src/output/fields.h @@ -43,7 +43,7 @@ namespace out { [[nodiscard]] auto is_moment() const -> bool { return (id() == FldsID::T || id() == FldsID::Rho || id() == FldsID::Nppc || - id() == FldsID::N || id() == FldsID::Charge); + id() == FldsID::N || id() == FldsID::Charge || id() == FldsID::V); } [[nodiscard]] @@ -94,6 +94,8 @@ namespace out { tmp += m_name.substr(1, 2); } else if (id() == FldsID::A) { tmp += "3"; + } else if (id() == FldsID::V) { + tmp += m_name.substr(1, 1); } else if (is_field()) { tmp += "i"; } From 334a8b18de7341586321e0989ffa565852e87ba7 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Mon, 26 Aug 2024 11:46:57 -0400 Subject: [PATCH 064/183] Update input. --- input.example.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/input.example.toml b/input.example.toml index 788c30685..a49067811 100644 --- a/input.example.toml +++ b/input.example.toml @@ -340,7 +340,7 @@ # Field quantities to output: # @type: array of strings # @valid: fields: "E", "B", "J", "divE" - # @valid: moments: "Rho", "Charge", "N", "Nppc", "T0i", "Tij" + # @valid: moments: "Rho", "Charge", "N", "Nppc", "T0i", "Tij", "Vi" # @valid: for GR: "D", "H", "divD", "A" # @default: [] # @note: For T, you can use unspecified indices, e.g., Tij, T0i, or specific ones, e.g., Ttt, T00, T02, T23 From d0c98aede75239499743b63c409572cb1cc50d16 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Wed, 16 Oct 2024 08:53:39 -0400 Subject: [PATCH 065/183] Bugfix in moment calculation. --- src/kernels/particle_moments.hpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/kernels/particle_moments.hpp b/src/kernels/particle_moments.hpp index 3c6cd37c9..368873cde 100644 --- a/src/kernels/particle_moments.hpp +++ b/src/kernels/particle_moments.hpp @@ -261,10 +261,7 @@ namespace kernel { } // compute the corresponding moment coeff = u_Phys[c1 - 1] / gamma; - } else { - // for other cases, use the `contrib` defined above - coeff = contrib; - } + } if constexpr (F != FldsID::Nppc) { // for nppc calculation ... From e7976ca49c5f3f2e00f4c7316efba5517b5840b0 Mon Sep 17 00:00:00 2001 From: haykh Date: Sun, 9 Mar 2025 10:57:45 -0400 Subject: [PATCH 066/183] minor bookkeeping for 3vel output --- src/framework/domain/output.cpp | 12 ++++++------ src/global/enums.h | 19 ++++++++++--------- src/global/tests/enums.cpp | 6 +++--- src/output/fields.cpp | 17 ++++++++++++----- src/output/fields.h | 4 +--- 5 files changed, 32 insertions(+), 26 deletions(-) diff --git a/src/framework/domain/output.cpp b/src/framework/domain/output.cpp index 8283be825..40871dd2a 100644 --- a/src/framework/domain/output.cpp +++ b/src/framework/domain/output.cpp @@ -330,12 +330,12 @@ namespace ntt { c); } else if (fld.id() == FldsID::V) { ComputeMoments(params, - local_domain->mesh, - local_domain->species, - fld.species, - fld.comp[0], - local_domain->fields.bckp, - c); + local_domain->mesh, + local_domain->species, + fld.species, + fld.comp[0], + local_domain->fields.bckp, + c); } else { raise::Error("Wrong moment requested for output", HERE); } diff --git a/src/global/enums.h b/src/global/enums.h index 4d2348244..6c6a4ec8f 100644 --- a/src/global/enums.h +++ b/src/global/enums.h @@ -13,7 +13,7 @@ * - enum ntt::PrtlPusher // boris, vay, photon, none * - enum ntt::Cooling // synchrotron, none * - enum ntt::FldsID // e, dive, d, divd, b, h, j, - * a, t, rho, charge, n, nppc, custom + * a, t, rho, charge, n, nppc, v, custom * @namespaces: * - ntt:: * @note Enums of the same type can be compared with each other and with strings @@ -287,18 +287,19 @@ namespace ntt { Charge = 11, N = 12, Nppc = 13, - Custom = 14, - V = 15, + V = 14, + Custom = 15, }; constexpr FldsID(uint8_t c) : enums_hidden::BaseEnum { c } {} - static constexpr type variants[] = { E, divE, D, divD, B, H, J, - A, T, Rho, Charge, N, Nppc, Custom , V}; - static constexpr const char* lookup[] = { "e", "dive", "d", "divd", - "b", "h", "j", "a", - "t", "rho", "charge", "n", - "nppc", "custom", "v" }; + static constexpr type variants[] = { E, divE, D, divD, B, + H, J, A, T, Rho, + Charge, N, Nppc, V, Custom }; + static constexpr const char* lookup[] = { "e", "dive", "d", "divd", + "b", "h", "j", "a", + "t", "rho", "charge", "n", + "nppc", "v", "custom" }; static constexpr std::size_t total = sizeof(variants) / sizeof(variants[0]); }; diff --git a/src/global/tests/enums.cpp b/src/global/tests/enums.cpp index d5eeb76e2..673efaf34 100644 --- a/src/global/tests/enums.cpp +++ b/src/global/tests/enums.cpp @@ -66,9 +66,9 @@ auto main() -> int { enum_str_t all_particle_pushers = { "boris", "vay", "photon", "none" }; enum_str_t all_coolings = { "synchrotron", "none" }; - enum_str_t all_out_flds = { "e", "dive", "d", "divd", "b", - "h", "j", "a", "t", "rho", - "charge", "n", "nppc", "custom" , "v"}; + enum_str_t all_out_flds = { "e", "dive", "d", "divd", "b", + "h", "j", "a", "t", "rho", + "charge", "n", "nppc", "v", "custom" }; checkEnum(all_coords); checkEnum(all_metrics); diff --git a/src/output/fields.cpp b/src/output/fields.cpp index 0c2ea5e50..678ca20f6 100644 --- a/src/output/fields.cpp +++ b/src/output/fields.cpp @@ -29,14 +29,24 @@ namespace out { } else { m_id = FldsID::Custom; } + // check compatibility + raise::ErrorIf(is_gr_aux_field() and S != SimEngine::GRPIC, + "GR auxiliary field output not supported for non-GRPIC", + HERE); + raise::ErrorIf(id() == FldsID::A and S != SimEngine::GRPIC, + "Output of A_phi not supported for non-GRPIC", + HERE); + raise::ErrorIf(id() == FldsID::V and S == SimEngine::GRPIC, + "Output of bulk 3-vel not supported for GRPIC", + HERE); // determine the species and components to output if (is_moment()) { species = InterpretSpecies(name); } else { species = {}; } - if (is_field() || is_current()) { - // always write all the field/current components + if (is_field() || is_current() || id() == FldsID::V) { + // always write all the field/current/bulk vel components comp = { { 1 }, { 2 }, { 3 } }; } else if (id() == FldsID::A) { // only write A3 @@ -44,9 +54,6 @@ namespace out { } else if (id() == FldsID::T) { // energy-momentum tensor comp = InterpretComponents({ name.substr(1, 1), name.substr(2, 1) }); - } else if (id() == FldsID::V) { - // energy-momentum tensor - comp = InterpretComponents({ name.substr(1, 1) }); } else { // scalar (Rho, divE, Custom, etc.) comp = {}; diff --git a/src/output/fields.h b/src/output/fields.h index 4fde18ed2..0e8e31d08 100644 --- a/src/output/fields.h +++ b/src/output/fields.h @@ -94,9 +94,7 @@ namespace out { tmp += m_name.substr(1, 2); } else if (id() == FldsID::A) { tmp += "3"; - } else if (id() == FldsID::V) { - tmp += m_name.substr(1, 1); - } else if (is_field()) { + } else if (is_field() || id() == FldsID::V) { tmp += "i"; } if (species.size() > 0) { From c9d69ce682e660e7ed0f3c43d414856717094550 Mon Sep 17 00:00:00 2001 From: haykh Date: Sun, 9 Mar 2025 14:26:41 -0400 Subject: [PATCH 067/183] implemented and tested bulk V output --- src/framework/domain/output.cpp | 69 ++++++++++---- src/kernels/particle_moments.hpp | 154 ++++++++++++++++++++----------- src/output/fields.cpp | 3 - 3 files changed, 151 insertions(+), 75 deletions(-) diff --git a/src/framework/domain/output.cpp b/src/framework/domain/output.cpp index 40871dd2a..685099e55 100644 --- a/src/framework/domain/output.cpp +++ b/src/framework/domain/output.cpp @@ -328,14 +328,6 @@ namespace ntt { {}, local_domain->fields.bckp, c); - } else if (fld.id() == FldsID::V) { - ComputeMoments(params, - local_domain->mesh, - local_domain->species, - fld.species, - fld.comp[0], - local_domain->fields.bckp, - c); } else { raise::Error("Wrong moment requested for output", HERE); } @@ -365,16 +357,35 @@ namespace ntt { if (fld.is_moment()) { for (auto i = 0; i < 3; ++i) { const auto c = static_cast(addresses[i]); - raise::ErrorIf(fld.comp[i].size() != 2, - "Wrong # of components requested for moment", - HERE); - ComputeMoments(params, - local_domain->mesh, - local_domain->species, - fld.species, - fld.comp[i], - local_domain->fields.bckp, - c); + if (fld.id() == FldsID::T) { + raise::ErrorIf(fld.comp[i].size() != 2, + "Wrong # of components requested for moment", + HERE); + ComputeMoments(params, + local_domain->mesh, + local_domain->species, + fld.species, + fld.comp[i], + local_domain->fields.bckp, + c); + } else if (fld.id() == FldsID::V) { + raise::ErrorIf(fld.comp[i].size() != 1, + "Wrong # of components requested for 3vel", + HERE); + if constexpr (S == SimEngine::SRPIC) { + ComputeMoments(params, + local_domain->mesh, + local_domain->species, + fld.species, + fld.comp[i], + local_domain->fields.bckp, + c); + } else { + raise::Error("Bulk velocity not supported for GRPIC", HERE); + } + } else { + raise::Error("Wrong moment requested for output", HERE); + } } raise::ErrorIf(addresses[1] - addresses[0] != addresses[2] - addresses[1], @@ -383,6 +394,28 @@ namespace ntt { SynchronizeFields(*local_domain, Comm::Bckp, { addresses[0], addresses[2] + 1 }); + if constexpr (S == SimEngine::SRPIC) { + if (fld.id() == FldsID::V) { + // normalize 3vel * rho (combuted above) by rho + ComputeMoments(params, + local_domain->mesh, + local_domain->species, + fld.species, + {}, + local_domain->fields.bckp, + 0u); + SynchronizeFields(*local_domain, Comm::Bckp, { 0, 1 }); + Kokkos::parallel_for("NormalizeVectorByRho", + local_domain->mesh.rangeActiveCells(), + kernel::NormalizeVectorByRho_kernel( + local_domain->fields.bckp, + local_domain->fields.bckp, + 0, + addresses[0], + addresses[1], + addresses[2])); + } + } } else { // copy fields to bckp (:, 0, 1, 2) // if as-is specified ==> copy directly to 3, 4, 5 diff --git a/src/kernels/particle_moments.hpp b/src/kernels/particle_moments.hpp index 368873cde..e52c68486 100644 --- a/src/kernels/particle_moments.hpp +++ b/src/kernels/particle_moments.hpp @@ -14,6 +14,7 @@ #include "global.h" #include "arch/kokkos_aliases.h" +#include "utils/comparators.h" #include "utils/error.h" #include "utils/numeric.h" @@ -40,8 +41,10 @@ namespace kernel { static_assert(M::is_metric, "M must be a metric class"); static constexpr auto D = M::Dim; - static_assert((F == FldsID::Rho) || (F == FldsID::Charge) || - (F == FldsID::N) || (F == FldsID::Nppc) || (F == FldsID::T) || (F == FldsID::V), + static_assert(!((S == SimEngine::GRPIC) && (F == FldsID::V)), + "Bulk velocity not supported for GRPIC"); + static_assert((F == FldsID::Rho) || (F == FldsID::Charge) || (F == FldsID::N) || + (F == FldsID::Nppc) || (F == FldsID::T) || (F == FldsID::V), "Invalid field ID"); const unsigned short c1, c2; @@ -90,7 +93,7 @@ namespace kernel { real_t inv_n0, unsigned short window) : c1 { (components.size() > 0) ? components[0] - : static_cast(0) } + : static_cast(0) } , c2 { (components.size() == 2) ? components[1] : static_cast(0) } , Buff { scatter_buff } @@ -200,68 +203,38 @@ namespace kernel { coeff *= u_Phys[c - 1]; } } - } else { - // for other cases, use the `contrib` defined above - coeff = contrib; - } - - if constexpr (F == FldsID::V) { + } else if constexpr (F == FldsID::V) { real_t gamma { ZERO }; - // for stress-energy tensor + // for bulk 3vel (tetrad basis) vec_t u_Phys { ZERO }; - if constexpr (S == SimEngine::SRPIC) { - // SR - // stress-energy tensor for SR is computed in the tetrad (hatted) basis - if constexpr (M::CoordType == Coord::Cart) { - u_Phys[0] = ux1(p); - u_Phys[1] = ux2(p); - u_Phys[2] = ux3(p); - } else { - static_assert(D != Dim::_1D, "non-Cartesian SRPIC 1D"); - coord_t x_Code { ZERO }; - x_Code[0] = static_cast(i1(p)) + static_cast(dx1(p)); - x_Code[1] = static_cast(i2(p)) + static_cast(dx2(p)); - if constexpr (D == Dim::_3D) { - x_Code[2] = static_cast(i3(p)) + static_cast(dx3(p)); - } else { - x_Code[2] = phi(p); - } - metric.template transform_xyz( - x_Code, - { ux1(p), ux2(p), ux3(p) }, - u_Phys); - } - if (mass == ZERO) { - gamma = NORM(u_Phys[0], u_Phys[1], u_Phys[2]); - } else { - gamma = math::sqrt(ONE + NORM_SQR(u_Phys[0], u_Phys[1], u_Phys[2])); - } + if constexpr (M::CoordType == Coord::Cart) { + u_Phys[0] = ux1(p); + u_Phys[1] = ux2(p); + u_Phys[2] = ux3(p); } else { - // GR - // stress-energy tensor for GR is computed in contravariant basis - static_assert(D != Dim::_1D, "GRPIC 1D"); - coord_t x_Code { ZERO }; + coord_t x_Code { ZERO }; x_Code[0] = static_cast(i1(p)) + static_cast(dx1(p)); x_Code[1] = static_cast(i2(p)) + static_cast(dx2(p)); if constexpr (D == Dim::_3D) { x_Code[2] = static_cast(i3(p)) + static_cast(dx3(p)); - } - vec_t u_Cntrv { ZERO }; - // compute u_i u^i for energy - metric.template transform(x_Code, - { ux1(p), ux2(p), ux3(p) }, - u_Cntrv); - gamma = u_Cntrv[0] * ux1(p) + u_Cntrv[1] * ux2(p) + u_Cntrv[2] * ux3(p); - if (mass == ZERO) { - gamma = math::sqrt(gamma); } else { - gamma = math::sqrt(ONE + gamma); + x_Code[2] = phi(p); } - metric.template transform(x_Code, u_Cntrv, u_Phys); + metric.template transform_xyz(x_Code, + { ux1(p), ux2(p), ux3(p) }, + u_Phys); + } + if (mass == ZERO) { + gamma = NORM(u_Phys[0], u_Phys[1], u_Phys[2]); + } else { + gamma = math::sqrt(ONE + NORM_SQR(u_Phys[0], u_Phys[1], u_Phys[2])); } // compute the corresponding moment - coeff = u_Phys[c1 - 1] / gamma; - } + coeff = (mass == ZERO ? ONE : mass) * u_Phys[c1 - 1] / gamma; + } else { + // for other cases, use the `contrib` defined above + coeff = contrib; + } if constexpr (F != FldsID::Nppc) { // for nppc calculation ... @@ -346,6 +319,79 @@ namespace kernel { } }; + template + class NormalizeVectorByRho_kernel { + const ndfield_t Rho; + ndfield_t Vector; + const unsigned short c_rho, c_v1, c_v2, c_v3; + + public: + NormalizeVectorByRho_kernel(const ndfield_t& rho, + const ndfield_t& vector, + unsigned short crho, + unsigned short cv1, + unsigned short cv2, + unsigned short cv3) + : Rho { rho } + , Vector { vector } + , c_rho { crho } + , c_v1 { cv1 } + , c_v2 { cv2 } + , c_v3 { cv3 } { + raise::ErrorIf(c_rho >= N or c_v1 >= N or c_v2 >= N or c_v3 >= N, + "Invalid component index", + HERE); + raise::ErrorIf(c_rho == c_v1 or c_rho == c_v2 or c_rho == c_v3, + "Invalid component index", + HERE); + raise::ErrorIf(c_v1 == c_v2 or c_v1 == c_v3 or c_v2 == c_v3, + "Invalid component index", + HERE); + } + + Inline void operator()(index_t i1) const { + if constexpr (D == Dim::_1D) { + if (not cmp::AlmostZero(Rho(i1, c_rho))) { + Vector(i1, c_v1) /= Rho(i1, c_rho); + Vector(i1, c_v2) /= Rho(i1, c_rho); + Vector(i1, c_v3) /= Rho(i1, c_rho); + } + } else { + raise::KernelError( + HERE, + "1D implementation of NormalizeVectorByRho_kernel called for non-1D"); + } + } + + Inline void operator()(index_t i1, index_t i2) const { + if constexpr (D == Dim::_2D) { + if (not cmp::AlmostZero(Rho(i1, i2, c_rho))) { + Vector(i1, i2, c_v1) /= Rho(i1, i2, c_rho); + Vector(i1, i2, c_v2) /= Rho(i1, i2, c_rho); + Vector(i1, i2, c_v3) /= Rho(i1, i2, c_rho); + } + } else { + raise::KernelError( + HERE, + "2D implementation of NormalizeVectorByRho_kernel called for non-2D"); + } + } + + Inline void operator()(index_t i1, index_t i2, index_t i3) const { + if constexpr (D == Dim::_3D) { + if (not cmp::AlmostZero(Rho(i1, i2, i3, c_rho))) { + Vector(i1, i2, i3, c_v1) /= Rho(i1, i2, i3, c_rho); + Vector(i1, i2, i3, c_v2) /= Rho(i1, i2, i3, c_rho); + Vector(i1, i2, i3, c_v3) /= Rho(i1, i2, i3, c_rho); + } + } else { + raise::KernelError( + HERE, + "3D implementation of NormalizeVectorByRho_kernel called for non-3D"); + } + } + }; + } // namespace kernel #endif // KERNELS_PARTICLE_MOMENTS_HPP diff --git a/src/output/fields.cpp b/src/output/fields.cpp index 678ca20f6..091f04cd9 100644 --- a/src/output/fields.cpp +++ b/src/output/fields.cpp @@ -30,9 +30,6 @@ namespace out { m_id = FldsID::Custom; } // check compatibility - raise::ErrorIf(is_gr_aux_field() and S != SimEngine::GRPIC, - "GR auxiliary field output not supported for non-GRPIC", - HERE); raise::ErrorIf(id() == FldsID::A and S != SimEngine::GRPIC, "Output of A_phi not supported for non-GRPIC", HERE); From b6fe114ab719e45d58ebb6a4241cac3407df5040 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 10 Mar 2025 12:00:38 -0500 Subject: [PATCH 068/183] revert to zero at boundary --- src/engines/srpic.hpp | 28 +----------------------- src/kernels/fields_bcs.hpp | 44 ++++++++++++-------------------------- 2 files changed, 15 insertions(+), 57 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 891336309..96747e429 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -890,36 +890,10 @@ namespace ntt { raise::Error("Invalid dimension", HERE); } - auto match_fields = m_pgen.MatchFields(time); Kokkos::parallel_for( "ConductorFields", range, - kernel::bc::ConductorBoundaries_kernel(domain.fields.em, - match_fields, tags)); - - // if constexpr (M::Dim == Dim::_1D) { - // Kokkos::parallel_for( - // "MatchFields", - // CreateRangePolicy({ xi_min[0] }, { xi_max[0] }), - // kernel::bc::ConductorBoundaries_kernel(domain.fields.em, - // tags)); - // } else if constexpr (M::Dim == Dim::_2D) { - // Kokkos::parallel_for( - // "MatchFields", - // CreateRangePolicy({ xi_min[0], xi_min[1] }, - // { xi_max[0], xi_max[1] }), - // kernel::bc::ConductorBoundaries_kernel(domain.fields.em, - // tags)); - // } else if constexpr (M::Dim == Dim::_3D) { - // Kokkos::parallel_for( - // "MatchFields", - // CreateRangePolicy({ xi_min[0], xi_min[1], xi_min[2] }, - // { xi_max[0], xi_max[1], xi_max[2] }), - // kernel::bc::ConductorBoundaries_kernel(domain.fields.em, - // tags)); - // } else { - // raise::Error("Invalid dimension", HERE); - // } + kernel::bc::ConductorBoundaries_kernel(domain.fields.em, tags)); } } diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 4d6be9380..b1ee999d3 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -486,32 +486,24 @@ namespace kernel::bc { } }; - template + template struct ConductorBoundaries_kernel { static_assert(static_cast(o) < static_cast(D), "Invalid component index"); - static constexpr bool defines_ex2 = traits::has_method::value; - static constexpr bool defines_ex3 = traits::has_method::value; - static constexpr bool defines_bx1 = traits::has_method::value; - static_assert(( defines_ex2 or defines_ex3 or defines_bx1), - "none of the components of E or B are specified in PGEN"); ndfield_t Fld; - const I fset; const BCTags tags; - ConductorBoundaries_kernel(ndfield_t Fld, const I& fset, BCTags tags) + ConductorBoundaries_kernel(ndfield_t Fld, BCTags tags) : Fld { Fld } - , fset { fset } , tags { tags } {} Inline void operator()(index_t i1) const { if constexpr (D == Dim::_1D) { - coord_t x_Ph_H { ZERO }; if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, em::ex2) = fset.ex2(x_Ph_H); - Fld(N_GHOSTS, em::ex3) = fset.ex3(x_Ph_H); + Fld(N_GHOSTS, em::ex2) = ZERO; + Fld(N_GHOSTS, em::ex3) = ZERO; } else { Fld(N_GHOSTS - i1, em::ex1) = Fld(N_GHOSTS + i1 - 1, em::ex1); Fld(N_GHOSTS - i1, em::ex2) = -Fld(N_GHOSTS + i1, em::ex2); @@ -521,7 +513,7 @@ namespace kernel::bc { if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, em::bx1) = fset.bx1(x_Ph_H); + Fld(N_GHOSTS, em::bx1) = ZERO; } else { Fld(N_GHOSTS - i1, em::bx1) = -Fld(N_GHOSTS + i1, em::bx1); Fld(N_GHOSTS - i1, em::bx2) = Fld(N_GHOSTS + i1 - 1, em::bx2); @@ -537,11 +529,10 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2) const { if constexpr (D == Dim::_2D) { - coord_t x_Ph_H { ZERO }; if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, i2, em::ex2) = fset.ex2(x_Ph_H); - Fld(N_GHOSTS, i2, em::ex3) = fset.ex3(x_Ph_H); + Fld(N_GHOSTS, i2, em::ex2) = ZERO; + Fld(N_GHOSTS, i2, em::ex3) = ZERO; } else { Fld(N_GHOSTS - i1, i2, em::ex1) = Fld(N_GHOSTS + i1 - 1, i2, em::ex1); Fld(N_GHOSTS - i1, i2, em::ex2) = -Fld(N_GHOSTS + i1, i2, em::ex2); @@ -551,7 +542,7 @@ namespace kernel::bc { if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, i2, em::bx1) = fset.bx1(x_Ph_H); + Fld(N_GHOSTS, i2, em::bx1) = ZERO; } else { Fld(N_GHOSTS - i1, i2, em::bx1) = -Fld(N_GHOSTS + i1, i2, em::bx1); Fld(N_GHOSTS - i1, i2, em::bx2) = Fld(N_GHOSTS + i1 - 1, i2, em::bx2); @@ -567,16 +558,13 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2, index_t i3) const { if constexpr (D == Dim::_3D) { - coord_t x_Ph_H { ZERO }; if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, i2, i3, em::ex2) = fset.ex2(x_Ph_H); - Fld(N_GHOSTS, i2, i3, em::ex3) = fset.ex3(x_Ph_H); + Fld(N_GHOSTS, i2, i3, em::ex2) = ZERO; + Fld(N_GHOSTS, i2, i3, em::ex3) = ZERO; } else { Fld(N_GHOSTS - i1, i2, i3, em::ex1) = Fld(N_GHOSTS + i1 - 1, - i2, - i3, - em::ex1); + i2, i3, em::ex1); Fld(N_GHOSTS - i1, i2, i3, em::ex2) = -Fld(N_GHOSTS + i1, i2, i3, em::ex2); Fld(N_GHOSTS - i1, i2, i3, em::ex3) = -Fld(N_GHOSTS + i1, i2, i3, em::ex3); } @@ -584,17 +572,13 @@ namespace kernel::bc { if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, i2, i3, em::bx1) = fset.bx1(x_Ph_H); + Fld(N_GHOSTS, i2, i3, em::bx1) = ZERO; } else { Fld(N_GHOSTS - i1, i2, i3, em::bx1) = -Fld(N_GHOSTS + i1, i2, i3, em::bx1); Fld(N_GHOSTS - i1, i2, i3, em::bx2) = Fld(N_GHOSTS + i1 - 1, - i2, - i3, - em::bx2); + i2, i3, em::bx2); Fld(N_GHOSTS - i1, i2, i3, em::bx3) = Fld(N_GHOSTS + i1 - 1, - i2, - i3, - em::bx3); + i2, i3, em::bx3); } } } else { From 15ffa1b7a86d9e562d226320788882939b9a8758 Mon Sep 17 00:00:00 2001 From: haykh Date: Thu, 13 Mar 2025 01:56:06 -0400 Subject: [PATCH 069/183] types made uniform --- src/archetypes/energy_dist.h | 10 +- src/archetypes/particle_injector.h | 6 +- src/archetypes/spatial_dist.h | 12 +-- src/checkpoint/reader.cpp | 50 +++++----- src/checkpoint/reader.h | 12 +-- src/checkpoint/writer.cpp | 80 ++++++++-------- src/checkpoint/writer.h | 24 ++--- src/engines/engine.hpp | 27 +++--- src/engines/engine_init.cpp | 2 +- src/engines/engine_printer.cpp | 15 ++- src/engines/engine_run.cpp | 6 +- src/engines/srpic.hpp | 24 ++--- src/framework/containers/fields.cpp | 6 +- src/framework/containers/fields.h | 2 +- src/framework/containers/particles.cpp | 49 +++++----- src/framework/containers/particles.h | 18 ++-- src/framework/containers/species.h | 6 +- src/framework/domain/checkpoint.cpp | 30 +++--- src/framework/domain/comm_mpi.hpp | 38 ++++---- src/framework/domain/communications.cpp | 35 ++++--- src/framework/domain/domain.h | 12 +-- src/framework/domain/grid.cpp | 14 +-- src/framework/domain/grid.h | 20 ++-- src/framework/domain/mesh.h | 40 ++++---- src/framework/domain/metadomain.cpp | 18 ++-- src/framework/domain/metadomain.h | 18 ++-- src/framework/domain/output.cpp | 40 ++++---- src/framework/parameters.cpp | 46 +++++----- src/framework/parameters.h | 2 +- src/framework/simulation.cpp | 16 ++-- src/framework/tests/comm_mpi.cpp | 4 +- src/framework/tests/metadomain.cpp | 30 +++--- src/global/arch/directions.h | 10 +- src/global/arch/kokkos_aliases.cpp | 28 +++--- src/global/arch/kokkos_aliases.h | 23 +++-- src/global/defaults.h | 12 +-- src/global/enums.h | 4 +- src/global/global.h | 14 ++- src/global/utils/diag.cpp | 34 +++---- src/global/utils/diag.h | 14 +-- src/global/utils/param_container.cpp | 74 ++++++++------- src/global/utils/progressbar.cpp | 39 ++++---- src/global/utils/progressbar.h | 19 ++-- src/global/utils/timer.cpp | 24 ++--- src/global/utils/timer.h | 26 +++--- src/global/utils/tools.h | 58 ++++++------ src/kernels/ampere_gr.hpp | 20 ++-- src/kernels/ampere_sr.hpp | 40 ++++---- src/kernels/comm.hpp | 117 ++++++++++++------------ src/kernels/digital_filter.hpp | 6 +- src/kernels/faraday_gr.hpp | 2 +- src/kernels/faraday_sr.hpp | 6 +- src/kernels/fields_bcs.hpp | 18 ++-- src/kernels/injectors.hpp | 65 +++++++------ src/kernels/particle_moments.hpp | 2 +- src/kernels/prtls_to_phys.hpp | 4 +- src/metrics/kerr_schild.h | 2 +- src/metrics/kerr_schild_0.h | 4 +- src/metrics/metric_base.h | 2 +- src/metrics/minkowski.h | 7 +- src/metrics/qkerr_schild.h | 2 +- src/metrics/qspherical.h | 5 +- src/metrics/spherical.h | 7 +- src/output/fields.h | 2 +- src/output/tests/writer-nompi.cpp | 21 ++--- src/output/utils/attr_writer.h | 10 +- src/output/writer.cpp | 73 ++++++++------- src/output/writer.h | 24 ++--- 68 files changed, 779 insertions(+), 751 deletions(-) diff --git a/src/archetypes/energy_dist.h b/src/archetypes/energy_dist.h index e9bc9051a..231ea5b0b 100644 --- a/src/archetypes/energy_dist.h +++ b/src/archetypes/energy_dist.h @@ -78,14 +78,14 @@ namespace arch { real_t g_max, real_t pl_ind) : EnergyDistribution { metric } - , pool { pool } , g_min { g_min } , g_max { g_max } - , pl_ind { pl_ind } {} + , pl_ind { pl_ind } + , pool { pool } {} Inline void operator()(const coord_t& x_Code, vec_t& v, - unsigned short sp = 0) const override { + unsigned short = 0) const override { auto rand_gen = pool.get_state(); auto rand_X1 = Random(rand_gen); auto rand_gam = ONE; @@ -225,7 +225,7 @@ namespace arch { Inline void operator()(const coord_t& x_Code, vec_t& v, - unsigned short s = 0) const override { + unsigned short sp = 0) const override { if (cmp::AlmostZero(temperature)) { v[0] = ZERO; v[1] = ZERO; @@ -245,7 +245,7 @@ namespace arch { // boost only when using cartesian coordinates if (not cmp::AlmostZero(boost_velocity)) { boost(v); - if (not zero_current and s % 2 == 0) { + if (not zero_current and sp % 2 == 0) { v[0] = -v[0]; v[1] = -v[1]; v[2] = -v[2]; diff --git a/src/archetypes/particle_injector.h b/src/archetypes/particle_injector.h index cbcbbd389..3884e6ced 100644 --- a/src/archetypes/particle_injector.h +++ b/src/archetypes/particle_injector.h @@ -208,13 +208,13 @@ namespace arch { auto ppc0 = params.template get("particles.ppc0"); array_t ni { "ni", M::Dim }; auto ni_h = Kokkos::create_mirror_view(ni); - std::size_t ncells = 1; + ncells_t ncells = 1; for (auto d = 0; d < M::Dim; ++d) { ni_h(d) = domain.mesh.n_active()[d]; ncells *= domain.mesh.n_active()[d]; } Kokkos::deep_copy(ni, ni_h); - const auto nparticles = static_cast( + const auto nparticles = static_cast( (long double)(ppc0 * number_density * 0.5) * (long double)(ncells)); Kokkos::parallel_for( @@ -320,7 +320,7 @@ namespace arch { incl_ghosts.push_back({ false, false }); } const auto extent = domain.mesh.ExtentToRange(box, incl_ghosts); - tuple_t x_min { 0 }, x_max { 0 }; + tuple_t x_min { 0 }, x_max { 0 }; for (auto d = 0; d < M::Dim; ++d) { x_min[d] = extent[d].first; x_max[d] = extent[d].second; diff --git a/src/archetypes/spatial_dist.h b/src/archetypes/spatial_dist.h index be2836da2..d036c0166 100644 --- a/src/archetypes/spatial_dist.h +++ b/src/archetypes/spatial_dist.h @@ -74,15 +74,15 @@ namespace arch { metric.template convert(x_Ph, x_Cd); real_t dens { ZERO }; if constexpr (M::Dim == Dim::_1D) { - dens = density(static_cast(x_Cd[0]) + N_GHOSTS, idx); + dens = density(static_cast(x_Cd[0]) + N_GHOSTS, idx); } else if constexpr (M::Dim == Dim::_2D) { - dens = density(static_cast(x_Cd[0]) + N_GHOSTS, - static_cast(x_Cd[1]) + N_GHOSTS, + dens = density(static_cast(x_Cd[0]) + N_GHOSTS, + static_cast(x_Cd[1]) + N_GHOSTS, idx); } else if constexpr (M::Dim == Dim::_3D) { - dens = density(static_cast(x_Cd[0]) + N_GHOSTS, - static_cast(x_Cd[1]) + N_GHOSTS, - static_cast(x_Cd[2]) + N_GHOSTS, + dens = density(static_cast(x_Cd[0]) + N_GHOSTS, + static_cast(x_Cd[1]) + N_GHOSTS, + static_cast(x_Cd[2]) + N_GHOSTS, idx); } else { raise::KernelError(HERE, "Invalid dimension"); diff --git a/src/checkpoint/reader.cpp b/src/checkpoint/reader.cpp index 9fc2d2640..6e32cb3b9 100644 --- a/src/checkpoint/reader.cpp +++ b/src/checkpoint/reader.cpp @@ -40,15 +40,13 @@ namespace checkpoint { } } - auto ReadParticleCount( - adios2::IO& io, - adios2::Engine& reader, - unsigned short s, - std::size_t local_dom, - std::size_t ndomains) -> std::pair { + auto ReadParticleCount(adios2::IO& io, + adios2::Engine& reader, + unsigned short s, + std::size_t local_dom, + std::size_t ndomains) -> std::pair { logger::Checkpoint(fmt::format("Reading particle count for: %d", s + 1), HERE); - auto npart_var = io.InquireVariable( - fmt::format("s%d_npart", s + 1)); + auto npart_var = io.InquireVariable(fmt::format("s%d_npart", s + 1)); if (npart_var) { raise::ErrorIf(npart_var.Shape()[0] != ndomains, "npart_var.Shape()[0] != ndomains", @@ -57,21 +55,21 @@ namespace checkpoint { "npart_var.Shape().size() != 1", HERE); npart_var.SetSelection(adios2::Box({ local_dom }, { 1 })); - std::size_t npart; + npart_t npart; reader.Get(npart_var, &npart, adios2::Mode::Sync); const auto loc_npart = npart; #if !defined(MPI_ENABLED) - std::size_t offset_npart = 0; + npart_t offset_npart = 0; #else - std::vector glob_nparts(ndomains); + std::vector glob_nparts(ndomains); MPI_Allgather(&loc_npart, 1, - mpi::get_type(), + mpi::get_type(), glob_nparts.data(), 1, - mpi::get_type(), + mpi::get_type(), MPI_COMM_WORLD); - std::size_t offset_npart = 0; + npart_t offset_npart = 0; for (auto d { 0u }; d < local_dom; ++d) { offset_npart += glob_nparts[d]; } @@ -89,8 +87,8 @@ namespace checkpoint { const std::string& quantity, unsigned short s, array_t& array, - std::size_t count, - std::size_t offset) { + npart_t count, + npart_t offset) { logger::Checkpoint( fmt::format("Reading quantity: s%d_%s", s + 1, quantity.c_str()), HERE); @@ -115,8 +113,8 @@ namespace checkpoint { unsigned short s, array_t& array, std::size_t nplds, - std::size_t count, - std::size_t offset) { + npart_t count, + npart_t offset) { logger::Checkpoint(fmt::format("Reading quantity: s%d_plds", s + 1), HERE); auto var = io.InquireVariable(fmt::format("s%d_plds", s + 1)); if (var) { @@ -168,28 +166,28 @@ namespace checkpoint { const std::string&, unsigned short, array_t&, - std::size_t, - std::size_t); + npart_t, + npart_t); template void ReadParticleData(adios2::IO&, adios2::Engine&, const std::string&, unsigned short, array_t&, - std::size_t, - std::size_t); + npart_t, + npart_t); template void ReadParticleData(adios2::IO&, adios2::Engine&, const std::string&, unsigned short, array_t&, - std::size_t, - std::size_t); + npart_t, + npart_t); template void ReadParticleData(adios2::IO&, adios2::Engine&, const std::string&, unsigned short, array_t&, - std::size_t, - std::size_t); + npart_t, + npart_t); } // namespace checkpoint diff --git a/src/checkpoint/reader.h b/src/checkpoint/reader.h index e5a91ab75..883a1d125 100644 --- a/src/checkpoint/reader.h +++ b/src/checkpoint/reader.h @@ -4,7 +4,7 @@ * @implements * - checkpoint::ReadFields -> void * - checkpoint::ReadParticleData -> void - * - checkpoint::ReadParticleCount -> std::pair + * - checkpoint::ReadParticleCount -> std::pair * @cpp: * - reader.cpp * @namespaces: @@ -34,7 +34,7 @@ namespace checkpoint { adios2::Engine&, unsigned short, std::size_t, - std::size_t) -> std::pair; + std::size_t) -> std::pair; template void ReadParticleData(adios2::IO&, @@ -42,16 +42,16 @@ namespace checkpoint { const std::string&, unsigned short, array_t&, - std::size_t, - std::size_t); + npart_t, + npart_t); void ReadParticlePayloads(adios2::IO&, adios2::Engine&, unsigned short, array_t&, std::size_t, - std::size_t, - std::size_t); + npart_t, + npart_t); } // namespace checkpoint diff --git a/src/checkpoint/writer.cpp b/src/checkpoint/writer.cpp index a12e3ef26..c5f7e5181 100644 --- a/src/checkpoint/writer.cpp +++ b/src/checkpoint/writer.cpp @@ -21,8 +21,8 @@ namespace checkpoint { void Writer::init(adios2::ADIOS* ptr_adios, - std::size_t interval, - long double interval_time, + timestep_t interval, + simtime_t interval_time, int keep) { m_keep = keep; m_enabled = keep != 0; @@ -36,8 +36,8 @@ namespace checkpoint { m_io = p_adios->DeclareIO("Entity::Checkpoint"); m_io.SetEngine("BPFile"); - m_io.DefineVariable("Step"); - m_io.DefineVariable("Time"); + m_io.DefineVariable("Step"); + m_io.DefineVariable("Time"); m_io.DefineAttribute("NGhosts", ntt::N_GHOSTS); CallOnce([]() { @@ -48,13 +48,13 @@ namespace checkpoint { }); } - void Writer::defineFieldVariables(const ntt::SimEngine& S, - const std::vector& glob_shape, - const std::vector& loc_corner, - const std::vector& loc_shape) { - auto gs6 = std::vector(glob_shape.begin(), glob_shape.end()); - auto lc6 = std::vector(loc_corner.begin(), loc_corner.end()); - auto ls6 = std::vector(loc_shape.begin(), loc_shape.end()); + void Writer::defineFieldVariables(const ntt::SimEngine& S, + const std::vector& glob_shape, + const std::vector& loc_corner, + const std::vector& loc_shape) { + auto gs6 = std::vector(glob_shape.begin(), glob_shape.end()); + auto lc6 = std::vector(loc_corner.begin(), loc_corner.end()); + auto ls6 = std::vector(loc_shape.begin(), loc_shape.end()); gs6.push_back(6); lc6.push_back(0); ls6.push_back(6); @@ -62,9 +62,9 @@ namespace checkpoint { m_io.DefineVariable("em", gs6, lc6, ls6); if (S == ntt::SimEngine::GRPIC) { m_io.DefineVariable("em0", gs6, lc6, ls6); - auto gs3 = std::vector(glob_shape.begin(), glob_shape.end()); - auto lc3 = std::vector(loc_corner.begin(), loc_corner.end()); - auto ls3 = std::vector(loc_shape.begin(), loc_shape.end()); + auto gs3 = std::vector(glob_shape.begin(), glob_shape.end()); + auto lc3 = std::vector(loc_corner.begin(), loc_corner.end()); + auto ls3 = std::vector(loc_shape.begin(), loc_shape.end()); gs3.push_back(3); lc3.push_back(0); ls3.push_back(3); @@ -80,10 +80,10 @@ namespace checkpoint { "Number of payloads does not match the number of species", HERE); for (auto s { 0u }; s < nspec; ++s) { - m_io.DefineVariable(fmt::format("s%d_npart", s + 1), - { adios2::UnknownDim }, - { adios2::UnknownDim }, - { adios2::UnknownDim }); + m_io.DefineVariable(fmt::format("s%d_npart", s + 1), + { adios2::UnknownDim }, + { adios2::UnknownDim }, + { adios2::UnknownDim }); for (auto d { 0u }; d < dim; ++d) { m_io.DefineVariable(fmt::format("s%d_i%d", s + 1, d + 1), @@ -135,11 +135,11 @@ namespace checkpoint { } } - auto Writer::shouldSave(std::size_t step, long double time) -> bool { + auto Writer::shouldSave(timestep_t step, simtime_t time) -> bool { return m_enabled and m_tracker.shouldWrite(step, time); } - void Writer::beginSaving(std::size_t step, long double time) { + void Writer::beginSaving(timestep_t step, simtime_t time) { raise::ErrorIf(!m_enabled, "Checkpoint is not enabled", HERE); raise::ErrorIf(p_adios == nullptr, "ADIOS pointer is null", HERE); if (m_writing_mode) { @@ -160,8 +160,8 @@ namespace checkpoint { } m_writer.BeginStep(); - m_writer.Put(m_io.InquireVariable("Step"), &step); - m_writer.Put(m_io.InquireVariable("Time"), &time); + m_writer.Put(m_io.InquireVariable("Step"), &step); + m_writer.Put(m_io.InquireVariable("Time"), &time); } void Writer::endSaving() { @@ -226,9 +226,9 @@ namespace checkpoint { template void Writer::saveParticleQuantity(const std::string& quantity, - std::size_t glob_total, - std::size_t loc_offset, - std::size_t loc_size, + npart_t glob_total, + npart_t loc_offset, + npart_t loc_size, const array_t& data) { const auto slice = range_tuple_t(0, loc_size); auto var = m_io.InquireVariable(quantity); @@ -244,9 +244,9 @@ namespace checkpoint { void Writer::saveParticlePayloads(const std::string& quantity, std::size_t nplds, - std::size_t glob_total, - std::size_t loc_offset, - std::size_t loc_size, + npart_t glob_total, + npart_t loc_offset, + npart_t loc_size, const array_t& data) { const auto slice = range_tuple_t(0, loc_size); auto var = m_io.InquireVariable(quantity); @@ -292,23 +292,23 @@ namespace checkpoint { const ndfield_t&); template void Writer::saveParticleQuantity(const std::string&, - std::size_t, - std::size_t, - std::size_t, + npart_t, + npart_t, + npart_t, const array_t&); template void Writer::saveParticleQuantity(const std::string&, - std::size_t, - std::size_t, - std::size_t, + npart_t, + npart_t, + npart_t, const array_t&); template void Writer::saveParticleQuantity(const std::string&, - std::size_t, - std::size_t, - std::size_t, + npart_t, + npart_t, + npart_t, const array_t&); template void Writer::saveParticleQuantity(const std::string&, - std::size_t, - std::size_t, - std::size_t, + npart_t, + npart_t, + npart_t, const array_t&); } // namespace checkpoint diff --git a/src/checkpoint/writer.h b/src/checkpoint/writer.h index 346bee24a..992c54c96 100644 --- a/src/checkpoint/writer.h +++ b/src/checkpoint/writer.h @@ -47,11 +47,11 @@ namespace checkpoint { ~Writer() = default; - void init(adios2::ADIOS*, std::size_t, long double, int); + void init(adios2::ADIOS*, timestep_t, simtime_t, int); - auto shouldSave(std::size_t, long double) -> bool; + auto shouldSave(timestep_t, simtime_t) -> bool; - void beginSaving(std::size_t, long double); + void beginSaving(timestep_t, simtime_t); void endSaving(); void saveAttrs(const ntt::SimulationParams&, long double); @@ -64,22 +64,22 @@ namespace checkpoint { template void saveParticleQuantity(const std::string&, - std::size_t, - std::size_t, - std::size_t, + npart_t, + npart_t, + npart_t, const array_t&); void saveParticlePayloads(const std::string&, std::size_t, - std::size_t, - std::size_t, - std::size_t, + npart_t, + npart_t, + npart_t, const array_t&); void defineFieldVariables(const ntt::SimEngine&, - const std::vector&, - const std::vector&, - const std::vector&); + const std::vector&, + const std::vector&, + const std::vector&); void defineParticleVariables(const ntt::Coord&, Dimension, diff --git a/src/engines/engine.hpp b/src/engines/engine.hpp index dac553dcd..9525874a6 100644 --- a/src/engines/engine.hpp +++ b/src/engines/engine.hpp @@ -67,14 +67,14 @@ namespace ntt { Metadomain m_metadomain; user::PGen m_pgen; - const bool is_resuming; - const long double runtime; - const real_t dt; - const std::size_t max_steps; - const std::size_t start_step; - const long double start_time; - long double time; - std::size_t step; + const bool is_resuming; + const simtime_t runtime; + const real_t dt; + const timestep_t max_steps; + const timestep_t start_step; + const simtime_t start_time; + simtime_t time; + timestep_t step; public: static constexpr bool pgen_is_ok { @@ -91,8 +91,7 @@ namespace ntt { , m_metadomain { m_params.get("simulation.domain.number"), m_params.get>( "simulation.domain.decomposition"), - m_params.get>( - "grid.resolution"), + m_params.get>("grid.resolution"), m_params.get>("grid.extent"), m_params.get>( "grid.boundaries.fields"), @@ -104,11 +103,11 @@ namespace ntt { "particles.species") } , m_pgen { m_params, m_metadomain } , is_resuming { m_params.get("checkpoint.is_resuming") } - , runtime { m_params.get("simulation.runtime") } + , runtime { m_params.get("simulation.runtime") } , dt { m_params.get("algorithms.timestep.dt") } - , max_steps { static_cast(runtime / dt) } - , start_step { m_params.get("checkpoint.start_step") } - , start_time { m_params.get("checkpoint.start_time") } + , max_steps { static_cast(runtime / dt) } + , start_step { m_params.get("checkpoint.start_step") } + , start_time { m_params.get("checkpoint.start_time") } , time { start_time } , step { start_step } { raise::ErrorIf(not pgen_is_ok, "Problem generator is not compatible with the picked engine/metric/dimension", HERE); diff --git a/src/engines/engine_init.cpp b/src/engines/engine_init.cpp index 0239724e1..7ce242bc6 100644 --- a/src/engines/engine_init.cpp +++ b/src/engines/engine_init.cpp @@ -53,7 +53,7 @@ namespace ntt { #if defined(OUTPUT_ENABLED) // read simulation data from the checkpoint raise::ErrorIf( - m_params.template get("checkpoint.start_step") == 0, + m_params.template get("checkpoint.start_step") == 0, "Resuming simulation from a checkpoint requires a valid start_step", HERE); logger::Checkpoint("Resuming simulation from a checkpoint", HERE); diff --git a/src/engines/engine_printer.cpp b/src/engines/engine_printer.cpp index 2a7ee4405..c1a36a323 100644 --- a/src/engines/engine_printer.cpp +++ b/src/engines/engine_printer.cpp @@ -38,7 +38,7 @@ namespace ntt { color::BRIGHT_BLACK, fmt::repeat("═", 58).c_str(), color::RESET); - for (std::size_t i { 0 }; i < lines.size(); ++i) { + for (auto i { 0u }; i < lines.size(); ++i) { report += fmt::format("%s║%s %s%s%s%s%s║%s\n", color::BRIGHT_BLACK, color::RESET, @@ -108,7 +108,7 @@ namespace ntt { auto bytes_to_human_readable( std::size_t bytes) -> std::pair { const std::vector units { "B", "KB", "MB", "GB", "TB" }; - std::size_t unit_idx = 0; + idx_t unit_idx = 0; auto size = static_cast(bytes); while ((size >= 1024) && (unit_idx < units.size() - 1)) { size /= 1024; @@ -236,12 +236,11 @@ namespace ntt { add_param(report, 4, "Runtime", "%.3Le [%d steps]", runtime, max_steps); report += "\n"; add_category(report, 4, "Global domain"); - add_param( - report, - 4, - "Resolution", - "%s", - params.template stringize("grid.resolution").c_str()); + add_param(report, + 4, + "Resolution", + "%s", + params.template stringize("grid.resolution").c_str()); add_param(report, 4, "Extent", diff --git a/src/engines/engine_run.cpp b/src/engines/engine_run.cpp index 506fd121d..2d4b0d5ed 100644 --- a/src/engines/engine_run.cpp +++ b/src/engines/engine_run.cpp @@ -34,11 +34,11 @@ namespace ntt { }, m_params.get("diagnostics.blocking_timers") }; - const auto diag_interval = m_params.get( + const auto diag_interval = m_params.get( "diagnostics.interval"); auto time_history = pbar::DurationHistory { 1000 }; - const auto clear_interval = m_params.template get( + const auto clear_interval = m_params.template get( "particles.clear_interval"); // main algorithm loop @@ -71,7 +71,7 @@ namespace ntt { traits::has_method::value) { auto lambda_custom_field_output = [&](const std::string& name, ndfield_t& buff, - std::size_t idx, + index_t idx, const Domain& dom) { m_pgen.CustomFieldOutput(name, buff, idx, dom); }; diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 9f5e4551f..0a9cc311b 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -79,7 +79,7 @@ namespace ntt { "algorithms.toggles.fieldsolver"); const auto deposit_enabled = m_params.template get( "algorithms.toggles.deposit"); - const auto clear_interval = m_params.template get( + const auto clear_interval = m_params.template get( "particles.clear_interval"); if (step == 0) { @@ -558,7 +558,7 @@ namespace ntt { auto range = range_with_axis_BCs(domain); const auto nfilter = m_params.template get( "algorithms.current_filters"); - tuple_t size; + tuple_t size; if constexpr (M::Dim == Dim::_1D || M::Dim == Dim::_2D || M::Dim == Dim::_3D) { size[0] = domain.mesh.n_active(in::x1); } @@ -644,8 +644,8 @@ namespace ntt { return; } const auto intersect_range = domain.mesh.ExtentToRange(box, incl_ghosts); - tuple_t range_min { 0 }; - tuple_t range_max { 0 }; + tuple_t range_min { 0 }; + tuple_t range_max { 0 }; for (unsigned short d { 0 }; d < M::Dim; ++d) { range_min[d] = intersect_range[d].first; @@ -757,8 +757,8 @@ namespace ntt { } else { raise::Error("Invalid dimension", HERE); } - std::vector xi_min, xi_max; - const std::vector all_dirs { in::x1, in::x2, in::x3 }; + std::vector xi_min, xi_max; + const std::vector all_dirs { in::x1, in::x2, in::x3 }; for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { const auto dd = all_dirs[d]; if (dim == dd) { @@ -862,15 +862,15 @@ namespace ntt { return; } const auto intersect_range = domain.mesh.ExtentToRange(box, incl_ghosts); - tuple_t range_min { 0 }; - tuple_t range_max { 0 }; + tuple_t range_min { 0 }; + tuple_t range_max { 0 }; for (unsigned short d { 0 }; d < M::Dim; ++d) { range_min[d] = intersect_range[d].first; range_max[d] = intersect_range[d].second; } - auto atm_fields = m_pgen.AtmFields(time); - std::size_t il_edge; + auto atm_fields = m_pgen.AtmFields(time); + ncells_t il_edge; if (sign > 0) { il_edge = range_min[dd] - N_GHOSTS; } else { @@ -1202,8 +1202,8 @@ namespace ntt { "possible only in -x1 (@ rmin)", HERE); } - real_t xg_min { ZERO }, xg_max { ZERO }; - std::size_t ig_min, ig_max; + real_t xg_min { ZERO }, xg_max { ZERO }; + ncells_t ig_min, ig_max; if (sign > 0) { // + direction ig_min = m_metadomain.mesh().n_active(dim) - buffer_ncells; ig_max = m_metadomain.mesh().n_active(dim); diff --git a/src/framework/containers/fields.cpp b/src/framework/containers/fields.cpp index a62886b06..7202ff282 100644 --- a/src/framework/containers/fields.cpp +++ b/src/framework/containers/fields.cpp @@ -8,8 +8,8 @@ namespace ntt { template - Fields::Fields(const std::vector& res) { - std::size_t nx1, nx2, nx3; + Fields::Fields(const std::vector& res) { + ncells_t nx1, nx2, nx3; nx1 = res[0] + 2 * N_GHOSTS; if constexpr ((D == Dim::_3D) || (D == Dim::_2D)) { nx2 = res[1] + 2 * N_GHOSTS; @@ -52,4 +52,4 @@ namespace ntt { template struct Fields; template struct Fields; -} // namespace ntt \ No newline at end of file +} // namespace ntt diff --git a/src/framework/containers/fields.h b/src/framework/containers/fields.h index d0bd7d020..ee9d656d6 100644 --- a/src/framework/containers/fields.h +++ b/src/framework/containers/fields.h @@ -109,7 +109,7 @@ namespace ntt { */ Fields() {} - Fields(const std::vector& res); + Fields(const std::vector& res); Fields(Fields&& other) noexcept : em { std::move(other.em) } diff --git a/src/framework/containers/particles.cpp b/src/framework/containers/particles.cpp index d78055824..048d57cde 100644 --- a/src/framework/containers/particles.cpp +++ b/src/framework/containers/particles.cpp @@ -25,7 +25,7 @@ namespace ntt { const std::string& label, float m, float ch, - std::size_t maxnpart, + npart_t maxnpart, const PrtlPusher& pusher, bool use_gca, const Cooling& cooling, @@ -72,10 +72,10 @@ namespace ntt { template auto Particles::NpartsPerTagAndOffsets() const - -> std::pair, array_t> { - auto this_tag = tag; - const auto num_tags = ntags(); - array_t npptag { "nparts_per_tag", ntags() }; + -> std::pair, array_t> { + auto this_tag = tag; + const auto num_tags = ntags(); + array_t npptag { "nparts_per_tag", ntags() }; // count # of particles per each tag auto npptag_scat = Kokkos::Experimental::create_scatter_view(npptag); @@ -94,14 +94,14 @@ namespace ntt { // copy the count to a vector on the host auto npptag_h = Kokkos::create_mirror_view(npptag); Kokkos::deep_copy(npptag_h, npptag); - std::vector npptag_vec(num_tags); + std::vector npptag_vec(num_tags); for (auto t { 0u }; t < num_tags; ++t) { npptag_vec[t] = npptag_h(t); } // count the offsets on the host and copy to device - array_t tag_offsets("tag_offsets", num_tags - 3); - auto tag_offsets_h = Kokkos::create_mirror_view(tag_offsets); + array_t tag_offsets("tag_offsets", num_tags - 3); + auto tag_offsets_h = Kokkos::create_mirror_view(tag_offsets); tag_offsets_h(0) = npptag_vec[2]; // offset for tag = 3 for (auto t { 1u }; t < num_tags - 3; ++t) { @@ -113,25 +113,23 @@ namespace ntt { } template - void RemoveDeadInArray(array_t& arr, - const array_t& indices_alive) { - auto n_alive = indices_alive.extent(0); - auto buffer = Kokkos::View("buffer", n_alive); + void RemoveDeadInArray(array_t& arr, const array_t& indices_alive) { + npart_t n_alive = indices_alive.extent(0); + auto buffer = Kokkos::View("buffer", n_alive); Kokkos::parallel_for( "PopulateBufferAlive", n_alive, Lambda(index_t p) { buffer(p) = arr(indices_alive(p)); }); Kokkos::deep_copy( - Kokkos::subview(arr, std::make_pair(static_cast(0), n_alive)), + Kokkos::subview(arr, std::make_pair(static_cast(0), n_alive)), buffer); } template - void RemoveDeadInArray(array_t& arr, - const array_t& indices_alive) { - auto n_alive = indices_alive.extent(0); - auto buffer = array_t { "buffer", n_alive, arr.extent(1) }; + void RemoveDeadInArray(array_t& arr, const array_t& indices_alive) { + npart_t n_alive = indices_alive.extent(0); + auto buffer = array_t { "buffer", n_alive, arr.extent(1) }; Kokkos::parallel_for( "PopulateBufferAlive", CreateRangePolicy({ 0, 0 }, { n_alive, arr.extent(1) }), @@ -139,21 +137,21 @@ namespace ntt { Kokkos::deep_copy( Kokkos::subview(arr, - std::make_pair(static_cast(0), n_alive), + std::make_pair(static_cast(0), n_alive), Kokkos::ALL), buffer); } template void Particles::RemoveDead() { - const auto n_part = npart(); - std::size_t n_alive = 0, n_dead = 0; - auto& this_tag = tag; + const auto n_part = npart(); + npart_t n_alive = 0, n_dead = 0; + auto& this_tag = tag; Kokkos::parallel_reduce( "CountDeadAlive", rangeActiveParticles(), - Lambda(index_t p, std::size_t & nalive, std::size_t & ndead) { + Lambda(index_t p, npart_t & nalive, npart_t & ndead) { nalive += (this_tag(p) == ParticleTag::alive); ndead += (this_tag(p) == ParticleTag::dead); if (this_tag(p) != ParticleTag::alive and this_tag(p) != ParticleTag::dead) { @@ -163,8 +161,8 @@ namespace ntt { n_alive, n_dead); - array_t indices_alive { "indices_alive", n_alive }; - array_t alive_counter { "counter_alive", 1 }; + array_t indices_alive { "indices_alive", n_alive }; + array_t alive_counter { "counter_alive", 1 }; Kokkos::parallel_for( "AliveIndices", @@ -221,8 +219,7 @@ namespace ntt { Kokkos::Experimental::fill( "TagAliveParticles", AccelExeSpace(), - Kokkos::subview(this_tag, - std::make_pair(static_cast(0), n_alive)), + Kokkos::subview(this_tag, std::make_pair(static_cast(0), n_alive)), ParticleTag::alive); Kokkos::Experimental::fill( diff --git a/src/framework/containers/particles.h b/src/framework/containers/particles.h index d84bd0cc9..5241822e2 100644 --- a/src/framework/containers/particles.h +++ b/src/framework/containers/particles.h @@ -37,8 +37,8 @@ namespace ntt { struct Particles : public ParticleSpecies { private: // Number of currently active (used) particles - std::size_t m_npart { 0 }; - bool m_is_sorted { false }; + npart_t m_npart { 0 }; + bool m_is_sorted { false }; #if !defined(MPI_ENABLED) const std::size_t m_ntags { 2 }; @@ -84,7 +84,7 @@ namespace ntt { const std::string& label, float m, float ch, - std::size_t maxnpart, + npart_t maxnpart, const PrtlPusher& pusher, bool use_gca, const Cooling& cooling, @@ -116,7 +116,7 @@ namespace ntt { * @returns A 1D Kokkos range policy of size of `npart` */ inline auto rangeActiveParticles() const -> range_t { - return CreateRangePolicy({ 0 }, { npart() }); + return CreateParticleRangePolicy(0u, npart()); } /** @@ -124,7 +124,7 @@ namespace ntt { * @returns A 1D Kokkos range policy of size of `npart` */ inline auto rangeAllParticles() const -> range_t { - return CreateRangePolicy({ 0 }, { maxnpart() }); + return CreateParticleRangePolicy(0u, maxnpart()); } /* getters -------------------------------------------------------------- */ @@ -132,7 +132,7 @@ namespace ntt { * @brief Get the number of active particles */ [[nodiscard]] - auto npart() const -> std::size_t { + auto npart() const -> npart_t { return m_npart; } @@ -188,14 +188,14 @@ namespace ntt { * ... etc. */ auto NpartsPerTagAndOffsets() const - -> std::pair, array_t>; + -> std::pair, array_t>; /* setters -------------------------------------------------------------- */ /** * @brief Set the number of particles - * @param npart The number of particles as a std::size_t + * @param npart The number of particles as a npart_t */ - void set_npart(std::size_t n) { + void set_npart(npart_t n) { raise::ErrorIf( n > maxnpart(), fmt::format( diff --git a/src/framework/containers/species.h b/src/framework/containers/species.h index 1f52733aa..6dd437819 100644 --- a/src/framework/containers/species.h +++ b/src/framework/containers/species.h @@ -28,7 +28,7 @@ namespace ntt { // Species charge in units of q0 const float m_charge; // Max number of allocated particles for the species - std::size_t m_maxnpart; + npart_t m_maxnpart; // Pusher assigned for the species const PrtlPusher m_pusher; @@ -68,7 +68,7 @@ namespace ntt { const std::string& label, float m, float ch, - std::size_t maxnpart, + npart_t maxnpart, const PrtlPusher& pusher, bool use_gca, const Cooling& cooling, @@ -111,7 +111,7 @@ namespace ntt { } [[nodiscard]] - auto maxnpart() const -> std::size_t { + auto maxnpart() const -> npart_t { return m_maxnpart; } diff --git a/src/framework/domain/checkpoint.cpp b/src/framework/domain/checkpoint.cpp index 6dfb137db..978a1ad10 100644 --- a/src/framework/domain/checkpoint.cpp +++ b/src/framework/domain/checkpoint.cpp @@ -32,7 +32,7 @@ namespace ntt { "local_domain is a placeholder", HERE); - std::vector glob_shape_with_ghosts, off_ncells_with_ghosts; + std::vector glob_shape_with_ghosts, off_ncells_with_ghosts; for (auto d { 0u }; d < M::Dim; ++d) { off_ncells_with_ghosts.push_back( local_domain->offset_ncells()[d] + @@ -49,8 +49,8 @@ namespace ntt { g_checkpoint_writer.init( ptr_adios, - params.template get("checkpoint.interval"), - params.template get("checkpoint.interval_time"), + params.template get("checkpoint.interval"), + params.template get("checkpoint.interval_time"), params.template get("checkpoint.keep")); if (g_checkpoint_writer.enabled()) { g_checkpoint_writer.defineFieldVariables(S, @@ -66,10 +66,10 @@ namespace ntt { template auto Metadomain::WriteCheckpoint(const SimulationParams& params, - std::size_t current_step, - std::size_t finished_step, - long double current_time, - long double finished_time) -> bool { + timestep_t current_step, + timestep_t finished_step, + simtime_t current_time, + simtime_t finished_time) -> bool { raise::ErrorIf( l_subdomain_indices().size() != 1, "Checkpointing for now is only supported for one subdomain per rank", @@ -98,17 +98,17 @@ namespace ntt { #endif // MPI_ENABLED for (auto s { 0u }; s < local_domain->species.size(); ++s) { - auto npart = local_domain->species[s].npart(); - std::size_t offset = 0; - auto glob_tot = npart; + auto npart = local_domain->species[s].npart(); + npart_t offset = 0; + auto glob_tot = npart; #if defined(MPI_ENABLED) - auto glob_npart = std::vector(g_ndomains); + auto glob_npart = std::vector(g_ndomains); MPI_Allgather(&npart, 1, - mpi::get_type(), + mpi::get_type(), glob_npart.data(), 1, - mpi::get_type(), + mpi::get_type(), MPI_COMM_WORLD); glob_tot = 0; for (auto r = 0; r < g_mpi_size; ++r) { @@ -118,7 +118,7 @@ namespace ntt { glob_tot += glob_npart[r]; } #endif // MPI_ENABLED - g_checkpoint_writer.savePerDomainVariable( + g_checkpoint_writer.savePerDomainVariable( fmt::format("s%d_npart", s + 1), dom_tot, dom_offset, @@ -263,7 +263,7 @@ namespace ntt { raise::ErrorIf(ptr_adios == nullptr, "adios == nullptr", HERE); auto fname = fmt::format( "checkpoints/step-%08lu.bp", - params.template get("checkpoint.start_step")); + params.template get("checkpoint.start_step")); logger::Checkpoint(fmt::format("Reading checkpoint from %s", fname.c_str()), HERE); diff --git a/src/framework/domain/comm_mpi.hpp b/src/framework/domain/comm_mpi.hpp index e5bc2d21e..8c6e532de 100644 --- a/src/framework/domain/comm_mpi.hpp +++ b/src/framework/domain/comm_mpi.hpp @@ -129,7 +129,7 @@ namespace comm { } } } else { - std::size_t nsend { comps.second - comps.first }, + ncells_t nsend { comps.second - comps.first }, nrecv { comps.second - comps.first }; ndarray_t(D) + 1> send_fld, recv_fld; @@ -287,29 +287,29 @@ namespace comm { } } - void ParticleSendRecvCount(int send_rank, - int recv_rank, - std::size_t send_count, - std::size_t& recv_count) { + void ParticleSendRecvCount(int send_rank, + int recv_rank, + npart_t send_count, + npart_t& recv_count) { if ((send_rank >= 0) && (recv_rank >= 0)) { MPI_Sendrecv(&send_count, 1, - mpi::get_type(), + mpi::get_type(), send_rank, 0, &recv_count, 1, - mpi::get_type(), + mpi::get_type(), recv_rank, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } else if (send_rank >= 0) { - MPI_Send(&send_count, 1, mpi::get_type(), send_rank, 0, MPI_COMM_WORLD); + MPI_Send(&send_count, 1, mpi::get_type(), send_rank, 0, MPI_COMM_WORLD); } else if (recv_rank >= 0) { MPI_Recv(&recv_count, 1, - mpi::get_type(), + mpi::get_type(), recv_rank, 0, MPI_COMM_WORLD, @@ -320,14 +320,14 @@ namespace comm { } template - void CommunicateParticles(Particles& species, - const array_t& outgoing_indices, - const array_t& tag_offsets, - const std::vector& npptag_vec, - const std::vector& npptag_recv_vec, - const std::vector& send_ranks, - const std::vector& recv_ranks, - const dir::dirs_t& dirs_to_comm) { + void CommunicateParticles(Particles& species, + const array_t& outgoing_indices, + const array_t& tag_offsets, + const std::vector& npptag_vec, + const std::vector& npptag_recv_vec, + const std::vector& send_ranks, + const std::vector& recv_ranks, + const dir::dirs_t& dirs_to_comm) { // number of arrays of each type to send/recv const unsigned short NREALS = 4 + static_cast( D == Dim::_2D and C != Coord::Cart); @@ -341,7 +341,7 @@ namespace comm { const auto npart_send = outgoing_indices.extent(0) - npart_dead; const auto npart_recv = std::accumulate(npptag_recv_vec.begin(), npptag_recv_vec.end(), - static_cast(0)); + static_cast(0)); array_t recv_buff_int { "recv_buff_int", npart_recv * NINTS }; array_t recv_buff_real { "recv_buff_real", npart_recv * NREALS }; array_t recv_buff_prtldx { "recv_buff_prtldx", npart_recv * NPRTLDX }; @@ -376,7 +376,7 @@ namespace comm { auto tag_offsets_h = Kokkos::create_mirror_view(tag_offsets); Kokkos::deep_copy(tag_offsets_h, tag_offsets); - std::size_t idx_offset = npart_dead; + npart_t idx_offset = npart_dead; if (tag_send > 2) { idx_offset += tag_offsets_h(tag_send - 3); } diff --git a/src/framework/domain/communications.cpp b/src/framework/domain/communications.cpp index 7dc5d285a..946756f49 100644 --- a/src/framework/domain/communications.cpp +++ b/src/framework/domain/communications.cpp @@ -36,10 +36,10 @@ namespace ntt { using comm_params_t = std::pair>; template - auto GetSendRecvRanks(Metadomain* metadomain, - Domain& domain, - dir::direction_t direction) - -> std::pair { + auto GetSendRecvRanks( + Metadomain* metadomain, + Domain& domain, + dir::direction_t direction) -> std::pair { Domain* send_to_nghbr_ptr = nullptr; Domain* recv_from_nghbr_ptr = nullptr; // set pointers to the correct send/recv domains @@ -119,11 +119,11 @@ namespace ntt { } template - auto GetSendRecvParams(Metadomain* metadomain, - Domain& domain, - dir::direction_t direction, - bool synchronize) - -> std::pair { + auto GetSendRecvParams( + Metadomain* metadomain, + Domain& domain, + dir::direction_t direction, + bool synchronize) -> std::pair { const auto [send_indrank, recv_indrank] = GetSendRecvRanks(metadomain, domain, direction); const auto [send_ind, send_rank] = send_indrank; @@ -140,7 +140,7 @@ namespace ntt { auto recv_slice = std::vector {}; const in components[] = { in::x1, in::x2, in::x3 }; // find the field components and indices to be sent/received - for (std::size_t d { 0 }; d < direction.size(); ++d) { + for (auto d { 0u }; d < direction.size(); ++d) { const auto c = components[d]; const auto dir = direction[d]; if (not synchronize) { @@ -510,11 +510,11 @@ namespace ntt { const auto npart_holes = npart - npart_alive; // # of particles to receive per each tag (direction) - std::vector npptag_recv_vec(ntags - 2, 0); + std::vector npptag_recv_vec(ntags - 2, 0); // coordinate shifts per each direction - array_t shifts_in_x1 { "shifts_in_x1", ntags - 2 }; - array_t shifts_in_x2 { "shifts_in_x2", ntags - 2 }; - array_t shifts_in_x3 { "shifts_in_x3", ntags - 2 }; + array_t shifts_in_x1 { "shifts_in_x1", ntags - 2 }; + array_t shifts_in_x2 { "shifts_in_x2", ntags - 2 }; + array_t shifts_in_x3 { "shifts_in_x3", ntags - 2 }; auto shifts_in_x1_h = Kokkos::create_mirror_view(shifts_in_x1); auto shifts_in_x2_h = Kokkos::create_mirror_view(shifts_in_x2); auto shifts_in_x3_h = Kokkos::create_mirror_view(shifts_in_x3); @@ -527,7 +527,7 @@ namespace ntt { std::vector recv_ranks, recv_inds; // total # of reaceived particles from all directions - std::size_t npart_recv = 0u; + npart_t npart_recv = 0u; for (const auto& direction : dir::Directions::all) { // tags corresponding to the direction (both send & recv) @@ -559,7 +559,7 @@ namespace ntt { // request the # of particles to-be-received ... // ... and send the # of particles to-be-sent - std::size_t nrecv = 0; + npart_t nrecv = 0; comm::ParticleSendRecvCount(send_rank, recv_rank, nsend, nrecv); npart_recv += nrecv; npptag_recv_vec[tag_recv - 2] = nrecv; @@ -604,8 +604,7 @@ namespace ntt { Kokkos::deep_copy(shifts_in_x2, shifts_in_x2_h); Kokkos::deep_copy(shifts_in_x3, shifts_in_x3_h); - array_t outgoing_indices { "outgoing_indices", - npart - npart_alive }; + array_t outgoing_indices { "outgoing_indices", npart - npart_alive }; // clang-format off Kokkos::parallel_for( "PrepareOutgoingPrtls", diff --git a/src/framework/domain/domain.h b/src/framework/domain/domain.h index bc7c6e4b5..7966cdb54 100644 --- a/src/framework/domain/domain.h +++ b/src/framework/domain/domain.h @@ -73,8 +73,8 @@ namespace ntt { Domain(bool, unsigned int index, const std::vector& offset_ndomains, - const std::vector& offset_ncells, - const std::vector& ncells, + const std::vector& offset_ncells, + const std::vector& ncells, const boundaries_t& extent, const std::map& metric_params, const std::vector&) @@ -88,8 +88,8 @@ namespace ntt { Domain(unsigned int index, const std::vector& offset_ndomains, - const std::vector& offset_ncells, - const std::vector& ncells, + const std::vector& offset_ncells, + const std::vector& ncells, const boundaries_t& extent, const std::map& metric_params, const std::vector& species_params) @@ -124,7 +124,7 @@ namespace ntt { } [[nodiscard]] - auto offset_ncells() const -> std::vector { + auto offset_ncells() const -> std::vector { return m_offset_ncells; } @@ -156,7 +156,7 @@ namespace ntt { // offset of the domain in # of domains std::vector m_offset_ndomains; // offset of the domain in cells (# of cells in each dimension) - std::vector m_offset_ncells; + std::vector m_offset_ncells; // neighboring domain indices dir::map_t m_neighbor_idx; // MPI rank of the domain (used only when MPI enabled) diff --git a/src/framework/domain/grid.cpp b/src/framework/domain/grid.cpp index 9302386e1..baa23fb5c 100644 --- a/src/framework/domain/grid.cpp +++ b/src/framework/domain/grid.cpp @@ -50,7 +50,7 @@ namespace ntt { template auto Grid::rangeCells(const box_region_t& region) const -> range_t { - tuple_t imin, imax; + tuple_t imin, imax; for (unsigned short i = 0; i < (unsigned short)D; i++) { switch (region[i]) { case CellLayer::allLayer: @@ -87,9 +87,9 @@ namespace ntt { // !TODO: too ugly, implement a better solution (combine with device) template - auto Grid::rangeCellsOnHost(const box_region_t& region) const - -> range_h_t { - tuple_t imin, imax; + auto Grid::rangeCellsOnHost( + const box_region_t& region) const -> range_h_t { + tuple_t imin, imax; for (unsigned short i = 0; i < (unsigned short)D; i++) { switch (region[i]) { case CellLayer::allLayer: @@ -164,9 +164,9 @@ namespace ntt { } template - auto Grid::rangeCells(const tuple_t, D>& ranges) const - -> range_t { - tuple_t imin, imax; + auto Grid::rangeCells( + const tuple_t, D>& ranges) const -> range_t { + tuple_t imin, imax; for (unsigned short i = 0; i < (unsigned short)D; i++) { raise::ErrorIf((ranges[i][0] < -(int)N_GHOSTS) || (ranges[i][1] > (int)N_GHOSTS), diff --git a/src/framework/domain/grid.h b/src/framework/domain/grid.h index 97a939117..af5b6c8d5 100644 --- a/src/framework/domain/grid.h +++ b/src/framework/domain/grid.h @@ -73,7 +73,7 @@ namespace ntt { template struct Grid { - Grid(const std::vector& res) : m_resolution { res } { + Grid(const std::vector& res) : m_resolution { res } { raise::ErrorIf(m_resolution.size() != D, "invalid dimension", HERE); } @@ -81,7 +81,7 @@ namespace ntt { /* getters -------------------------------------------------------------- */ [[nodiscard]] - auto i_min(in i) const -> std::size_t { + auto i_min(in i) const -> ncells_t { switch (i) { case in::x1: return (m_resolution.size() > 0) ? N_GHOSTS : 0; @@ -96,7 +96,7 @@ namespace ntt { } [[nodiscard]] - auto i_max(in i) const -> std::size_t { + auto i_max(in i) const -> ncells_t { switch (i) { case in::x1: return (m_resolution.size() > 0) ? (m_resolution[0] + N_GHOSTS) : 1; @@ -111,7 +111,7 @@ namespace ntt { } [[nodiscard]] - auto n_active(in i) const -> std::size_t { + auto n_active(in i) const -> ncells_t { switch (i) { case in::x1: return (m_resolution.size() > 0) ? m_resolution[0] : 1; @@ -126,12 +126,12 @@ namespace ntt { } [[nodiscard]] - auto n_active() const -> std::vector { + auto n_active() const -> std::vector { return m_resolution; } [[nodiscard]] - auto n_all(in i) const -> std::size_t { + auto n_all(in i) const -> ncells_t { switch (i) { case in::x1: return (m_resolution.size() > 0) ? (m_resolution[0] + 2 * N_GHOSTS) : 1; @@ -146,9 +146,9 @@ namespace ntt { } [[nodiscard]] - auto n_all() const -> std::vector { - std::vector nall; - for (std::size_t i = 0; i < D; ++i) { + auto n_all() const -> std::vector { + std::vector nall; + for (auto i = 0u; i < D; ++i) { nall.push_back(m_resolution[i] + 2 * N_GHOSTS); } return nall; @@ -204,7 +204,7 @@ namespace ntt { auto rangeCellsOnHost(const box_region_t&) const -> range_h_t; protected: - std::vector m_resolution; + std::vector m_resolution; }; } // namespace ntt diff --git a/src/framework/domain/mesh.h b/src/framework/domain/mesh.h index b0bd1a567..98fe68895 100644 --- a/src/framework/domain/mesh.h +++ b/src/framework/domain/mesh.h @@ -38,14 +38,14 @@ namespace ntt { M metric; - Mesh(const std::vector& res, + Mesh(const std::vector& res, const boundaries_t& ext, const std::map& metric_params) : Grid { res } , metric { res, ext, metric_params } , m_extent { ext } {} - Mesh(const std::vector& res, + Mesh(const std::vector& res, const boundaries_t& ext, const std::map& metric_params, const boundaries_t& flds_bc, @@ -131,15 +131,15 @@ namespace ntt { * @note indices are already shifted by N_GHOSTS (i.e. they start at N_GHOSTS not 0) */ [[nodiscard]] - auto ExtentToRange(boundaries_t box, boundaries_t incl_ghosts) - -> boundaries_t { + auto ExtentToRange(boundaries_t box, + boundaries_t incl_ghosts) -> boundaries_t { raise::ErrorIf(box.size() != M::Dim, "Invalid box dimension", HERE); raise::ErrorIf(incl_ghosts.size() != M::Dim, "Invalid incl_ghosts dimension", HERE); - boundaries_t range; + boundaries_t range; if (not Intersects(box)) { - for (std::size_t i { 0 }; i < box.size(); ++i) { + for (auto i { 0u }; i < box.size(); ++i) { range.push_back({ 0, 0 }); } return range; @@ -184,9 +184,9 @@ namespace ntt { raise::Error("invalid dimension", HERE); throw; } - range.push_back({ static_cast(xi_min_Cd) + + range.push_back({ static_cast(xi_min_Cd) + (incl_ghosts[d].first ? 0 : N_GHOSTS), - static_cast(xi_max_Cd) + + static_cast(xi_max_Cd) + (incl_ghosts[d].second ? 2 * N_GHOSTS : N_GHOSTS) }); } ++d; @@ -222,18 +222,18 @@ namespace ntt { auto flds_bc() const -> boundaries_t { if constexpr (D == Dim::_1D) { return { - {flds_bc_in({ -1 }), flds_bc_in({ -1 })} + { flds_bc_in({ -1 }), flds_bc_in({ -1 }) } }; } else if constexpr (D == Dim::_2D) { return { - {flds_bc_in({ -1, 0 }), flds_bc_in({ 1, 0 })}, - {flds_bc_in({ 0, -1 }), flds_bc_in({ 0, 1 })} + { flds_bc_in({ -1, 0 }), flds_bc_in({ 1, 0 }) }, + { flds_bc_in({ 0, -1 }), flds_bc_in({ 0, 1 }) } }; } else if constexpr (D == Dim::_3D) { return { - {flds_bc_in({ -1, 0, 0 }), flds_bc_in({ 1, 0, 0 })}, - {flds_bc_in({ 0, -1, 0 }), flds_bc_in({ 0, 1, 0 })}, - {flds_bc_in({ 0, 0, -1 }), flds_bc_in({ 0, 0, 1 })} + { flds_bc_in({ -1, 0, 0 }), flds_bc_in({ 1, 0, 0 }) }, + { flds_bc_in({ 0, -1, 0 }), flds_bc_in({ 0, 1, 0 }) }, + { flds_bc_in({ 0, 0, -1 }), flds_bc_in({ 0, 0, 1 }) } }; } else { raise::Error("invalid dimension", HERE); @@ -245,18 +245,18 @@ namespace ntt { auto prtl_bc() const -> boundaries_t { if constexpr (D == Dim::_1D) { return { - {prtl_bc_in({ -1 }), prtl_bc_in({ -1 })} + { prtl_bc_in({ -1 }), prtl_bc_in({ -1 }) } }; } else if constexpr (D == Dim::_2D) { return { - {prtl_bc_in({ -1, 0 }), prtl_bc_in({ 1, 0 })}, - {prtl_bc_in({ 0, -1 }), prtl_bc_in({ 0, 1 })} + { prtl_bc_in({ -1, 0 }), prtl_bc_in({ 1, 0 }) }, + { prtl_bc_in({ 0, -1 }), prtl_bc_in({ 0, 1 }) } }; } else if constexpr (D == Dim::_3D) { return { - {prtl_bc_in({ -1, 0, 0 }), prtl_bc_in({ 1, 0, 0 })}, - {prtl_bc_in({ 0, -1, 0 }), prtl_bc_in({ 0, 1, 0 })}, - {prtl_bc_in({ 0, 0, -1 }), prtl_bc_in({ 0, 0, 1 })} + { prtl_bc_in({ -1, 0, 0 }), prtl_bc_in({ 1, 0, 0 }) }, + { prtl_bc_in({ 0, -1, 0 }), prtl_bc_in({ 0, 1, 0 }) }, + { prtl_bc_in({ 0, 0, -1 }), prtl_bc_in({ 0, 0, 1 }) } }; } else { raise::Error("invalid dimension", HERE); diff --git a/src/framework/domain/metadomain.cpp b/src/framework/domain/metadomain.cpp index ed4373df2..dd0d6ffe7 100644 --- a/src/framework/domain/metadomain.cpp +++ b/src/framework/domain/metadomain.cpp @@ -32,10 +32,10 @@ namespace ntt { template Metadomain::Metadomain(unsigned int global_ndomains, const std::vector& global_decomposition, - const std::vector& global_ncells, - const boundaries_t& global_extent, - const boundaries_t& global_flds_bc, - const boundaries_t& global_prtl_bc, + const std::vector& global_ncells, + const boundaries_t& global_extent, + const boundaries_t& global_flds_bc, + const boundaries_t& global_prtl_bc, const std::map& metric_params, const std::vector& species_params) : g_ndomains { global_ndomains } @@ -104,13 +104,13 @@ namespace ntt { raise::ErrorIf(d_ncells.size() != (std::size_t)D, "Invalid number of dimensions received", HERE); - auto d_offset_ncells = std::vector> {}; + auto d_offset_ncells = std::vector> {}; auto d_offset_ndoms = std::vector> {}; for (auto& d : d_ncells) { g_ndomains_per_dim.push_back(d.size()); - auto offset_ncell = std::vector { 0 }; + auto offset_ncell = std::vector { 0 }; auto offset_ndom = std::vector { 0 }; - for (std::size_t i { 1 }; i < d.size(); ++i) { + for (auto i { 1u }; i < d.size(); ++i) { auto di = d[i - 1]; offset_ncell.push_back(offset_ncell.back() + di); offset_ndom.push_back(offset_ndom.back() + 1); @@ -121,8 +121,8 @@ namespace ntt { /* compute tensor products of the domain decompositions --------------- */ // works similar to np.meshgrid() - const auto domain_ncells = tools::TensorProduct(d_ncells); - const auto domain_offset_ncells = tools::TensorProduct( + const auto domain_ncells = tools::TensorProduct(d_ncells); + const auto domain_offset_ncells = tools::TensorProduct( d_offset_ncells); const auto domain_offset_ndoms = tools::TensorProduct( d_offset_ndoms); diff --git a/src/framework/domain/metadomain.h b/src/framework/domain/metadomain.h index 5177571d0..e7e1340e2 100644 --- a/src/framework/domain/metadomain.h +++ b/src/framework/domain/metadomain.h @@ -103,7 +103,7 @@ namespace ntt { */ Metadomain(unsigned int, const std::vector&, - const std::vector&, + const std::vector&, const boundaries_t&, const boundaries_t&, const boundaries_t&, @@ -118,20 +118,20 @@ namespace ntt { #if defined(OUTPUT_ENABLED) void InitWriter(adios2::ADIOS*, const SimulationParams&, bool is_resuming); auto Write(const SimulationParams&, - std::size_t, - std::size_t, - long double, - long double, + timestep_t, + timestep_t, + simtime_t, + simtime_t, std::function&, std::size_t, const Domain&)> = {}) -> bool; void InitCheckpointWriter(adios2::ADIOS*, const SimulationParams&); auto WriteCheckpoint(const SimulationParams&, - std::size_t, - std::size_t, - long double, - long double) -> bool; + timestep_t, + timestep_t, + simtime_t, + simtime_t) -> bool; void ContinueFromCheckpoint(adios2::ADIOS*, const SimulationParams&); #endif diff --git a/src/framework/domain/output.cpp b/src/framework/domain/output.cpp index 6961d2826..1b29dcf7d 100644 --- a/src/framework/domain/output.cpp +++ b/src/framework/domain/output.cpp @@ -96,9 +96,9 @@ namespace ntt { g_writer.defineSpectraOutputs(spectra_species); for (const auto& type : { "fields", "particles", "spectra" }) { g_writer.addTracker(type, - params.template get( + params.template get( "output." + std::string(type) + ".interval"), - params.template get( + params.template get( "output." + std::string(type) + ".interval_time")); } if (is_resuming and std::filesystem::exists(g_writer.fname())) { @@ -182,10 +182,10 @@ namespace ntt { template auto Metadomain::Write( const SimulationParams& params, - std::size_t current_step, - std::size_t finished_step, - long double current_time, - long double finished_time, + timestep_t current_step, + timestep_t finished_step, + simtime_t current_time, + simtime_t finished_time, std::function< void(const std::string&, ndfield_t&, std::size_t, const Domain&)> CustomFieldOutput) -> bool { @@ -234,8 +234,8 @@ namespace ntt { const double l = l_offset; const double f = math::ceil(l / d) * d - l; - const auto first_cell = static_cast(f); - const auto l_size_dwn = static_cast(math::ceil((n - f) / d)); + const auto first_cell = static_cast(f); + const auto l_size_dwn = static_cast(math::ceil((n - f) / d)); const auto is_last = l_offset + l_size == g_size; @@ -472,19 +472,19 @@ namespace ntt { if (write_particles) { g_writer.beginWriting(WriteMode::Particles, current_step, current_time); - const auto prtl_stride = params.template get( + const auto prtl_stride = params.template get( "output.particles.stride"); for (const auto& prtl : g_writer.speciesWriters()) { auto& species = local_domain->species[prtl.species() - 1]; if (not species.is_sorted()) { species.RemoveDead(); } - const std::size_t nout = species.npart() / prtl_stride; - array_t buff_x1, buff_x2, buff_x3; - array_t buff_ux1 { "u1", nout }; - array_t buff_ux2 { "ux2", nout }; - array_t buff_ux3 { "ux3", nout }; - array_t buff_wei { "w", nout }; + const npart_t nout = species.npart() / prtl_stride; + array_t buff_x1, buff_x2, buff_x3; + array_t buff_ux1 { "u1", nout }; + array_t buff_ux2 { "ux2", nout }; + array_t buff_ux3 { "ux3", nout }; + array_t buff_wei { "w", nout }; if constexpr (M::Dim == Dim::_1D or M::Dim == Dim::_2D or M::Dim == Dim::_3D) { buff_x1 = array_t { "x1", nout }; @@ -512,16 +512,16 @@ namespace ntt { local_domain->mesh.metric)); // clang-format on } - std::size_t offset = 0; - std::size_t glob_tot = nout; + npart_t offset = 0; + npart_t glob_tot = nout; #if defined(MPI_ENABLED) - auto glob_nout = std::vector(g_ndomains); + auto glob_nout = std::vector(g_ndomains); MPI_Allgather(&nout, 1, - mpi::get_type(), + mpi::get_type(), glob_nout.data(), 1, - mpi::get_type(), + mpi::get_type(), MPI_COMM_WORLD); glob_tot = 0; for (auto r = 0; r < g_mpi_size; ++r) { diff --git a/src/framework/parameters.cpp b/src/framework/parameters.cpp index 4a9b3056a..e3bddfd70 100644 --- a/src/framework/parameters.cpp +++ b/src/framework/parameters.cpp @@ -32,7 +32,7 @@ namespace ntt { template auto get_dx0_V0( - const std::vector& resolution, + const std::vector& resolution, const boundaries_t& extent, const std::map& params) -> std::pair { const auto metric = M(resolution, extent, params); @@ -78,9 +78,9 @@ namespace ntt { promiseToDefine("simulation.domain.decomposition"); /* [grid] --------------------------------------------------------------- */ - const auto res = toml::find>(toml_data, - "grid", - "resolution"); + const auto res = toml::find>(toml_data, + "grid", + "resolution"); raise::ErrorIf(res.size() < 1 || res.size() > 3, "invalid `grid.resolution`", HERE); @@ -198,7 +198,7 @@ namespace ntt { const auto def_pusher = (is_massless ? defaults::ph_pusher : defaults::em_pusher); const auto maxnpart_real = toml::find(sp, "maxnpart"); - const auto maxnpart = static_cast(maxnpart_real); + const auto maxnpart = static_cast(maxnpart_real); auto pusher = toml::find_or(sp, "pusher", std::string(def_pusher)); const auto npayloads = toml::find_or(sp, "n_payloads", @@ -335,7 +335,7 @@ namespace ntt { set("simulation.name", toml::find(toml_data, "simulation", "name")); set("simulation.runtime", - toml::find(toml_data, "simulation", "runtime")); + toml::find(toml_data, "simulation", "runtime")); /* [grid.boundaraies] --------------------------------------------------- */ auto flds_bc = toml::find>>( @@ -454,7 +454,7 @@ namespace ntt { set("output.interval", toml::find_or(toml_data, "output", "interval", defaults::output::interval)); set("output.interval_time", - toml::find_or(toml_data, "output", "interval_time", -1.0)); + toml::find_or(toml_data, "output", "interval_time", -1.0)); set("output.separate_files", toml::find_or(toml_data, "output", "separate_files", true)); @@ -542,23 +542,23 @@ namespace ntt { // intervals for (const auto& type : { "fields", "particles", "spectra" }) { - const auto q_int = toml::find_or(toml_data, - "output", - std::string(type), - "interval", - 0); - const auto q_int_time = toml::find_or(toml_data, - "output", - std::string(type), - "interval_time", - -1.0); + const auto q_int = toml::find_or(toml_data, + "output", + std::string(type), + "interval", + 0); + const auto q_int_time = toml::find_or(toml_data, + "output", + std::string(type), + "interval_time", + -1.0); set("output." + std::string(type) + ".enable", toml::find_or(toml_data, "output", std::string(type), "enable", true)); if (q_int == 0 && q_int_time == -1.0) { set("output." + std::string(type) + ".interval", - get("output.interval")); + get("output.interval")); set("output." + std::string(type) + ".interval_time", - get("output.interval_time")); + get("output.interval_time")); } else { set("output." + std::string(type) + ".interval", q_int); set("output." + std::string(type) + ".interval_time", q_int_time); @@ -590,7 +590,7 @@ namespace ntt { "interval", defaults::checkpoint::interval)); set("checkpoint.interval_time", - toml::find_or(toml_data, "checkpoint", "interval_time", -1.0)); + toml::find_or(toml_data, "checkpoint", "interval_time", -1.0)); set("checkpoint.keep", toml::find_or(toml_data, "checkpoint", "keep", defaults::checkpoint::keep)); @@ -888,9 +888,9 @@ namespace ntt { } } - void SimulationParams::setCheckpointParams(bool is_resuming, - std::size_t start_step, - long double start_time) { + void SimulationParams::setCheckpointParams(bool is_resuming, + timestep_t start_step, + simtime_t start_time) { set("checkpoint.is_resuming", is_resuming); set("checkpoint.start_step", start_step); set("checkpoint.start_time", start_time); diff --git a/src/framework/parameters.h b/src/framework/parameters.h index 0f9e29370..83192f447 100644 --- a/src/framework/parameters.h +++ b/src/framework/parameters.h @@ -37,7 +37,7 @@ namespace ntt { void setImmutableParams(const toml::value&); void setMutableParams(const toml::value&); - void setCheckpointParams(bool, std::size_t, long double); + void setCheckpointParams(bool, timestep_t, simtime_t); void setSetupParams(const toml::value&); void checkPromises() const; diff --git a/src/framework/simulation.cpp b/src/framework/simulation.cpp index 9961eb3eb..bea50ff09 100644 --- a/src/framework/simulation.cpp +++ b/src/framework/simulation.cpp @@ -40,9 +40,9 @@ namespace ntt { fmt::toLower(toml::find(raw_params, "grid", "metric", "metric")) .c_str()); - const auto res = toml::find>(raw_params, - "grid", - "resolution"); + const auto res = toml::find>(raw_params, + "grid", + "resolution"); raise::ErrorIf(res.size() < 1 || res.size() > 3, "invalid `grid.resolution`", HERE); @@ -51,7 +51,7 @@ namespace ntt { // !TODO: when mixing checkpoint metadata with input, // ... need to properly take care of the diffs m_params.setRawData(raw_params); - std::size_t checkpoint_step = 0; + timestep_t checkpoint_step = 0; if (is_resuming) { logger::Checkpoint("Reading params from a checkpoint", HERE); if (not std::filesystem::exists("checkpoints")) { @@ -61,7 +61,7 @@ namespace ntt { std::filesystem::directory_iterator("checkpoints")) { const auto fname = entry.path().filename().string(); if (fname.find("step-") == 0) { - const std::size_t step = std::stoi(fname.substr(5, fname.size() - 5 - 3)); + const timestep_t step = std::stoi(fname.substr(5, fname.size() - 5 - 3)); if (step > checkpoint_step) { checkpoint_step = step; } @@ -78,9 +78,9 @@ namespace ntt { } logger::Checkpoint(fmt::format("Using %08lu", checkpoint_step), HERE); const auto raw_checkpoint_params = toml::parse(checkpoint_inputfname); - const auto start_time = toml::find(raw_checkpoint_params, - "metadata", - "time"); + const auto start_time = toml::find(raw_checkpoint_params, + "metadata", + "time"); m_params.setImmutableParams(raw_checkpoint_params); m_params.setMutableParams(raw_params); m_params.setCheckpointParams(true, checkpoint_step, start_time); diff --git a/src/framework/tests/comm_mpi.cpp b/src/framework/tests/comm_mpi.cpp index 2f65defd6..5d2c8d4f0 100644 --- a/src/framework/tests/comm_mpi.cpp +++ b/src/framework/tests/comm_mpi.cpp @@ -15,7 +15,7 @@ auto main(int argc, char* argv[]) -> int { Kokkos::initialize(argc, argv); try { - const std::size_t nx1 = 15, nx2 = 15; + const ncells_t nx1 = 15, nx2 = 15; ndfield_t fld_b1 { "fld", nx1 + 2 * N_GHOSTS, nx2 + 2 * N_GHOSTS }; ndfield_t fld_b2 { "fld", nx1 + 2 * N_GHOSTS, nx2 + 2 * N_GHOSTS }; @@ -58,4 +58,4 @@ auto main(int argc, char* argv[]) -> int { Kokkos::finalize(); return 0; -} \ No newline at end of file +} diff --git a/src/framework/tests/metadomain.cpp b/src/framework/tests/metadomain.cpp index 8f5865499..829a2b82f 100644 --- a/src/framework/tests/metadomain.cpp +++ b/src/framework/tests/metadomain.cpp @@ -22,31 +22,31 @@ auto main(int argc, char* argv[]) -> int { using namespace ntt; using namespace metric; { - const std::vector res { 64, 32 }; - const boundaries_t extent { - {1.0, 10.0}, - {0.0, constant::PI} + const std::vector res { 64, 32 }; + const boundaries_t extent { + { 1.0, 10.0 }, + { 0.0, constant::PI } }; const boundaries_t fldsbc { - {FldsBC::ATMOSPHERE, FldsBC::ABSORB}, - { FldsBC::AXIS, FldsBC::AXIS} + { FldsBC::ATMOSPHERE, FldsBC::MATCH }, + { FldsBC::AXIS, FldsBC::AXIS } }; const boundaries_t prtlbc { - {PrtlBC::ATMOSPHERE, PrtlBC::ABSORB}, - { PrtlBC::AXIS, PrtlBC::AXIS} + { PrtlBC::ATMOSPHERE, PrtlBC::ABSORB }, + { PrtlBC::AXIS, PrtlBC::AXIS } }; const std::map params { - {"r0", -ONE}, - { "h", (real_t)0.25} + { "r0", -ONE }, + { "h", (real_t)0.25 } }; #if defined(OUTPUT_ENABLED) Metadomain> metadomain { - 4, { -1, -1 }, - res, extent, fldsbc, prtlbc, params, {}, "disabled" + 4u, { -1, -1 }, + res, extent, fldsbc, prtlbc, params, {} }; #else Metadomain> metadomain { - 4, { -1, -1 }, + 4u, { -1, -1 }, res, extent, fldsbc, prtlbc, params, {} }; #endif @@ -132,7 +132,7 @@ auto main(int argc, char* argv[]) -> int { raise::ErrorIf(self.offset_ndomains()[0] != 1, "Domain::offset_ndomains() failed", HERE); - raise::ErrorIf(self.mesh.flds_bc_in({ +1, 0 }) != FldsBC::ABSORB, + raise::ErrorIf(self.mesh.flds_bc_in({ +1, 0 }) != FldsBC::MATCH, "Mesh::flds_bc_in() failed", HERE); raise::ErrorIf(self.mesh.prtl_bc_in({ +1, 0 }) != PrtlBC::ABSORB, @@ -203,4 +203,4 @@ auto main(int argc, char* argv[]) -> int { } Kokkos::finalize(); return 0; -} \ No newline at end of file +} diff --git a/src/global/arch/directions.h b/src/global/arch/directions.h index ccd4e67b0..2ea009814 100644 --- a/src/global/arch/directions.h +++ b/src/global/arch/directions.h @@ -50,14 +50,14 @@ namespace dir { auto operator-() const -> direction_t { auto result = direction_t {}; - for (std::size_t i = 0; i < (short)D; ++i) { + for (auto i = 0u; i < (unsigned short)D; ++i) { result[i] = -(*this)[i]; } return result; } auto operator==(const direction_t& other) const -> bool { - for (std::size_t i = 0; i < (short)D; ++i) { + for (auto i = 0u; i < (unsigned short)D; ++i) { if ((*this)[i] != other[i]) { return false; } @@ -79,7 +79,7 @@ namespace dir { */ auto get_assoc_orth() const -> std::vector { auto result = std::vector {}; - for (std::size_t i = 0; i < this->size(); ++i) { + for (auto i = 0u; i < this->size(); ++i) { if ((*this)[i] != 0) { direction_t dir; dir[i] = (*this)[i]; @@ -91,7 +91,7 @@ namespace dir { auto get_sign() const -> short { short sign = 0; - for (std::size_t i = 0; i < this->size(); ++i) { + for (auto i = 0u; i < this->size(); ++i) { if ((*this)[i] != 0) { raise::ErrorIf(sign != 0, "Undefined signature for non-orth direction", @@ -105,7 +105,7 @@ namespace dir { auto get_dim() const -> in { short dir = -1; - for (std::size_t i = 0; i < this->size(); ++i) { + for (auto i = 0u; i < this->size(); ++i) { if ((*this)[i] != 0) { raise::ErrorIf(dir > 0, "Undefined dim for non-orth direction", HERE); dir = i; diff --git a/src/global/arch/kokkos_aliases.cpp b/src/global/arch/kokkos_aliases.cpp index 6c15e3d52..d6622b0e5 100644 --- a/src/global/arch/kokkos_aliases.cpp +++ b/src/global/arch/kokkos_aliases.cpp @@ -4,10 +4,14 @@ #include +auto CreateParticleRangePolicy(npart_t p1, npart_t p2) -> range_t { + return Kokkos::RangePolicy(p1, p2); +} + template <> auto CreateRangePolicy( - const tuple_t& i1, - const tuple_t& i2) -> range_t { + const tuple_t& i1, + const tuple_t& i2) -> range_t { index_t i1min = i1[0]; index_t i1max = i2[0]; return Kokkos::RangePolicy(i1min, i1max); @@ -15,8 +19,8 @@ auto CreateRangePolicy( template <> auto CreateRangePolicy( - const tuple_t& i1, - const tuple_t& i2) -> range_t { + const tuple_t& i1, + const tuple_t& i2) -> range_t { index_t i1min = i1[0]; index_t i1max = i2[0]; index_t i2min = i1[1]; @@ -27,8 +31,8 @@ auto CreateRangePolicy( template <> auto CreateRangePolicy( - const tuple_t& i1, - const tuple_t& i2) -> range_t { + const tuple_t& i1, + const tuple_t& i2) -> range_t { index_t i1min = i1[0]; index_t i1max = i2[0]; index_t i2min = i1[1]; @@ -42,8 +46,8 @@ auto CreateRangePolicy( template <> auto CreateRangePolicyOnHost( - const tuple_t& i1, - const tuple_t& i2) -> range_h_t { + const tuple_t& i1, + const tuple_t& i2) -> range_h_t { index_t i1min = i1[0]; index_t i1max = i2[0]; return Kokkos::RangePolicy(i1min, i1max); @@ -51,8 +55,8 @@ auto CreateRangePolicyOnHost( template <> auto CreateRangePolicyOnHost( - const tuple_t& i1, - const tuple_t& i2) -> range_h_t { + const tuple_t& i1, + const tuple_t& i2) -> range_h_t { index_t i1min = i1[0]; index_t i1max = i2[0]; index_t i2min = i1[1]; @@ -63,8 +67,8 @@ auto CreateRangePolicyOnHost( template <> auto CreateRangePolicyOnHost( - const tuple_t& i1, - const tuple_t& i2) -> range_h_t { + const tuple_t& i1, + const tuple_t& i2) -> range_h_t { index_t i1min = i1[0]; index_t i1max = i2[0]; index_t i2min = i1[1]; diff --git a/src/global/arch/kokkos_aliases.h b/src/global/arch/kokkos_aliases.h index f9aac9685..e1a759bed 100644 --- a/src/global/arch/kokkos_aliases.h +++ b/src/global/arch/kokkos_aliases.h @@ -219,27 +219,34 @@ namespace kokkos_aliases_hidden { template using range_h_t = typename kokkos_aliases_hidden::range_h_impl::type; +/** + * @brief Function template for generating 1D Kokkos range policy for particles. + * @param p1 `npart_t`: min. + * @param p2 `npart_t`: max. + */ +auto CreateParticleRangePolicy(npart_t, npart_t) -> range_t; + /** * @brief Function template for generating ND Kokkos range policy. * @tparam D Dimension - * @param i1 array of size D `std::size_t`: { min }. - * @param i2 array of size D `std::size_t`: { max }. + * @param i1 array of size D `ncells_t`: { min }. + * @param i2 array of size D `ncells_t`: { max }. * @returns Kokkos::RangePolicy or Kokkos::MDRangePolicy in the accelerator execution space. */ template -auto CreateRangePolicy(const tuple_t&, - const tuple_t&) -> range_t; +auto CreateRangePolicy(const tuple_t&, + const tuple_t&) -> range_t; /** * @brief Function template for generating ND Kokkos range policy on the host. * @tparam D Dimension - * @param i1 array of size D `std::size_t`: { min }. - * @param i2 array of size D `std::size_t`: { max }. + * @param i1 array of size D `ncells_t`: { min }. + * @param i2 array of size D `ncells_t`: { max }. * @returns Kokkos::RangePolicy or Kokkos::MDRangePolicy in the host execution space. */ template -auto CreateRangePolicyOnHost(const tuple_t&, - const tuple_t&) -> range_h_t; +auto CreateRangePolicyOnHost(const tuple_t&, + const tuple_t&) -> range_h_t; // Random number pool/generator type alias using random_number_pool_t = Kokkos::Random_XorShift1024_Pool; diff --git a/src/global/defaults.h b/src/global/defaults.h index f44fd1844..b81369022 100644 --- a/src/global/defaults.h +++ b/src/global/defaults.h @@ -24,7 +24,7 @@ namespace ntt::defaults { const std::string em_pusher = "Boris"; const std::string ph_pusher = "Photon"; - const std::size_t clear_interval = 100; + const timestep_t clear_interval = 100; namespace qsph { const real_t r0 = 0.0; @@ -53,9 +53,9 @@ namespace ntt::defaults { namespace output { const std::string format = "hdf5"; - const std::size_t interval = 100; + const timestep_t interval = 100; const unsigned short mom_smooth = 0; - const std::size_t prtl_stride = 100; + const npart_t prtl_stride = 100; const real_t spec_emin = 1e-3; const real_t spec_emax = 1e3; const bool spec_log = true; @@ -63,12 +63,12 @@ namespace ntt::defaults { } // namespace output namespace checkpoint { - const std::size_t interval = 1000; - const int keep = 2; + const timestep_t interval = 1000; + const int keep = 2; } // namespace checkpoint namespace diag { - const std::size_t interval = 1; + const timestep_t interval = 1; } // namespace diag namespace gca { diff --git a/src/global/enums.h b/src/global/enums.h index 8f2495c13..5860f8346 100644 --- a/src/global/enums.h +++ b/src/global/enums.h @@ -56,7 +56,7 @@ namespace ntt { const char* const* arr_c, const std::size_t n, const char* elem) -> T { - for (std::size_t i = 0; i < n; ++i) { + for (auto i = 0u; i < n; ++i) { if (strcmp(arr_c[i], elem) == 0) { return (T)(arr[i]); } @@ -70,7 +70,7 @@ namespace ntt { constexpr auto baseContains(const char* const* arr_c, const std::size_t n, const char* elem) -> bool { - for (std::size_t i = 0; i < n; ++i) { + for (auto i = 0u; i < n; ++i) { if (strcmp(arr_c[i], elem) == 0) { return true; } diff --git a/src/global/global.h b/src/global/global.h index 77fa8c51c..577b13f1a 100644 --- a/src/global/global.h +++ b/src/global/global.h @@ -92,6 +92,7 @@ #ifndef GLOBAL_GLOBAL_H #define GLOBAL_GLOBAL_H +#include #include #include #include @@ -111,7 +112,7 @@ namespace files { namespace ntt { - inline constexpr unsigned int N_GHOSTS = 2; + inline constexpr std::size_t N_GHOSTS = 2; // Coordinate shift to account for ghost cells #define COORD(I) \ (static_cast(static_cast((I)) - static_cast(N_GHOSTS))) @@ -339,10 +340,19 @@ using coord_t = tuple_t; template using vec_t = tuple_t; +// time/duration +using timestamp_t = std::chrono::time_point; +using duration_t = double; +using simtime_t = double; +using timestep_t = std::size_t; +using ncells_t = std::size_t; +using npart_t = unsigned long long int; + +// index/number using index_t = const std::size_t; using idx_t = unsigned short; -using range_tuple_t = std::pair; +using range_tuple_t = std::pair; template using boundaries_t = std::vector>; diff --git a/src/global/utils/diag.cpp b/src/global/utils/diag.cpp index c053cdacf..fbd0c4215 100644 --- a/src/global/utils/diag.cpp +++ b/src/global/utils/diag.cpp @@ -22,9 +22,9 @@ namespace diag { auto npart_stats( - std::size_t npart, - std::size_t maxnpart) -> std::vector> { - auto stats = std::vector>(); + npart_t npart, + npart_t maxnpart) -> std::vector> { + auto stats = std::vector>(); #if !defined(MPI_ENABLED) stats.push_back( { npart, @@ -34,22 +34,22 @@ namespace diag { int rank, size; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); - std::vector mpi_npart(size, 0); - std::vector mpi_maxnpart(size, 0); + std::vector mpi_npart(size, 0); + std::vector mpi_maxnpart(size, 0); MPI_Gather(&npart, 1, - mpi::get_type(), + mpi::get_type(), mpi_npart.data(), 1, - mpi::get_type(), + mpi::get_type(), MPI_ROOT_RANK, MPI_COMM_WORLD); MPI_Gather(&maxnpart, 1, - mpi::get_type(), + mpi::get_type(), mpi_maxnpart.data(), 1, - mpi::get_type(), + mpi::get_type(), MPI_ROOT_RANK, MPI_COMM_WORLD); if (rank != MPI_ROOT_RANK) { @@ -75,16 +75,16 @@ namespace diag { return stats; } - void printDiagnostics(std::size_t step, - std::size_t tot_steps, - long double time, - long double dt, + void printDiagnostics(timestep_t step, + timestep_t tot_steps, + simtime_t time, + simtime_t dt, timer::Timers& timers, pbar::DurationHistory& time_history, - std::size_t ncells, + ncells_t ncells, const std::vector& species_labels, - const std::vector& species_npart, - const std::vector& species_maxnpart, + const std::vector& species_npart, + const std::vector& species_maxnpart, bool print_prtl_clear, bool print_output, bool print_checkpoint, @@ -168,7 +168,7 @@ namespace diag { c_reset); }); #endif - for (std::size_t i = 0; i < species_labels.size(); ++i) { + for (auto i = 0u; i < species_labels.size(); ++i) { const auto part_stats = npart_stats(species_npart[i], species_maxnpart[i]); if (part_stats.size() == 0) { continue; diff --git a/src/global/utils/diag.h b/src/global/utils/diag.h index 30cca5705..6d3c5937d 100644 --- a/src/global/utils/diag.h +++ b/src/global/utils/diag.h @@ -39,16 +39,16 @@ namespace diag { * @param checkpoint (if true, checkpoint was written) * @param colorful_print (if true, print with colors) */ - void printDiagnostics(std::size_t, - std::size_t, - long double, - long double, + void printDiagnostics(timestep_t, + timestep_t, + simtime_t, + simtime_t, timer::Timers&, pbar::DurationHistory&, - std::size_t, + ncells_t, const std::vector&, - const std::vector&, - const std::vector&, + const std::vector&, + const std::vector&, bool, bool, bool, diff --git a/src/global/utils/param_container.cpp b/src/global/utils/param_container.cpp index 6e4f5e0f4..e1dd57c9e 100644 --- a/src/global/utils/param_container.cpp +++ b/src/global/utils/param_container.cpp @@ -32,8 +32,8 @@ namespace prm { } template - auto write(adios2::IO& io, const std::string& name, T var) - -> decltype(void(T()), void()) { + auto write(adios2::IO& io, const std::string& name, T var) -> decltype(void(T()), + void()) { io.DefineAttribute(name, var); } @@ -48,8 +48,8 @@ namespace prm { } template - auto write_pair(adios2::IO& io, const std::string& name, std::pair var) - -> typename std::enable_if::value, void>::type { + auto write_pair(adios2::IO& io, const std::string& name, std::pair var) -> + typename std::enable_if::value, void>::type { std::vector var_str; var_str.push_back(var.first.to_string()); var_str.push_back(var.second.to_string()); @@ -57,8 +57,9 @@ namespace prm { } template - auto write_pair(adios2::IO& io, const std::string& name, std::pair var) - -> decltype(void(T()), void()) { + auto write_pair(adios2::IO& io, + const std::string& name, + std::pair var) -> decltype(void(T()), void()) { std::vector var_vec; var_vec.push_back(var.first); var_vec.push_back(var.second); @@ -76,8 +77,9 @@ namespace prm { } template - auto write_vec(adios2::IO& io, const std::string& name, std::vector var) - -> decltype(void(T()), void()) { + auto write_vec(adios2::IO& io, + const std::string& name, + std::vector var) -> decltype(void(T()), void()) { io.DefineAttribute(name, var.data(), var.size()); } @@ -97,8 +99,8 @@ namespace prm { template auto write_vec_pair(adios2::IO& io, const std::string& name, - std::vector> var) - -> decltype(void(T()), void()) { + std::vector> var) -> decltype(void(T()), + void()) { std::vector var_vec; for (const auto& v : var) { var_vec.push_back(v.first); @@ -124,8 +126,8 @@ namespace prm { template auto write_vec_vec(adios2::IO& io, const std::string& name, - std::vector> var) - -> decltype(void(T()), void()) { + std::vector> var) -> decltype(void(T()), + void()) { std::vector var_vec; for (const auto& vec : var) { for (const auto& v : vec) { @@ -215,34 +217,36 @@ namespace prm { } void Parameters::write(adios2::IO& io) const { - register_write_function(); + register_write_function(); register_write_function(); + register_write_function(); + register_write_function(); register_write_function(); - register_write_function(); register_write_function(); register_write_function(); - register_write_function(); register_write_function(); + register_write_function(); + register_write_function(); register_write_function(); - register_write_function(); register_write_function(); + register_write_function(); + register_write_function(); register_write_function(); register_write_function(); register_write_function(); register_write_function(); register_write_function(); register_write_function(); - register_write_function(); - register_write_function(); - register_write_function_for_pair(); register_write_function_for_pair(); + register_write_function_for_pair(); + register_write_function_for_pair(); register_write_function_for_pair(); - register_write_function_for_pair(); register_write_function_for_pair(); register_write_function_for_pair(); - register_write_function_for_pair(); register_write_function_for_pair(); + register_write_function_for_pair(); + register_write_function_for_pair(); register_write_function_for_pair(); register_write_function_for_pair(); register_write_function_for_pair(); @@ -253,14 +257,15 @@ namespace prm { register_write_function_for_pair(); register_write_function_for_pair(); - register_write_function_for_vector(); register_write_function_for_vector(); + register_write_function_for_vector(); + register_write_function_for_vector(); register_write_function_for_vector(); - register_write_function_for_vector(); register_write_function_for_vector(); register_write_function_for_vector(); - register_write_function_for_vector(); register_write_function_for_vector(); + register_write_function_for_vector(); + register_write_function_for_vector(); register_write_function_for_vector(); register_write_function_for_vector(); register_write_function_for_vector(); @@ -271,14 +276,15 @@ namespace prm { register_write_function_for_vector(); register_write_function_for_vector(); - register_write_function_for_vector_of_pair(); register_write_function_for_vector_of_pair(); + register_write_function_for_vector_of_pair(); + register_write_function_for_vector_of_pair(); register_write_function_for_vector_of_pair(); - register_write_function_for_vector_of_pair(); register_write_function_for_vector_of_pair(); register_write_function_for_vector_of_pair(); - register_write_function_for_vector_of_pair(); register_write_function_for_vector_of_pair(); + register_write_function_for_vector_of_pair(); + register_write_function_for_vector_of_pair(); register_write_function_for_vector_of_pair(); register_write_function_for_vector_of_pair(); register_write_function_for_vector_of_pair(); @@ -289,14 +295,15 @@ namespace prm { register_write_function_for_vector_of_pair(); register_write_function_for_vector_of_pair(); - register_write_function_for_vector_of_vector(); register_write_function_for_vector_of_vector(); + register_write_function_for_vector_of_vector(); + register_write_function_for_vector_of_vector(); register_write_function_for_vector_of_vector(); - register_write_function_for_vector_of_vector(); register_write_function_for_vector_of_vector(); register_write_function_for_vector_of_vector(); - register_write_function_for_vector_of_vector(); register_write_function_for_vector_of_vector(); + register_write_function_for_vector_of_vector(); + register_write_function_for_vector_of_vector(); register_write_function_for_vector_of_vector(); register_write_function_for_vector_of_vector(); register_write_function_for_vector_of_vector(); @@ -307,14 +314,15 @@ namespace prm { register_write_function_for_vector_of_vector(); register_write_function_for_vector_of_vector(); - register_write_function_for_dict(); register_write_function_for_dict(); + register_write_function_for_dict(); + register_write_function_for_dict(); register_write_function_for_dict(); - register_write_function_for_dict(); register_write_function_for_dict(); register_write_function_for_dict(); - register_write_function_for_dict(); register_write_function_for_dict(); + register_write_function_for_dict(); + register_write_function_for_dict(); register_write_function_for_dict(); register_write_function_for_dict(); register_write_function_for_dict(); diff --git a/src/global/utils/progressbar.cpp b/src/global/utils/progressbar.cpp index 74f952382..eaa8118fc 100644 --- a/src/global/utils/progressbar.cpp +++ b/src/global/utils/progressbar.cpp @@ -1,5 +1,7 @@ #include "utils/progressbar.h" +#include "global.h" + #include "utils/error.h" #include "utils/formatting.h" @@ -19,14 +21,15 @@ namespace pbar { - auto normalize_duration_fmt(long double t, const std::string& u) - -> std::pair { - const std::vector> units { - {"µs", 1e0}, - { "ms", 1e3}, - { "s", 1e6}, - {"min", 6e7}, - { "hr", 3.6e9} + auto normalize_duration_fmt( + duration_t t, + const std::string& u) -> std::pair { + const std::vector> units { + { "µs", 1e0 }, + { "ms", 1e3 }, + { "s", 1e6 }, + { "min", 6e7 }, + { "hr", 3.6e9 } }; auto it = std::find_if(units.begin(), units.end(), [&u](const auto& pr) { return pr.first == u; @@ -51,17 +54,17 @@ namespace pbar { units[newu_idx].first }; } - auto to_human_readable(long double t, const std::string& u) -> std::string { + auto to_human_readable(duration_t t, const std::string& u) -> std::string { const auto [tt, tu] = normalize_duration_fmt(t, u); const auto t1 = static_cast(tt); - const auto t2 = tt - static_cast(t1); + const auto t2 = tt - static_cast(t1); const auto [tt2, tu2] = normalize_duration_fmt(t2, tu); return fmt::format("%d%s %d%s", t1, tu.c_str(), static_cast(tt2), tu2.c_str()); } auto ProgressBar(const DurationHistory& history, - std::size_t step, - std::size_t max_steps, + timestep_t step, + timestep_t max_steps, DiagFlags& flags) -> std::string { auto avg_duration = history.average(); @@ -69,13 +72,13 @@ namespace pbar { int rank, size; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); - std::vector mpi_avg_durations(size, 0.0); + std::vector mpi_avg_durations(size, 0.0); MPI_Gather(&avg_duration, 1, - mpi::get_type(), + mpi::get_type(), mpi_avg_durations.data(), 1, - mpi::get_type(), + mpi::get_type(), MPI_ROOT_RANK, MPI_COMM_WORLD); if (rank != MPI_ROOT_RANK) { @@ -88,11 +91,11 @@ namespace pbar { const auto avg = to_human_readable(avg_duration, "µs"); const auto elapsed = to_human_readable(history.elapsed(), "µs"); const auto remain = to_human_readable( - static_cast(max_steps - step) * avg_duration, + static_cast(max_steps - step) * avg_duration, "µs"); - const auto pct = static_cast(step) / - static_cast(max_steps); + const auto pct = static_cast(step) / + static_cast(max_steps); const int nfilled = std::min(static_cast(pct * params::width), params::width); const int nempty = params::width - nfilled; diff --git a/src/global/utils/progressbar.h b/src/global/utils/progressbar.h index f218bc3f4..588413cb4 100644 --- a/src/global/utils/progressbar.h +++ b/src/global/utils/progressbar.h @@ -38,7 +38,7 @@ namespace pbar { class DurationHistory { std::size_t capacity; - std::vector durations; + std::vector durations; const std::chrono::time_point start; std::chrono::time_point prev_start; @@ -60,30 +60,31 @@ namespace pbar { prev_start = now; } - auto average() const -> long double { + auto average() const -> duration_t { if (durations.size() > 0) { return std::accumulate(durations.begin(), durations.end(), 0.0) / - static_cast(durations.size()); + static_cast(durations.size()); } else { return 0.0; } } - auto elapsed() const -> long double { + auto elapsed() const -> duration_t { return std::chrono::duration_cast( std::chrono::system_clock::now() - start) .count(); } }; - auto normalize_duration_fmt(long double t, const std::string& u) - -> std::pair; + auto normalize_duration_fmt( + duration_t t, + const std::string& u) -> std::pair; - auto to_human_readable(long double t, const std::string& u) -> std::string; + auto to_human_readable(duration_t t, const std::string& u) -> std::string; auto ProgressBar(const DurationHistory& history, - std::size_t step, - std::size_t max_steps, + timestep_t step, + timestep_t max_steps, DiagFlags& flags) -> std::string; } // namespace pbar diff --git a/src/global/utils/timer.cpp b/src/global/utils/timer.cpp index a12d79b96..d3c14d05e 100644 --- a/src/global/utils/timer.cpp +++ b/src/global/utils/timer.cpp @@ -1,5 +1,7 @@ #include "utils/timer.h" +#include "global.h" + #include "utils/colors.h" #include "utils/formatting.h" @@ -18,8 +20,8 @@ namespace timer { auto Timers::gather(const std::vector& ignore_in_tot, - std::size_t npart, - std::size_t ncells) const + npart_t npart, + ncells_t ncells) const -> std::map> { auto timer_stats = std::map< @@ -44,22 +46,22 @@ namespace timer { MPI_COMM_WORLD); } // accumulate nparts and ncells from MPI blocks - auto all_nparts = std::vector(size, 0); - auto all_ncells = std::vector(size, 0); + auto all_nparts = std::vector(size, 0); + auto all_ncells = std::vector(size, 0); MPI_Gather(&npart, 1, - mpi::get_type(), + mpi::get_type(), all_nparts.data(), 1, - mpi::get_type(), + mpi::get_type(), MPI_ROOT_RANK, MPI_COMM_WORLD); MPI_Gather(&ncells, 1, - mpi::get_type(), + mpi::get_type(), all_ncells.data(), 1, - mpi::get_type(), + mpi::get_type(), MPI_ROOT_RANK, MPI_COMM_WORLD); if (rank != MPI_ROOT_RANK) { @@ -128,9 +130,9 @@ namespace timer { return timer_stats; } - auto Timers::printAll(TimerFlags flags, - std::size_t npart, - std::size_t ncells) const -> std::string { + auto Timers::printAll(TimerFlags flags, + npart_t npart, + ncells_t ncells) const -> std::string { const std::vector extras { "PrtlClear", "Output", "Checkpoint" }; const auto stats = gather(extras, npart, ncells); if (stats.empty()) { diff --git a/src/global/utils/timer.h b/src/global/utils/timer.h index 3e9c1433e..8abe8ab0a 100644 --- a/src/global/utils/timer.h +++ b/src/global/utils/timer.h @@ -41,8 +41,6 @@ #include namespace timer { - using timestamp = std::chrono::time_point; - using duration_t = long double; inline void convertTime(duration_t& value, std::string& units) { if (value > 1e6) { @@ -58,10 +56,10 @@ namespace timer { } class Timers { - std::map> m_timers; - std::vector m_names; - const bool m_blocking; - const std::function m_synchronize; + std::map> m_timers; + std::vector m_names; + const bool m_blocking; + const std::function m_synchronize; public: Timers(std::initializer_list names, @@ -75,7 +73,7 @@ namespace timer { for (const auto& name : names) { m_timers.insert({ name, - {std::chrono::system_clock::now(), 0.0} + { std::chrono::system_clock::now(), 0.0 } }); m_names.push_back(name); } @@ -89,7 +87,9 @@ namespace timer { void stop(const std::string& name) { if (m_blocking) { - m_synchronize(); + if (m_synchronize != nullptr) { + m_synchronize(); + } #if defined(MPI_ENABLED) MPI_Barrier(MPI_COMM_WORLD); #endif @@ -139,15 +139,15 @@ namespace timer { */ [[nodiscard]] auto gather(const std::vector& ignore_in_tot, - std::size_t npart, - std::size_t ncells) const + npart_t npart, + ncells_t ncells) const -> std::map>; [[nodiscard]] - auto printAll(TimerFlags flags = Timer::Default, - std::size_t npart = 0, - std::size_t ncells = 0) const -> std::string; + auto printAll(TimerFlags flags = Timer::Default, + npart_t npart = 0, + ncells_t ncells = 0) const -> std::string; }; } // namespace timer diff --git a/src/global/utils/tools.h b/src/global/utils/tools.h index 30687d01b..95765e2de 100644 --- a/src/global/utils/tools.h +++ b/src/global/utils/tools.h @@ -4,10 +4,10 @@ * @implements * - tools::ArrayImbalance -> unsigned short * - tools::TensorProduct<> -> boundaries_t - * - tools::decompose1D -> std::vector + * - tools::decompose1D -> std::vector * - tools::divideInProportions2D -> std::tuple * - tools::divideInProportions3D -> std::tuple - * - tools::Decompose -> std::vector> + * - tools::Decompose -> std::vector> * - tools::Tracker * @namespaces: * - tools:: @@ -60,8 +60,8 @@ namespace tools { * @return Tensor product of list */ template - inline auto TensorProduct(const std::vector>& list) - -> std::vector> { + inline auto TensorProduct( + const std::vector>& list) -> std::vector> { std::vector> result = { {} }; for (const auto& sublist : list) { std::vector> temp; @@ -81,16 +81,16 @@ namespace tools { * @param ndomains Number of domains * @param ncells Number of cells */ - inline auto decompose1D(unsigned int ndomains, std::size_t ncells) - -> std::vector { - auto size = (std::size_t)((double)ncells / (double)ndomains); - auto ncells_domain = std::vector(ndomains, size); - for (std::size_t i { 0 }; i < ncells - size * ndomains; ++i) { + inline auto decompose1D(unsigned int ndomains, + ncells_t ncells) -> std::vector { + auto size = (ncells_t)((double)ncells / (double)ndomains); + auto ncells_domain = std::vector(ndomains, size); + for (auto i { 0u }; i < ncells - size * ndomains; ++i) { ncells_domain[i] += 1; } auto sum = std::accumulate(ncells_domain.begin(), ncells_domain.end(), - (std::size_t)0); + (ncells_t)0); raise::ErrorIf(sum != ncells, "Decomposition error: sum != ncells", HERE); raise::ErrorIf(ncells_domain.size() != (std::size_t)ndomains, "Decomposition error: size != ndomains", @@ -107,8 +107,10 @@ namespace tools { * @param s1 Proportion of the first dimension * @param s2 Proportion of the second dimension */ - inline auto divideInProportions2D(unsigned int ntot, unsigned int s1, unsigned int s2) - -> std::tuple { + inline auto divideInProportions2D( + unsigned int ntot, + unsigned int s1, + unsigned int s2) -> std::tuple { auto n1 = (unsigned int)(std::sqrt((double)ntot * (double)s1 / (double)s2)); if (n1 == 0) { return { 1, ntot }; @@ -130,11 +132,11 @@ namespace tools { * @param s2 Proportion of the second dimension * @param s3 Proportion of the third dimension */ - inline auto divideInProportions3D(unsigned int ntot, - unsigned int s1, - unsigned int s2, - unsigned int s3) - -> std::tuple { + inline auto divideInProportions3D( + unsigned int ntot, + unsigned int s1, + unsigned int s2, + unsigned int s3) -> std::tuple { auto n1 = (unsigned int)(std::cbrt( (double)ntot * (double)(SQR(s1)) / (double)(s2 * s3))); if (n1 > ntot) { @@ -163,10 +165,10 @@ namespace tools { * * @note If decomposition has -1, it will be calculated automatically */ - inline auto Decompose(unsigned int ndomains, - const std::vector& ncells, - const std::vector& decomposition) - -> std::vector> { + inline auto Decompose( + unsigned int ndomains, + const std::vector& ncells, + const std::vector& decomposition) -> std::vector> { const auto dimension = ncells.size(); raise::ErrorIf(dimension != decomposition.size(), "Decomposition error: dimension != decomposition.size", @@ -277,16 +279,16 @@ namespace tools { bool m_initialized { false }; std::string m_type; - std::size_t m_interval; - long double m_interval_time; + timestep_t m_interval; + simtime_t m_interval_time; bool m_use_time; - long double m_last_output_time { -1.0 }; + simtime_t m_last_output_time { -1.0 }; public: Tracker() = default; - Tracker(const std::string& type, std::size_t interval, long double interval_time) + Tracker(const std::string& type, timestep_t interval, simtime_t interval_time) : m_initialized { true } , m_type { type } , m_interval { interval } @@ -295,9 +297,7 @@ namespace tools { ~Tracker() = default; - void init(const std::string& type, - std::size_t interval, - long double interval_time) { + void init(const std::string& type, timestep_t interval, simtime_t interval_time) { m_type = type; m_interval = interval; m_interval_time = interval_time; @@ -305,7 +305,7 @@ namespace tools { m_initialized = true; } - auto shouldWrite(std::size_t step, long double time) -> bool { + auto shouldWrite(timestep_t step, simtime_t time) -> bool { raise::ErrorIf(!m_initialized, "Tracker not initialized", HERE); if (m_use_time) { if ((m_last_output_time < 0) or diff --git a/src/kernels/ampere_gr.hpp b/src/kernels/ampere_gr.hpp index 5af0fa4ef..782dece0d 100644 --- a/src/kernels/ampere_gr.hpp +++ b/src/kernels/ampere_gr.hpp @@ -37,7 +37,7 @@ namespace kernel::gr { ndfield_t Dout; const ndfield_t H; const M metric; - const std::size_t i2max; + const ncells_t i2max; const real_t coeff; bool is_axis_i2min { false }, is_axis_i2max { false }; @@ -47,7 +47,7 @@ namespace kernel::gr { const ndfield_t& H, const M& metric, real_t coeff, - std::size_t ni2, + ncells_t ni2, const boundaries_t& boundaries) : Din { Din } , Dout { Dout } @@ -67,9 +67,9 @@ namespace kernel::gr { Inline void operator()(index_t i1, index_t i2) const { if constexpr (D == Dim::_2D) { - constexpr std::size_t i2min { N_GHOSTS }; - const real_t i1_ { COORD(i1) }; - const real_t i2_ { COORD(i2) }; + constexpr ncells_t i2min { N_GHOSTS }; + const real_t i1_ { COORD(i1) }; + const real_t i2_ { COORD(i2) }; const real_t inv_sqrt_detH_0pH { ONE / metric.sqrt_det_h({ i1_, i2_ + HALF }) }; @@ -118,7 +118,7 @@ namespace kernel::gr { ndfield_t Df; const ndfield_t J; const M metric; - const std::size_t i2max; + const ncells_t i2max; const real_t coeff; bool is_axis_i2min { false }; bool is_axis_i2max { false }; @@ -132,7 +132,7 @@ namespace kernel::gr { const ndfield_t& J, const M& metric, real_t coeff, - std::size_t ni2, + ncells_t ni2, const boundaries_t& boundaries) : Df { Df } , J { J } @@ -148,9 +148,9 @@ namespace kernel::gr { Inline void operator()(index_t i1, index_t i2) const { if constexpr (D == Dim::_2D) { - constexpr std::size_t i2min { N_GHOSTS }; - const real_t i1_ { COORD(i1) }; - const real_t i2_ { COORD(i2) }; + constexpr ncells_t i2min { N_GHOSTS }; + const real_t i1_ { COORD(i1) }; + const real_t i2_ { COORD(i2) }; const real_t inv_sqrt_detH_0pH { ONE / metric.sqrt_det_h({ i1_, i2_ + HALF }) }; diff --git a/src/kernels/ampere_sr.hpp b/src/kernels/ampere_sr.hpp index e4faec6ce..dbe9d3dbf 100644 --- a/src/kernels/ampere_sr.hpp +++ b/src/kernels/ampere_sr.hpp @@ -32,17 +32,17 @@ namespace kernel::sr { static_assert(M::is_metric, "M must be a metric class"); static constexpr auto D = M::Dim; - ndfield_t EB; - const M metric; - const std::size_t i2max; - const real_t coeff; - bool is_axis_i2min { false }, is_axis_i2max { false }; + ndfield_t EB; + const M metric; + const ncells_t i2max; + const real_t coeff; + bool is_axis_i2min { false }, is_axis_i2max { false }; public: Ampere_kernel(const ndfield_t& EB, const M& metric, real_t coeff, - std::size_t ni2, + ncells_t ni2, const boundaries_t& boundaries) : EB { EB } , metric { metric } @@ -57,9 +57,9 @@ namespace kernel::sr { Inline void operator()(index_t i1, index_t i2) const { if constexpr (D == Dim::_2D) { - constexpr std::size_t i2min { N_GHOSTS }; - const real_t i1_ { COORD(i1) }; - const real_t i2_ { COORD(i2) }; + constexpr ncells_t i2min { N_GHOSTS }; + const real_t i1_ { COORD(i1) }; + const real_t i2_ { COORD(i2) }; const real_t inv_sqrt_detH_0pH { ONE / metric.sqrt_det_h({ i1_, i2_ + HALF }) }; @@ -122,18 +122,18 @@ namespace kernel::sr { */ template class CurrentsAmpere_kernel { - static constexpr auto D = M::Dim; - static constexpr std::size_t i2min = N_GHOSTS; + static constexpr auto D = M::Dim; + static constexpr ncells_t i2min = N_GHOSTS; - ndfield_t E; - ndfield_t J; - const M metric; - const std::size_t i2max; + ndfield_t E; + ndfield_t J; + const M metric; + const ncells_t i2max; // coeff = -dt * q0 * n0 / B0 - const real_t coeff; - const real_t inv_n0; - bool is_axis_i2min { false }; - bool is_axis_i2max { false }; + const real_t coeff; + const real_t inv_n0; + bool is_axis_i2min { false }; + bool is_axis_i2max { false }; public: /** @@ -145,7 +145,7 @@ namespace kernel::sr { const M& metric, real_t coeff, real_t inv_n0, - std::size_t ni2, + ncells_t ni2, const boundaries_t& boundaries) : E { E } , J { J } diff --git a/src/kernels/comm.hpp b/src/kernels/comm.hpp index b280ce38b..7873870b1 100644 --- a/src/kernels/comm.hpp +++ b/src/kernels/comm.hpp @@ -24,35 +24,36 @@ namespace kernel::comm { template class PrepareOutgoingPrtls_kernel { - const array_t shifts_in_x1, shifts_in_x2, shifts_in_x3; - array_t outgoing_indices; + const array_t shifts_in_x1, shifts_in_x2, shifts_in_x3; + array_t outgoing_indices; - const std::size_t npart, npart_alive, npart_dead, ntags; + const npart_t npart, npart_alive, npart_dead; + const std::size_t ntags; array_t i1, i1_prev, i2, i2_prev, i3, i3_prev; const array_t tag; - const array_t tag_offsets; + const array_t tag_offsets; - array_t current_offset; + array_t current_offset; public: - PrepareOutgoingPrtls_kernel(const array_t& shifts_in_x1, - const array_t& shifts_in_x2, - const array_t& shifts_in_x3, - array_t& outgoing_indices, - std::size_t npart, - std::size_t npart_alive, - std::size_t npart_dead, - std::size_t ntags, - array_t& i1, - array_t& i1_prev, - array_t& i2, - array_t& i2_prev, - array_t& i3, - array_t& i3_prev, - const array_t& tag, - const array_t& tag_offsets) + PrepareOutgoingPrtls_kernel(const array_t& shifts_in_x1, + const array_t& shifts_in_x2, + const array_t& shifts_in_x3, + array_t& outgoing_indices, + npart_t npart, + npart_t npart_alive, + npart_t npart_dead, + std::size_t ntags, + array_t& i1, + array_t& i1_prev, + array_t& i2, + array_t& i2_prev, + array_t& i3, + array_t& i3_prev, + const array_t& tag, + const array_t& tag_offsets) : shifts_in_x1 { shifts_in_x1 } , shifts_in_x2 { shifts_in_x2 } , shifts_in_x3 { shifts_in_x3 } @@ -225,45 +226,45 @@ namespace kernel::comm { const array_t recv_buff_pld; const unsigned short NINTS, NREALS, NPRTLDX, NPLDS; - const std::size_t npart, npart_holes; + const npart_t npart, npart_holes; - array_t i1, i1_prev, i2, i2_prev, i3, i3_prev; - array_t dx1, dx1_prev, dx2, dx2_prev, dx3, dx3_prev; - array_t ux1, ux2, ux3, weight, phi; - array_t pld; - array_t tag; - const array_t outgoing_indices; + array_t i1, i1_prev, i2, i2_prev, i3, i3_prev; + array_t dx1, dx1_prev, dx2, dx2_prev, dx3, dx3_prev; + array_t ux1, ux2, ux3, weight, phi; + array_t pld; + array_t tag; + const array_t outgoing_indices; public: - ExtractReceivedPrtls_kernel(const array_t& recv_buff_int, - const array_t& recv_buff_real, - const array_t& recv_buff_prtldx, - const array_t& recv_buff_pld, - unsigned short NINTS, - unsigned short NREALS, - unsigned short NPRTLDX, - unsigned short NPLDS, - std::size_t npart, - array_t& i1, - array_t& i1_prev, - array_t& dx1, - array_t& dx1_prev, - array_t& i2, - array_t& i2_prev, - array_t& dx2, - array_t& dx2_prev, - array_t& i3, - array_t& i3_prev, - array_t& dx3, - array_t& dx3_prev, - array_t& ux1, - array_t& ux2, - array_t& ux3, - array_t& weight, - array_t& phi, - array_t& pld, - array_t& tag, - const array_t& outgoing_indices) + ExtractReceivedPrtls_kernel(const array_t& recv_buff_int, + const array_t& recv_buff_real, + const array_t& recv_buff_prtldx, + const array_t& recv_buff_pld, + unsigned short NINTS, + unsigned short NREALS, + unsigned short NPRTLDX, + unsigned short NPLDS, + npart_t npart, + array_t& i1, + array_t& i1_prev, + array_t& dx1, + array_t& dx1_prev, + array_t& i2, + array_t& i2_prev, + array_t& dx2, + array_t& dx2_prev, + array_t& i3, + array_t& i3_prev, + array_t& dx3, + array_t& dx3_prev, + array_t& ux1, + array_t& ux2, + array_t& ux3, + array_t& weight, + array_t& phi, + array_t& pld, + array_t& tag, + const array_t& outgoing_indices) : recv_buff_int { recv_buff_int } , recv_buff_real { recv_buff_real } , recv_buff_prtldx { recv_buff_prtldx } @@ -296,7 +297,7 @@ namespace kernel::comm { , outgoing_indices { outgoing_indices } {} Inline void operator()(index_t p) const { - std::size_t idx; + npart_t idx; if (p >= npart_holes) { idx = npart + p - npart_holes; } else { diff --git a/src/kernels/digital_filter.hpp b/src/kernels/digital_filter.hpp index 5d05fad2d..f01eb0fb8 100644 --- a/src/kernels/digital_filter.hpp +++ b/src/kernels/digital_filter.hpp @@ -19,7 +19,7 @@ #define FILTER_IN_I1(ARR, COMP, I, J) \ INV_2*(ARR)((I), (J), (COMP)) + \ - INV_4*((ARR)((I)-1, (J), (COMP)) + (ARR)((I) + 1, (J), (COMP))) + INV_4*((ARR)((I) - 1, (J), (COMP)) + (ARR)((I) + 1, (J), (COMP))) namespace kernel { using namespace ntt; @@ -30,12 +30,12 @@ namespace kernel { const ndfield_t buffer; bool is_axis_i2min { false }, is_axis_i2max { false }; static constexpr auto i2_min = N_GHOSTS; - const std::size_t i2_max; + const ncells_t i2_max; public: DigitalFilter_kernel(ndfield_t& array, const ndfield_t& buffer, - const std::size_t (&size_)[D], + const ncells_t (&size_)[D], const boundaries_t& boundaries) : array { array } , buffer { buffer } diff --git a/src/kernels/faraday_gr.hpp b/src/kernels/faraday_gr.hpp index 19eede5f2..7fb327e61 100644 --- a/src/kernels/faraday_gr.hpp +++ b/src/kernels/faraday_gr.hpp @@ -36,7 +36,7 @@ namespace kernel::gr { ndfield_t Bout; const ndfield_t E; const M metric; - const std::size_t i2max; + const ncells_t i2max; const real_t coeff; bool is_axis_i2min { false }; diff --git a/src/kernels/faraday_sr.hpp b/src/kernels/faraday_sr.hpp index 727974248..be3e60dd0 100644 --- a/src/kernels/faraday_sr.hpp +++ b/src/kernels/faraday_sr.hpp @@ -52,9 +52,9 @@ namespace kernel::sr { Inline void operator()(index_t i1, index_t i2) const { if constexpr (D == Dim::_2D) { - constexpr std::size_t i2min { N_GHOSTS }; - const real_t i1_ { COORD(i1) }; - const real_t i2_ { COORD(i2) }; + constexpr ncells_t i2min { N_GHOSTS }; + const real_t i1_ { COORD(i1) }; + const real_t i2_ { COORD(i2) }; const real_t inv_sqrt_detH_0pH { ONE / metric.sqrt_det_h({ i1_, i2_ + HALF }) }; diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 363ff3ad2..dbb47f42c 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -493,11 +493,11 @@ namespace kernel::bc { */ template struct AxisBoundaries_kernel { - ndfield_t Fld; - const std::size_t i_edge; - const bool setE, setB; + ndfield_t Fld; + const ncells_t i_edge; + const bool setE, setB; - AxisBoundaries_kernel(ndfield_t Fld, std::size_t i_edge, BCTags tags) + AxisBoundaries_kernel(ndfield_t Fld, ncells_t i_edge, BCTags tags) : Fld { Fld } , i_edge { i_edge } , setE { tags & BC::Ex1 or tags & BC::Ex2 or tags & BC::Ex3 } @@ -558,15 +558,15 @@ namespace kernel::bc { static_cast(M::Dim), "Invalid Orientation"); - ndfield_t Fld; - const I fset; - const M metric; - const std::size_t i_edge; + ndfield_t Fld; + const I fset; + const M metric; + const ncells_t i_edge; EnforcedBoundaries_kernel(ndfield_t& Fld, const I& fset, const M& metric, - std::size_t i_edge, + ncells_t i_edge, BCTags tags) : Fld { Fld } , fset { fset } diff --git a/src/kernels/injectors.hpp b/src/kernels/injectors.hpp index 9d3fd7d81..09bc7a180 100644 --- a/src/kernels/injectors.hpp +++ b/src/kernels/injectors.hpp @@ -47,7 +47,7 @@ namespace kernel { array_t weights_2; array_t tags_2; - std::size_t offset1, offset2; + npart_t offset1, offset2; const M metric; const array_t ni; const ED energy_dist; @@ -58,8 +58,8 @@ namespace kernel { spidx_t spidx2, Particles& species1, Particles& species2, - std::size_t offset1, - std::size_t offset2, + npart_t offset1, + npart_t offset2, const M& metric, const array_t& ni, const ED& energy_dist, @@ -201,20 +201,20 @@ namespace kernel { array_t in_phi; array_t in_wei; - array_t idx { "idx" }; - array_t i1s, i2s, i3s; - array_t dx1s, dx2s, dx3s; - array_t ux1s, ux2s, ux3s; - array_t phis; - array_t weights; - array_t tags; + array_t idx { "idx" }; + array_t i1s, i2s, i3s; + array_t dx1s, dx2s, dx3s; + array_t ux1s, ux2s, ux3s; + array_t phis; + array_t weights; + array_t tags; - const std::size_t offset; + const npart_t offset; M global_metric; - real_t x1_min, x1_max, x2_min, x2_max, x3_min, x3_max; - std::size_t i1_offset, i2_offset, i3_offset; + real_t x1_min, x1_max, x2_min, x2_max, x3_min, x3_max; + ncells_t i1_offset, i2_offset, i3_offset; GlobalInjector_kernel(Particles& species, const M& global_metric, @@ -269,18 +269,18 @@ namespace kernel { void copy_from_vector(const std::string& name, array_t& arr, const std::map>& data, - std::size_t n_inject) { + npart_t n_inject) { raise::ErrorIf(data.find(name) == data.end(), name + " not found in data", HERE); raise::ErrorIf(data.at(name).size() != n_inject, "Inconsistent data size", HERE); arr = array_t { name, n_inject }; auto arr_h = Kokkos::create_mirror_view(arr); - for (std::size_t i = 0; i < data.at(name).size(); ++i) { + for (auto i = 0u; i < data.at(name).size(); ++i) { arr_h(i) = data.at(name)[i]; } Kokkos::deep_copy(arr, arr_h); } - auto number_injected() const -> std::size_t { + auto number_injected() const -> npart_t { auto idx_h = Kokkos::create_mirror_view(idx); Kokkos::deep_copy(idx_h, idx); return idx_h(); @@ -298,7 +298,7 @@ namespace kernel { global_metric.template transform_xyz(x_Cd, u_Ph, u_XYZ); const auto i1 = static_cast( - static_cast(x_Cd[0]) - i1_offset); + static_cast(x_Cd[0]) - i1_offset); const auto dx1 = static_cast( x_Cd[0] - static_cast(i1 + i1_offset)); @@ -322,9 +322,8 @@ namespace kernel { vec_t u_Ph { in_ux1(p), in_ux2(p), in_ux3(p) }; coord_t x_Cd_ { ZERO }; - auto index { - offset + Kokkos::atomic_fetch_add(&idx(), static_cast(1)) - }; + auto index { offset + + Kokkos::atomic_fetch_add(&idx(), static_cast(1)) }; global_metric.template convert({ in_x1(p), in_x2(p) }, x_Cd); x_Cd_[0] = x_Cd[0]; @@ -340,11 +339,11 @@ namespace kernel { raise::KernelError(HERE, "Unknown simulation engine"); } const auto i1 = static_cast( - static_cast(x_Cd[0]) - i1_offset); + static_cast(x_Cd[0]) - i1_offset); const auto dx1 = static_cast( x_Cd[0] - static_cast(i1 + i1_offset)); const auto i2 = static_cast( - static_cast(x_Cd[1]) - i2_offset); + static_cast(x_Cd[1]) - i2_offset); const auto dx2 = static_cast( x_Cd[1] - static_cast(i2 + i2_offset)); @@ -385,15 +384,15 @@ namespace kernel { raise::KernelError(HERE, "Unknown simulation engine"); } const auto i1 = static_cast( - static_cast(x_Cd[0]) - i1_offset); + static_cast(x_Cd[0]) - i1_offset); const auto dx1 = static_cast( x_Cd[0] - static_cast(i1 + i1_offset)); const auto i2 = static_cast( - static_cast(x_Cd[1]) - i2_offset); + static_cast(x_Cd[1]) - i2_offset); const auto dx2 = static_cast( x_Cd[1] - static_cast(i2 + i2_offset)); const auto i3 = static_cast( - static_cast(x_Cd[2]) - i3_offset); + static_cast(x_Cd[2]) - i3_offset); const auto dx3 = static_cast( x_Cd[2] - static_cast(i3 + i3_offset)); @@ -440,9 +439,9 @@ namespace kernel { array_t weights_2; array_t tags_2; - array_t idx { "idx" }; + array_t idx { "idx" }; - std::size_t offset1, offset2; + npart_t offset1, offset2; M metric; const ED energy_dist; const SD spatial_dist; @@ -454,8 +453,8 @@ namespace kernel { spidx_t spidx2, Particles& species1, Particles& species2, - std::size_t offset1, - std::size_t offset2, + npart_t offset1, + npart_t offset2, const M& metric, const ED& energy_dist, const SD& spatial_dist, @@ -496,7 +495,7 @@ namespace kernel { , inv_V0 { inv_V0 } , random_pool { random_pool } {} - auto number_injected() const -> std::size_t { + auto number_injected() const -> npart_t { auto idx_h = Kokkos::create_mirror_view(idx); Kokkos::deep_copy(idx_h, idx); return idx_h(); @@ -508,7 +507,7 @@ namespace kernel { coord_t x_Cd { i1_ + HALF }; coord_t x_Ph { ZERO }; metric.template convert(x_Cd, x_Ph); - const auto ppc = static_cast(ppc0 * spatial_dist(x_Ph)); + const auto ppc = static_cast(ppc0 * spatial_dist(x_Ph)); if (ppc == 0) { return; } @@ -564,7 +563,7 @@ namespace kernel { x_Cd_[2] = ZERO; } metric.template convert(x_Cd, x_Ph); - const auto ppc = static_cast(ppc0 * spatial_dist(x_Ph)); + const auto ppc = static_cast(ppc0 * spatial_dist(x_Ph)); if (ppc == 0) { return; } @@ -631,7 +630,7 @@ namespace kernel { coord_t x_Cd { i1_ + HALF, i2_ + HALF, i3_ + HALF }; coord_t x_Ph { ZERO }; metric.template convert(x_Cd, x_Ph); - const auto ppc = static_cast(ppc0 * spatial_dist(x_Ph)); + const auto ppc = static_cast(ppc0 * spatial_dist(x_Ph)); if (ppc == 0) { return; } diff --git a/src/kernels/particle_moments.hpp b/src/kernels/particle_moments.hpp index 0621646ad..948e46e1e 100644 --- a/src/kernels/particle_moments.hpp +++ b/src/kernels/particle_moments.hpp @@ -86,7 +86,7 @@ namespace kernel { bool use_weights, const M& metric, const boundaries_t& boundaries, - std::size_t ni2, + ncells_t ni2, real_t inv_n0, unsigned short window) : c1 { (components.size() == 2) ? components[0] diff --git a/src/kernels/prtls_to_phys.hpp b/src/kernels/prtls_to_phys.hpp index f12eefc95..4dd7d88b0 100644 --- a/src/kernels/prtls_to_phys.hpp +++ b/src/kernels/prtls_to_phys.hpp @@ -31,7 +31,7 @@ namespace kernel { static constexpr Dimension D = M::Dim; protected: - const std::size_t stride; + const npart_t stride; array_t buff_x1; array_t buff_x2; array_t buff_x3; @@ -47,7 +47,7 @@ namespace kernel { const M metric; public: - PrtlToPhys_kernel(std::size_t stride, + PrtlToPhys_kernel(npart_t stride, array_t& buff_x1, array_t& buff_x2, array_t& buff_x3, diff --git a/src/metrics/kerr_schild.h b/src/metrics/kerr_schild.h index 5a60def53..13294dd7d 100644 --- a/src/metrics/kerr_schild.h +++ b/src/metrics/kerr_schild.h @@ -72,7 +72,7 @@ namespace metric { using MetricBase::nx3; using MetricBase::set_dxMin; - KerrSchild(std::vector res, + KerrSchild(std::vector res, boundaries_t ext, const std::map& params) : MetricBase { res, ext } diff --git a/src/metrics/kerr_schild_0.h b/src/metrics/kerr_schild_0.h index 70689f4f0..fdd2fd847 100644 --- a/src/metrics/kerr_schild_0.h +++ b/src/metrics/kerr_schild_0.h @@ -53,8 +53,8 @@ namespace metric { using MetricBase::nx3; using MetricBase::set_dxMin; - KerrSchild0(std::vector res, - boundaries_t ext, + KerrSchild0(std::vector res, + boundaries_t ext, const std::map& = {}) : MetricBase { res, ext } , dr { (x1_max - x1_min) / nx1 } diff --git a/src/metrics/metric_base.h b/src/metrics/metric_base.h index 321d39bbd..66cc28918 100644 --- a/src/metrics/metric_base.h +++ b/src/metrics/metric_base.h @@ -61,7 +61,7 @@ namespace metric { static constexpr bool is_metric { true }; static constexpr Dimension Dim { D }; - MetricBase(std::vector res, boundaries_t ext) + MetricBase(std::vector res, boundaries_t ext) : nx1 { res.size() > 0 ? (real_t)(res[0]) : ONE } , nx2 { res.size() > 1 ? (real_t)(res[1]) : ONE } , nx3 { res.size() > 2 ? (real_t)(res[2]) : ONE } diff --git a/src/metrics/minkowski.h b/src/metrics/minkowski.h index c22ac1ad9..367689e0f 100644 --- a/src/metrics/minkowski.h +++ b/src/metrics/minkowski.h @@ -47,8 +47,8 @@ namespace metric { using MetricBase::nx3; using MetricBase::set_dxMin; - Minkowski(std::vector res, - boundaries_t ext, + Minkowski(std::vector res, + boundaries_t ext, const std::map& = {}) : MetricBase { res, ext } , dx { (x1_max - x1_min) / nx1 } @@ -240,8 +240,7 @@ namespace metric { * @note tetrad/cart <-> cntrv <-> cov */ template - Inline auto transform(const coord_t& xi, const real_t& v_in) const - -> real_t { + Inline auto transform(const coord_t& xi, const real_t& v_in) const -> real_t { static_assert(i > 0 && i <= 3, "Invalid index i"); static_assert(in != out, "Invalid vector transformation"); if constexpr (i > static_cast(D)) { diff --git a/src/metrics/qkerr_schild.h b/src/metrics/qkerr_schild.h index d531b8b3b..d64fe05da 100644 --- a/src/metrics/qkerr_schild.h +++ b/src/metrics/qkerr_schild.h @@ -72,7 +72,7 @@ namespace metric { using MetricBase::nx3; using MetricBase::set_dxMin; - QKerrSchild(std::vector res, + QKerrSchild(std::vector res, boundaries_t ext, const std::map& params) : MetricBase { res, ext } diff --git a/src/metrics/qspherical.h b/src/metrics/qspherical.h index 8062f7589..4df5db866 100644 --- a/src/metrics/qspherical.h +++ b/src/metrics/qspherical.h @@ -55,7 +55,7 @@ namespace metric { using MetricBase::nx3; using MetricBase::set_dxMin; - QSpherical(std::vector res, + QSpherical(std::vector res, boundaries_t ext, const std::map& params) : MetricBase { res, ext } @@ -284,8 +284,7 @@ namespace metric { * @note tetrad/sph <-> cntrv <-> cov */ template - Inline auto transform(const coord_t& xi, const real_t& v_in) const - -> real_t { + Inline auto transform(const coord_t& xi, const real_t& v_in) const -> real_t { static_assert(i > 0 && i <= 3, "Invalid index i"); static_assert(in != out, "Invalid vector transformation"); if constexpr ((in == Idx::T && out == Idx::Sph) || diff --git a/src/metrics/spherical.h b/src/metrics/spherical.h index f4bbe2eea..54e12367d 100644 --- a/src/metrics/spherical.h +++ b/src/metrics/spherical.h @@ -50,8 +50,8 @@ namespace metric { using MetricBase::nx3; using MetricBase::set_dxMin; - Spherical(std::vector res, - boundaries_t ext, + Spherical(std::vector res, + boundaries_t ext, const std::map& = {}) : MetricBase { res, ext } , dr((x1_max - x1_min) / nx1) @@ -252,8 +252,7 @@ namespace metric { * @note tetrad/sph <-> cntrv <-> cov */ template - Inline auto transform(const coord_t& xi, const real_t& v_in) const - -> real_t { + Inline auto transform(const coord_t& xi, const real_t& v_in) const -> real_t { static_assert(i > 0 && i <= 3, "Invalid index i"); static_assert(in != out, "Invalid vector transformation"); if constexpr ((in == Idx::T && out == Idx::Sph) || diff --git a/src/output/fields.h b/src/output/fields.h index a520a246d..811c5adc2 100644 --- a/src/output/fields.h +++ b/src/output/fields.h @@ -112,7 +112,7 @@ namespace out { } [[nodiscard]] - inline auto name(const std::size_t& ci) const -> std::string { + inline auto name(std::size_t ci) const -> std::string { raise::ErrorIf( comp.size() == 0, "OutputField::name(ci) called but no components were available", diff --git a/src/output/tests/writer-nompi.cpp b/src/output/tests/writer-nompi.cpp index 8fb2ac026..b063539ca 100644 --- a/src/output/tests/writer-nompi.cpp +++ b/src/output/tests/writer-nompi.cpp @@ -103,22 +103,21 @@ auto main(int argc, char* argv[]) -> int { const auto layoutRight = io.InquireAttribute("LayoutRight").Data()[0] == 1; - raise::ErrorIf(io.InquireAttribute("NGhosts").Data()[0] != 0, + raise::ErrorIf(io.InquireAttribute("NGhosts").Data()[0] != 0, "NGhosts is not correct", HERE); raise::ErrorIf(io.InquireAttribute("Dimension").Data()[0] != 3, "Dimension is not correct", HERE); - for (std::size_t step = 0; reader.BeginStep() == adios2::StepStatus::OK; - ++step) { - std::size_t step_read; - long double time_read; + for (auto step = 0u; reader.BeginStep() == adios2::StepStatus::OK; ++step) { + timestep_t step_read; + simtime_t time_read; - reader.Get(io.InquireVariable("Step"), + reader.Get(io.InquireVariable("Step"), &step_read, adios2::Mode::Sync); - reader.Get(io.InquireVariable("Time"), + reader.Get(io.InquireVariable("Time"), &time_read, adios2::Mode::Sync); raise::ErrorIf(step_read != (step + 1) * 10, "Step is not correct", HERE); @@ -136,10 +135,10 @@ auto main(int argc, char* argv[]) -> int { fmt::format("%s is not 3D", name.c_str()), HERE); - auto dims = fieldVar.Shape(); - std::size_t nx1_r = dims[0]; - std::size_t nx2_r = dims[1]; - std::size_t nx3_r = dims[2]; + auto dims = fieldVar.Shape(); + ncells_t nx1_r = dims[0]; + ncells_t nx2_r = dims[1]; + ncells_t nx3_r = dims[2]; if (!layoutRight) { std::swap(nx1_r, nx3_r); } diff --git a/src/output/utils/attr_writer.h b/src/output/utils/attr_writer.h index 47f269d55..c8b21e4c2 100644 --- a/src/output/utils/attr_writer.h +++ b/src/output/utils/attr_writer.h @@ -40,7 +40,10 @@ namespace out { {typeid(int), defineAttribute}, {typeid(short), defineAttribute}, {typeid(unsigned int), defineAttribute}, - {typeid(std::size_t), defineAttribute}, + {typeid(long int), defineAttribute}, + {typeid(unsigned long int), defineAttribute}, + {typeid(long long int), defineAttribute}, + {typeid(unsigned long long int), defineAttribute}, {typeid(unsigned short), defineAttribute}, {typeid(float), defineAttribute}, {typeid(double), defineAttribute}, @@ -49,7 +52,10 @@ namespace out { {typeid(std::vector), defineAttribute>}, {typeid(std::vector), defineAttribute>}, {typeid(std::vector), defineAttribute>}, - {typeid(std::vector), defineAttribute>}, + {typeid(std::vector), defineAttribute>}, + {typeid(std::vector), defineAttribute>}, + {typeid(std::vector), defineAttribute>}, + {typeid(std::vector), defineAttribute>}, {typeid(std::vector), defineAttribute>}, {typeid(std::vector), defineAttribute>}, {typeid(std::vector), defineAttribute>}, diff --git a/src/output/writer.cpp b/src/output/writer.cpp index 95965c864..4350e8442 100644 --- a/src/output/writer.cpp +++ b/src/output/writer.cpp @@ -37,20 +37,20 @@ namespace out { m_io = p_adios->DeclareIO("Entity::Output"); m_io.SetEngine(engine); - m_io.DefineVariable("Step"); - m_io.DefineVariable("Time"); + m_io.DefineVariable("Step"); + m_io.DefineVariable("Time"); m_fname = title; } void Writer::addTracker(const std::string& type, - std::size_t interval, - long double interval_time) { + timestep_t interval, + simtime_t interval_time) { m_trackers.insert({ type, tools::Tracker(type, interval, interval_time) }); } auto Writer::shouldWrite(const std::string& type, - std::size_t step, - long double time) -> bool { + timestep_t step, + simtime_t time) -> bool { if (m_trackers.find(type) != m_trackers.end()) { return m_trackers.at(type).shouldWrite(step, time); } else { @@ -63,9 +63,9 @@ namespace out { m_mode = mode; } - void Writer::defineMeshLayout(const std::vector& glob_shape, - const std::vector& loc_corner, - const std::vector& loc_shape, + void Writer::defineMeshLayout(const std::vector& glob_shape, + const std::vector& loc_corner, + const std::vector& loc_shape, const std::vector& dwn, bool incl_ghosts, Coord coords) { @@ -76,7 +76,7 @@ namespace out { m_flds_l_corner = loc_corner; m_flds_l_shape = loc_shape; - for (std::size_t i { 0 }; i < glob_shape.size(); ++i) { + for (auto i { 0u }; i < glob_shape.size(); ++i) { raise::ErrorIf(dwn[i] != 1 && incl_ghosts, "Downsampling with ghosts not supported", HERE); @@ -86,18 +86,17 @@ namespace out { const double l = loc_corner[i]; const double n = loc_shape[i]; const double f = math::ceil(l / d) * d - l; - m_flds_g_shape_dwn.push_back(static_cast(math::ceil(g / d))); - m_flds_l_corner_dwn.push_back(static_cast(math::ceil(l / d))); - m_flds_l_first.push_back(static_cast(f)); - m_flds_l_shape_dwn.push_back( - static_cast(math::ceil((n - f) / d))); + m_flds_g_shape_dwn.push_back(static_cast(math::ceil(g / d))); + m_flds_l_corner_dwn.push_back(static_cast(math::ceil(l / d))); + m_flds_l_first.push_back(static_cast(f)); + m_flds_l_shape_dwn.push_back(static_cast(math::ceil((n - f) / d))); } m_io.DefineAttribute("NGhosts", incl_ghosts ? N_GHOSTS : 0); m_io.DefineAttribute("Dimension", m_flds_g_shape.size()); m_io.DefineAttribute("Coordinates", std::string(coords.to_string())); - for (std::size_t i { 0 }; i < m_flds_g_shape.size(); ++i) { + for (auto i { 0u }; i < m_flds_g_shape.size(); ++i) { // cell-centers adios2::Dims g_shape = { m_flds_g_shape_dwn[i] }; adios2::Dims l_corner = { m_flds_l_corner_dwn[i] }; @@ -151,7 +150,7 @@ namespace out { adios2::ConstantDims); } else { // vector or tensor - for (std::size_t i { 0 }; i < fld.comp.size(); ++i) { + for (auto i { 0u }; i < fld.comp.size(); ++i) { m_io.DefineVariable(fld.name(i), m_flds_g_shape_dwn, m_flds_l_corner_dwn, @@ -210,7 +209,7 @@ namespace out { const ndfield_t& field, std::size_t comp, std::vector dwn, - std::vector first_cell, + std::vector first_cell, bool ghosts) { // when dwn != 1 in any direction, it is assumed that ghosts == false auto var = io.InquireVariable(varname); @@ -230,7 +229,7 @@ namespace out { const double nx1_full = field.extent(0) - 2 * N_GHOSTS; const auto first_cell1 = first_cell[0]; - const auto nx1_dwn = static_cast( + const auto nx1_dwn = static_cast( math::ceil((nx1_full - first_cell1_d) / dwn1)); output_field = array_t { "output_field", nx1_dwn }; @@ -260,9 +259,9 @@ namespace out { const auto first_cell1 = first_cell[0]; const auto first_cell2 = first_cell[1]; - const auto nx1_dwn = static_cast( + const auto nx1_dwn = static_cast( math::ceil((nx1_full - first_cell1_d) / dwn1)); - const auto nx2_dwn = static_cast( + const auto nx2_dwn = static_cast( math::ceil((nx2_full - first_cell2_d) / dwn2)); output_field = array_t { "output_field", nx1_dwn, nx2_dwn }; Kokkos::parallel_for( @@ -299,11 +298,11 @@ namespace out { const auto first_cell2 = first_cell[1]; const auto first_cell3 = first_cell[2]; - const auto nx1_dwn = static_cast( + const auto nx1_dwn = static_cast( math::ceil((nx1_full - first_cell1_d) / dwn1)); - const auto nx2_dwn = static_cast( + const auto nx2_dwn = static_cast( math::ceil((nx2_full - first_cell2_d) / dwn2)); - const auto nx3_dwn = static_cast( + const auto nx3_dwn = static_cast( math::ceil((nx3_full - first_cell3_d) / dwn3)); output_field = array_t { "output_field", nx1_dwn, nx2_dwn, nx3_dwn }; @@ -333,7 +332,7 @@ namespace out { raise::ErrorIf(names.size() != addresses.size(), "# of names != # of addresses ", HERE); - for (std::size_t i { 0 }; i < addresses.size(); ++i) { + for (auto i { 0u }; i < addresses.size(); ++i) { WriteField(m_io, m_writer, names[i], @@ -346,8 +345,8 @@ namespace out { } void Writer::writeParticleQuantity(const array_t& array, - std::size_t glob_total, - std::size_t loc_offset, + npart_t glob_total, + npart_t loc_offset, const std::string& varname) { auto var = m_io.InquireVariable(varname); var.SetShape({ glob_total }); @@ -416,8 +415,8 @@ namespace out { } void Writer::beginWriting(WriteModeTags write_mode, - std::size_t tstep, - long double time) { + timestep_t tstep, + simtime_t time) { raise::ErrorIf(write_mode == WriteMode::None, "None is not a valid mode", HERE); raise::ErrorIf(p_adios == nullptr, "ADIOS pointer is null", HERE); if (m_active_mode != WriteMode::None) { @@ -468,8 +467,8 @@ namespace out { raise::Fatal(e.what(), HERE); } m_writer.BeginStep(); - m_writer.Put(m_io.InquireVariable("Step"), &tstep); - m_writer.Put(m_io.InquireVariable("Time"), &time); + m_writer.Put(m_io.InquireVariable("Step"), &tstep); + m_writer.Put(m_io.InquireVariable("Time"), &time); } void Writer::endWriting(WriteModeTags write_mode) { @@ -511,7 +510,7 @@ namespace out { const ndfield_t&, std::size_t, std::vector, - std::vector, + std::vector, bool); template void WriteField(adios2::IO&, adios2::Engine&, @@ -519,7 +518,7 @@ namespace out { const ndfield_t&, std::size_t, std::vector, - std::vector, + std::vector, bool); template void WriteField(adios2::IO&, adios2::Engine&, @@ -527,7 +526,7 @@ namespace out { const ndfield_t&, std::size_t, std::vector, - std::vector, + std::vector, bool); template void WriteField(adios2::IO&, adios2::Engine&, @@ -535,7 +534,7 @@ namespace out { const ndfield_t&, std::size_t, std::vector, - std::vector, + std::vector, bool); template void WriteField(adios2::IO&, adios2::Engine&, @@ -543,7 +542,7 @@ namespace out { const ndfield_t&, std::size_t, std::vector, - std::vector, + std::vector, bool); template void WriteField(adios2::IO&, adios2::Engine&, @@ -551,7 +550,7 @@ namespace out { const ndfield_t&, std::size_t, std::vector, - std::vector, + std::vector, bool); } // namespace out diff --git a/src/output/writer.h b/src/output/writer.h index a8abf4b12..5484aa6d7 100644 --- a/src/output/writer.h +++ b/src/output/writer.h @@ -39,16 +39,16 @@ namespace out { bool m_separate_files; // global shape of the fields array to output - std::vector m_flds_g_shape; + std::vector m_flds_g_shape; // local corner of the fields array to output - std::vector m_flds_l_corner; + std::vector m_flds_l_corner; // local shape of the fields array to output - std::vector m_flds_l_shape; + std::vector m_flds_l_shape; // downsampling factors for each dimension std::vector m_dwn; // starting cell in each dimension (not including ghosts) - std::vector m_flds_l_first; + std::vector m_flds_l_first; // same but downsampled adios2::Dims m_flds_g_shape_dwn; @@ -78,14 +78,14 @@ namespace out { void setMode(adios2::Mode); - void addTracker(const std::string&, std::size_t, long double); - auto shouldWrite(const std::string&, std::size_t, long double) -> bool; + void addTracker(const std::string&, timestep_t, simtime_t); + auto shouldWrite(const std::string&, timestep_t, simtime_t) -> bool; void writeAttrs(const prm::Parameters&); - void defineMeshLayout(const std::vector&, - const std::vector&, - const std::vector&, + void defineMeshLayout(const std::vector&, + const std::vector&, + const std::vector&, const std::vector&, bool, Coord); @@ -102,13 +102,13 @@ namespace out { const std::vector&); void writeParticleQuantity(const array_t&, - std::size_t, - std::size_t, + npart_t, + npart_t, const std::string&); void writeSpectrum(const array_t&, const std::string&); void writeSpectrumBins(const array_t&, const std::string&); - void beginWriting(WriteModeTags, std::size_t, long double); + void beginWriting(WriteModeTags, timestep_t, simtime_t); void endWriting(WriteModeTags); /* getters -------------------------------------------------------------- */ From 85811c9bdd670dd44d8c50123240881dc2407293 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Thu, 13 Mar 2025 10:09:36 -0400 Subject: [PATCH 070/183] Minor fixes of npart_t --- src/checkpoint/writer.cpp | 4 ++-- src/framework/domain/metadomain.h | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/checkpoint/writer.cpp b/src/checkpoint/writer.cpp index c5f7e5181..a70d8de09 100644 --- a/src/checkpoint/writer.cpp +++ b/src/checkpoint/writer.cpp @@ -273,10 +273,10 @@ namespace checkpoint { std::size_t, std::size_t, double); - template void Writer::savePerDomainVariable(const std::string&, + template void Writer::savePerDomainVariable(const std::string&, std::size_t, std::size_t, - std::size_t); + npart_t); template void Writer::saveField(const std::string&, const ndfield_t&); diff --git a/src/framework/domain/metadomain.h b/src/framework/domain/metadomain.h index e7e1340e2..80f546664 100644 --- a/src/framework/domain/metadomain.h +++ b/src/framework/domain/metadomain.h @@ -177,8 +177,8 @@ namespace ntt { } [[nodiscard]] - auto l_npart_perspec() const -> std::vector { - std::vector npart(g_species_params.size(), 0); + auto l_npart_perspec() const -> std::vector { + std::vector npart(g_species_params.size(), 0); for (const auto& ldidx : l_subdomain_indices()) { for (std::size_t i = 0; i < g_species_params.size(); ++i) { npart[i] += g_subdomains[ldidx].species[i].npart(); @@ -188,8 +188,8 @@ namespace ntt { } [[nodiscard]] - auto l_maxnpart_perspec() const -> std::vector { - std::vector maxnpart(g_species_params.size(), 0); + auto l_maxnpart_perspec() const -> std::vector { + std::vector maxnpart(g_species_params.size(), 0); for (const auto& ldidx : l_subdomain_indices()) { for (std::size_t i = 0; i < g_species_params.size(); ++i) { maxnpart[i] += g_subdomains[ldidx].species[i].maxnpart(); From 777d1fbff3cf66a7b0e5e49260cb1acb954d5761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Thu, 13 Mar 2025 10:00:56 -0500 Subject: [PATCH 071/183] cleanup of pgen --- setups/srpic/shock/pgen.hpp | 46 ------------------------------------- 1 file changed, 46 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 59c5590c9..b8f169521 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -106,52 +106,6 @@ namespace user { } } - - auto PerfectConductorFieldsConst(const bc_in&, const em& comp) const -> real_t{ - - // electric field components - if (comp == em::ex1) { - return ONE; - } else if (comp == em::ex2) { - return -ONE; - } else if (comp == em::ex3) { - return -ONE; } - // magentic field components - else if (comp == em::bx1) { - return -ONE; - } else if (comp == em::bx2) { - return ONE; - } else if (comp == em::bx3) { - return ONE;} - // should never be the case - else - { - return ZERO; - } - } - - // auto PerfectConductorCurrentsConst(const bc_in &, const cur &comp) const - // -> std::pair - // { - // // ToDo - // if (comp == cur::jx1) - // { - // return ZERO; - // } - // else if (comp == cur::jx2) - // { - // return ZERO; - // } - // else if (comp == cur::jx3) - // { - // return ZERO; - // } - // else - // { - // return ZERO; - // } - // } - auto MatchFields(real_t time) const -> InitFields { return init_flds; } From ef5c85409d3422f1cb71337166c719f8b5e76ed2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Thu, 13 Mar 2025 11:03:56 -0500 Subject: [PATCH 072/183] update example parameter file --- setups/srpic/shock/shock.toml | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/setups/srpic/shock/shock.toml b/setups/srpic/shock/shock.toml index 4ed3a2b9e..4e03721f7 100644 --- a/setups/srpic/shock/shock.toml +++ b/setups/srpic/shock/shock.toml @@ -1,22 +1,22 @@ [simulation] name = "shock" engine = "srpic" - runtime = 50.0 + runtime = 30000.0 [grid] - resolution = [2048, 128] - extent = [[0.0, 10.0], [-0.3125, 0.3125]] + resolution = [16384, 128] + extent = [[0.0, 8000.0], [-31.25, 31.25]] [grid.metric] metric = "minkowski" [grid.boundaries] - fields = [["ABSORB", "FIXED"], ["PERIODIC"]] + fields = [["CONDUCTOR", "MATCH"], ["PERIODIC"]] particles = [["REFLECT", "ABSORB"], ["PERIODIC"]] - + [scales] - larmor0 = 1e-2 - skindepth0 = 1e-2 + larmor0 = 100.0 + skindepth0 = 1.0 [algorithms] current_filters = 8 @@ -25,7 +25,7 @@ CFL = 0.5 [particles] - ppc0 = 16.0 + ppc0 = 8.0 [[particles.species]] label = "e-" @@ -34,21 +34,29 @@ maxnpart = 1e8 [[particles.species]] - label = "e+" + label = "p+" mass = 1.0 charge = 1.0 maxnpart = 1e8 [setup] drift_ux = 0.1 - temperature = 1e-3 + temperature = 1e-4 Bmag = 1.0 Btheta = 0.0 Bphi = 0.0 [output] - interval_time = 0.1 + interval_time = 500.0 format = "hdf5" - + [output.fields] - quantities = ["N_1", "N_2", "E", "B", "T0i_1", "T0i_2", "J"] + quantities = ["N_1", "N_2", "B", "E"] + + [output.particles] + enable = true + stride = 10 + + [output.spectra] + enable = false + From 66cb136a4f3fa46cca535befce004d1102b4a17d Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Thu, 13 Mar 2025 15:35:15 -0400 Subject: [PATCH 073/183] Minor fixes of npart_t --- src/kernels/comm.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/kernels/comm.hpp b/src/kernels/comm.hpp index 7873870b1..6a47e56b4 100644 --- a/src/kernels/comm.hpp +++ b/src/kernels/comm.hpp @@ -113,14 +113,14 @@ namespace kernel::comm { array_t send_buff_pld; const unsigned short NINTS, NREALS, NPRTLDX, NPLDS; - const std::size_t idx_offset; + const npart_t idx_offset; const array_t i1, i1_prev, i2, i2_prev, i3, i3_prev; const array_t dx1, dx1_prev, dx2, dx2_prev, dx3, dx3_prev; const array_t ux1, ux2, ux3, weight, phi; const array_t pld; array_t tag; - const array_t outgoing_indices; + const array_t outgoing_indices; public: PopulatePrtlSendBuffer_kernel(array_t& send_buff_int, @@ -131,7 +131,7 @@ namespace kernel::comm { unsigned short NREALS, unsigned short NPRTLDX, unsigned short NPLDS, - std::size_t idx_offset, + npart_t idx_offset, const array_t& i1, const array_t& i1_prev, const array_t& dx1, @@ -151,7 +151,7 @@ namespace kernel::comm { const array_t& phi, const array_t& pld, array_t& tag, - const array_t& outgoing_indices) + const array_t& outgoing_indices) : send_buff_int { send_buff_int } , send_buff_real { send_buff_real } , send_buff_prtldx { send_buff_prtldx } From 38e76bd7763d00743f547a07b5ed2682dc70ea22 Mon Sep 17 00:00:00 2001 From: haykh Date: Thu, 13 Mar 2025 19:25:55 -0400 Subject: [PATCH 074/183] minor bugfix dx_min comparison + nonexistent species --- src/framework/domain/metadomain.cpp | 4 +++- src/framework/domain/output.cpp | 5 +++++ src/framework/tests/comm_mpi.cpp | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/framework/domain/metadomain.cpp b/src/framework/domain/metadomain.cpp index ed4373df2..57b2ece17 100644 --- a/src/framework/domain/metadomain.cpp +++ b/src/framework/domain/metadomain.cpp @@ -369,6 +369,8 @@ namespace ntt { template void Metadomain::metricCompatibilityCheck() const { + const auto epsilon = std::numeric_limits::epsilon() * + static_cast(100.0); const auto dx_min = g_mesh.metric.dxMin(); auto dx_min_from_domains = std::numeric_limits::infinity(); for (unsigned int idx { 0 }; idx < g_ndomains; ++idx) { @@ -377,7 +379,7 @@ namespace ntt { dx_min_from_domains = std::min(dx_min_from_domains, current_dx_min); } raise::ErrorIf( - not cmp::AlmostEqual(dx_min, dx_min_from_domains), + not cmp::AlmostEqual(dx_min / dx_min_from_domains, ONE, epsilon), "dx_min is not the same across all domains: " + std::to_string(dx_min) + " " + std::to_string(dx_min_from_domains), HERE); diff --git a/src/framework/domain/output.cpp b/src/framework/domain/output.cpp index 6961d2826..3dc95c18a 100644 --- a/src/framework/domain/output.cpp +++ b/src/framework/domain/output.cpp @@ -125,6 +125,11 @@ namespace ntt { } } } + for (const auto& sp : specs) { + raise::ErrorIf((sp > prtl_species.size()) or (sp == 0), + "Invalid species index " + std::to_string(sp), + HERE); + } auto scatter_buff = Kokkos::Experimental::create_scatter_view(buffer); // some parameters diff --git a/src/framework/tests/comm_mpi.cpp b/src/framework/tests/comm_mpi.cpp index 2f65defd6..157a9692b 100644 --- a/src/framework/tests/comm_mpi.cpp +++ b/src/framework/tests/comm_mpi.cpp @@ -58,4 +58,4 @@ auto main(int argc, char* argv[]) -> int { Kokkos::finalize(); return 0; -} \ No newline at end of file +} From 2d4e2bfda13a700283e5b8712c894c3fa158938f Mon Sep 17 00:00:00 2001 From: haykh Date: Thu, 13 Mar 2025 19:26:07 -0400 Subject: [PATCH 075/183] bugfix in mpihdf5 test --- src/output/tests/writer-mpi.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/output/tests/writer-mpi.cpp b/src/output/tests/writer-mpi.cpp index 6ab16305f..f6d3ee88a 100644 --- a/src/output/tests/writer-mpi.cpp +++ b/src/output/tests/writer-mpi.cpp @@ -60,7 +60,7 @@ auto main(int argc, char* argv[]) -> int { { // write auto writer = out::Writer(); - writer.init(&adios, "hdf5", "test"); + writer.init(&adios, "hdf5", "test", false); writer.defineMeshLayout({ static_cast(mpi_size) * nx1 }, { static_cast(mpi_rank) * nx1 }, { nx1 }, @@ -74,13 +74,13 @@ auto main(int argc, char* argv[]) -> int { field_names.push_back(writer.fieldWriters()[0].name(i)); addresses.push_back(i); } - writer.beginWriting(0, 0.0); + writer.beginWriting(WriteMode::Fields, 0, 0.0); writer.writeField(field_names, field, addresses); - writer.endWriting(); + writer.endWriting(WriteMode::Fields); - writer.beginWriting(1, 0.1); + writer.beginWriting(WriteMode::Fields, 1, 0.1); writer.writeField(field_names, field, addresses); - writer.endWriting(); + writer.endWriting(WriteMode::Fields); adios.ExitComputationBlock(); } From 84e047c0f741d3886cc7fcdf94c1addc36d6e0aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Wed, 19 Mar 2025 15:40:05 -0500 Subject: [PATCH 076/183] revert to cleaner injector for shocktest --- setups/srpic/shocktest/pgen.hpp | 152 +++++++------------------------- 1 file changed, 32 insertions(+), 120 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index ff54923d1..a7bcf5c3d 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -93,16 +93,16 @@ namespace user { : arch::ProblemGenerator { p } , drift_ux { p.template get("setup.drift_ux") } , temperature { p.template get("setup.temperature") } - // , x1arr_e { p.template get>("setup.x_e") } - // , x2arr_e { p.template get>("setup.y_e") } - // , ux1arr_e { p.template get>("setup.ux_e") } - // , ux2arr_e { p.template get>("setup.uy_e") } - // , ux3arr_e { p.template get>("setup.uz_e") } - // , x1arr_i { p.template get>("setup.x_i") } - // , x2arr_i { p.template get>("setup.y_i") } - // , ux1arr_i { p.template get>("setup.ux_i") } - // , ux2arr_i { p.template get>("setup.uy_i") } - // , ux3arr_i { p.template get>("setup.uz_i") } + , x1arr_e { p.template get>("setup.x_e") } + , x2arr_e { p.template get>("setup.y_e") } + , ux1arr_e { p.template get>("setup.ux_e") } + , ux2arr_e { p.template get>("setup.uy_e") } + , ux3arr_e { p.template get>("setup.uz_e") } + , x1arr_i { p.template get>("setup.x_i") } + , x2arr_i { p.template get>("setup.y_i") } + , ux1arr_i { p.template get>("setup.ux_i") } + , ux2arr_i { p.template get>("setup.uy_i") } + , ux3arr_i { p.template get>("setup.uz_i") } , Btheta { p.template get("setup.Btheta", ZERO) } , Bmag { p.template get("setup.Bmag", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } @@ -126,116 +126,28 @@ namespace user { return init_flds; } - // inline void InitPrtls(Domain& domain) { - // arch::InjectGlobally(*metadomain, - // domain, - // 1, - // { - // { "x1", x1arr_e }, - // { "x2", x2arr_e }, - // { "ux1", ux1arr_e }, - // { "ux2", ux1arr_e }, - // { "ux3", ux3arr_e } - // }); - // arch::InjectGlobally(*metadomain, - // domain, - // 2, - // { - // { "x1", x1arr_i }, - // { "x2", x2arr_i }, - // { "ux1", ux1arr_i }, - // { "ux2", ux1arr_i }, - // { "ux3", ux3arr_i } - // }); - // } - - inline void InitPrtls(Domain& domain) { - - // auto& species_e = domain.species[0]; - // auto& species_p = domain.species[1]; - auto& species_e = domain.species[0]; - auto& species_p = domain.species[1]; - - // array_t elec_ind("elec_ind"); - // array_t pos_ind("pos_ind"); - array_t elec_ind("elec_ind"); - array_t pos_ind("pos_ind"); - - auto offset_e = species_e.npart(); - auto offset_p = species_p.npart(); - - auto ux1_e = species_e.ux1; - auto ux2_e = species_e.ux2; - auto ux3_e = species_e.ux3; - auto i1_e = species_e.i1; - auto i2_e = species_e.i2; - auto dx1_e = species_e.dx1; - auto dx2_e = species_e.dx2; - auto phi_e = species_e.phi; - auto weight_e = species_e.weight; - auto tag_e = species_e.tag; - - auto ux1_p = species_p.ux1; - auto ux2_p = species_p.ux2; - auto ux3_p = species_p.ux3; - auto i1_p = species_p.i1; - auto i2_p = species_p.i2; - auto dx1_p = species_p.dx1; - auto dx2_p = species_p.dx2; - auto phi_p = species_p.phi; - auto weight_p = species_p.weight; - auto tag_p = species_p.tag; - - int nseed = 1; - - Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { - - // ToDo: fix this - auto i1_ = math::floor(10); - auto i2_ = math::floor(64); - auto dx1_ = HALF; - auto dx2_ = HALF; - - - auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); - auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); - - i1_e(elec_p + offset_e) = i1_; - dx1_e(elec_p + offset_e) = dx1_; - i2_e(elec_p + offset_e) = i2_; - dx2_e(elec_p + offset_e) = 2*dx2_; - // ux1_e(elec_p + offset_e) = -0.5; - // ux2_e(elec_p + offset_e) = 0.5; - ux1_e(elec_p + offset_e) = -drift_ux; - ux2_e(elec_p + offset_e) = ZERO; - ux3_e(elec_p + offset_e) = ZERO; - weight_e(elec_p + offset_e) = ONE; - tag_e(elec_p + offset_e) = ParticleTag::alive; - - i1_p(pos_p + offset_p) = i1_; - dx1_p(pos_p + offset_p) = dx1_; - i2_p(pos_p + offset_p) = i2_; - dx2_p(pos_p + offset_p) = -2*dx2_; - // ux1_p(pos_p + offset_p) = 0.5; - // ux2_p(pos_p + offset_p) = -0.5; - ux1_p(pos_p + offset_p) = -drift_ux; - ux2_p(pos_p + offset_p) = ZERO; - ux3_p(pos_p + offset_p) = ZERO; - weight_p(pos_p + offset_p) = ONE; - tag_p(pos_p + offset_p) = ParticleTag::alive; - - - }); - - - auto elec_ind_h = Kokkos::create_mirror(elec_ind); - Kokkos::deep_copy(elec_ind_h, elec_ind); - species_e.set_npart(offset_e + elec_ind_h()); - - auto pos_ind_h = Kokkos::create_mirror(pos_ind); - Kokkos::deep_copy(pos_ind_h, pos_ind); - species_p.set_npart(offset_p + pos_ind_h()); - } + inline void InitPrtls(Domain& domain) { + arch::InjectGlobally(*metadomain, + domain, + 1, + { + { "x1", x1arr_e }, + { "x2", x2arr_e }, + { "ux1", ux1arr_e }, + { "ux2", ux1arr_e }, + { "ux3", ux3arr_e } + }); + arch::InjectGlobally(*metadomain, + domain, + 2, + { + { "x1", x1arr_i }, + { "x2", x2arr_i }, + { "ux1", ux1arr_i }, + { "ux2", ux1arr_i }, + { "ux3", ux3arr_i } + }); + } }; } // namespace user From 69d4a0d180f3ae3b18600981e87a49b66929c495 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Thu, 20 Mar 2025 17:06:25 -0500 Subject: [PATCH 077/183] implemented new `arch::Piston` spatial distribution to set up shock partially filled with plasma --- setups/srpic/shock/pgen.hpp | 59 ++++++++++++++++++++++------------- setups/srpic/shock/shock.toml | 11 ++++--- src/archetypes/spatial_dist.h | 30 ++++++++++++++++++ 3 files changed, 74 insertions(+), 26 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index b8f169521..9bf51b866 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -79,7 +79,7 @@ namespace user { using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; - const real_t drift_ux, temperature; + const real_t drift_ux, temperature, filling_fraction; const real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -91,40 +91,57 @@ namespace user { , Bmag { p.template get("setup.Bmag", ZERO) } , Btheta { p.template get("setup.Btheta", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } - , init_flds { Bmag, Btheta, Bphi, drift_ux } {} + , init_flds { Bmag, Btheta, Bphi, drift_ux } + , filling_fraction { params.template get("setup.filling_fraction", 1.0) }{} inline PGen() {} - auto FixFieldsConst(const bc_in&, const em& comp) const - -> std::pair { - if (comp == em::ex2) { - return { init_flds.ex2({ ZERO }), true }; - } else if (comp == em::ex3) { - return { init_flds.ex3({ ZERO }), true }; - } else { - return { ZERO, false }; - } - } - auto MatchFields(real_t time) const -> InitFields { return init_flds; } inline void InitPrtls(Domain& local_domain) { + + // minimum and maximum position of particles + real_t xg_min = local_domain.mesh.extent(in::x1).first; + real_t xg_max = local_domain.mesh.extent(in::x1).second * filling_fraction; + + // define box to inject into + boundaries_t box; + // loop over all dimensions + for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { + // compute the range for the x-direction + if (d == static_cast(in::x1)) { + box.push_back({xg_min, xg_max}); + } else { + // inject into full range in other directions + box.push_back(Range::All); + } + } + + // spatial distribution of the particles + // -> hack to use the uniform distribution in NonUniformInjector + const auto spatial_dist = arch::Piston(local_domain.mesh.metric, xg_min, xg_max, in::x1); + + // energy distribution of the particles const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, local_domain.random_pool, temperature, -drift_ux, in::x1); - const auto injector = arch::UniformInjector( - energy_dist, - { 1, 2 }); - arch::InjectUniform>( - params, - local_domain, - injector, - 1.0); + const auto injector = arch::NonUniformInjector( + energy_dist, + spatial_dist, + {1, 2}); + + arch::InjectNonUniform>( + params, + local_domain, + injector, + 1.0, // target density + false, // no weights + box); } }; diff --git a/setups/srpic/shock/shock.toml b/setups/srpic/shock/shock.toml index 4e03721f7..f954345c2 100644 --- a/setups/srpic/shock/shock.toml +++ b/setups/srpic/shock/shock.toml @@ -40,11 +40,12 @@ maxnpart = 1e8 [setup] - drift_ux = 0.1 - temperature = 1e-4 - Bmag = 1.0 - Btheta = 0.0 - Bphi = 0.0 + drift_ux = 0.1 # speed towards the wall [c] + temperature = 1e-4 # temeperature of maxwell distribution [m_e c^2] + Bmag = 1.0 # magnetic field strength as fraction of magnetisation + Btheta = 0.0 # magnetic field angle in the plane + Bphi = 0.0 # magnetic field angle out of plane + filling_fraction = 1.0 # fraction of the shock piston filled with plasma [output] interval_time = 500.0 diff --git a/src/archetypes/spatial_dist.h b/src/archetypes/spatial_dist.h index be2836da2..098e1c5bb 100644 --- a/src/archetypes/spatial_dist.h +++ b/src/archetypes/spatial_dist.h @@ -4,6 +4,7 @@ * @implements * - arch::SpatialDistribution<> * - arch::Uniform<> : arch::SpatialDistribution<> + * - arch::Piston<> : arch::SpatialDistribution<> * - arch::Replenish<> : arch::SpatialDistribution<> * @namespace * - arch:: @@ -49,6 +50,35 @@ namespace arch { } }; + template + struct Piston : public arch::SpatialDistribution + { + Piston(const M &metric, real_t xmin, real_t xmax, in piston_direction = in::x1) + : arch::SpatialDistribution{metric} + , xmin {xmin} + , xmax {xmax} + , piston_direction {piston_direction} {} + + Inline auto operator()(const coord_t &x_Ph) const -> real_t override + { + // dimentsion to fill + const auto fill_dim = static_cast(piston_direction); + + if (x_Ph[fill_dim] < xmin || x_Ph[fill_dim] > xmax) + { + return 0.0; + } + else + { + return 1.0; + } + } + + private: + real_t xmin, xmax; + in piston_direction; + }; + template struct Replenish : public SpatialDistribution { using SpatialDistribution::metric; From e4d597968bc3411d20696ef3ca15890e06eeed7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Thu, 20 Mar 2025 17:43:25 -0500 Subject: [PATCH 078/183] fix inconsistency in return type --- src/archetypes/spatial_dist.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/archetypes/spatial_dist.h b/src/archetypes/spatial_dist.h index 098e1c5bb..3731922de 100644 --- a/src/archetypes/spatial_dist.h +++ b/src/archetypes/spatial_dist.h @@ -66,11 +66,11 @@ namespace arch { if (x_Ph[fill_dim] < xmin || x_Ph[fill_dim] > xmax) { - return 0.0; + return ZERO; } else { - return 1.0; + return ONE; } } From cbc20219b85a340644cc836be2cdb3a8d608f05b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Mar 2025 16:24:35 -0500 Subject: [PATCH 079/183] Added `MovingInjector` (wip) --- src/archetypes/particle_injector.h | 69 ++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/src/archetypes/particle_injector.h b/src/archetypes/particle_injector.h index cbcbbd389..4a6e72526 100644 --- a/src/archetypes/particle_injector.h +++ b/src/archetypes/particle_injector.h @@ -170,6 +170,75 @@ namespace arch { ~AtmosphereInjector() = default; }; + template + struct MovingInjector + { + struct TargetDensityProfile + { + const real_t nmax, xinj, xdrift; + + TargetDensityProfile(real_t xinj, real_t xdrift, real_t nmax) + : xinj{xinj}, xdrift{xdrift}, nmax{nmax} {} + + Inline auto operator()(const coord_t &x_Ph) const -> real_t + { + if constexpr ((O == in::x1) or + (O == in::x2 and (M::Dim == Dim::_2D or M::Dim == Dim::_3D)) or + (O == in::x3 and M::Dim == Dim::_3D)) + { + const auto xi = x_Ph[static_cast(O)]; + // + direction + if (xi < xinj - xdrift or xi >= xinj) + { + return ZERO; + } + else + { + if constexpr (M::CoordType == Coord::Cart) + { + return nmax; + } + else + { + raise::KernelError( + HERE, + "Moving injector in +x cannot be applied for non-cartesian"); + return ZERO; + } + } + } + else + { + raise::KernelError(HERE, "Wrong direction"); + return ZERO; + } + } + }; + using energy_dist_t = Maxwellian; + using spatial_dist_t = Replenish; + static_assert(M::is_metric, "M must be a metric class"); + static constexpr bool is_nonuniform_injector{true}; + static constexpr Dimension D{M::Dim}; + static constexpr Coord C{M::CoordType}; + + const energy_dist_t energy_dist; + const TargetDensityProfile target_density; + const spatial_dist_t spatial_dist; + const std::pair species; + + MovingInjector(const M &metric, + const ndfield_t &density, + const energy_dist_t &energy_dist, + real_t xinj, + real_t xdrift, + real_t nmax, + const std::pair &species) + : energy_dist{energy_dist}, + target_density{xinj, xdrift, nmax}, spatial_dist{metric, density, 0, target_density, nmax}, species{species} {} + + ~MovingInjector() = default; + }; + /** * @brief Injects uniform number density of particles everywhere in the domain * @param domain Domain object From 70ea9af6342afc63e25464369c68aac71dada3fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Mar 2025 16:25:10 -0500 Subject: [PATCH 080/183] Added `CustomPostStep` for shock pgen to continually inject particles (wip) --- setups/srpic/shock/pgen.hpp | 78 ++++++++++++++++++++++++++++++++++- setups/srpic/shock/shock.toml | 4 +- src/engines/engine_run.cpp | 2 +- 3 files changed, 79 insertions(+), 5 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 9bf51b866..7c0d750e3 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -79,7 +79,7 @@ namespace user { using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; - const real_t drift_ux, temperature, filling_fraction; + const real_t drift_ux, temperature, filling_fraction, injection_rate; const real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -92,7 +92,8 @@ namespace user { , Btheta { p.template get("setup.Btheta", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } , init_flds { Bmag, Btheta, Bphi, drift_ux } - , filling_fraction { params.template get("setup.filling_fraction", 1.0) }{} + , filling_fraction { params.template get("setup.filling_fraction", 1.0) } + , injection_rate { params.template get("setup.injection_rate", 1.0) } {} inline PGen() {} @@ -100,6 +101,11 @@ namespace user { return init_flds; } + // Inline auto TargetDensity(const coord_t &x_Ph) const -> real_t + // { + // return ONE; + // } + inline void InitPrtls(Domain& local_domain) { // minimum and maximum position of particles @@ -143,6 +149,74 @@ namespace user { false, // no weights box); } + + void CustomPostStep(std::size_t, long double time, real_t dt, Domain &domain) + { + /* + Moving injector for the particles + */ + + // energy distribution of the particles + const auto energy_dist = arch::Maxwellian(domain.mesh.metric, + domain.random_pool, + temperature, + -drift_ux, + in::x1); + + // minimum and maximum position of particles + // ToDo: Add time offset for start of movement? + real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction + + injection_rate * time - drift_ux * dt; + real_t xg_max = domain.mesh.extent(in::x1).second * filling_fraction + + injection_rate * time; + + + /* + I thought this option would be better, but I can't get it to work + */ + + // // define box to inject into + // boundaries_t box; + // // loop over all dimensions + // for (unsigned short d{0}; d < static_cast(M::Dim); ++d) + // { + // // compute the range for the x-direction + // if (d == static_cast(in::x1)) + // { + // box.push_back({xg_min, xg_max}); + // } else { + // // inject into full range in other directions + // box.push_back(Range::All); + // } + // } + + // const auto spatial_dist = arch::Replenish(domain.mesh.metric, + // domain.fields.bckp, + // box, + // TargetDensity, + // 1.0); + + // const auto injector = arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // {1, 2}); + + const auto injector = arch::MovingInjector{ + domain.mesh.metric, + domain.fields.bckp, + energy_dist, + xg_max, + xg_min, + 1.0, + {1, 2}}; + + arch::InjectNonUniform( + params, + domain, + injector, + 1.0, // target density + false); + } }; } // namespace user diff --git a/setups/srpic/shock/shock.toml b/setups/srpic/shock/shock.toml index f954345c2..772b452c9 100644 --- a/setups/srpic/shock/shock.toml +++ b/setups/srpic/shock/shock.toml @@ -45,10 +45,10 @@ Bmag = 1.0 # magnetic field strength as fraction of magnetisation Btheta = 0.0 # magnetic field angle in the plane Bphi = 0.0 # magnetic field angle out of plane - filling_fraction = 1.0 # fraction of the shock piston filled with plasma + filling_fraction = 0.1 # fraction of the shock piston filled with plasma [output] - interval_time = 500.0 + interval_time = 10.0 format = "hdf5" [output.fields] diff --git a/src/engines/engine_run.cpp b/src/engines/engine_run.cpp index 506fd121d..3ace709e9 100644 --- a/src/engines/engine_run.cpp +++ b/src/engines/engine_run.cpp @@ -52,7 +52,7 @@ namespace ntt { traits::has_method::value) { timers.start("Custom"); m_metadomain.runOnLocalDomains([&timers, this](auto& dom) { - m_pgen.CustomPostStep(step, time, dom); + m_pgen.CustomPostStep(step, time, dt, dom); }); timers.stop("Custom"); } From 972750763f79a15c5f4cb764b9e7f4901e394fe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Sat, 22 Mar 2025 11:42:48 -0500 Subject: [PATCH 081/183] bugfix --- src/archetypes/particle_injector.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/archetypes/particle_injector.h b/src/archetypes/particle_injector.h index 4a6e72526..967a77c75 100644 --- a/src/archetypes/particle_injector.h +++ b/src/archetypes/particle_injector.h @@ -188,7 +188,7 @@ namespace arch { { const auto xi = x_Ph[static_cast(O)]; // + direction - if (xi < xinj - xdrift or xi >= xinj) + if (xi < xdrift or xi >= xinj) { return ZERO; } From d7d12a31742c3303cda25a354fd724d99b9c3f5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Sat, 22 Mar 2025 11:56:32 -0500 Subject: [PATCH 082/183] added start time of injection and explicitly enforce injection region (wip) --- setups/srpic/shock/pgen.hpp | 102 +++++++++++++++++----------------- setups/srpic/shock/shock.toml | 2 + 2 files changed, 54 insertions(+), 50 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 7c0d750e3..04de58406 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -79,7 +79,8 @@ namespace user { using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; - const real_t drift_ux, temperature, filling_fraction, injection_rate; + const real_t drift_ux, temperature, filling_fraction, + injector_velocity, injection_start; const real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -93,7 +94,8 @@ namespace user { , Bphi { p.template get("setup.Bphi", ZERO) } , init_flds { Bmag, Btheta, Bphi, drift_ux } , filling_fraction { params.template get("setup.filling_fraction", 1.0) } - , injection_rate { params.template get("setup.injection_rate", 1.0) } {} + , injector_velocity { params.template get("setup.injector_velocity", 1.0) } + , injection_start { params.template get("setup.injection_start", 0.0) } {} inline PGen() {} @@ -164,58 +166,58 @@ namespace user { in::x1); // minimum and maximum position of particles - // ToDo: Add time offset for start of movement? - real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction + - injection_rate * time - drift_ux * dt; + real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction + + injector_velocity * (time - injection_start) + - drift_ux; // distance particles have moved in the last time step real_t xg_max = domain.mesh.extent(in::x1).second * filling_fraction + - injection_rate * time; + injector_velocity * (time - injection_start); + /* + I thought this option would be better, but I can't get it to work + */ - /* - I thought this option would be better, but I can't get it to work - */ - - // // define box to inject into - // boundaries_t box; - // // loop over all dimensions - // for (unsigned short d{0}; d < static_cast(M::Dim); ++d) - // { - // // compute the range for the x-direction - // if (d == static_cast(in::x1)) - // { - // box.push_back({xg_min, xg_max}); - // } else { - // // inject into full range in other directions - // box.push_back(Range::All); - // } - // } - - // const auto spatial_dist = arch::Replenish(domain.mesh.metric, - // domain.fields.bckp, - // box, - // TargetDensity, - // 1.0); - - // const auto injector = arch::NonUniformInjector( - // energy_dist, - // spatial_dist, - // {1, 2}); - - const auto injector = arch::MovingInjector{ - domain.mesh.metric, - domain.fields.bckp, - energy_dist, - xg_max, - xg_min, - 1.0, - {1, 2}}; + // define box to inject into + boundaries_t box; + // loop over all dimensions + for (unsigned short d{0}; d < static_cast(M::Dim); ++d) + { + // compute the range for the x-direction + if (d == static_cast(in::x1)) + { + box.push_back({xg_min, xg_max}); + } else { + // inject into full range in other directions + box.push_back(Range::All); + } + } - arch::InjectNonUniform( - params, - domain, - injector, - 1.0, // target density - false); + // const auto spatial_dist = arch::Replenish(domain.mesh.metric, + // domain.fields.bckp, + // box, + // TargetDensity, + // 1.0); + + // const auto injector = arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // {1, 2}); + + const auto injector = arch::MovingInjector{ + domain.mesh.metric, + domain.fields.bckp, + energy_dist, + xg_max, + xg_min, + 1.0, + {1, 2}}; + + arch::InjectNonUniform( + params, + domain, + injector, + 1.0, // target density + false, + box); } }; diff --git a/setups/srpic/shock/shock.toml b/setups/srpic/shock/shock.toml index 772b452c9..43da87d99 100644 --- a/setups/srpic/shock/shock.toml +++ b/setups/srpic/shock/shock.toml @@ -46,6 +46,8 @@ Btheta = 0.0 # magnetic field angle in the plane Bphi = 0.0 # magnetic field angle out of plane filling_fraction = 0.1 # fraction of the shock piston filled with plasma + injector_velocity = 1.0 # speed of injector [c] + injection_start = 0.0 # start time of moving injector [output] interval_time = 10.0 From 6cf0dc8e5eb44528d049b31111ff7ff81bb52bc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Sat, 22 Mar 2025 12:12:00 -0500 Subject: [PATCH 083/183] small bugfix --- setups/srpic/shock/pgen.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 04de58406..1c9c9a4d3 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -167,8 +167,8 @@ namespace user { // minimum and maximum position of particles real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction - + injector_velocity * (time - injection_start) - - drift_ux; // distance particles have moved in the last time step + + injector_velocity * (time - injection_start - dt) + - drift_ux * dt; // distance particles have moved in the last time step real_t xg_max = domain.mesh.extent(in::x1).second * filling_fraction + injector_velocity * (time - injection_start); From 22a0479855dad05a77b5d139e6c2d0c692dd2ead Mon Sep 17 00:00:00 2001 From: hayk Date: Mon, 24 Mar 2025 13:14:51 -0400 Subject: [PATCH 084/183] shock pgen changed --- setups/srpic/shock/pgen.hpp | 200 ++++++++++++++++++++-------------- src/archetypes/spatial_dist.h | 39 +++---- src/engines/engine_run.cpp | 2 +- 3 files changed, 134 insertions(+), 107 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 1c9c9a4d3..cc6f1b812 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -18,6 +18,16 @@ namespace user { using namespace ntt; + template + struct SpatialUniform : public arch::SpatialDistribution { + SpatialUniform(const M& metric) + : arch::SpatialDistribution { metric } {} + + Inline auto operator()(const coord_t&) const -> real_t override { + return ONE; + } + }; + template struct InitFields { /* @@ -79,8 +89,8 @@ namespace user { using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; - const real_t drift_ux, temperature, filling_fraction, - injector_velocity, injection_start; + const real_t drift_ux, temperature, filling_fraction, injector_velocity, + injection_start, dt; const real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -92,10 +102,14 @@ namespace user { , Bmag { p.template get("setup.Bmag", ZERO) } , Btheta { p.template get("setup.Btheta", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } - , init_flds { Bmag, Btheta, Bphi, drift_ux } - , filling_fraction { params.template get("setup.filling_fraction", 1.0) } - , injector_velocity { params.template get("setup.injector_velocity", 1.0) } - , injection_start { params.template get("setup.injection_start", 0.0) } {} + , init_flds { Bmag, Btheta, Bphi, drift_ux } + , filling_fraction { params.template get("setup.filling_fraction", + 1.0) } + , injector_velocity { params.template get( + "setup.injector_velocity", + 1.0) } + , injection_start { params.template get("setup.injection_start", 0.0) } + , dt { params.template get("algorithms.timestep.dt") } {} inline PGen() {} @@ -109,7 +123,7 @@ namespace user { // } inline void InitPrtls(Domain& local_domain) { - + // minimum and maximum position of particles real_t xg_min = local_domain.mesh.extent(in::x1).first; real_t xg_max = local_domain.mesh.extent(in::x1).second * filling_fraction; @@ -120,16 +134,19 @@ namespace user { for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { // compute the range for the x-direction if (d == static_cast(in::x1)) { - box.push_back({xg_min, xg_max}); + box.push_back({ xg_min, xg_max }); } else { // inject into full range in other directions box.push_back(Range::All); } } - // spatial distribution of the particles + // spatial distribution of the particles // -> hack to use the uniform distribution in NonUniformInjector - const auto spatial_dist = arch::Piston(local_domain.mesh.metric, xg_min, xg_max, in::x1); + const auto spatial_dist = arch::Piston(local_domain.mesh.metric, + xg_min, + xg_max, + in::x1); // energy distribution of the particles const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, @@ -139,85 +156,100 @@ namespace user { in::x1); const auto injector = arch::NonUniformInjector( - energy_dist, - spatial_dist, - {1, 2}); + energy_dist, + spatial_dist, + { 1, 2 }); arch::InjectNonUniform>( - params, - local_domain, - injector, - 1.0, // target density - false, // no weights - box); + params, + local_domain, + injector, + 1.0, // target density + false, // no weights + box); } - void CustomPostStep(std::size_t, long double time, real_t dt, Domain &domain) - { - /* - Moving injector for the particles - */ - - // energy distribution of the particles - const auto energy_dist = arch::Maxwellian(domain.mesh.metric, - domain.random_pool, - temperature, - -drift_ux, - in::x1); - - // minimum and maximum position of particles - real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction - + injector_velocity * (time - injection_start - dt) - - drift_ux * dt; // distance particles have moved in the last time step - real_t xg_max = domain.mesh.extent(in::x1).second * filling_fraction + - injector_velocity * (time - injection_start); - - /* - I thought this option would be better, but I can't get it to work - */ - - // define box to inject into - boundaries_t box; - // loop over all dimensions - for (unsigned short d{0}; d < static_cast(M::Dim); ++d) - { - // compute the range for the x-direction - if (d == static_cast(in::x1)) - { - box.push_back({xg_min, xg_max}); - } else { - // inject into full range in other directions - box.push_back(Range::All); - } - } + void CustomPostStep(std::size_t, long double time, Domain& domain) { + const auto energy_dist = arch::Maxwellian(domain.mesh.metric, + domain.random_pool, + temperature, + -drift_ux, + in::x1); + const auto spatial_dist = SpatialUniform(domain.mesh.metric); + const auto injector = arch::NonUniformInjector( + energy_dist, + spatial_dist, + { 1, 2 }); + + boundaries_t box; + + const auto x_init = domain.mesh.extent(in::x1).first + + filling_fraction * (domain.mesh.extent(in::x1).second - + domain.mesh.extent(in::x1).first); + + box[0].first = x_init + injector_velocity * (time - injection_start); + box[0].second = x_init + injector_velocity * (time - injection_start + dt) - + drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt; + for (auto d = 1u; d < M::Dim; ++d) { + box[d] = Range::All; + } - // const auto spatial_dist = arch::Replenish(domain.mesh.metric, - // domain.fields.bckp, - // box, - // TargetDensity, - // 1.0); - - // const auto injector = arch::NonUniformInjector( - // energy_dist, - // spatial_dist, - // {1, 2}); - - const auto injector = arch::MovingInjector{ - domain.mesh.metric, - domain.fields.bckp, - energy_dist, - xg_max, - xg_min, - 1.0, - {1, 2}}; - - arch::InjectNonUniform( - params, - domain, - injector, - 1.0, // target density - false, - box); + arch::InjectNonUniform(params, + domain, + injector, + ONE, + false, + box); + + // /* + // Moving injector for the particles + // */ + // + // // minimum and maximum position of particles + // real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction + + // injector_velocity * (time - injection_start - dt) - + // drift_ux * + // dt; // distance particles have moved in the last time step + // real_t xg_max = domain.mesh.extent(in::x1).second * filling_fraction + + // injector_velocity * (time - injection_start); + // + // /* + // I thought this option would be better, but I can't get it to work + // */ + // + // // define box to inject into + // // loop over all dimensions + // for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { + // // compute the range for the x-direction + // if (d == static_cast(in::x1)) { + // box.push_back({ xg_min, xg_max }); + // } else { + // // inject into full range in other directions + // box.push_back(Range::All); + // } + // } + + // const auto spatial_dist = arch::Replenish(domain.mesh.metric, + // domain.fields.bckp, + // box, + // TargetDensity, + // 1.0); + + // const auto injector = arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // {1, 2}); + + // const auto injector = arch::MovingInjector { + // domain.mesh.metric, + // domain.fields.bckp, + // energy_dist, + // xg_max, + // xg_min, + // 1.0, + // { 1, 2 } + // }; } }; diff --git a/src/archetypes/spatial_dist.h b/src/archetypes/spatial_dist.h index 3731922de..3555aaca5 100644 --- a/src/archetypes/spatial_dist.h +++ b/src/archetypes/spatial_dist.h @@ -51,32 +51,27 @@ namespace arch { }; template - struct Piston : public arch::SpatialDistribution - { - Piston(const M &metric, real_t xmin, real_t xmax, in piston_direction = in::x1) - : arch::SpatialDistribution{metric} - , xmin {xmin} - , xmax {xmax} - , piston_direction {piston_direction} {} - - Inline auto operator()(const coord_t &x_Ph) const -> real_t override - { - // dimentsion to fill - const auto fill_dim = static_cast(piston_direction); - - if (x_Ph[fill_dim] < xmin || x_Ph[fill_dim] > xmax) - { - return ZERO; - } - else - { - return ONE; - } + struct Piston : public arch::SpatialDistribution { + Piston(const M& metric, real_t xmin, real_t xmax, in piston_direction = in::x1) + : arch::SpatialDistribution { metric } + , xmin { xmin } + , xmax { xmax } + , piston_direction { piston_direction } {} + + Inline auto operator()(const coord_t& x_Ph) const -> real_t override { + // dimentsion to fill + const auto fill_dim = static_cast(piston_direction); + + if (x_Ph[fill_dim] < xmin || x_Ph[fill_dim] > xmax) { + return ZERO; + } else { + return ONE; } + } private: real_t xmin, xmax; - in piston_direction; + in piston_direction; }; template diff --git a/src/engines/engine_run.cpp b/src/engines/engine_run.cpp index 3ace709e9..506fd121d 100644 --- a/src/engines/engine_run.cpp +++ b/src/engines/engine_run.cpp @@ -52,7 +52,7 @@ namespace ntt { traits::has_method::value) { timers.start("Custom"); m_metadomain.runOnLocalDomains([&timers, this](auto& dom) { - m_pgen.CustomPostStep(step, time, dt, dom); + m_pgen.CustomPostStep(step, time, dom); }); timers.stop("Custom"); } From 13da0dda1a67d2fba6c7e6d7111965b75f0f0ce0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 24 Mar 2025 15:31:41 -0500 Subject: [PATCH 085/183] reduce resolution for faster test --- setups/srpic/shock/shock.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setups/srpic/shock/shock.toml b/setups/srpic/shock/shock.toml index 43da87d99..ca19a4078 100644 --- a/setups/srpic/shock/shock.toml +++ b/setups/srpic/shock/shock.toml @@ -4,8 +4,8 @@ runtime = 30000.0 [grid] - resolution = [16384, 128] - extent = [[0.0, 8000.0], [-31.25, 31.25]] + resolution = [4096, 128] + extent = [[0.0, 2000.0], [-31.25, 31.25]] [grid.metric] metric = "minkowski" From 54f083fe77d849e0c9658ab5714f10dd38915926 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 24 Mar 2025 15:31:52 -0500 Subject: [PATCH 086/183] cleanup and bugfix --- setups/srpic/shock/pgen.hpp | 195 +++++++++++++++++------------------- 1 file changed, 92 insertions(+), 103 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index cc6f1b812..2454c9037 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -18,16 +18,6 @@ namespace user { using namespace ntt; - template - struct SpatialUniform : public arch::SpatialDistribution { - SpatialUniform(const M& metric) - : arch::SpatialDistribution { metric } {} - - Inline auto operator()(const coord_t&) const -> real_t override { - return ONE; - } - }; - template struct InitFields { /* @@ -89,10 +79,13 @@ namespace user { using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; - const real_t drift_ux, temperature, filling_fraction, injector_velocity, - injection_start, dt; + // gas properties + const real_t drift_ux, temperature, filling_fraction; + // injector properties + const real_t injector_velocity, injection_start, dt; + const bool injection_on; - const real_t Btheta, Bphi, Bmag; + real_t Btheta, Bphi, Bmag; InitFields init_flds; inline PGen(const SimulationParams& p, const Metadomain& m) @@ -103,13 +96,11 @@ namespace user { , Btheta { p.template get("setup.Btheta", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } , init_flds { Bmag, Btheta, Bphi, drift_ux } - , filling_fraction { params.template get("setup.filling_fraction", - 1.0) } - , injector_velocity { params.template get( - "setup.injector_velocity", - 1.0) } - , injection_start { params.template get("setup.injection_start", 0.0) } - , dt { params.template get("algorithms.timestep.dt") } {} + , filling_fraction { p.template get("setup.filling_fraction", 1.0) } + , injector_velocity { p.template get("setup.injector_velocity", 1.0) } + , injection_start { p.template get("setup.injection_start", 0.0) } + , injection_on { p.template get("setup.continuous_injection", true) } + , dt { p.template get("algorithms.timestep.dt") } {} inline PGen() {} @@ -117,16 +108,13 @@ namespace user { return init_flds; } - // Inline auto TargetDensity(const coord_t &x_Ph) const -> real_t - // { - // return ONE; - // } - inline void InitPrtls(Domain& local_domain) { // minimum and maximum position of particles real_t xg_min = local_domain.mesh.extent(in::x1).first; - real_t xg_max = local_domain.mesh.extent(in::x1).second * filling_fraction; + real_t xg_max = local_domain.mesh.extent(in::x1).first + + filling_fraction * (local_domain.mesh.extent(in::x1).second - + local_domain.mesh.extent(in::x1).first); // define box to inject into boundaries_t box; @@ -170,86 +158,87 @@ namespace user { } void CustomPostStep(std::size_t, long double time, Domain& domain) { - const auto energy_dist = arch::Maxwellian(domain.mesh.metric, - domain.random_pool, - temperature, - -drift_ux, - in::x1); - const auto spatial_dist = SpatialUniform(domain.mesh.metric); - const auto injector = arch::NonUniformInjector( - energy_dist, - spatial_dist, - { 1, 2 }); - boundaries_t box; - - const auto x_init = domain.mesh.extent(in::x1).first + - filling_fraction * (domain.mesh.extent(in::x1).second - - domain.mesh.extent(in::x1).first); + // ToDo: more performant version? + if (injection_on) { + // same maxwell distribution as above + const auto energy_dist = arch::Maxwellian(domain.mesh.metric, + domain.random_pool, + temperature, + -drift_ux, + in::x1); + + // initial position of injector + const auto x_init = domain.mesh.extent(in::x1).first + + filling_fraction * (domain.mesh.extent(in::x1).second - + domain.mesh.extent(in::x1).first); + + // check if injector is supposed to start moving already + const auto dt_inj = time - injection_start > ZERO ? time - injection_start + : ZERO; + + // define box to inject into + boundaries_t box; + + // loop over all dimension + for (auto d = 0u; d < M::Dim; ++d) { + if (d == 0) { + box.push_back({ x_init + injector_velocity * (dt_inj)-drift_ux / + math::sqrt(1 + SQR(drift_ux)) * dt, + x_init + injector_velocity * (dt_inj + dt) }); + } else { + box.push_back(Range::All); + } + } - box[0].first = x_init + injector_velocity * (time - injection_start); - box[0].second = x_init + injector_velocity * (time - injection_start + dt) - - drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt; - for (auto d = 1u; d < M::Dim; ++d) { - box[d] = Range::All; + // spatial distribution of the particles + // -> hack to use the uniform distribution in NonUniformInjector + const auto spatial_dist = arch::Piston(domain.mesh.metric, + box[0].first, + box[0].second, + in::x1); + + // ToDo: extend Replenish to replace the current injector + const auto injector = + arch::NonUniformInjector( + energy_dist, + spatial_dist, + { 1, 2 }); + + // inject non-uniformly within the defined box + arch::InjectNonUniform(params, + domain, + injector, + ONE, + false, + box); + + /* + I thought this option would be better, but I can't get it to work + */ + + // const auto spatial_dist = arch::Replenish(domain.mesh.metric, + // domain.fields.bckp, + // box, + // TargetDensity, + // 1.0); + + // const auto injector = arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // {1, 2}); + + // const auto injector = arch::MovingInjector { + // domain.mesh.metric, + // domain.fields.bckp, + // energy_dist, + // box[0].first, + // box[0].second, + // 1.0, + // { 1, 2 } + // }; } - - arch::InjectNonUniform(params, - domain, - injector, - ONE, - false, - box); - - // /* - // Moving injector for the particles - // */ - // - // // minimum and maximum position of particles - // real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction + - // injector_velocity * (time - injection_start - dt) - - // drift_ux * - // dt; // distance particles have moved in the last time step - // real_t xg_max = domain.mesh.extent(in::x1).second * filling_fraction + - // injector_velocity * (time - injection_start); - // - // /* - // I thought this option would be better, but I can't get it to work - // */ - // - // // define box to inject into - // // loop over all dimensions - // for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { - // // compute the range for the x-direction - // if (d == static_cast(in::x1)) { - // box.push_back({ xg_min, xg_max }); - // } else { - // // inject into full range in other directions - // box.push_back(Range::All); - // } - // } - - // const auto spatial_dist = arch::Replenish(domain.mesh.metric, - // domain.fields.bckp, - // box, - // TargetDensity, - // 1.0); - - // const auto injector = arch::NonUniformInjector( - // energy_dist, - // spatial_dist, - // {1, 2}); - - // const auto injector = arch::MovingInjector { - // domain.mesh.metric, - // domain.fields.bckp, - // energy_dist, - // xg_max, - // xg_min, - // 1.0, - // { 1, 2 } - // }; } }; From 9f45d04b528ff15f850a86584a58287e52631748 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 24 Mar 2025 15:45:43 -0500 Subject: [PATCH 087/183] removed unnecessary parameter --- setups/srpic/shock/pgen.hpp | 156 +++++++++++++++++------------------- 1 file changed, 75 insertions(+), 81 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 2454c9037..30ab01538 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -83,8 +83,7 @@ namespace user { const real_t drift_ux, temperature, filling_fraction; // injector properties const real_t injector_velocity, injection_start, dt; - const bool injection_on; - + // magnetic field properties real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -99,7 +98,6 @@ namespace user { , filling_fraction { p.template get("setup.filling_fraction", 1.0) } , injector_velocity { p.template get("setup.injector_velocity", 1.0) } , injection_start { p.template get("setup.injection_start", 0.0) } - , injection_on { p.template get("setup.continuous_injection", true) } , dt { p.template get("algorithms.timestep.dt") } {} inline PGen() {} @@ -159,86 +157,82 @@ namespace user { void CustomPostStep(std::size_t, long double time, Domain& domain) { - // ToDo: more performant version? - if (injection_on) { - // same maxwell distribution as above - const auto energy_dist = arch::Maxwellian(domain.mesh.metric, - domain.random_pool, - temperature, - -drift_ux, - in::x1); - - // initial position of injector - const auto x_init = domain.mesh.extent(in::x1).first + - filling_fraction * (domain.mesh.extent(in::x1).second - - domain.mesh.extent(in::x1).first); - - // check if injector is supposed to start moving already - const auto dt_inj = time - injection_start > ZERO ? time - injection_start - : ZERO; - - // define box to inject into - boundaries_t box; - - // loop over all dimension - for (auto d = 0u; d < M::Dim; ++d) { - if (d == 0) { - box.push_back({ x_init + injector_velocity * (dt_inj)-drift_ux / - math::sqrt(1 + SQR(drift_ux)) * dt, - x_init + injector_velocity * (dt_inj + dt) }); - } else { - box.push_back(Range::All); - } - } + // same maxwell distribution as above + const auto energy_dist = arch::Maxwellian(domain.mesh.metric, + domain.random_pool, + temperature, + -drift_ux, + in::x1); + + // initial position of injector + const auto x_init = domain.mesh.extent(in::x1).first + + filling_fraction * (domain.mesh.extent(in::x1).second - + domain.mesh.extent(in::x1).first); - // spatial distribution of the particles - // -> hack to use the uniform distribution in NonUniformInjector - const auto spatial_dist = arch::Piston(domain.mesh.metric, - box[0].first, - box[0].second, - in::x1); - - // ToDo: extend Replenish to replace the current injector - const auto injector = - arch::NonUniformInjector( - energy_dist, - spatial_dist, - { 1, 2 }); - - // inject non-uniformly within the defined box - arch::InjectNonUniform(params, - domain, - injector, - ONE, - false, - box); - - /* - I thought this option would be better, but I can't get it to work - */ - - // const auto spatial_dist = arch::Replenish(domain.mesh.metric, - // domain.fields.bckp, - // box, - // TargetDensity, - // 1.0); - - // const auto injector = arch::NonUniformInjector( - // energy_dist, - // spatial_dist, - // {1, 2}); - - // const auto injector = arch::MovingInjector { - // domain.mesh.metric, - // domain.fields.bckp, - // energy_dist, - // box[0].first, - // box[0].second, - // 1.0, - // { 1, 2 } - // }; + // check if injector is supposed to start moving already + const auto dt_inj = time - injection_start > ZERO ? + time - injection_start : ZERO; + + // define box to inject into + boundaries_t box; + + // loop over all dimension + for (auto d = 0u; d < M::Dim; ++d) { + if (d == 0) { + box.push_back({ x_init + injector_velocity * dt_inj - + drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt, + x_init + injector_velocity * (dt_inj + dt) }); + } else { + box.push_back(Range::All); + } } + + // spatial distribution of the particles + // -> hack to use the uniform distribution in NonUniformInjector + const auto spatial_dist = arch::Piston(domain.mesh.metric, + box[0].first, + box[0].second, + in::x1); + + // ToDo: extend Replenish to replace the current injector + const auto injector = arch::NonUniformInjector( + energy_dist, + spatial_dist, + { 1, 2 }); + + // inject non-uniformly within the defined box + arch::InjectNonUniform(params, + domain, + injector, + ONE, + false, + box); + + /* + I thought this option would be better, but I can't get it to work + */ + + // const auto spatial_dist = arch::Replenish(domain.mesh.metric, + // domain.fields.bckp, + // box, + // TargetDensity, + // 1.0); + + // const auto injector = arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // {1, 2}); + + // const auto injector = arch::MovingInjector { + // domain.mesh.metric, + // domain.fields.bckp, + // energy_dist, + // box[0].first, + // box[0].second, + // 1.0, + // { 1, 2 } + // }; } }; From aa8b8e7f10edc6316476b37297d8b980bb5c79d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 24 Mar 2025 16:06:42 -0500 Subject: [PATCH 088/183] applied formatting to `MovingInjector` --- src/archetypes/particle_injector.h | 123 ++++++++++++++--------------- 1 file changed, 58 insertions(+), 65 deletions(-) diff --git a/src/archetypes/particle_injector.h b/src/archetypes/particle_injector.h index 967a77c75..818ab5bf5 100644 --- a/src/archetypes/particle_injector.h +++ b/src/archetypes/particle_injector.h @@ -171,72 +171,65 @@ namespace arch { }; template - struct MovingInjector - { - struct TargetDensityProfile - { - const real_t nmax, xinj, xdrift; - - TargetDensityProfile(real_t xinj, real_t xdrift, real_t nmax) - : xinj{xinj}, xdrift{xdrift}, nmax{nmax} {} - - Inline auto operator()(const coord_t &x_Ph) const -> real_t - { - if constexpr ((O == in::x1) or - (O == in::x2 and (M::Dim == Dim::_2D or M::Dim == Dim::_3D)) or - (O == in::x3 and M::Dim == Dim::_3D)) - { - const auto xi = x_Ph[static_cast(O)]; - // + direction - if (xi < xdrift or xi >= xinj) - { - return ZERO; - } - else - { - if constexpr (M::CoordType == Coord::Cart) - { - return nmax; - } - else - { - raise::KernelError( - HERE, - "Moving injector in +x cannot be applied for non-cartesian"); - return ZERO; - } - } - } - else - { - raise::KernelError(HERE, "Wrong direction"); - return ZERO; - } + struct MovingInjector { + struct TargetDensityProfile { + const real_t nmax, xinj, xdrift; + + TargetDensityProfile(real_t xinj, real_t xdrift, real_t nmax) + : xinj { xinj } + , xdrift { xdrift } + , nmax { nmax } {} + + Inline auto operator()(const coord_t& x_Ph) const -> real_t { + if constexpr ((O == in::x1) or + (O == in::x2 and (M::Dim == Dim::_2D or M::Dim == Dim::_3D)) or + (O == in::x3 and M::Dim == Dim::_3D)) { + const auto xi = x_Ph[static_cast(O)]; + // + direction + if (xi < xdrift or xi >= xinj) { + return ZERO; + } else { + if constexpr (M::CoordType == Coord::Cart) { + return nmax; + } else { + raise::KernelError( + HERE, + "Moving injector in +x cannot be applied for non-cartesian"); + return ZERO; } - }; - using energy_dist_t = Maxwellian; - using spatial_dist_t = Replenish; - static_assert(M::is_metric, "M must be a metric class"); - static constexpr bool is_nonuniform_injector{true}; - static constexpr Dimension D{M::Dim}; - static constexpr Coord C{M::CoordType}; - - const energy_dist_t energy_dist; - const TargetDensityProfile target_density; - const spatial_dist_t spatial_dist; - const std::pair species; - - MovingInjector(const M &metric, - const ndfield_t &density, - const energy_dist_t &energy_dist, - real_t xinj, - real_t xdrift, - real_t nmax, - const std::pair &species) - : energy_dist{energy_dist}, - target_density{xinj, xdrift, nmax}, spatial_dist{metric, density, 0, target_density, nmax}, species{species} {} - - ~MovingInjector() = default; + } + } else { + raise::KernelError(HERE, "Wrong direction"); + return ZERO; + } + } + }; + + using energy_dist_t = Maxwellian; + using spatial_dist_t = Replenish; + static_assert(M::is_metric, "M must be a metric class"); + static constexpr bool is_nonuniform_injector { true }; + static constexpr Dimension D { M::Dim }; + static constexpr Coord C { M::CoordType }; + + const energy_dist_t energy_dist; + const TargetDensityProfile target_density; + const spatial_dist_t spatial_dist; + const std::pair species; + + MovingInjector(const M& metric, + const ndfield_t& density, + const energy_dist_t& energy_dist, + real_t xinj, + real_t xdrift, + real_t nmax, + const std::pair& species) + : energy_dist { energy_dist } + , target_density { xinj, xdrift, nmax } + , spatial_dist { metric, density, 0, target_density, nmax } + , species { species } {} + + ~MovingInjector() = default; }; /** From d0fb4ca29ece804e9acfb3e0bc2d2438342b6267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 31 Mar 2025 11:03:57 -0500 Subject: [PATCH 089/183] first attempt at moving-window injection --- setups/srpic/shock/pgen.hpp | 123 +++++++++++++++++++++++++++++++++--- 1 file changed, 114 insertions(+), 9 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 30ab01538..9568767ff 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -83,6 +83,7 @@ namespace user { const real_t drift_ux, temperature, filling_fraction; // injector properties const real_t injector_velocity, injection_start, dt; + const int injection_frequency; // magnetic field properties real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -98,6 +99,7 @@ namespace user { , filling_fraction { p.template get("setup.filling_fraction", 1.0) } , injector_velocity { p.template get("setup.injector_velocity", 1.0) } , injection_start { p.template get("setup.injection_start", 0.0) } + , injection_frequency { p.template get("setup.injection_frequency", 100) } , dt { p.template get("algorithms.timestep.dt") } {} inline PGen() {} @@ -106,6 +108,25 @@ namespace user { return init_flds; } + auto ResetFields(const em& comp) const -> real_t { + if (comp == em::ex1) { + return init_flds.ex1({ ZERO }); + } else if (comp == em::ex2) { + return init_flds.ex2({ ZERO }); + } else if (comp == em::ex3) { + return init_flds.ex3({ ZERO }); + } else if (comp == em::bx1) { + return init_flds.bx1({ ZERO }); + } else if (comp == em::bx2) { + return init_flds.bx2({ ZERO }); + } else if (comp == em::bx3) { + return init_flds.bx3({ ZERO }); + } else { + raise::Error("Invalid component", HERE); + return ZERO; + } + } + inline void InitPrtls(Domain& local_domain) { // minimum and maximum position of particles @@ -155,14 +176,12 @@ namespace user { box); } - void CustomPostStep(std::size_t, long double time, Domain& domain) { + void CustomPostStep(std::size_t step, long double time, Domain& domain) { - // same maxwell distribution as above - const auto energy_dist = arch::Maxwellian(domain.mesh.metric, - domain.random_pool, - temperature, - -drift_ux, - in::x1); + // check if the injector should be active + if (step % injection_frequency != 0) { + return; + } // initial position of injector const auto x_init = domain.mesh.extent(in::x1).first + @@ -180,13 +199,99 @@ namespace user { for (auto d = 0u; d < M::Dim; ++d) { if (d == 0) { box.push_back({ x_init + injector_velocity * dt_inj - - drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt, + drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt - + 1.5 * injection_frequency * dt, x_init + injector_velocity * (dt_inj + dt) }); } else { box.push_back(Range::All); } } + // define indice range to reset fields + boundaries_t incl_ghosts; + for (auto d = 0; d < M::Dim; ++d) { + incl_ghosts.push_back({ true, true }); + } + const auto extent = domain.mesh.ExtentToRange(box, incl_ghosts); + tuple_t x_min { 0 }, x_max { 0 }; + for (auto d = 0; d < M::Dim; ++d) { + x_min[d] = extent[d].first; + x_max[d] = extent[d].second; + } + + // reset fields + std::vector comps = { em::ex1, em::ex2, em::ex3, + em::bx1, em::bx2, em::bx3 }; + + // loop over all components + for (const auto& comp : comps) { + auto value = ResetFields((em)comp); + + if constexpr (M::Dim == Dim::_1D) { + Kokkos::deep_copy(Kokkos::subview(domain.fields.em, + std::make_pair(x_min[0], x_max[0]), + comp), + value); + } else if constexpr (M::Dim == Dim::_2D) { + Kokkos::deep_copy(Kokkos::subview(domain.fields.em, + std::make_pair(x_min[0], x_max[0]), + std::make_pair(x_min[1], x_max[1]), + comp), + value); + } else if constexpr (M::Dim == Dim::_3D) { + Kokkos::deep_copy(Kokkos::subview(domain.fields.em, + std::make_pair(x_min[0], x_max[0]), + std::make_pair(x_min[1], x_max[1]), + std::make_pair(x_min[2], x_max[2]), + comp), + value); + } else { + raise::Error("Invalid dimension", HERE); + } + } + + /* + tag particles inside the injection zone as dead + */ + + // loop over particle species + for (std::size_t s { 0 }; s < 2; ++s) { + + // get particle properties + auto& species = domain.species[s]; + auto i1 = species.i1; + auto tag = species.tag; + + + // tag all particles with x > box[0].first as dead + Kokkos::parallel_for( + "RemoveParticles", + species.rangeActiveParticles(), + Lambda(index_t p) { + // check if the particle is already dead + if (tag(p) == ParticleTag::dead) { + return; + } + // select the x-coordinate index + auto x_i1 = i1(p); + // check if the particle is inside the box of new plasma + if (x_i1 > x_min[0]) { + tag(p) = ParticleTag::dead; + } + } + ); + } + + /* + Inject piston of fresh plasma + */ + + // same maxwell distribution as above + const auto energy_dist = arch::Maxwellian(domain.mesh.metric, + domain.random_pool, + temperature, + -drift_ux, + in::x1); // spatial distribution of the particles // -> hack to use the uniform distribution in NonUniformInjector const auto spatial_dist = arch::Piston(domain.mesh.metric, @@ -194,7 +299,7 @@ namespace user { box[0].second, in::x1); - // ToDo: extend Replenish to replace the current injector + // inject piston of fresh plasma const auto injector = arch::NonUniformInjector( energy_dist, spatial_dist, From 4bcb1a2ebf6ba5d100b1cc0961b2102bc305b8eb Mon Sep 17 00:00:00 2001 From: LudwigBoess Date: Mon, 31 Mar 2025 21:52:04 -0500 Subject: [PATCH 090/183] extended field reset box to the right to squash waves propagating into the empty box --- setups/srpic/shock/pgen.hpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 9568767ff..a25c3891a 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -190,17 +190,16 @@ namespace user { // check if injector is supposed to start moving already const auto dt_inj = time - injection_start > ZERO ? - time - injection_start : ZERO; + time - injection_start : ZERO; // define box to inject into boundaries_t box; - // loop over all dimension for (auto d = 0u; d < M::Dim; ++d) { if (d == 0) { box.push_back({ x_init + injector_velocity * dt_inj - drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt - - 1.5 * injection_frequency * dt, + injection_frequency * dt, x_init + injector_velocity * (dt_inj + dt) }); } else { box.push_back(Range::All); @@ -212,7 +211,9 @@ namespace user { for (auto d = 0; d < M::Dim; ++d) { incl_ghosts.push_back({ true, true }); } - const auto extent = domain.mesh.ExtentToRange(box, incl_ghosts); + auto fields_box = box; + fields_box[0].second += injection_frequency * dt; + const auto extent = domain.mesh.ExtentToRange(fields_box, incl_ghosts); tuple_t x_min { 0 }, x_max { 0 }; for (auto d = 0; d < M::Dim; ++d) { x_min[d] = extent[d].first; @@ -220,11 +221,13 @@ namespace user { } // reset fields - std::vector comps = { em::ex1, em::ex2, em::ex3, - em::bx1, em::bx2, em::bx3 }; + std::vector comps = { em::bx1, em::bx2, em::bx3, + em::ex1, em::ex2, em::ex3 }; // loop over all components for (const auto& comp : comps) { + + // get initial field value of component auto value = ResetFields((em)comp); if constexpr (M::Dim == Dim::_1D) { @@ -262,7 +265,6 @@ namespace user { auto i1 = species.i1; auto tag = species.tag; - // tag all particles with x > box[0].first as dead Kokkos::parallel_for( "RemoveParticles", From ebe60809b3ab5021dd690f97f6a94ef6bdfd0d41 Mon Sep 17 00:00:00 2001 From: hayk Date: Tue, 1 Apr 2025 13:34:59 -0400 Subject: [PATCH 091/183] contribs --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index e0d1fe016..7287d52db 100644 --- a/README.md +++ b/README.md @@ -24,10 +24,14 @@ Our [detailed documentation](https://entity-toolkit.github.io/) includes everyth ## Contributors (alphabetical) +🎸 __Ludwig Böss__ {[@LudwigBoess](https://github.com/LudwigBoess): PIC, framework} + 👀 __Yangyang Cai__ {[@StaticObserver](https://github.com/StaticObserver): GRPIC} 🍵 __Benjamin Crinquand__ {[@bcrinquand](https://github.com/bcrinquand): GRPIC, cubed-sphere} +🚂 __Evgeny Gorbunov__ {[@Alcauchy](https://github.com/Alcauchy): PIC, framework} + :radio: __Siddhant Solanki__ {[@sidruns30](https://github.com/sidruns30): framework} 🤷 __Arno Vanthieghem__ {[@vanthieg](https://github.com/vanthieg): framework, PIC} From 08de6896e6af7761a2ed31f0979cf373435825b9 Mon Sep 17 00:00:00 2001 From: hayk Date: Tue, 1 Apr 2025 13:36:29 -0400 Subject: [PATCH 092/183] rm tasks (RUNTEST) --- TASKLIST.md | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 TASKLIST.md diff --git a/TASKLIST.md b/TASKLIST.md deleted file mode 100644 index c12f60f4c..000000000 --- a/TASKLIST.md +++ /dev/null @@ -1,9 +0,0 @@ -### Performance improvements to try - -- [ ] removing temporary variables in interpolation -- [ ] passing by value vs const ref in metric -- [ ] return physical coords one-by-one instead of by passing full vector - -### Things to look into - -1. _h fields in mpi communication From 2132b3333bbc3168d31b5559b52626ead96d66b9 Mon Sep 17 00:00:00 2001 From: hayk Date: Tue, 1 Apr 2025 13:42:15 -0400 Subject: [PATCH 093/183] bulk grpic prohibit (RUNTEST) --- src/framework/domain/output.cpp | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/framework/domain/output.cpp b/src/framework/domain/output.cpp index 9cca7cf11..8fb756130 100644 --- a/src/framework/domain/output.cpp +++ b/src/framework/domain/output.cpp @@ -329,13 +329,17 @@ namespace ntt { local_domain->fields.bckp, c); } else if (fld.id() == FldsID::V) { - ComputeMoments(params, - local_domain->mesh, - local_domain->species, - fld.species, - fld.comp[0], - local_domain->fields.bckp, - c); + if constexpr (S != SimEngine::GRPIC) { + ComputeMoments(params, + local_domain->mesh, + local_domain->species, + fld.species, + fld.comp[0], + local_domain->fields.bckp, + c); + } else { + raise::Error("Bulk velocity not supported for GRPIC", HERE); + } } else { raise::Error("Wrong moment requested for output", HERE); } From 0f2c91ea3a5b7b9ed5f6b6134f830e21956c9122 Mon Sep 17 00:00:00 2001 From: hayk Date: Tue, 1 Apr 2025 13:45:30 -0400 Subject: [PATCH 094/183] minor bug in tests (RUNTEST) --- src/output/tests/writer-mpi.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/output/tests/writer-mpi.cpp b/src/output/tests/writer-mpi.cpp index 6ab16305f..0c3000f3a 100644 --- a/src/output/tests/writer-mpi.cpp +++ b/src/output/tests/writer-mpi.cpp @@ -60,7 +60,7 @@ auto main(int argc, char* argv[]) -> int { { // write auto writer = out::Writer(); - writer.init(&adios, "hdf5", "test"); + writer.init(&adios, "hdf5", "test", false); writer.defineMeshLayout({ static_cast(mpi_size) * nx1 }, { static_cast(mpi_rank) * nx1 }, { nx1 }, From a22cd9684f722a1daf83469c454c46af9a98525e Mon Sep 17 00:00:00 2001 From: hayk Date: Tue, 1 Apr 2025 13:48:10 -0400 Subject: [PATCH 095/183] minor bug in mpi-output tests (RUNTEST) --- src/output/tests/writer-mpi.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/output/tests/writer-mpi.cpp b/src/output/tests/writer-mpi.cpp index 0c3000f3a..f6d3ee88a 100644 --- a/src/output/tests/writer-mpi.cpp +++ b/src/output/tests/writer-mpi.cpp @@ -74,13 +74,13 @@ auto main(int argc, char* argv[]) -> int { field_names.push_back(writer.fieldWriters()[0].name(i)); addresses.push_back(i); } - writer.beginWriting(0, 0.0); + writer.beginWriting(WriteMode::Fields, 0, 0.0); writer.writeField(field_names, field, addresses); - writer.endWriting(); + writer.endWriting(WriteMode::Fields); - writer.beginWriting(1, 0.1); + writer.beginWriting(WriteMode::Fields, 1, 0.1); writer.writeField(field_names, field, addresses); - writer.endWriting(); + writer.endWriting(WriteMode::Fields); adios.ExitComputationBlock(); } From 420f36f86b41a8315bc200c0fadd09d4a98a857f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Tue, 1 Apr 2025 15:32:04 -0500 Subject: [PATCH 096/183] bugfix: added check to truncate the injection box at the end of the domain --- setups/srpic/shock/pgen.hpp | 48 ++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index a25c3891a..cf58b94c6 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -80,10 +80,10 @@ namespace user { using arch::ProblemGenerator::params; // gas properties - const real_t drift_ux, temperature, filling_fraction; + const real_t drift_ux, temperature, filling_fraction; // injector properties - const real_t injector_velocity, injection_start, dt; - const int injection_frequency; + const real_t injector_velocity, injection_start, dt; + const int injection_frequency; // magnetic field properties real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -189,18 +189,23 @@ namespace user { domain.mesh.extent(in::x1).first); // check if injector is supposed to start moving already - const auto dt_inj = time - injection_start > ZERO ? - time - injection_start : ZERO; + const auto dt_inj = time - injection_start > ZERO ? time - injection_start + : ZERO; + + // compute the position of the injector + auto xmax = x_init + injector_velocity * (dt_inj + dt); + if (xmax >= domain.mesh.extent(in::x1).second) { + xmax = domain.mesh.extent(in::x1).second; + } // define box to inject into boundaries_t box; // loop over all dimension for (auto d = 0u; d < M::Dim; ++d) { if (d == 0) { - box.push_back({ x_init + injector_velocity * dt_inj - - drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt - - injection_frequency * dt, - x_init + injector_velocity * (dt_inj + dt) }); + box.push_back({ xmax - drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt - + injection_frequency * dt, + xmax }); } else { box.push_back(Range::All); } @@ -212,7 +217,13 @@ namespace user { incl_ghosts.push_back({ true, true }); } auto fields_box = box; - fields_box[0].second += injection_frequency * dt; + // check if the box is still inside the domain + if (xmax + injection_frequency * dt < domain.mesh.extent(in::x1).second) { + fields_box[0].second += injection_frequency * dt; + } else { + // if right side of the box is outside of the domain -> truncate box + fields_box[0].second = domain.mesh.extent(in::x1).second; + } const auto extent = domain.mesh.ExtentToRange(fields_box, incl_ghosts); tuple_t x_min { 0 }, x_max { 0 }; for (auto d = 0; d < M::Dim; ++d) { @@ -221,7 +232,7 @@ namespace user { } // reset fields - std::vector comps = { em::bx1, em::bx2, em::bx3, + std::vector comps = { em::bx1, em::bx2, em::bx3, em::ex1, em::ex2, em::ex3 }; // loop over all components @@ -253,8 +264,8 @@ namespace user { } } - /* - tag particles inside the injection zone as dead + /* + tag particles inside the injection zone as dead */ // loop over particle species @@ -262,8 +273,8 @@ namespace user { // get particle properties auto& species = domain.species[s]; - auto i1 = species.i1; - auto tag = species.tag; + auto i1 = species.i1; + auto tag = species.tag; // tag all particles with x > box[0].first as dead Kokkos::parallel_for( @@ -277,14 +288,13 @@ namespace user { // select the x-coordinate index auto x_i1 = i1(p); // check if the particle is inside the box of new plasma - if (x_i1 > x_min[0]) { + if (x_i1 >= x_min[0]) { tag(p) = ParticleTag::dead; } - } - ); + }); } - /* + /* Inject piston of fresh plasma */ From 76867488994438bf98d4b41a2b9209d4764d9de9 Mon Sep 17 00:00:00 2001 From: hayk Date: Tue, 1 Apr 2025 17:31:28 -0400 Subject: [PATCH 097/183] Lf -> f in fmt --- src/engines/engine_printer.cpp | 8 ++++---- src/global/utils/diag.cpp | 7 +++---- src/global/utils/timer.cpp | 34 ++++++++++++++++------------------ 3 files changed, 23 insertions(+), 26 deletions(-) diff --git a/src/engines/engine_printer.cpp b/src/engines/engine_printer.cpp index c1a36a323..f94715d09 100644 --- a/src/engines/engine_printer.cpp +++ b/src/engines/engine_printer.cpp @@ -105,8 +105,8 @@ namespace ntt { color::RESET); } - auto bytes_to_human_readable( - std::size_t bytes) -> std::pair { + auto bytes_to_human_readable(std::size_t bytes) + -> std::pair { const std::vector units { "B", "KB", "MB", "GB", "TB" }; idx_t unit_idx = 0; auto size = static_cast(bytes); @@ -391,7 +391,7 @@ namespace ntt { add_subcategory(report, 6, "Memory footprint"); auto flds_footprint = domain.fields.memory_footprint(); auto [flds_size, flds_unit] = bytes_to_human_readable(flds_footprint); - add_param(report, 8, "Fields", "%.2Lf %s", flds_size, flds_unit.c_str()); + add_param(report, 8, "Fields", "%.2f %s", flds_size, flds_unit.c_str()); if (domain.species.size() > 0) { add_subcategory(report, 8, "Particles"); } @@ -400,7 +400,7 @@ namespace ntt { species.index(), species.label().c_str()); auto [size, unit] = bytes_to_human_readable(species.memory_footprint()); - add_param(report, 10, str.c_str(), "%.2Lf %s", size, unit.c_str()); + add_param(report, 10, str.c_str(), "%.2f %s", size, unit.c_str()); } report.pop_back(); if (idx == m_metadomain.ndomains() - 1) { diff --git a/src/global/utils/diag.cpp b/src/global/utils/diag.cpp index fbd0c4215..9a35e30c9 100644 --- a/src/global/utils/diag.cpp +++ b/src/global/utils/diag.cpp @@ -21,9 +21,8 @@ #include namespace diag { - auto npart_stats( - npart_t npart, - npart_t maxnpart) -> std::vector> { + auto npart_stats(npart_t npart, npart_t maxnpart) + -> std::vector> { auto stats = std::vector>(); #if !defined(MPI_ENABLED) stats.push_back( @@ -127,7 +126,7 @@ namespace diag { c_reset); ss << fmt::alignedTable( - { "Time:", fmt::format("%.4Lf", time), fmt::format("[Δt = %.4Lf]", dt) }, + { "Time:", fmt::format("%.4f", time), fmt::format("[Δt = %.4f]", dt) }, { c_reset, c_bgreen, c_bblack }, { 0, -6, -15 }, { ' ', ' ', ' ' }, diff --git a/src/global/utils/timer.cpp b/src/global/utils/timer.cpp index d3c14d05e..bc133b4b7 100644 --- a/src/global/utils/timer.cpp +++ b/src/global/utils/timer.cpp @@ -130,9 +130,8 @@ namespace timer { return timer_stats; } - auto Timers::printAll(TimerFlags flags, - npart_t npart, - ncells_t ncells) const -> std::string { + auto Timers::printAll(TimerFlags flags, npart_t npart, ncells_t ncells) const + -> std::string { const std::vector extras { "PrtlClear", "Output", "Checkpoint" }; const auto stats = gather(extras, npart, ncells); if (stats.empty()) { @@ -192,11 +191,11 @@ namespace timer { if (multi_rank) { ss << fmt::alignedTable( { name, - fmt::format("%.2Lf", time) + " " + units, + fmt::format("%.2f", time) + " " + units, std::to_string(tot_pct) + "%", std::to_string(var_pct) + "%", - fmt::format("%.2Lf", per_npart) + " " + units_npart, - fmt::format("%.2Lf", per_ncells) + " " + units_ncells }, + fmt::format("%.2f", per_npart) + " " + units_npart, + fmt::format("%.2f", per_ncells) + " " + units_ncells }, { c_reset, c_yellow, ((tot_pct > 60) ? c_red : ((tot_pct > 40) ? c_yellow : c_green)), @@ -210,10 +209,10 @@ namespace timer { } else { ss << fmt::alignedTable( { name, - fmt::format("%.2Lf", time) + " " + units, + fmt::format("%.2f", time) + " " + units, std::to_string(tot_pct) + "%", - fmt::format("%.2Lf", per_npart) + " " + units_npart, - fmt::format("%.2Lf", per_ncells) + " " + units_ncells }, + fmt::format("%.2f", per_npart) + " " + units_npart, + fmt::format("%.2f", per_ncells) + " " + units_ncells }, { c_reset, c_yellow, ((tot_pct > 60) ? c_red : ((tot_pct > 40) ? c_yellow : c_green)), @@ -237,7 +236,7 @@ namespace timer { if (multi_rank) { ss << fmt::alignedTable( { "Total", - fmt::format("%.2Lf", time) + " " + units, + fmt::format("%.2f", time) + " " + units, std::to_string(var_pct) + "%" }, { c_reset, c_blue, @@ -247,13 +246,12 @@ namespace timer { c_bblack, c_reset); } else { - ss << fmt::alignedTable( - { "Total", fmt::format("%.2Lf", time) + " " + units }, - { c_reset, c_blue }, - { 0, 37 }, - { ' ', ' ' }, - c_bblack, - c_reset); + ss << fmt::alignedTable({ "Total", fmt::format("%.2f", time) + " " + units }, + { c_reset, c_blue }, + { 0, 37 }, + { ' ', ' ' }, + c_bblack, + c_reset); } } @@ -271,7 +269,7 @@ namespace timer { convertTime(time, units); } ss << fmt::alignedTable({ name, - fmt::format("%.2Lf", time) + " " + units, + fmt::format("%.2f", time) + " " + units, std::to_string(tot_pct) + "%" }, { (active ? c_reset : c_bblack), (active ? c_byellow : c_bblack), From 4ee44eca54a4eba205d160176043f0ea16a75cbe Mon Sep 17 00:00:00 2001 From: hayk Date: Tue, 1 Apr 2025 17:38:48 -0400 Subject: [PATCH 098/183] dx_min between domains *10 acc --- src/framework/domain/metadomain.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/framework/domain/metadomain.cpp b/src/framework/domain/metadomain.cpp index dd0d6ffe7..a4ff1a905 100644 --- a/src/framework/domain/metadomain.cpp +++ b/src/framework/domain/metadomain.cpp @@ -392,7 +392,10 @@ namespace ntt { mpi::get_type(), MPI_COMM_WORLD); for (const auto& dx : dx_mins) { - raise::ErrorIf(!cmp::AlmostEqual(dx, dx_min), + raise::ErrorIf(!cmp::AlmostEqual(dx, + dx_min, + std::numeric_limits::epsilon() * + static_cast(10.0)), "dx_min is not the same across all MPI ranks", HERE); } From 18f02f64fc353752bc70d90d1d5ad31cc99e9d8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Tue, 1 Apr 2025 19:46:46 -0500 Subject: [PATCH 099/183] fix unit conversion bug in field reset --- setups/srpic/shock/pgen.hpp | 40 ++++++++----------------------------- 1 file changed, 8 insertions(+), 32 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index cf58b94c6..ad260bda0 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -9,6 +9,7 @@ #include "utils/numeric.h" #include "archetypes/energy_dist.h" +#include "archetypes/field_setter.h" #include "archetypes/particle_injector.h" #include "archetypes/problem_generator.h" #include "framework/domain/metadomain.h" @@ -231,38 +232,13 @@ namespace user { x_max[d] = extent[d].second; } - // reset fields - std::vector comps = { em::bx1, em::bx2, em::bx3, - em::ex1, em::ex2, em::ex3 }; - - // loop over all components - for (const auto& comp : comps) { - - // get initial field value of component - auto value = ResetFields((em)comp); - - if constexpr (M::Dim == Dim::_1D) { - Kokkos::deep_copy(Kokkos::subview(domain.fields.em, - std::make_pair(x_min[0], x_max[0]), - comp), - value); - } else if constexpr (M::Dim == Dim::_2D) { - Kokkos::deep_copy(Kokkos::subview(domain.fields.em, - std::make_pair(x_min[0], x_max[0]), - std::make_pair(x_min[1], x_max[1]), - comp), - value); - } else if constexpr (M::Dim == Dim::_3D) { - Kokkos::deep_copy(Kokkos::subview(domain.fields.em, - std::make_pair(x_min[0], x_max[0]), - std::make_pair(x_min[1], x_max[1]), - std::make_pair(x_min[2], x_max[2]), - comp), - value); - } else { - raise::Error("Invalid dimension", HERE); - } - } + Kokkos::parallel_for("ResetFields", + CreateRangePolicy(x_min, x_max), + arch::SetEMFields_kernel { + domain.fields.em, + init_flds, + domain.mesh.metric }); + /* tag particles inside the injection zone as dead From e96a2d38d8bace74a06d170bb3254fadc23b3c3a Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 4 Apr 2025 15:40:17 -0400 Subject: [PATCH 100/183] conductor in all directions --- src/engines/srpic.hpp | 58 ++++++-- src/kernels/fields_bcs.hpp | 288 ++++++++++++++++++++++++++++++------- 2 files changed, 290 insertions(+), 56 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 96747e429..a1228acda 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -854,9 +854,6 @@ namespace ntt { } else { const auto sign = direction.get_sign(); const auto dim = direction.get_dim(); - raise::ErrorIf(dim != in::x1, - "Perfect conductor BCs only implemented for x1", - HERE); std::vector xi_min, xi_max; @@ -866,7 +863,7 @@ namespace ntt { const auto dd = all_dirs[d]; if (dim == dd) { xi_min.push_back(0); - xi_max.push_back(N_GHOSTS + 1); + xi_max.push_back((sign < 0) ? (N_GHOSTS + 1) : N_GHOSTS); } else { xi_min.push_back(0); xi_max.push_back(domain.mesh.n_all(dd)); @@ -890,10 +887,55 @@ namespace ntt { raise::Error("Invalid dimension", HERE); } - Kokkos::parallel_for( - "ConductorFields", - range, - kernel::bc::ConductorBoundaries_kernel(domain.fields.em, tags)); + if (dim == in::x1) { + if (sign > 0) { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } else { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } + } else if (dim == in::x2) { + if (sign > 0) { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } else { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } + } else { + if (sign > 0) { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } else { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } + } } } diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index b1ee999d3..a69193d42 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -486,38 +486,52 @@ namespace kernel::bc { } }; - template + template struct ConductorBoundaries_kernel { static_assert(static_cast(o) < static_cast(D), "Invalid component index"); - ndfield_t Fld; - const BCTags tags; + ndfield_t Fld; + const BCTags tags; + const std::size_t i_edge; - ConductorBoundaries_kernel(ndfield_t Fld, BCTags tags) + ConductorBoundaries_kernel(ndfield_t Fld, std::size_t i_edge, BCTags tags) : Fld { Fld } + , i_edge { i_edge } , tags { tags } {} Inline void operator()(index_t i1) const { if constexpr (D == Dim::_1D) { if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, em::ex2) = ZERO; - Fld(N_GHOSTS, em::ex3) = ZERO; + Fld(i_edge, em::ex2) = ZERO; + Fld(i_edge, em::ex3) = ZERO; } else { - Fld(N_GHOSTS - i1, em::ex1) = Fld(N_GHOSTS + i1 - 1, em::ex1); - Fld(N_GHOSTS - i1, em::ex2) = -Fld(N_GHOSTS + i1, em::ex2); - Fld(N_GHOSTS - i1, em::ex3) = -Fld(N_GHOSTS + i1, em::ex3); + if constexpr (not P) { + Fld(i_edge - i1, em::ex1) = Fld(i_edge + i1 - 1, em::ex1); + Fld(i_edge - i1, em::ex2) = -Fld(i_edge + i1, em::ex2); + Fld(i_edge - i1, em::ex3) = -Fld(i_edge + i1, em::ex3); + } else { + Fld(i_edge + i1 - 1, em::ex1) = Fld(i_edge - i1, em::ex1); + Fld(i_edge + i1, em::ex2) = -Fld(i_edge - i1, em::ex2); + Fld(i_edge + i1, em::ex3) = -Fld(i_edge - i1, em::ex3); + } } } if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, em::bx1) = ZERO; + Fld(i_edge, em::bx1) = ZERO; } else { - Fld(N_GHOSTS - i1, em::bx1) = -Fld(N_GHOSTS + i1, em::bx1); - Fld(N_GHOSTS - i1, em::bx2) = Fld(N_GHOSTS + i1 - 1, em::bx2); - Fld(N_GHOSTS - i1, em::bx3) = Fld(N_GHOSTS + i1 - 1, em::bx3); + if constexpr (not P) { + Fld(i_edge - i1, em::bx1) = -Fld(i_edge + i1, em::bx1); + Fld(i_edge - i1, em::bx2) = Fld(i_edge + i1 - 1, em::bx2); + Fld(i_edge - i1, em::bx3) = Fld(i_edge + i1 - 1, em::bx3); + } else { + Fld(i_edge + i1, em::bx1) = -Fld(i_edge - i1, em::bx1); + Fld(i_edge + i1 - 1, em::bx2) = Fld(i_edge - i1, em::bx2); + Fld(i_edge + i1 - 1, em::bx3) = Fld(i_edge - i1, em::bx3); + } } } } else { @@ -529,24 +543,71 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2) const { if constexpr (D == Dim::_2D) { - if (tags & BC::E) { - if (i1 == 0) { - Fld(N_GHOSTS, i2, em::ex2) = ZERO; - Fld(N_GHOSTS, i2, em::ex3) = ZERO; - } else { - Fld(N_GHOSTS - i1, i2, em::ex1) = Fld(N_GHOSTS + i1 - 1, i2, em::ex1); - Fld(N_GHOSTS - i1, i2, em::ex2) = -Fld(N_GHOSTS + i1, i2, em::ex2); - Fld(N_GHOSTS - i1, i2, em::ex3) = -Fld(N_GHOSTS + i1, i2, em::ex3); + if constexpr (o == in::x1) { + if (tags & BC::E) { + if (i1 == 0) { + Fld(i_edge, i2, em::ex2) = ZERO; + Fld(i_edge, i2, em::ex3) = ZERO; + } else { + if constexpr (not P) { + Fld(i_edge - i1, i2, em::ex1) = Fld(i_edge + i1 - 1, i2, em::ex1); + Fld(i_edge - i1, i2, em::ex2) = -Fld(i_edge + i1, i2, em::ex2); + Fld(i_edge - i1, i2, em::ex3) = -Fld(i_edge + i1, i2, em::ex3); + } else { + Fld(i_edge + i1 - 1, i2, em::ex1) = Fld(i_edge - i1, i2, em::ex1); + Fld(i_edge + i1, i2, em::ex2) = -Fld(i_edge - i1, i2, em::ex2); + Fld(i_edge + i1, i2, em::ex3) = -Fld(i_edge - i1, i2, em::ex3); + } + } } - } - if (tags & BC::B) { - if (i1 == 0) { - Fld(N_GHOSTS, i2, em::bx1) = ZERO; - } else { - Fld(N_GHOSTS - i1, i2, em::bx1) = -Fld(N_GHOSTS + i1, i2, em::bx1); - Fld(N_GHOSTS - i1, i2, em::bx2) = Fld(N_GHOSTS + i1 - 1, i2, em::bx2); - Fld(N_GHOSTS - i1, i2, em::bx3) = Fld(N_GHOSTS + i1 - 1, i2, em::bx3); + if (tags & BC::B) { + if (i1 == 0) { + Fld(i_edge, i2, em::bx1) = ZERO; + } else { + if constexpr (not P) { + Fld(i_edge - i1, i2, em::bx1) = -Fld(i_edge + i1, i2, em::bx1); + Fld(i_edge - i1, i2, em::bx2) = Fld(i_edge + i1 - 1, i2, em::bx2); + Fld(i_edge - i1, i2, em::bx3) = Fld(i_edge + i1 - 1, i2, em::bx3); + } else { + Fld(i_edge + i1, i2, em::bx1) = -Fld(i_edge - i1, i2, em::bx1); + Fld(i_edge + i1 - 1, i2, em::bx2) = Fld(i_edge - i1, i2, em::bx2); + Fld(i_edge + i1 - 1, i2, em::bx3) = Fld(i_edge - i1, i2, em::bx3); + } + } + } + } else { + if (tags & BC::E) { + if (i2 == 0) { + Fld(i1, i_edge, em::ex1) = ZERO; + Fld(i1, i_edge, em::ex3) = ZERO; + } else { + if constexpr (not P) { + Fld(i1, i_edge - i2, em::ex1) = -Fld(i1, i_edge + i2, em::ex1); + Fld(i1, i_edge - i2, em::ex2) = Fld(i1, i_edge + i2 - 1, em::ex2); + Fld(i1, i_edge - i2, em::ex3) = -Fld(i1, i_edge + i2, em::ex3); + } else { + Fld(i1, i_edge + i2, em::ex1) = -Fld(i1, i_edge - i2, em::ex1); + Fld(i1, i_edge + i2 - 1, em::ex2) = Fld(i1, i_edge - i2, em::ex2); + Fld(i1, i_edge + i2, em::ex3) = -Fld(i1, i_edge - i2, em::ex3); + } + } + } + + if (tags & BC::B) { + if (i2 == 0) { + Fld(i1, i_edge, em::bx2) = ZERO; + } else { + if constexpr (not P) { + Fld(i1, i_edge - i2, em::bx1) = Fld(i1, i_edge + i2 - 1, em::bx1); + Fld(i1, i_edge - i2, em::bx2) = -Fld(i1, i_edge + i2, em::bx2); + Fld(i1, i_edge - i2, em::bx3) = Fld(i1, i_edge + i2 - 1, em::bx3); + } else { + Fld(i1, i_edge + i2 - 1, em::bx1) = Fld(i1, i_edge - i2, em::bx1); + Fld(i1, i_edge + i2, em::bx2) = -Fld(i1, i_edge - i2, em::bx2); + Fld(i1, i_edge + i2 - 1, em::bx3) = Fld(i1, i_edge - i2, em::bx3); + } + } } } } else { @@ -558,27 +619,158 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2, index_t i3) const { if constexpr (D == Dim::_3D) { - if (tags & BC::E) { - if (i1 == 0) { - Fld(N_GHOSTS, i2, i3, em::ex2) = ZERO; - Fld(N_GHOSTS, i2, i3, em::ex3) = ZERO; - } else { - Fld(N_GHOSTS - i1, i2, i3, em::ex1) = Fld(N_GHOSTS + i1 - 1, - i2, i3, em::ex1); - Fld(N_GHOSTS - i1, i2, i3, em::ex2) = -Fld(N_GHOSTS + i1, i2, i3, em::ex2); - Fld(N_GHOSTS - i1, i2, i3, em::ex3) = -Fld(N_GHOSTS + i1, i2, i3, em::ex3); + if constexpr (o == in::x1) { + if (tags & BC::E) { + if (i1 == 0) { + Fld(i_edge, i2, i3, em::ex2) = ZERO; + Fld(i_edge, i2, i3, em::ex3) = ZERO; + } else { + if constexpr (not P) { + Fld(i_edge - i1, i2, i3, em::ex1) = Fld(i_edge + i1 - 1, + i2, + i3, + em::ex1); + Fld(i_edge - i1, i2, i3, em::ex2) = -Fld(i_edge + i1, i2, i3, em::ex2); + Fld(i_edge - i1, i2, i3, em::ex3) = -Fld(i_edge + i1, i2, i3, em::ex3); + } else { + Fld(i_edge + i1 - 1, i2, i3, em::ex1) = Fld(i_edge - i1, + i2, + i3, + em::ex1); + Fld(i_edge + i1, i2, i3, em::ex2) = -Fld(i_edge - i1, i2, i3, em::ex2); + Fld(i_edge + i1, i2, i3, em::ex3) = -Fld(i_edge - i1, i2, i3, em::ex3); + } + } } - } - if (tags & BC::B) { - if (i1 == 0) { - Fld(N_GHOSTS, i2, i3, em::bx1) = ZERO; - } else { - Fld(N_GHOSTS - i1, i2, i3, em::bx1) = -Fld(N_GHOSTS + i1, i2, i3, em::bx1); - Fld(N_GHOSTS - i1, i2, i3, em::bx2) = Fld(N_GHOSTS + i1 - 1, - i2, i3, em::bx2); - Fld(N_GHOSTS - i1, i2, i3, em::bx3) = Fld(N_GHOSTS + i1 - 1, - i2, i3, em::bx3); + if (tags & BC::B) { + if (i1 == 0) { + Fld(i_edge, i2, i3, em::bx1) = ZERO; + } else { + if constexpr (not P) { + Fld(i_edge - i1, i2, i3, em::bx1) = -Fld(i_edge + i1, i2, i3, em::bx1); + Fld(i_edge - i1, i2, i3, em::bx2) = Fld(i_edge + i1 - 1, + i2, + i3, + em::bx2); + Fld(i_edge - i1, i2, i3, em::bx3) = Fld(i_edge + i1 - 1, + i2, + i3, + em::bx3); + } else { + Fld(i_edge + i1, i2, i3, em::bx1) = -Fld(i_edge - i1, i2, i3, em::bx1); + Fld(i_edge + i1 - 1, i2, i3, em::bx2) = Fld(i_edge - i1, + i2, + i3, + em::bx2); + Fld(i_edge + i1 - 1, i2, i3, em::bx3) = Fld(i_edge - i1, + i2, + i3, + em::bx3); + } + } + } + } else if (o == in::x2) { + if (tags & BC::E) { + if (i2 == 0) { + Fld(i1, i_edge, i3, em::ex1) = ZERO; + Fld(i1, i_edge, i3, em::ex3) = ZERO; + } else { + if constexpr (not P) { + Fld(i1, i_edge - i2, i3, em::ex1) = -Fld(i1, i_edge + i2, i3, em::ex1); + Fld(i1, i_edge - i2, i3, em::ex2) = Fld(i1, + i_edge + i2 - 1, + i3, + em::ex2); + Fld(i1, i_edge - i2, i3, em::ex3) = -Fld(i1, i_edge + i2, i3, em::ex3); + } else { + Fld(i1, i_edge + i2, i3, em::ex1) = -Fld(i1, i_edge - i2, i3, em::ex1); + Fld(i1, i_edge + i2 - 1, i3, em::ex2) = Fld(i1, + i_edge - i2, + i3, + em::ex2); + Fld(i1, i_edge + i2, i3, em::ex3) = -Fld(i1, i_edge - i2, i3, em::ex3); + } + } + } + + if (tags & BC::B) { + if (i2 == 0) { + Fld(i1, i_edge, i3, em::bx2) = ZERO; + } else { + if constexpr (not P) { + Fld(i1, i_edge - i2, i3, em::bx1) = Fld(i1, + i_edge + i2 - 1, + i3, + em::bx1); + Fld(i1, i_edge - i2, i3, em::bx2) = -Fld(i1, i_edge + i2, i3, em::bx2); + Fld(i1, i_edge - i2, i3, em::bx3) = Fld(i1, + i_edge + i2 - 1, + i3, + em::bx3); + } else { + Fld(i1, i_edge + i2 - 1, i3, em::bx1) = Fld(i1, + i_edge - i2, + i3, + em::bx1); + Fld(i1, i_edge + i2, i3, em::bx2) = -Fld(i1, i_edge - i2, i3, em::bx2); + Fld(i1, i_edge + i2 - 1, i3, em::bx3) = Fld(i1, + i_edge - i2, + i3, + em::bx3); + } + } + } + } else { + if (tags & BC::E) { + if (i3 == 0) { + Fld(i1, i2, i_edge, em::ex1) = ZERO; + Fld(i1, i2, i_edge, em::ex2) = ZERO; + } else { + if constexpr (not P) { + Fld(i1, i2, i_edge - i3, em::ex1) = -Fld(i1, i2, i_edge + i3, em::ex1); + Fld(i1, i2, i_edge - i3, em::ex2) = -Fld(i1, i2, i_edge + i3, em::ex2); + Fld(i1, i2, i_edge - i3, em::ex3) = Fld(i1, + i2, + i_edge + i3 - 1, + em::ex3); + } else { + Fld(i1, i2, i_edge + i3, em::ex1) = -Fld(i1, i2, i_edge - i3, em::ex1); + Fld(i1, i2, i_edge + i3, em::ex2) = -Fld(i1, i2, i_edge - i3, em::ex2); + Fld(i1, i2, i_edge + i3 - 1, em::ex3) = Fld(i1, + i2, + i_edge - i3, + em::ex3); + } + } + } + + if (tags & BC::B) { + if (i3 == 0) { + Fld(i1, i2, i_edge, em::bx3) = ZERO; + } else { + if constexpr (not P) { + Fld(i1, i2, i_edge - i3, em::bx1) = Fld(i1, + i2, + i_edge + i3 - 1, + em::bx1); + Fld(i1, i2, i_edge - i3, em::bx2) = Fld(i1, + i2, + i_edge + i3 - 1, + em::bx2); + Fld(i1, i2, i_edge - i3, em::bx3) = -Fld(i1, i2, i_edge + i3, em::bx3); + } else { + Fld(i1, i2, i_edge + i3 - 1, em::bx1) = Fld(i1, + i2, + i_edge - i3, + em::bx1); + Fld(i1, i2, i_edge + i3 - 1, em::bx2) = Fld(i1, + i2, + i_edge - i3, + em::bx2); + Fld(i1, i2, i_edge + i3, em::bx3) = -Fld(i1, i2, i_edge - i3, em::bx3); + } + } } } } else { From af420129db2c35dd87b9e3ad0ea3a5249a305495 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Feb 2025 10:43:05 -0600 Subject: [PATCH 101/183] prep for conductor boundaries --- src/global/arch/traits.h | 12 ++++++++++++ src/global/enums.h | 7 ++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/global/arch/traits.h b/src/global/arch/traits.h index 4cde4fca5..65cc63cf8 100644 --- a/src/global/arch/traits.h +++ b/src/global/arch/traits.h @@ -112,6 +112,18 @@ namespace traits { template using fix_fields_const_t = decltype(&T::FixFieldsConst); + template + using perfect_conductor_fields_t = decltype(&T::PerfectConductorFields); + + template + using perfect_conductor_fields_const_t = decltype(&T::PerfectConductorFieldsConst); + + template + using perfect_conductor_currents_t = decltype(&T::PerfectConductorCurrents); + + template + using perfect_conductor_currents_const_t = decltype(&T::PerfectConductorCurrentsConst); + template using custom_fields_t = decltype(&T::CustomFields); diff --git a/src/global/enums.h b/src/global/enums.h index 3afa0497a..8407d0b66 100644 --- a/src/global/enums.h +++ b/src/global/enums.h @@ -222,15 +222,16 @@ namespace ntt { HORIZON = 6, AXIS = 7, SYNC = 8, // <- SYNC means synchronization with other domains + CONDUCTOR = 9 }; constexpr FldsBC(uint8_t c) : enums_hidden::BaseEnum { c } {} - static constexpr type variants[] = { PERIODIC, MATCH, FIXED, ATMOSPHERE, - CUSTOM, HORIZON, AXIS, SYNC }; + static constexpr type variants[] = { PERIODIC, MATCH, FIXED, ATMOSPHERE, + CUSTOM, HORIZON, AXIS, SYNC, CONDUCTOR }; static constexpr const char* lookup[] = { "periodic", "match", "fixed", "atmosphere", "custom", "horizon", - "axis", "sync" }; + "axis", "sync", "conductor"}; static constexpr std::size_t total = sizeof(variants) / sizeof(variants[0]); }; From 7c6ddf46887ee75b87a291a4444c07a06da8691e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Feb 2025 10:43:32 -0600 Subject: [PATCH 102/183] first stubborn attempt at conductor boundaries (broken) --- setups/srpic/shock/pgen.hpp | 46 +++++++++++++ src/engines/srpic.hpp | 133 ++++++++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index b8f169521..59c5590c9 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -106,6 +106,52 @@ namespace user { } } + + auto PerfectConductorFieldsConst(const bc_in&, const em& comp) const -> real_t{ + + // electric field components + if (comp == em::ex1) { + return ONE; + } else if (comp == em::ex2) { + return -ONE; + } else if (comp == em::ex3) { + return -ONE; } + // magentic field components + else if (comp == em::bx1) { + return -ONE; + } else if (comp == em::bx2) { + return ONE; + } else if (comp == em::bx3) { + return ONE;} + // should never be the case + else + { + return ZERO; + } + } + + // auto PerfectConductorCurrentsConst(const bc_in &, const cur &comp) const + // -> std::pair + // { + // // ToDo + // if (comp == cur::jx1) + // { + // return ZERO; + // } + // else if (comp == cur::jx2) + // { + // return ZERO; + // } + // else if (comp == cur::jx3) + // { + // return ZERO; + // } + // else + // { + // return ZERO; + // } + // } + auto MatchFields(real_t time) const -> InitFields { return init_flds; } diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 0a9cc311b..c5b48310e 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -596,6 +596,10 @@ namespace ntt { if (domain.mesh.flds_bc_in(direction) == FldsBC::FIXED) { FixedFieldsIn(direction, domain, tags); } + } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::CONDUCTOR) { + if (domain.mesh.flds_bc_in(direction) == FldsBC::CONDUCTOR) { + PerfectConductorFieldsIn(direction, domain, tags); + } } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::CUSTOM) { if (domain.mesh.flds_bc_in(direction) == FldsBC::CUSTOM) { CustomFieldsIn(direction, domain, tags); @@ -834,6 +838,135 @@ namespace ntt { } } + void PerfectConductorFieldsIn(dir::direction_t direction, + domain_t& domain, + BCTags tags) { + /** + * perfect conductor field boundaries + */ + const auto sign = direction.get_sign(); + const auto dim = direction.get_dim(); + raise::ErrorIf(dim != in::x1 and M::CoordType != Coord::Cart, + "Perfect conductor BCs only implemented for x1 in " + "non-cartesian coordinates", + HERE); + + // magnetic and electron field components + em normal_b_comp, tang_b_comp1, tang_b_comp2, + normal_e_comp, tang_e_comp1, tang_e_comp2; + + // current components + // cur normal_j_comp, tang_j_comp1, tang_j_comp2; + + if (dim == in::x1) { + normal_b_comp = em::bx1; + tang_b_comp1 = em::bx2; + tang_b_comp2 = em::bx3; + + normal_e_comp = em::ex1; + tang_e_comp1 = em::ex2; + tang_e_comp2 = em::ex3; + } else if (dim == in::x2) { + normal_b_comp = em::bx2; + tang_b_comp1 = em::bx1; + tang_b_comp2 = em::bx3; + + normal_e_comp = em::ex2; + tang_e_comp1 = em::ex1; + tang_e_comp2 = em::ex3; + } else if (dim == in::x3) { + normal_b_comp = em::bx3; + tang_b_comp1 = em::bx1; + tang_b_comp2 = em::bx2; + + normal_e_comp = em::ex3; + tang_e_comp1 = em::ex1; + tang_e_comp2 = em::ex2; + } else { + raise::Error("Invalid dimension", HERE); + } + + std::vector origin_xi_min, origin_xi_max, + target_xi_min, target_xi_max; + const std::vector all_dirs { in::x1, in::x2, in::x3 }; + + for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { + const auto dd = all_dirs[d]; + if (dim == dd) { + // origin: right side of boundary + origin_xi_min.push_back(N_GHOSTS+1); + origin_xi_max.push_back(2*N_GHOSTS); + // target: left side of boundary + target_xi_min.push_back(0); + target_xi_max.push_back(N_GHOSTS); + + } else { + origin_xi_min.push_back(0); + origin_xi_max.push_back(domain.mesh.n_all(dd)); + + target_xi_min.push_back(0); + target_xi_max.push_back(domain.mesh.n_all(dd)); + } + } + raise::ErrorIf(target_xi_min.size() != origin_xi_min.size() or + origin_xi_min.size() != static_cast(M::Dim), + "Invalid range size", + HERE); + + std::vector comps; + if (tags & BC::E) { + comps.push_back(normal_e_comp); + comps.push_back(tang_e_comp1); + comps.push_back(tang_e_comp2); + } + if (tags & BC::B) { + comps.push_back(normal_b_comp); + comps.push_back(tang_b_comp1); + comps.push_back(tang_b_comp2); + } + + // ToDo: smarter loop/views + auto EB = domain.fields.em; + + // loop over all components + for (const auto& comp : comps) { + + // store sign of component behind boundary + auto new_sign = m_pgen.PerfectConductorFieldsConst( + (bc_in)(sign * ((short)dim + 1)), + (em)comp); + // to do: Kokkos::parallel_for + for (int i = 0; i < N_GHOSTS; i++) + { + if constexpr (M::Dim == Dim::_1D) { + // multiply with correct sign + EB(target_xi_min[0]+i, comp) = new_sign * EB(origin_xi_max[0]-i, comp); + + } else if constexpr (M::Dim == Dim::_2D) { + for (int j = 0; j < domain.mesh.n_all(in::x2); j++) + { + EB(target_xi_min[0]+i, j, comp) = + new_sign * EB(origin_xi_max[0]-i, j, comp); + } + } else if constexpr (M::Dim == Dim::_3D) { + for (int j = 0; j < domain.mesh.n_all(in::x2); j++) + { + for (int k = 0; k < domain.mesh.n_all(in::x3); k++) + { + EB(target_xi_min[0]+i, j, k, comp) = + new_sign * EB(origin_xi_max[0]-i, j, k, comp); + } + } + } else { + raise::Error("Invalid dimension", HERE); + } + } + + // ToDo: set zero at boundary + } + } + + void AtmosphereFieldsIn(dir::direction_t direction, domain_t& domain, BCTags tags) { From 9333b76b33cc08dbcc5427331ec37b202bda963f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Feb 2025 16:41:20 -0600 Subject: [PATCH 103/183] first attempt at ConductorBoundaries_kernel --- src/kernels/fields_bcs.hpp | 107 +++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index dbb47f42c..cadc9cfda 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -485,6 +485,113 @@ namespace kernel::bc { } }; + template + struct ConductorBoundaries_kernel { + static_assert(M::is_metric, "M must be a metric class"); + static_assert(static_cast(o) < + static_cast(M::Dim), + "Invalid component index"); + static constexpr idx_t i = static_cast(o) + 1u; + + ndfield_t Fld; + const I fset; + const M metric; + const BCTags tags; + + ConductorBoundaries_kernel(ndfield_t Fld, + BCTags tags) + : Fld { Fld } + , tags { tags } {} + + Inline void operator()(index_t i1) const { + if constexpr (M::Dim == Dim::_1D) { + + if constexpr (S == SimEngine::SRPIC) { + + if (tags & BC::E) { + Fld((N_GHOSTS-1)-i1, em::ex1) = Fld(N_GHOSTS+i1, em::ex1); + Fld((N_GHOSTS-1)-i1, em::ex2) = -Fld(N_GHOSTS+i1, em::ex2); + Fld((N_GHOSTS-1)-i1, em::ex3) = -Fld(N_GHOSTS+i1, em::ex3); + } + + if (tags & BC::B) + { + Fld((N_GHOSTS-1)-i1, em::bx1) = -Fld(N_GHOSTS+i1, em::bx1); + Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); + Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); + } + + } else { + // GRPIC + raise::KernelError(HERE, "1D GRPIC not implemented"); + } + } else { + raise::KernelError( + HERE, + "ConductorBoundaries_kernel: 1D implementation called for D != 1"); + } + } + + Inline void operator()(index_t i1, index_t i2) const { + if constexpr (M::Dim == Dim::_2D) { + + if constexpr (S == SimEngine::SRPIC) { + // SRPIC + if (tags & BC::E) { + Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); + Fld((N_GHOSTS-1)-i1, i2, em::ex2) = -Fld(N_GHOSTS+i1, i2, em::ex2); + Fld((N_GHOSTS-1)-i1, i2, em::ex3) = -Fld(N_GHOSTS+i1, i2, em::ex3); + } + + if (tags & BC::B) + { + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = -Fld(N_GHOSTS+i1, i2, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); + } + } else { + // GRPIC + raise::KernelError(HERE, "2D GRPIC not implemented"); + } + } else { + raise::KernelError( + HERE, + "ConductorBoundaries_kernel: 2D implementation called for D != 2"); + } + } + + Inline void operator()(index_t i1, index_t i2, index_t i3) const { + if constexpr (M::Dim == Dim::_3D) { + const auto i1_ = COORD(i1); + const auto i2_ = COORD(i2); + const auto i3_ = COORD(i3); + + if constexpr (S == SimEngine::SRPIC) { + // SRPIC + if (tags & BC::E) { + Fld((N_GHOSTS-1)-i1, i2, i3, em::ex1) = Fld(N_GHOSTS+i1, i2, i3, em::ex1); + Fld((N_GHOSTS-1)-i1, i2, i3, em::ex2) = -Fld(N_GHOSTS+i1, i2, i3, em::ex2); + Fld((N_GHOSTS-1)-i1, i2, i3, em::ex3) = -Fld(N_GHOSTS+i1, i2, i3, em::ex3); + } + + if (tags & BC::B) + { + Fld((N_GHOSTS-1)-i1, i2, i3, em::bx1) = -Fld(N_GHOSTS+i1, i2, i3, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, i3, em::bx2) = Fld(N_GHOSTS+i1, i2, i3, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, i3, em::bx3) = Fld(N_GHOSTS+i1, i2, i3, em::bx3); + } + } else { + // GRPIC + raise::KernelError(HERE, "3D GRPIC not implemented"); + } + } else { + raise::KernelError( + HERE, + "ConductorBoundaries_kernel: 3D implementation called for D != 3"); + } + } + }; + /* * @tparam D: Dimension * @tparam P: Positive/Negative direction From 156507077a91473816ba5b3503c2f4b09bd20a15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Feb 2025 16:55:21 -0600 Subject: [PATCH 104/183] bugfix --- src/kernels/fields_bcs.hpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index cadc9cfda..95c32d894 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -485,7 +485,7 @@ namespace kernel::bc { } }; - template + template struct ConductorBoundaries_kernel { static_assert(M::is_metric, "M must be a metric class"); static_assert(static_cast(o) < @@ -494,7 +494,6 @@ namespace kernel::bc { static constexpr idx_t i = static_cast(o) + 1u; ndfield_t Fld; - const I fset; const M metric; const BCTags tags; @@ -562,13 +561,10 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2, index_t i3) const { if constexpr (M::Dim == Dim::_3D) { - const auto i1_ = COORD(i1); - const auto i2_ = COORD(i2); - const auto i3_ = COORD(i3); if constexpr (S == SimEngine::SRPIC) { // SRPIC - if (tags & BC::E) { + if (tags & BC::E) { Fld((N_GHOSTS-1)-i1, i2, i3, em::ex1) = Fld(N_GHOSTS+i1, i2, i3, em::ex1); Fld((N_GHOSTS-1)-i1, i2, i3, em::ex2) = -Fld(N_GHOSTS+i1, i2, i3, em::ex2); Fld((N_GHOSTS-1)-i1, i2, i3, em::ex3) = -Fld(N_GHOSTS+i1, i2, i3, em::ex3); From 6a136fbda06f66448bd2e29791cb46ca18b98948 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Feb 2025 17:02:04 -0600 Subject: [PATCH 105/183] first attempt at Kokkos::parallel_for loop (broken) --- src/engines/srpic.hpp | 117 ++++++------------------------------------ 1 file changed, 17 insertions(+), 100 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index c5b48310e..0a2ae360b 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -851,122 +851,39 @@ namespace ntt { "non-cartesian coordinates", HERE); - // magnetic and electron field components - em normal_b_comp, tang_b_comp1, tang_b_comp2, - normal_e_comp, tang_e_comp1, tang_e_comp2; - // current components - // cur normal_j_comp, tang_j_comp1, tang_j_comp2; + std::vector xi_min, xi_max; - if (dim == in::x1) { - normal_b_comp = em::bx1; - tang_b_comp1 = em::bx2; - tang_b_comp2 = em::bx3; - - normal_e_comp = em::ex1; - tang_e_comp1 = em::ex2; - tang_e_comp2 = em::ex3; - } else if (dim == in::x2) { - normal_b_comp = em::bx2; - tang_b_comp1 = em::bx1; - tang_b_comp2 = em::bx3; - - normal_e_comp = em::ex2; - tang_e_comp1 = em::ex1; - tang_e_comp2 = em::ex3; - } else if (dim == in::x3) { - normal_b_comp = em::bx3; - tang_b_comp1 = em::bx1; - tang_b_comp2 = em::bx2; - - normal_e_comp = em::ex3; - tang_e_comp1 = em::ex1; - tang_e_comp2 = em::ex2; - } else { - raise::Error("Invalid dimension", HERE); - } - - std::vector origin_xi_min, origin_xi_max, - target_xi_min, target_xi_max; const std::vector all_dirs { in::x1, in::x2, in::x3 }; for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { const auto dd = all_dirs[d]; if (dim == dd) { - // origin: right side of boundary - origin_xi_min.push_back(N_GHOSTS+1); - origin_xi_max.push_back(2*N_GHOSTS); - // target: left side of boundary - target_xi_min.push_back(0); - target_xi_max.push_back(N_GHOSTS); - + xi_min.push_back(0); + xi_max.push_back(N_GHOSTS); } else { - origin_xi_min.push_back(0); - origin_xi_max.push_back(domain.mesh.n_all(dd)); - - target_xi_min.push_back(0); - target_xi_max.push_back(domain.mesh.n_all(dd)); + xi_min.push_back(0); + xi_max.push_back(domain.mesh.n_all(dd)); } } - raise::ErrorIf(target_xi_min.size() != origin_xi_min.size() or - origin_xi_min.size() != static_cast(M::Dim), + raise::ErrorIf(xi_min.size() != xi_max.size() or + xi_min.size() != static_cast(M::Dim), "Invalid range size", HERE); - std::vector comps; - if (tags & BC::E) { - comps.push_back(normal_e_comp); - comps.push_back(tang_e_comp1); - comps.push_back(tang_e_comp2); - } - if (tags & BC::B) { - comps.push_back(normal_b_comp); - comps.push_back(tang_b_comp1); - comps.push_back(tang_b_comp2); - } - - // ToDo: smarter loop/views - auto EB = domain.fields.em; - - // loop over all components - for (const auto& comp : comps) { - - // store sign of component behind boundary - auto new_sign = m_pgen.PerfectConductorFieldsConst( - (bc_in)(sign * ((short)dim + 1)), - (em)comp); - // to do: Kokkos::parallel_for - for (int i = 0; i < N_GHOSTS; i++) - { - if constexpr (M::Dim == Dim::_1D) { - // multiply with correct sign - EB(target_xi_min[0]+i, comp) = new_sign * EB(origin_xi_max[0]-i, comp); - - } else if constexpr (M::Dim == Dim::_2D) { - for (int j = 0; j < domain.mesh.n_all(in::x2); j++) - { - EB(target_xi_min[0]+i, j, comp) = - new_sign * EB(origin_xi_max[0]-i, j, comp); - } - } else if constexpr (M::Dim == Dim::_3D) { - for (int j = 0; j < domain.mesh.n_all(in::x2); j++) - { - for (int k = 0; k < domain.mesh.n_all(in::x3); k++) - { - EB(target_xi_min[0]+i, j, k, comp) = - new_sign * EB(origin_xi_max[0]-i, j, k, comp); - } - } - } else { - raise::Error("Invalid dimension", HERE); - } - } - - // ToDo: set zero at boundary + if (dim == in::x1) + { + Kokkos::parallel_for( + "MatchFields", + CreateRangePolicy(xi_min, xi_max), + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); } + + } - void AtmosphereFieldsIn(dir::direction_t direction, domain_t& domain, BCTags tags) { From 861e78345702dd606f9a16161ed78306fd5834ea Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 21 Feb 2025 22:12:56 -0500 Subject: [PATCH 106/183] Ongoing. --- src/engines/srpic.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 0a2ae360b..a317aa7ea 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -871,11 +871,11 @@ namespace ntt { "Invalid range size", HERE); - if (dim == in::x1) + if constexpr (M::Dim == Dim::_1D) { { Kokkos::parallel_for( "MatchFields", - CreateRangePolicy(xi_min, xi_max), + CreateRangePolicy(xi_min[0], xi_max[0]), kernel::bc::ConductorBoundaries_kernel( domain.fields.em, tags)); From cd81a5b467bee1ea8379cf90764a78af51ec3006 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 21 Feb 2025 22:16:54 -0500 Subject: [PATCH 107/183] Ongoing. --- src/engines/srpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index a317aa7ea..9d015aa7e 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -871,7 +871,7 @@ namespace ntt { "Invalid range size", HERE); - if constexpr (M::Dim == Dim::_1D) { + if constexpr (M::Dim == Dim::_1D) { Kokkos::parallel_for( "MatchFields", From aec0d414a13ee09f92afc6c701782a265b8c8faf Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 21 Feb 2025 22:19:01 -0500 Subject: [PATCH 108/183] Ongoing. --- src/engines/srpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 9d015aa7e..c997f026a 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -875,7 +875,7 @@ namespace ntt { { Kokkos::parallel_for( "MatchFields", - CreateRangePolicy(xi_min[0], xi_max[0]), + CreateRangePolicy( { xi_min[0] } , { xi_max[0] } ), kernel::bc::ConductorBoundaries_kernel( domain.fields.em, tags)); From 1c7105bb47d3dbcc9eb66d9ea30920d7627fde40 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 21 Feb 2025 22:35:06 -0500 Subject: [PATCH 109/183] Ongoing. --- src/engines/srpic.hpp | 2 +- src/kernels/fields_bcs.hpp | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index c997f026a..4bc16d273 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -876,7 +876,7 @@ namespace ntt { Kokkos::parallel_for( "MatchFields", CreateRangePolicy( { xi_min[0] } , { xi_max[0] } ), - kernel::bc::ConductorBoundaries_kernel( + kernel::bc::ConductorBoundaries_kernel( domain.fields.em, tags)); } diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 95c32d894..bb851b45f 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -494,7 +494,6 @@ namespace kernel::bc { static constexpr idx_t i = static_cast(o) + 1u; ndfield_t Fld; - const M metric; const BCTags tags; ConductorBoundaries_kernel(ndfield_t Fld, From de66f25dc33795fcf7412bce377176e2863d5898 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 21 Feb 2025 22:35:54 -0500 Subject: [PATCH 110/183] Ongoing. --- src/engines/srpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 4bc16d273..c997f026a 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -876,7 +876,7 @@ namespace ntt { Kokkos::parallel_for( "MatchFields", CreateRangePolicy( { xi_min[0] } , { xi_max[0] } ), - kernel::bc::ConductorBoundaries_kernel( + kernel::bc::ConductorBoundaries_kernel( domain.fields.em, tags)); } From 2f8a01e374665c89d89b7e8deea6e8b235deac32 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 11:16:58 -0500 Subject: [PATCH 111/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 179 ++++++++++++++++++++++++++++++ setups/srpic/shocktest/shock.py | 75 +++++++++++++ setups/srpic/shocktest/shock.toml | 56 ++++++++++ 3 files changed, 310 insertions(+) create mode 100644 setups/srpic/shocktest/pgen.hpp create mode 100644 setups/srpic/shocktest/shock.py create mode 100644 setups/srpic/shocktest/shock.toml diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp new file mode 100644 index 000000000..83f19df52 --- /dev/null +++ b/setups/srpic/shocktest/pgen.hpp @@ -0,0 +1,179 @@ +#ifndef PROBLEM_GENERATOR_H +#define PROBLEM_GENERATOR_H + +#include "enums.h" +#include "global.h" + +#include "arch/traits.h" +#include "utils/error.h" +#include "utils/numeric.h" + +#include "archetypes/energy_dist.h" +#include "archetypes/particle_injector.h" +#include "archetypes/problem_generator.h" +#include "framework/domain/metadomain.h" + +#include + +namespace user { + using namespace ntt; + + template + struct InitFields { + /* + Sets up magnetic and electric field components for the simulation. + Must satisfy E = -v x B for Lorentz Force to be zero. + + @param bmag: magnetic field scaling + @param btheta: magnetic field polar angle + @param bphi: magnetic field azimuthal angle + @param drift_ux: drift velocity in the x direction + */ + InitFields(real_t bmag, real_t btheta, real_t bphi, real_t drift_ux) + : Bmag { bmag } + , Btheta { btheta * static_cast(convert::deg2rad) } + , Bphi { bphi * static_cast(convert::deg2rad) } + , Vx { drift_ux } {} + + // magnetic field components + Inline auto bx1(const coord_t& x_Ph) const -> real_t { + return x_Ph[0]; + } + + Inline auto bx2(const coord_t&) const -> real_t { + return x_Ph[0]; + } + + Inline auto bx3(const coord_t&) const -> real_t { + return x_Ph[0]; + } + + // electric field components + Inline auto ex1(const coord_t&) const -> real_t { + return x_Ph[0]; + } + + Inline auto ex2(const coord_t&) const -> real_t { + return x_Ph[0]; + } + + Inline auto ex3(const coord_t&) const -> real_t { + return x_Ph[0]; + } + + private: + const real_t Btheta, Bphi, Vx, Bmag; + }; + + template + struct PGen : public arch::ProblemGenerator { + // compatibility traits for the problem generator + static constexpr auto engines { traits::compatible_with::value }; + static constexpr auto metrics { traits::compatible_with::value }; + static constexpr auto dimensions { + traits::compatible_with::value + }; + + // for easy access to variables in the child class + using arch::ProblemGenerator::D; + using arch::ProblemGenerator::C; + using arch::ProblemGenerator::params; + + const real_t drift_ux, temperature; + + const real_t Btheta, Bphi, Bmag; + InitFields init_flds; + + inline PGen(const SimulationParams& p, const Metadomain& m) + : arch::ProblemGenerator { p } + , drift_ux { p.template get("setup.drift_ux") } + , temperature { p.template get("setup.temperature") } + , Bmag { p.template get("setup.Bmag", ZERO) } + , Btheta { p.template get("setup.Btheta", ZERO) } + , Bphi { p.template get("setup.Bphi", ZERO) } + , init_flds { Bmag, Btheta, Bphi, drift_ux } {} + + inline PGen() {} + + auto FixFieldsConst(const bc_in&, const em& comp) const + -> std::pair { + if (comp == em::ex2) { + return { init_flds.ex2({ ZERO }), true }; + } else if (comp == em::ex3) { + return { init_flds.ex3({ ZERO }), true }; + } else { + return { ZERO, false }; + } + } + + + auto PerfectConductorFieldsConst(const bc_in&, const em& comp) const -> real_t{ + + // electric field components + if (comp == em::ex1) { + return ONE; + } else if (comp == em::ex2) { + return -ONE; + } else if (comp == em::ex3) { + return -ONE; } + // magentic field components + else if (comp == em::bx1) { + return -ONE; + } else if (comp == em::bx2) { + return ONE; + } else if (comp == em::bx3) { + return ONE;} + // should never be the case + else + { + return ZERO; + } + } + + // auto PerfectConductorCurrentsConst(const bc_in &, const cur &comp) const + // -> std::pair + // { + // // ToDo + // if (comp == cur::jx1) + // { + // return ZERO; + // } + // else if (comp == cur::jx2) + // { + // return ZERO; + // } + // else if (comp == cur::jx3) + // { + // return ZERO; + // } + // else + // { + // return ZERO; + // } + // } + + auto MatchFields(real_t time) const -> InitFields { + return init_flds; + } + + inline void InitPrtls(Domain& local_domain) { + const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, + local_domain.random_pool, + temperature, + -drift_ux, + in::x1); + + const auto injector = arch::UniformInjector( + energy_dist, + { 1, 2 }); + // arch::InjectUniform>( + // params, + // local_domain, + // injector, + // 1.0); + } + }; + +} // namespace user + +#endif diff --git a/setups/srpic/shocktest/shock.py b/setups/srpic/shocktest/shock.py new file mode 100644 index 000000000..dc1565572 --- /dev/null +++ b/setups/srpic/shocktest/shock.py @@ -0,0 +1,75 @@ +import nt2.read as nt2r +import matplotlib.pyplot as plt +import matplotlib as mpl + +data = nt2r.Data("shock.h5") + + +def frame(ti, f): + quantities = [ + { + "name": "density", + "compute": lambda f: f.N_2 + f.N_1, + "cmap": "inferno", + "norm": mpl.colors.Normalize(0, 5), + }, + { + "name": r"$E_x$", + "compute": lambda f: f.Ex, + "cmap": "RdBu_r", + "norm": mpl.colors.Normalize(-0.05, 0.05), + }, + { + "name": r"$E_y$", + "compute": lambda f: f.Ey, + "cmap": "RdBu_r", + "norm": mpl.colors.Normalize(-0.05, 0.05), + }, + { + "name": r"$E_z$", + "compute": lambda f: f.Ez, + "cmap": "RdBu_r", + "norm": mpl.colors.Normalize(-0.05, 0.05), + }, + { + "name": r"$B_x$", + "compute": lambda f: f.Bx, + "cmap": "BrBG", + "norm": mpl.colors.Normalize(-0.05, 0.05), + }, + { + "name": r"$B_y$", + "compute": lambda f: f.By, + "cmap": "BrBG", + "norm": mpl.colors.Normalize(-0.05, 0.05), + }, + { + "name": r"$B_z$", + "compute": lambda f: f.Bz, + "cmap": "BrBG", + "norm": mpl.colors.Normalize(-0.05, 0.05), + }, + ] + fig = plt.figure(figsize=(12, 5.5), dpi=300) + gs = fig.add_gridspec(len(quantities), 1, hspace=0.02) + axs = [fig.add_subplot(gs[i]) for i in range(len(quantities))] + + for ax, q in zip(axs, quantities): + q["compute"](f.isel(t=ti)).plot( + ax=ax, + cmap=q["cmap"], + norm=q["norm"], + cbar_kwargs={"label": q["name"], "shrink": 0.8, "aspect": 10, "pad": 0.005}, + ) + for i, ax in enumerate(axs): + ax.set(aspect=1) + if i != 0: + ax.set(title=None) + if i != len(axs) - 1: + ax.set( + xticks=[], + xticklabels=[], + xlabel=None, + title=ax.get_title().split(",")[0], + ) + return fig diff --git a/setups/srpic/shocktest/shock.toml b/setups/srpic/shocktest/shock.toml new file mode 100644 index 000000000..fdbc44465 --- /dev/null +++ b/setups/srpic/shocktest/shock.toml @@ -0,0 +1,56 @@ +[simulation] + name = "shock" + engine = "srpic" + runtime = 50.0 + +[grid] + resolution = [2048, 128] + extent = [[0.0, 10.0], [-0.3125, 0.3125]] + + [grid.metric] + metric = "minkowski" + + [grid.boundaries] + fields = [["CONDUCTOR", "FIXED"], ["PERIODIC"]] + particles = [["REFLECT", "ABSORB"], ["PERIODIC"]] + +[scales] + larmor0 = 1e-2 + skindepth0 = 1e-2 + +[algorithms] + current_filters = 8 + fieldsolver = "false" + deposit = "false" + + [algorithms.timestep] + CFL = 0.5 + +[particles] + ppc0 = 16.0 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = -1.0 + maxnpart = 1e8 + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = 1.0 + maxnpart = 1e8 + +[setup] + drift_ux = 0.1 + temperature = 1e-3 + Bmag = 1.0 + Btheta = 0.0 + Bphi = 0.0 + +[output] + interval = 1 + format = "hdf5" + + [output.fields] + quantities = ["N_1", "N_2", "E", "B", "T0i_1", "T0i_2", "J"] From 2c7019b58b1d717e4a126a75382274be29aa63a4 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 11:20:28 -0500 Subject: [PATCH 112/183] Ongoing. --- setups/srpic/shocktest/shock.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setups/srpic/shocktest/shock.toml b/setups/srpic/shocktest/shock.toml index fdbc44465..a77f5c2e9 100644 --- a/setups/srpic/shocktest/shock.toml +++ b/setups/srpic/shocktest/shock.toml @@ -54,3 +54,6 @@ [output.fields] quantities = ["N_1", "N_2", "E", "B", "T0i_1", "T0i_2", "J"] + + [output.debug] + ghosts = true From c0414671c6986b015f589c5832dd83fa7432a1c0 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 14:01:06 -0500 Subject: [PATCH 113/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 83f19df52..7724893a1 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -40,24 +40,24 @@ namespace user { return x_Ph[0]; } - Inline auto bx2(const coord_t&) const -> real_t { + Inline auto bx2(const coord_t& x_Ph) const -> real_t { return x_Ph[0]; } - Inline auto bx3(const coord_t&) const -> real_t { + Inline auto bx3(const coord_t& x_Ph) const -> real_t { return x_Ph[0]; } // electric field components - Inline auto ex1(const coord_t&) const -> real_t { + Inline auto ex1(const coord_t& x_Ph) const -> real_t { return x_Ph[0]; } - Inline auto ex2(const coord_t&) const -> real_t { + Inline auto ex2(const coord_t& x_Ph) const -> real_t { return x_Ph[0]; } - Inline auto ex3(const coord_t&) const -> real_t { + Inline auto ex3(const coord_t& x_Ph) const -> real_t { return x_Ph[0]; } From b85346ebc97ff7a5f59e17248187e8df71704528 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 17:15:38 -0500 Subject: [PATCH 114/183] Ongoing. --- src/kernels/fields_bcs.hpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index bb851b45f..e8993907c 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,9 +543,12 @@ namespace kernel::bc { if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = -Fld(N_GHOSTS+i1, i2, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); + // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = -Fld(N_GHOSTS+i1, i2, em::bx1); + // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); + // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; } } else { // GRPIC From b51a8bb69e885b2281e1b563254fd293289a7e8a Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 17:21:07 -0500 Subject: [PATCH 115/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 7724893a1..0dc91d3e2 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -41,7 +41,7 @@ namespace user { } Inline auto bx2(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return -x_Ph[0]; } Inline auto bx3(const coord_t& x_Ph) const -> real_t { From f94e0fb79c268a20bfdba1c5d4161142ba5ce17a Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 18:50:58 -0500 Subject: [PATCH 116/183] Ongoing. --- src/engines/srpic.hpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index c997f026a..86b85de32 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -880,6 +880,16 @@ namespace ntt { domain.fields.em, tags)); } + + if constexpr (M::Dim == Dim::_2D) + { + Kokkos::parallel_for( + "MatchFields", + CreateRangePolicy( { xi_min[0], xi_min[1] } , { xi_max[0], xi_max[1] } ), + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } } From 7536825ef6942ff41e85c01abfe9ce7db19c1947 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 19:05:19 -0500 Subject: [PATCH 117/183] Ongoing. --- src/kernels/fields_bcs.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index e8993907c..c85115f12 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,12 +543,12 @@ namespace kernel::bc { if (tags & BC::B) { - // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = -Fld(N_GHOSTS+i1, i2, em::bx1); - // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); - // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = Fld(N_GHOSTS+i1, i2, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); + // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; + // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; + // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; } } else { // GRPIC From f313624b9171651d79eb888435106edfba028000 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 19:10:18 -0500 Subject: [PATCH 118/183] Ongoing. --- src/kernels/fields_bcs.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index c85115f12..da6e88dbd 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,9 +543,9 @@ namespace kernel::bc { if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = Fld(N_GHOSTS+i1, i2, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; From dd6056edef8e71ed274ea78879028d03a07ec4d7 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 20:01:27 -0500 Subject: [PATCH 119/183] Ongoing. --- src/kernels/fields_bcs.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index da6e88dbd..58e5af4b4 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,9 +543,9 @@ namespace kernel::bc { if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); + Fld((N_GHOSTS)-i1, i2, em::bx1) = - Fld(N_GHOSTS+1+i1, i2, em::bx1); + Fld((N_GHOSTS)-i1, i2, em::bx2) = - Fld(N_GHOSTS+1+i1, i2, em::bx2); + Fld((N_GHOSTS)-i1, i2, em::bx3) = - Fld(N_GHOSTS+1+i1, i2, em::bx3); // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; From 67a6a191ec16043c14e5079da2942b2f431e446b Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 20:54:37 -0500 Subject: [PATCH 120/183] Ongoing. --- src/kernels/fields_bcs.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 58e5af4b4..da6e88dbd 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,9 +543,9 @@ namespace kernel::bc { if (tags & BC::B) { - Fld((N_GHOSTS)-i1, i2, em::bx1) = - Fld(N_GHOSTS+1+i1, i2, em::bx1); - Fld((N_GHOSTS)-i1, i2, em::bx2) = - Fld(N_GHOSTS+1+i1, i2, em::bx2); - Fld((N_GHOSTS)-i1, i2, em::bx3) = - Fld(N_GHOSTS+1+i1, i2, em::bx3); + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; From 679d0374d52ea2a1d56cffb4fe8d72514ed1c4de Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 21:02:54 -0500 Subject: [PATCH 121/183] Ongoing. --- src/kernels/fields_bcs.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index da6e88dbd..31cc55268 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,12 +543,12 @@ namespace kernel::bc { if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); - // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; - // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; - // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; + // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); + // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); + // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; } } else { // GRPIC From 80043ef77450790525d694ffe204b826fabf3f0a Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 21:13:50 -0500 Subject: [PATCH 122/183] Ongoing. --- src/kernels/fields_bcs.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 31cc55268..0c6d0782c 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,12 +543,12 @@ namespace kernel::bc { if (tags & BC::B) { - // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); - // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); - // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+1+i1, i2, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); + // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; + // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; + // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; } } else { // GRPIC From aff570eec4c46872be891aa87a02d972d0235973 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 21:43:02 -0500 Subject: [PATCH 123/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 0dc91d3e2..685521f09 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -45,7 +45,7 @@ namespace user { } Inline auto bx3(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return ONE; } // electric field components From 6d4eb13bed081dd157c0527e65ef3461572f7ec9 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 22:06:25 -0500 Subject: [PATCH 124/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 685521f09..0dc91d3e2 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -45,7 +45,7 @@ namespace user { } Inline auto bx3(const coord_t& x_Ph) const -> real_t { - return ONE; + return x_Ph[0]; } // electric field components From c3aa545c58fb323651a03999298131fd28fc94b0 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 22:16:17 -0500 Subject: [PATCH 125/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 0dc91d3e2..f8bf67dab 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -37,7 +37,7 @@ namespace user { // magnetic field components Inline auto bx1(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return ZERO; } Inline auto bx2(const coord_t& x_Ph) const -> real_t { @@ -50,15 +50,15 @@ namespace user { // electric field components Inline auto ex1(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return ZERO; } Inline auto ex2(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return ZERO; } Inline auto ex3(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return ZERO; } private: From f5d9bec2182ded70cee21ec9db214f74c2f75951 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 22:34:56 -0500 Subject: [PATCH 126/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index f8bf67dab..39ce8ea03 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -45,7 +45,7 @@ namespace user { } Inline auto bx3(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return -x_Ph[0]; } // electric field components From 2c2d595947601bba212ea37432f911ff304089e2 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 22:58:19 -0500 Subject: [PATCH 127/183] Ongoing. --- src/kernels/fields_bcs.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 0c6d0782c..7f9a054a5 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -536,9 +536,9 @@ namespace kernel::bc { if constexpr (S == SimEngine::SRPIC) { // SRPIC if (tags & BC::E) { - Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); - Fld((N_GHOSTS-1)-i1, i2, em::ex2) = -Fld(N_GHOSTS+i1, i2, em::ex2); - Fld((N_GHOSTS-1)-i1, i2, em::ex3) = -Fld(N_GHOSTS+i1, i2, em::ex3); + Fld((N_GHOSTS-1)-i1, i2, em::ex1) = - Fld(N_GHOSTS+i1, i2, em::ex1); + Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+1+i1, i2, em::ex2); + Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+1+i1, i2, em::ex3); } if (tags & BC::B) From 54b02223075e7bb01204c31e76967e96a9aed182 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Tue, 25 Feb 2025 23:51:36 -0500 Subject: [PATCH 128/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 39ce8ea03..2cb66539f 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -36,16 +36,16 @@ namespace user { , Vx { drift_ux } {} // magnetic field components - Inline auto bx1(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto bx1(const coord_t&) const -> real_t { + return Bmag * math::cos(Btheta); } - Inline auto bx2(const coord_t& x_Ph) const -> real_t { - return -x_Ph[0]; + Inline auto bx2(const coord_t&) const -> real_t { + return Bmag * math::sin(Btheta) * math::sin(Bphi); } - Inline auto bx3(const coord_t& x_Ph) const -> real_t { - return -x_Ph[0]; + Inline auto bx3(const coord_t&) const -> real_t { + return Bmag * math::sin(Btheta) * math::cos(Bphi); } // electric field components From e023e37ff798d0838d09d343b9fde596e1bd2056 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Tue, 25 Feb 2025 23:59:44 -0500 Subject: [PATCH 129/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 2cb66539f..b77583ccb 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -49,16 +49,16 @@ namespace user { } // electric field components - Inline auto ex1(const coord_t& x_Ph) const -> real_t { + Inline auto ex1(const coord_t&) const -> real_t { return ZERO; } - Inline auto ex2(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto ex2(const coord_t&) const -> real_t { + return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); } - Inline auto ex3(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto ex3(const coord_t&) const -> real_t { + return Vx * Bmag * math::sin(Btheta) * math::sin(Bphi); } private: From 42549067c3744a0c43aef401c5c09f821ed0b344 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Thu, 27 Feb 2025 11:42:46 -0500 Subject: [PATCH 130/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index b77583ccb..f92086528 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -36,29 +36,34 @@ namespace user { , Vx { drift_ux } {} // magnetic field components - Inline auto bx1(const coord_t&) const -> real_t { - return Bmag * math::cos(Btheta); + Inline auto bx1(const coord_t& x_ph) const -> real_t { + // return Bmag * math::cos(Btheta); + return ZERO; } - Inline auto bx2(const coord_t&) const -> real_t { - return Bmag * math::sin(Btheta) * math::sin(Bphi); + Inline auto bx2(const coord_t& x_ph) const -> real_t { + // return Bmag * math::sin(Btheta) * math::sin(Bphi); + return ZERO; } - Inline auto bx3(const coord_t&) const -> real_t { - return Bmag * math::sin(Btheta) * math::cos(Bphi); + Inline auto bx3(const coord_t& x_ph) const -> real_t { + // return Bmag * math::sin(Btheta) * math::cos(Bphi); + return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x))*math::tanh(20.*(-0.5 + x)))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); } // electric field components - Inline auto ex1(const coord_t&) const -> real_t { + Inline auto ex1(const coord_t& x_ph) const -> real_t { return ZERO; } - Inline auto ex2(const coord_t&) const -> real_t { - return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); + Inline auto ex2(const coord_t& x_ph) const -> real_t { + // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); + return ZERO + 0.01 * (ONE - math::tanh(20.*(-1.5 + x))*math::tanh(20.*(-0.5 + x)))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); } - Inline auto ex3(const coord_t&) const -> real_t { - return Vx * Bmag * math::sin(Btheta) * math::sin(Bphi); + Inline auto ex3(const coord_t& x_ph) const -> real_t { + // return Vx * Bmag * math::sin(Btheta) * math::sin(Bphi); + return ZERO; } private: From 20ae9a4f73aef85be0c8ce588d17cb1e562ed8e3 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Thu, 27 Feb 2025 12:01:42 -0500 Subject: [PATCH 131/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index f92086528..c43688d02 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -48,7 +48,7 @@ namespace user { Inline auto bx3(const coord_t& x_ph) const -> real_t { // return Bmag * math::sin(Btheta) * math::cos(Bphi); - return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x))*math::tanh(20.*(-0.5 + x)))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); } // electric field components @@ -58,7 +58,7 @@ namespace user { Inline auto ex2(const coord_t& x_ph) const -> real_t { // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - return ZERO + 0.01 * (ONE - math::tanh(20.*(-1.5 + x))*math::tanh(20.*(-0.5 + x)))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ZERO + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); } Inline auto ex3(const coord_t& x_ph) const -> real_t { From 9facba71628d0b8295b06710ac8d6519c606652d Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Thu, 27 Feb 2025 13:13:12 -0500 Subject: [PATCH 132/183] Ongoing. --- src/kernels/fields_bcs.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 7f9a054a5..77497114c 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -536,7 +536,7 @@ namespace kernel::bc { if constexpr (S == SimEngine::SRPIC) { // SRPIC if (tags & BC::E) { - Fld((N_GHOSTS-1)-i1, i2, em::ex1) = - Fld(N_GHOSTS+i1, i2, em::ex1); + Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+1+i1, i2, em::ex2); Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+1+i1, i2, em::ex3); } @@ -544,8 +544,8 @@ namespace kernel::bc { if (tags & BC::B) { Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+1+i1, i2, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; From 8f0e72c1bacd22ba022645bd8de1052ed6c8af92 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Thu, 27 Feb 2025 13:18:43 -0500 Subject: [PATCH 133/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index c43688d02..10d4c6b59 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -58,7 +58,7 @@ namespace user { Inline auto ex2(const coord_t& x_ph) const -> real_t { // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - return ZERO + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); } Inline auto ex3(const coord_t& x_ph) const -> real_t { From cedf47a613ee4eb2e5edb31ccdbba9d36c54cad5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Thu, 27 Feb 2025 18:04:22 -0600 Subject: [PATCH 134/183] first attempt at single particle injection --- setups/srpic/shocktest/pgen.hpp | 164 ++++++++++++++++++++------------ src/kernels/fields_bcs.hpp | 27 +++--- 2 files changed, 116 insertions(+), 75 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 10d4c6b59..4d4b1cb37 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -111,72 +111,116 @@ namespace user { } } - - auto PerfectConductorFieldsConst(const bc_in&, const em& comp) const -> real_t{ - - // electric field components - if (comp == em::ex1) { - return ONE; - } else if (comp == em::ex2) { - return -ONE; - } else if (comp == em::ex3) { - return -ONE; } - // magentic field components - else if (comp == em::bx1) { - return -ONE; - } else if (comp == em::bx2) { - return ONE; - } else if (comp == em::bx3) { - return ONE;} - // should never be the case - else - { - return ZERO; - } - } - - // auto PerfectConductorCurrentsConst(const bc_in &, const cur &comp) const - // -> std::pair - // { - // // ToDo - // if (comp == cur::jx1) - // { - // return ZERO; - // } - // else if (comp == cur::jx2) - // { - // return ZERO; - // } - // else if (comp == cur::jx3) - // { - // return ZERO; - // } - // else - // { - // return ZERO; - // } - // } auto MatchFields(real_t time) const -> InitFields { return init_flds; } - inline void InitPrtls(Domain& local_domain) { - const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, - local_domain.random_pool, - temperature, - -drift_ux, - in::x1); - - const auto injector = arch::UniformInjector( - energy_dist, - { 1, 2 }); - // arch::InjectUniform>( - // params, - // local_domain, - // injector, - // 1.0); + inline void InitPrtls(Domain& domain) { + + auto& species_e = domain.species[0]; + auto& species_p = domain.species[1]; + auto metric = domain.mesh.metric; + auto m = domain.mesh.metric; + + array_t elec_ind("elec_ind"); + array_t pos_ind("pos_ind"); + + auto offset_e = species_e.npart(); + auto offset_p = species_p.npart(); + + auto ux1_e = species_e.ux1; + auto ux2_e = species_e.ux2; + auto ux3_e = species_e.ux3; + auto i1_e = species_e.i1; + auto i2_e = species_e.i2; + auto dx1_e = species_e.dx1; + auto dx2_e = species_e.dx2; + auto phi_e = species_e.phi; + auto weight_e = species_e.weight; + auto tag_e = species_e.tag; + + auto ux1_p = species_p.ux1; + auto ux2_p = species_p.ux2; + auto ux3_p = species_p.ux3; + auto i1_p = species_p.i1; + auto i2_p = species_p.i2; + auto dx1_p = species_p.dx1; + auto dx2_p = species_p.dx2; + auto phi_p = species_p.phi; + auto weight_p = species_p.weight; + auto tag_p = species_p.tag; + + int nseed = 1; + auto dseed = HALF * constant::PI / static_cast(nseed); + + Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { + + // ToDo: fix this + auto i1_ = ONE; + auto i2_ = ONE; + auto dx1_ = ONE - HALF; + auto dx2_ = ONE - HALF; + + + auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); + auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); + + i1_e(elec_p + offset_e) = i1_; + dx1_e(elec_p + offset_e) = dx1_; + i2_e(elec_p + offset_e) = i2_; + dx2_e(elec_p + offset_e) = dx2_; + phi_e(elec_p + offset_e) = ZERO; + ux1_e(elec_p + offset_e) = -drift_ux; + ux2_e(elec_p + offset_e) = -drift_ux; + ux3_e(elec_p + offset_e) = ZERO; + weight_e(elec_p + offset_e) = ONE; + tag_e(elec_p + offset_e) = ParticleTag::alive; + + i1_p(pos_p + offset_p) = i1_; + dx1_p(pos_p + offset_p) = dx1_; + i2_p(pos_p + offset_p) = i2_; + dx2_p(pos_p + offset_p) = dx2_; + phi_p(pos_p + offset_p) = ZERO; + ux1_p(pos_p + offset_p) = -drift_ux; + ux2_p(pos_p + offset_p) = drift_ux; + ux3_p(pos_p + offset_p) = ZERO; + weight_p(pos_p + offset_p) = ONE; + tag_p(pos_p + offset_p) = ParticleTag::alive; + + + }); + + + auto elec_ind_h = Kokkos::create_mirror(elec_ind); + Kokkos::deep_copy(elec_ind_h, elec_ind); + species_e.set_npart(offset_e + elec_ind_h()); + + auto pos_ind_h = Kokkos::create_mirror(pos_ind); + Kokkos::deep_copy(pos_ind_h, pos_ind); + species_p.set_npart(offset_p + pos_ind_h()); + + } + + + // inline void InitPrtls(Domain& local_domain) { + // const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, + // local_domain.random_pool, + // temperature, + // -drift_ux, + // in::x1); + + // const auto injector = arch::UniformInjector( + // energy_dist, + // { 1, 2 }); + // arch::InjectUniform>( + // params, + // local_domain, + // injector, + // 1.0); + // } + }; } // namespace user diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 77497114c..e3fe4f2ba 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -506,19 +506,19 @@ namespace kernel::bc { if constexpr (S == SimEngine::SRPIC) { - if (tags & BC::E) { - Fld((N_GHOSTS-1)-i1, em::ex1) = Fld(N_GHOSTS+i1, em::ex1); - Fld((N_GHOSTS-1)-i1, em::ex2) = -Fld(N_GHOSTS+i1, em::ex2); - Fld((N_GHOSTS-1)-i1, em::ex3) = -Fld(N_GHOSTS+i1, em::ex3); - } - - if (tags & BC::B) - { - Fld((N_GHOSTS-1)-i1, em::bx1) = -Fld(N_GHOSTS+i1, em::bx1); - Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); - Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); - } + if (tags & BC::E) + { + Fld((N_GHOSTS - 1) - i1, em::ex1) = Fld(N_GHOSTS + i1, em::ex1); + Fld((N_GHOSTS - 1) - i1, em::ex2) = -Fld(N_GHOSTS + 1 + i1, em::ex2); + Fld((N_GHOSTS - 1) - i1, em::ex3) = -Fld(N_GHOSTS + 1 + i1, em::ex3); + } + if (tags & BC::B) + { + Fld((N_GHOSTS - 1) - i1, em::bx1) = -Fld(N_GHOSTS + 1 + i1, em::bx1); + Fld((N_GHOSTS - 1) - i1, em::bx2) = Fld(N_GHOSTS + i1, em::bx2); + Fld((N_GHOSTS - 1) - i1, em::bx3) = Fld(N_GHOSTS + i1, em::bx3); + } } else { // GRPIC raise::KernelError(HERE, "1D GRPIC not implemented"); @@ -546,9 +546,6 @@ namespace kernel::bc { Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+1+i1, i2, em::bx1); Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); - // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; - // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; - // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; } } else { // GRPIC From 77dc646323348d720c9e8348c458aba78808b58e Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 07:44:22 -0500 Subject: [PATCH 135/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 4d4b1cb37..14c2006e9 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -48,7 +48,8 @@ namespace user { Inline auto bx3(const coord_t& x_ph) const -> real_t { // return Bmag * math::sin(Btheta) * math::cos(Bphi); - return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + // return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ZERO; } // electric field components @@ -58,7 +59,8 @@ namespace user { Inline auto ex2(const coord_t& x_ph) const -> real_t { // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + // return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ZERO; } Inline auto ex3(const coord_t& x_ph) const -> real_t { @@ -120,8 +122,6 @@ namespace user { auto& species_e = domain.species[0]; auto& species_p = domain.species[1]; - auto metric = domain.mesh.metric; - auto m = domain.mesh.metric; array_t elec_ind("elec_ind"); array_t pos_ind("pos_ind"); @@ -152,15 +152,14 @@ namespace user { auto tag_p = species_p.tag; int nseed = 1; - auto dseed = HALF * constant::PI / static_cast(nseed); Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { // ToDo: fix this auto i1_ = ONE; auto i2_ = ONE; - auto dx1_ = ONE - HALF; - auto dx2_ = ONE - HALF; + auto dx1_ = HALF; + auto dx2_ = HALF; auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); From afc0fe976718d89ca5e8b7dbbfd630410564ebe1 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 07:47:32 -0500 Subject: [PATCH 136/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 14c2006e9..85d08214a 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -156,8 +156,8 @@ namespace user { Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { // ToDo: fix this - auto i1_ = ONE; - auto i2_ = ONE; + auto i1_ = 100; + auto i2_ = 100; auto dx1_ = HALF; auto dx2_ = HALF; From c6a01f8546df922a91afccd662075e704a2436e8 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 09:42:48 -0500 Subject: [PATCH 137/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 90 ++++++++++++++++----------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 85d08214a..b682cd621 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -153,51 +153,51 @@ namespace user { int nseed = 1; - Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { - - // ToDo: fix this - auto i1_ = 100; - auto i2_ = 100; - auto dx1_ = HALF; - auto dx2_ = HALF; - - - auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); - auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); - - i1_e(elec_p + offset_e) = i1_; - dx1_e(elec_p + offset_e) = dx1_; - i2_e(elec_p + offset_e) = i2_; - dx2_e(elec_p + offset_e) = dx2_; - phi_e(elec_p + offset_e) = ZERO; - ux1_e(elec_p + offset_e) = -drift_ux; - ux2_e(elec_p + offset_e) = -drift_ux; - ux3_e(elec_p + offset_e) = ZERO; - weight_e(elec_p + offset_e) = ONE; - tag_e(elec_p + offset_e) = ParticleTag::alive; - - i1_p(pos_p + offset_p) = i1_; - dx1_p(pos_p + offset_p) = dx1_; - i2_p(pos_p + offset_p) = i2_; - dx2_p(pos_p + offset_p) = dx2_; - phi_p(pos_p + offset_p) = ZERO; - ux1_p(pos_p + offset_p) = -drift_ux; - ux2_p(pos_p + offset_p) = drift_ux; - ux3_p(pos_p + offset_p) = ZERO; - weight_p(pos_p + offset_p) = ONE; - tag_p(pos_p + offset_p) = ParticleTag::alive; - - - }); - - - auto elec_ind_h = Kokkos::create_mirror(elec_ind); - Kokkos::deep_copy(elec_ind_h, elec_ind); - species_e.set_npart(offset_e + elec_ind_h()); - - auto pos_ind_h = Kokkos::create_mirror(pos_ind); - Kokkos::deep_copy(pos_ind_h, pos_ind); - species_p.set_npart(offset_p + pos_ind_h()); + // Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { + + // // ToDo: fix this + // auto i1_ = 100; + // auto i2_ = 100; + // auto dx1_ = HALF; + // auto dx2_ = HALF; + + + // auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); + // auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); + + // i1_e(elec_p + offset_e) = i1_; + // dx1_e(elec_p + offset_e) = dx1_; + // i2_e(elec_p + offset_e) = i2_; + // dx2_e(elec_p + offset_e) = dx2_; + // phi_e(elec_p + offset_e) = ZERO; + // ux1_e(elec_p + offset_e) = -drift_ux; + // ux2_e(elec_p + offset_e) = -drift_ux; + // ux3_e(elec_p + offset_e) = ZERO; + // weight_e(elec_p + offset_e) = ONE; + // tag_e(elec_p + offset_e) = ParticleTag::alive; + + // i1_p(pos_p + offset_p) = i1_; + // dx1_p(pos_p + offset_p) = dx1_; + // i2_p(pos_p + offset_p) = i2_; + // dx2_p(pos_p + offset_p) = dx2_; + // phi_p(pos_p + offset_p) = ZERO; + // ux1_p(pos_p + offset_p) = -drift_ux; + // ux2_p(pos_p + offset_p) = drift_ux; + // ux3_p(pos_p + offset_p) = ZERO; + // weight_p(pos_p + offset_p) = ONE; + // tag_p(pos_p + offset_p) = ParticleTag::alive; + + + // }); + + + // auto elec_ind_h = Kokkos::create_mirror(elec_ind); + // Kokkos::deep_copy(elec_ind_h, elec_ind); + // species_e.set_npart(offset_e + elec_ind_h()); + + // auto pos_ind_h = Kokkos::create_mirror(pos_ind); + // Kokkos::deep_copy(pos_ind_h, pos_ind); + // species_p.set_npart(offset_p + pos_ind_h()); } From 4f3d261f6bb7e40544f97bddc6753059ddeac54b Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 09:51:22 -0500 Subject: [PATCH 138/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 70 ++++++++++++++++----------------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index b682cd621..f74e45080 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -151,53 +151,51 @@ namespace user { auto weight_p = species_p.weight; auto tag_p = species_p.tag; - int nseed = 1; + int nseed = 10; - // Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { + Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { - // // ToDo: fix this - // auto i1_ = 100; - // auto i2_ = 100; - // auto dx1_ = HALF; - // auto dx2_ = HALF; + // ToDo: fix this + auto i1_ = math::floor(100); + auto i2_ = math::floor(64); + auto dx1_ = HALF; + auto dx2_ = HALF; - // auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); - // auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); + auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); + auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); - // i1_e(elec_p + offset_e) = i1_; - // dx1_e(elec_p + offset_e) = dx1_; - // i2_e(elec_p + offset_e) = i2_; - // dx2_e(elec_p + offset_e) = dx2_; - // phi_e(elec_p + offset_e) = ZERO; - // ux1_e(elec_p + offset_e) = -drift_ux; - // ux2_e(elec_p + offset_e) = -drift_ux; - // ux3_e(elec_p + offset_e) = ZERO; - // weight_e(elec_p + offset_e) = ONE; - // tag_e(elec_p + offset_e) = ParticleTag::alive; + i1_e(elec_p + offset_e) = i1_; + dx1_e(elec_p + offset_e) = dx1_; + i2_e(elec_p + offset_e) = i2_; + dx2_e(elec_p + offset_e) = dx2_; + ux1_e(elec_p + offset_e) = ZERO; + ux2_e(elec_p + offset_e) = ZERO; + ux3_e(elec_p + offset_e) = ZERO; + weight_e(elec_p + offset_e) = ONE; + tag_e(elec_p + offset_e) = ParticleTag::alive; - // i1_p(pos_p + offset_p) = i1_; - // dx1_p(pos_p + offset_p) = dx1_; - // i2_p(pos_p + offset_p) = i2_; - // dx2_p(pos_p + offset_p) = dx2_; - // phi_p(pos_p + offset_p) = ZERO; - // ux1_p(pos_p + offset_p) = -drift_ux; - // ux2_p(pos_p + offset_p) = drift_ux; - // ux3_p(pos_p + offset_p) = ZERO; - // weight_p(pos_p + offset_p) = ONE; - // tag_p(pos_p + offset_p) = ParticleTag::alive; + i1_p(pos_p + offset_p) = i1_; + dx1_p(pos_p + offset_p) = dx1_; + i2_p(pos_p + offset_p) = i2_; + dx2_p(pos_p + offset_p) = dx2_; + ux1_p(pos_p + offset_p) = ZERO; + ux2_p(pos_p + offset_p) = ZERO; + ux3_p(pos_p + offset_p) = ZERO; + weight_p(pos_p + offset_p) = ONE; + tag_p(pos_p + offset_p) = ParticleTag::alive; - // }); + }); - // auto elec_ind_h = Kokkos::create_mirror(elec_ind); - // Kokkos::deep_copy(elec_ind_h, elec_ind); - // species_e.set_npart(offset_e + elec_ind_h()); + auto elec_ind_h = Kokkos::create_mirror(elec_ind); + Kokkos::deep_copy(elec_ind_h, elec_ind); + species_e.set_npart(offset_e + elec_ind_h()); - // auto pos_ind_h = Kokkos::create_mirror(pos_ind); - // Kokkos::deep_copy(pos_ind_h, pos_ind); - // species_p.set_npart(offset_p + pos_ind_h()); + auto pos_ind_h = Kokkos::create_mirror(pos_ind); + Kokkos::deep_copy(pos_ind_h, pos_ind); + species_p.set_npart(offset_p + pos_ind_h()); } From bec35fd8c64cbb39a78252410646a566ec9a5118 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 09:56:29 -0500 Subject: [PATCH 139/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index f74e45080..a835b7990 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -151,7 +151,7 @@ namespace user { auto weight_p = species_p.weight; auto tag_p = species_p.tag; - int nseed = 10; + int nseed = 1; Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { @@ -169,8 +169,8 @@ namespace user { dx1_e(elec_p + offset_e) = dx1_; i2_e(elec_p + offset_e) = i2_; dx2_e(elec_p + offset_e) = dx2_; - ux1_e(elec_p + offset_e) = ZERO; - ux2_e(elec_p + offset_e) = ZERO; + ux1_e(elec_p + offset_e) = -drift_ux; + ux2_e(elec_p + offset_e) = drift_ux; ux3_e(elec_p + offset_e) = ZERO; weight_e(elec_p + offset_e) = ONE; tag_e(elec_p + offset_e) = ParticleTag::alive; @@ -179,8 +179,8 @@ namespace user { dx1_p(pos_p + offset_p) = dx1_; i2_p(pos_p + offset_p) = i2_; dx2_p(pos_p + offset_p) = dx2_; - ux1_p(pos_p + offset_p) = ZERO; - ux2_p(pos_p + offset_p) = ZERO; + ux1_p(pos_p + offset_p) = -drift_ux; + ux2_p(pos_p + offset_p) = drift_ux; ux3_p(pos_p + offset_p) = ZERO; weight_p(pos_p + offset_p) = ONE; tag_p(pos_p + offset_p) = ParticleTag::alive; From 400feb03cd2482abae63bf20ba3fe00916ab553b Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 09:58:00 -0500 Subject: [PATCH 140/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index a835b7990..9711646e3 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -169,8 +169,8 @@ namespace user { dx1_e(elec_p + offset_e) = dx1_; i2_e(elec_p + offset_e) = i2_; dx2_e(elec_p + offset_e) = dx2_; - ux1_e(elec_p + offset_e) = -drift_ux; - ux2_e(elec_p + offset_e) = drift_ux; + ux1_e(elec_p + offset_e) = -0.5; + ux2_e(elec_p + offset_e) = 0.5; ux3_e(elec_p + offset_e) = ZERO; weight_e(elec_p + offset_e) = ONE; tag_e(elec_p + offset_e) = ParticleTag::alive; @@ -179,8 +179,8 @@ namespace user { dx1_p(pos_p + offset_p) = dx1_; i2_p(pos_p + offset_p) = i2_; dx2_p(pos_p + offset_p) = dx2_; - ux1_p(pos_p + offset_p) = -drift_ux; - ux2_p(pos_p + offset_p) = drift_ux; + ux1_p(pos_p + offset_p) = -0.5; + ux2_p(pos_p + offset_p) = 0.5; ux3_p(pos_p + offset_p) = ZERO; weight_p(pos_p + offset_p) = ONE; tag_p(pos_p + offset_p) = ParticleTag::alive; From 906d2766a8a21e24b438967151f1f2115f701a7c Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 11:05:00 -0500 Subject: [PATCH 141/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 9711646e3..4cf89ac69 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -156,7 +156,7 @@ namespace user { Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { // ToDo: fix this - auto i1_ = math::floor(100); + auto i1_ = math::floor(10); auto i2_ = math::floor(64); auto dx1_ = HALF; auto dx2_ = HALF; From 8bf2718953707cab4241aa152cc1b1734beb9659 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 11:18:31 -0500 Subject: [PATCH 142/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 4cf89ac69..a1ce9a1a9 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -179,8 +179,8 @@ namespace user { dx1_p(pos_p + offset_p) = dx1_; i2_p(pos_p + offset_p) = i2_; dx2_p(pos_p + offset_p) = dx2_; - ux1_p(pos_p + offset_p) = -0.5; - ux2_p(pos_p + offset_p) = 0.5; + ux1_p(pos_p + offset_p) = 0.5; + ux2_p(pos_p + offset_p) = -0.5; ux3_p(pos_p + offset_p) = ZERO; weight_p(pos_p + offset_p) = ONE; tag_p(pos_p + offset_p) = ParticleTag::alive; From 5534983abaf90b99e7f076ac7ad80815e78cd67f Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 13:20:21 -0500 Subject: [PATCH 143/183] Ongoing. --- src/kernels/fields_bcs.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index e3fe4f2ba..6f157c4aa 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -537,13 +537,13 @@ namespace kernel::bc { // SRPIC if (tags & BC::E) { Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); - Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+1+i1, i2, em::ex2); - Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+1+i1, i2, em::ex3); + Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+i1, i2, em::ex2); + Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+i1, i2, em::ex3); } if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+1+i1, i2, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); } From 1a60a7aa64c26fd14984788c68268b7ca9ce46d0 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 13:34:59 -0500 Subject: [PATCH 144/183] Ongoing. --- src/kernels/fields_bcs.hpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 6f157c4aa..12dbe45ed 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -536,16 +536,19 @@ namespace kernel::bc { if constexpr (S == SimEngine::SRPIC) { // SRPIC if (tags & BC::E) { - Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); - Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+i1, i2, em::ex2); - Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+i1, i2, em::ex3); + // Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); + // Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+1+i1, i2, em::ex2); + // Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+1+i1, i2, em::ex3); + Fld((N_GHOSTS-1)-i1, i2, em::ex1) = ZERO; + Fld((N_GHOSTS-1)-i1, i2, em::ex2) = ZERO; + Fld((N_GHOSTS-1)-i1, i2, em::ex3) = ZERO; } if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = ZERO; + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = ZERO; + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = ZERO; } } else { // GRPIC From d50abcdbef8bcf9949aa5c7564745f21f6e231e6 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 13:36:55 -0500 Subject: [PATCH 145/183] Ongoing. --- src/kernels/fields_bcs.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 12dbe45ed..def4668c6 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -542,6 +542,8 @@ namespace kernel::bc { Fld((N_GHOSTS-1)-i1, i2, em::ex1) = ZERO; Fld((N_GHOSTS-1)-i1, i2, em::ex2) = ZERO; Fld((N_GHOSTS-1)-i1, i2, em::ex3) = ZERO; + Fld((N_GHOSTS), i2, em::ex2) = ZERO; + Fld((N_GHOSTS), i2, em::ex3) = ZERO; } if (tags & BC::B) From fbdfb920b2df113f5c380ad55f31e30bf4d21d93 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 14:53:48 -0500 Subject: [PATCH 146/183] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 156 ++++++++++++++++---------------- 1 file changed, 78 insertions(+), 78 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index a1ce9a1a9..1b8926c67 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -48,8 +48,8 @@ namespace user { Inline auto bx3(const coord_t& x_ph) const -> real_t { // return Bmag * math::sin(Btheta) * math::cos(Bphi); - // return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); - return ZERO; + return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + // return ZERO; } // electric field components @@ -59,8 +59,8 @@ namespace user { Inline auto ex2(const coord_t& x_ph) const -> real_t { // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - // return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); - return ZERO; + return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + // return ZERO; } Inline auto ex3(const coord_t& x_ph) const -> real_t { @@ -120,82 +120,82 @@ namespace user { inline void InitPrtls(Domain& domain) { - auto& species_e = domain.species[0]; - auto& species_p = domain.species[1]; + // auto& species_e = domain.species[0]; + // auto& species_p = domain.species[1]; - array_t elec_ind("elec_ind"); - array_t pos_ind("pos_ind"); + // array_t elec_ind("elec_ind"); + // array_t pos_ind("pos_ind"); - auto offset_e = species_e.npart(); - auto offset_p = species_p.npart(); - - auto ux1_e = species_e.ux1; - auto ux2_e = species_e.ux2; - auto ux3_e = species_e.ux3; - auto i1_e = species_e.i1; - auto i2_e = species_e.i2; - auto dx1_e = species_e.dx1; - auto dx2_e = species_e.dx2; - auto phi_e = species_e.phi; - auto weight_e = species_e.weight; - auto tag_e = species_e.tag; - - auto ux1_p = species_p.ux1; - auto ux2_p = species_p.ux2; - auto ux3_p = species_p.ux3; - auto i1_p = species_p.i1; - auto i2_p = species_p.i2; - auto dx1_p = species_p.dx1; - auto dx2_p = species_p.dx2; - auto phi_p = species_p.phi; - auto weight_p = species_p.weight; - auto tag_p = species_p.tag; - - int nseed = 1; - - Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { - - // ToDo: fix this - auto i1_ = math::floor(10); - auto i2_ = math::floor(64); - auto dx1_ = HALF; - auto dx2_ = HALF; - - - auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); - auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); - - i1_e(elec_p + offset_e) = i1_; - dx1_e(elec_p + offset_e) = dx1_; - i2_e(elec_p + offset_e) = i2_; - dx2_e(elec_p + offset_e) = dx2_; - ux1_e(elec_p + offset_e) = -0.5; - ux2_e(elec_p + offset_e) = 0.5; - ux3_e(elec_p + offset_e) = ZERO; - weight_e(elec_p + offset_e) = ONE; - tag_e(elec_p + offset_e) = ParticleTag::alive; - - i1_p(pos_p + offset_p) = i1_; - dx1_p(pos_p + offset_p) = dx1_; - i2_p(pos_p + offset_p) = i2_; - dx2_p(pos_p + offset_p) = dx2_; - ux1_p(pos_p + offset_p) = 0.5; - ux2_p(pos_p + offset_p) = -0.5; - ux3_p(pos_p + offset_p) = ZERO; - weight_p(pos_p + offset_p) = ONE; - tag_p(pos_p + offset_p) = ParticleTag::alive; - - - }); - - - auto elec_ind_h = Kokkos::create_mirror(elec_ind); - Kokkos::deep_copy(elec_ind_h, elec_ind); - species_e.set_npart(offset_e + elec_ind_h()); - - auto pos_ind_h = Kokkos::create_mirror(pos_ind); - Kokkos::deep_copy(pos_ind_h, pos_ind); - species_p.set_npart(offset_p + pos_ind_h()); + // auto offset_e = species_e.npart(); + // auto offset_p = species_p.npart(); + + // auto ux1_e = species_e.ux1; + // auto ux2_e = species_e.ux2; + // auto ux3_e = species_e.ux3; + // auto i1_e = species_e.i1; + // auto i2_e = species_e.i2; + // auto dx1_e = species_e.dx1; + // auto dx2_e = species_e.dx2; + // auto phi_e = species_e.phi; + // auto weight_e = species_e.weight; + // auto tag_e = species_e.tag; + + // auto ux1_p = species_p.ux1; + // auto ux2_p = species_p.ux2; + // auto ux3_p = species_p.ux3; + // auto i1_p = species_p.i1; + // auto i2_p = species_p.i2; + // auto dx1_p = species_p.dx1; + // auto dx2_p = species_p.dx2; + // auto phi_p = species_p.phi; + // auto weight_p = species_p.weight; + // auto tag_p = species_p.tag; + + // int nseed = 1; + + // Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { + + // // ToDo: fix this + // auto i1_ = math::floor(10); + // auto i2_ = math::floor(64); + // auto dx1_ = HALF; + // auto dx2_ = HALF; + + + // auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); + // auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); + + // i1_e(elec_p + offset_e) = i1_; + // dx1_e(elec_p + offset_e) = dx1_; + // i2_e(elec_p + offset_e) = i2_; + // dx2_e(elec_p + offset_e) = dx2_; + // ux1_e(elec_p + offset_e) = -0.5; + // ux2_e(elec_p + offset_e) = 0.5; + // ux3_e(elec_p + offset_e) = ZERO; + // weight_e(elec_p + offset_e) = ONE; + // tag_e(elec_p + offset_e) = ParticleTag::alive; + + // i1_p(pos_p + offset_p) = i1_; + // dx1_p(pos_p + offset_p) = dx1_; + // i2_p(pos_p + offset_p) = i2_; + // dx2_p(pos_p + offset_p) = dx2_; + // ux1_p(pos_p + offset_p) = 0.5; + // ux2_p(pos_p + offset_p) = -0.5; + // ux3_p(pos_p + offset_p) = ZERO; + // weight_p(pos_p + offset_p) = ONE; + // tag_p(pos_p + offset_p) = ParticleTag::alive; + + + // }); + + + // auto elec_ind_h = Kokkos::create_mirror(elec_ind); + // Kokkos::deep_copy(elec_ind_h, elec_ind); + // species_e.set_npart(offset_e + elec_ind_h()); + + // auto pos_ind_h = Kokkos::create_mirror(pos_ind); + // Kokkos::deep_copy(pos_ind_h, pos_ind); + // species_p.set_npart(offset_p + pos_ind_h()); } From b7bed425e6ba920ae46cf63fd1cc6a55cc94e894 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 3 Mar 2025 13:11:03 -0600 Subject: [PATCH 147/183] cleanup of conductor BCs --- src/kernels/fields_bcs.hpp | 101 ++++++++++++++++++++----------------- 1 file changed, 56 insertions(+), 45 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index def4668c6..7a499210f 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -506,19 +506,19 @@ namespace kernel::bc { if constexpr (S == SimEngine::SRPIC) { - if (tags & BC::E) - { - Fld((N_GHOSTS - 1) - i1, em::ex1) = Fld(N_GHOSTS + i1, em::ex1); - Fld((N_GHOSTS - 1) - i1, em::ex2) = -Fld(N_GHOSTS + 1 + i1, em::ex2); - Fld((N_GHOSTS - 1) - i1, em::ex3) = -Fld(N_GHOSTS + 1 + i1, em::ex3); - } + if (tags & BC::E) { + Fld((N_GHOSTS-1)-i1, em::ex1) = Fld(N_GHOSTS+i1, em::ex1); + Fld((N_GHOSTS-1)-i1, em::ex2) = -Fld(N_GHOSTS+i1+1, em::ex2); + Fld((N_GHOSTS-1)-i1, em::ex3) = -Fld(N_GHOSTS+i1+1, em::ex3); + } + + if (tags & BC::B) + { + Fld((N_GHOSTS-1)-i1, em::bx1) = -Fld(N_GHOSTS+i1+1, em::bx1); + Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); + Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); + } - if (tags & BC::B) - { - Fld((N_GHOSTS - 1) - i1, em::bx1) = -Fld(N_GHOSTS + 1 + i1, em::bx1); - Fld((N_GHOSTS - 1) - i1, em::bx2) = Fld(N_GHOSTS + i1, em::bx2); - Fld((N_GHOSTS - 1) - i1, em::bx3) = Fld(N_GHOSTS + i1, em::bx3); - } } else { // GRPIC raise::KernelError(HERE, "1D GRPIC not implemented"); @@ -530,64 +530,75 @@ namespace kernel::bc { } } - Inline void operator()(index_t i1, index_t i2) const { - if constexpr (M::Dim == Dim::_2D) { + Inline void operator()(index_t i1, index_t i2) const + { + if constexpr (M::Dim == Dim::_2D) + { - if constexpr (S == SimEngine::SRPIC) { + if constexpr (S == SimEngine::SRPIC) + { // SRPIC - if (tags & BC::E) { - // Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); - // Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+1+i1, i2, em::ex2); - // Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+1+i1, i2, em::ex3); - Fld((N_GHOSTS-1)-i1, i2, em::ex1) = ZERO; - Fld((N_GHOSTS-1)-i1, i2, em::ex2) = ZERO; - Fld((N_GHOSTS-1)-i1, i2, em::ex3) = ZERO; - Fld((N_GHOSTS), i2, em::ex2) = ZERO; - Fld((N_GHOSTS), i2, em::ex3) = ZERO; + if (tags & BC::E) + { + Fld((N_GHOSTS - 1) - i1, i2, em::ex1) = Fld(N_GHOSTS + i1, i2, em::ex1); + Fld((N_GHOSTS - 1) - i1, i2, em::ex2) = -Fld(N_GHOSTS + 1 + i1, i2, em::ex2); + Fld((N_GHOSTS - 1) - i1, i2, em::ex3) = -Fld(N_GHOSTS + 1 + i1, i2, em::ex3); } if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = ZERO; - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = ZERO; - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = ZERO; + Fld((N_GHOSTS - 1) - i1, i2, em::bx1) = -Fld(N_GHOSTS + 1 + i1, i2, em::bx1); + Fld((N_GHOSTS - 1) - i1, i2, em::bx2) = Fld(N_GHOSTS + i1, i2, em::bx2); + Fld((N_GHOSTS - 1) - i1, i2, em::bx3) = Fld(N_GHOSTS + i1, i2, em::bx3); } - } else { + } + else + { // GRPIC raise::KernelError(HERE, "2D GRPIC not implemented"); } - } else { + } + else + { raise::KernelError( - HERE, - "ConductorBoundaries_kernel: 2D implementation called for D != 2"); + HERE, + "ConductorBoundaries_kernel: 2D implementation called for D != 2"); } } - Inline void operator()(index_t i1, index_t i2, index_t i3) const { - if constexpr (M::Dim == Dim::_3D) { + Inline void operator()(index_t i1, index_t i2, index_t i3) const + { + if constexpr (M::Dim == Dim::_3D) + { - if constexpr (S == SimEngine::SRPIC) { + if constexpr (S == SimEngine::SRPIC) + { // SRPIC - if (tags & BC::E) { - Fld((N_GHOSTS-1)-i1, i2, i3, em::ex1) = Fld(N_GHOSTS+i1, i2, i3, em::ex1); - Fld((N_GHOSTS-1)-i1, i2, i3, em::ex2) = -Fld(N_GHOSTS+i1, i2, i3, em::ex2); - Fld((N_GHOSTS-1)-i1, i2, i3, em::ex3) = -Fld(N_GHOSTS+i1, i2, i3, em::ex3); + if (tags & BC::E) + { + Fld((N_GHOSTS - 1) - i1, i2, i3, em::ex1) = Fld(N_GHOSTS + i1, i2, i3, em::ex1); + Fld((N_GHOSTS - 1) - i1, i2, i3, em::ex2) = -Fld(N_GHOSTS + i1 + 1, i2, i3, em::ex2); + Fld((N_GHOSTS - 1) - i1, i2, i3, em::ex3) = -Fld(N_GHOSTS + i1 + 1, i2, i3, em::ex3); } if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, i3, em::bx1) = -Fld(N_GHOSTS+i1, i2, i3, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, i3, em::bx2) = Fld(N_GHOSTS+i1, i2, i3, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, i3, em::bx3) = Fld(N_GHOSTS+i1, i2, i3, em::bx3); + Fld((N_GHOSTS - 1) - i1, i2, i3, em::bx1) = -Fld(N_GHOSTS + i1 + 1, i2, i3, em::bx1); + Fld((N_GHOSTS - 1) - i1, i2, i3, em::bx2) = Fld(N_GHOSTS + i1, i2, i3, em::bx2); + Fld((N_GHOSTS - 1) - i1, i2, i3, em::bx3) = Fld(N_GHOSTS + i1, i2, i3, em::bx3); } - } else { + } + else + { // GRPIC raise::KernelError(HERE, "3D GRPIC not implemented"); } - } else { + } + else + { raise::KernelError( - HERE, - "ConductorBoundaries_kernel: 3D implementation called for D != 3"); + HERE, + "ConductorBoundaries_kernel: 3D implementation called for D != 3"); } } }; From 624991803f2d9a520fdb0cd3770806d18f4b9dd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 3 Mar 2025 13:11:50 -0600 Subject: [PATCH 148/183] add 3D case and set correct boundary cells to zero --- src/engines/srpic.hpp | 137 ++++++++++++++++++++++++++++-------------- 1 file changed, 91 insertions(+), 46 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 86b85de32..b1fca46da 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -839,59 +839,104 @@ namespace ntt { } void PerfectConductorFieldsIn(dir::direction_t direction, - domain_t& domain, - BCTags tags) { + domain_t& domain, + BCTags tags) { /** * perfect conductor field boundaries */ - const auto sign = direction.get_sign(); - const auto dim = direction.get_dim(); - raise::ErrorIf(dim != in::x1 and M::CoordType != Coord::Cart, - "Perfect conductor BCs only implemented for x1 in " - "non-cartesian coordinates", - HERE); + if constexpr (M::CoordType != Coord::Cart) { + (void)direction; + (void)domain; + (void)tags; + raise::Error( + "Perfect conductor BCs only applicable to cartesian coordinates", + HERE); + } else { + const auto sign = direction.get_sign(); + const auto dim = direction.get_dim(); + std::vector xi_min, xi_max; - std::vector xi_min, xi_max; + const std::vector all_dirs { in::x1, in::x2, in::x3 }; - const std::vector all_dirs { in::x1, in::x2, in::x3 }; - - for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { - const auto dd = all_dirs[d]; - if (dim == dd) { + for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { + const auto dd = all_dirs[d]; + if (dim == dd) { xi_min.push_back(0); - xi_max.push_back(N_GHOSTS); - } else { - xi_min.push_back(0); - xi_max.push_back(domain.mesh.n_all(dd)); + xi_max.push_back((sign < 0) ? (N_GHOSTS + 1) : N_GHOSTS); + } else { + xi_min.push_back(0); + xi_max.push_back(domain.mesh.n_all(dd)); + } } - } - raise::ErrorIf(xi_min.size() != xi_max.size() or - xi_min.size() != static_cast(M::Dim), - "Invalid range size", - HERE); + raise::ErrorIf(xi_min.size() != xi_max.size() or + xi_min.size() != static_cast(M::Dim), + "Invalid range size", + HERE); - if constexpr (M::Dim == Dim::_1D) - { - Kokkos::parallel_for( - "MatchFields", - CreateRangePolicy( { xi_min[0] } , { xi_max[0] } ), - kernel::bc::ConductorBoundaries_kernel( - domain.fields.em, - tags)); - } + range_t range; + if constexpr (M::Dim == Dim::_1D) { + range = CreateRangePolicy({ xi_min[0] }, { xi_max[0] }); + } else if constexpr (M::Dim == Dim::_2D) { + range = CreateRangePolicy({ xi_min[0], xi_min[1] }, + { xi_max[0], xi_max[1] }); + } else if constexpr (M::Dim == Dim::_3D) { + range = CreateRangePolicy({ xi_min[0], xi_min[1], xi_min[2] }, + { xi_max[0], xi_max[1], xi_max[2] }); + } else { + raise::Error("Invalid dimension", HERE); + } - if constexpr (M::Dim == Dim::_2D) - { - Kokkos::parallel_for( - "MatchFields", - CreateRangePolicy( { xi_min[0], xi_min[1] } , { xi_max[0], xi_max[1] } ), - kernel::bc::ConductorBoundaries_kernel( - domain.fields.em, - tags)); + if (dim == in::x1) { + if (sign > 0) { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } else { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } + } else if (dim == in::x2) { + if (sign > 0) { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } else { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } + } else { + if (sign > 0) { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } else { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } + } } - - } void AtmosphereFieldsIn(dir::direction_t direction, @@ -922,15 +967,15 @@ namespace ntt { return; } const auto intersect_range = domain.mesh.ExtentToRange(box, incl_ghosts); - tuple_t range_min { 0 }; - tuple_t range_max { 0 }; + tuple_t range_min { 0 }; + tuple_t range_max { 0 }; for (unsigned short d { 0 }; d < M::Dim; ++d) { range_min[d] = intersect_range[d].first; range_max[d] = intersect_range[d].second; } - auto atm_fields = m_pgen.AtmFields(time); - ncells_t il_edge; + auto atm_fields = m_pgen.AtmFields(time); + std::size_t il_edge; if (sign > 0) { il_edge = range_min[dd] - N_GHOSTS; } else { From 268f35afd100d05d34603cfa9cdd8d292e8d0380 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Tue, 4 Mar 2025 13:22:38 -0600 Subject: [PATCH 149/183] Ongoing --- setups/srpic/shocktest/pgen.hpp | 156 ++++++++++++++++---------------- 1 file changed, 78 insertions(+), 78 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 1b8926c67..a1ce9a1a9 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -48,8 +48,8 @@ namespace user { Inline auto bx3(const coord_t& x_ph) const -> real_t { // return Bmag * math::sin(Btheta) * math::cos(Bphi); - return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); - // return ZERO; + // return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ZERO; } // electric field components @@ -59,8 +59,8 @@ namespace user { Inline auto ex2(const coord_t& x_ph) const -> real_t { // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); - // return ZERO; + // return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ZERO; } Inline auto ex3(const coord_t& x_ph) const -> real_t { @@ -120,82 +120,82 @@ namespace user { inline void InitPrtls(Domain& domain) { - // auto& species_e = domain.species[0]; - // auto& species_p = domain.species[1]; + auto& species_e = domain.species[0]; + auto& species_p = domain.species[1]; - // array_t elec_ind("elec_ind"); - // array_t pos_ind("pos_ind"); + array_t elec_ind("elec_ind"); + array_t pos_ind("pos_ind"); - // auto offset_e = species_e.npart(); - // auto offset_p = species_p.npart(); - - // auto ux1_e = species_e.ux1; - // auto ux2_e = species_e.ux2; - // auto ux3_e = species_e.ux3; - // auto i1_e = species_e.i1; - // auto i2_e = species_e.i2; - // auto dx1_e = species_e.dx1; - // auto dx2_e = species_e.dx2; - // auto phi_e = species_e.phi; - // auto weight_e = species_e.weight; - // auto tag_e = species_e.tag; - - // auto ux1_p = species_p.ux1; - // auto ux2_p = species_p.ux2; - // auto ux3_p = species_p.ux3; - // auto i1_p = species_p.i1; - // auto i2_p = species_p.i2; - // auto dx1_p = species_p.dx1; - // auto dx2_p = species_p.dx2; - // auto phi_p = species_p.phi; - // auto weight_p = species_p.weight; - // auto tag_p = species_p.tag; - - // int nseed = 1; - - // Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { - - // // ToDo: fix this - // auto i1_ = math::floor(10); - // auto i2_ = math::floor(64); - // auto dx1_ = HALF; - // auto dx2_ = HALF; - - - // auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); - // auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); - - // i1_e(elec_p + offset_e) = i1_; - // dx1_e(elec_p + offset_e) = dx1_; - // i2_e(elec_p + offset_e) = i2_; - // dx2_e(elec_p + offset_e) = dx2_; - // ux1_e(elec_p + offset_e) = -0.5; - // ux2_e(elec_p + offset_e) = 0.5; - // ux3_e(elec_p + offset_e) = ZERO; - // weight_e(elec_p + offset_e) = ONE; - // tag_e(elec_p + offset_e) = ParticleTag::alive; - - // i1_p(pos_p + offset_p) = i1_; - // dx1_p(pos_p + offset_p) = dx1_; - // i2_p(pos_p + offset_p) = i2_; - // dx2_p(pos_p + offset_p) = dx2_; - // ux1_p(pos_p + offset_p) = 0.5; - // ux2_p(pos_p + offset_p) = -0.5; - // ux3_p(pos_p + offset_p) = ZERO; - // weight_p(pos_p + offset_p) = ONE; - // tag_p(pos_p + offset_p) = ParticleTag::alive; - - - // }); - - - // auto elec_ind_h = Kokkos::create_mirror(elec_ind); - // Kokkos::deep_copy(elec_ind_h, elec_ind); - // species_e.set_npart(offset_e + elec_ind_h()); - - // auto pos_ind_h = Kokkos::create_mirror(pos_ind); - // Kokkos::deep_copy(pos_ind_h, pos_ind); - // species_p.set_npart(offset_p + pos_ind_h()); + auto offset_e = species_e.npart(); + auto offset_p = species_p.npart(); + + auto ux1_e = species_e.ux1; + auto ux2_e = species_e.ux2; + auto ux3_e = species_e.ux3; + auto i1_e = species_e.i1; + auto i2_e = species_e.i2; + auto dx1_e = species_e.dx1; + auto dx2_e = species_e.dx2; + auto phi_e = species_e.phi; + auto weight_e = species_e.weight; + auto tag_e = species_e.tag; + + auto ux1_p = species_p.ux1; + auto ux2_p = species_p.ux2; + auto ux3_p = species_p.ux3; + auto i1_p = species_p.i1; + auto i2_p = species_p.i2; + auto dx1_p = species_p.dx1; + auto dx2_p = species_p.dx2; + auto phi_p = species_p.phi; + auto weight_p = species_p.weight; + auto tag_p = species_p.tag; + + int nseed = 1; + + Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { + + // ToDo: fix this + auto i1_ = math::floor(10); + auto i2_ = math::floor(64); + auto dx1_ = HALF; + auto dx2_ = HALF; + + + auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); + auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); + + i1_e(elec_p + offset_e) = i1_; + dx1_e(elec_p + offset_e) = dx1_; + i2_e(elec_p + offset_e) = i2_; + dx2_e(elec_p + offset_e) = dx2_; + ux1_e(elec_p + offset_e) = -0.5; + ux2_e(elec_p + offset_e) = 0.5; + ux3_e(elec_p + offset_e) = ZERO; + weight_e(elec_p + offset_e) = ONE; + tag_e(elec_p + offset_e) = ParticleTag::alive; + + i1_p(pos_p + offset_p) = i1_; + dx1_p(pos_p + offset_p) = dx1_; + i2_p(pos_p + offset_p) = i2_; + dx2_p(pos_p + offset_p) = dx2_; + ux1_p(pos_p + offset_p) = 0.5; + ux2_p(pos_p + offset_p) = -0.5; + ux3_p(pos_p + offset_p) = ZERO; + weight_p(pos_p + offset_p) = ONE; + tag_p(pos_p + offset_p) = ParticleTag::alive; + + + }); + + + auto elec_ind_h = Kokkos::create_mirror(elec_ind); + Kokkos::deep_copy(elec_ind_h, elec_ind); + species_e.set_npart(offset_e + elec_ind_h()); + + auto pos_ind_h = Kokkos::create_mirror(pos_ind); + Kokkos::deep_copy(pos_ind_h, pos_ind); + species_p.set_npart(offset_p + pos_ind_h()); } From c4325e70b9bf4b15004f2b0b56b54dea58badf3e Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 7 Mar 2025 11:30:47 -0500 Subject: [PATCH 150/183] minor reformatting of conductor BC --- input.example.toml | 2 +- setups/srpic/shocktest/pgen.hpp | 149 +++++++++------------------- setups/srpic/shocktest/shock.toml | 26 +++-- src/engines/srpic.hpp | 52 ++++++++++ src/global/enums.h | 19 ++-- src/global/tests/enums.cpp | 5 +- src/kernels/fields_bcs.hpp | 157 ++++++++++++++---------------- 7 files changed, 204 insertions(+), 206 deletions(-) diff --git a/input.example.toml b/input.example.toml index a49067811..c71049b84 100644 --- a/input.example.toml +++ b/input.example.toml @@ -90,7 +90,7 @@ # Boundary conditions for fields: # @required # @type: 1/2/3-size array of string tuples, each of size 1 or 2 - # @valid: "PERIODIC", "MATCH", "FIXED", "ATMOSPHERE", "CUSTOM", "HORIZON" + # @valid: "PERIODIC", "MATCH", "FIXED", "ATMOSPHERE", "CUSTOM", "HORIZON", "CONDUCTOR" # @example: [["CUSTOM", "MATCH"]] (for 2D spherical [[rmin, rmax]]) # @note: When periodic in any of the directions, you should only set one value: [..., ["PERIODIC"], ...] # @note: In spherical, bondaries in theta/phi are set automatically (only specify bc @ [rmin, rmax]): [["ATMOSPHERE", "MATCH"]] diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index a1ce9a1a9..4f6decc76 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -14,6 +14,7 @@ #include "framework/domain/metadomain.h" #include +#include namespace user { using namespace ntt; @@ -48,7 +49,8 @@ namespace user { Inline auto bx3(const coord_t& x_ph) const -> real_t { // return Bmag * math::sin(Btheta) * math::cos(Bphi); - // return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + // return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + // + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); return ZERO; } @@ -59,7 +61,8 @@ namespace user { Inline auto ex2(const coord_t& x_ph) const -> real_t { // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - // return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + // return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + // + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); return ZERO; } @@ -88,17 +91,32 @@ namespace user { const real_t drift_ux, temperature; - const real_t Btheta, Bphi, Bmag; - InitFields init_flds; + const std::vector x1arr_e, x2arr_e, ux1arr_e, ux2arr_e, ux3arr_e; + const std::vector x1arr_i, x2arr_i, ux1arr_i, ux2arr_i, ux3arr_i; + + const real_t Btheta, Bphi, Bmag; + InitFields init_flds; + const Metadomain* metadomain; inline PGen(const SimulationParams& p, const Metadomain& m) : arch::ProblemGenerator { p } , drift_ux { p.template get("setup.drift_ux") } , temperature { p.template get("setup.temperature") } - , Bmag { p.template get("setup.Bmag", ZERO) } + , x1arr_e { p.template get>("setup.x_e") } + , x2arr_e { p.template get>("setup.y_e") } + , ux1arr_e { p.template get>("setup.ux_e") } + , ux2arr_e { p.template get>("setup.uy_e") } + , ux3arr_e { p.template get>("setup.uz_e") } + , x1arr_i { p.template get>("setup.x_i") } + , x2arr_i { p.template get>("setup.y_i") } + , ux1arr_i { p.template get>("setup.ux_i") } + , ux2arr_i { p.template get>("setup.uy_i") } + , ux3arr_i { p.template get>("setup.uz_i") } , Btheta { p.template get("setup.Btheta", ZERO) } + , Bmag { p.template get("setup.Bmag", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } - , init_flds { Bmag, Btheta, Bphi, drift_ux } {} + , init_flds { Bmag, Btheta, Bphi, drift_ux } + , metadomain { &m } {} inline PGen() {} @@ -113,111 +131,32 @@ namespace user { } } - auto MatchFields(real_t time) const -> InitFields { return init_flds; } inline void InitPrtls(Domain& domain) { - - auto& species_e = domain.species[0]; - auto& species_p = domain.species[1]; - - array_t elec_ind("elec_ind"); - array_t pos_ind("pos_ind"); - - auto offset_e = species_e.npart(); - auto offset_p = species_p.npart(); - - auto ux1_e = species_e.ux1; - auto ux2_e = species_e.ux2; - auto ux3_e = species_e.ux3; - auto i1_e = species_e.i1; - auto i2_e = species_e.i2; - auto dx1_e = species_e.dx1; - auto dx2_e = species_e.dx2; - auto phi_e = species_e.phi; - auto weight_e = species_e.weight; - auto tag_e = species_e.tag; - - auto ux1_p = species_p.ux1; - auto ux2_p = species_p.ux2; - auto ux3_p = species_p.ux3; - auto i1_p = species_p.i1; - auto i2_p = species_p.i2; - auto dx1_p = species_p.dx1; - auto dx2_p = species_p.dx2; - auto phi_p = species_p.phi; - auto weight_p = species_p.weight; - auto tag_p = species_p.tag; - - int nseed = 1; - - Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { - - // ToDo: fix this - auto i1_ = math::floor(10); - auto i2_ = math::floor(64); - auto dx1_ = HALF; - auto dx2_ = HALF; - - - auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); - auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); - - i1_e(elec_p + offset_e) = i1_; - dx1_e(elec_p + offset_e) = dx1_; - i2_e(elec_p + offset_e) = i2_; - dx2_e(elec_p + offset_e) = dx2_; - ux1_e(elec_p + offset_e) = -0.5; - ux2_e(elec_p + offset_e) = 0.5; - ux3_e(elec_p + offset_e) = ZERO; - weight_e(elec_p + offset_e) = ONE; - tag_e(elec_p + offset_e) = ParticleTag::alive; - - i1_p(pos_p + offset_p) = i1_; - dx1_p(pos_p + offset_p) = dx1_; - i2_p(pos_p + offset_p) = i2_; - dx2_p(pos_p + offset_p) = dx2_; - ux1_p(pos_p + offset_p) = 0.5; - ux2_p(pos_p + offset_p) = -0.5; - ux3_p(pos_p + offset_p) = ZERO; - weight_p(pos_p + offset_p) = ONE; - tag_p(pos_p + offset_p) = ParticleTag::alive; - - - }); - - - auto elec_ind_h = Kokkos::create_mirror(elec_ind); - Kokkos::deep_copy(elec_ind_h, elec_ind); - species_e.set_npart(offset_e + elec_ind_h()); - - auto pos_ind_h = Kokkos::create_mirror(pos_ind); - Kokkos::deep_copy(pos_ind_h, pos_ind); - species_p.set_npart(offset_p + pos_ind_h()); - - + arch::InjectGlobally(*metadomain, + domain, + 1, + { + { "x1", x1arr_e }, + { "x2", x2arr_e }, + { "ux1", ux1arr_e }, + { "ux2", ux1arr_e }, + { "ux3", ux3arr_e } + }); + arch::InjectGlobally(*metadomain, + domain, + 2, + { + { "x1", x1arr_i }, + { "x2", x2arr_i }, + { "ux1", ux1arr_i }, + { "ux2", ux1arr_i }, + { "ux3", ux3arr_i } + }); } - - - // inline void InitPrtls(Domain& local_domain) { - // const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, - // local_domain.random_pool, - // temperature, - // -drift_ux, - // in::x1); - - // const auto injector = arch::UniformInjector( - // energy_dist, - // { 1, 2 }); - // arch::InjectUniform>( - // params, - // local_domain, - // injector, - // 1.0); - // } - }; } // namespace user diff --git a/setups/srpic/shocktest/shock.toml b/setups/srpic/shocktest/shock.toml index a77f5c2e9..6c0c9a3a0 100644 --- a/setups/srpic/shocktest/shock.toml +++ b/setups/srpic/shocktest/shock.toml @@ -11,7 +11,7 @@ metric = "minkowski" [grid.boundaries] - fields = [["CONDUCTOR", "FIXED"], ["PERIODIC"]] + fields = [["CONDUCTOR", "FIXED"], ["PERIODIC"]] particles = [["REFLECT", "ABSORB"], ["PERIODIC"]] [scales] @@ -20,8 +20,8 @@ [algorithms] current_filters = 8 - fieldsolver = "false" - deposit = "false" + fieldsolver = "false" + deposit = "false" [algorithms.timestep] CFL = 0.5 @@ -44,13 +44,23 @@ [setup] drift_ux = 0.1 temperature = 1e-3 - Bmag = 1.0 - Btheta = 0.0 - Bphi = 0.0 + Bmag = 1.0 + Btheta = 0.0 + Bphi = 0.0 + x_e = [0.05] + y_e = [0.0] + ux_e = [-0.01] + uy_e = [0.01] + uz_e = [0.001] + x_i = [0.05] + y_i = [0.0] + ux_i = [0.01] + uy_i = [-0.01] + uz_i = [-0.001] [output] - interval = 1 - format = "hdf5" + interval = 1 + format = "hdf5" [output.fields] quantities = ["N_1", "N_2", "E", "B", "T0i_1", "T0i_2", "J"] diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index b1fca46da..f75989739 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -854,6 +854,19 @@ namespace ntt { } else { const auto sign = direction.get_sign(); const auto dim = direction.get_dim(); +<<<<<<< HEAD +||||||| parent of bd723f39 (minor reformatting of conductor BC) + const auto sign = direction.get_sign(); + const auto dim = direction.get_dim(); + raise::ErrorIf(dim != in::x1 and M::CoordType != Coord::Cart, + "Perfect conductor BCs only implemented for x1 in " + "non-cartesian coordinates", + HERE); +======= + raise::ErrorIf(dim != in::x1, + "Perfect conductor BCs only implemented for x1", + HERE); +>>>>>>> bd723f39 (minor reformatting of conductor BC) std::vector xi_min, xi_max; @@ -863,7 +876,13 @@ namespace ntt { const auto dd = all_dirs[d]; if (dim == dd) { xi_min.push_back(0); +<<<<<<< HEAD xi_max.push_back((sign < 0) ? (N_GHOSTS + 1) : N_GHOSTS); +||||||| parent of bd723f39 (minor reformatting of conductor BC) + xi_max.push_back(N_GHOSTS); +======= + xi_max.push_back(N_GHOSTS + 1); +>>>>>>> bd723f39 (minor reformatting of conductor BC) } else { xi_min.push_back(0); xi_max.push_back(domain.mesh.n_all(dd)); @@ -886,6 +905,7 @@ namespace ntt { } else { raise::Error("Invalid dimension", HERE); } +<<<<<<< HEAD if (dim == in::x1) { if (sign > 0) { @@ -936,6 +956,38 @@ namespace ntt { tags)); } } +||||||| parent of bd723f39 (minor reformatting of conductor BC) +======= + Kokkos::parallel_for( + "MatchFields", + range, + kernel::bc::ConductorBoundaries_kernel(domain.fields.em, + tags)); + + // if constexpr (M::Dim == Dim::_1D) { + // Kokkos::parallel_for( + // "MatchFields", + // CreateRangePolicy({ xi_min[0] }, { xi_max[0] }), + // kernel::bc::ConductorBoundaries_kernel(domain.fields.em, + // tags)); + // } else if constexpr (M::Dim == Dim::_2D) { + // Kokkos::parallel_for( + // "MatchFields", + // CreateRangePolicy({ xi_min[0], xi_min[1] }, + // { xi_max[0], xi_max[1] }), + // kernel::bc::ConductorBoundaries_kernel(domain.fields.em, + // tags)); + // } else if constexpr (M::Dim == Dim::_3D) { + // Kokkos::parallel_for( + // "MatchFields", + // CreateRangePolicy({ xi_min[0], xi_min[1], xi_min[2] }, + // { xi_max[0], xi_max[1], xi_max[2] }), + // kernel::bc::ConductorBoundaries_kernel(domain.fields.em, + // tags)); + // } else { + // raise::Error("Invalid dimension", HERE); + // } +>>>>>>> bd723f39 (minor reformatting of conductor BC) } } diff --git a/src/global/enums.h b/src/global/enums.h index 8407d0b66..d80297d8d 100644 --- a/src/global/enums.h +++ b/src/global/enums.h @@ -9,7 +9,7 @@ * - enum ntt::PrtlBC // periodic, absorb, atmosphere, custom, * reflect, horizon, axis, sync * - enum ntt::FldsBC // periodic, match, fixed, atmosphere, - * custom, horizon, axis, sync + * custom, horizon, axis, conductor, sync * - enum ntt::PrtlPusher // boris, vay, photon, none * - enum ntt::Cooling // synchrotron, none * - enum ntt::FldsID // e, dive, d, divd, b, h, j, @@ -221,17 +221,20 @@ namespace ntt { CUSTOM = 5, HORIZON = 6, AXIS = 7, - SYNC = 8, // <- SYNC means synchronization with other domains - CONDUCTOR = 9 + CONDUCTOR = 8, + SYNC = 9 // <- SYNC means synchronization with other domains }; constexpr FldsBC(uint8_t c) : enums_hidden::BaseEnum { c } {} - static constexpr type variants[] = { PERIODIC, MATCH, FIXED, ATMOSPHERE, - CUSTOM, HORIZON, AXIS, SYNC, CONDUCTOR }; - static constexpr const char* lookup[] = { "periodic", "match", "fixed", - "atmosphere", "custom", "horizon", - "axis", "sync", "conductor"}; + static constexpr type variants[] = { + PERIODIC, MATCH, FIXED, ATMOSPHERE, CUSTOM, + HORIZON, AXIS, CONDUCTOR, SYNC, + }; + static constexpr const char* lookup[] = { + "periodic", "match", "fixed", "atmosphere", "custom", + "horizon", "axis", "conductor", "sync" + }; static constexpr std::size_t total = sizeof(variants) / sizeof(variants[0]); }; diff --git a/src/global/tests/enums.cpp b/src/global/tests/enums.cpp index 673efaf34..ebc074b72 100644 --- a/src/global/tests/enums.cpp +++ b/src/global/tests/enums.cpp @@ -61,8 +61,9 @@ auto main() -> int { enum_str_t all_simulation_engines = { "srpic", "grpic" }; enum_str_t all_particle_bcs = { "periodic", "absorb", "atmosphere", "custom", "reflect", "horizon", "axis", "sync" }; - enum_str_t all_fields_bcs = { "periodic", "match", "fixed", "atmosphere", - "custom", "horizon", "axis", "sync" }; + enum_str_t all_fields_bcs = { "periodic", "match", "fixed", + "atmosphere", "custom", "horizon", + "axis", "conductor", "sync" }; enum_str_t all_particle_pushers = { "boris", "vay", "photon", "none" }; enum_str_t all_coolings = { "synchrotron", "none" }; diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 7a499210f..901f8e02a 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -5,6 +5,7 @@ * - kernel::bc::MatchBoundaries_kernel<> * - kernel::bc::AxisBoundaries_kernel<> * - kernel::bc::EnforcedBoundaries_kernel<> + * - kernel::bc::ConductorBoundaries_kernel<> * @namespaces: * - kernel::bc:: */ @@ -485,43 +486,40 @@ namespace kernel::bc { } }; - template + template struct ConductorBoundaries_kernel { - static_assert(M::is_metric, "M must be a metric class"); - static_assert(static_cast(o) < - static_cast(M::Dim), + static_assert(static_cast(o) < static_cast(D), "Invalid component index"); - static constexpr idx_t i = static_cast(o) + 1u; + // static constexpr idx_t i = static_cast(o) + 1u; - ndfield_t Fld; - const BCTags tags; + ndfield_t Fld; + const BCTags tags; - ConductorBoundaries_kernel(ndfield_t Fld, - BCTags tags) + ConductorBoundaries_kernel(ndfield_t Fld, BCTags tags) : Fld { Fld } , tags { tags } {} Inline void operator()(index_t i1) const { - if constexpr (M::Dim == Dim::_1D) { - - if constexpr (S == SimEngine::SRPIC) { - + if constexpr (D == Dim::_1D) { if (tags & BC::E) { - Fld((N_GHOSTS-1)-i1, em::ex1) = Fld(N_GHOSTS+i1, em::ex1); - Fld((N_GHOSTS-1)-i1, em::ex2) = -Fld(N_GHOSTS+i1+1, em::ex2); - Fld((N_GHOSTS-1)-i1, em::ex3) = -Fld(N_GHOSTS+i1+1, em::ex3); - } - - if (tags & BC::B) - { - Fld((N_GHOSTS-1)-i1, em::bx1) = -Fld(N_GHOSTS+i1+1, em::bx1); - Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); - Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); + if (i1 == 0) { + Fld(N_GHOSTS, em::ex2) = ZERO; + Fld(N_GHOSTS, em::ex3) = ZERO; + } else { + Fld(N_GHOSTS - i1, em::ex1) = Fld(N_GHOSTS + i1 - 1, em::ex1); + Fld(N_GHOSTS - i1, em::ex2) = -Fld(N_GHOSTS + i1, em::ex2); + Fld(N_GHOSTS - i1, em::ex3) = -Fld(N_GHOSTS + i1, em::ex3); + } } - } else { - // GRPIC - raise::KernelError(HERE, "1D GRPIC not implemented"); + if (tags & BC::B) { + if (i1 == 0) { + Fld(N_GHOSTS, em::bx1) = ZERO; + } else { + Fld(N_GHOSTS - i1, em::bx1) = -Fld(N_GHOSTS + i1, em::bx1); + Fld(N_GHOSTS - i1, em::bx2) = Fld(N_GHOSTS + i1 - 1, em::bx2); + Fld(N_GHOSTS - i1, em::bx3) = Fld(N_GHOSTS + i1 - 1, em::bx3); + } } } else { raise::KernelError( @@ -530,75 +528,70 @@ namespace kernel::bc { } } - Inline void operator()(index_t i1, index_t i2) const - { - if constexpr (M::Dim == Dim::_2D) - { - - if constexpr (S == SimEngine::SRPIC) - { - // SRPIC - if (tags & BC::E) - { - Fld((N_GHOSTS - 1) - i1, i2, em::ex1) = Fld(N_GHOSTS + i1, i2, em::ex1); - Fld((N_GHOSTS - 1) - i1, i2, em::ex2) = -Fld(N_GHOSTS + 1 + i1, i2, em::ex2); - Fld((N_GHOSTS - 1) - i1, i2, em::ex3) = -Fld(N_GHOSTS + 1 + i1, i2, em::ex3); + Inline void operator()(index_t i1, index_t i2) const { + if constexpr (D == Dim::_2D) { + if (tags & BC::E) { + if (i1 == 0) { + Fld(N_GHOSTS, i2, em::ex2) = ZERO; + Fld(N_GHOSTS, i2, em::ex3) = ZERO; + } else { + Fld(N_GHOSTS - i1, i2, em::ex1) = Fld(N_GHOSTS + i1 - 1, i2, em::ex1); + Fld(N_GHOSTS - i1, i2, em::ex2) = -Fld(N_GHOSTS + i1, i2, em::ex2); + Fld(N_GHOSTS - i1, i2, em::ex3) = -Fld(N_GHOSTS + i1, i2, em::ex3); } + } - if (tags & BC::B) - { - Fld((N_GHOSTS - 1) - i1, i2, em::bx1) = -Fld(N_GHOSTS + 1 + i1, i2, em::bx1); - Fld((N_GHOSTS - 1) - i1, i2, em::bx2) = Fld(N_GHOSTS + i1, i2, em::bx2); - Fld((N_GHOSTS - 1) - i1, i2, em::bx3) = Fld(N_GHOSTS + i1, i2, em::bx3); + if (tags & BC::B) { + if (i1 == 0) { + Fld(N_GHOSTS, i2, em::bx1) = ZERO; + } else { + Fld(N_GHOSTS - i1, i2, em::bx1) = -Fld(N_GHOSTS + i1, i2, em::bx1); + Fld(N_GHOSTS - i1, i2, em::bx2) = Fld(N_GHOSTS + i1 - 1, i2, em::bx2); + Fld(N_GHOSTS - i1, i2, em::bx3) = Fld(N_GHOSTS + i1 - 1, i2, em::bx3); } } - else - { - // GRPIC - raise::KernelError(HERE, "2D GRPIC not implemented"); - } - } - else - { + } else { raise::KernelError( - HERE, - "ConductorBoundaries_kernel: 2D implementation called for D != 2"); + HERE, + "ConductorBoundaries_kernel: 2D implementation called for D != 2"); } } - Inline void operator()(index_t i1, index_t i2, index_t i3) const - { - if constexpr (M::Dim == Dim::_3D) - { - - if constexpr (S == SimEngine::SRPIC) - { - // SRPIC - if (tags & BC::E) - { - Fld((N_GHOSTS - 1) - i1, i2, i3, em::ex1) = Fld(N_GHOSTS + i1, i2, i3, em::ex1); - Fld((N_GHOSTS - 1) - i1, i2, i3, em::ex2) = -Fld(N_GHOSTS + i1 + 1, i2, i3, em::ex2); - Fld((N_GHOSTS - 1) - i1, i2, i3, em::ex3) = -Fld(N_GHOSTS + i1 + 1, i2, i3, em::ex3); + Inline void operator()(index_t i1, index_t i2, index_t i3) const { + if constexpr (D == Dim::_3D) { + if (tags & BC::E) { + if (i1 == 0) { + Fld(N_GHOSTS, i2, i3, em::ex2) = ZERO; + Fld(N_GHOSTS, i2, i3, em::ex3) = ZERO; + } else { + Fld(N_GHOSTS - i1, i2, i3, em::ex1) = Fld(N_GHOSTS + i1 - 1, + i2, + i3, + em::ex1); + Fld(N_GHOSTS - i1, i2, i3, em::ex2) = -Fld(N_GHOSTS + i1, i2, i3, em::ex2); + Fld(N_GHOSTS - i1, i2, i3, em::ex3) = -Fld(N_GHOSTS + i1, i2, i3, em::ex3); } + } - if (tags & BC::B) - { - Fld((N_GHOSTS - 1) - i1, i2, i3, em::bx1) = -Fld(N_GHOSTS + i1 + 1, i2, i3, em::bx1); - Fld((N_GHOSTS - 1) - i1, i2, i3, em::bx2) = Fld(N_GHOSTS + i1, i2, i3, em::bx2); - Fld((N_GHOSTS - 1) - i1, i2, i3, em::bx3) = Fld(N_GHOSTS + i1, i2, i3, em::bx3); + if (tags & BC::B) { + if (i1 == 0) { + Fld(N_GHOSTS, i2, i3, em::bx1) = ZERO; + } else { + Fld(N_GHOSTS - i1, i2, i3, em::bx1) = -Fld(N_GHOSTS + i1, i2, i3, em::bx1); + Fld(N_GHOSTS - i1, i2, i3, em::bx2) = Fld(N_GHOSTS + i1 - 1, + i2, + i3, + em::bx2); + Fld(N_GHOSTS - i1, i2, i3, em::bx3) = Fld(N_GHOSTS + i1 - 1, + i2, + i3, + em::bx3); } } - else - { - // GRPIC - raise::KernelError(HERE, "3D GRPIC not implemented"); - } - } - else - { + } else { raise::KernelError( - HERE, - "ConductorBoundaries_kernel: 3D implementation called for D != 3"); + HERE, + "ConductorBoundaries_kernel: 3D implementation called for D != 3"); } } }; From 6e66aa27097095bea992672762b20a7a9093f865 Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 7 Mar 2025 12:11:31 -0500 Subject: [PATCH 151/183] filters adapted for conducting BCs --- src/kernels/digital_filter.hpp | 129 ++++++++++++++++++++++----------- 1 file changed, 86 insertions(+), 43 deletions(-) diff --git a/src/kernels/digital_filter.hpp b/src/kernels/digital_filter.hpp index f01eb0fb8..0ec6dcdbc 100644 --- a/src/kernels/digital_filter.hpp +++ b/src/kernels/digital_filter.hpp @@ -17,10 +17,14 @@ #include "utils/error.h" #include "utils/numeric.h" -#define FILTER_IN_I1(ARR, COMP, I, J) \ +#define FILTER2D_IN_I1(ARR, COMP, I, J) \ INV_2*(ARR)((I), (J), (COMP)) + \ INV_4*((ARR)((I) - 1, (J), (COMP)) + (ARR)((I) + 1, (J), (COMP))) +#define FILTER2D_IN_I2(ARR, COMP, I, J) \ + INV_2*(ARR)((I), (J), (COMP)) + \ + INV_4*((ARR)((I), (J) - 1, (COMP)) + (ARR)((I), (J) + 1, (COMP))) + namespace kernel { using namespace ntt; @@ -31,6 +35,10 @@ namespace kernel { bool is_axis_i2min { false }, is_axis_i2max { false }; static constexpr auto i2_min = N_GHOSTS; const ncells_t i2_max; + const bool is_axis_i2min, is_axis_i2max; + const bool is_conductor_i1min; + static constexpr auto i1_min = N_GHOSTS, i2_min = N_GHOSTS; + const std::size_t i2_max; public: DigitalFilter_kernel(ndfield_t& array, @@ -39,20 +47,31 @@ namespace kernel { const boundaries_t& boundaries) : array { array } , buffer { buffer } - , i2_max { (short)D > 1 ? size_[1] + N_GHOSTS : 0 } { - if constexpr ((C != Coord::Cart) && (D != Dim::_1D)) { - raise::ErrorIf(boundaries.size() < 2, "boundaries defined incorrectly", HERE); - is_axis_i2min = (boundaries[1].first == FldsBC::AXIS); - is_axis_i2max = (boundaries[1].second == FldsBC::AXIS); - } - } + , is_axis_i2min { (D == Dim::_2D) and (boundaries[1].first == FldsBC::AXIS) } + , is_axis_i2max { (D == Dim::_2D) and (boundaries[1].second == FldsBC::AXIS) } + , is_conductor_i1min { boundaries[0].first == FldsBC::CONDUCTOR } + , i2_max { (short)D > 1 ? size_[1] + N_GHOSTS : 0 } {} Inline void operator()(index_t i1) const { if constexpr ((D == Dim::_1D) && (C == Coord::Cart)) { + if (is_conductor_i1min and i1 == i1_min) { + array(i1, cur::jx1) = (THREE * INV_4) * buffer(i1, cur::jx1) + + (INV_4)*buffer(i1 + 1, cur::jx1); + } else if (is_conductor_i1min and i1 == i1_min + 1) { + array(i1, cur::jx1) = INV_2 * buffer(i1, cur::jx1) + + INV_4 * (buffer(i1 - 1, cur::jx1) + + buffer(i1 + 1, cur::jx1)); + array(i1, cur::jx2) = (INV_2)*buffer(i1, cur::jx2) + + (INV_4)*buffer(i1 + 1, cur::jx2); + array(i1, cur::jx3) = (INV_2)*buffer(i1, cur::jx3) + + (INV_4)*buffer(i1 + 1, cur::jx3); + } else { #pragma unroll - for (const auto& comp : { cur::jx1, cur::jx2, cur::jx3 }) { - array(i1, comp) = INV_2 * buffer(i1, comp) + - INV_4 * (buffer(i1 - 1, comp) + buffer(i1 + 1, comp)); + for (const auto& comp : { cur::jx1, cur::jx2, cur::jx3 }) { + array(i1, comp) = INV_2 * buffer(i1, comp) + + INV_4 * + (buffer(i1 - 1, comp) + buffer(i1 + 1, comp)); + } } } else { raise::KernelError(HERE, "DigitalFilter_kernel: 1D implementation called for D != 1 or for non-Cartesian metric"); @@ -62,25 +81,49 @@ namespace kernel { Inline void operator()(index_t i1, index_t i2) const { if constexpr (D == Dim::_2D) { if constexpr (C == Coord::Cart) { + if (is_conductor_i1min and i1 == i1_min) { + array(i1, i2, cur::jx1) = + (THREE * INV_4) * (FILTER2D_IN_I2(buffer, cur::jx1, i1, i2)) + + (INV_4) * (FILTER2D_IN_I2(buffer, cur::jx1, i1 + 1, i2)); + } else if (is_conductor_i1min and i1 == i1_min + 1) { + array(i1, + i2, + cur::jx1) = INV_2 * (FILTER2D_IN_I2(buffer, cur::jx1, i1, i2)) + + INV_4 * + ((FILTER2D_IN_I2(buffer, cur::jx1, i1 - 1, i2)) + + (FILTER2D_IN_I2(buffer, cur::jx1, i1 + 1, i2))); + array(i1, + i2, + cur::jx2) = INV_2 * (FILTER2D_IN_I2(buffer, cur::jx2, i1, i2)) + + INV_4 * + (FILTER2D_IN_I2(buffer, cur::jx2, i1 + 1, i2)); + array(i1, + i2, + cur::jx3) = INV_2 * (FILTER2D_IN_I2(buffer, cur::jx3, i1, i2)) + + INV_4 * + (FILTER2D_IN_I2(buffer, cur::jx3, i1 + 1, i2)); + } else { #pragma unroll - for (const auto comp : { cur::jx1, cur::jx2, cur::jx3 }) { - array(i1, i2, comp) = INV_4 * buffer(i1, i2, comp) + - INV_8 * (buffer(i1 - 1, i2, comp) + - buffer(i1 + 1, i2, comp) + - buffer(i1, i2 - 1, comp) + - buffer(i1, i2 + 1, comp)) + - INV_16 * (buffer(i1 - 1, i2 - 1, comp) + - buffer(i1 + 1, i2 + 1, comp) + - buffer(i1 - 1, i2 + 1, comp) + - buffer(i1 + 1, i2 - 1, comp)); + for (const auto comp : { cur::jx1, cur::jx2, cur::jx3 }) { + array(i1, i2, comp) = INV_4 * buffer(i1, i2, comp) + + INV_8 * (buffer(i1 - 1, i2, comp) + + buffer(i1 + 1, i2, comp) + + buffer(i1, i2 - 1, comp) + + buffer(i1, i2 + 1, comp)) + + INV_16 * (buffer(i1 - 1, i2 - 1, comp) + + buffer(i1 + 1, i2 + 1, comp) + + buffer(i1 - 1, i2 + 1, comp) + + buffer(i1 + 1, i2 - 1, comp)); + } } } else { // spherical + // @TODO: get rid of temporary variables real_t cur_00, cur_0p1, cur_0m1; if (is_axis_i2min && (i2 == i2_min)) { /* --------------------------------- r, phi --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx1, i1, i2); - cur_0p1 = FILTER_IN_I1(buffer, cur::jx1, i1, i2 + 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2); + cur_0p1 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2 + 1); // ... filter in theta array(i1, i2, cur::jx1) = INV_2 * cur_00 + INV_2 * cur_0p1; @@ -88,58 +131,58 @@ namespace kernel { /* ---------------------------------- theta --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx2, i1, i2); - cur_0p1 = FILTER_IN_I1(buffer, cur::jx2, i1, i2 + 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2); + cur_0p1 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2 + 1); // ... filter in theta array(i1, i2, cur::jx2) = INV_4 * (cur_00 + cur_0p1); } else if (is_axis_i2min && (i2 == i2_min + 1)) { /* --------------------------------- r, phi --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx1, i1, i2); - cur_0p1 = FILTER_IN_I1(buffer, cur::jx1, i1, i2 + 1); - cur_0m1 = FILTER_IN_I1(buffer, cur::jx1, i1, i2 - 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2); + cur_0p1 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2 + 1); + cur_0m1 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2 - 1); // ... filter in theta array(i1, i2, cur::jx1) = INV_2 * cur_00 + INV_4 * (cur_0p1 + cur_0m1); // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx3, i1, i2); - cur_0p1 = FILTER_IN_I1(buffer, cur::jx3, i1, i2 + 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx3, i1, i2); + cur_0p1 = FILTER2D_IN_I1(buffer, cur::jx3, i1, i2 + 1); // ... filter in theta array(i1, i2, cur::jx3) = INV_2 * cur_00 + INV_4 * cur_0p1; /* ---------------------------------- theta --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx2, i1, i2); - cur_0p1 = FILTER_IN_I1(buffer, cur::jx2, i1, i2 + 1); - cur_0m1 = FILTER_IN_I1(buffer, cur::jx2, i1, i2 - 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2); + cur_0p1 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2 + 1); + cur_0m1 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2 - 1); // ... filter in theta array(i1, i2, cur::jx2) = INV_2 * cur_00 + INV_4 * (cur_0m1 + cur_0p1); } else if (is_axis_i2max && (i2 == i2_max - 1)) { /* --------------------------------- r, phi --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx1, i1, i2); - cur_0p1 = FILTER_IN_I1(buffer, cur::jx1, i1, i2 + 1); - cur_0m1 = FILTER_IN_I1(buffer, cur::jx1, i1, i2 - 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2); + cur_0p1 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2 + 1); + cur_0m1 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2 - 1); // ... filter in theta array(i1, i2, cur::jx1) = INV_2 * cur_00 + INV_4 * (cur_0m1 + cur_0p1); // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx3, i1, i2); - cur_0m1 = FILTER_IN_I1(buffer, cur::jx3, i1, i2 - 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx3, i1, i2); + cur_0m1 = FILTER2D_IN_I1(buffer, cur::jx3, i1, i2 - 1); // ... filter in theta array(i1, i2, cur::jx3) = INV_2 * cur_00 + INV_4 * cur_0m1; /* ---------------------------------- theta --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx2, i1, i2); - cur_0m1 = FILTER_IN_I1(buffer, cur::jx2, i1, i2 - 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2); + cur_0m1 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2 - 1); // ... filter in theta array(i1, i2, cur::jx2) = INV_4 * (cur_00 + cur_0m1); } else if (is_axis_i2max && (i2 == i2_max)) { /* --------------------------------- r, phi --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx1, i1, i2); - cur_0m1 = FILTER_IN_I1(buffer, cur::jx1, i1, i2 - 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2); + cur_0m1 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2 - 1); // ... filter in theta array(i1, i2, cur::jx1) = INV_2 * cur_00 + INV_2 * cur_0m1; @@ -210,6 +253,6 @@ namespace kernel { } // namespace kernel -#undef FILTER_IN_I1 +#undef FILTER2D_IN_I1 #endif // DIGITAL_FILTER_HPP From 81d51f99f1c7d1b1b293ddbeda575f03c45e5aeb Mon Sep 17 00:00:00 2001 From: hayk Date: Sat, 8 Mar 2025 16:16:31 -0500 Subject: [PATCH 152/183] filter test fixed --- src/kernels/tests/digital_filter.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/kernels/tests/digital_filter.cpp b/src/kernels/tests/digital_filter.cpp index e0cc352f5..03a63753b 100644 --- a/src/kernels/tests/digital_filter.cpp +++ b/src/kernels/tests/digital_filter.cpp @@ -40,8 +40,13 @@ void testFilter(const std::vector& res, auto boundaries = boundaries_t {}; if constexpr (M::CoordType != Coord::Cart) { boundaries = { - {FldsBC::CUSTOM, FldsBC::CUSTOM}, - { FldsBC::AXIS, FldsBC::AXIS} + { FldsBC::CUSTOM, FldsBC::CUSTOM }, + { FldsBC::AXIS, FldsBC::AXIS } + }; + } else { + boundaries = { + { FldsBC::PERIODIC, FldsBC::PERIODIC }, + { FldsBC::PERIODIC, FldsBC::PERIODIC } }; } @@ -183,4 +188,4 @@ auto main(int argc, char* argv[]) -> int { } Kokkos::finalize(); return 0; -} \ No newline at end of file +} From 8bb8428204469b52823347b3ab84aed2254475b3 Mon Sep 17 00:00:00 2001 From: hayk Date: Sun, 9 Mar 2025 01:59:57 -0500 Subject: [PATCH 153/183] removed extra kokkos flags --- cmake/kokkosConfig.cmake | 35 ----------------- legacy/tests/kernels-sr.cpp | 10 ++--- src/framework/containers/particles.cpp | 2 +- src/framework/domain/comm_mpi.hpp | 12 +++--- src/framework/domain/comm_nompi.hpp | 6 +-- src/global/arch/kokkos_aliases.cpp | 54 +++++++++++++------------- src/global/arch/kokkos_aliases.h | 18 ++++----- src/kernels/tests/prtls_to_phys.cpp | 45 ++++++++++----------- 8 files changed, 75 insertions(+), 107 deletions(-) diff --git a/cmake/kokkosConfig.cmake b/cmake/kokkosConfig.cmake index 63c32622d..8800f21a0 100644 --- a/cmake/kokkosConfig.cmake +++ b/cmake/kokkosConfig.cmake @@ -37,41 +37,6 @@ set(Kokkos_ENABLE_OPENMP ${default_KOKKOS_ENABLE_OPENMP} CACHE BOOL "Enable OpenMP") -# set memory space -if(${Kokkos_ENABLE_CUDA}) - add_compile_definitions(CUDA_ENABLED) - set(ACC_MEM_SPACE Kokkos::CudaSpace) -elseif(${Kokkos_ENABLE_HIP}) - add_compile_definitions(HIP_ENABLED) - set(ACC_MEM_SPACE Kokkos::HIPSpace) -else() - set(ACC_MEM_SPACE Kokkos::HostSpace) -endif() - -set(HOST_MEM_SPACE Kokkos::HostSpace) - -# set execution space -if(${Kokkos_ENABLE_CUDA}) - set(ACC_EXE_SPACE Kokkos::Cuda) -elseif(${Kokkos_ENABLE_HIP}) - set(ACC_EXE_SPACE Kokkos::HIP) -elseif(${Kokkos_ENABLE_OPENMP}) - set(ACC_EXE_SPACE Kokkos::OpenMP) -else() - set(ACC_EXE_SPACE Kokkos::Serial) -endif() - -if(${Kokkos_ENABLE_OPENMP}) - set(HOST_EXE_SPACE Kokkos::OpenMP) -else() - set(HOST_EXE_SPACE Kokkos::Serial) -endif() - -add_compile_options("-D AccelExeSpace=${ACC_EXE_SPACE}") -add_compile_options("-D AccelMemSpace=${ACC_MEM_SPACE}") -add_compile_options("-D HostExeSpace=${HOST_EXE_SPACE}") -add_compile_options("-D HostMemSpace=${HOST_MEM_SPACE}") - if(${BUILD_TESTING} STREQUAL "OFF") set(Kokkos_ENABLE_TESTS OFF diff --git a/legacy/tests/kernels-sr.cpp b/legacy/tests/kernels-sr.cpp index 59ce0646b..d765799e3 100644 --- a/legacy/tests/kernels-sr.cpp +++ b/legacy/tests/kernels-sr.cpp @@ -1,17 +1,17 @@ -#include "wrapper.h" - #include #include #include +#include "wrapper.h" + #include METRIC_HEADER #include PGEN_HEADER -#include "particle_macros.h" - #include "kernels/particle_pusher_sr.hpp" +#include "particle_macros.h" + template void put_value(ntt::array_t& arr, T value, int i) { auto arr_h = Kokkos::create_mirror_view(arr); @@ -221,4 +221,4 @@ auto main(int argc, char* argv[]) -> int { ntt::GlobalFinalize(); return 0; -} \ No newline at end of file +} diff --git a/src/framework/containers/particles.cpp b/src/framework/containers/particles.cpp index 048d57cde..634a34862 100644 --- a/src/framework/containers/particles.cpp +++ b/src/framework/containers/particles.cpp @@ -224,7 +224,7 @@ namespace ntt { Kokkos::Experimental::fill( "TagDeadParticles", - AccelExeSpace(), + Kokkos::DefaultExecutionSpace(), Kokkos::subview(this_tag, std::make_pair(n_alive, n_alive + n_dead)), ParticleTag::dead); diff --git a/src/framework/domain/comm_mpi.hpp b/src/framework/domain/comm_mpi.hpp index 8c6e532de..4f382be4a 100644 --- a/src/framework/domain/comm_mpi.hpp +++ b/src/framework/domain/comm_mpi.hpp @@ -83,7 +83,7 @@ namespace comm { (long int)(send_slice[0].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, comps.first }, { recv_slice[0].second, comps.second }), Lambda(index_t i1, index_t ci) { @@ -96,7 +96,7 @@ namespace comm { (long int)(send_slice[1].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, comps.first }, { recv_slice[0].second, recv_slice[1].second, comps.second }), Lambda(index_t i1, index_t i2, index_t ci) { @@ -111,7 +111,7 @@ namespace comm { (long int)(send_slice[2].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, recv_slice[2].first, @@ -239,7 +239,7 @@ namespace comm { const auto offset_c = comps.first; Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, comps.first }, { recv_slice[0].second, comps.second }), Lambda(index_t i1, index_t ci) { @@ -251,7 +251,7 @@ namespace comm { const auto offset_c = comps.first; Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, comps.first }, { recv_slice[0].second, recv_slice[1].second, comps.second }), Lambda(index_t i1, index_t i2, index_t ci) { @@ -266,7 +266,7 @@ namespace comm { const auto offset_c = comps.first; Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, recv_slice[2].first, diff --git a/src/framework/domain/comm_nompi.hpp b/src/framework/domain/comm_nompi.hpp index 197d336fa..b477ac176 100644 --- a/src/framework/domain/comm_nompi.hpp +++ b/src/framework/domain/comm_nompi.hpp @@ -70,7 +70,7 @@ namespace comm { (long int)(send_slice[0].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, comps.first }, { recv_slice[0].second, comps.second }), Lambda(index_t i1, index_t ci) { @@ -83,7 +83,7 @@ namespace comm { (long int)(send_slice[1].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, comps.first }, { recv_slice[0].second, recv_slice[1].second, comps.second }), Lambda(index_t i1, index_t i2, index_t ci) { @@ -98,7 +98,7 @@ namespace comm { (long int)(send_slice[2].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, recv_slice[2].first, diff --git a/src/global/arch/kokkos_aliases.cpp b/src/global/arch/kokkos_aliases.cpp index d6622b0e5..25397af8e 100644 --- a/src/global/arch/kokkos_aliases.cpp +++ b/src/global/arch/kokkos_aliases.cpp @@ -9,73 +9,75 @@ auto CreateParticleRangePolicy(npart_t p1, npart_t p2) -> range_t { } template <> -auto CreateRangePolicy( - const tuple_t& i1, - const tuple_t& i2) -> range_t { +auto CreateRangePolicy(const tuple_t& i1, + const tuple_t& i2) + -> range_t { index_t i1min = i1[0]; index_t i1max = i2[0]; - return Kokkos::RangePolicy(i1min, i1max); + return Kokkos::RangePolicy(i1min, i1max); } template <> -auto CreateRangePolicy( - const tuple_t& i1, - const tuple_t& i2) -> range_t { +auto CreateRangePolicy(const tuple_t& i1, + const tuple_t& i2) + -> range_t { index_t i1min = i1[0]; index_t i1max = i2[0]; index_t i2min = i1[1]; index_t i2max = i2[1]; - return Kokkos::MDRangePolicy, AccelExeSpace>({ i1min, i2min }, - { i1max, i2max }); + return Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( + { i1min, i2min }, + { i1max, i2max }); } template <> -auto CreateRangePolicy( - const tuple_t& i1, - const tuple_t& i2) -> range_t { +auto CreateRangePolicy(const tuple_t& i1, + const tuple_t& i2) + -> range_t { index_t i1min = i1[0]; index_t i1max = i2[0]; index_t i2min = i1[1]; index_t i2max = i2[1]; index_t i3min = i1[2]; index_t i3max = i2[2]; - return Kokkos::MDRangePolicy, AccelExeSpace>( + return Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { i1min, i2min, i3min }, { i1max, i2max, i3max }); } template <> -auto CreateRangePolicyOnHost( - const tuple_t& i1, - const tuple_t& i2) -> range_h_t { +auto CreateRangePolicyOnHost(const tuple_t& i1, + const tuple_t& i2) + -> range_h_t { index_t i1min = i1[0]; index_t i1max = i2[0]; - return Kokkos::RangePolicy(i1min, i1max); + return Kokkos::RangePolicy(i1min, i1max); } template <> -auto CreateRangePolicyOnHost( - const tuple_t& i1, - const tuple_t& i2) -> range_h_t { +auto CreateRangePolicyOnHost(const tuple_t& i1, + const tuple_t& i2) + -> range_h_t { index_t i1min = i1[0]; index_t i1max = i2[0]; index_t i2min = i1[1]; index_t i2max = i2[1]; - return Kokkos::MDRangePolicy, HostExeSpace>({ i1min, i2min }, - { i1max, i2max }); + return Kokkos::MDRangePolicy, Kokkos::DefaultHostExecutionSpace>( + { i1min, i2min }, + { i1max, i2max }); } template <> -auto CreateRangePolicyOnHost( - const tuple_t& i1, - const tuple_t& i2) -> range_h_t { +auto CreateRangePolicyOnHost(const tuple_t& i1, + const tuple_t& i2) + -> range_h_t { index_t i1min = i1[0]; index_t i1max = i2[0]; index_t i2min = i1[1]; index_t i2max = i2[1]; index_t i3min = i1[2]; index_t i3max = i2[2]; - return Kokkos::MDRangePolicy, HostExeSpace>( + return Kokkos::MDRangePolicy, Kokkos::DefaultHostExecutionSpace>( { i1min, i2min, i3min }, { i1max, i2max, i3max }); } diff --git a/src/global/arch/kokkos_aliases.h b/src/global/arch/kokkos_aliases.h index e1a759bed..adb0b6451 100644 --- a/src/global/arch/kokkos_aliases.h +++ b/src/global/arch/kokkos_aliases.h @@ -34,7 +34,7 @@ namespace math = Kokkos; template -using array_t = Kokkos::View; +using array_t = Kokkos::View; // Array mirror alias of arbitrary type template @@ -174,17 +174,17 @@ namespace kokkos_aliases_hidden { template <> struct range_impl { - using type = Kokkos::RangePolicy; + using type = Kokkos::RangePolicy; }; template <> struct range_impl { - using type = Kokkos::MDRangePolicy, AccelExeSpace>; + using type = Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>; }; template <> struct range_impl { - using type = Kokkos::MDRangePolicy, AccelExeSpace>; + using type = Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>; }; } // namespace kokkos_aliases_hidden @@ -201,17 +201,17 @@ namespace kokkos_aliases_hidden { template <> struct range_h_impl { - using type = Kokkos::RangePolicy; + using type = Kokkos::RangePolicy; }; template <> struct range_h_impl { - using type = Kokkos::MDRangePolicy, HostExeSpace>; + using type = Kokkos::MDRangePolicy, Kokkos::DefaultHostExecutionSpace>; }; template <> struct range_h_impl { - using type = Kokkos::MDRangePolicy, HostExeSpace>; + using type = Kokkos::MDRangePolicy, Kokkos::DefaultHostExecutionSpace>; }; } // namespace kokkos_aliases_hidden @@ -249,8 +249,8 @@ auto CreateRangePolicyOnHost(const tuple_t&, const tuple_t&) -> range_h_t; // Random number pool/generator type alias -using random_number_pool_t = Kokkos::Random_XorShift1024_Pool; -using random_generator_t = typename random_number_pool_t::generator_type; +using random_number_pool_t = Kokkos::Random_XorShift1024_Pool; +using random_generator_t = typename random_number_pool_t::generator_type; // Random number generator functions template diff --git a/src/kernels/tests/prtls_to_phys.cpp b/src/kernels/tests/prtls_to_phys.cpp index 4719fe6a1..0ceb88cd1 100644 --- a/src/kernels/tests/prtls_to_phys.cpp +++ b/src/kernels/tests/prtls_to_phys.cpp @@ -177,28 +177,29 @@ void testPrtl2PhysSR(const std::vector& res, array_t buff_ux3 { "buff_ux3", nprtl / stride }; array_t buff_wei { "buff_wei", nprtl / stride }; - Kokkos::parallel_for("Init", - Kokkos::RangePolicy(0, nprtl / stride), - kernel::PrtlToPhys_kernel(stride, - buff_x1, - buff_x2, - buff_x3, - buff_ux1, - buff_ux2, - buff_ux3, - buff_wei, - i1, - i2, - i3, - dx1, - dx2, - dx3, - ux1, - ux2, - ux3, - phi, - weight, - metric)); + Kokkos::parallel_for( + "Init", + Kokkos::RangePolicy(0, nprtl / stride), + kernel::PrtlToPhys_kernel(stride, + buff_x1, + buff_x2, + buff_x3, + buff_ux1, + buff_ux2, + buff_ux3, + buff_wei, + i1, + i2, + i3, + dx1, + dx2, + dx3, + ux1, + ux2, + ux3, + phi, + weight, + metric)); Kokkos::parallel_for("Check", nprtl / stride, Checker(metric, From b167d373c568f8e66597bac3b97a6d68061f345a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 7 Mar 2025 20:26:49 -0600 Subject: [PATCH 154/183] revert to old pgen for testing --- setups/srpic/shocktest/pgen.hpp | 171 +++++++++++++++++++++++--------- 1 file changed, 125 insertions(+), 46 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 4f6decc76..ff54923d1 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -38,20 +38,15 @@ namespace user { // magnetic field components Inline auto bx1(const coord_t& x_ph) const -> real_t { - // return Bmag * math::cos(Btheta); - return ZERO; + return Bmag * math::cos(Btheta); } Inline auto bx2(const coord_t& x_ph) const -> real_t { - // return Bmag * math::sin(Btheta) * math::sin(Bphi); - return ZERO; + return Bmag * math::sin(Btheta) * math::sin(Bphi); } Inline auto bx3(const coord_t& x_ph) const -> real_t { - // return Bmag * math::sin(Btheta) * math::cos(Bphi); - // return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 - // + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); - return ZERO; + return Bmag * math::sin(Btheta) * math::cos(Bphi); } // electric field components @@ -60,15 +55,11 @@ namespace user { } Inline auto ex2(const coord_t& x_ph) const -> real_t { - // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - // return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 - // + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); - return ZERO; + return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); } Inline auto ex3(const coord_t& x_ph) const -> real_t { - // return Vx * Bmag * math::sin(Btheta) * math::sin(Bphi); - return ZERO; + return Vx * Bmag * math::sin(Btheta) * math::sin(Bphi); } private: @@ -102,16 +93,16 @@ namespace user { : arch::ProblemGenerator { p } , drift_ux { p.template get("setup.drift_ux") } , temperature { p.template get("setup.temperature") } - , x1arr_e { p.template get>("setup.x_e") } - , x2arr_e { p.template get>("setup.y_e") } - , ux1arr_e { p.template get>("setup.ux_e") } - , ux2arr_e { p.template get>("setup.uy_e") } - , ux3arr_e { p.template get>("setup.uz_e") } - , x1arr_i { p.template get>("setup.x_i") } - , x2arr_i { p.template get>("setup.y_i") } - , ux1arr_i { p.template get>("setup.ux_i") } - , ux2arr_i { p.template get>("setup.uy_i") } - , ux3arr_i { p.template get>("setup.uz_i") } + // , x1arr_e { p.template get>("setup.x_e") } + // , x2arr_e { p.template get>("setup.y_e") } + // , ux1arr_e { p.template get>("setup.ux_e") } + // , ux2arr_e { p.template get>("setup.uy_e") } + // , ux3arr_e { p.template get>("setup.uz_e") } + // , x1arr_i { p.template get>("setup.x_i") } + // , x2arr_i { p.template get>("setup.y_i") } + // , ux1arr_i { p.template get>("setup.ux_i") } + // , ux2arr_i { p.template get>("setup.uy_i") } + // , ux3arr_i { p.template get>("setup.uz_i") } , Btheta { p.template get("setup.Btheta", ZERO) } , Bmag { p.template get("setup.Bmag", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } @@ -135,28 +126,116 @@ namespace user { return init_flds; } - inline void InitPrtls(Domain& domain) { - arch::InjectGlobally(*metadomain, - domain, - 1, - { - { "x1", x1arr_e }, - { "x2", x2arr_e }, - { "ux1", ux1arr_e }, - { "ux2", ux1arr_e }, - { "ux3", ux3arr_e } - }); - arch::InjectGlobally(*metadomain, - domain, - 2, - { - { "x1", x1arr_i }, - { "x2", x2arr_i }, - { "ux1", ux1arr_i }, - { "ux2", ux1arr_i }, - { "ux3", ux3arr_i } - }); - } + // inline void InitPrtls(Domain& domain) { + // arch::InjectGlobally(*metadomain, + // domain, + // 1, + // { + // { "x1", x1arr_e }, + // { "x2", x2arr_e }, + // { "ux1", ux1arr_e }, + // { "ux2", ux1arr_e }, + // { "ux3", ux3arr_e } + // }); + // arch::InjectGlobally(*metadomain, + // domain, + // 2, + // { + // { "x1", x1arr_i }, + // { "x2", x2arr_i }, + // { "ux1", ux1arr_i }, + // { "ux2", ux1arr_i }, + // { "ux3", ux3arr_i } + // }); + // } + + inline void InitPrtls(Domain& domain) { + + // auto& species_e = domain.species[0]; + // auto& species_p = domain.species[1]; + auto& species_e = domain.species[0]; + auto& species_p = domain.species[1]; + + // array_t elec_ind("elec_ind"); + // array_t pos_ind("pos_ind"); + array_t elec_ind("elec_ind"); + array_t pos_ind("pos_ind"); + + auto offset_e = species_e.npart(); + auto offset_p = species_p.npart(); + + auto ux1_e = species_e.ux1; + auto ux2_e = species_e.ux2; + auto ux3_e = species_e.ux3; + auto i1_e = species_e.i1; + auto i2_e = species_e.i2; + auto dx1_e = species_e.dx1; + auto dx2_e = species_e.dx2; + auto phi_e = species_e.phi; + auto weight_e = species_e.weight; + auto tag_e = species_e.tag; + + auto ux1_p = species_p.ux1; + auto ux2_p = species_p.ux2; + auto ux3_p = species_p.ux3; + auto i1_p = species_p.i1; + auto i2_p = species_p.i2; + auto dx1_p = species_p.dx1; + auto dx2_p = species_p.dx2; + auto phi_p = species_p.phi; + auto weight_p = species_p.weight; + auto tag_p = species_p.tag; + + int nseed = 1; + + Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { + + // ToDo: fix this + auto i1_ = math::floor(10); + auto i2_ = math::floor(64); + auto dx1_ = HALF; + auto dx2_ = HALF; + + + auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); + auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); + + i1_e(elec_p + offset_e) = i1_; + dx1_e(elec_p + offset_e) = dx1_; + i2_e(elec_p + offset_e) = i2_; + dx2_e(elec_p + offset_e) = 2*dx2_; + // ux1_e(elec_p + offset_e) = -0.5; + // ux2_e(elec_p + offset_e) = 0.5; + ux1_e(elec_p + offset_e) = -drift_ux; + ux2_e(elec_p + offset_e) = ZERO; + ux3_e(elec_p + offset_e) = ZERO; + weight_e(elec_p + offset_e) = ONE; + tag_e(elec_p + offset_e) = ParticleTag::alive; + + i1_p(pos_p + offset_p) = i1_; + dx1_p(pos_p + offset_p) = dx1_; + i2_p(pos_p + offset_p) = i2_; + dx2_p(pos_p + offset_p) = -2*dx2_; + // ux1_p(pos_p + offset_p) = 0.5; + // ux2_p(pos_p + offset_p) = -0.5; + ux1_p(pos_p + offset_p) = -drift_ux; + ux2_p(pos_p + offset_p) = ZERO; + ux3_p(pos_p + offset_p) = ZERO; + weight_p(pos_p + offset_p) = ONE; + tag_p(pos_p + offset_p) = ParticleTag::alive; + + + }); + + + auto elec_ind_h = Kokkos::create_mirror(elec_ind); + Kokkos::deep_copy(elec_ind_h, elec_ind); + species_e.set_npart(offset_e + elec_ind_h()); + + auto pos_ind_h = Kokkos::create_mirror(pos_ind); + Kokkos::deep_copy(pos_ind_h, pos_ind); + species_p.set_npart(offset_p + pos_ind_h()); + } }; } // namespace user From ec88a4fa8f0e16118cfd08bdcab9dd80546a15f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 7 Mar 2025 21:26:16 -0600 Subject: [PATCH 155/183] enforce B0/E0 at boundary --- src/engines/srpic.hpp | 52 -------------------------------------- src/kernels/fields_bcs.hpp | 33 +++++++++++++++--------- 2 files changed, 21 insertions(+), 64 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index f75989739..b1fca46da 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -854,19 +854,6 @@ namespace ntt { } else { const auto sign = direction.get_sign(); const auto dim = direction.get_dim(); -<<<<<<< HEAD -||||||| parent of bd723f39 (minor reformatting of conductor BC) - const auto sign = direction.get_sign(); - const auto dim = direction.get_dim(); - raise::ErrorIf(dim != in::x1 and M::CoordType != Coord::Cart, - "Perfect conductor BCs only implemented for x1 in " - "non-cartesian coordinates", - HERE); -======= - raise::ErrorIf(dim != in::x1, - "Perfect conductor BCs only implemented for x1", - HERE); ->>>>>>> bd723f39 (minor reformatting of conductor BC) std::vector xi_min, xi_max; @@ -876,13 +863,7 @@ namespace ntt { const auto dd = all_dirs[d]; if (dim == dd) { xi_min.push_back(0); -<<<<<<< HEAD xi_max.push_back((sign < 0) ? (N_GHOSTS + 1) : N_GHOSTS); -||||||| parent of bd723f39 (minor reformatting of conductor BC) - xi_max.push_back(N_GHOSTS); -======= - xi_max.push_back(N_GHOSTS + 1); ->>>>>>> bd723f39 (minor reformatting of conductor BC) } else { xi_min.push_back(0); xi_max.push_back(domain.mesh.n_all(dd)); @@ -905,7 +886,6 @@ namespace ntt { } else { raise::Error("Invalid dimension", HERE); } -<<<<<<< HEAD if (dim == in::x1) { if (sign > 0) { @@ -956,38 +936,6 @@ namespace ntt { tags)); } } -||||||| parent of bd723f39 (minor reformatting of conductor BC) -======= - Kokkos::parallel_for( - "MatchFields", - range, - kernel::bc::ConductorBoundaries_kernel(domain.fields.em, - tags)); - - // if constexpr (M::Dim == Dim::_1D) { - // Kokkos::parallel_for( - // "MatchFields", - // CreateRangePolicy({ xi_min[0] }, { xi_max[0] }), - // kernel::bc::ConductorBoundaries_kernel(domain.fields.em, - // tags)); - // } else if constexpr (M::Dim == Dim::_2D) { - // Kokkos::parallel_for( - // "MatchFields", - // CreateRangePolicy({ xi_min[0], xi_min[1] }, - // { xi_max[0], xi_max[1] }), - // kernel::bc::ConductorBoundaries_kernel(domain.fields.em, - // tags)); - // } else if constexpr (M::Dim == Dim::_3D) { - // Kokkos::parallel_for( - // "MatchFields", - // CreateRangePolicy({ xi_min[0], xi_min[1], xi_min[2] }, - // { xi_max[0], xi_max[1], xi_max[2] }), - // kernel::bc::ConductorBoundaries_kernel(domain.fields.em, - // tags)); - // } else { - // raise::Error("Invalid dimension", HERE); - // } ->>>>>>> bd723f39 (minor reformatting of conductor BC) } } diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 901f8e02a..dd926580d 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -486,25 +486,32 @@ namespace kernel::bc { } }; - template + template struct ConductorBoundaries_kernel { static_assert(static_cast(o) < static_cast(D), "Invalid component index"); - // static constexpr idx_t i = static_cast(o) + 1u; + static constexpr bool defines_ex2 = traits::has_method::value; + static constexpr bool defines_ex3 = traits::has_method::value; + static constexpr bool defines_bx1 = traits::has_method::value; + static_assert(( defines_ex2 or defines_ex3 or defines_bx1), + "none of the components of E or B are specified in PGEN"); ndfield_t Fld; + const I fset; const BCTags tags; - ConductorBoundaries_kernel(ndfield_t Fld, BCTags tags) + ConductorBoundaries_kernel(ndfield_t Fld, const I& fset, BCTags tags) : Fld { Fld } + , fset { fset } , tags { tags } {} Inline void operator()(index_t i1) const { if constexpr (D == Dim::_1D) { + coord_t x_Ph_H { ZERO }; if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, em::ex2) = ZERO; - Fld(N_GHOSTS, em::ex3) = ZERO; + Fld(N_GHOSTS, em::ex2) = fset.ex2(x_Ph_H); + Fld(N_GHOSTS, em::ex3) = fset.ex3(x_Ph_H); } else { Fld(N_GHOSTS - i1, em::ex1) = Fld(N_GHOSTS + i1 - 1, em::ex1); Fld(N_GHOSTS - i1, em::ex2) = -Fld(N_GHOSTS + i1, em::ex2); @@ -514,7 +521,7 @@ namespace kernel::bc { if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, em::bx1) = ZERO; + Fld(N_GHOSTS, em::bx1) = fset.bx1(x_Ph_H); } else { Fld(N_GHOSTS - i1, em::bx1) = -Fld(N_GHOSTS + i1, em::bx1); Fld(N_GHOSTS - i1, em::bx2) = Fld(N_GHOSTS + i1 - 1, em::bx2); @@ -530,10 +537,11 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2) const { if constexpr (D == Dim::_2D) { + coord_t x_Ph_H { ZERO }; if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, i2, em::ex2) = ZERO; - Fld(N_GHOSTS, i2, em::ex3) = ZERO; + Fld(N_GHOSTS, i2, em::ex2) = fset.ex2(x_Ph_H); + Fld(N_GHOSTS, i2, em::ex3) = fset.ex3(x_Ph_H); } else { Fld(N_GHOSTS - i1, i2, em::ex1) = Fld(N_GHOSTS + i1 - 1, i2, em::ex1); Fld(N_GHOSTS - i1, i2, em::ex2) = -Fld(N_GHOSTS + i1, i2, em::ex2); @@ -543,7 +551,7 @@ namespace kernel::bc { if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, i2, em::bx1) = ZERO; + Fld(N_GHOSTS, i2, em::bx1) = fset.bx1(x_Ph_H); } else { Fld(N_GHOSTS - i1, i2, em::bx1) = -Fld(N_GHOSTS + i1, i2, em::bx1); Fld(N_GHOSTS - i1, i2, em::bx2) = Fld(N_GHOSTS + i1 - 1, i2, em::bx2); @@ -559,10 +567,11 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2, index_t i3) const { if constexpr (D == Dim::_3D) { + coord_t x_Ph_H { ZERO }; if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, i2, i3, em::ex2) = ZERO; - Fld(N_GHOSTS, i2, i3, em::ex3) = ZERO; + Fld(N_GHOSTS, i2, i3, em::ex2) = fset.ex2(x_Ph_H); + Fld(N_GHOSTS, i2, i3, em::ex3) = fset.ex3(x_Ph_H); } else { Fld(N_GHOSTS - i1, i2, i3, em::ex1) = Fld(N_GHOSTS + i1 - 1, i2, @@ -575,7 +584,7 @@ namespace kernel::bc { if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, i2, i3, em::bx1) = ZERO; + Fld(N_GHOSTS, i2, i3, em::bx1) = fset.bx1(x_Ph_H); } else { Fld(N_GHOSTS - i1, i2, i3, em::bx1) = -Fld(N_GHOSTS + i1, i2, i3, em::bx1); Fld(N_GHOSTS - i1, i2, i3, em::bx2) = Fld(N_GHOSTS + i1 - 1, From 680b950db0a8bdaf88d75c405296024daf116790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 10 Mar 2025 12:00:38 -0500 Subject: [PATCH 156/183] revert to zero at boundary --- src/kernels/fields_bcs.hpp | 44 ++++++++++++-------------------------- 1 file changed, 14 insertions(+), 30 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index dd926580d..8ea6b72d8 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -486,32 +486,24 @@ namespace kernel::bc { } }; - template + template struct ConductorBoundaries_kernel { static_assert(static_cast(o) < static_cast(D), "Invalid component index"); - static constexpr bool defines_ex2 = traits::has_method::value; - static constexpr bool defines_ex3 = traits::has_method::value; - static constexpr bool defines_bx1 = traits::has_method::value; - static_assert(( defines_ex2 or defines_ex3 or defines_bx1), - "none of the components of E or B are specified in PGEN"); ndfield_t Fld; - const I fset; const BCTags tags; - ConductorBoundaries_kernel(ndfield_t Fld, const I& fset, BCTags tags) + ConductorBoundaries_kernel(ndfield_t Fld, BCTags tags) : Fld { Fld } - , fset { fset } , tags { tags } {} Inline void operator()(index_t i1) const { if constexpr (D == Dim::_1D) { - coord_t x_Ph_H { ZERO }; if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, em::ex2) = fset.ex2(x_Ph_H); - Fld(N_GHOSTS, em::ex3) = fset.ex3(x_Ph_H); + Fld(N_GHOSTS, em::ex2) = ZERO; + Fld(N_GHOSTS, em::ex3) = ZERO; } else { Fld(N_GHOSTS - i1, em::ex1) = Fld(N_GHOSTS + i1 - 1, em::ex1); Fld(N_GHOSTS - i1, em::ex2) = -Fld(N_GHOSTS + i1, em::ex2); @@ -521,7 +513,7 @@ namespace kernel::bc { if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, em::bx1) = fset.bx1(x_Ph_H); + Fld(N_GHOSTS, em::bx1) = ZERO; } else { Fld(N_GHOSTS - i1, em::bx1) = -Fld(N_GHOSTS + i1, em::bx1); Fld(N_GHOSTS - i1, em::bx2) = Fld(N_GHOSTS + i1 - 1, em::bx2); @@ -537,11 +529,10 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2) const { if constexpr (D == Dim::_2D) { - coord_t x_Ph_H { ZERO }; if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, i2, em::ex2) = fset.ex2(x_Ph_H); - Fld(N_GHOSTS, i2, em::ex3) = fset.ex3(x_Ph_H); + Fld(N_GHOSTS, i2, em::ex2) = ZERO; + Fld(N_GHOSTS, i2, em::ex3) = ZERO; } else { Fld(N_GHOSTS - i1, i2, em::ex1) = Fld(N_GHOSTS + i1 - 1, i2, em::ex1); Fld(N_GHOSTS - i1, i2, em::ex2) = -Fld(N_GHOSTS + i1, i2, em::ex2); @@ -551,7 +542,7 @@ namespace kernel::bc { if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, i2, em::bx1) = fset.bx1(x_Ph_H); + Fld(N_GHOSTS, i2, em::bx1) = ZERO; } else { Fld(N_GHOSTS - i1, i2, em::bx1) = -Fld(N_GHOSTS + i1, i2, em::bx1); Fld(N_GHOSTS - i1, i2, em::bx2) = Fld(N_GHOSTS + i1 - 1, i2, em::bx2); @@ -567,16 +558,13 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2, index_t i3) const { if constexpr (D == Dim::_3D) { - coord_t x_Ph_H { ZERO }; if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, i2, i3, em::ex2) = fset.ex2(x_Ph_H); - Fld(N_GHOSTS, i2, i3, em::ex3) = fset.ex3(x_Ph_H); + Fld(N_GHOSTS, i2, i3, em::ex2) = ZERO; + Fld(N_GHOSTS, i2, i3, em::ex3) = ZERO; } else { Fld(N_GHOSTS - i1, i2, i3, em::ex1) = Fld(N_GHOSTS + i1 - 1, - i2, - i3, - em::ex1); + i2, i3, em::ex1); Fld(N_GHOSTS - i1, i2, i3, em::ex2) = -Fld(N_GHOSTS + i1, i2, i3, em::ex2); Fld(N_GHOSTS - i1, i2, i3, em::ex3) = -Fld(N_GHOSTS + i1, i2, i3, em::ex3); } @@ -584,17 +572,13 @@ namespace kernel::bc { if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, i2, i3, em::bx1) = fset.bx1(x_Ph_H); + Fld(N_GHOSTS, i2, i3, em::bx1) = ZERO; } else { Fld(N_GHOSTS - i1, i2, i3, em::bx1) = -Fld(N_GHOSTS + i1, i2, i3, em::bx1); Fld(N_GHOSTS - i1, i2, i3, em::bx2) = Fld(N_GHOSTS + i1 - 1, - i2, - i3, - em::bx2); + i2, i3, em::bx2); Fld(N_GHOSTS - i1, i2, i3, em::bx3) = Fld(N_GHOSTS + i1 - 1, - i2, - i3, - em::bx3); + i2, i3, em::bx3); } } } else { From de24703779b97f928042853abfcbbf211666b08a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Thu, 13 Mar 2025 10:00:56 -0500 Subject: [PATCH 157/183] cleanup of pgen --- setups/srpic/shock/pgen.hpp | 46 ------------------------------------- 1 file changed, 46 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 59c5590c9..b8f169521 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -106,52 +106,6 @@ namespace user { } } - - auto PerfectConductorFieldsConst(const bc_in&, const em& comp) const -> real_t{ - - // electric field components - if (comp == em::ex1) { - return ONE; - } else if (comp == em::ex2) { - return -ONE; - } else if (comp == em::ex3) { - return -ONE; } - // magentic field components - else if (comp == em::bx1) { - return -ONE; - } else if (comp == em::bx2) { - return ONE; - } else if (comp == em::bx3) { - return ONE;} - // should never be the case - else - { - return ZERO; - } - } - - // auto PerfectConductorCurrentsConst(const bc_in &, const cur &comp) const - // -> std::pair - // { - // // ToDo - // if (comp == cur::jx1) - // { - // return ZERO; - // } - // else if (comp == cur::jx2) - // { - // return ZERO; - // } - // else if (comp == cur::jx3) - // { - // return ZERO; - // } - // else - // { - // return ZERO; - // } - // } - auto MatchFields(real_t time) const -> InitFields { return init_flds; } From a41bbe2b726d2d785f0e48ccd82686f1d20aba77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Thu, 13 Mar 2025 11:03:56 -0500 Subject: [PATCH 158/183] update example parameter file --- setups/srpic/shock/shock.toml | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/setups/srpic/shock/shock.toml b/setups/srpic/shock/shock.toml index 4ed3a2b9e..4e03721f7 100644 --- a/setups/srpic/shock/shock.toml +++ b/setups/srpic/shock/shock.toml @@ -1,22 +1,22 @@ [simulation] name = "shock" engine = "srpic" - runtime = 50.0 + runtime = 30000.0 [grid] - resolution = [2048, 128] - extent = [[0.0, 10.0], [-0.3125, 0.3125]] + resolution = [16384, 128] + extent = [[0.0, 8000.0], [-31.25, 31.25]] [grid.metric] metric = "minkowski" [grid.boundaries] - fields = [["ABSORB", "FIXED"], ["PERIODIC"]] + fields = [["CONDUCTOR", "MATCH"], ["PERIODIC"]] particles = [["REFLECT", "ABSORB"], ["PERIODIC"]] - + [scales] - larmor0 = 1e-2 - skindepth0 = 1e-2 + larmor0 = 100.0 + skindepth0 = 1.0 [algorithms] current_filters = 8 @@ -25,7 +25,7 @@ CFL = 0.5 [particles] - ppc0 = 16.0 + ppc0 = 8.0 [[particles.species]] label = "e-" @@ -34,21 +34,29 @@ maxnpart = 1e8 [[particles.species]] - label = "e+" + label = "p+" mass = 1.0 charge = 1.0 maxnpart = 1e8 [setup] drift_ux = 0.1 - temperature = 1e-3 + temperature = 1e-4 Bmag = 1.0 Btheta = 0.0 Bphi = 0.0 [output] - interval_time = 0.1 + interval_time = 500.0 format = "hdf5" - + [output.fields] - quantities = ["N_1", "N_2", "E", "B", "T0i_1", "T0i_2", "J"] + quantities = ["N_1", "N_2", "B", "E"] + + [output.particles] + enable = true + stride = 10 + + [output.spectra] + enable = false + From 5d9fe735066c5a0e3cb33d2b67c215475f1a02fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Wed, 19 Mar 2025 15:40:05 -0500 Subject: [PATCH 159/183] revert to cleaner injector for shocktest --- setups/srpic/shocktest/pgen.hpp | 152 +++++++------------------------- 1 file changed, 32 insertions(+), 120 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index ff54923d1..a7bcf5c3d 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -93,16 +93,16 @@ namespace user { : arch::ProblemGenerator { p } , drift_ux { p.template get("setup.drift_ux") } , temperature { p.template get("setup.temperature") } - // , x1arr_e { p.template get>("setup.x_e") } - // , x2arr_e { p.template get>("setup.y_e") } - // , ux1arr_e { p.template get>("setup.ux_e") } - // , ux2arr_e { p.template get>("setup.uy_e") } - // , ux3arr_e { p.template get>("setup.uz_e") } - // , x1arr_i { p.template get>("setup.x_i") } - // , x2arr_i { p.template get>("setup.y_i") } - // , ux1arr_i { p.template get>("setup.ux_i") } - // , ux2arr_i { p.template get>("setup.uy_i") } - // , ux3arr_i { p.template get>("setup.uz_i") } + , x1arr_e { p.template get>("setup.x_e") } + , x2arr_e { p.template get>("setup.y_e") } + , ux1arr_e { p.template get>("setup.ux_e") } + , ux2arr_e { p.template get>("setup.uy_e") } + , ux3arr_e { p.template get>("setup.uz_e") } + , x1arr_i { p.template get>("setup.x_i") } + , x2arr_i { p.template get>("setup.y_i") } + , ux1arr_i { p.template get>("setup.ux_i") } + , ux2arr_i { p.template get>("setup.uy_i") } + , ux3arr_i { p.template get>("setup.uz_i") } , Btheta { p.template get("setup.Btheta", ZERO) } , Bmag { p.template get("setup.Bmag", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } @@ -126,116 +126,28 @@ namespace user { return init_flds; } - // inline void InitPrtls(Domain& domain) { - // arch::InjectGlobally(*metadomain, - // domain, - // 1, - // { - // { "x1", x1arr_e }, - // { "x2", x2arr_e }, - // { "ux1", ux1arr_e }, - // { "ux2", ux1arr_e }, - // { "ux3", ux3arr_e } - // }); - // arch::InjectGlobally(*metadomain, - // domain, - // 2, - // { - // { "x1", x1arr_i }, - // { "x2", x2arr_i }, - // { "ux1", ux1arr_i }, - // { "ux2", ux1arr_i }, - // { "ux3", ux3arr_i } - // }); - // } - - inline void InitPrtls(Domain& domain) { - - // auto& species_e = domain.species[0]; - // auto& species_p = domain.species[1]; - auto& species_e = domain.species[0]; - auto& species_p = domain.species[1]; - - // array_t elec_ind("elec_ind"); - // array_t pos_ind("pos_ind"); - array_t elec_ind("elec_ind"); - array_t pos_ind("pos_ind"); - - auto offset_e = species_e.npart(); - auto offset_p = species_p.npart(); - - auto ux1_e = species_e.ux1; - auto ux2_e = species_e.ux2; - auto ux3_e = species_e.ux3; - auto i1_e = species_e.i1; - auto i2_e = species_e.i2; - auto dx1_e = species_e.dx1; - auto dx2_e = species_e.dx2; - auto phi_e = species_e.phi; - auto weight_e = species_e.weight; - auto tag_e = species_e.tag; - - auto ux1_p = species_p.ux1; - auto ux2_p = species_p.ux2; - auto ux3_p = species_p.ux3; - auto i1_p = species_p.i1; - auto i2_p = species_p.i2; - auto dx1_p = species_p.dx1; - auto dx2_p = species_p.dx2; - auto phi_p = species_p.phi; - auto weight_p = species_p.weight; - auto tag_p = species_p.tag; - - int nseed = 1; - - Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { - - // ToDo: fix this - auto i1_ = math::floor(10); - auto i2_ = math::floor(64); - auto dx1_ = HALF; - auto dx2_ = HALF; - - - auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); - auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); - - i1_e(elec_p + offset_e) = i1_; - dx1_e(elec_p + offset_e) = dx1_; - i2_e(elec_p + offset_e) = i2_; - dx2_e(elec_p + offset_e) = 2*dx2_; - // ux1_e(elec_p + offset_e) = -0.5; - // ux2_e(elec_p + offset_e) = 0.5; - ux1_e(elec_p + offset_e) = -drift_ux; - ux2_e(elec_p + offset_e) = ZERO; - ux3_e(elec_p + offset_e) = ZERO; - weight_e(elec_p + offset_e) = ONE; - tag_e(elec_p + offset_e) = ParticleTag::alive; - - i1_p(pos_p + offset_p) = i1_; - dx1_p(pos_p + offset_p) = dx1_; - i2_p(pos_p + offset_p) = i2_; - dx2_p(pos_p + offset_p) = -2*dx2_; - // ux1_p(pos_p + offset_p) = 0.5; - // ux2_p(pos_p + offset_p) = -0.5; - ux1_p(pos_p + offset_p) = -drift_ux; - ux2_p(pos_p + offset_p) = ZERO; - ux3_p(pos_p + offset_p) = ZERO; - weight_p(pos_p + offset_p) = ONE; - tag_p(pos_p + offset_p) = ParticleTag::alive; - - - }); - - - auto elec_ind_h = Kokkos::create_mirror(elec_ind); - Kokkos::deep_copy(elec_ind_h, elec_ind); - species_e.set_npart(offset_e + elec_ind_h()); - - auto pos_ind_h = Kokkos::create_mirror(pos_ind); - Kokkos::deep_copy(pos_ind_h, pos_ind); - species_p.set_npart(offset_p + pos_ind_h()); - } + inline void InitPrtls(Domain& domain) { + arch::InjectGlobally(*metadomain, + domain, + 1, + { + { "x1", x1arr_e }, + { "x2", x2arr_e }, + { "ux1", ux1arr_e }, + { "ux2", ux1arr_e }, + { "ux3", ux3arr_e } + }); + arch::InjectGlobally(*metadomain, + domain, + 2, + { + { "x1", x1arr_i }, + { "x2", x2arr_i }, + { "ux1", ux1arr_i }, + { "ux2", ux1arr_i }, + { "ux3", ux3arr_i } + }); + } }; } // namespace user From 6d43018085d3923ab7aae5d25b9917d2e690321f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Thu, 20 Mar 2025 17:06:25 -0500 Subject: [PATCH 160/183] implemented new `arch::Piston` spatial distribution to set up shock partially filled with plasma --- setups/srpic/shock/pgen.hpp | 59 ++++++++++++++++++++++------------- setups/srpic/shock/shock.toml | 11 ++++--- src/archetypes/spatial_dist.h | 30 ++++++++++++++++++ 3 files changed, 74 insertions(+), 26 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index b8f169521..9bf51b866 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -79,7 +79,7 @@ namespace user { using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; - const real_t drift_ux, temperature; + const real_t drift_ux, temperature, filling_fraction; const real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -91,40 +91,57 @@ namespace user { , Bmag { p.template get("setup.Bmag", ZERO) } , Btheta { p.template get("setup.Btheta", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } - , init_flds { Bmag, Btheta, Bphi, drift_ux } {} + , init_flds { Bmag, Btheta, Bphi, drift_ux } + , filling_fraction { params.template get("setup.filling_fraction", 1.0) }{} inline PGen() {} - auto FixFieldsConst(const bc_in&, const em& comp) const - -> std::pair { - if (comp == em::ex2) { - return { init_flds.ex2({ ZERO }), true }; - } else if (comp == em::ex3) { - return { init_flds.ex3({ ZERO }), true }; - } else { - return { ZERO, false }; - } - } - auto MatchFields(real_t time) const -> InitFields { return init_flds; } inline void InitPrtls(Domain& local_domain) { + + // minimum and maximum position of particles + real_t xg_min = local_domain.mesh.extent(in::x1).first; + real_t xg_max = local_domain.mesh.extent(in::x1).second * filling_fraction; + + // define box to inject into + boundaries_t box; + // loop over all dimensions + for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { + // compute the range for the x-direction + if (d == static_cast(in::x1)) { + box.push_back({xg_min, xg_max}); + } else { + // inject into full range in other directions + box.push_back(Range::All); + } + } + + // spatial distribution of the particles + // -> hack to use the uniform distribution in NonUniformInjector + const auto spatial_dist = arch::Piston(local_domain.mesh.metric, xg_min, xg_max, in::x1); + + // energy distribution of the particles const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, local_domain.random_pool, temperature, -drift_ux, in::x1); - const auto injector = arch::UniformInjector( - energy_dist, - { 1, 2 }); - arch::InjectUniform>( - params, - local_domain, - injector, - 1.0); + const auto injector = arch::NonUniformInjector( + energy_dist, + spatial_dist, + {1, 2}); + + arch::InjectNonUniform>( + params, + local_domain, + injector, + 1.0, // target density + false, // no weights + box); } }; diff --git a/setups/srpic/shock/shock.toml b/setups/srpic/shock/shock.toml index 4e03721f7..f954345c2 100644 --- a/setups/srpic/shock/shock.toml +++ b/setups/srpic/shock/shock.toml @@ -40,11 +40,12 @@ maxnpart = 1e8 [setup] - drift_ux = 0.1 - temperature = 1e-4 - Bmag = 1.0 - Btheta = 0.0 - Bphi = 0.0 + drift_ux = 0.1 # speed towards the wall [c] + temperature = 1e-4 # temeperature of maxwell distribution [m_e c^2] + Bmag = 1.0 # magnetic field strength as fraction of magnetisation + Btheta = 0.0 # magnetic field angle in the plane + Bphi = 0.0 # magnetic field angle out of plane + filling_fraction = 1.0 # fraction of the shock piston filled with plasma [output] interval_time = 500.0 diff --git a/src/archetypes/spatial_dist.h b/src/archetypes/spatial_dist.h index d036c0166..2407d98cd 100644 --- a/src/archetypes/spatial_dist.h +++ b/src/archetypes/spatial_dist.h @@ -4,6 +4,7 @@ * @implements * - arch::SpatialDistribution<> * - arch::Uniform<> : arch::SpatialDistribution<> + * - arch::Piston<> : arch::SpatialDistribution<> * - arch::Replenish<> : arch::SpatialDistribution<> * @namespace * - arch:: @@ -49,6 +50,35 @@ namespace arch { } }; + template + struct Piston : public arch::SpatialDistribution + { + Piston(const M &metric, real_t xmin, real_t xmax, in piston_direction = in::x1) + : arch::SpatialDistribution{metric} + , xmin {xmin} + , xmax {xmax} + , piston_direction {piston_direction} {} + + Inline auto operator()(const coord_t &x_Ph) const -> real_t override + { + // dimentsion to fill + const auto fill_dim = static_cast(piston_direction); + + if (x_Ph[fill_dim] < xmin || x_Ph[fill_dim] > xmax) + { + return 0.0; + } + else + { + return 1.0; + } + } + + private: + real_t xmin, xmax; + in piston_direction; + }; + template struct Replenish : public SpatialDistribution { using SpatialDistribution::metric; From 4f5b1d87e3b6f5321e3c51e53bfc1de6bd45dc6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Thu, 20 Mar 2025 17:43:25 -0500 Subject: [PATCH 161/183] fix inconsistency in return type --- src/archetypes/spatial_dist.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/archetypes/spatial_dist.h b/src/archetypes/spatial_dist.h index 2407d98cd..2edbf5b1d 100644 --- a/src/archetypes/spatial_dist.h +++ b/src/archetypes/spatial_dist.h @@ -66,11 +66,11 @@ namespace arch { if (x_Ph[fill_dim] < xmin || x_Ph[fill_dim] > xmax) { - return 0.0; + return ZERO; } else { - return 1.0; + return ONE; } } From 3e9c4acc7236a4c912e507708657e491f34c25fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Mar 2025 16:24:35 -0500 Subject: [PATCH 162/183] Added `MovingInjector` (wip) --- src/archetypes/particle_injector.h | 69 ++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/src/archetypes/particle_injector.h b/src/archetypes/particle_injector.h index 3884e6ced..daa41506c 100644 --- a/src/archetypes/particle_injector.h +++ b/src/archetypes/particle_injector.h @@ -170,6 +170,75 @@ namespace arch { ~AtmosphereInjector() = default; }; + template + struct MovingInjector + { + struct TargetDensityProfile + { + const real_t nmax, xinj, xdrift; + + TargetDensityProfile(real_t xinj, real_t xdrift, real_t nmax) + : xinj{xinj}, xdrift{xdrift}, nmax{nmax} {} + + Inline auto operator()(const coord_t &x_Ph) const -> real_t + { + if constexpr ((O == in::x1) or + (O == in::x2 and (M::Dim == Dim::_2D or M::Dim == Dim::_3D)) or + (O == in::x3 and M::Dim == Dim::_3D)) + { + const auto xi = x_Ph[static_cast(O)]; + // + direction + if (xi < xinj - xdrift or xi >= xinj) + { + return ZERO; + } + else + { + if constexpr (M::CoordType == Coord::Cart) + { + return nmax; + } + else + { + raise::KernelError( + HERE, + "Moving injector in +x cannot be applied for non-cartesian"); + return ZERO; + } + } + } + else + { + raise::KernelError(HERE, "Wrong direction"); + return ZERO; + } + } + }; + using energy_dist_t = Maxwellian; + using spatial_dist_t = Replenish; + static_assert(M::is_metric, "M must be a metric class"); + static constexpr bool is_nonuniform_injector{true}; + static constexpr Dimension D{M::Dim}; + static constexpr Coord C{M::CoordType}; + + const energy_dist_t energy_dist; + const TargetDensityProfile target_density; + const spatial_dist_t spatial_dist; + const std::pair species; + + MovingInjector(const M &metric, + const ndfield_t &density, + const energy_dist_t &energy_dist, + real_t xinj, + real_t xdrift, + real_t nmax, + const std::pair &species) + : energy_dist{energy_dist}, + target_density{xinj, xdrift, nmax}, spatial_dist{metric, density, 0, target_density, nmax}, species{species} {} + + ~MovingInjector() = default; + }; + /** * @brief Injects uniform number density of particles everywhere in the domain * @param domain Domain object From e724b506f2acef8869c1a92bb476ff4c7065ac1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Mar 2025 16:25:10 -0500 Subject: [PATCH 163/183] Added `CustomPostStep` for shock pgen to continually inject particles (wip) --- setups/srpic/shock/pgen.hpp | 78 ++++++++++++++++++++++++++++++++++- setups/srpic/shock/shock.toml | 4 +- src/engines/engine_run.cpp | 2 +- 3 files changed, 79 insertions(+), 5 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 9bf51b866..7c0d750e3 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -79,7 +79,7 @@ namespace user { using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; - const real_t drift_ux, temperature, filling_fraction; + const real_t drift_ux, temperature, filling_fraction, injection_rate; const real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -92,7 +92,8 @@ namespace user { , Btheta { p.template get("setup.Btheta", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } , init_flds { Bmag, Btheta, Bphi, drift_ux } - , filling_fraction { params.template get("setup.filling_fraction", 1.0) }{} + , filling_fraction { params.template get("setup.filling_fraction", 1.0) } + , injection_rate { params.template get("setup.injection_rate", 1.0) } {} inline PGen() {} @@ -100,6 +101,11 @@ namespace user { return init_flds; } + // Inline auto TargetDensity(const coord_t &x_Ph) const -> real_t + // { + // return ONE; + // } + inline void InitPrtls(Domain& local_domain) { // minimum and maximum position of particles @@ -143,6 +149,74 @@ namespace user { false, // no weights box); } + + void CustomPostStep(std::size_t, long double time, real_t dt, Domain &domain) + { + /* + Moving injector for the particles + */ + + // energy distribution of the particles + const auto energy_dist = arch::Maxwellian(domain.mesh.metric, + domain.random_pool, + temperature, + -drift_ux, + in::x1); + + // minimum and maximum position of particles + // ToDo: Add time offset for start of movement? + real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction + + injection_rate * time - drift_ux * dt; + real_t xg_max = domain.mesh.extent(in::x1).second * filling_fraction + + injection_rate * time; + + + /* + I thought this option would be better, but I can't get it to work + */ + + // // define box to inject into + // boundaries_t box; + // // loop over all dimensions + // for (unsigned short d{0}; d < static_cast(M::Dim); ++d) + // { + // // compute the range for the x-direction + // if (d == static_cast(in::x1)) + // { + // box.push_back({xg_min, xg_max}); + // } else { + // // inject into full range in other directions + // box.push_back(Range::All); + // } + // } + + // const auto spatial_dist = arch::Replenish(domain.mesh.metric, + // domain.fields.bckp, + // box, + // TargetDensity, + // 1.0); + + // const auto injector = arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // {1, 2}); + + const auto injector = arch::MovingInjector{ + domain.mesh.metric, + domain.fields.bckp, + energy_dist, + xg_max, + xg_min, + 1.0, + {1, 2}}; + + arch::InjectNonUniform( + params, + domain, + injector, + 1.0, // target density + false); + } }; } // namespace user diff --git a/setups/srpic/shock/shock.toml b/setups/srpic/shock/shock.toml index f954345c2..772b452c9 100644 --- a/setups/srpic/shock/shock.toml +++ b/setups/srpic/shock/shock.toml @@ -45,10 +45,10 @@ Bmag = 1.0 # magnetic field strength as fraction of magnetisation Btheta = 0.0 # magnetic field angle in the plane Bphi = 0.0 # magnetic field angle out of plane - filling_fraction = 1.0 # fraction of the shock piston filled with plasma + filling_fraction = 0.1 # fraction of the shock piston filled with plasma [output] - interval_time = 500.0 + interval_time = 10.0 format = "hdf5" [output.fields] diff --git a/src/engines/engine_run.cpp b/src/engines/engine_run.cpp index 2d4b0d5ed..a3ae21ccb 100644 --- a/src/engines/engine_run.cpp +++ b/src/engines/engine_run.cpp @@ -52,7 +52,7 @@ namespace ntt { traits::has_method::value) { timers.start("Custom"); m_metadomain.runOnLocalDomains([&timers, this](auto& dom) { - m_pgen.CustomPostStep(step, time, dom); + m_pgen.CustomPostStep(step, time, dt, dom); }); timers.stop("Custom"); } From 236060b4b9b092c5e568b5691a76ac2f76772fce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Sat, 22 Mar 2025 11:42:48 -0500 Subject: [PATCH 164/183] bugfix --- src/archetypes/particle_injector.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/archetypes/particle_injector.h b/src/archetypes/particle_injector.h index daa41506c..676cf2857 100644 --- a/src/archetypes/particle_injector.h +++ b/src/archetypes/particle_injector.h @@ -188,7 +188,7 @@ namespace arch { { const auto xi = x_Ph[static_cast(O)]; // + direction - if (xi < xinj - xdrift or xi >= xinj) + if (xi < xdrift or xi >= xinj) { return ZERO; } From 2391bee2aad7d3d22427fe3481675aab9ebf4d3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Sat, 22 Mar 2025 11:56:32 -0500 Subject: [PATCH 165/183] added start time of injection and explicitly enforce injection region (wip) --- setups/srpic/shock/pgen.hpp | 102 +++++++++++++++++----------------- setups/srpic/shock/shock.toml | 2 + 2 files changed, 54 insertions(+), 50 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 7c0d750e3..04de58406 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -79,7 +79,8 @@ namespace user { using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; - const real_t drift_ux, temperature, filling_fraction, injection_rate; + const real_t drift_ux, temperature, filling_fraction, + injector_velocity, injection_start; const real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -93,7 +94,8 @@ namespace user { , Bphi { p.template get("setup.Bphi", ZERO) } , init_flds { Bmag, Btheta, Bphi, drift_ux } , filling_fraction { params.template get("setup.filling_fraction", 1.0) } - , injection_rate { params.template get("setup.injection_rate", 1.0) } {} + , injector_velocity { params.template get("setup.injector_velocity", 1.0) } + , injection_start { params.template get("setup.injection_start", 0.0) } {} inline PGen() {} @@ -164,58 +166,58 @@ namespace user { in::x1); // minimum and maximum position of particles - // ToDo: Add time offset for start of movement? - real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction + - injection_rate * time - drift_ux * dt; + real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction + + injector_velocity * (time - injection_start) + - drift_ux; // distance particles have moved in the last time step real_t xg_max = domain.mesh.extent(in::x1).second * filling_fraction + - injection_rate * time; + injector_velocity * (time - injection_start); + /* + I thought this option would be better, but I can't get it to work + */ - /* - I thought this option would be better, but I can't get it to work - */ - - // // define box to inject into - // boundaries_t box; - // // loop over all dimensions - // for (unsigned short d{0}; d < static_cast(M::Dim); ++d) - // { - // // compute the range for the x-direction - // if (d == static_cast(in::x1)) - // { - // box.push_back({xg_min, xg_max}); - // } else { - // // inject into full range in other directions - // box.push_back(Range::All); - // } - // } - - // const auto spatial_dist = arch::Replenish(domain.mesh.metric, - // domain.fields.bckp, - // box, - // TargetDensity, - // 1.0); - - // const auto injector = arch::NonUniformInjector( - // energy_dist, - // spatial_dist, - // {1, 2}); - - const auto injector = arch::MovingInjector{ - domain.mesh.metric, - domain.fields.bckp, - energy_dist, - xg_max, - xg_min, - 1.0, - {1, 2}}; + // define box to inject into + boundaries_t box; + // loop over all dimensions + for (unsigned short d{0}; d < static_cast(M::Dim); ++d) + { + // compute the range for the x-direction + if (d == static_cast(in::x1)) + { + box.push_back({xg_min, xg_max}); + } else { + // inject into full range in other directions + box.push_back(Range::All); + } + } - arch::InjectNonUniform( - params, - domain, - injector, - 1.0, // target density - false); + // const auto spatial_dist = arch::Replenish(domain.mesh.metric, + // domain.fields.bckp, + // box, + // TargetDensity, + // 1.0); + + // const auto injector = arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // {1, 2}); + + const auto injector = arch::MovingInjector{ + domain.mesh.metric, + domain.fields.bckp, + energy_dist, + xg_max, + xg_min, + 1.0, + {1, 2}}; + + arch::InjectNonUniform( + params, + domain, + injector, + 1.0, // target density + false, + box); } }; diff --git a/setups/srpic/shock/shock.toml b/setups/srpic/shock/shock.toml index 772b452c9..43da87d99 100644 --- a/setups/srpic/shock/shock.toml +++ b/setups/srpic/shock/shock.toml @@ -46,6 +46,8 @@ Btheta = 0.0 # magnetic field angle in the plane Bphi = 0.0 # magnetic field angle out of plane filling_fraction = 0.1 # fraction of the shock piston filled with plasma + injector_velocity = 1.0 # speed of injector [c] + injection_start = 0.0 # start time of moving injector [output] interval_time = 10.0 From d67f6c746e360b6e27aa7aabdb97d314e9c5d467 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Sat, 22 Mar 2025 12:12:00 -0500 Subject: [PATCH 166/183] small bugfix --- setups/srpic/shock/pgen.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 04de58406..1c9c9a4d3 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -167,8 +167,8 @@ namespace user { // minimum and maximum position of particles real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction - + injector_velocity * (time - injection_start) - - drift_ux; // distance particles have moved in the last time step + + injector_velocity * (time - injection_start - dt) + - drift_ux * dt; // distance particles have moved in the last time step real_t xg_max = domain.mesh.extent(in::x1).second * filling_fraction + injector_velocity * (time - injection_start); From bb8e30810d140863dcbbad1def9ce1b1ae5da202 Mon Sep 17 00:00:00 2001 From: hayk Date: Mon, 24 Mar 2025 13:14:51 -0400 Subject: [PATCH 167/183] shock pgen changed --- setups/srpic/shock/pgen.hpp | 200 ++++++++++++++++++++-------------- src/archetypes/spatial_dist.h | 39 +++---- src/engines/engine_run.cpp | 2 +- 3 files changed, 134 insertions(+), 107 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 1c9c9a4d3..cc6f1b812 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -18,6 +18,16 @@ namespace user { using namespace ntt; + template + struct SpatialUniform : public arch::SpatialDistribution { + SpatialUniform(const M& metric) + : arch::SpatialDistribution { metric } {} + + Inline auto operator()(const coord_t&) const -> real_t override { + return ONE; + } + }; + template struct InitFields { /* @@ -79,8 +89,8 @@ namespace user { using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; - const real_t drift_ux, temperature, filling_fraction, - injector_velocity, injection_start; + const real_t drift_ux, temperature, filling_fraction, injector_velocity, + injection_start, dt; const real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -92,10 +102,14 @@ namespace user { , Bmag { p.template get("setup.Bmag", ZERO) } , Btheta { p.template get("setup.Btheta", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } - , init_flds { Bmag, Btheta, Bphi, drift_ux } - , filling_fraction { params.template get("setup.filling_fraction", 1.0) } - , injector_velocity { params.template get("setup.injector_velocity", 1.0) } - , injection_start { params.template get("setup.injection_start", 0.0) } {} + , init_flds { Bmag, Btheta, Bphi, drift_ux } + , filling_fraction { params.template get("setup.filling_fraction", + 1.0) } + , injector_velocity { params.template get( + "setup.injector_velocity", + 1.0) } + , injection_start { params.template get("setup.injection_start", 0.0) } + , dt { params.template get("algorithms.timestep.dt") } {} inline PGen() {} @@ -109,7 +123,7 @@ namespace user { // } inline void InitPrtls(Domain& local_domain) { - + // minimum and maximum position of particles real_t xg_min = local_domain.mesh.extent(in::x1).first; real_t xg_max = local_domain.mesh.extent(in::x1).second * filling_fraction; @@ -120,16 +134,19 @@ namespace user { for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { // compute the range for the x-direction if (d == static_cast(in::x1)) { - box.push_back({xg_min, xg_max}); + box.push_back({ xg_min, xg_max }); } else { // inject into full range in other directions box.push_back(Range::All); } } - // spatial distribution of the particles + // spatial distribution of the particles // -> hack to use the uniform distribution in NonUniformInjector - const auto spatial_dist = arch::Piston(local_domain.mesh.metric, xg_min, xg_max, in::x1); + const auto spatial_dist = arch::Piston(local_domain.mesh.metric, + xg_min, + xg_max, + in::x1); // energy distribution of the particles const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, @@ -139,85 +156,100 @@ namespace user { in::x1); const auto injector = arch::NonUniformInjector( - energy_dist, - spatial_dist, - {1, 2}); + energy_dist, + spatial_dist, + { 1, 2 }); arch::InjectNonUniform>( - params, - local_domain, - injector, - 1.0, // target density - false, // no weights - box); + params, + local_domain, + injector, + 1.0, // target density + false, // no weights + box); } - void CustomPostStep(std::size_t, long double time, real_t dt, Domain &domain) - { - /* - Moving injector for the particles - */ - - // energy distribution of the particles - const auto energy_dist = arch::Maxwellian(domain.mesh.metric, - domain.random_pool, - temperature, - -drift_ux, - in::x1); - - // minimum and maximum position of particles - real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction - + injector_velocity * (time - injection_start - dt) - - drift_ux * dt; // distance particles have moved in the last time step - real_t xg_max = domain.mesh.extent(in::x1).second * filling_fraction + - injector_velocity * (time - injection_start); - - /* - I thought this option would be better, but I can't get it to work - */ - - // define box to inject into - boundaries_t box; - // loop over all dimensions - for (unsigned short d{0}; d < static_cast(M::Dim); ++d) - { - // compute the range for the x-direction - if (d == static_cast(in::x1)) - { - box.push_back({xg_min, xg_max}); - } else { - // inject into full range in other directions - box.push_back(Range::All); - } - } + void CustomPostStep(std::size_t, long double time, Domain& domain) { + const auto energy_dist = arch::Maxwellian(domain.mesh.metric, + domain.random_pool, + temperature, + -drift_ux, + in::x1); + const auto spatial_dist = SpatialUniform(domain.mesh.metric); + const auto injector = arch::NonUniformInjector( + energy_dist, + spatial_dist, + { 1, 2 }); + + boundaries_t box; + + const auto x_init = domain.mesh.extent(in::x1).first + + filling_fraction * (domain.mesh.extent(in::x1).second - + domain.mesh.extent(in::x1).first); + + box[0].first = x_init + injector_velocity * (time - injection_start); + box[0].second = x_init + injector_velocity * (time - injection_start + dt) - + drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt; + for (auto d = 1u; d < M::Dim; ++d) { + box[d] = Range::All; + } - // const auto spatial_dist = arch::Replenish(domain.mesh.metric, - // domain.fields.bckp, - // box, - // TargetDensity, - // 1.0); - - // const auto injector = arch::NonUniformInjector( - // energy_dist, - // spatial_dist, - // {1, 2}); - - const auto injector = arch::MovingInjector{ - domain.mesh.metric, - domain.fields.bckp, - energy_dist, - xg_max, - xg_min, - 1.0, - {1, 2}}; - - arch::InjectNonUniform( - params, - domain, - injector, - 1.0, // target density - false, - box); + arch::InjectNonUniform(params, + domain, + injector, + ONE, + false, + box); + + // /* + // Moving injector for the particles + // */ + // + // // minimum and maximum position of particles + // real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction + + // injector_velocity * (time - injection_start - dt) - + // drift_ux * + // dt; // distance particles have moved in the last time step + // real_t xg_max = domain.mesh.extent(in::x1).second * filling_fraction + + // injector_velocity * (time - injection_start); + // + // /* + // I thought this option would be better, but I can't get it to work + // */ + // + // // define box to inject into + // // loop over all dimensions + // for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { + // // compute the range for the x-direction + // if (d == static_cast(in::x1)) { + // box.push_back({ xg_min, xg_max }); + // } else { + // // inject into full range in other directions + // box.push_back(Range::All); + // } + // } + + // const auto spatial_dist = arch::Replenish(domain.mesh.metric, + // domain.fields.bckp, + // box, + // TargetDensity, + // 1.0); + + // const auto injector = arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // {1, 2}); + + // const auto injector = arch::MovingInjector { + // domain.mesh.metric, + // domain.fields.bckp, + // energy_dist, + // xg_max, + // xg_min, + // 1.0, + // { 1, 2 } + // }; } }; diff --git a/src/archetypes/spatial_dist.h b/src/archetypes/spatial_dist.h index 2edbf5b1d..ad9404ea3 100644 --- a/src/archetypes/spatial_dist.h +++ b/src/archetypes/spatial_dist.h @@ -51,32 +51,27 @@ namespace arch { }; template - struct Piston : public arch::SpatialDistribution - { - Piston(const M &metric, real_t xmin, real_t xmax, in piston_direction = in::x1) - : arch::SpatialDistribution{metric} - , xmin {xmin} - , xmax {xmax} - , piston_direction {piston_direction} {} - - Inline auto operator()(const coord_t &x_Ph) const -> real_t override - { - // dimentsion to fill - const auto fill_dim = static_cast(piston_direction); - - if (x_Ph[fill_dim] < xmin || x_Ph[fill_dim] > xmax) - { - return ZERO; - } - else - { - return ONE; - } + struct Piston : public arch::SpatialDistribution { + Piston(const M& metric, real_t xmin, real_t xmax, in piston_direction = in::x1) + : arch::SpatialDistribution { metric } + , xmin { xmin } + , xmax { xmax } + , piston_direction { piston_direction } {} + + Inline auto operator()(const coord_t& x_Ph) const -> real_t override { + // dimentsion to fill + const auto fill_dim = static_cast(piston_direction); + + if (x_Ph[fill_dim] < xmin || x_Ph[fill_dim] > xmax) { + return ZERO; + } else { + return ONE; } + } private: real_t xmin, xmax; - in piston_direction; + in piston_direction; }; template diff --git a/src/engines/engine_run.cpp b/src/engines/engine_run.cpp index a3ae21ccb..2d4b0d5ed 100644 --- a/src/engines/engine_run.cpp +++ b/src/engines/engine_run.cpp @@ -52,7 +52,7 @@ namespace ntt { traits::has_method::value) { timers.start("Custom"); m_metadomain.runOnLocalDomains([&timers, this](auto& dom) { - m_pgen.CustomPostStep(step, time, dt, dom); + m_pgen.CustomPostStep(step, time, dom); }); timers.stop("Custom"); } From 31ef1000e7e814e06cd027b79104bc8be6cdba9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 24 Mar 2025 15:31:41 -0500 Subject: [PATCH 168/183] reduce resolution for faster test --- setups/srpic/shock/shock.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setups/srpic/shock/shock.toml b/setups/srpic/shock/shock.toml index 43da87d99..ca19a4078 100644 --- a/setups/srpic/shock/shock.toml +++ b/setups/srpic/shock/shock.toml @@ -4,8 +4,8 @@ runtime = 30000.0 [grid] - resolution = [16384, 128] - extent = [[0.0, 8000.0], [-31.25, 31.25]] + resolution = [4096, 128] + extent = [[0.0, 2000.0], [-31.25, 31.25]] [grid.metric] metric = "minkowski" From b32468754eaf799e94da7d3a531a2926afc9d5c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 24 Mar 2025 15:31:52 -0500 Subject: [PATCH 169/183] cleanup and bugfix --- setups/srpic/shock/pgen.hpp | 195 +++++++++++++++++------------------- 1 file changed, 92 insertions(+), 103 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index cc6f1b812..2454c9037 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -18,16 +18,6 @@ namespace user { using namespace ntt; - template - struct SpatialUniform : public arch::SpatialDistribution { - SpatialUniform(const M& metric) - : arch::SpatialDistribution { metric } {} - - Inline auto operator()(const coord_t&) const -> real_t override { - return ONE; - } - }; - template struct InitFields { /* @@ -89,10 +79,13 @@ namespace user { using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; - const real_t drift_ux, temperature, filling_fraction, injector_velocity, - injection_start, dt; + // gas properties + const real_t drift_ux, temperature, filling_fraction; + // injector properties + const real_t injector_velocity, injection_start, dt; + const bool injection_on; - const real_t Btheta, Bphi, Bmag; + real_t Btheta, Bphi, Bmag; InitFields init_flds; inline PGen(const SimulationParams& p, const Metadomain& m) @@ -103,13 +96,11 @@ namespace user { , Btheta { p.template get("setup.Btheta", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } , init_flds { Bmag, Btheta, Bphi, drift_ux } - , filling_fraction { params.template get("setup.filling_fraction", - 1.0) } - , injector_velocity { params.template get( - "setup.injector_velocity", - 1.0) } - , injection_start { params.template get("setup.injection_start", 0.0) } - , dt { params.template get("algorithms.timestep.dt") } {} + , filling_fraction { p.template get("setup.filling_fraction", 1.0) } + , injector_velocity { p.template get("setup.injector_velocity", 1.0) } + , injection_start { p.template get("setup.injection_start", 0.0) } + , injection_on { p.template get("setup.continuous_injection", true) } + , dt { p.template get("algorithms.timestep.dt") } {} inline PGen() {} @@ -117,16 +108,13 @@ namespace user { return init_flds; } - // Inline auto TargetDensity(const coord_t &x_Ph) const -> real_t - // { - // return ONE; - // } - inline void InitPrtls(Domain& local_domain) { // minimum and maximum position of particles real_t xg_min = local_domain.mesh.extent(in::x1).first; - real_t xg_max = local_domain.mesh.extent(in::x1).second * filling_fraction; + real_t xg_max = local_domain.mesh.extent(in::x1).first + + filling_fraction * (local_domain.mesh.extent(in::x1).second - + local_domain.mesh.extent(in::x1).first); // define box to inject into boundaries_t box; @@ -170,86 +158,87 @@ namespace user { } void CustomPostStep(std::size_t, long double time, Domain& domain) { - const auto energy_dist = arch::Maxwellian(domain.mesh.metric, - domain.random_pool, - temperature, - -drift_ux, - in::x1); - const auto spatial_dist = SpatialUniform(domain.mesh.metric); - const auto injector = arch::NonUniformInjector( - energy_dist, - spatial_dist, - { 1, 2 }); - boundaries_t box; - - const auto x_init = domain.mesh.extent(in::x1).first + - filling_fraction * (domain.mesh.extent(in::x1).second - - domain.mesh.extent(in::x1).first); + // ToDo: more performant version? + if (injection_on) { + // same maxwell distribution as above + const auto energy_dist = arch::Maxwellian(domain.mesh.metric, + domain.random_pool, + temperature, + -drift_ux, + in::x1); + + // initial position of injector + const auto x_init = domain.mesh.extent(in::x1).first + + filling_fraction * (domain.mesh.extent(in::x1).second - + domain.mesh.extent(in::x1).first); + + // check if injector is supposed to start moving already + const auto dt_inj = time - injection_start > ZERO ? time - injection_start + : ZERO; + + // define box to inject into + boundaries_t box; + + // loop over all dimension + for (auto d = 0u; d < M::Dim; ++d) { + if (d == 0) { + box.push_back({ x_init + injector_velocity * (dt_inj)-drift_ux / + math::sqrt(1 + SQR(drift_ux)) * dt, + x_init + injector_velocity * (dt_inj + dt) }); + } else { + box.push_back(Range::All); + } + } - box[0].first = x_init + injector_velocity * (time - injection_start); - box[0].second = x_init + injector_velocity * (time - injection_start + dt) - - drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt; - for (auto d = 1u; d < M::Dim; ++d) { - box[d] = Range::All; + // spatial distribution of the particles + // -> hack to use the uniform distribution in NonUniformInjector + const auto spatial_dist = arch::Piston(domain.mesh.metric, + box[0].first, + box[0].second, + in::x1); + + // ToDo: extend Replenish to replace the current injector + const auto injector = + arch::NonUniformInjector( + energy_dist, + spatial_dist, + { 1, 2 }); + + // inject non-uniformly within the defined box + arch::InjectNonUniform(params, + domain, + injector, + ONE, + false, + box); + + /* + I thought this option would be better, but I can't get it to work + */ + + // const auto spatial_dist = arch::Replenish(domain.mesh.metric, + // domain.fields.bckp, + // box, + // TargetDensity, + // 1.0); + + // const auto injector = arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // {1, 2}); + + // const auto injector = arch::MovingInjector { + // domain.mesh.metric, + // domain.fields.bckp, + // energy_dist, + // box[0].first, + // box[0].second, + // 1.0, + // { 1, 2 } + // }; } - - arch::InjectNonUniform(params, - domain, - injector, - ONE, - false, - box); - - // /* - // Moving injector for the particles - // */ - // - // // minimum and maximum position of particles - // real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction + - // injector_velocity * (time - injection_start - dt) - - // drift_ux * - // dt; // distance particles have moved in the last time step - // real_t xg_max = domain.mesh.extent(in::x1).second * filling_fraction + - // injector_velocity * (time - injection_start); - // - // /* - // I thought this option would be better, but I can't get it to work - // */ - // - // // define box to inject into - // // loop over all dimensions - // for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { - // // compute the range for the x-direction - // if (d == static_cast(in::x1)) { - // box.push_back({ xg_min, xg_max }); - // } else { - // // inject into full range in other directions - // box.push_back(Range::All); - // } - // } - - // const auto spatial_dist = arch::Replenish(domain.mesh.metric, - // domain.fields.bckp, - // box, - // TargetDensity, - // 1.0); - - // const auto injector = arch::NonUniformInjector( - // energy_dist, - // spatial_dist, - // {1, 2}); - - // const auto injector = arch::MovingInjector { - // domain.mesh.metric, - // domain.fields.bckp, - // energy_dist, - // xg_max, - // xg_min, - // 1.0, - // { 1, 2 } - // }; } }; From 2d1a45436cd677560cd93bab56c255eea251db43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 24 Mar 2025 15:45:43 -0500 Subject: [PATCH 170/183] removed unnecessary parameter --- setups/srpic/shock/pgen.hpp | 156 +++++++++++++++++------------------- 1 file changed, 75 insertions(+), 81 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 2454c9037..30ab01538 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -83,8 +83,7 @@ namespace user { const real_t drift_ux, temperature, filling_fraction; // injector properties const real_t injector_velocity, injection_start, dt; - const bool injection_on; - + // magnetic field properties real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -99,7 +98,6 @@ namespace user { , filling_fraction { p.template get("setup.filling_fraction", 1.0) } , injector_velocity { p.template get("setup.injector_velocity", 1.0) } , injection_start { p.template get("setup.injection_start", 0.0) } - , injection_on { p.template get("setup.continuous_injection", true) } , dt { p.template get("algorithms.timestep.dt") } {} inline PGen() {} @@ -159,86 +157,82 @@ namespace user { void CustomPostStep(std::size_t, long double time, Domain& domain) { - // ToDo: more performant version? - if (injection_on) { - // same maxwell distribution as above - const auto energy_dist = arch::Maxwellian(domain.mesh.metric, - domain.random_pool, - temperature, - -drift_ux, - in::x1); - - // initial position of injector - const auto x_init = domain.mesh.extent(in::x1).first + - filling_fraction * (domain.mesh.extent(in::x1).second - - domain.mesh.extent(in::x1).first); - - // check if injector is supposed to start moving already - const auto dt_inj = time - injection_start > ZERO ? time - injection_start - : ZERO; - - // define box to inject into - boundaries_t box; - - // loop over all dimension - for (auto d = 0u; d < M::Dim; ++d) { - if (d == 0) { - box.push_back({ x_init + injector_velocity * (dt_inj)-drift_ux / - math::sqrt(1 + SQR(drift_ux)) * dt, - x_init + injector_velocity * (dt_inj + dt) }); - } else { - box.push_back(Range::All); - } - } + // same maxwell distribution as above + const auto energy_dist = arch::Maxwellian(domain.mesh.metric, + domain.random_pool, + temperature, + -drift_ux, + in::x1); + + // initial position of injector + const auto x_init = domain.mesh.extent(in::x1).first + + filling_fraction * (domain.mesh.extent(in::x1).second - + domain.mesh.extent(in::x1).first); - // spatial distribution of the particles - // -> hack to use the uniform distribution in NonUniformInjector - const auto spatial_dist = arch::Piston(domain.mesh.metric, - box[0].first, - box[0].second, - in::x1); - - // ToDo: extend Replenish to replace the current injector - const auto injector = - arch::NonUniformInjector( - energy_dist, - spatial_dist, - { 1, 2 }); - - // inject non-uniformly within the defined box - arch::InjectNonUniform(params, - domain, - injector, - ONE, - false, - box); - - /* - I thought this option would be better, but I can't get it to work - */ - - // const auto spatial_dist = arch::Replenish(domain.mesh.metric, - // domain.fields.bckp, - // box, - // TargetDensity, - // 1.0); - - // const auto injector = arch::NonUniformInjector( - // energy_dist, - // spatial_dist, - // {1, 2}); - - // const auto injector = arch::MovingInjector { - // domain.mesh.metric, - // domain.fields.bckp, - // energy_dist, - // box[0].first, - // box[0].second, - // 1.0, - // { 1, 2 } - // }; + // check if injector is supposed to start moving already + const auto dt_inj = time - injection_start > ZERO ? + time - injection_start : ZERO; + + // define box to inject into + boundaries_t box; + + // loop over all dimension + for (auto d = 0u; d < M::Dim; ++d) { + if (d == 0) { + box.push_back({ x_init + injector_velocity * dt_inj - + drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt, + x_init + injector_velocity * (dt_inj + dt) }); + } else { + box.push_back(Range::All); + } } + + // spatial distribution of the particles + // -> hack to use the uniform distribution in NonUniformInjector + const auto spatial_dist = arch::Piston(domain.mesh.metric, + box[0].first, + box[0].second, + in::x1); + + // ToDo: extend Replenish to replace the current injector + const auto injector = arch::NonUniformInjector( + energy_dist, + spatial_dist, + { 1, 2 }); + + // inject non-uniformly within the defined box + arch::InjectNonUniform(params, + domain, + injector, + ONE, + false, + box); + + /* + I thought this option would be better, but I can't get it to work + */ + + // const auto spatial_dist = arch::Replenish(domain.mesh.metric, + // domain.fields.bckp, + // box, + // TargetDensity, + // 1.0); + + // const auto injector = arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // {1, 2}); + + // const auto injector = arch::MovingInjector { + // domain.mesh.metric, + // domain.fields.bckp, + // energy_dist, + // box[0].first, + // box[0].second, + // 1.0, + // { 1, 2 } + // }; } }; From b3ee77b9714c2e83fb1b3be6caf397cf30255fbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 24 Mar 2025 16:06:42 -0500 Subject: [PATCH 171/183] applied formatting to `MovingInjector` --- src/archetypes/particle_injector.h | 123 ++++++++++++++--------------- 1 file changed, 58 insertions(+), 65 deletions(-) diff --git a/src/archetypes/particle_injector.h b/src/archetypes/particle_injector.h index 676cf2857..62b9249c3 100644 --- a/src/archetypes/particle_injector.h +++ b/src/archetypes/particle_injector.h @@ -171,72 +171,65 @@ namespace arch { }; template - struct MovingInjector - { - struct TargetDensityProfile - { - const real_t nmax, xinj, xdrift; - - TargetDensityProfile(real_t xinj, real_t xdrift, real_t nmax) - : xinj{xinj}, xdrift{xdrift}, nmax{nmax} {} - - Inline auto operator()(const coord_t &x_Ph) const -> real_t - { - if constexpr ((O == in::x1) or - (O == in::x2 and (M::Dim == Dim::_2D or M::Dim == Dim::_3D)) or - (O == in::x3 and M::Dim == Dim::_3D)) - { - const auto xi = x_Ph[static_cast(O)]; - // + direction - if (xi < xdrift or xi >= xinj) - { - return ZERO; - } - else - { - if constexpr (M::CoordType == Coord::Cart) - { - return nmax; - } - else - { - raise::KernelError( - HERE, - "Moving injector in +x cannot be applied for non-cartesian"); - return ZERO; - } - } - } - else - { - raise::KernelError(HERE, "Wrong direction"); - return ZERO; - } + struct MovingInjector { + struct TargetDensityProfile { + const real_t nmax, xinj, xdrift; + + TargetDensityProfile(real_t xinj, real_t xdrift, real_t nmax) + : xinj { xinj } + , xdrift { xdrift } + , nmax { nmax } {} + + Inline auto operator()(const coord_t& x_Ph) const -> real_t { + if constexpr ((O == in::x1) or + (O == in::x2 and (M::Dim == Dim::_2D or M::Dim == Dim::_3D)) or + (O == in::x3 and M::Dim == Dim::_3D)) { + const auto xi = x_Ph[static_cast(O)]; + // + direction + if (xi < xdrift or xi >= xinj) { + return ZERO; + } else { + if constexpr (M::CoordType == Coord::Cart) { + return nmax; + } else { + raise::KernelError( + HERE, + "Moving injector in +x cannot be applied for non-cartesian"); + return ZERO; } - }; - using energy_dist_t = Maxwellian; - using spatial_dist_t = Replenish; - static_assert(M::is_metric, "M must be a metric class"); - static constexpr bool is_nonuniform_injector{true}; - static constexpr Dimension D{M::Dim}; - static constexpr Coord C{M::CoordType}; - - const energy_dist_t energy_dist; - const TargetDensityProfile target_density; - const spatial_dist_t spatial_dist; - const std::pair species; - - MovingInjector(const M &metric, - const ndfield_t &density, - const energy_dist_t &energy_dist, - real_t xinj, - real_t xdrift, - real_t nmax, - const std::pair &species) - : energy_dist{energy_dist}, - target_density{xinj, xdrift, nmax}, spatial_dist{metric, density, 0, target_density, nmax}, species{species} {} - - ~MovingInjector() = default; + } + } else { + raise::KernelError(HERE, "Wrong direction"); + return ZERO; + } + } + }; + + using energy_dist_t = Maxwellian; + using spatial_dist_t = Replenish; + static_assert(M::is_metric, "M must be a metric class"); + static constexpr bool is_nonuniform_injector { true }; + static constexpr Dimension D { M::Dim }; + static constexpr Coord C { M::CoordType }; + + const energy_dist_t energy_dist; + const TargetDensityProfile target_density; + const spatial_dist_t spatial_dist; + const std::pair species; + + MovingInjector(const M& metric, + const ndfield_t& density, + const energy_dist_t& energy_dist, + real_t xinj, + real_t xdrift, + real_t nmax, + const std::pair& species) + : energy_dist { energy_dist } + , target_density { xinj, xdrift, nmax } + , spatial_dist { metric, density, 0, target_density, nmax } + , species { species } {} + + ~MovingInjector() = default; }; /** From 8d61ce85a70cc3fed3b4a57db2f84a4c7bccad45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 31 Mar 2025 11:03:57 -0500 Subject: [PATCH 172/183] first attempt at moving-window injection --- setups/srpic/shock/pgen.hpp | 123 +++++++++++++++++++++++++++++++++--- 1 file changed, 114 insertions(+), 9 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 30ab01538..9568767ff 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -83,6 +83,7 @@ namespace user { const real_t drift_ux, temperature, filling_fraction; // injector properties const real_t injector_velocity, injection_start, dt; + const int injection_frequency; // magnetic field properties real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -98,6 +99,7 @@ namespace user { , filling_fraction { p.template get("setup.filling_fraction", 1.0) } , injector_velocity { p.template get("setup.injector_velocity", 1.0) } , injection_start { p.template get("setup.injection_start", 0.0) } + , injection_frequency { p.template get("setup.injection_frequency", 100) } , dt { p.template get("algorithms.timestep.dt") } {} inline PGen() {} @@ -106,6 +108,25 @@ namespace user { return init_flds; } + auto ResetFields(const em& comp) const -> real_t { + if (comp == em::ex1) { + return init_flds.ex1({ ZERO }); + } else if (comp == em::ex2) { + return init_flds.ex2({ ZERO }); + } else if (comp == em::ex3) { + return init_flds.ex3({ ZERO }); + } else if (comp == em::bx1) { + return init_flds.bx1({ ZERO }); + } else if (comp == em::bx2) { + return init_flds.bx2({ ZERO }); + } else if (comp == em::bx3) { + return init_flds.bx3({ ZERO }); + } else { + raise::Error("Invalid component", HERE); + return ZERO; + } + } + inline void InitPrtls(Domain& local_domain) { // minimum and maximum position of particles @@ -155,14 +176,12 @@ namespace user { box); } - void CustomPostStep(std::size_t, long double time, Domain& domain) { + void CustomPostStep(std::size_t step, long double time, Domain& domain) { - // same maxwell distribution as above - const auto energy_dist = arch::Maxwellian(domain.mesh.metric, - domain.random_pool, - temperature, - -drift_ux, - in::x1); + // check if the injector should be active + if (step % injection_frequency != 0) { + return; + } // initial position of injector const auto x_init = domain.mesh.extent(in::x1).first + @@ -180,13 +199,99 @@ namespace user { for (auto d = 0u; d < M::Dim; ++d) { if (d == 0) { box.push_back({ x_init + injector_velocity * dt_inj - - drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt, + drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt - + 1.5 * injection_frequency * dt, x_init + injector_velocity * (dt_inj + dt) }); } else { box.push_back(Range::All); } } + // define indice range to reset fields + boundaries_t incl_ghosts; + for (auto d = 0; d < M::Dim; ++d) { + incl_ghosts.push_back({ true, true }); + } + const auto extent = domain.mesh.ExtentToRange(box, incl_ghosts); + tuple_t x_min { 0 }, x_max { 0 }; + for (auto d = 0; d < M::Dim; ++d) { + x_min[d] = extent[d].first; + x_max[d] = extent[d].second; + } + + // reset fields + std::vector comps = { em::ex1, em::ex2, em::ex3, + em::bx1, em::bx2, em::bx3 }; + + // loop over all components + for (const auto& comp : comps) { + auto value = ResetFields((em)comp); + + if constexpr (M::Dim == Dim::_1D) { + Kokkos::deep_copy(Kokkos::subview(domain.fields.em, + std::make_pair(x_min[0], x_max[0]), + comp), + value); + } else if constexpr (M::Dim == Dim::_2D) { + Kokkos::deep_copy(Kokkos::subview(domain.fields.em, + std::make_pair(x_min[0], x_max[0]), + std::make_pair(x_min[1], x_max[1]), + comp), + value); + } else if constexpr (M::Dim == Dim::_3D) { + Kokkos::deep_copy(Kokkos::subview(domain.fields.em, + std::make_pair(x_min[0], x_max[0]), + std::make_pair(x_min[1], x_max[1]), + std::make_pair(x_min[2], x_max[2]), + comp), + value); + } else { + raise::Error("Invalid dimension", HERE); + } + } + + /* + tag particles inside the injection zone as dead + */ + + // loop over particle species + for (std::size_t s { 0 }; s < 2; ++s) { + + // get particle properties + auto& species = domain.species[s]; + auto i1 = species.i1; + auto tag = species.tag; + + + // tag all particles with x > box[0].first as dead + Kokkos::parallel_for( + "RemoveParticles", + species.rangeActiveParticles(), + Lambda(index_t p) { + // check if the particle is already dead + if (tag(p) == ParticleTag::dead) { + return; + } + // select the x-coordinate index + auto x_i1 = i1(p); + // check if the particle is inside the box of new plasma + if (x_i1 > x_min[0]) { + tag(p) = ParticleTag::dead; + } + } + ); + } + + /* + Inject piston of fresh plasma + */ + + // same maxwell distribution as above + const auto energy_dist = arch::Maxwellian(domain.mesh.metric, + domain.random_pool, + temperature, + -drift_ux, + in::x1); // spatial distribution of the particles // -> hack to use the uniform distribution in NonUniformInjector const auto spatial_dist = arch::Piston(domain.mesh.metric, @@ -194,7 +299,7 @@ namespace user { box[0].second, in::x1); - // ToDo: extend Replenish to replace the current injector + // inject piston of fresh plasma const auto injector = arch::NonUniformInjector( energy_dist, spatial_dist, From b2c738af48dce1d73b7021ae3fbf401c96984817 Mon Sep 17 00:00:00 2001 From: LudwigBoess Date: Mon, 31 Mar 2025 21:52:04 -0500 Subject: [PATCH 173/183] extended field reset box to the right to squash waves propagating into the empty box --- setups/srpic/shock/pgen.hpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 9568767ff..a25c3891a 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -190,17 +190,16 @@ namespace user { // check if injector is supposed to start moving already const auto dt_inj = time - injection_start > ZERO ? - time - injection_start : ZERO; + time - injection_start : ZERO; // define box to inject into boundaries_t box; - // loop over all dimension for (auto d = 0u; d < M::Dim; ++d) { if (d == 0) { box.push_back({ x_init + injector_velocity * dt_inj - drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt - - 1.5 * injection_frequency * dt, + injection_frequency * dt, x_init + injector_velocity * (dt_inj + dt) }); } else { box.push_back(Range::All); @@ -212,7 +211,9 @@ namespace user { for (auto d = 0; d < M::Dim; ++d) { incl_ghosts.push_back({ true, true }); } - const auto extent = domain.mesh.ExtentToRange(box, incl_ghosts); + auto fields_box = box; + fields_box[0].second += injection_frequency * dt; + const auto extent = domain.mesh.ExtentToRange(fields_box, incl_ghosts); tuple_t x_min { 0 }, x_max { 0 }; for (auto d = 0; d < M::Dim; ++d) { x_min[d] = extent[d].first; @@ -220,11 +221,13 @@ namespace user { } // reset fields - std::vector comps = { em::ex1, em::ex2, em::ex3, - em::bx1, em::bx2, em::bx3 }; + std::vector comps = { em::bx1, em::bx2, em::bx3, + em::ex1, em::ex2, em::ex3 }; // loop over all components for (const auto& comp : comps) { + + // get initial field value of component auto value = ResetFields((em)comp); if constexpr (M::Dim == Dim::_1D) { @@ -262,7 +265,6 @@ namespace user { auto i1 = species.i1; auto tag = species.tag; - // tag all particles with x > box[0].first as dead Kokkos::parallel_for( "RemoveParticles", From fb1692875ad050ba73dee745d0800dd356d9f8bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Tue, 1 Apr 2025 15:32:04 -0500 Subject: [PATCH 174/183] bugfix: added check to truncate the injection box at the end of the domain --- setups/srpic/shock/pgen.hpp | 48 ++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index a25c3891a..cf58b94c6 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -80,10 +80,10 @@ namespace user { using arch::ProblemGenerator::params; // gas properties - const real_t drift_ux, temperature, filling_fraction; + const real_t drift_ux, temperature, filling_fraction; // injector properties - const real_t injector_velocity, injection_start, dt; - const int injection_frequency; + const real_t injector_velocity, injection_start, dt; + const int injection_frequency; // magnetic field properties real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -189,18 +189,23 @@ namespace user { domain.mesh.extent(in::x1).first); // check if injector is supposed to start moving already - const auto dt_inj = time - injection_start > ZERO ? - time - injection_start : ZERO; + const auto dt_inj = time - injection_start > ZERO ? time - injection_start + : ZERO; + + // compute the position of the injector + auto xmax = x_init + injector_velocity * (dt_inj + dt); + if (xmax >= domain.mesh.extent(in::x1).second) { + xmax = domain.mesh.extent(in::x1).second; + } // define box to inject into boundaries_t box; // loop over all dimension for (auto d = 0u; d < M::Dim; ++d) { if (d == 0) { - box.push_back({ x_init + injector_velocity * dt_inj - - drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt - - injection_frequency * dt, - x_init + injector_velocity * (dt_inj + dt) }); + box.push_back({ xmax - drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt - + injection_frequency * dt, + xmax }); } else { box.push_back(Range::All); } @@ -212,7 +217,13 @@ namespace user { incl_ghosts.push_back({ true, true }); } auto fields_box = box; - fields_box[0].second += injection_frequency * dt; + // check if the box is still inside the domain + if (xmax + injection_frequency * dt < domain.mesh.extent(in::x1).second) { + fields_box[0].second += injection_frequency * dt; + } else { + // if right side of the box is outside of the domain -> truncate box + fields_box[0].second = domain.mesh.extent(in::x1).second; + } const auto extent = domain.mesh.ExtentToRange(fields_box, incl_ghosts); tuple_t x_min { 0 }, x_max { 0 }; for (auto d = 0; d < M::Dim; ++d) { @@ -221,7 +232,7 @@ namespace user { } // reset fields - std::vector comps = { em::bx1, em::bx2, em::bx3, + std::vector comps = { em::bx1, em::bx2, em::bx3, em::ex1, em::ex2, em::ex3 }; // loop over all components @@ -253,8 +264,8 @@ namespace user { } } - /* - tag particles inside the injection zone as dead + /* + tag particles inside the injection zone as dead */ // loop over particle species @@ -262,8 +273,8 @@ namespace user { // get particle properties auto& species = domain.species[s]; - auto i1 = species.i1; - auto tag = species.tag; + auto i1 = species.i1; + auto tag = species.tag; // tag all particles with x > box[0].first as dead Kokkos::parallel_for( @@ -277,14 +288,13 @@ namespace user { // select the x-coordinate index auto x_i1 = i1(p); // check if the particle is inside the box of new plasma - if (x_i1 > x_min[0]) { + if (x_i1 >= x_min[0]) { tag(p) = ParticleTag::dead; } - } - ); + }); } - /* + /* Inject piston of fresh plasma */ From 884a1571f2c12f096fe966b81bf066294e499f03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Tue, 1 Apr 2025 19:46:46 -0500 Subject: [PATCH 175/183] fix unit conversion bug in field reset --- setups/srpic/shock/pgen.hpp | 40 ++++++++----------------------------- 1 file changed, 8 insertions(+), 32 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index cf58b94c6..ad260bda0 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -9,6 +9,7 @@ #include "utils/numeric.h" #include "archetypes/energy_dist.h" +#include "archetypes/field_setter.h" #include "archetypes/particle_injector.h" #include "archetypes/problem_generator.h" #include "framework/domain/metadomain.h" @@ -231,38 +232,13 @@ namespace user { x_max[d] = extent[d].second; } - // reset fields - std::vector comps = { em::bx1, em::bx2, em::bx3, - em::ex1, em::ex2, em::ex3 }; - - // loop over all components - for (const auto& comp : comps) { - - // get initial field value of component - auto value = ResetFields((em)comp); - - if constexpr (M::Dim == Dim::_1D) { - Kokkos::deep_copy(Kokkos::subview(domain.fields.em, - std::make_pair(x_min[0], x_max[0]), - comp), - value); - } else if constexpr (M::Dim == Dim::_2D) { - Kokkos::deep_copy(Kokkos::subview(domain.fields.em, - std::make_pair(x_min[0], x_max[0]), - std::make_pair(x_min[1], x_max[1]), - comp), - value); - } else if constexpr (M::Dim == Dim::_3D) { - Kokkos::deep_copy(Kokkos::subview(domain.fields.em, - std::make_pair(x_min[0], x_max[0]), - std::make_pair(x_min[1], x_max[1]), - std::make_pair(x_min[2], x_max[2]), - comp), - value); - } else { - raise::Error("Invalid dimension", HERE); - } - } + Kokkos::parallel_for("ResetFields", + CreateRangePolicy(x_min, x_max), + arch::SetEMFields_kernel { + domain.fields.em, + init_flds, + domain.mesh.metric }); + /* tag particles inside the injection zone as dead From 3f4c5659a95c2d30af22530a8198676de6d44e9c Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 4 Apr 2025 15:40:17 -0400 Subject: [PATCH 176/183] conductor in all directions --- src/kernels/fields_bcs.hpp | 288 ++++++++++++++++++++++++++++++------- 1 file changed, 240 insertions(+), 48 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 8ea6b72d8..970ab559e 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -486,38 +486,52 @@ namespace kernel::bc { } }; - template + template struct ConductorBoundaries_kernel { static_assert(static_cast(o) < static_cast(D), "Invalid component index"); - ndfield_t Fld; - const BCTags tags; + ndfield_t Fld; + const BCTags tags; + const std::size_t i_edge; - ConductorBoundaries_kernel(ndfield_t Fld, BCTags tags) + ConductorBoundaries_kernel(ndfield_t Fld, std::size_t i_edge, BCTags tags) : Fld { Fld } + , i_edge { i_edge } , tags { tags } {} Inline void operator()(index_t i1) const { if constexpr (D == Dim::_1D) { if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, em::ex2) = ZERO; - Fld(N_GHOSTS, em::ex3) = ZERO; + Fld(i_edge, em::ex2) = ZERO; + Fld(i_edge, em::ex3) = ZERO; } else { - Fld(N_GHOSTS - i1, em::ex1) = Fld(N_GHOSTS + i1 - 1, em::ex1); - Fld(N_GHOSTS - i1, em::ex2) = -Fld(N_GHOSTS + i1, em::ex2); - Fld(N_GHOSTS - i1, em::ex3) = -Fld(N_GHOSTS + i1, em::ex3); + if constexpr (not P) { + Fld(i_edge - i1, em::ex1) = Fld(i_edge + i1 - 1, em::ex1); + Fld(i_edge - i1, em::ex2) = -Fld(i_edge + i1, em::ex2); + Fld(i_edge - i1, em::ex3) = -Fld(i_edge + i1, em::ex3); + } else { + Fld(i_edge + i1 - 1, em::ex1) = Fld(i_edge - i1, em::ex1); + Fld(i_edge + i1, em::ex2) = -Fld(i_edge - i1, em::ex2); + Fld(i_edge + i1, em::ex3) = -Fld(i_edge - i1, em::ex3); + } } } if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, em::bx1) = ZERO; + Fld(i_edge, em::bx1) = ZERO; } else { - Fld(N_GHOSTS - i1, em::bx1) = -Fld(N_GHOSTS + i1, em::bx1); - Fld(N_GHOSTS - i1, em::bx2) = Fld(N_GHOSTS + i1 - 1, em::bx2); - Fld(N_GHOSTS - i1, em::bx3) = Fld(N_GHOSTS + i1 - 1, em::bx3); + if constexpr (not P) { + Fld(i_edge - i1, em::bx1) = -Fld(i_edge + i1, em::bx1); + Fld(i_edge - i1, em::bx2) = Fld(i_edge + i1 - 1, em::bx2); + Fld(i_edge - i1, em::bx3) = Fld(i_edge + i1 - 1, em::bx3); + } else { + Fld(i_edge + i1, em::bx1) = -Fld(i_edge - i1, em::bx1); + Fld(i_edge + i1 - 1, em::bx2) = Fld(i_edge - i1, em::bx2); + Fld(i_edge + i1 - 1, em::bx3) = Fld(i_edge - i1, em::bx3); + } } } } else { @@ -529,24 +543,71 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2) const { if constexpr (D == Dim::_2D) { - if (tags & BC::E) { - if (i1 == 0) { - Fld(N_GHOSTS, i2, em::ex2) = ZERO; - Fld(N_GHOSTS, i2, em::ex3) = ZERO; - } else { - Fld(N_GHOSTS - i1, i2, em::ex1) = Fld(N_GHOSTS + i1 - 1, i2, em::ex1); - Fld(N_GHOSTS - i1, i2, em::ex2) = -Fld(N_GHOSTS + i1, i2, em::ex2); - Fld(N_GHOSTS - i1, i2, em::ex3) = -Fld(N_GHOSTS + i1, i2, em::ex3); + if constexpr (o == in::x1) { + if (tags & BC::E) { + if (i1 == 0) { + Fld(i_edge, i2, em::ex2) = ZERO; + Fld(i_edge, i2, em::ex3) = ZERO; + } else { + if constexpr (not P) { + Fld(i_edge - i1, i2, em::ex1) = Fld(i_edge + i1 - 1, i2, em::ex1); + Fld(i_edge - i1, i2, em::ex2) = -Fld(i_edge + i1, i2, em::ex2); + Fld(i_edge - i1, i2, em::ex3) = -Fld(i_edge + i1, i2, em::ex3); + } else { + Fld(i_edge + i1 - 1, i2, em::ex1) = Fld(i_edge - i1, i2, em::ex1); + Fld(i_edge + i1, i2, em::ex2) = -Fld(i_edge - i1, i2, em::ex2); + Fld(i_edge + i1, i2, em::ex3) = -Fld(i_edge - i1, i2, em::ex3); + } + } } - } - if (tags & BC::B) { - if (i1 == 0) { - Fld(N_GHOSTS, i2, em::bx1) = ZERO; - } else { - Fld(N_GHOSTS - i1, i2, em::bx1) = -Fld(N_GHOSTS + i1, i2, em::bx1); - Fld(N_GHOSTS - i1, i2, em::bx2) = Fld(N_GHOSTS + i1 - 1, i2, em::bx2); - Fld(N_GHOSTS - i1, i2, em::bx3) = Fld(N_GHOSTS + i1 - 1, i2, em::bx3); + if (tags & BC::B) { + if (i1 == 0) { + Fld(i_edge, i2, em::bx1) = ZERO; + } else { + if constexpr (not P) { + Fld(i_edge - i1, i2, em::bx1) = -Fld(i_edge + i1, i2, em::bx1); + Fld(i_edge - i1, i2, em::bx2) = Fld(i_edge + i1 - 1, i2, em::bx2); + Fld(i_edge - i1, i2, em::bx3) = Fld(i_edge + i1 - 1, i2, em::bx3); + } else { + Fld(i_edge + i1, i2, em::bx1) = -Fld(i_edge - i1, i2, em::bx1); + Fld(i_edge + i1 - 1, i2, em::bx2) = Fld(i_edge - i1, i2, em::bx2); + Fld(i_edge + i1 - 1, i2, em::bx3) = Fld(i_edge - i1, i2, em::bx3); + } + } + } + } else { + if (tags & BC::E) { + if (i2 == 0) { + Fld(i1, i_edge, em::ex1) = ZERO; + Fld(i1, i_edge, em::ex3) = ZERO; + } else { + if constexpr (not P) { + Fld(i1, i_edge - i2, em::ex1) = -Fld(i1, i_edge + i2, em::ex1); + Fld(i1, i_edge - i2, em::ex2) = Fld(i1, i_edge + i2 - 1, em::ex2); + Fld(i1, i_edge - i2, em::ex3) = -Fld(i1, i_edge + i2, em::ex3); + } else { + Fld(i1, i_edge + i2, em::ex1) = -Fld(i1, i_edge - i2, em::ex1); + Fld(i1, i_edge + i2 - 1, em::ex2) = Fld(i1, i_edge - i2, em::ex2); + Fld(i1, i_edge + i2, em::ex3) = -Fld(i1, i_edge - i2, em::ex3); + } + } + } + + if (tags & BC::B) { + if (i2 == 0) { + Fld(i1, i_edge, em::bx2) = ZERO; + } else { + if constexpr (not P) { + Fld(i1, i_edge - i2, em::bx1) = Fld(i1, i_edge + i2 - 1, em::bx1); + Fld(i1, i_edge - i2, em::bx2) = -Fld(i1, i_edge + i2, em::bx2); + Fld(i1, i_edge - i2, em::bx3) = Fld(i1, i_edge + i2 - 1, em::bx3); + } else { + Fld(i1, i_edge + i2 - 1, em::bx1) = Fld(i1, i_edge - i2, em::bx1); + Fld(i1, i_edge + i2, em::bx2) = -Fld(i1, i_edge - i2, em::bx2); + Fld(i1, i_edge + i2 - 1, em::bx3) = Fld(i1, i_edge - i2, em::bx3); + } + } } } } else { @@ -558,27 +619,158 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2, index_t i3) const { if constexpr (D == Dim::_3D) { - if (tags & BC::E) { - if (i1 == 0) { - Fld(N_GHOSTS, i2, i3, em::ex2) = ZERO; - Fld(N_GHOSTS, i2, i3, em::ex3) = ZERO; - } else { - Fld(N_GHOSTS - i1, i2, i3, em::ex1) = Fld(N_GHOSTS + i1 - 1, - i2, i3, em::ex1); - Fld(N_GHOSTS - i1, i2, i3, em::ex2) = -Fld(N_GHOSTS + i1, i2, i3, em::ex2); - Fld(N_GHOSTS - i1, i2, i3, em::ex3) = -Fld(N_GHOSTS + i1, i2, i3, em::ex3); + if constexpr (o == in::x1) { + if (tags & BC::E) { + if (i1 == 0) { + Fld(i_edge, i2, i3, em::ex2) = ZERO; + Fld(i_edge, i2, i3, em::ex3) = ZERO; + } else { + if constexpr (not P) { + Fld(i_edge - i1, i2, i3, em::ex1) = Fld(i_edge + i1 - 1, + i2, + i3, + em::ex1); + Fld(i_edge - i1, i2, i3, em::ex2) = -Fld(i_edge + i1, i2, i3, em::ex2); + Fld(i_edge - i1, i2, i3, em::ex3) = -Fld(i_edge + i1, i2, i3, em::ex3); + } else { + Fld(i_edge + i1 - 1, i2, i3, em::ex1) = Fld(i_edge - i1, + i2, + i3, + em::ex1); + Fld(i_edge + i1, i2, i3, em::ex2) = -Fld(i_edge - i1, i2, i3, em::ex2); + Fld(i_edge + i1, i2, i3, em::ex3) = -Fld(i_edge - i1, i2, i3, em::ex3); + } + } } - } - if (tags & BC::B) { - if (i1 == 0) { - Fld(N_GHOSTS, i2, i3, em::bx1) = ZERO; - } else { - Fld(N_GHOSTS - i1, i2, i3, em::bx1) = -Fld(N_GHOSTS + i1, i2, i3, em::bx1); - Fld(N_GHOSTS - i1, i2, i3, em::bx2) = Fld(N_GHOSTS + i1 - 1, - i2, i3, em::bx2); - Fld(N_GHOSTS - i1, i2, i3, em::bx3) = Fld(N_GHOSTS + i1 - 1, - i2, i3, em::bx3); + if (tags & BC::B) { + if (i1 == 0) { + Fld(i_edge, i2, i3, em::bx1) = ZERO; + } else { + if constexpr (not P) { + Fld(i_edge - i1, i2, i3, em::bx1) = -Fld(i_edge + i1, i2, i3, em::bx1); + Fld(i_edge - i1, i2, i3, em::bx2) = Fld(i_edge + i1 - 1, + i2, + i3, + em::bx2); + Fld(i_edge - i1, i2, i3, em::bx3) = Fld(i_edge + i1 - 1, + i2, + i3, + em::bx3); + } else { + Fld(i_edge + i1, i2, i3, em::bx1) = -Fld(i_edge - i1, i2, i3, em::bx1); + Fld(i_edge + i1 - 1, i2, i3, em::bx2) = Fld(i_edge - i1, + i2, + i3, + em::bx2); + Fld(i_edge + i1 - 1, i2, i3, em::bx3) = Fld(i_edge - i1, + i2, + i3, + em::bx3); + } + } + } + } else if (o == in::x2) { + if (tags & BC::E) { + if (i2 == 0) { + Fld(i1, i_edge, i3, em::ex1) = ZERO; + Fld(i1, i_edge, i3, em::ex3) = ZERO; + } else { + if constexpr (not P) { + Fld(i1, i_edge - i2, i3, em::ex1) = -Fld(i1, i_edge + i2, i3, em::ex1); + Fld(i1, i_edge - i2, i3, em::ex2) = Fld(i1, + i_edge + i2 - 1, + i3, + em::ex2); + Fld(i1, i_edge - i2, i3, em::ex3) = -Fld(i1, i_edge + i2, i3, em::ex3); + } else { + Fld(i1, i_edge + i2, i3, em::ex1) = -Fld(i1, i_edge - i2, i3, em::ex1); + Fld(i1, i_edge + i2 - 1, i3, em::ex2) = Fld(i1, + i_edge - i2, + i3, + em::ex2); + Fld(i1, i_edge + i2, i3, em::ex3) = -Fld(i1, i_edge - i2, i3, em::ex3); + } + } + } + + if (tags & BC::B) { + if (i2 == 0) { + Fld(i1, i_edge, i3, em::bx2) = ZERO; + } else { + if constexpr (not P) { + Fld(i1, i_edge - i2, i3, em::bx1) = Fld(i1, + i_edge + i2 - 1, + i3, + em::bx1); + Fld(i1, i_edge - i2, i3, em::bx2) = -Fld(i1, i_edge + i2, i3, em::bx2); + Fld(i1, i_edge - i2, i3, em::bx3) = Fld(i1, + i_edge + i2 - 1, + i3, + em::bx3); + } else { + Fld(i1, i_edge + i2 - 1, i3, em::bx1) = Fld(i1, + i_edge - i2, + i3, + em::bx1); + Fld(i1, i_edge + i2, i3, em::bx2) = -Fld(i1, i_edge - i2, i3, em::bx2); + Fld(i1, i_edge + i2 - 1, i3, em::bx3) = Fld(i1, + i_edge - i2, + i3, + em::bx3); + } + } + } + } else { + if (tags & BC::E) { + if (i3 == 0) { + Fld(i1, i2, i_edge, em::ex1) = ZERO; + Fld(i1, i2, i_edge, em::ex2) = ZERO; + } else { + if constexpr (not P) { + Fld(i1, i2, i_edge - i3, em::ex1) = -Fld(i1, i2, i_edge + i3, em::ex1); + Fld(i1, i2, i_edge - i3, em::ex2) = -Fld(i1, i2, i_edge + i3, em::ex2); + Fld(i1, i2, i_edge - i3, em::ex3) = Fld(i1, + i2, + i_edge + i3 - 1, + em::ex3); + } else { + Fld(i1, i2, i_edge + i3, em::ex1) = -Fld(i1, i2, i_edge - i3, em::ex1); + Fld(i1, i2, i_edge + i3, em::ex2) = -Fld(i1, i2, i_edge - i3, em::ex2); + Fld(i1, i2, i_edge + i3 - 1, em::ex3) = Fld(i1, + i2, + i_edge - i3, + em::ex3); + } + } + } + + if (tags & BC::B) { + if (i3 == 0) { + Fld(i1, i2, i_edge, em::bx3) = ZERO; + } else { + if constexpr (not P) { + Fld(i1, i2, i_edge - i3, em::bx1) = Fld(i1, + i2, + i_edge + i3 - 1, + em::bx1); + Fld(i1, i2, i_edge - i3, em::bx2) = Fld(i1, + i2, + i_edge + i3 - 1, + em::bx2); + Fld(i1, i2, i_edge - i3, em::bx3) = -Fld(i1, i2, i_edge + i3, em::bx3); + } else { + Fld(i1, i2, i_edge + i3 - 1, em::bx1) = Fld(i1, + i2, + i_edge - i3, + em::bx1); + Fld(i1, i2, i_edge + i3 - 1, em::bx2) = Fld(i1, + i2, + i_edge - i3, + em::bx2); + Fld(i1, i2, i_edge + i3, em::bx3) = -Fld(i1, i2, i_edge - i3, em::bx3); + } + } } } } else { From 6ae91c91fc8176b01964589caffaab8e8225cb73 Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 4 Apr 2025 15:44:54 -0400 Subject: [PATCH 177/183] minor formatting --- src/engines/srpic.hpp | 72 ++++++++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 31 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index b1fca46da..3684fb573 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -651,7 +651,7 @@ namespace ntt { tuple_t range_min { 0 }; tuple_t range_max { 0 }; - for (unsigned short d { 0 }; d < M::Dim; ++d) { + for (auto d { 0u }; d < M::Dim; ++d) { range_min[d] = intersect_range[d].first; range_max[d] = intersect_range[d].second; } @@ -708,28 +708,32 @@ namespace ntt { /** * axis boundaries */ - raise::ErrorIf(M::CoordType == Coord::Cart, - "Invalid coordinate type for axis BCs", - HERE); - raise::ErrorIf(direction.get_dim() != in::x2, - "Invalid axis direction, should be x2", - HERE); - const auto i2_min = domain.mesh.i_min(in::x2); - const auto i2_max = domain.mesh.i_max(in::x2); - if (direction.get_sign() < 0) { - Kokkos::parallel_for( - "AxisBCFields", - domain.mesh.n_all(in::x1), - kernel::bc::AxisBoundaries_kernel(domain.fields.em, - i2_min, - tags)); + if constexpr (M::CoordType != Coord::Cart) { + raise::ErrorIf(direction.get_dim() != in::x2, + "Invalid axis direction, should be x2", + HERE); + const auto i2_min = domain.mesh.i_min(in::x2); + const auto i2_max = domain.mesh.i_max(in::x2); + if (direction.get_sign() < 0) { + Kokkos::parallel_for( + "AxisBCFields", + domain.mesh.n_all(in::x1), + kernel::bc::AxisBoundaries_kernel(domain.fields.em, + i2_min, + tags)); + } else { + Kokkos::parallel_for( + "AxisBCFields", + domain.mesh.n_all(in::x1), + kernel::bc::AxisBoundaries_kernel(domain.fields.em, + i2_max, + tags)); + } } else { - Kokkos::parallel_for( - "AxisBCFields", - domain.mesh.n_all(in::x1), - kernel::bc::AxisBoundaries_kernel(domain.fields.em, - i2_max, - tags)); + (void)direction; + (void)domain; + (void)tags; + raise::Error("Invalid coordinate type for axis BCs", HERE); } } @@ -834,6 +838,9 @@ namespace ntt { } } } else { + (void)direction; + (void)domain; + (void)tags; raise::Error("Fixed fields not present (both const and non-const)", HERE); } } @@ -844,14 +851,7 @@ namespace ntt { /** * perfect conductor field boundaries */ - if constexpr (M::CoordType != Coord::Cart) { - (void)direction; - (void)domain; - (void)tags; - raise::Error( - "Perfect conductor BCs only applicable to cartesian coordinates", - HERE); - } else { + if constexpr (M::CoordType == Coord::Cart) { const auto sign = direction.get_sign(); const auto dim = direction.get_dim(); @@ -859,7 +859,7 @@ namespace ntt { const std::vector all_dirs { in::x1, in::x2, in::x3 }; - for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { + for (auto d { 0u }; d < static_cast(M::Dim); ++d) { const auto dd = all_dirs[d]; if (dim == dd) { xi_min.push_back(0); @@ -936,6 +936,13 @@ namespace ntt { tags)); } } + } else { + (void)direction; + (void)domain; + (void)tags; + raise::Error( + "Perfect conductor BCs only applicable to cartesian coordinates", + HERE); } } @@ -1060,6 +1067,9 @@ namespace ntt { raise::Error("Invalid dimension", HERE); } } else { + (void)direction; + (void)domain; + (void)tags; raise::Error("Atm fields not implemented in PGEN for atmosphere BCs", HERE); } } From a31c9c72dd36e6fec2e5428c81eb1e18bb6cbc93 Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 4 Apr 2025 16:50:03 -0400 Subject: [PATCH 178/183] rm TODOs --- src/framework/domain/comm_mpi.hpp | 1 - src/framework/domain/grid.cpp | 1 - src/framework/domain/metadomain.cpp | 1 - src/framework/simulation.cpp | 2 -- 4 files changed, 5 deletions(-) diff --git a/src/framework/domain/comm_mpi.hpp b/src/framework/domain/comm_mpi.hpp index 4f382be4a..159502e7a 100644 --- a/src/framework/domain/comm_mpi.hpp +++ b/src/framework/domain/comm_mpi.hpp @@ -220,7 +220,6 @@ namespace comm { if (recv_rank >= 0) { - // !TODO: perhaps directly recv to the fld? if (not additive) { if constexpr (D == Dim::_1D) { Kokkos::deep_copy(Kokkos::subview(fld, recv_slice[0], comps), recv_fld); diff --git a/src/framework/domain/grid.cpp b/src/framework/domain/grid.cpp index baa23fb5c..c022184b1 100644 --- a/src/framework/domain/grid.cpp +++ b/src/framework/domain/grid.cpp @@ -85,7 +85,6 @@ namespace ntt { return CreateRangePolicy(imin, imax); } - // !TODO: too ugly, implement a better solution (combine with device) template auto Grid::rangeCellsOnHost( const box_region_t& region) const -> range_h_t { diff --git a/src/framework/domain/metadomain.cpp b/src/framework/domain/metadomain.cpp index 10e3e4fb0..eec6c8fa0 100644 --- a/src/framework/domain/metadomain.cpp +++ b/src/framework/domain/metadomain.cpp @@ -234,7 +234,6 @@ namespace ntt { template void Metadomain::redefineBoundaries() { - // !TODO: not setting CommBC for now for (unsigned int idx { 0 }; idx < g_ndomains; ++idx) { // offset of the subdomain[idx] auto& current_domain = g_subdomains[idx]; diff --git a/src/framework/simulation.cpp b/src/framework/simulation.cpp index bea50ff09..560539c25 100644 --- a/src/framework/simulation.cpp +++ b/src/framework/simulation.cpp @@ -48,8 +48,6 @@ namespace ntt { HERE); m_requested_dimension = static_cast(res.size()); - // !TODO: when mixing checkpoint metadata with input, - // ... need to properly take care of the diffs m_params.setRawData(raw_params); timestep_t checkpoint_step = 0; if (is_resuming) { From 58a6214d728ad779b6809754856a7b286a223749 Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 4 Apr 2025 16:50:21 -0400 Subject: [PATCH 179/183] conductor filters in full 3D --- src/kernels/digital_filter.hpp | 225 +++++++++++++++++++++++++++------ 1 file changed, 185 insertions(+), 40 deletions(-) diff --git a/src/kernels/digital_filter.hpp b/src/kernels/digital_filter.hpp index 0ec6dcdbc..4f1385c16 100644 --- a/src/kernels/digital_filter.hpp +++ b/src/kernels/digital_filter.hpp @@ -25,6 +25,33 @@ INV_2*(ARR)((I), (J), (COMP)) + \ INV_4*((ARR)((I), (J) - 1, (COMP)) + (ARR)((I), (J) + 1, (COMP))) +#define FILTER3D_IN_I1_I2(ARR, COMP, I, J, K) \ + INV_4*(ARR)(I, J, K, (COMP)) + \ + INV_8*((ARR)((I) - 1, (J), (K), (COMP)) + (ARR)((I) + 1, (J), (K), (COMP)) + \ + (ARR)((I), (J) - 1, (K), (COMP)) + (ARR)((I), (J) + 1, (K), (COMP))) + \ + INV_16*((ARR)((I) - 1, (J) - 1, (K), (COMP)) + \ + (ARR)((I) + 1, (J) + 1, (K), (COMP)) + \ + (ARR)((I) - 1, (J) + 1, (K), (COMP)) + \ + (ARR)((I) + 1, (J) - 1, (K), (COMP))) + +#define FILTER3D_IN_I2_I3(ARR, COMP, I, J, K) \ + INV_4*(ARR)(I, J, K, (COMP)) + \ + INV_8*((ARR)((I), (J) - 1, (K), (COMP)) + (ARR)((I), (J) + 1, (K), (COMP)) + \ + (ARR)((I), (J), (K) - 1, (COMP)) + (ARR)((I), (J), (K) + 1, (COMP))) + \ + INV_16*((ARR)((I), (J) - 1, (K) - 1, (COMP)) + \ + (ARR)((I), (J) + 1, (K) + 1, (COMP)) + \ + (ARR)((I), (J) - 1, (K) + 1, (COMP)) + \ + (ARR)((I), (J) + 1, (K) - 1, (COMP))) + +#define FILTER3D_IN_I1_I3(ARR, COMP, I, J, K) \ + INV_4*(ARR)(I, J, K, (COMP)) + \ + INV_8*((ARR)((I) - 1, (J), (K), (COMP)) + (ARR)((I) + 1, (J), (K), (COMP)) + \ + (ARR)((I), (J), (K) - 1, (COMP)) + (ARR)((I), (J), (K) + 1, (COMP))) + \ + INV_16*((ARR)((I) - 1, (J), (K) - 1, (COMP)) + \ + (ARR)((I) + 1, (J), (K) + 1, (COMP)) + \ + (ARR)((I) - 1, (J), (K) + 1, (COMP)) + \ + (ARR)((I) + 1, (J), (K) - 1, (COMP))) + namespace kernel { using namespace ntt; @@ -36,9 +63,14 @@ namespace kernel { static constexpr auto i2_min = N_GHOSTS; const ncells_t i2_max; const bool is_axis_i2min, is_axis_i2max; - const bool is_conductor_i1min; - static constexpr auto i1_min = N_GHOSTS, i2_min = N_GHOSTS; - const std::size_t i2_max; + const bool is_conductor_i1min, is_conductor_i1max; + const bool is_conductor_i2min, is_conductor_i2max; + const bool is_conductor_i3min, is_conductor_i3max; + static constexpr auto i1_min = N_GHOSTS, i2_min = N_GHOSTS, i3_min = N_GHOSTS; + const std::size_t i1_max, i2_max, i3_max; + + // @TODO: Current implementation might have issues + // ... at the corners between two conductors public: DigitalFilter_kernel(ndfield_t& array, @@ -50,21 +82,40 @@ namespace kernel { , is_axis_i2min { (D == Dim::_2D) and (boundaries[1].first == FldsBC::AXIS) } , is_axis_i2max { (D == Dim::_2D) and (boundaries[1].second == FldsBC::AXIS) } , is_conductor_i1min { boundaries[0].first == FldsBC::CONDUCTOR } - , i2_max { (short)D > 1 ? size_[1] + N_GHOSTS : 0 } {} + , is_conductor_i1max { boundaries[0].second == FldsBC::CONDUCTOR } + , is_conductor_i2min { (short)D > 1 + ? (boundaries[1].first == FldsBC::CONDUCTOR) + : false } + , is_conductor_i2max { (short)D > 1 + ? (boundaries[1].second == FldsBC::CONDUCTOR) + : false } + , is_conductor_i3min { (short)D > 2 + ? (boundaries[2].first == FldsBC::CONDUCTOR) + : false } + , is_conductor_i3max { (short)D > 2 + ? (boundaries[2].second == FldsBC::CONDUCTOR) + : false } + , i1_max { size_[0] + N_GHOSTS } + , i2_max { (short)D > 1 ? (size_[1] + N_GHOSTS) : 0 } + , i3_max { (short)D > 2 ? (size_[2] + N_GHOSTS) : 0 } {} Inline void operator()(index_t i1) const { if constexpr ((D == Dim::_1D) && (C == Coord::Cart)) { - if (is_conductor_i1min and i1 == i1_min) { + if ((is_conductor_i1min and i1 == i1_min) or + (is_conductor_i1max and i1 == i1_max - 1)) { + const auto i1side = is_conductor_i1min ? (i1 + 1) : (i1 - 1); array(i1, cur::jx1) = (THREE * INV_4) * buffer(i1, cur::jx1) + - (INV_4)*buffer(i1 + 1, cur::jx1); - } else if (is_conductor_i1min and i1 == i1_min + 1) { + (INV_4)*buffer(i1side, cur::jx1); + } else if ((is_conductor_i1min and i1 == i1_min + 1) or + (is_conductor_i1max and i1 == i1_max - 2)) { + const auto i1side = is_conductor_i1min ? (i1 + 1) : (i1 - 1); array(i1, cur::jx1) = INV_2 * buffer(i1, cur::jx1) + INV_4 * (buffer(i1 - 1, cur::jx1) + buffer(i1 + 1, cur::jx1)); array(i1, cur::jx2) = (INV_2)*buffer(i1, cur::jx2) + - (INV_4)*buffer(i1 + 1, cur::jx2); + (INV_4)*buffer(i1side, cur::jx2); array(i1, cur::jx3) = (INV_2)*buffer(i1, cur::jx3) + - (INV_4)*buffer(i1 + 1, cur::jx3); + (INV_4)*buffer(i1side, cur::jx3); } else { #pragma unroll for (const auto& comp : { cur::jx1, cur::jx2, cur::jx3 }) { @@ -81,11 +132,15 @@ namespace kernel { Inline void operator()(index_t i1, index_t i2) const { if constexpr (D == Dim::_2D) { if constexpr (C == Coord::Cart) { - if (is_conductor_i1min and i1 == i1_min) { + if ((is_conductor_i1min and i1 == i1_min) or + (is_conductor_i1max and i1 == i1_max - 1)) { + const auto i1side = is_conductor_i1min ? (i1 + 1) : (i1 - 1); array(i1, i2, cur::jx1) = (THREE * INV_4) * (FILTER2D_IN_I2(buffer, cur::jx1, i1, i2)) + - (INV_4) * (FILTER2D_IN_I2(buffer, cur::jx1, i1 + 1, i2)); - } else if (is_conductor_i1min and i1 == i1_min + 1) { + (INV_4) * (FILTER2D_IN_I2(buffer, cur::jx1, i1side, i2)); + } else if ((is_conductor_i1min and i1 == i1_min + 1) or + (is_conductor_i1max and i1 == i1_max - 2)) { + const auto i1side = is_conductor_i1min ? (i1 + 1) : (i1 - 1); array(i1, i2, cur::jx1) = INV_2 * (FILTER2D_IN_I2(buffer, cur::jx1, i1, i2)) + @@ -96,12 +151,37 @@ namespace kernel { i2, cur::jx2) = INV_2 * (FILTER2D_IN_I2(buffer, cur::jx2, i1, i2)) + INV_4 * - (FILTER2D_IN_I2(buffer, cur::jx2, i1 + 1, i2)); + (FILTER2D_IN_I2(buffer, cur::jx2, i1side, i2)); array(i1, i2, cur::jx3) = INV_2 * (FILTER2D_IN_I2(buffer, cur::jx3, i1, i2)) + INV_4 * - (FILTER2D_IN_I2(buffer, cur::jx3, i1 + 1, i2)); + (FILTER2D_IN_I2(buffer, cur::jx3, i1side, i2)); + } else if ((is_conductor_i2min and i2 == i2_min) or + (is_conductor_i2max and i2 == i2_max - 1)) { + const auto i2side = is_conductor_i2min ? (i2 + 1) : (i2 - 1); + array(i1, i2, cur::jx2) = + (THREE * INV_4) * (FILTER2D_IN_I1(buffer, cur::jx2, i1, i2)) + + (INV_4) * (FILTER2D_IN_I1(buffer, cur::jx2, i1, i2side)); + } else if ((is_conductor_i2min and i2 == i2_min + 1) or + (is_conductor_i2max and i2 == i2_max - 2)) { + const auto i2side = is_conductor_i2min ? (i2 + 1) : (i2 - 1); + array(i1, + i2, + cur::jx1) = INV_2 * (FILTER2D_IN_I1(buffer, cur::jx1, i1, i2)) + + INV_4 * + (FILTER2D_IN_I1(buffer, cur::jx1, i1, i2side)); + array(i1, + i2, + cur::jx2) = INV_2 * (FILTER2D_IN_I1(buffer, cur::jx2, i1, i2)) + + INV_4 * + ((FILTER2D_IN_I1(buffer, cur::jx2, i1, i2 - 1)) + + (FILTER2D_IN_I1(buffer, cur::jx2, i1, i2 + 1))); + array(i1, + i2, + cur::jx3) = INV_2 * (FILTER2D_IN_I1(buffer, cur::jx3, i1, i2)) + + INV_4 * + (FILTER2D_IN_I1(buffer, cur::jx3, i1, i2side)); } else { #pragma unroll for (const auto comp : { cur::jx1, cur::jx2, cur::jx3 }) { @@ -212,33 +292,93 @@ namespace kernel { Inline void operator()(index_t i1, index_t i2, index_t i3) const { if constexpr (D == Dim::_3D) { if constexpr (C == Coord::Cart) { + if ((is_conductor_i1min and i1 == i1_min) or + (is_conductor_i1max and i1 == i1_max - 1)) { + const auto i1side = is_conductor_i1min ? (i1 + 1) : (i1 - 1); + array(i1, i2, i3, cur::jx1) = + (THREE * INV_4) * (FILTER3D_IN_I2_I3(buffer, cur::jx1, i1, i2, i3)) + + (INV_4) * (FILTER3D_IN_I2_I3(buffer, cur::jx1, i1side, i2, i3)); + } else if ((is_conductor_i1min and i1 == i1_min + 1) or + (is_conductor_i1max and i1 == i1_max - 2)) { + const auto i1side = is_conductor_i1min ? (i1 + 1) : (i1 - 1); + array(i1, i2, i3, cur::jx1) = + INV_2 * (FILTER3D_IN_I2_I3(buffer, cur::jx1, i1, i2, i3)) + + INV_4 * ((FILTER3D_IN_I2_I3(buffer, cur::jx1, i1 - 1, i2, i3)) + + (FILTER3D_IN_I2_I3(buffer, cur::jx1, i1 + 1, i2, i3))); + array(i1, i2, i3, cur::jx2) = + INV_2 * (FILTER3D_IN_I2_I3(buffer, cur::jx2, i1, i2, i3)) + + INV_4 * (FILTER3D_IN_I2_I3(buffer, cur::jx2, i1side, i2, i3)); + array(i1, i2, i3, cur::jx3) = + INV_2 * (FILTER3D_IN_I2_I3(buffer, cur::jx3, i1, i2, i3)) + + INV_4 * (FILTER3D_IN_I2_I3(buffer, cur::jx3, i1side, i2, i3)); + } else if ((is_conductor_i2min and i2 == i2_min) or + (is_conductor_i2max and i2 == i2_max - 1)) { + const auto i2side = is_conductor_i2min ? (i2 + 1) : (i2 - 1); + array(i1, i2, i3, cur::jx2) = + (THREE * INV_4) * (FILTER3D_IN_I1_I3(buffer, cur::jx2, i1, i2, i3)) + + (INV_4) * (FILTER3D_IN_I1_I3(buffer, cur::jx2, i1, i2side, i3)); + } else if ((is_conductor_i2min and i2 == i2_min + 1) or + (is_conductor_i2max and i2 == i2_max - 2)) { + const auto i2side = is_conductor_i2min ? (i2 + 1) : (i2 - 1); + array(i1, i2, i3, cur::jx1) = + INV_2 * (FILTER3D_IN_I1_I3(buffer, cur::jx1, i1, i2, i3)) + + INV_4 * (FILTER3D_IN_I1_I3(buffer, cur::jx1, i1, i2side, i3)); + array(i1, i2, i3, cur::jx2) = + INV_2 * (FILTER3D_IN_I1_I3(buffer, cur::jx2, i1, i2, i3)) + + INV_4 * ((FILTER3D_IN_I1_I3(buffer, cur::jx2, i1, i2 - 1, i3)) + + (FILTER3D_IN_I1_I3(buffer, cur::jx2, i1, i2 + 1, i3))); + array(i1, i2, i3, cur::jx3) = + INV_2 * (FILTER3D_IN_I1_I3(buffer, cur::jx3, i1, i2, i3)) + + INV_4 * (FILTER3D_IN_I1_I3(buffer, cur::jx3, i1, i2side, i3)); + } else if ((is_conductor_i3min and i3 == i3_min) or + (is_conductor_i3max and i3 == i3_max - 1)) { + const auto i3side = is_conductor_i3min ? (i3 + 1) : (i3 - 1); + array(i1, i2, i3, cur::jx3) = + (THREE * INV_4) * (FILTER3D_IN_I1_I2(buffer, cur::jx3, i1, i2, i3)) + + (INV_4) * (FILTER3D_IN_I1_I2(buffer, cur::jx3, i1, i2, i3side)); + } else if ((is_conductor_i3min and i3 == i3_min + 1) or + (is_conductor_i3max and i3 == i3_max - 2)) { + const auto i3side = is_conductor_i3min ? (i3 + 1) : (i3 - 1); + array(i1, i2, i3, cur::jx1) = + INV_2 * (FILTER3D_IN_I1_I2(buffer, cur::jx1, i1, i2, i3)) + + INV_4 * (FILTER3D_IN_I1_I2(buffer, cur::jx1, i1, i2, i3side)); + array(i1, i2, i3, cur::jx2) = + INV_2 * (FILTER3D_IN_I1_I2(buffer, cur::jx2, i1, i2, i3)) + + INV_4 * (FILTER3D_IN_I1_I2(buffer, cur::jx2, i1, i2, i3side)); + array(i1, i2, i3, cur::jx3) = + INV_2 * (FILTER3D_IN_I1_I2(buffer, cur::jx3, i1, i2, i3)) + + INV_4 * ((FILTER3D_IN_I1_I2(buffer, cur::jx3, i1, i2, i3 - 1)) + + (FILTER3D_IN_I1_I2(buffer, cur::jx3, i1, i2, i3 + 1))); + } else { #pragma unroll - for (auto& comp : { cur::jx1, cur::jx2, cur::jx3 }) { - array(i1, i2, i3, comp) = - INV_8 * buffer(i1, i2, i3, comp) + - INV_16 * - (buffer(i1 - 1, i2, i3, comp) + buffer(i1 + 1, i2, i3, comp) + - buffer(i1, i2 - 1, i3, comp) + buffer(i1, i2 + 1, i3, comp) + - buffer(i1, i2, i3 - 1, comp) + buffer(i1, i2, i3 + 1, comp)) + - INV_32 * - (buffer(i1 - 1, i2 - 1, i3, comp) + - buffer(i1 + 1, i2 + 1, i3, comp) + - buffer(i1 - 1, i2 + 1, i3, comp) + - buffer(i1 + 1, i2 - 1, i3, comp) + - buffer(i1, i2 - 1, i3 - 1, comp) + - buffer(i1, i2 + 1, i3 + 1, comp) + buffer(i1, i2, i3 - 1, comp) + - buffer(i1, i2, i3 + 1, comp) + buffer(i1 - 1, i2, i3 - 1, comp) + - buffer(i1 + 1, i2, i3 + 1, comp) + - buffer(i1 - 1, i2, i3 + 1, comp) + - buffer(i1 + 1, i2, i3 - 1, comp)) + - INV_64 * (buffer(i1 - 1, i2 - 1, i3 - 1, comp) + - buffer(i1 + 1, i2 + 1, i3 + 1, comp) + - buffer(i1 - 1, i2 + 1, i3 + 1, comp) + - buffer(i1 + 1, i2 - 1, i3 - 1, comp) + - buffer(i1 - 1, i2 - 1, i3 + 1, comp) + - buffer(i1 + 1, i2 + 1, i3 - 1, comp) + - buffer(i1 - 1, i2 + 1, i3 - 1, comp) + - buffer(i1 + 1, i2 - 1, i3 + 1, comp)); + for (auto& comp : { cur::jx1, cur::jx2, cur::jx3 }) { + array(i1, i2, i3, comp) = + INV_8 * buffer(i1, i2, i3, comp) + + INV_16 * + (buffer(i1 - 1, i2, i3, comp) + buffer(i1 + 1, i2, i3, comp) + + buffer(i1, i2 - 1, i3, comp) + buffer(i1, i2 + 1, i3, comp) + + buffer(i1, i2, i3 - 1, comp) + buffer(i1, i2, i3 + 1, comp)) + + INV_32 * + (buffer(i1 - 1, i2 - 1, i3, comp) + + buffer(i1 + 1, i2 + 1, i3, comp) + + buffer(i1 - 1, i2 + 1, i3, comp) + + buffer(i1 + 1, i2 - 1, i3, comp) + + buffer(i1, i2 - 1, i3 - 1, comp) + + buffer(i1, i2 + 1, i3 + 1, comp) + + buffer(i1, i2, i3 - 1, comp) + buffer(i1, i2, i3 + 1, comp) + + buffer(i1 - 1, i2, i3 - 1, comp) + + buffer(i1 + 1, i2, i3 + 1, comp) + + buffer(i1 - 1, i2, i3 + 1, comp) + + buffer(i1 + 1, i2, i3 - 1, comp)) + + INV_64 * (buffer(i1 - 1, i2 - 1, i3 - 1, comp) + + buffer(i1 + 1, i2 + 1, i3 + 1, comp) + + buffer(i1 - 1, i2 + 1, i3 + 1, comp) + + buffer(i1 + 1, i2 - 1, i3 - 1, comp) + + buffer(i1 - 1, i2 - 1, i3 + 1, comp) + + buffer(i1 + 1, i2 + 1, i3 - 1, comp) + + buffer(i1 - 1, i2 + 1, i3 - 1, comp) + + buffer(i1 + 1, i2 - 1, i3 + 1, comp)); + } } } else { raise::KernelNotImplementedError(HERE); @@ -253,6 +393,11 @@ namespace kernel { } // namespace kernel +#undef FILTER3D_IN_I1_I3 +#undef FILTER3D_IN_I2_I3 +#undef FILTER3D_IN_I1_I2 + +#undef FILTER2D_IN_I2 #undef FILTER2D_IN_I1 #endif // DIGITAL_FILTER_HPP From 0303acf48d29e64faeb982b0106fc37f08db2d0c Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 4 Apr 2025 17:26:12 -0400 Subject: [PATCH 180/183] rebase to 1.2.0rc --- src/checkpoint/tests/checkpoint-nompi.cpp | 4 ++-- src/framework/containers/particles.cpp | 2 +- src/kernels/digital_filter.hpp | 5 +---- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/checkpoint/tests/checkpoint-nompi.cpp b/src/checkpoint/tests/checkpoint-nompi.cpp index 23dbd8871..be5e97e24 100644 --- a/src/checkpoint/tests/checkpoint-nompi.cpp +++ b/src/checkpoint/tests/checkpoint-nompi.cpp @@ -105,8 +105,8 @@ auto main(int argc, char* argv[]) -> int { writer.saveField("em", field1); writer.saveField("em0", field2); - writer.savePerDomainVariable("s1_npart", 1, 0, npart1); - writer.savePerDomainVariable("s2_npart", 1, 0, npart2); + writer.savePerDomainVariable("s1_npart", 1, 0, npart1); + writer.savePerDomainVariable("s2_npart", 1, 0, npart2); writer.saveParticleQuantity("s1_i1", npart1, 0, npart1, i1); writer.saveParticleQuantity("s1_ux1", npart1, 0, npart1, u1); diff --git a/src/framework/containers/particles.cpp b/src/framework/containers/particles.cpp index 634a34862..9ec8810dc 100644 --- a/src/framework/containers/particles.cpp +++ b/src/framework/containers/particles.cpp @@ -218,7 +218,7 @@ namespace ntt { Kokkos::Experimental::fill( "TagAliveParticles", - AccelExeSpace(), + Kokkos::DefaultExecutionSpace(), Kokkos::subview(this_tag, std::make_pair(static_cast(0), n_alive)), ParticleTag::alive); diff --git a/src/kernels/digital_filter.hpp b/src/kernels/digital_filter.hpp index 4f1385c16..5ac60327d 100644 --- a/src/kernels/digital_filter.hpp +++ b/src/kernels/digital_filter.hpp @@ -59,15 +59,12 @@ namespace kernel { class DigitalFilter_kernel { ndfield_t array; const ndfield_t buffer; - bool is_axis_i2min { false }, is_axis_i2max { false }; - static constexpr auto i2_min = N_GHOSTS; - const ncells_t i2_max; const bool is_axis_i2min, is_axis_i2max; const bool is_conductor_i1min, is_conductor_i1max; const bool is_conductor_i2min, is_conductor_i2max; const bool is_conductor_i3min, is_conductor_i3max; static constexpr auto i1_min = N_GHOSTS, i2_min = N_GHOSTS, i3_min = N_GHOSTS; - const std::size_t i1_max, i2_max, i3_max; + const ncells_t i1_max, i2_max, i3_max; // @TODO: Current implementation might have issues // ... at the corners between two conductors From dd8a35b9ccf419bccbfdbb0c6e7f7a7890364b96 Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 4 Apr 2025 17:35:17 -0400 Subject: [PATCH 181/183] (RUNTEST) From cb34e94bb928c2e7b8df3cb8d191d3628377eefc Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 4 Apr 2025 17:50:03 -0400 Subject: [PATCH 182/183] mpi tests fixed (RUNTEST) --- src/checkpoint/tests/checkpoint-mpi.cpp | 4 ++-- src/output/tests/writer-mpi.cpp | 24 ++++++++++++------------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/checkpoint/tests/checkpoint-mpi.cpp b/src/checkpoint/tests/checkpoint-mpi.cpp index f97202ab1..bc6d6038a 100644 --- a/src/checkpoint/tests/checkpoint-mpi.cpp +++ b/src/checkpoint/tests/checkpoint-mpi.cpp @@ -144,8 +144,8 @@ auto main(int argc, char* argv[]) -> int { writer.saveField("em", field1); writer.saveField("em0", field2); - writer.savePerDomainVariable("s1_npart", 1, 0, npart1); - writer.savePerDomainVariable("s2_npart", 1, 0, npart2); + writer.savePerDomainVariable("s1_npart", 1, 0, npart1); + writer.savePerDomainVariable("s2_npart", 1, 0, npart2); writer.saveParticleQuantity("s1_i1", npart1_globtot, diff --git a/src/output/tests/writer-mpi.cpp b/src/output/tests/writer-mpi.cpp index f6d3ee88a..cc9208889 100644 --- a/src/output/tests/writer-mpi.cpp +++ b/src/output/tests/writer-mpi.cpp @@ -61,8 +61,8 @@ auto main(int argc, char* argv[]) -> int { // write auto writer = out::Writer(); writer.init(&adios, "hdf5", "test", false); - writer.defineMeshLayout({ static_cast(mpi_size) * nx1 }, - { static_cast(mpi_rank) * nx1 }, + writer.defineMeshLayout({ static_cast(mpi_size) * nx1 }, + { static_cast(mpi_rank) * nx1 }, { nx1 }, { dwn1 }, false, @@ -89,9 +89,9 @@ auto main(int argc, char* argv[]) -> int { { // read adios2::IO io = adios.DeclareIO("read-test"); - io.SetEngine("hdf5"); - adios2::Engine reader = io.Open("test.h5", adios2::Mode::Read, MPI_COMM_SELF); - raise::ErrorIf(io.InquireAttribute("NGhosts").Data()[0] != 0, + io.SetEngine("HDF5"); + adios2::Engine reader = io.Open("test.h5", adios2::Mode::Read); + raise::ErrorIf(io.InquireAttribute("NGhosts").Data()[0] != 0, "NGhosts is not correct", HERE); raise::ErrorIf(io.InquireAttribute("Dimension").Data()[0] != 1, @@ -99,13 +99,13 @@ auto main(int argc, char* argv[]) -> int { HERE); for (std::size_t step = 0; reader.BeginStep() == adios2::StepStatus::OK; ++step) { - std::size_t step_read; - long double time_read; + timestep_t step_read; + simtime_t time_read; - reader.Get(io.InquireVariable("Step"), + reader.Get(io.InquireVariable("Step"), &step_read, adios2::Mode::Sync); - reader.Get(io.InquireVariable("Time"), + reader.Get(io.InquireVariable("Time"), &time_read, adios2::Mode::Sync); raise::ErrorIf(step_read != step, "Step is not correct", HERE); @@ -122,9 +122,9 @@ auto main(int argc, char* argv[]) -> int { const double l = l_offset; const double f = math::ceil(l / d) * d - l; - const auto first_cell = static_cast(f); - const auto l_size_dwn = static_cast(math::ceil((n - f) / d)); - const auto l_corner_dwn = static_cast(math::ceil(l / d)); + const auto first_cell = static_cast(f); + const auto l_size_dwn = static_cast(math::ceil((n - f) / d)); + const auto l_corner_dwn = static_cast(math::ceil(l / d)); array_t field_read {}; int cntr = 0; From 0bc510f7c03c3897ec29249f3c8486005e3af125 Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 4 Apr 2025 17:56:58 -0400 Subject: [PATCH 183/183] added non-mpi cpu test action (RUNTEST) --- .github/workflows/actions.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index f60ee9061..a5c546328 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -27,9 +27,14 @@ jobs: matrix: device: [cpu, amd-gpu, nvidia-gpu] precision: [double, single] + mpi: [serial, parallel] exclude: # my AMD GPU doesn't support fp64 atomics : ( - device: amd-gpu precision: double + - device: amd-gpu + mpi: parallel + - device: nvidia-gpu + mpi: parallel runs-on: [self-hosted, "${{ matrix.device }}"] steps: - name: Checkout @@ -47,7 +52,7 @@ jobs: fi elif [ "${{ matrix.device }}" = "amd-gpu" ]; then FLAGS="-D Kokkos_ENABLE_HIP=ON -D Kokkos_ARCH_AMD_GFX1100=ON" - elif [ "${{ matrix.device }}" = "cpu" ]; then + elif [ "${{ matrix.mpi }}" = "parallel" ]; then FLAGS="-D mpi=ON" fi cmake -B build -D TESTS=ON -D output=ON -D precision=${{ matrix.precision }} $FLAGS