From 3e1848aaba12da9b4c2844f0c2aba95d2102a47b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Feb 2025 10:43:05 -0600 Subject: [PATCH 001/176] prep for conductor boundaries --- src/global/arch/traits.h | 12 ++++++++++++ src/global/enums.h | 7 ++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/global/arch/traits.h b/src/global/arch/traits.h index 4cde4fca5..65cc63cf8 100644 --- a/src/global/arch/traits.h +++ b/src/global/arch/traits.h @@ -112,6 +112,18 @@ namespace traits { template using fix_fields_const_t = decltype(&T::FixFieldsConst); + template + using perfect_conductor_fields_t = decltype(&T::PerfectConductorFields); + + template + using perfect_conductor_fields_const_t = decltype(&T::PerfectConductorFieldsConst); + + template + using perfect_conductor_currents_t = decltype(&T::PerfectConductorCurrents); + + template + using perfect_conductor_currents_const_t = decltype(&T::PerfectConductorCurrentsConst); + template using custom_fields_t = decltype(&T::CustomFields); diff --git a/src/global/enums.h b/src/global/enums.h index 8f2495c13..2b3bf5936 100644 --- a/src/global/enums.h +++ b/src/global/enums.h @@ -222,15 +222,16 @@ namespace ntt { HORIZON = 6, AXIS = 7, SYNC = 8, // <- SYNC means synchronization with other domains + CONDUCTOR = 9 }; constexpr FldsBC(uint8_t c) : enums_hidden::BaseEnum { c } {} - static constexpr type variants[] = { PERIODIC, MATCH, FIXED, ATMOSPHERE, - CUSTOM, HORIZON, AXIS, SYNC }; + static constexpr type variants[] = { PERIODIC, MATCH, FIXED, ATMOSPHERE, + CUSTOM, HORIZON, AXIS, SYNC, CONDUCTOR }; static constexpr const char* lookup[] = { "periodic", "match", "fixed", "atmosphere", "custom", "horizon", - "axis", "sync" }; + "axis", "sync", "conductor"}; static constexpr std::size_t total = sizeof(variants) / sizeof(variants[0]); }; From bacbe24117dff5ebb832465aa9c539c9e0231900 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Feb 2025 10:43:32 -0600 Subject: [PATCH 002/176] first stubborn attempt at conductor boundaries (broken) --- setups/srpic/shock/pgen.hpp | 46 +++++++++++++ src/engines/srpic.hpp | 133 ++++++++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index b8f169521..59c5590c9 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -106,6 +106,52 @@ namespace user { } } + + auto PerfectConductorFieldsConst(const bc_in&, const em& comp) const -> real_t{ + + // electric field components + if (comp == em::ex1) { + return ONE; + } else if (comp == em::ex2) { + return -ONE; + } else if (comp == em::ex3) { + return -ONE; } + // magentic field components + else if (comp == em::bx1) { + return -ONE; + } else if (comp == em::bx2) { + return ONE; + } else if (comp == em::bx3) { + return ONE;} + // should never be the case + else + { + return ZERO; + } + } + + // auto PerfectConductorCurrentsConst(const bc_in &, const cur &comp) const + // -> std::pair + // { + // // ToDo + // if (comp == cur::jx1) + // { + // return ZERO; + // } + // else if (comp == cur::jx2) + // { + // return ZERO; + // } + // else if (comp == cur::jx3) + // { + // return ZERO; + // } + // else + // { + // return ZERO; + // } + // } + auto MatchFields(real_t time) const -> InitFields { return init_flds; } diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 9f5e4551f..4fca96674 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -596,6 +596,10 @@ namespace ntt { if (domain.mesh.flds_bc_in(direction) == FldsBC::FIXED) { FixedFieldsIn(direction, domain, tags); } + } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::CONDUCTOR) { + if (domain.mesh.flds_bc_in(direction) == FldsBC::CONDUCTOR) { + PerfectConductorFieldsIn(direction, domain, tags); + } } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::CUSTOM) { if (domain.mesh.flds_bc_in(direction) == FldsBC::CUSTOM) { CustomFieldsIn(direction, domain, tags); @@ -834,6 +838,135 @@ namespace ntt { } } + void PerfectConductorFieldsIn(dir::direction_t direction, + domain_t& domain, + BCTags tags) { + /** + * perfect conductor field boundaries + */ + const auto sign = direction.get_sign(); + const auto dim = direction.get_dim(); + raise::ErrorIf(dim != in::x1 and M::CoordType != Coord::Cart, + "Perfect conductor BCs only implemented for x1 in " + "non-cartesian coordinates", + HERE); + + // magnetic and electron field components + em normal_b_comp, tang_b_comp1, tang_b_comp2, + normal_e_comp, tang_e_comp1, tang_e_comp2; + + // current components + // cur normal_j_comp, tang_j_comp1, tang_j_comp2; + + if (dim == in::x1) { + normal_b_comp = em::bx1; + tang_b_comp1 = em::bx2; + tang_b_comp2 = em::bx3; + + normal_e_comp = em::ex1; + tang_e_comp1 = em::ex2; + tang_e_comp2 = em::ex3; + } else if (dim == in::x2) { + normal_b_comp = em::bx2; + tang_b_comp1 = em::bx1; + tang_b_comp2 = em::bx3; + + normal_e_comp = em::ex2; + tang_e_comp1 = em::ex1; + tang_e_comp2 = em::ex3; + } else if (dim == in::x3) { + normal_b_comp = em::bx3; + tang_b_comp1 = em::bx1; + tang_b_comp2 = em::bx2; + + normal_e_comp = em::ex3; + tang_e_comp1 = em::ex1; + tang_e_comp2 = em::ex2; + } else { + raise::Error("Invalid dimension", HERE); + } + + std::vector origin_xi_min, origin_xi_max, + target_xi_min, target_xi_max; + const std::vector all_dirs { in::x1, in::x2, in::x3 }; + + for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { + const auto dd = all_dirs[d]; + if (dim == dd) { + // origin: right side of boundary + origin_xi_min.push_back(N_GHOSTS+1); + origin_xi_max.push_back(2*N_GHOSTS); + // target: left side of boundary + target_xi_min.push_back(0); + target_xi_max.push_back(N_GHOSTS); + + } else { + origin_xi_min.push_back(0); + origin_xi_max.push_back(domain.mesh.n_all(dd)); + + target_xi_min.push_back(0); + target_xi_max.push_back(domain.mesh.n_all(dd)); + } + } + raise::ErrorIf(target_xi_min.size() != origin_xi_min.size() or + origin_xi_min.size() != static_cast(M::Dim), + "Invalid range size", + HERE); + + std::vector comps; + if (tags & BC::E) { + comps.push_back(normal_e_comp); + comps.push_back(tang_e_comp1); + comps.push_back(tang_e_comp2); + } + if (tags & BC::B) { + comps.push_back(normal_b_comp); + comps.push_back(tang_b_comp1); + comps.push_back(tang_b_comp2); + } + + // ToDo: smarter loop/views + auto EB = domain.fields.em; + + // loop over all components + for (const auto& comp : comps) { + + // store sign of component behind boundary + auto new_sign = m_pgen.PerfectConductorFieldsConst( + (bc_in)(sign * ((short)dim + 1)), + (em)comp); + // to do: Kokkos::parallel_for + for (int i = 0; i < N_GHOSTS; i++) + { + if constexpr (M::Dim == Dim::_1D) { + // multiply with correct sign + EB(target_xi_min[0]+i, comp) = new_sign * EB(origin_xi_max[0]-i, comp); + + } else if constexpr (M::Dim == Dim::_2D) { + for (int j = 0; j < domain.mesh.n_all(in::x2); j++) + { + EB(target_xi_min[0]+i, j, comp) = + new_sign * EB(origin_xi_max[0]-i, j, comp); + } + } else if constexpr (M::Dim == Dim::_3D) { + for (int j = 0; j < domain.mesh.n_all(in::x2); j++) + { + for (int k = 0; k < domain.mesh.n_all(in::x3); k++) + { + EB(target_xi_min[0]+i, j, k, comp) = + new_sign * EB(origin_xi_max[0]-i, j, k, comp); + } + } + } else { + raise::Error("Invalid dimension", HERE); + } + } + + // ToDo: set zero at boundary + } + } + + void AtmosphereFieldsIn(dir::direction_t direction, domain_t& domain, BCTags tags) { From 2081a39b3aea9b4c9fc5d9c122bd168879781aca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Feb 2025 16:41:20 -0600 Subject: [PATCH 003/176] first attempt at ConductorBoundaries_kernel --- src/kernels/fields_bcs.hpp | 107 +++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 363ff3ad2..fa0b7ec0d 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -485,6 +485,113 @@ namespace kernel::bc { } }; + template + struct ConductorBoundaries_kernel { + static_assert(M::is_metric, "M must be a metric class"); + static_assert(static_cast(o) < + static_cast(M::Dim), + "Invalid component index"); + static constexpr idx_t i = static_cast(o) + 1u; + + ndfield_t Fld; + const I fset; + const M metric; + const BCTags tags; + + ConductorBoundaries_kernel(ndfield_t Fld, + BCTags tags) + : Fld { Fld } + , tags { tags } {} + + Inline void operator()(index_t i1) const { + if constexpr (M::Dim == Dim::_1D) { + + if constexpr (S == SimEngine::SRPIC) { + + if (tags & BC::E) { + Fld((N_GHOSTS-1)-i1, em::ex1) = Fld(N_GHOSTS+i1, em::ex1); + Fld((N_GHOSTS-1)-i1, em::ex2) = -Fld(N_GHOSTS+i1, em::ex2); + Fld((N_GHOSTS-1)-i1, em::ex3) = -Fld(N_GHOSTS+i1, em::ex3); + } + + if (tags & BC::B) + { + Fld((N_GHOSTS-1)-i1, em::bx1) = -Fld(N_GHOSTS+i1, em::bx1); + Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); + Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); + } + + } else { + // GRPIC + raise::KernelError(HERE, "1D GRPIC not implemented"); + } + } else { + raise::KernelError( + HERE, + "ConductorBoundaries_kernel: 1D implementation called for D != 1"); + } + } + + Inline void operator()(index_t i1, index_t i2) const { + if constexpr (M::Dim == Dim::_2D) { + + if constexpr (S == SimEngine::SRPIC) { + // SRPIC + if (tags & BC::E) { + Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); + Fld((N_GHOSTS-1)-i1, i2, em::ex2) = -Fld(N_GHOSTS+i1, i2, em::ex2); + Fld((N_GHOSTS-1)-i1, i2, em::ex3) = -Fld(N_GHOSTS+i1, i2, em::ex3); + } + + if (tags & BC::B) + { + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = -Fld(N_GHOSTS+i1, i2, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); + } + } else { + // GRPIC + raise::KernelError(HERE, "2D GRPIC not implemented"); + } + } else { + raise::KernelError( + HERE, + "ConductorBoundaries_kernel: 2D implementation called for D != 2"); + } + } + + Inline void operator()(index_t i1, index_t i2, index_t i3) const { + if constexpr (M::Dim == Dim::_3D) { + const auto i1_ = COORD(i1); + const auto i2_ = COORD(i2); + const auto i3_ = COORD(i3); + + if constexpr (S == SimEngine::SRPIC) { + // SRPIC + if (tags & BC::E) { + Fld((N_GHOSTS-1)-i1, i2, i3, em::ex1) = Fld(N_GHOSTS+i1, i2, i3, em::ex1); + Fld((N_GHOSTS-1)-i1, i2, i3, em::ex2) = -Fld(N_GHOSTS+i1, i2, i3, em::ex2); + Fld((N_GHOSTS-1)-i1, i2, i3, em::ex3) = -Fld(N_GHOSTS+i1, i2, i3, em::ex3); + } + + if (tags & BC::B) + { + Fld((N_GHOSTS-1)-i1, i2, i3, em::bx1) = -Fld(N_GHOSTS+i1, i2, i3, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, i3, em::bx2) = Fld(N_GHOSTS+i1, i2, i3, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, i3, em::bx3) = Fld(N_GHOSTS+i1, i2, i3, em::bx3); + } + } else { + // GRPIC + raise::KernelError(HERE, "3D GRPIC not implemented"); + } + } else { + raise::KernelError( + HERE, + "ConductorBoundaries_kernel: 3D implementation called for D != 3"); + } + } + }; + /* * @tparam D: Dimension * @tparam P: Positive/Negative direction From b7a863c6de52fe5e32bdc0fa89010e07237f5be7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Feb 2025 16:55:21 -0600 Subject: [PATCH 004/176] bugfix --- src/kernels/fields_bcs.hpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index fa0b7ec0d..3a253820a 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -485,7 +485,7 @@ namespace kernel::bc { } }; - template + template struct ConductorBoundaries_kernel { static_assert(M::is_metric, "M must be a metric class"); static_assert(static_cast(o) < @@ -494,7 +494,6 @@ namespace kernel::bc { static constexpr idx_t i = static_cast(o) + 1u; ndfield_t Fld; - const I fset; const M metric; const BCTags tags; @@ -562,13 +561,10 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2, index_t i3) const { if constexpr (M::Dim == Dim::_3D) { - const auto i1_ = COORD(i1); - const auto i2_ = COORD(i2); - const auto i3_ = COORD(i3); if constexpr (S == SimEngine::SRPIC) { // SRPIC - if (tags & BC::E) { + if (tags & BC::E) { Fld((N_GHOSTS-1)-i1, i2, i3, em::ex1) = Fld(N_GHOSTS+i1, i2, i3, em::ex1); Fld((N_GHOSTS-1)-i1, i2, i3, em::ex2) = -Fld(N_GHOSTS+i1, i2, i3, em::ex2); Fld((N_GHOSTS-1)-i1, i2, i3, em::ex3) = -Fld(N_GHOSTS+i1, i2, i3, em::ex3); From f383e492633417d53f7ddd266b29fe07004f03d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Feb 2025 17:02:04 -0600 Subject: [PATCH 005/176] first attempt at Kokkos::parallel_for loop (broken) --- src/engines/srpic.hpp | 117 ++++++------------------------------------ 1 file changed, 17 insertions(+), 100 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 4fca96674..4198c9531 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -851,122 +851,39 @@ namespace ntt { "non-cartesian coordinates", HERE); - // magnetic and electron field components - em normal_b_comp, tang_b_comp1, tang_b_comp2, - normal_e_comp, tang_e_comp1, tang_e_comp2; - // current components - // cur normal_j_comp, tang_j_comp1, tang_j_comp2; - - if (dim == in::x1) { - normal_b_comp = em::bx1; - tang_b_comp1 = em::bx2; - tang_b_comp2 = em::bx3; - - normal_e_comp = em::ex1; - tang_e_comp1 = em::ex2; - tang_e_comp2 = em::ex3; - } else if (dim == in::x2) { - normal_b_comp = em::bx2; - tang_b_comp1 = em::bx1; - tang_b_comp2 = em::bx3; - - normal_e_comp = em::ex2; - tang_e_comp1 = em::ex1; - tang_e_comp2 = em::ex3; - } else if (dim == in::x3) { - normal_b_comp = em::bx3; - tang_b_comp1 = em::bx1; - tang_b_comp2 = em::bx2; - - normal_e_comp = em::ex3; - tang_e_comp1 = em::ex1; - tang_e_comp2 = em::ex2; - } else { - raise::Error("Invalid dimension", HERE); - } + std::vector xi_min, xi_max; - std::vector origin_xi_min, origin_xi_max, - target_xi_min, target_xi_max; const std::vector all_dirs { in::x1, in::x2, in::x3 }; for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { const auto dd = all_dirs[d]; if (dim == dd) { - // origin: right side of boundary - origin_xi_min.push_back(N_GHOSTS+1); - origin_xi_max.push_back(2*N_GHOSTS); - // target: left side of boundary - target_xi_min.push_back(0); - target_xi_max.push_back(N_GHOSTS); - + xi_min.push_back(0); + xi_max.push_back(N_GHOSTS); } else { - origin_xi_min.push_back(0); - origin_xi_max.push_back(domain.mesh.n_all(dd)); - - target_xi_min.push_back(0); - target_xi_max.push_back(domain.mesh.n_all(dd)); + xi_min.push_back(0); + xi_max.push_back(domain.mesh.n_all(dd)); } } - raise::ErrorIf(target_xi_min.size() != origin_xi_min.size() or - origin_xi_min.size() != static_cast(M::Dim), + raise::ErrorIf(xi_min.size() != xi_max.size() or + xi_min.size() != static_cast(M::Dim), "Invalid range size", HERE); - std::vector comps; - if (tags & BC::E) { - comps.push_back(normal_e_comp); - comps.push_back(tang_e_comp1); - comps.push_back(tang_e_comp2); - } - if (tags & BC::B) { - comps.push_back(normal_b_comp); - comps.push_back(tang_b_comp1); - comps.push_back(tang_b_comp2); - } - - // ToDo: smarter loop/views - auto EB = domain.fields.em; - - // loop over all components - for (const auto& comp : comps) { - - // store sign of component behind boundary - auto new_sign = m_pgen.PerfectConductorFieldsConst( - (bc_in)(sign * ((short)dim + 1)), - (em)comp); - // to do: Kokkos::parallel_for - for (int i = 0; i < N_GHOSTS; i++) - { - if constexpr (M::Dim == Dim::_1D) { - // multiply with correct sign - EB(target_xi_min[0]+i, comp) = new_sign * EB(origin_xi_max[0]-i, comp); - - } else if constexpr (M::Dim == Dim::_2D) { - for (int j = 0; j < domain.mesh.n_all(in::x2); j++) - { - EB(target_xi_min[0]+i, j, comp) = - new_sign * EB(origin_xi_max[0]-i, j, comp); - } - } else if constexpr (M::Dim == Dim::_3D) { - for (int j = 0; j < domain.mesh.n_all(in::x2); j++) - { - for (int k = 0; k < domain.mesh.n_all(in::x3); k++) - { - EB(target_xi_min[0]+i, j, k, comp) = - new_sign * EB(origin_xi_max[0]-i, j, k, comp); - } - } - } else { - raise::Error("Invalid dimension", HERE); - } - } - - // ToDo: set zero at boundary + if (dim == in::x1) + { + Kokkos::parallel_for( + "MatchFields", + CreateRangePolicy(xi_min, xi_max), + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); } + + } - void AtmosphereFieldsIn(dir::direction_t direction, domain_t& domain, BCTags tags) { From 7666918b4117ab726422a6440fcf0135aa39cb05 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 21 Feb 2025 22:12:56 -0500 Subject: [PATCH 006/176] Ongoing. --- src/engines/srpic.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 4198c9531..fadb55087 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -871,11 +871,11 @@ namespace ntt { "Invalid range size", HERE); - if (dim == in::x1) + if constexpr (M::Dim == Dim::_1D) { { Kokkos::parallel_for( "MatchFields", - CreateRangePolicy(xi_min, xi_max), + CreateRangePolicy(xi_min[0], xi_max[0]), kernel::bc::ConductorBoundaries_kernel( domain.fields.em, tags)); From af065442ded1c0fa3ada6ab01398ec522cde1e16 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 21 Feb 2025 22:16:54 -0500 Subject: [PATCH 007/176] Ongoing. --- src/engines/srpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index fadb55087..da111256a 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -871,7 +871,7 @@ namespace ntt { "Invalid range size", HERE); - if constexpr (M::Dim == Dim::_1D) { + if constexpr (M::Dim == Dim::_1D) { Kokkos::parallel_for( "MatchFields", From 0d3cc1cf90ce489d23f918c1b657b98d8b44e0fe Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 21 Feb 2025 22:19:01 -0500 Subject: [PATCH 008/176] Ongoing. --- src/engines/srpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index da111256a..7f38d567a 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -875,7 +875,7 @@ namespace ntt { { Kokkos::parallel_for( "MatchFields", - CreateRangePolicy(xi_min[0], xi_max[0]), + CreateRangePolicy( { xi_min[0] } , { xi_max[0] } ), kernel::bc::ConductorBoundaries_kernel( domain.fields.em, tags)); From 4a57c38538916967860bdbbbfe34e452ed785942 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 21 Feb 2025 22:35:06 -0500 Subject: [PATCH 009/176] Ongoing. --- src/engines/srpic.hpp | 2 +- src/kernels/fields_bcs.hpp | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 7f38d567a..6e4f51945 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -876,7 +876,7 @@ namespace ntt { Kokkos::parallel_for( "MatchFields", CreateRangePolicy( { xi_min[0] } , { xi_max[0] } ), - kernel::bc::ConductorBoundaries_kernel( + kernel::bc::ConductorBoundaries_kernel( domain.fields.em, tags)); } diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 3a253820a..48f2aa7d3 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -494,7 +494,6 @@ namespace kernel::bc { static constexpr idx_t i = static_cast(o) + 1u; ndfield_t Fld; - const M metric; const BCTags tags; ConductorBoundaries_kernel(ndfield_t Fld, From ff5a3ee8dbb2ca2bc60279f8885918497a1acb5a Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 21 Feb 2025 22:35:54 -0500 Subject: [PATCH 010/176] Ongoing. --- src/engines/srpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 6e4f51945..7f38d567a 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -876,7 +876,7 @@ namespace ntt { Kokkos::parallel_for( "MatchFields", CreateRangePolicy( { xi_min[0] } , { xi_max[0] } ), - kernel::bc::ConductorBoundaries_kernel( + kernel::bc::ConductorBoundaries_kernel( domain.fields.em, tags)); } From d6b2b1c266a127763a60ee6706f59f64214f7795 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 11:16:58 -0500 Subject: [PATCH 011/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 179 ++++++++++++++++++++++++++++++ setups/srpic/shocktest/shock.py | 75 +++++++++++++ setups/srpic/shocktest/shock.toml | 56 ++++++++++ 3 files changed, 310 insertions(+) create mode 100644 setups/srpic/shocktest/pgen.hpp create mode 100644 setups/srpic/shocktest/shock.py create mode 100644 setups/srpic/shocktest/shock.toml diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp new file mode 100644 index 000000000..83f19df52 --- /dev/null +++ b/setups/srpic/shocktest/pgen.hpp @@ -0,0 +1,179 @@ +#ifndef PROBLEM_GENERATOR_H +#define PROBLEM_GENERATOR_H + +#include "enums.h" +#include "global.h" + +#include "arch/traits.h" +#include "utils/error.h" +#include "utils/numeric.h" + +#include "archetypes/energy_dist.h" +#include "archetypes/particle_injector.h" +#include "archetypes/problem_generator.h" +#include "framework/domain/metadomain.h" + +#include + +namespace user { + using namespace ntt; + + template + struct InitFields { + /* + Sets up magnetic and electric field components for the simulation. + Must satisfy E = -v x B for Lorentz Force to be zero. + + @param bmag: magnetic field scaling + @param btheta: magnetic field polar angle + @param bphi: magnetic field azimuthal angle + @param drift_ux: drift velocity in the x direction + */ + InitFields(real_t bmag, real_t btheta, real_t bphi, real_t drift_ux) + : Bmag { bmag } + , Btheta { btheta * static_cast(convert::deg2rad) } + , Bphi { bphi * static_cast(convert::deg2rad) } + , Vx { drift_ux } {} + + // magnetic field components + Inline auto bx1(const coord_t& x_Ph) const -> real_t { + return x_Ph[0]; + } + + Inline auto bx2(const coord_t&) const -> real_t { + return x_Ph[0]; + } + + Inline auto bx3(const coord_t&) const -> real_t { + return x_Ph[0]; + } + + // electric field components + Inline auto ex1(const coord_t&) const -> real_t { + return x_Ph[0]; + } + + Inline auto ex2(const coord_t&) const -> real_t { + return x_Ph[0]; + } + + Inline auto ex3(const coord_t&) const -> real_t { + return x_Ph[0]; + } + + private: + const real_t Btheta, Bphi, Vx, Bmag; + }; + + template + struct PGen : public arch::ProblemGenerator { + // compatibility traits for the problem generator + static constexpr auto engines { traits::compatible_with::value }; + static constexpr auto metrics { traits::compatible_with::value }; + static constexpr auto dimensions { + traits::compatible_with::value + }; + + // for easy access to variables in the child class + using arch::ProblemGenerator::D; + using arch::ProblemGenerator::C; + using arch::ProblemGenerator::params; + + const real_t drift_ux, temperature; + + const real_t Btheta, Bphi, Bmag; + InitFields init_flds; + + inline PGen(const SimulationParams& p, const Metadomain& m) + : arch::ProblemGenerator { p } + , drift_ux { p.template get("setup.drift_ux") } + , temperature { p.template get("setup.temperature") } + , Bmag { p.template get("setup.Bmag", ZERO) } + , Btheta { p.template get("setup.Btheta", ZERO) } + , Bphi { p.template get("setup.Bphi", ZERO) } + , init_flds { Bmag, Btheta, Bphi, drift_ux } {} + + inline PGen() {} + + auto FixFieldsConst(const bc_in&, const em& comp) const + -> std::pair { + if (comp == em::ex2) { + return { init_flds.ex2({ ZERO }), true }; + } else if (comp == em::ex3) { + return { init_flds.ex3({ ZERO }), true }; + } else { + return { ZERO, false }; + } + } + + + auto PerfectConductorFieldsConst(const bc_in&, const em& comp) const -> real_t{ + + // electric field components + if (comp == em::ex1) { + return ONE; + } else if (comp == em::ex2) { + return -ONE; + } else if (comp == em::ex3) { + return -ONE; } + // magentic field components + else if (comp == em::bx1) { + return -ONE; + } else if (comp == em::bx2) { + return ONE; + } else if (comp == em::bx3) { + return ONE;} + // should never be the case + else + { + return ZERO; + } + } + + // auto PerfectConductorCurrentsConst(const bc_in &, const cur &comp) const + // -> std::pair + // { + // // ToDo + // if (comp == cur::jx1) + // { + // return ZERO; + // } + // else if (comp == cur::jx2) + // { + // return ZERO; + // } + // else if (comp == cur::jx3) + // { + // return ZERO; + // } + // else + // { + // return ZERO; + // } + // } + + auto MatchFields(real_t time) const -> InitFields { + return init_flds; + } + + inline void InitPrtls(Domain& local_domain) { + const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, + local_domain.random_pool, + temperature, + -drift_ux, + in::x1); + + const auto injector = arch::UniformInjector( + energy_dist, + { 1, 2 }); + // arch::InjectUniform>( + // params, + // local_domain, + // injector, + // 1.0); + } + }; + +} // namespace user + +#endif diff --git a/setups/srpic/shocktest/shock.py b/setups/srpic/shocktest/shock.py new file mode 100644 index 000000000..dc1565572 --- /dev/null +++ b/setups/srpic/shocktest/shock.py @@ -0,0 +1,75 @@ +import nt2.read as nt2r +import matplotlib.pyplot as plt +import matplotlib as mpl + +data = nt2r.Data("shock.h5") + + +def frame(ti, f): + quantities = [ + { + "name": "density", + "compute": lambda f: f.N_2 + f.N_1, + "cmap": "inferno", + "norm": mpl.colors.Normalize(0, 5), + }, + { + "name": r"$E_x$", + "compute": lambda f: f.Ex, + "cmap": "RdBu_r", + "norm": mpl.colors.Normalize(-0.05, 0.05), + }, + { + "name": r"$E_y$", + "compute": lambda f: f.Ey, + "cmap": "RdBu_r", + "norm": mpl.colors.Normalize(-0.05, 0.05), + }, + { + "name": r"$E_z$", + "compute": lambda f: f.Ez, + "cmap": "RdBu_r", + "norm": mpl.colors.Normalize(-0.05, 0.05), + }, + { + "name": r"$B_x$", + "compute": lambda f: f.Bx, + "cmap": "BrBG", + "norm": mpl.colors.Normalize(-0.05, 0.05), + }, + { + "name": r"$B_y$", + "compute": lambda f: f.By, + "cmap": "BrBG", + "norm": mpl.colors.Normalize(-0.05, 0.05), + }, + { + "name": r"$B_z$", + "compute": lambda f: f.Bz, + "cmap": "BrBG", + "norm": mpl.colors.Normalize(-0.05, 0.05), + }, + ] + fig = plt.figure(figsize=(12, 5.5), dpi=300) + gs = fig.add_gridspec(len(quantities), 1, hspace=0.02) + axs = [fig.add_subplot(gs[i]) for i in range(len(quantities))] + + for ax, q in zip(axs, quantities): + q["compute"](f.isel(t=ti)).plot( + ax=ax, + cmap=q["cmap"], + norm=q["norm"], + cbar_kwargs={"label": q["name"], "shrink": 0.8, "aspect": 10, "pad": 0.005}, + ) + for i, ax in enumerate(axs): + ax.set(aspect=1) + if i != 0: + ax.set(title=None) + if i != len(axs) - 1: + ax.set( + xticks=[], + xticklabels=[], + xlabel=None, + title=ax.get_title().split(",")[0], + ) + return fig diff --git a/setups/srpic/shocktest/shock.toml b/setups/srpic/shocktest/shock.toml new file mode 100644 index 000000000..fdbc44465 --- /dev/null +++ b/setups/srpic/shocktest/shock.toml @@ -0,0 +1,56 @@ +[simulation] + name = "shock" + engine = "srpic" + runtime = 50.0 + +[grid] + resolution = [2048, 128] + extent = [[0.0, 10.0], [-0.3125, 0.3125]] + + [grid.metric] + metric = "minkowski" + + [grid.boundaries] + fields = [["CONDUCTOR", "FIXED"], ["PERIODIC"]] + particles = [["REFLECT", "ABSORB"], ["PERIODIC"]] + +[scales] + larmor0 = 1e-2 + skindepth0 = 1e-2 + +[algorithms] + current_filters = 8 + fieldsolver = "false" + deposit = "false" + + [algorithms.timestep] + CFL = 0.5 + +[particles] + ppc0 = 16.0 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = -1.0 + maxnpart = 1e8 + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = 1.0 + maxnpart = 1e8 + +[setup] + drift_ux = 0.1 + temperature = 1e-3 + Bmag = 1.0 + Btheta = 0.0 + Bphi = 0.0 + +[output] + interval = 1 + format = "hdf5" + + [output.fields] + quantities = ["N_1", "N_2", "E", "B", "T0i_1", "T0i_2", "J"] From f0ed6c7ba59988ecb52de96760643a3d34d513bf Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 11:20:28 -0500 Subject: [PATCH 012/176] Ongoing. --- setups/srpic/shocktest/shock.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setups/srpic/shocktest/shock.toml b/setups/srpic/shocktest/shock.toml index fdbc44465..a77f5c2e9 100644 --- a/setups/srpic/shocktest/shock.toml +++ b/setups/srpic/shocktest/shock.toml @@ -54,3 +54,6 @@ [output.fields] quantities = ["N_1", "N_2", "E", "B", "T0i_1", "T0i_2", "J"] + + [output.debug] + ghosts = true From 79d323741ef5eeeaf939ee502a56629842baeac1 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 14:01:06 -0500 Subject: [PATCH 013/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 83f19df52..7724893a1 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -40,24 +40,24 @@ namespace user { return x_Ph[0]; } - Inline auto bx2(const coord_t&) const -> real_t { + Inline auto bx2(const coord_t& x_Ph) const -> real_t { return x_Ph[0]; } - Inline auto bx3(const coord_t&) const -> real_t { + Inline auto bx3(const coord_t& x_Ph) const -> real_t { return x_Ph[0]; } // electric field components - Inline auto ex1(const coord_t&) const -> real_t { + Inline auto ex1(const coord_t& x_Ph) const -> real_t { return x_Ph[0]; } - Inline auto ex2(const coord_t&) const -> real_t { + Inline auto ex2(const coord_t& x_Ph) const -> real_t { return x_Ph[0]; } - Inline auto ex3(const coord_t&) const -> real_t { + Inline auto ex3(const coord_t& x_Ph) const -> real_t { return x_Ph[0]; } From 17b469bfa3137317145a50853271773e3ec0c8b2 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 17:15:38 -0500 Subject: [PATCH 014/176] Ongoing. --- src/kernels/fields_bcs.hpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 48f2aa7d3..8eda1b34a 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,9 +543,12 @@ namespace kernel::bc { if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = -Fld(N_GHOSTS+i1, i2, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); + // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = -Fld(N_GHOSTS+i1, i2, em::bx1); + // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); + // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; } } else { // GRPIC From e234173bd8688c8585f4ef6d51c1cc721ff49e38 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 17:21:07 -0500 Subject: [PATCH 015/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 7724893a1..0dc91d3e2 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -41,7 +41,7 @@ namespace user { } Inline auto bx2(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return -x_Ph[0]; } Inline auto bx3(const coord_t& x_Ph) const -> real_t { From de510ea1abdc4e7381f5705e38fbe33510e3bc9a Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 18:50:58 -0500 Subject: [PATCH 016/176] Ongoing. --- src/engines/srpic.hpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 7f38d567a..90aa271a3 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -880,6 +880,16 @@ namespace ntt { domain.fields.em, tags)); } + + if constexpr (M::Dim == Dim::_2D) + { + Kokkos::parallel_for( + "MatchFields", + CreateRangePolicy( { xi_min[0], xi_min[1] } , { xi_max[0], xi_max[1] } ), + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } } From 0524d782381c66c6fcf291027e85009ccdfa8cd6 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 19:05:19 -0500 Subject: [PATCH 017/176] Ongoing. --- src/kernels/fields_bcs.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 8eda1b34a..490042304 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,12 +543,12 @@ namespace kernel::bc { if (tags & BC::B) { - // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = -Fld(N_GHOSTS+i1, i2, em::bx1); - // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); - // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = Fld(N_GHOSTS+i1, i2, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); + // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; + // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; + // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; } } else { // GRPIC From 9d78b9e2a43740cc1a1fb7d2faffc6871c8ba0b1 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 19:10:18 -0500 Subject: [PATCH 018/176] Ongoing. --- src/kernels/fields_bcs.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 490042304..33f2813ca 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,9 +543,9 @@ namespace kernel::bc { if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = Fld(N_GHOSTS+i1, i2, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; From cec22b04960cad3889473c956a17b4e270ef83c0 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 20:01:27 -0500 Subject: [PATCH 019/176] Ongoing. --- src/kernels/fields_bcs.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 33f2813ca..aebe35794 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,9 +543,9 @@ namespace kernel::bc { if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); + Fld((N_GHOSTS)-i1, i2, em::bx1) = - Fld(N_GHOSTS+1+i1, i2, em::bx1); + Fld((N_GHOSTS)-i1, i2, em::bx2) = - Fld(N_GHOSTS+1+i1, i2, em::bx2); + Fld((N_GHOSTS)-i1, i2, em::bx3) = - Fld(N_GHOSTS+1+i1, i2, em::bx3); // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; From b3754e29a4bf20ea04a11e494866dd7c5323d59c Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 20:54:37 -0500 Subject: [PATCH 020/176] Ongoing. --- src/kernels/fields_bcs.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index aebe35794..33f2813ca 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,9 +543,9 @@ namespace kernel::bc { if (tags & BC::B) { - Fld((N_GHOSTS)-i1, i2, em::bx1) = - Fld(N_GHOSTS+1+i1, i2, em::bx1); - Fld((N_GHOSTS)-i1, i2, em::bx2) = - Fld(N_GHOSTS+1+i1, i2, em::bx2); - Fld((N_GHOSTS)-i1, i2, em::bx3) = - Fld(N_GHOSTS+1+i1, i2, em::bx3); + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; From 83dba373e226994a77d5845b574b12c43e69e947 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 21:02:54 -0500 Subject: [PATCH 021/176] Ongoing. --- src/kernels/fields_bcs.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 33f2813ca..3e6458877 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,12 +543,12 @@ namespace kernel::bc { if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); - // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; - // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; - // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; + // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); + // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); + // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; } } else { // GRPIC From a7ef051440b5695e3c30bc88dc63bba7865abf2c Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 21:13:50 -0500 Subject: [PATCH 022/176] Ongoing. --- src/kernels/fields_bcs.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 3e6458877..082d06329 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,12 +543,12 @@ namespace kernel::bc { if (tags & BC::B) { - // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); - // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); - // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+1+i1, i2, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); + // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; + // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; + // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; } } else { // GRPIC From d219fc43854fcaafc099d3889587403f1b2d1a38 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 21:43:02 -0500 Subject: [PATCH 023/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 0dc91d3e2..685521f09 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -45,7 +45,7 @@ namespace user { } Inline auto bx3(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return ONE; } // electric field components From 044ccea51284764cc110069d64d6702f2c028913 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 22:06:25 -0500 Subject: [PATCH 024/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 685521f09..0dc91d3e2 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -45,7 +45,7 @@ namespace user { } Inline auto bx3(const coord_t& x_Ph) const -> real_t { - return ONE; + return x_Ph[0]; } // electric field components From 2af41d83e01527f3bdbb2548e95ab7d098648b25 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 22:16:17 -0500 Subject: [PATCH 025/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 0dc91d3e2..f8bf67dab 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -37,7 +37,7 @@ namespace user { // magnetic field components Inline auto bx1(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return ZERO; } Inline auto bx2(const coord_t& x_Ph) const -> real_t { @@ -50,15 +50,15 @@ namespace user { // electric field components Inline auto ex1(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return ZERO; } Inline auto ex2(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return ZERO; } Inline auto ex3(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return ZERO; } private: From 056c6294df2c23d568805e402eb28a1ae9952fb7 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 22:34:56 -0500 Subject: [PATCH 026/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index f8bf67dab..39ce8ea03 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -45,7 +45,7 @@ namespace user { } Inline auto bx3(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return -x_Ph[0]; } // electric field components From 7d9a1a39d75d239bb8ff387bc47cee4c5305f333 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 22:58:19 -0500 Subject: [PATCH 027/176] Ongoing. --- src/kernels/fields_bcs.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 082d06329..fe5ae4122 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -536,9 +536,9 @@ namespace kernel::bc { if constexpr (S == SimEngine::SRPIC) { // SRPIC if (tags & BC::E) { - Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); - Fld((N_GHOSTS-1)-i1, i2, em::ex2) = -Fld(N_GHOSTS+i1, i2, em::ex2); - Fld((N_GHOSTS-1)-i1, i2, em::ex3) = -Fld(N_GHOSTS+i1, i2, em::ex3); + Fld((N_GHOSTS-1)-i1, i2, em::ex1) = - Fld(N_GHOSTS+i1, i2, em::ex1); + Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+1+i1, i2, em::ex2); + Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+1+i1, i2, em::ex3); } if (tags & BC::B) From d8abf1b2688f61636fb604b017e5c7a85073c0df Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Tue, 25 Feb 2025 23:51:36 -0500 Subject: [PATCH 028/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 39ce8ea03..2cb66539f 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -36,16 +36,16 @@ namespace user { , Vx { drift_ux } {} // magnetic field components - Inline auto bx1(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto bx1(const coord_t&) const -> real_t { + return Bmag * math::cos(Btheta); } - Inline auto bx2(const coord_t& x_Ph) const -> real_t { - return -x_Ph[0]; + Inline auto bx2(const coord_t&) const -> real_t { + return Bmag * math::sin(Btheta) * math::sin(Bphi); } - Inline auto bx3(const coord_t& x_Ph) const -> real_t { - return -x_Ph[0]; + Inline auto bx3(const coord_t&) const -> real_t { + return Bmag * math::sin(Btheta) * math::cos(Bphi); } // electric field components From a17db6280545241cd28c85936d772314369606e6 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Tue, 25 Feb 2025 23:59:44 -0500 Subject: [PATCH 029/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 2cb66539f..b77583ccb 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -49,16 +49,16 @@ namespace user { } // electric field components - Inline auto ex1(const coord_t& x_Ph) const -> real_t { + Inline auto ex1(const coord_t&) const -> real_t { return ZERO; } - Inline auto ex2(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto ex2(const coord_t&) const -> real_t { + return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); } - Inline auto ex3(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto ex3(const coord_t&) const -> real_t { + return Vx * Bmag * math::sin(Btheta) * math::sin(Bphi); } private: From 1d109a062753f09885239f289c4c0cf2280320a7 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Thu, 27 Feb 2025 11:42:46 -0500 Subject: [PATCH 030/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index b77583ccb..f92086528 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -36,29 +36,34 @@ namespace user { , Vx { drift_ux } {} // magnetic field components - Inline auto bx1(const coord_t&) const -> real_t { - return Bmag * math::cos(Btheta); + Inline auto bx1(const coord_t& x_ph) const -> real_t { + // return Bmag * math::cos(Btheta); + return ZERO; } - Inline auto bx2(const coord_t&) const -> real_t { - return Bmag * math::sin(Btheta) * math::sin(Bphi); + Inline auto bx2(const coord_t& x_ph) const -> real_t { + // return Bmag * math::sin(Btheta) * math::sin(Bphi); + return ZERO; } - Inline auto bx3(const coord_t&) const -> real_t { - return Bmag * math::sin(Btheta) * math::cos(Bphi); + Inline auto bx3(const coord_t& x_ph) const -> real_t { + // return Bmag * math::sin(Btheta) * math::cos(Bphi); + return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x))*math::tanh(20.*(-0.5 + x)))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); } // electric field components - Inline auto ex1(const coord_t&) const -> real_t { + Inline auto ex1(const coord_t& x_ph) const -> real_t { return ZERO; } - Inline auto ex2(const coord_t&) const -> real_t { - return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); + Inline auto ex2(const coord_t& x_ph) const -> real_t { + // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); + return ZERO + 0.01 * (ONE - math::tanh(20.*(-1.5 + x))*math::tanh(20.*(-0.5 + x)))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); } - Inline auto ex3(const coord_t&) const -> real_t { - return Vx * Bmag * math::sin(Btheta) * math::sin(Bphi); + Inline auto ex3(const coord_t& x_ph) const -> real_t { + // return Vx * Bmag * math::sin(Btheta) * math::sin(Bphi); + return ZERO; } private: From bee7530bf59a8386b14799114a393a015105f7f7 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Thu, 27 Feb 2025 12:01:42 -0500 Subject: [PATCH 031/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index f92086528..c43688d02 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -48,7 +48,7 @@ namespace user { Inline auto bx3(const coord_t& x_ph) const -> real_t { // return Bmag * math::sin(Btheta) * math::cos(Bphi); - return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x))*math::tanh(20.*(-0.5 + x)))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); } // electric field components @@ -58,7 +58,7 @@ namespace user { Inline auto ex2(const coord_t& x_ph) const -> real_t { // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - return ZERO + 0.01 * (ONE - math::tanh(20.*(-1.5 + x))*math::tanh(20.*(-0.5 + x)))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ZERO + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); } Inline auto ex3(const coord_t& x_ph) const -> real_t { From 2a269fde2d70d3e2b05b7627b30eaa4484ac83f6 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Thu, 27 Feb 2025 13:13:12 -0500 Subject: [PATCH 032/176] Ongoing. --- src/kernels/fields_bcs.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index fe5ae4122..f201d7cc8 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -536,7 +536,7 @@ namespace kernel::bc { if constexpr (S == SimEngine::SRPIC) { // SRPIC if (tags & BC::E) { - Fld((N_GHOSTS-1)-i1, i2, em::ex1) = - Fld(N_GHOSTS+i1, i2, em::ex1); + Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+1+i1, i2, em::ex2); Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+1+i1, i2, em::ex3); } @@ -544,8 +544,8 @@ namespace kernel::bc { if (tags & BC::B) { Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+1+i1, i2, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; From 49820ff2a0d67ea68b7b3cbee687e1531df96641 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Thu, 27 Feb 2025 13:18:43 -0500 Subject: [PATCH 033/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index c43688d02..10d4c6b59 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -58,7 +58,7 @@ namespace user { Inline auto ex2(const coord_t& x_ph) const -> real_t { // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - return ZERO + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); } Inline auto ex3(const coord_t& x_ph) const -> real_t { From d2167da1bc2c1c0a88ba94f2dfb4b80ca363447f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Thu, 27 Feb 2025 18:04:22 -0600 Subject: [PATCH 034/176] first attempt at single particle injection --- setups/srpic/shocktest/pgen.hpp | 164 ++++++++++++++++++++------------ src/kernels/fields_bcs.hpp | 27 +++--- 2 files changed, 116 insertions(+), 75 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 10d4c6b59..4d4b1cb37 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -111,72 +111,116 @@ namespace user { } } - - auto PerfectConductorFieldsConst(const bc_in&, const em& comp) const -> real_t{ - - // electric field components - if (comp == em::ex1) { - return ONE; - } else if (comp == em::ex2) { - return -ONE; - } else if (comp == em::ex3) { - return -ONE; } - // magentic field components - else if (comp == em::bx1) { - return -ONE; - } else if (comp == em::bx2) { - return ONE; - } else if (comp == em::bx3) { - return ONE;} - // should never be the case - else - { - return ZERO; - } - } - - // auto PerfectConductorCurrentsConst(const bc_in &, const cur &comp) const - // -> std::pair - // { - // // ToDo - // if (comp == cur::jx1) - // { - // return ZERO; - // } - // else if (comp == cur::jx2) - // { - // return ZERO; - // } - // else if (comp == cur::jx3) - // { - // return ZERO; - // } - // else - // { - // return ZERO; - // } - // } auto MatchFields(real_t time) const -> InitFields { return init_flds; } - inline void InitPrtls(Domain& local_domain) { - const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, - local_domain.random_pool, - temperature, - -drift_ux, - in::x1); - - const auto injector = arch::UniformInjector( - energy_dist, - { 1, 2 }); - // arch::InjectUniform>( - // params, - // local_domain, - // injector, - // 1.0); + inline void InitPrtls(Domain& domain) { + + auto& species_e = domain.species[0]; + auto& species_p = domain.species[1]; + auto metric = domain.mesh.metric; + auto m = domain.mesh.metric; + + array_t elec_ind("elec_ind"); + array_t pos_ind("pos_ind"); + + auto offset_e = species_e.npart(); + auto offset_p = species_p.npart(); + + auto ux1_e = species_e.ux1; + auto ux2_e = species_e.ux2; + auto ux3_e = species_e.ux3; + auto i1_e = species_e.i1; + auto i2_e = species_e.i2; + auto dx1_e = species_e.dx1; + auto dx2_e = species_e.dx2; + auto phi_e = species_e.phi; + auto weight_e = species_e.weight; + auto tag_e = species_e.tag; + + auto ux1_p = species_p.ux1; + auto ux2_p = species_p.ux2; + auto ux3_p = species_p.ux3; + auto i1_p = species_p.i1; + auto i2_p = species_p.i2; + auto dx1_p = species_p.dx1; + auto dx2_p = species_p.dx2; + auto phi_p = species_p.phi; + auto weight_p = species_p.weight; + auto tag_p = species_p.tag; + + int nseed = 1; + auto dseed = HALF * constant::PI / static_cast(nseed); + + Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { + + // ToDo: fix this + auto i1_ = ONE; + auto i2_ = ONE; + auto dx1_ = ONE - HALF; + auto dx2_ = ONE - HALF; + + + auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); + auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); + + i1_e(elec_p + offset_e) = i1_; + dx1_e(elec_p + offset_e) = dx1_; + i2_e(elec_p + offset_e) = i2_; + dx2_e(elec_p + offset_e) = dx2_; + phi_e(elec_p + offset_e) = ZERO; + ux1_e(elec_p + offset_e) = -drift_ux; + ux2_e(elec_p + offset_e) = -drift_ux; + ux3_e(elec_p + offset_e) = ZERO; + weight_e(elec_p + offset_e) = ONE; + tag_e(elec_p + offset_e) = ParticleTag::alive; + + i1_p(pos_p + offset_p) = i1_; + dx1_p(pos_p + offset_p) = dx1_; + i2_p(pos_p + offset_p) = i2_; + dx2_p(pos_p + offset_p) = dx2_; + phi_p(pos_p + offset_p) = ZERO; + ux1_p(pos_p + offset_p) = -drift_ux; + ux2_p(pos_p + offset_p) = drift_ux; + ux3_p(pos_p + offset_p) = ZERO; + weight_p(pos_p + offset_p) = ONE; + tag_p(pos_p + offset_p) = ParticleTag::alive; + + + }); + + + auto elec_ind_h = Kokkos::create_mirror(elec_ind); + Kokkos::deep_copy(elec_ind_h, elec_ind); + species_e.set_npart(offset_e + elec_ind_h()); + + auto pos_ind_h = Kokkos::create_mirror(pos_ind); + Kokkos::deep_copy(pos_ind_h, pos_ind); + species_p.set_npart(offset_p + pos_ind_h()); + + } + + + // inline void InitPrtls(Domain& local_domain) { + // const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, + // local_domain.random_pool, + // temperature, + // -drift_ux, + // in::x1); + + // const auto injector = arch::UniformInjector( + // energy_dist, + // { 1, 2 }); + // arch::InjectUniform>( + // params, + // local_domain, + // injector, + // 1.0); + // } + }; } // namespace user diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index f201d7cc8..39dfaf5dc 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -506,19 +506,19 @@ namespace kernel::bc { if constexpr (S == SimEngine::SRPIC) { - if (tags & BC::E) { - Fld((N_GHOSTS-1)-i1, em::ex1) = Fld(N_GHOSTS+i1, em::ex1); - Fld((N_GHOSTS-1)-i1, em::ex2) = -Fld(N_GHOSTS+i1, em::ex2); - Fld((N_GHOSTS-1)-i1, em::ex3) = -Fld(N_GHOSTS+i1, em::ex3); - } - - if (tags & BC::B) - { - Fld((N_GHOSTS-1)-i1, em::bx1) = -Fld(N_GHOSTS+i1, em::bx1); - Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); - Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); - } + if (tags & BC::E) + { + Fld((N_GHOSTS - 1) - i1, em::ex1) = Fld(N_GHOSTS + i1, em::ex1); + Fld((N_GHOSTS - 1) - i1, em::ex2) = -Fld(N_GHOSTS + 1 + i1, em::ex2); + Fld((N_GHOSTS - 1) - i1, em::ex3) = -Fld(N_GHOSTS + 1 + i1, em::ex3); + } + if (tags & BC::B) + { + Fld((N_GHOSTS - 1) - i1, em::bx1) = -Fld(N_GHOSTS + 1 + i1, em::bx1); + Fld((N_GHOSTS - 1) - i1, em::bx2) = Fld(N_GHOSTS + i1, em::bx2); + Fld((N_GHOSTS - 1) - i1, em::bx3) = Fld(N_GHOSTS + i1, em::bx3); + } } else { // GRPIC raise::KernelError(HERE, "1D GRPIC not implemented"); @@ -546,9 +546,6 @@ namespace kernel::bc { Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+1+i1, i2, em::bx1); Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); - // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; - // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; - // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; } } else { // GRPIC From fc3e647f62b596501107776847a9f6a9019bfb65 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 07:44:22 -0500 Subject: [PATCH 035/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 4d4b1cb37..14c2006e9 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -48,7 +48,8 @@ namespace user { Inline auto bx3(const coord_t& x_ph) const -> real_t { // return Bmag * math::sin(Btheta) * math::cos(Bphi); - return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + // return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ZERO; } // electric field components @@ -58,7 +59,8 @@ namespace user { Inline auto ex2(const coord_t& x_ph) const -> real_t { // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + // return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ZERO; } Inline auto ex3(const coord_t& x_ph) const -> real_t { @@ -120,8 +122,6 @@ namespace user { auto& species_e = domain.species[0]; auto& species_p = domain.species[1]; - auto metric = domain.mesh.metric; - auto m = domain.mesh.metric; array_t elec_ind("elec_ind"); array_t pos_ind("pos_ind"); @@ -152,15 +152,14 @@ namespace user { auto tag_p = species_p.tag; int nseed = 1; - auto dseed = HALF * constant::PI / static_cast(nseed); Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { // ToDo: fix this auto i1_ = ONE; auto i2_ = ONE; - auto dx1_ = ONE - HALF; - auto dx2_ = ONE - HALF; + auto dx1_ = HALF; + auto dx2_ = HALF; auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); From 83c144f4f99960c3c381e2945d1b319ecf15ef04 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 07:47:32 -0500 Subject: [PATCH 036/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 14c2006e9..85d08214a 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -156,8 +156,8 @@ namespace user { Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { // ToDo: fix this - auto i1_ = ONE; - auto i2_ = ONE; + auto i1_ = 100; + auto i2_ = 100; auto dx1_ = HALF; auto dx2_ = HALF; From 7c7c3f9cecd887948c9b42e73f01cfb83da17056 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 09:42:48 -0500 Subject: [PATCH 037/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 90 ++++++++++++++++----------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 85d08214a..b682cd621 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -153,51 +153,51 @@ namespace user { int nseed = 1; - Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { - - // ToDo: fix this - auto i1_ = 100; - auto i2_ = 100; - auto dx1_ = HALF; - auto dx2_ = HALF; - - - auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); - auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); - - i1_e(elec_p + offset_e) = i1_; - dx1_e(elec_p + offset_e) = dx1_; - i2_e(elec_p + offset_e) = i2_; - dx2_e(elec_p + offset_e) = dx2_; - phi_e(elec_p + offset_e) = ZERO; - ux1_e(elec_p + offset_e) = -drift_ux; - ux2_e(elec_p + offset_e) = -drift_ux; - ux3_e(elec_p + offset_e) = ZERO; - weight_e(elec_p + offset_e) = ONE; - tag_e(elec_p + offset_e) = ParticleTag::alive; - - i1_p(pos_p + offset_p) = i1_; - dx1_p(pos_p + offset_p) = dx1_; - i2_p(pos_p + offset_p) = i2_; - dx2_p(pos_p + offset_p) = dx2_; - phi_p(pos_p + offset_p) = ZERO; - ux1_p(pos_p + offset_p) = -drift_ux; - ux2_p(pos_p + offset_p) = drift_ux; - ux3_p(pos_p + offset_p) = ZERO; - weight_p(pos_p + offset_p) = ONE; - tag_p(pos_p + offset_p) = ParticleTag::alive; - - - }); - - - auto elec_ind_h = Kokkos::create_mirror(elec_ind); - Kokkos::deep_copy(elec_ind_h, elec_ind); - species_e.set_npart(offset_e + elec_ind_h()); - - auto pos_ind_h = Kokkos::create_mirror(pos_ind); - Kokkos::deep_copy(pos_ind_h, pos_ind); - species_p.set_npart(offset_p + pos_ind_h()); + // Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { + + // // ToDo: fix this + // auto i1_ = 100; + // auto i2_ = 100; + // auto dx1_ = HALF; + // auto dx2_ = HALF; + + + // auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); + // auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); + + // i1_e(elec_p + offset_e) = i1_; + // dx1_e(elec_p + offset_e) = dx1_; + // i2_e(elec_p + offset_e) = i2_; + // dx2_e(elec_p + offset_e) = dx2_; + // phi_e(elec_p + offset_e) = ZERO; + // ux1_e(elec_p + offset_e) = -drift_ux; + // ux2_e(elec_p + offset_e) = -drift_ux; + // ux3_e(elec_p + offset_e) = ZERO; + // weight_e(elec_p + offset_e) = ONE; + // tag_e(elec_p + offset_e) = ParticleTag::alive; + + // i1_p(pos_p + offset_p) = i1_; + // dx1_p(pos_p + offset_p) = dx1_; + // i2_p(pos_p + offset_p) = i2_; + // dx2_p(pos_p + offset_p) = dx2_; + // phi_p(pos_p + offset_p) = ZERO; + // ux1_p(pos_p + offset_p) = -drift_ux; + // ux2_p(pos_p + offset_p) = drift_ux; + // ux3_p(pos_p + offset_p) = ZERO; + // weight_p(pos_p + offset_p) = ONE; + // tag_p(pos_p + offset_p) = ParticleTag::alive; + + + // }); + + + // auto elec_ind_h = Kokkos::create_mirror(elec_ind); + // Kokkos::deep_copy(elec_ind_h, elec_ind); + // species_e.set_npart(offset_e + elec_ind_h()); + + // auto pos_ind_h = Kokkos::create_mirror(pos_ind); + // Kokkos::deep_copy(pos_ind_h, pos_ind); + // species_p.set_npart(offset_p + pos_ind_h()); } From 939369eb65a4aa66e6335054451d816a09ac63f6 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 09:51:22 -0500 Subject: [PATCH 038/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 70 ++++++++++++++++----------------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index b682cd621..f74e45080 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -151,53 +151,51 @@ namespace user { auto weight_p = species_p.weight; auto tag_p = species_p.tag; - int nseed = 1; + int nseed = 10; - // Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { + Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { - // // ToDo: fix this - // auto i1_ = 100; - // auto i2_ = 100; - // auto dx1_ = HALF; - // auto dx2_ = HALF; + // ToDo: fix this + auto i1_ = math::floor(100); + auto i2_ = math::floor(64); + auto dx1_ = HALF; + auto dx2_ = HALF; - // auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); - // auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); + auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); + auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); - // i1_e(elec_p + offset_e) = i1_; - // dx1_e(elec_p + offset_e) = dx1_; - // i2_e(elec_p + offset_e) = i2_; - // dx2_e(elec_p + offset_e) = dx2_; - // phi_e(elec_p + offset_e) = ZERO; - // ux1_e(elec_p + offset_e) = -drift_ux; - // ux2_e(elec_p + offset_e) = -drift_ux; - // ux3_e(elec_p + offset_e) = ZERO; - // weight_e(elec_p + offset_e) = ONE; - // tag_e(elec_p + offset_e) = ParticleTag::alive; + i1_e(elec_p + offset_e) = i1_; + dx1_e(elec_p + offset_e) = dx1_; + i2_e(elec_p + offset_e) = i2_; + dx2_e(elec_p + offset_e) = dx2_; + ux1_e(elec_p + offset_e) = ZERO; + ux2_e(elec_p + offset_e) = ZERO; + ux3_e(elec_p + offset_e) = ZERO; + weight_e(elec_p + offset_e) = ONE; + tag_e(elec_p + offset_e) = ParticleTag::alive; - // i1_p(pos_p + offset_p) = i1_; - // dx1_p(pos_p + offset_p) = dx1_; - // i2_p(pos_p + offset_p) = i2_; - // dx2_p(pos_p + offset_p) = dx2_; - // phi_p(pos_p + offset_p) = ZERO; - // ux1_p(pos_p + offset_p) = -drift_ux; - // ux2_p(pos_p + offset_p) = drift_ux; - // ux3_p(pos_p + offset_p) = ZERO; - // weight_p(pos_p + offset_p) = ONE; - // tag_p(pos_p + offset_p) = ParticleTag::alive; + i1_p(pos_p + offset_p) = i1_; + dx1_p(pos_p + offset_p) = dx1_; + i2_p(pos_p + offset_p) = i2_; + dx2_p(pos_p + offset_p) = dx2_; + ux1_p(pos_p + offset_p) = ZERO; + ux2_p(pos_p + offset_p) = ZERO; + ux3_p(pos_p + offset_p) = ZERO; + weight_p(pos_p + offset_p) = ONE; + tag_p(pos_p + offset_p) = ParticleTag::alive; - // }); + }); - // auto elec_ind_h = Kokkos::create_mirror(elec_ind); - // Kokkos::deep_copy(elec_ind_h, elec_ind); - // species_e.set_npart(offset_e + elec_ind_h()); + auto elec_ind_h = Kokkos::create_mirror(elec_ind); + Kokkos::deep_copy(elec_ind_h, elec_ind); + species_e.set_npart(offset_e + elec_ind_h()); - // auto pos_ind_h = Kokkos::create_mirror(pos_ind); - // Kokkos::deep_copy(pos_ind_h, pos_ind); - // species_p.set_npart(offset_p + pos_ind_h()); + auto pos_ind_h = Kokkos::create_mirror(pos_ind); + Kokkos::deep_copy(pos_ind_h, pos_ind); + species_p.set_npart(offset_p + pos_ind_h()); } From fa5a38fe127df07335793ea0433e9e4e95d9ae76 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 09:56:29 -0500 Subject: [PATCH 039/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index f74e45080..a835b7990 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -151,7 +151,7 @@ namespace user { auto weight_p = species_p.weight; auto tag_p = species_p.tag; - int nseed = 10; + int nseed = 1; Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { @@ -169,8 +169,8 @@ namespace user { dx1_e(elec_p + offset_e) = dx1_; i2_e(elec_p + offset_e) = i2_; dx2_e(elec_p + offset_e) = dx2_; - ux1_e(elec_p + offset_e) = ZERO; - ux2_e(elec_p + offset_e) = ZERO; + ux1_e(elec_p + offset_e) = -drift_ux; + ux2_e(elec_p + offset_e) = drift_ux; ux3_e(elec_p + offset_e) = ZERO; weight_e(elec_p + offset_e) = ONE; tag_e(elec_p + offset_e) = ParticleTag::alive; @@ -179,8 +179,8 @@ namespace user { dx1_p(pos_p + offset_p) = dx1_; i2_p(pos_p + offset_p) = i2_; dx2_p(pos_p + offset_p) = dx2_; - ux1_p(pos_p + offset_p) = ZERO; - ux2_p(pos_p + offset_p) = ZERO; + ux1_p(pos_p + offset_p) = -drift_ux; + ux2_p(pos_p + offset_p) = drift_ux; ux3_p(pos_p + offset_p) = ZERO; weight_p(pos_p + offset_p) = ONE; tag_p(pos_p + offset_p) = ParticleTag::alive; From eeca02d46f7d63575fa923b23b16a003d2fac936 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 09:58:00 -0500 Subject: [PATCH 040/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index a835b7990..9711646e3 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -169,8 +169,8 @@ namespace user { dx1_e(elec_p + offset_e) = dx1_; i2_e(elec_p + offset_e) = i2_; dx2_e(elec_p + offset_e) = dx2_; - ux1_e(elec_p + offset_e) = -drift_ux; - ux2_e(elec_p + offset_e) = drift_ux; + ux1_e(elec_p + offset_e) = -0.5; + ux2_e(elec_p + offset_e) = 0.5; ux3_e(elec_p + offset_e) = ZERO; weight_e(elec_p + offset_e) = ONE; tag_e(elec_p + offset_e) = ParticleTag::alive; @@ -179,8 +179,8 @@ namespace user { dx1_p(pos_p + offset_p) = dx1_; i2_p(pos_p + offset_p) = i2_; dx2_p(pos_p + offset_p) = dx2_; - ux1_p(pos_p + offset_p) = -drift_ux; - ux2_p(pos_p + offset_p) = drift_ux; + ux1_p(pos_p + offset_p) = -0.5; + ux2_p(pos_p + offset_p) = 0.5; ux3_p(pos_p + offset_p) = ZERO; weight_p(pos_p + offset_p) = ONE; tag_p(pos_p + offset_p) = ParticleTag::alive; From c18840ca3a1f4ccab2992c320ff1f2a0d61fce03 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 11:05:00 -0500 Subject: [PATCH 041/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 9711646e3..4cf89ac69 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -156,7 +156,7 @@ namespace user { Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { // ToDo: fix this - auto i1_ = math::floor(100); + auto i1_ = math::floor(10); auto i2_ = math::floor(64); auto dx1_ = HALF; auto dx2_ = HALF; From 9085afd497f0535126f75ecf353d289c6a248682 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 11:18:31 -0500 Subject: [PATCH 042/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 4cf89ac69..a1ce9a1a9 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -179,8 +179,8 @@ namespace user { dx1_p(pos_p + offset_p) = dx1_; i2_p(pos_p + offset_p) = i2_; dx2_p(pos_p + offset_p) = dx2_; - ux1_p(pos_p + offset_p) = -0.5; - ux2_p(pos_p + offset_p) = 0.5; + ux1_p(pos_p + offset_p) = 0.5; + ux2_p(pos_p + offset_p) = -0.5; ux3_p(pos_p + offset_p) = ZERO; weight_p(pos_p + offset_p) = ONE; tag_p(pos_p + offset_p) = ParticleTag::alive; From 87b37beb08e476081bcc4e601f16d2281549d62c Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 13:20:21 -0500 Subject: [PATCH 043/176] Ongoing. --- src/kernels/fields_bcs.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 39dfaf5dc..e477c4eb1 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -537,13 +537,13 @@ namespace kernel::bc { // SRPIC if (tags & BC::E) { Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); - Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+1+i1, i2, em::ex2); - Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+1+i1, i2, em::ex3); + Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+i1, i2, em::ex2); + Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+i1, i2, em::ex3); } if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+1+i1, i2, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); } From 5712e3bd4bb628fa9df2480e2e31e9a0f885995c Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 13:34:59 -0500 Subject: [PATCH 044/176] Ongoing. --- src/kernels/fields_bcs.hpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index e477c4eb1..115aa3415 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -536,16 +536,19 @@ namespace kernel::bc { if constexpr (S == SimEngine::SRPIC) { // SRPIC if (tags & BC::E) { - Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); - Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+i1, i2, em::ex2); - Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+i1, i2, em::ex3); + // Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); + // Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+1+i1, i2, em::ex2); + // Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+1+i1, i2, em::ex3); + Fld((N_GHOSTS-1)-i1, i2, em::ex1) = ZERO; + Fld((N_GHOSTS-1)-i1, i2, em::ex2) = ZERO; + Fld((N_GHOSTS-1)-i1, i2, em::ex3) = ZERO; } if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = ZERO; + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = ZERO; + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = ZERO; } } else { // GRPIC From af258c6c91d800977523f6d26ad25c423be84b5b Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 13:36:55 -0500 Subject: [PATCH 045/176] Ongoing. --- src/kernels/fields_bcs.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 115aa3415..b15a16546 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -542,6 +542,8 @@ namespace kernel::bc { Fld((N_GHOSTS-1)-i1, i2, em::ex1) = ZERO; Fld((N_GHOSTS-1)-i1, i2, em::ex2) = ZERO; Fld((N_GHOSTS-1)-i1, i2, em::ex3) = ZERO; + Fld((N_GHOSTS), i2, em::ex2) = ZERO; + Fld((N_GHOSTS), i2, em::ex3) = ZERO; } if (tags & BC::B) From 86f05972d2623612b13e9d51b814b0b715e5f566 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 14:53:48 -0500 Subject: [PATCH 046/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 156 ++++++++++++++++---------------- 1 file changed, 78 insertions(+), 78 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index a1ce9a1a9..1b8926c67 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -48,8 +48,8 @@ namespace user { Inline auto bx3(const coord_t& x_ph) const -> real_t { // return Bmag * math::sin(Btheta) * math::cos(Bphi); - // return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); - return ZERO; + return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + // return ZERO; } // electric field components @@ -59,8 +59,8 @@ namespace user { Inline auto ex2(const coord_t& x_ph) const -> real_t { // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - // return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); - return ZERO; + return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + // return ZERO; } Inline auto ex3(const coord_t& x_ph) const -> real_t { @@ -120,82 +120,82 @@ namespace user { inline void InitPrtls(Domain& domain) { - auto& species_e = domain.species[0]; - auto& species_p = domain.species[1]; + // auto& species_e = domain.species[0]; + // auto& species_p = domain.species[1]; - array_t elec_ind("elec_ind"); - array_t pos_ind("pos_ind"); + // array_t elec_ind("elec_ind"); + // array_t pos_ind("pos_ind"); - auto offset_e = species_e.npart(); - auto offset_p = species_p.npart(); - - auto ux1_e = species_e.ux1; - auto ux2_e = species_e.ux2; - auto ux3_e = species_e.ux3; - auto i1_e = species_e.i1; - auto i2_e = species_e.i2; - auto dx1_e = species_e.dx1; - auto dx2_e = species_e.dx2; - auto phi_e = species_e.phi; - auto weight_e = species_e.weight; - auto tag_e = species_e.tag; - - auto ux1_p = species_p.ux1; - auto ux2_p = species_p.ux2; - auto ux3_p = species_p.ux3; - auto i1_p = species_p.i1; - auto i2_p = species_p.i2; - auto dx1_p = species_p.dx1; - auto dx2_p = species_p.dx2; - auto phi_p = species_p.phi; - auto weight_p = species_p.weight; - auto tag_p = species_p.tag; - - int nseed = 1; - - Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { - - // ToDo: fix this - auto i1_ = math::floor(10); - auto i2_ = math::floor(64); - auto dx1_ = HALF; - auto dx2_ = HALF; - - - auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); - auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); - - i1_e(elec_p + offset_e) = i1_; - dx1_e(elec_p + offset_e) = dx1_; - i2_e(elec_p + offset_e) = i2_; - dx2_e(elec_p + offset_e) = dx2_; - ux1_e(elec_p + offset_e) = -0.5; - ux2_e(elec_p + offset_e) = 0.5; - ux3_e(elec_p + offset_e) = ZERO; - weight_e(elec_p + offset_e) = ONE; - tag_e(elec_p + offset_e) = ParticleTag::alive; - - i1_p(pos_p + offset_p) = i1_; - dx1_p(pos_p + offset_p) = dx1_; - i2_p(pos_p + offset_p) = i2_; - dx2_p(pos_p + offset_p) = dx2_; - ux1_p(pos_p + offset_p) = 0.5; - ux2_p(pos_p + offset_p) = -0.5; - ux3_p(pos_p + offset_p) = ZERO; - weight_p(pos_p + offset_p) = ONE; - tag_p(pos_p + offset_p) = ParticleTag::alive; - - - }); - - - auto elec_ind_h = Kokkos::create_mirror(elec_ind); - Kokkos::deep_copy(elec_ind_h, elec_ind); - species_e.set_npart(offset_e + elec_ind_h()); - - auto pos_ind_h = Kokkos::create_mirror(pos_ind); - Kokkos::deep_copy(pos_ind_h, pos_ind); - species_p.set_npart(offset_p + pos_ind_h()); + // auto offset_e = species_e.npart(); + // auto offset_p = species_p.npart(); + + // auto ux1_e = species_e.ux1; + // auto ux2_e = species_e.ux2; + // auto ux3_e = species_e.ux3; + // auto i1_e = species_e.i1; + // auto i2_e = species_e.i2; + // auto dx1_e = species_e.dx1; + // auto dx2_e = species_e.dx2; + // auto phi_e = species_e.phi; + // auto weight_e = species_e.weight; + // auto tag_e = species_e.tag; + + // auto ux1_p = species_p.ux1; + // auto ux2_p = species_p.ux2; + // auto ux3_p = species_p.ux3; + // auto i1_p = species_p.i1; + // auto i2_p = species_p.i2; + // auto dx1_p = species_p.dx1; + // auto dx2_p = species_p.dx2; + // auto phi_p = species_p.phi; + // auto weight_p = species_p.weight; + // auto tag_p = species_p.tag; + + // int nseed = 1; + + // Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { + + // // ToDo: fix this + // auto i1_ = math::floor(10); + // auto i2_ = math::floor(64); + // auto dx1_ = HALF; + // auto dx2_ = HALF; + + + // auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); + // auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); + + // i1_e(elec_p + offset_e) = i1_; + // dx1_e(elec_p + offset_e) = dx1_; + // i2_e(elec_p + offset_e) = i2_; + // dx2_e(elec_p + offset_e) = dx2_; + // ux1_e(elec_p + offset_e) = -0.5; + // ux2_e(elec_p + offset_e) = 0.5; + // ux3_e(elec_p + offset_e) = ZERO; + // weight_e(elec_p + offset_e) = ONE; + // tag_e(elec_p + offset_e) = ParticleTag::alive; + + // i1_p(pos_p + offset_p) = i1_; + // dx1_p(pos_p + offset_p) = dx1_; + // i2_p(pos_p + offset_p) = i2_; + // dx2_p(pos_p + offset_p) = dx2_; + // ux1_p(pos_p + offset_p) = 0.5; + // ux2_p(pos_p + offset_p) = -0.5; + // ux3_p(pos_p + offset_p) = ZERO; + // weight_p(pos_p + offset_p) = ONE; + // tag_p(pos_p + offset_p) = ParticleTag::alive; + + + // }); + + + // auto elec_ind_h = Kokkos::create_mirror(elec_ind); + // Kokkos::deep_copy(elec_ind_h, elec_ind); + // species_e.set_npart(offset_e + elec_ind_h()); + + // auto pos_ind_h = Kokkos::create_mirror(pos_ind); + // Kokkos::deep_copy(pos_ind_h, pos_ind); + // species_p.set_npart(offset_p + pos_ind_h()); } From d0ce7c8e64caab5fe200a85b4479dd07d21b5657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 3 Mar 2025 13:11:03 -0600 Subject: [PATCH 047/176] cleanup of conductor BCs --- src/kernels/fields_bcs.hpp | 101 ++++++++++++++++++++----------------- 1 file changed, 56 insertions(+), 45 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index b15a16546..720408d05 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -506,19 +506,19 @@ namespace kernel::bc { if constexpr (S == SimEngine::SRPIC) { - if (tags & BC::E) - { - Fld((N_GHOSTS - 1) - i1, em::ex1) = Fld(N_GHOSTS + i1, em::ex1); - Fld((N_GHOSTS - 1) - i1, em::ex2) = -Fld(N_GHOSTS + 1 + i1, em::ex2); - Fld((N_GHOSTS - 1) - i1, em::ex3) = -Fld(N_GHOSTS + 1 + i1, em::ex3); - } + if (tags & BC::E) { + Fld((N_GHOSTS-1)-i1, em::ex1) = Fld(N_GHOSTS+i1, em::ex1); + Fld((N_GHOSTS-1)-i1, em::ex2) = -Fld(N_GHOSTS+i1+1, em::ex2); + Fld((N_GHOSTS-1)-i1, em::ex3) = -Fld(N_GHOSTS+i1+1, em::ex3); + } + + if (tags & BC::B) + { + Fld((N_GHOSTS-1)-i1, em::bx1) = -Fld(N_GHOSTS+i1+1, em::bx1); + Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); + Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); + } - if (tags & BC::B) - { - Fld((N_GHOSTS - 1) - i1, em::bx1) = -Fld(N_GHOSTS + 1 + i1, em::bx1); - Fld((N_GHOSTS - 1) - i1, em::bx2) = Fld(N_GHOSTS + i1, em::bx2); - Fld((N_GHOSTS - 1) - i1, em::bx3) = Fld(N_GHOSTS + i1, em::bx3); - } } else { // GRPIC raise::KernelError(HERE, "1D GRPIC not implemented"); @@ -530,64 +530,75 @@ namespace kernel::bc { } } - Inline void operator()(index_t i1, index_t i2) const { - if constexpr (M::Dim == Dim::_2D) { + Inline void operator()(index_t i1, index_t i2) const + { + if constexpr (M::Dim == Dim::_2D) + { - if constexpr (S == SimEngine::SRPIC) { + if constexpr (S == SimEngine::SRPIC) + { // SRPIC - if (tags & BC::E) { - // Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); - // Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+1+i1, i2, em::ex2); - // Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+1+i1, i2, em::ex3); - Fld((N_GHOSTS-1)-i1, i2, em::ex1) = ZERO; - Fld((N_GHOSTS-1)-i1, i2, em::ex2) = ZERO; - Fld((N_GHOSTS-1)-i1, i2, em::ex3) = ZERO; - Fld((N_GHOSTS), i2, em::ex2) = ZERO; - Fld((N_GHOSTS), i2, em::ex3) = ZERO; + if (tags & BC::E) + { + Fld((N_GHOSTS - 1) - i1, i2, em::ex1) = Fld(N_GHOSTS + i1, i2, em::ex1); + Fld((N_GHOSTS - 1) - i1, i2, em::ex2) = -Fld(N_GHOSTS + 1 + i1, i2, em::ex2); + Fld((N_GHOSTS - 1) - i1, i2, em::ex3) = -Fld(N_GHOSTS + 1 + i1, i2, em::ex3); } if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = ZERO; - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = ZERO; - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = ZERO; + Fld((N_GHOSTS - 1) - i1, i2, em::bx1) = -Fld(N_GHOSTS + 1 + i1, i2, em::bx1); + Fld((N_GHOSTS - 1) - i1, i2, em::bx2) = Fld(N_GHOSTS + i1, i2, em::bx2); + Fld((N_GHOSTS - 1) - i1, i2, em::bx3) = Fld(N_GHOSTS + i1, i2, em::bx3); } - } else { + } + else + { // GRPIC raise::KernelError(HERE, "2D GRPIC not implemented"); } - } else { + } + else + { raise::KernelError( - HERE, - "ConductorBoundaries_kernel: 2D implementation called for D != 2"); + HERE, + "ConductorBoundaries_kernel: 2D implementation called for D != 2"); } } - Inline void operator()(index_t i1, index_t i2, index_t i3) const { - if constexpr (M::Dim == Dim::_3D) { + Inline void operator()(index_t i1, index_t i2, index_t i3) const + { + if constexpr (M::Dim == Dim::_3D) + { - if constexpr (S == SimEngine::SRPIC) { + if constexpr (S == SimEngine::SRPIC) + { // SRPIC - if (tags & BC::E) { - Fld((N_GHOSTS-1)-i1, i2, i3, em::ex1) = Fld(N_GHOSTS+i1, i2, i3, em::ex1); - Fld((N_GHOSTS-1)-i1, i2, i3, em::ex2) = -Fld(N_GHOSTS+i1, i2, i3, em::ex2); - Fld((N_GHOSTS-1)-i1, i2, i3, em::ex3) = -Fld(N_GHOSTS+i1, i2, i3, em::ex3); + if (tags & BC::E) + { + Fld((N_GHOSTS - 1) - i1, i2, i3, em::ex1) = Fld(N_GHOSTS + i1, i2, i3, em::ex1); + Fld((N_GHOSTS - 1) - i1, i2, i3, em::ex2) = -Fld(N_GHOSTS + i1 + 1, i2, i3, em::ex2); + Fld((N_GHOSTS - 1) - i1, i2, i3, em::ex3) = -Fld(N_GHOSTS + i1 + 1, i2, i3, em::ex3); } if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, i3, em::bx1) = -Fld(N_GHOSTS+i1, i2, i3, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, i3, em::bx2) = Fld(N_GHOSTS+i1, i2, i3, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, i3, em::bx3) = Fld(N_GHOSTS+i1, i2, i3, em::bx3); + Fld((N_GHOSTS - 1) - i1, i2, i3, em::bx1) = -Fld(N_GHOSTS + i1 + 1, i2, i3, em::bx1); + Fld((N_GHOSTS - 1) - i1, i2, i3, em::bx2) = Fld(N_GHOSTS + i1, i2, i3, em::bx2); + Fld((N_GHOSTS - 1) - i1, i2, i3, em::bx3) = Fld(N_GHOSTS + i1, i2, i3, em::bx3); } - } else { + } + else + { // GRPIC raise::KernelError(HERE, "3D GRPIC not implemented"); } - } else { + } + else + { raise::KernelError( - HERE, - "ConductorBoundaries_kernel: 3D implementation called for D != 3"); + HERE, + "ConductorBoundaries_kernel: 3D implementation called for D != 3"); } } }; From 0fd05b9ea515024beb64ed7ddf9b92d941d09949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 3 Mar 2025 13:11:50 -0600 Subject: [PATCH 048/176] add 3D case and set correct boundary cells to zero --- src/engines/srpic.hpp | 963 ++++++++++++++++++++++++------------------ 1 file changed, 547 insertions(+), 416 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 90aa271a3..1c7ffa869 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -871,225 +871,321 @@ namespace ntt { "Invalid range size", HERE); + if constexpr (M::Dim == Dim::_1D) { - Kokkos::parallel_for( - "MatchFields", - CreateRangePolicy( { xi_min[0] } , { xi_max[0] } ), - kernel::bc::ConductorBoundaries_kernel( - domain.fields.em, - tags)); + Kokkos::parallel_for( + "MatchFields", + CreateRangePolicy({xi_min[0]}, {xi_max[0]}), + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); } if constexpr (M::Dim == Dim::_2D) { - Kokkos::parallel_for( - "MatchFields", - CreateRangePolicy( { xi_min[0], xi_min[1] } , { xi_max[0], xi_max[1] } ), - kernel::bc::ConductorBoundaries_kernel( - domain.fields.em, - tags)); + Kokkos::parallel_for( + "MatchFields", + CreateRangePolicy({xi_min[0], xi_min[1]}, {xi_max[0], xi_max[1]}), + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); } - - - } - void AtmosphereFieldsIn(dir::direction_t direction, - domain_t& domain, - BCTags tags) { - /** - * atmosphere field boundaries - */ - if constexpr (traits::has_member::value) { - const auto [sign, dim, xg_min, xg_max] = get_atm_extent(direction); - const auto dd = static_cast(dim); - boundaries_t box; - boundaries_t incl_ghosts; - for (unsigned short d { 0 }; d < M::Dim; ++d) { - if (d == dd) { - box.push_back({ xg_min, xg_max }); - if (sign > 0) { - incl_ghosts.push_back({ false, true }); - } else { - incl_ghosts.push_back({ true, false }); - } - } else { - box.push_back(Range::All); - incl_ghosts.push_back({ true, true }); - } - } - if (not domain.mesh.Intersects(box)) { - return; - } - const auto intersect_range = domain.mesh.ExtentToRange(box, incl_ghosts); - tuple_t range_min { 0 }; - tuple_t range_max { 0 }; + if constexpr (M::Dim == Dim::_3D) + { + Kokkos::parallel_for( + "MatchFields", + CreateRangePolicy({xi_min[0], xi_min[1], xi_min[2]}, {xi_max[0], xi_max[1], xi_max[2]}), + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } - for (unsigned short d { 0 }; d < M::Dim; ++d) { - range_min[d] = intersect_range[d].first; - range_max[d] = intersect_range[d].second; + // these components need to be set to zero at the boundary + std::vector comps = {em::ex2, em::ex3, em::bx1}; + for (const auto &comp : comps) + { + if constexpr (M::Dim == Dim::_1D) + { + Kokkos::deep_copy( + Kokkos::subview(domain.fields.em, + std::make_pair(N_GHOSTS, N_GHOSTS), + comp), + ZERO); } - auto atm_fields = m_pgen.AtmFields(time); - std::size_t il_edge; - if (sign > 0) { - il_edge = range_min[dd] - N_GHOSTS; - } else { - il_edge = range_max[dd] - 1 - N_GHOSTS; + else if constexpr (M::Dim == Dim::_2D) + { + Kokkos::deep_copy( + Kokkos::subview(domain.fields.em, + std::make_pair(N_GHOSTS, N_GHOSTS), + std::make_pair(xi_min[1], xi_max[1]), + comp), + ZERO); } - const auto range = CreateRangePolicy(range_min, range_max); - if (dim == in::x1) { - if (sign > 0) { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } else { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } - } else if (dim == in::x2) { - if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { - if (sign > 0) { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } else { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } - } else { - raise::Error("Invalid dimension", HERE); - } - } else if (dim == in::x3) { - if constexpr (M::Dim == Dim::_3D) { - if (sign > 0) { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } else { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } - } else { - raise::Error("Invalid dimension", HERE); - } - } else { + else if constexpr (M::Dim == Dim::_3D) + { + Kokkos::deep_copy( + Kokkos::subview(domain.fields.em, + std::make_pair(N_GHOSTS, N_GHOSTS), + std::make_pair(xi_min[1], xi_max[1]), + std::make_pair(xi_min[2], xi_max[2]), + comp), + ZERO); + } + else + { raise::Error("Invalid dimension", HERE); } - } else { - raise::Error("Atm fields not implemented in PGEN for atmosphere BCs", HERE); } } - void CustomFieldsIn(dir::direction_t direction, - domain_t& domain, - BCTags tags) { - (void)direction; - (void)domain; - (void)tags; - raise::Error("Custom boundaries not implemented", HERE); - // if constexpr ( - // traits::has_member::value) { - // const auto [box, custom_fields] = m_pgen.CustomFields(time); - // if (domain.mesh.Intersects(box)) { - // } - // - // } else { - // raise::Error("Custom boundaries not implemented", HERE); - // } - } + void AtmosphereFieldsIn(dir::direction_t direction, + domain_t & domain, + BCTags tags) + { + /** + * atmosphere field boundaries + */ + if constexpr (traits::has_member::value) + { + const auto [sign, dim, xg_min, xg_max] = get_atm_extent(direction); + const auto dd = static_cast(dim); + boundaries_t box; + boundaries_t incl_ghosts; + for (unsigned short d{0}; d < M::Dim; ++d) + { + if (d == dd) + { + box.push_back({xg_min, xg_max}); + if (sign > 0) + { + incl_ghosts.push_back({false, true}); + } + else + { + incl_ghosts.push_back({true, false}); + } + } + else + { + box.push_back(Range::All); + incl_ghosts.push_back({true, true}); + } + } + if (not domain.mesh.Intersects(box)) + { + return; + } + const auto intersect_range = domain.mesh.ExtentToRange(box, incl_ghosts); + tuple_t range_min{0}; + tuple_t range_max{0}; + + for (unsigned short d{0}; d < M::Dim; ++d) + { + range_min[d] = intersect_range[d].first; + range_max[d] = intersect_range[d].second; + } + auto atm_fields = m_pgen.AtmFields(time); + std::size_t il_edge; + if (sign > 0) + { + il_edge = range_min[dd] - N_GHOSTS; + } + else + { + il_edge = range_max[dd] - 1 - N_GHOSTS; + } + const auto range = CreateRangePolicy(range_min, range_max); + if (dim == in::x1) + { + if (sign > 0) + { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } + else + { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } + } + else if (dim == in::x2) + { + if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) + { + if (sign > 0) + { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } + else + { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } + } + else + { + raise::Error("Invalid dimension", HERE); + } + } + else if (dim == in::x3) + { + if constexpr (M::Dim == Dim::_3D) + { + if (sign > 0) + { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } + else + { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } + } + else + { + raise::Error("Invalid dimension", HERE); + } + } + else + { + raise::Error("Invalid dimension", HERE); + } + } + else + { + raise::Error("Atm fields not implemented in PGEN for atmosphere BCs", HERE); + } + } - void AtmosphereParticlesIn(const dir::direction_t& direction, - domain_t& domain, - InjTags tags) { - const auto [sign, dim, xg_min, xg_max] = get_atm_extent(direction); - - const auto x_surf = sign > 0 ? xg_min : xg_max; - const auto ds = m_params.template get( - "grid.boundaries.atmosphere.ds"); - const auto temp = m_params.template get( - "grid.boundaries.atmosphere.temperature"); - const auto height = m_params.template get( - "grid.boundaries.atmosphere.height"); - const auto species = - m_params.template get>( - "grid.boundaries.atmosphere.species"); - const auto nmax = m_params.template get( - "grid.boundaries.atmosphere.density"); - - Kokkos::deep_copy(domain.fields.bckp, ZERO); - auto scatter_bckp = Kokkos::Experimental::create_scatter_view( - domain.fields.bckp); - const auto use_weights = M::CoordType != Coord::Cart; - const auto ni2 = domain.mesh.n_active(in::x2); - const auto inv_n0 = ONE / m_params.template get("scales.n0"); - - // compute the density of the two species - if (tags & Inj::AssumeEmpty) { - if constexpr (M::Dim == Dim::_1D) { - Kokkos::deep_copy( - Kokkos::subview(domain.fields.bckp, Kokkos::ALL, std::make_pair(0, 1)), - ZERO); - } else if constexpr (M::Dim == Dim::_2D) { - Kokkos::deep_copy(Kokkos::subview(domain.fields.bckp, - Kokkos::ALL, - Kokkos::ALL, - std::make_pair(0, 1)), - ZERO); - } else if constexpr (M::Dim == Dim::_3D) { - Kokkos::deep_copy(Kokkos::subview(domain.fields.bckp, - Kokkos::ALL, - Kokkos::ALL, - Kokkos::ALL, - std::make_pair(0, 1)), - ZERO); + void CustomFieldsIn(dir::direction_t direction, + domain_t & domain, + BCTags tags) + { + (void)direction; + (void)domain; + (void)tags; + raise::Error("Custom boundaries not implemented", HERE); + // if constexpr ( + // traits::has_member::value) { + // const auto [box, custom_fields] = m_pgen.CustomFields(time); + // if (domain.mesh.Intersects(box)) { + // } + // + // } else { + // raise::Error("Custom boundaries not implemented", HERE); + // } } - } else { - for (const auto& sp : - std::vector({ species.first, species.second })) { - auto& prtl_spec = domain.species[sp - 1]; - if (prtl_spec.npart() == 0) { - continue; + + void AtmosphereParticlesIn(const dir::direction_t &direction, + domain_t &domain, + InjTags tags) + { + const auto [sign, dim, xg_min, xg_max] = get_atm_extent(direction); + + const auto x_surf = sign > 0 ? xg_min : xg_max; + const auto ds = m_params.template get( + "grid.boundaries.atmosphere.ds"); + const auto temp = m_params.template get( + "grid.boundaries.atmosphere.temperature"); + const auto height = m_params.template get( + "grid.boundaries.atmosphere.height"); + const auto species = + m_params.template get>( + "grid.boundaries.atmosphere.species"); + const auto nmax = m_params.template get( + "grid.boundaries.atmosphere.density"); + + Kokkos::deep_copy(domain.fields.bckp, ZERO); + auto scatter_bckp = Kokkos::Experimental::create_scatter_view( + domain.fields.bckp); + const auto use_weights = M::CoordType != Coord::Cart; + const auto ni2 = domain.mesh.n_active(in::x2); + const auto inv_n0 = ONE / m_params.template get("scales.n0"); + + // compute the density of the two species + if (tags & Inj::AssumeEmpty) + { + if constexpr (M::Dim == Dim::_1D) + { + Kokkos::deep_copy( + Kokkos::subview(domain.fields.bckp, Kokkos::ALL, std::make_pair(0, 1)), + ZERO); + } + else if constexpr (M::Dim == Dim::_2D) + { + Kokkos::deep_copy(Kokkos::subview(domain.fields.bckp, + Kokkos::ALL, + Kokkos::ALL, + std::make_pair(0, 1)), + ZERO); + } + else if constexpr (M::Dim == Dim::_3D) + { + Kokkos::deep_copy(Kokkos::subview(domain.fields.bckp, + Kokkos::ALL, + Kokkos::ALL, + Kokkos::ALL, + std::make_pair(0, 1)), + ZERO); + } } - // clang-format off + else + { + for (const auto &sp : + std::vector({species.first, species.second})) + { + auto &prtl_spec = domain.species[sp - 1]; + if (prtl_spec.npart() == 0) + { + continue; + } + // clang-format off Kokkos::parallel_for( "ComputeMoments", prtl_spec.rangeActiveParticles(), @@ -1103,230 +1199,265 @@ namespace ntt { use_weights, domain.mesh.metric, domain.mesh.flds_bc(), ni2, inv_n0, 0)); - // clang-format on - prtl_spec.set_unsorted(); - } - Kokkos::Experimental::contribute(domain.fields.bckp, scatter_bckp); - m_metadomain.SynchronizeFields(domain, Comm::Bckp, { 0, 1 }); - } - - if (dim == in::x1) { - if (sign > 0) { - const auto atm_injector = - arch::AtmosphereInjector { - domain.mesh.metric, - domain.fields.bckp, - nmax, - height, - x_surf, - ds, - temp, - domain.random_pool, - species - }; - arch::InjectNonUniform(m_params, - domain, - atm_injector, - nmax, - use_weights); - } else { - const auto atm_injector = - arch::AtmosphereInjector { - domain.mesh.metric, - domain.fields.bckp, - nmax, - height, - x_surf, - ds, - temp, - domain.random_pool, - species - }; - arch::InjectNonUniform(m_params, - domain, - atm_injector, - nmax, - use_weights); - } - } else if (dim == in::x2) { - if (sign > 0) { - const auto atm_injector = - arch::AtmosphereInjector { - domain.mesh.metric, - domain.fields.bckp, - nmax, - height, - x_surf, - ds, - temp, - domain.random_pool, - species - }; - arch::InjectNonUniform(m_params, - domain, - atm_injector, - nmax, - use_weights); - } else { - const auto atm_injector = - arch::AtmosphereInjector { - domain.mesh.metric, - domain.fields.bckp, - nmax, - height, - x_surf, - ds, - temp, - domain.random_pool, - species - }; - arch::InjectNonUniform(m_params, - domain, - atm_injector, - nmax, - use_weights); - } - } else if (dim == in::x3) { - if (sign > 0) { - const auto atm_injector = - arch::AtmosphereInjector { - domain.mesh.metric, - domain.fields.bckp, - nmax, - height, - x_surf, - ds, - temp, - domain.random_pool, - species - }; - arch::InjectNonUniform(m_params, - domain, - atm_injector, - nmax, - use_weights); - } else { - const auto atm_injector = - arch::AtmosphereInjector { - domain.mesh.metric, - domain.fields.bckp, - nmax, - height, - x_surf, - ds, - temp, - domain.random_pool, - species - }; - arch::InjectNonUniform(m_params, - domain, - atm_injector, - nmax, - use_weights); - } - } else { - raise::Error("Invalid dimension", HERE); - } - return; - } - - private: - /** - * @brief Get the buffer region of the atmosphere and the direction - * @param direction direction in which the atmosphere is applied - * @return tuple: [sign of the direction, the direction (as in::), the min and max extent - * @note xg_min and xg_max are the extents where the fields are set, not the atmosphere itself - * @note i.e. - * - * fields set particles injected - * ghost zone | | - * v v v - * |....|...........|*******************..... -> x1 - * ^ ^ - * xg_min xg_max - * | | | - * |<-- buffer -->|<-- atmosphere -->| - * - * in this case the function returns { -1, in::x1, xg_min, xg_max } - */ - auto get_atm_extent(dir::direction_t direction) const - -> std::tuple { - const auto sign = direction.get_sign(); - const auto dim = direction.get_dim(); - const auto min_buff = m_params.template get( - "algorithms.current_filters") + - 2; - const auto buffer_ncells = min_buff > 5 ? min_buff : 5; - if (M::CoordType != Coord::Cart and (dim != in::x1 or sign > 0)) { - raise::Error("For non-cartesian coordinates atmosphere BCs is " - "possible only in -x1 (@ rmin)", - HERE); - } - real_t xg_min { ZERO }, xg_max { ZERO }; - std::size_t ig_min, ig_max; - if (sign > 0) { // + direction - ig_min = m_metadomain.mesh().n_active(dim) - buffer_ncells; - ig_max = m_metadomain.mesh().n_active(dim); - } else { // - direction - ig_min = 0; - ig_max = buffer_ncells; - } + // clang-format on + prtl_spec.set_unsorted(); + } + Kokkos::Experimental::contribute(domain.fields.bckp, scatter_bckp); + m_metadomain.SynchronizeFields(domain, Comm::Bckp, {0, 1}); + } - if (dim == in::x1) { - xg_min = m_metadomain.mesh().metric.template convert<1, Crd::Cd, Crd::Ph>( - static_cast(ig_min)); - xg_max = m_metadomain.mesh().metric.template convert<1, Crd::Cd, Crd::Ph>( - static_cast(ig_max)); - } else if (dim == in::x2) { - if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { - xg_min = m_metadomain.mesh().metric.template convert<2, Crd::Cd, Crd::Ph>( - static_cast(ig_min)); - xg_max = m_metadomain.mesh().metric.template convert<2, Crd::Cd, Crd::Ph>( - static_cast(ig_max)); - } else { - raise::Error("Invalid dimension", HERE); - } - } else if (dim == in::x3) { - if constexpr (M::Dim == Dim::_3D) { - xg_min = m_metadomain.mesh().metric.template convert<3, Crd::Cd, Crd::Ph>( - static_cast(ig_min)); - xg_max = m_metadomain.mesh().metric.template convert<3, Crd::Cd, Crd::Ph>( - static_cast(ig_max)); - } else { - raise::Error("Invalid dimension", HERE); + if (dim == in::x1) + { + if (sign > 0) + { + const auto atm_injector = + arch::AtmosphereInjector{ + domain.mesh.metric, + domain.fields.bckp, + nmax, + height, + x_surf, + ds, + temp, + domain.random_pool, + species}; + arch::InjectNonUniform(m_params, + domain, + atm_injector, + nmax, + use_weights); + } + else + { + const auto atm_injector = + arch::AtmosphereInjector{ + domain.mesh.metric, + domain.fields.bckp, + nmax, + height, + x_surf, + ds, + temp, + domain.random_pool, + species}; + arch::InjectNonUniform(m_params, + domain, + atm_injector, + nmax, + use_weights); + } + } + else if (dim == in::x2) + { + if (sign > 0) + { + const auto atm_injector = + arch::AtmosphereInjector{ + domain.mesh.metric, + domain.fields.bckp, + nmax, + height, + x_surf, + ds, + temp, + domain.random_pool, + species}; + arch::InjectNonUniform(m_params, + domain, + atm_injector, + nmax, + use_weights); + } + else + { + const auto atm_injector = + arch::AtmosphereInjector{ + domain.mesh.metric, + domain.fields.bckp, + nmax, + height, + x_surf, + ds, + temp, + domain.random_pool, + species}; + arch::InjectNonUniform(m_params, + domain, + atm_injector, + nmax, + use_weights); + } + } + else if (dim == in::x3) + { + if (sign > 0) + { + const auto atm_injector = + arch::AtmosphereInjector{ + domain.mesh.metric, + domain.fields.bckp, + nmax, + height, + x_surf, + ds, + temp, + domain.random_pool, + species}; + arch::InjectNonUniform(m_params, + domain, + atm_injector, + nmax, + use_weights); + } + else + { + const auto atm_injector = + arch::AtmosphereInjector{ + domain.mesh.metric, + domain.fields.bckp, + nmax, + height, + x_surf, + ds, + temp, + domain.random_pool, + species}; + arch::InjectNonUniform(m_params, + domain, + atm_injector, + nmax, + use_weights); + } + } + else + { + raise::Error("Invalid dimension", HERE); + } + return; } - } else { - raise::Error("Invalid dimension", HERE); - } - return { sign, dim, xg_min, xg_max }; - } - auto range_with_axis_BCs(const domain_t& domain) -> range_t { - auto range = domain.mesh.rangeActiveCells(); - if constexpr (M::CoordType != Coord::Cart) { + private: /** - * @brief taking one extra cell in the x2 direction if AXIS BCs + * @brief Get the buffer region of the atmosphere and the direction + * @param direction direction in which the atmosphere is applied + * @return tuple: [sign of the direction, the direction (as in::), the min and max extent + * @note xg_min and xg_max are the extents where the fields are set, not the atmosphere itself + * @note i.e. + * + * fields set particles injected + * ghost zone | | + * v v v + * |....|...........|*******************..... -> x1 + * ^ ^ + * xg_min xg_max + * | | | + * |<-- buffer -->|<-- atmosphere -->| + * + * in this case the function returns { -1, in::x1, xg_min, xg_max } */ - if constexpr (M::Dim == Dim::_2D) { - if (domain.mesh.flds_bc_in({ 0, +1 }) == FldsBC::AXIS) { - range = CreateRangePolicy( - { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) }, - { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1 }); + auto get_atm_extent(dir::direction_t direction) const + -> std::tuple + { + const auto sign = direction.get_sign(); + const auto dim = direction.get_dim(); + const auto min_buff = m_params.template get( + "algorithms.current_filters") + + 2; + const auto buffer_ncells = min_buff > 5 ? min_buff : 5; + if (M::CoordType != Coord::Cart and (dim != in::x1 or sign > 0)) + { + raise::Error("For non-cartesian coordinates atmosphere BCs is " + "possible only in -x1 (@ rmin)", + HERE); + } + real_t xg_min{ZERO}, xg_max{ZERO}; + std::size_t ig_min, ig_max; + if (sign > 0) + { // + direction + ig_min = m_metadomain.mesh().n_active(dim) - buffer_ncells; + ig_max = m_metadomain.mesh().n_active(dim); + } + else + { // - direction + ig_min = 0; + ig_max = buffer_ncells; + } + + if (dim == in::x1) + { + xg_min = m_metadomain.mesh().metric.template convert<1, Crd::Cd, Crd::Ph>( + static_cast(ig_min)); + xg_max = m_metadomain.mesh().metric.template convert<1, Crd::Cd, Crd::Ph>( + static_cast(ig_max)); } - } else if constexpr (M::Dim == Dim::_3D) { - if (domain.mesh.flds_bc_in({ 0, +1, 0 }) == FldsBC::AXIS) { - range = CreateRangePolicy({ domain.mesh.i_min(in::x1), - domain.mesh.i_min(in::x2), - domain.mesh.i_min(in::x3) }, - { domain.mesh.i_max(in::x1), - domain.mesh.i_max(in::x2) + 1, - domain.mesh.i_max(in::x3) }); + else if (dim == in::x2) + { + if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) + { + xg_min = m_metadomain.mesh().metric.template convert<2, Crd::Cd, Crd::Ph>( + static_cast(ig_min)); + xg_max = m_metadomain.mesh().metric.template convert<2, Crd::Cd, Crd::Ph>( + static_cast(ig_max)); + } + else + { + raise::Error("Invalid dimension", HERE); + } } + else if (dim == in::x3) + { + if constexpr (M::Dim == Dim::_3D) + { + xg_min = m_metadomain.mesh().metric.template convert<3, Crd::Cd, Crd::Ph>( + static_cast(ig_min)); + xg_max = m_metadomain.mesh().metric.template convert<3, Crd::Cd, Crd::Ph>( + static_cast(ig_max)); + } + else + { + raise::Error("Invalid dimension", HERE); + } + } + else + { + raise::Error("Invalid dimension", HERE); + } + return {sign, dim, xg_min, xg_max}; } - } - return range; - } - }; + + auto range_with_axis_BCs(const domain_t &domain) -> range_t + { + auto range = domain.mesh.rangeActiveCells(); + if constexpr (M::CoordType != Coord::Cart) + { + /** + * @brief taking one extra cell in the x2 direction if AXIS BCs + */ + if constexpr (M::Dim == Dim::_2D) + { + if (domain.mesh.flds_bc_in({0, +1}) == FldsBC::AXIS) + { + range = CreateRangePolicy( + {domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2)}, + {domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1}); + } + } + else if constexpr (M::Dim == Dim::_3D) + { + if (domain.mesh.flds_bc_in({0, +1, 0}) == FldsBC::AXIS) + { + range = CreateRangePolicy({domain.mesh.i_min(in::x1), + domain.mesh.i_min(in::x2), + domain.mesh.i_min(in::x3)}, + {domain.mesh.i_max(in::x1), + domain.mesh.i_max(in::x2) + 1, + domain.mesh.i_max(in::x3)}); + } + } + } + return range; + } + }; } // namespace ntt From 425abe5fcfbb040127e531c7e9d47efd6a6a7657 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Tue, 4 Mar 2025 13:22:38 -0600 Subject: [PATCH 049/176] Ongoing --- setups/srpic/shocktest/pgen.hpp | 156 ++++++++++++++++---------------- 1 file changed, 78 insertions(+), 78 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 1b8926c67..a1ce9a1a9 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -48,8 +48,8 @@ namespace user { Inline auto bx3(const coord_t& x_ph) const -> real_t { // return Bmag * math::sin(Btheta) * math::cos(Bphi); - return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); - // return ZERO; + // return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ZERO; } // electric field components @@ -59,8 +59,8 @@ namespace user { Inline auto ex2(const coord_t& x_ph) const -> real_t { // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); - // return ZERO; + // return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ZERO; } Inline auto ex3(const coord_t& x_ph) const -> real_t { @@ -120,82 +120,82 @@ namespace user { inline void InitPrtls(Domain& domain) { - // auto& species_e = domain.species[0]; - // auto& species_p = domain.species[1]; + auto& species_e = domain.species[0]; + auto& species_p = domain.species[1]; - // array_t elec_ind("elec_ind"); - // array_t pos_ind("pos_ind"); + array_t elec_ind("elec_ind"); + array_t pos_ind("pos_ind"); - // auto offset_e = species_e.npart(); - // auto offset_p = species_p.npart(); - - // auto ux1_e = species_e.ux1; - // auto ux2_e = species_e.ux2; - // auto ux3_e = species_e.ux3; - // auto i1_e = species_e.i1; - // auto i2_e = species_e.i2; - // auto dx1_e = species_e.dx1; - // auto dx2_e = species_e.dx2; - // auto phi_e = species_e.phi; - // auto weight_e = species_e.weight; - // auto tag_e = species_e.tag; - - // auto ux1_p = species_p.ux1; - // auto ux2_p = species_p.ux2; - // auto ux3_p = species_p.ux3; - // auto i1_p = species_p.i1; - // auto i2_p = species_p.i2; - // auto dx1_p = species_p.dx1; - // auto dx2_p = species_p.dx2; - // auto phi_p = species_p.phi; - // auto weight_p = species_p.weight; - // auto tag_p = species_p.tag; - - // int nseed = 1; - - // Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { - - // // ToDo: fix this - // auto i1_ = math::floor(10); - // auto i2_ = math::floor(64); - // auto dx1_ = HALF; - // auto dx2_ = HALF; - - - // auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); - // auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); - - // i1_e(elec_p + offset_e) = i1_; - // dx1_e(elec_p + offset_e) = dx1_; - // i2_e(elec_p + offset_e) = i2_; - // dx2_e(elec_p + offset_e) = dx2_; - // ux1_e(elec_p + offset_e) = -0.5; - // ux2_e(elec_p + offset_e) = 0.5; - // ux3_e(elec_p + offset_e) = ZERO; - // weight_e(elec_p + offset_e) = ONE; - // tag_e(elec_p + offset_e) = ParticleTag::alive; - - // i1_p(pos_p + offset_p) = i1_; - // dx1_p(pos_p + offset_p) = dx1_; - // i2_p(pos_p + offset_p) = i2_; - // dx2_p(pos_p + offset_p) = dx2_; - // ux1_p(pos_p + offset_p) = 0.5; - // ux2_p(pos_p + offset_p) = -0.5; - // ux3_p(pos_p + offset_p) = ZERO; - // weight_p(pos_p + offset_p) = ONE; - // tag_p(pos_p + offset_p) = ParticleTag::alive; - - - // }); - - - // auto elec_ind_h = Kokkos::create_mirror(elec_ind); - // Kokkos::deep_copy(elec_ind_h, elec_ind); - // species_e.set_npart(offset_e + elec_ind_h()); - - // auto pos_ind_h = Kokkos::create_mirror(pos_ind); - // Kokkos::deep_copy(pos_ind_h, pos_ind); - // species_p.set_npart(offset_p + pos_ind_h()); + auto offset_e = species_e.npart(); + auto offset_p = species_p.npart(); + + auto ux1_e = species_e.ux1; + auto ux2_e = species_e.ux2; + auto ux3_e = species_e.ux3; + auto i1_e = species_e.i1; + auto i2_e = species_e.i2; + auto dx1_e = species_e.dx1; + auto dx2_e = species_e.dx2; + auto phi_e = species_e.phi; + auto weight_e = species_e.weight; + auto tag_e = species_e.tag; + + auto ux1_p = species_p.ux1; + auto ux2_p = species_p.ux2; + auto ux3_p = species_p.ux3; + auto i1_p = species_p.i1; + auto i2_p = species_p.i2; + auto dx1_p = species_p.dx1; + auto dx2_p = species_p.dx2; + auto phi_p = species_p.phi; + auto weight_p = species_p.weight; + auto tag_p = species_p.tag; + + int nseed = 1; + + Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { + + // ToDo: fix this + auto i1_ = math::floor(10); + auto i2_ = math::floor(64); + auto dx1_ = HALF; + auto dx2_ = HALF; + + + auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); + auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); + + i1_e(elec_p + offset_e) = i1_; + dx1_e(elec_p + offset_e) = dx1_; + i2_e(elec_p + offset_e) = i2_; + dx2_e(elec_p + offset_e) = dx2_; + ux1_e(elec_p + offset_e) = -0.5; + ux2_e(elec_p + offset_e) = 0.5; + ux3_e(elec_p + offset_e) = ZERO; + weight_e(elec_p + offset_e) = ONE; + tag_e(elec_p + offset_e) = ParticleTag::alive; + + i1_p(pos_p + offset_p) = i1_; + dx1_p(pos_p + offset_p) = dx1_; + i2_p(pos_p + offset_p) = i2_; + dx2_p(pos_p + offset_p) = dx2_; + ux1_p(pos_p + offset_p) = 0.5; + ux2_p(pos_p + offset_p) = -0.5; + ux3_p(pos_p + offset_p) = ZERO; + weight_p(pos_p + offset_p) = ONE; + tag_p(pos_p + offset_p) = ParticleTag::alive; + + + }); + + + auto elec_ind_h = Kokkos::create_mirror(elec_ind); + Kokkos::deep_copy(elec_ind_h, elec_ind); + species_e.set_npart(offset_e + elec_ind_h()); + + auto pos_ind_h = Kokkos::create_mirror(pos_ind); + Kokkos::deep_copy(pos_ind_h, pos_ind); + species_p.set_npart(offset_p + pos_ind_h()); } From cfee51ecac9150e9c6c43668a83ed097f98a64ff Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Tue, 4 Mar 2025 15:06:18 -0600 Subject: [PATCH 050/176] Ongoing --- src/engines/srpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 1c7ffa869..21a44373d 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -918,7 +918,7 @@ namespace ntt { { Kokkos::deep_copy( Kokkos::subview(domain.fields.em, - std::make_pair(N_GHOSTS, N_GHOSTS), + std::make_pair(N_GHOSTS+1, N_GHOSTS+1), std::make_pair(xi_min[1], xi_max[1]), comp), ZERO); From 7d317e6bf1db35098a153bc8bccd5a5f90630500 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Tue, 4 Mar 2025 15:40:21 -0600 Subject: [PATCH 051/176] Ongoing --- src/engines/srpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 21a44373d..1c7ffa869 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -918,7 +918,7 @@ namespace ntt { { Kokkos::deep_copy( Kokkos::subview(domain.fields.em, - std::make_pair(N_GHOSTS+1, N_GHOSTS+1), + std::make_pair(N_GHOSTS, N_GHOSTS), std::make_pair(xi_min[1], xi_max[1]), comp), ZERO); From 21d99374744e333e5a6400058acab22347e0ecdd Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Tue, 4 Mar 2025 15:53:24 -0600 Subject: [PATCH 052/176] Ongoing --- src/engines/srpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 1c7ffa869..12872bf4e 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -918,7 +918,7 @@ namespace ntt { { Kokkos::deep_copy( Kokkos::subview(domain.fields.em, - std::make_pair(N_GHOSTS, N_GHOSTS), + std::make_pair(N_GHOSTS, N_GHOSTS+1), std::make_pair(xi_min[1], xi_max[1]), comp), ZERO); From bd723f3905bd3356b30faf6a8380106a1a890a9e Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 7 Mar 2025 11:30:47 -0500 Subject: [PATCH 053/176] minor reformatting of conductor BC --- input.example.toml | 2 +- setups/srpic/shocktest/pgen.hpp | 149 ++-- setups/srpic/shocktest/shock.toml | 26 +- src/engines/srpic.hpp | 1044 +++++++++++++---------------- src/global/enums.h | 19 +- src/global/tests/enums.cpp | 5 +- src/kernels/fields_bcs.hpp | 157 +++-- 7 files changed, 622 insertions(+), 780 deletions(-) diff --git a/input.example.toml b/input.example.toml index 788c30685..b327df54d 100644 --- a/input.example.toml +++ b/input.example.toml @@ -90,7 +90,7 @@ # Boundary conditions for fields: # @required # @type: 1/2/3-size array of string tuples, each of size 1 or 2 - # @valid: "PERIODIC", "MATCH", "FIXED", "ATMOSPHERE", "CUSTOM", "HORIZON" + # @valid: "PERIODIC", "MATCH", "FIXED", "ATMOSPHERE", "CUSTOM", "HORIZON", "CONDUCTOR" # @example: [["CUSTOM", "MATCH"]] (for 2D spherical [[rmin, rmax]]) # @note: When periodic in any of the directions, you should only set one value: [..., ["PERIODIC"], ...] # @note: In spherical, bondaries in theta/phi are set automatically (only specify bc @ [rmin, rmax]): [["ATMOSPHERE", "MATCH"]] diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index a1ce9a1a9..4f6decc76 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -14,6 +14,7 @@ #include "framework/domain/metadomain.h" #include +#include namespace user { using namespace ntt; @@ -48,7 +49,8 @@ namespace user { Inline auto bx3(const coord_t& x_ph) const -> real_t { // return Bmag * math::sin(Btheta) * math::cos(Bphi); - // return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + // return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + // + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); return ZERO; } @@ -59,7 +61,8 @@ namespace user { Inline auto ex2(const coord_t& x_ph) const -> real_t { // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - // return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + // return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + // + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); return ZERO; } @@ -88,17 +91,32 @@ namespace user { const real_t drift_ux, temperature; - const real_t Btheta, Bphi, Bmag; - InitFields init_flds; + const std::vector x1arr_e, x2arr_e, ux1arr_e, ux2arr_e, ux3arr_e; + const std::vector x1arr_i, x2arr_i, ux1arr_i, ux2arr_i, ux3arr_i; + + const real_t Btheta, Bphi, Bmag; + InitFields init_flds; + const Metadomain* metadomain; inline PGen(const SimulationParams& p, const Metadomain& m) : arch::ProblemGenerator { p } , drift_ux { p.template get("setup.drift_ux") } , temperature { p.template get("setup.temperature") } - , Bmag { p.template get("setup.Bmag", ZERO) } + , x1arr_e { p.template get>("setup.x_e") } + , x2arr_e { p.template get>("setup.y_e") } + , ux1arr_e { p.template get>("setup.ux_e") } + , ux2arr_e { p.template get>("setup.uy_e") } + , ux3arr_e { p.template get>("setup.uz_e") } + , x1arr_i { p.template get>("setup.x_i") } + , x2arr_i { p.template get>("setup.y_i") } + , ux1arr_i { p.template get>("setup.ux_i") } + , ux2arr_i { p.template get>("setup.uy_i") } + , ux3arr_i { p.template get>("setup.uz_i") } , Btheta { p.template get("setup.Btheta", ZERO) } + , Bmag { p.template get("setup.Bmag", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } - , init_flds { Bmag, Btheta, Bphi, drift_ux } {} + , init_flds { Bmag, Btheta, Bphi, drift_ux } + , metadomain { &m } {} inline PGen() {} @@ -113,111 +131,32 @@ namespace user { } } - auto MatchFields(real_t time) const -> InitFields { return init_flds; } inline void InitPrtls(Domain& domain) { - - auto& species_e = domain.species[0]; - auto& species_p = domain.species[1]; - - array_t elec_ind("elec_ind"); - array_t pos_ind("pos_ind"); - - auto offset_e = species_e.npart(); - auto offset_p = species_p.npart(); - - auto ux1_e = species_e.ux1; - auto ux2_e = species_e.ux2; - auto ux3_e = species_e.ux3; - auto i1_e = species_e.i1; - auto i2_e = species_e.i2; - auto dx1_e = species_e.dx1; - auto dx2_e = species_e.dx2; - auto phi_e = species_e.phi; - auto weight_e = species_e.weight; - auto tag_e = species_e.tag; - - auto ux1_p = species_p.ux1; - auto ux2_p = species_p.ux2; - auto ux3_p = species_p.ux3; - auto i1_p = species_p.i1; - auto i2_p = species_p.i2; - auto dx1_p = species_p.dx1; - auto dx2_p = species_p.dx2; - auto phi_p = species_p.phi; - auto weight_p = species_p.weight; - auto tag_p = species_p.tag; - - int nseed = 1; - - Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { - - // ToDo: fix this - auto i1_ = math::floor(10); - auto i2_ = math::floor(64); - auto dx1_ = HALF; - auto dx2_ = HALF; - - - auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); - auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); - - i1_e(elec_p + offset_e) = i1_; - dx1_e(elec_p + offset_e) = dx1_; - i2_e(elec_p + offset_e) = i2_; - dx2_e(elec_p + offset_e) = dx2_; - ux1_e(elec_p + offset_e) = -0.5; - ux2_e(elec_p + offset_e) = 0.5; - ux3_e(elec_p + offset_e) = ZERO; - weight_e(elec_p + offset_e) = ONE; - tag_e(elec_p + offset_e) = ParticleTag::alive; - - i1_p(pos_p + offset_p) = i1_; - dx1_p(pos_p + offset_p) = dx1_; - i2_p(pos_p + offset_p) = i2_; - dx2_p(pos_p + offset_p) = dx2_; - ux1_p(pos_p + offset_p) = 0.5; - ux2_p(pos_p + offset_p) = -0.5; - ux3_p(pos_p + offset_p) = ZERO; - weight_p(pos_p + offset_p) = ONE; - tag_p(pos_p + offset_p) = ParticleTag::alive; - - - }); - - - auto elec_ind_h = Kokkos::create_mirror(elec_ind); - Kokkos::deep_copy(elec_ind_h, elec_ind); - species_e.set_npart(offset_e + elec_ind_h()); - - auto pos_ind_h = Kokkos::create_mirror(pos_ind); - Kokkos::deep_copy(pos_ind_h, pos_ind); - species_p.set_npart(offset_p + pos_ind_h()); - - + arch::InjectGlobally(*metadomain, + domain, + 1, + { + { "x1", x1arr_e }, + { "x2", x2arr_e }, + { "ux1", ux1arr_e }, + { "ux2", ux1arr_e }, + { "ux3", ux3arr_e } + }); + arch::InjectGlobally(*metadomain, + domain, + 2, + { + { "x1", x1arr_i }, + { "x2", x2arr_i }, + { "ux1", ux1arr_i }, + { "ux2", ux1arr_i }, + { "ux3", ux3arr_i } + }); } - - - // inline void InitPrtls(Domain& local_domain) { - // const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, - // local_domain.random_pool, - // temperature, - // -drift_ux, - // in::x1); - - // const auto injector = arch::UniformInjector( - // energy_dist, - // { 1, 2 }); - // arch::InjectUniform>( - // params, - // local_domain, - // injector, - // 1.0); - // } - }; } // namespace user diff --git a/setups/srpic/shocktest/shock.toml b/setups/srpic/shocktest/shock.toml index a77f5c2e9..6c0c9a3a0 100644 --- a/setups/srpic/shocktest/shock.toml +++ b/setups/srpic/shocktest/shock.toml @@ -11,7 +11,7 @@ metric = "minkowski" [grid.boundaries] - fields = [["CONDUCTOR", "FIXED"], ["PERIODIC"]] + fields = [["CONDUCTOR", "FIXED"], ["PERIODIC"]] particles = [["REFLECT", "ABSORB"], ["PERIODIC"]] [scales] @@ -20,8 +20,8 @@ [algorithms] current_filters = 8 - fieldsolver = "false" - deposit = "false" + fieldsolver = "false" + deposit = "false" [algorithms.timestep] CFL = 0.5 @@ -44,13 +44,23 @@ [setup] drift_ux = 0.1 temperature = 1e-3 - Bmag = 1.0 - Btheta = 0.0 - Bphi = 0.0 + Bmag = 1.0 + Btheta = 0.0 + Bphi = 0.0 + x_e = [0.05] + y_e = [0.0] + ux_e = [-0.01] + uy_e = [0.01] + uz_e = [0.001] + x_i = [0.05] + y_i = [0.0] + ux_i = [0.01] + uy_i = [-0.01] + uz_i = [-0.001] [output] - interval = 1 - format = "hdf5" + interval = 1 + format = "hdf5" [output.fields] quantities = ["N_1", "N_2", "E", "B", "T0i_1", "T0i_2", "J"] diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 12872bf4e..dd2f5ccec 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -839,353 +839,284 @@ namespace ntt { } void PerfectConductorFieldsIn(dir::direction_t direction, - domain_t& domain, - BCTags tags) { + domain_t& domain, + BCTags tags) { /** * perfect conductor field boundaries */ - const auto sign = direction.get_sign(); - const auto dim = direction.get_dim(); - raise::ErrorIf(dim != in::x1 and M::CoordType != Coord::Cart, - "Perfect conductor BCs only implemented for x1 in " - "non-cartesian coordinates", - HERE); + if constexpr (M::CoordType != Coord::Cart) { + (void)direction; + (void)domain; + (void)tags; + raise::Error( + "Perfect conductor BCs only applicable to cartesian coordinates", + HERE); + } else { + const auto sign = direction.get_sign(); + const auto dim = direction.get_dim(); + raise::ErrorIf(dim != in::x1, + "Perfect conductor BCs only implemented for x1", + HERE); + std::vector xi_min, xi_max; - std::vector xi_min, xi_max; + const std::vector all_dirs { in::x1, in::x2, in::x3 }; - const std::vector all_dirs { in::x1, in::x2, in::x3 }; - - for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { - const auto dd = all_dirs[d]; - if (dim == dd) { + for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { + const auto dd = all_dirs[d]; + if (dim == dd) { xi_min.push_back(0); - xi_max.push_back(N_GHOSTS); + xi_max.push_back(N_GHOSTS + 1); + } else { + xi_min.push_back(0); + xi_max.push_back(domain.mesh.n_all(dd)); + } + } + raise::ErrorIf(xi_min.size() != xi_max.size() or + xi_min.size() != static_cast(M::Dim), + "Invalid range size", + HERE); + + range_t range; + if constexpr (M::Dim == Dim::_1D) { + range = CreateRangePolicy({ xi_min[0] }, { xi_max[0] }); + } else if constexpr (M::Dim == Dim::_2D) { + range = CreateRangePolicy({ xi_min[0], xi_min[1] }, + { xi_max[0], xi_max[1] }); + } else if constexpr (M::Dim == Dim::_3D) { + range = CreateRangePolicy({ xi_min[0], xi_min[1], xi_min[2] }, + { xi_max[0], xi_max[1], xi_max[2] }); } else { - xi_min.push_back(0); - xi_max.push_back(domain.mesh.n_all(dd)); + raise::Error("Invalid dimension", HERE); } - } - raise::ErrorIf(xi_min.size() != xi_max.size() or - xi_min.size() != static_cast(M::Dim), - "Invalid range size", - HERE); - - - if constexpr (M::Dim == Dim::_1D) - { Kokkos::parallel_for( - "MatchFields", - CreateRangePolicy({xi_min[0]}, {xi_max[0]}), - kernel::bc::ConductorBoundaries_kernel( - domain.fields.em, - tags)); + "MatchFields", + range, + kernel::bc::ConductorBoundaries_kernel(domain.fields.em, + tags)); + + // if constexpr (M::Dim == Dim::_1D) { + // Kokkos::parallel_for( + // "MatchFields", + // CreateRangePolicy({ xi_min[0] }, { xi_max[0] }), + // kernel::bc::ConductorBoundaries_kernel(domain.fields.em, + // tags)); + // } else if constexpr (M::Dim == Dim::_2D) { + // Kokkos::parallel_for( + // "MatchFields", + // CreateRangePolicy({ xi_min[0], xi_min[1] }, + // { xi_max[0], xi_max[1] }), + // kernel::bc::ConductorBoundaries_kernel(domain.fields.em, + // tags)); + // } else if constexpr (M::Dim == Dim::_3D) { + // Kokkos::parallel_for( + // "MatchFields", + // CreateRangePolicy({ xi_min[0], xi_min[1], xi_min[2] }, + // { xi_max[0], xi_max[1], xi_max[2] }), + // kernel::bc::ConductorBoundaries_kernel(domain.fields.em, + // tags)); + // } else { + // raise::Error("Invalid dimension", HERE); + // } } + } - if constexpr (M::Dim == Dim::_2D) - { - Kokkos::parallel_for( - "MatchFields", - CreateRangePolicy({xi_min[0], xi_min[1]}, {xi_max[0], xi_max[1]}), - kernel::bc::ConductorBoundaries_kernel( + void AtmosphereFieldsIn(dir::direction_t direction, + domain_t& domain, + BCTags tags) { + /** + * atmosphere field boundaries + */ + if constexpr (traits::has_member::value) { + const auto [sign, dim, xg_min, xg_max] = get_atm_extent(direction); + const auto dd = static_cast(dim); + boundaries_t box; + boundaries_t incl_ghosts; + for (unsigned short d { 0 }; d < M::Dim; ++d) { + if (d == dd) { + box.push_back({ xg_min, xg_max }); + if (sign > 0) { + incl_ghosts.push_back({ false, true }); + } else { + incl_ghosts.push_back({ true, false }); + } + } else { + box.push_back(Range::All); + incl_ghosts.push_back({ true, true }); + } + } + if (not domain.mesh.Intersects(box)) { + return; + } + const auto intersect_range = domain.mesh.ExtentToRange(box, incl_ghosts); + tuple_t range_min { 0 }; + tuple_t range_max { 0 }; + + for (unsigned short d { 0 }; d < M::Dim; ++d) { + range_min[d] = intersect_range[d].first; + range_max[d] = intersect_range[d].second; + } + auto atm_fields = m_pgen.AtmFields(time); + std::size_t il_edge; + if (sign > 0) { + il_edge = range_min[dd] - N_GHOSTS; + } else { + il_edge = range_max[dd] - 1 - N_GHOSTS; + } + const auto range = CreateRangePolicy(range_min, range_max); + if (dim == in::x1) { + if (sign > 0) { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, tags)); - } - - if constexpr (M::Dim == Dim::_3D) - { - Kokkos::parallel_for( - "MatchFields", - CreateRangePolicy({xi_min[0], xi_min[1], xi_min[2]}, {xi_max[0], xi_max[1], xi_max[2]}), - kernel::bc::ConductorBoundaries_kernel( + } else { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, tags)); - } - - // these components need to be set to zero at the boundary - std::vector comps = {em::ex2, em::ex3, em::bx1}; - for (const auto &comp : comps) - { - if constexpr (M::Dim == Dim::_1D) - { - Kokkos::deep_copy( - Kokkos::subview(domain.fields.em, - std::make_pair(N_GHOSTS, N_GHOSTS), - comp), - ZERO); - } - else if constexpr (M::Dim == Dim::_2D) - { - Kokkos::deep_copy( - Kokkos::subview(domain.fields.em, - std::make_pair(N_GHOSTS, N_GHOSTS+1), - std::make_pair(xi_min[1], xi_max[1]), - comp), - ZERO); - } - else if constexpr (M::Dim == Dim::_3D) - { - Kokkos::deep_copy( - Kokkos::subview(domain.fields.em, - std::make_pair(N_GHOSTS, N_GHOSTS), - std::make_pair(xi_min[1], xi_max[1]), - std::make_pair(xi_min[2], xi_max[2]), - comp), - ZERO); - } - else - { + } + } else if (dim == in::x2) { + if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { + if (sign > 0) { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } else { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } + } else { + raise::Error("Invalid dimension", HERE); + } + } else if (dim == in::x3) { + if constexpr (M::Dim == Dim::_3D) { + if (sign > 0) { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } else { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } + } else { + raise::Error("Invalid dimension", HERE); + } + } else { raise::Error("Invalid dimension", HERE); } + } else { + raise::Error("Atm fields not implemented in PGEN for atmosphere BCs", HERE); } } - void AtmosphereFieldsIn(dir::direction_t direction, - domain_t & domain, - BCTags tags) - { - /** - * atmosphere field boundaries - */ - if constexpr (traits::has_member::value) - { - const auto [sign, dim, xg_min, xg_max] = get_atm_extent(direction); - const auto dd = static_cast(dim); - boundaries_t box; - boundaries_t incl_ghosts; - for (unsigned short d{0}; d < M::Dim; ++d) - { - if (d == dd) - { - box.push_back({xg_min, xg_max}); - if (sign > 0) - { - incl_ghosts.push_back({false, true}); - } - else - { - incl_ghosts.push_back({true, false}); - } - } - else - { - box.push_back(Range::All); - incl_ghosts.push_back({true, true}); - } - } - if (not domain.mesh.Intersects(box)) - { - return; - } - const auto intersect_range = domain.mesh.ExtentToRange(box, incl_ghosts); - tuple_t range_min{0}; - tuple_t range_max{0}; - - for (unsigned short d{0}; d < M::Dim; ++d) - { - range_min[d] = intersect_range[d].first; - range_max[d] = intersect_range[d].second; - } - auto atm_fields = m_pgen.AtmFields(time); - std::size_t il_edge; - if (sign > 0) - { - il_edge = range_min[dd] - N_GHOSTS; - } - else - { - il_edge = range_max[dd] - 1 - N_GHOSTS; - } - const auto range = CreateRangePolicy(range_min, range_max); - if (dim == in::x1) - { - if (sign > 0) - { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } - else - { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } - } - else if (dim == in::x2) - { - if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) - { - if (sign > 0) - { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } - else - { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } - } - else - { - raise::Error("Invalid dimension", HERE); - } - } - else if (dim == in::x3) - { - if constexpr (M::Dim == Dim::_3D) - { - if (sign > 0) - { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } - else - { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } - } - else - { - raise::Error("Invalid dimension", HERE); - } - } - else - { - raise::Error("Invalid dimension", HERE); - } - } - else - { - raise::Error("Atm fields not implemented in PGEN for atmosphere BCs", HERE); - } - } + void CustomFieldsIn(dir::direction_t direction, + domain_t& domain, + BCTags tags) { + (void)direction; + (void)domain; + (void)tags; + raise::Error("Custom boundaries not implemented", HERE); + // if constexpr ( + // traits::has_member::value) { + // const auto [box, custom_fields] = m_pgen.CustomFields(time); + // if (domain.mesh.Intersects(box)) { + // } + // + // } else { + // raise::Error("Custom boundaries not implemented", HERE); + // } + } - void CustomFieldsIn(dir::direction_t direction, - domain_t & domain, - BCTags tags) - { - (void)direction; - (void)domain; - (void)tags; - raise::Error("Custom boundaries not implemented", HERE); - // if constexpr ( - // traits::has_member::value) { - // const auto [box, custom_fields] = m_pgen.CustomFields(time); - // if (domain.mesh.Intersects(box)) { - // } - // - // } else { - // raise::Error("Custom boundaries not implemented", HERE); - // } + void AtmosphereParticlesIn(const dir::direction_t& direction, + domain_t& domain, + InjTags tags) { + const auto [sign, dim, xg_min, xg_max] = get_atm_extent(direction); + + const auto x_surf = sign > 0 ? xg_min : xg_max; + const auto ds = m_params.template get( + "grid.boundaries.atmosphere.ds"); + const auto temp = m_params.template get( + "grid.boundaries.atmosphere.temperature"); + const auto height = m_params.template get( + "grid.boundaries.atmosphere.height"); + const auto species = + m_params.template get>( + "grid.boundaries.atmosphere.species"); + const auto nmax = m_params.template get( + "grid.boundaries.atmosphere.density"); + + Kokkos::deep_copy(domain.fields.bckp, ZERO); + auto scatter_bckp = Kokkos::Experimental::create_scatter_view( + domain.fields.bckp); + const auto use_weights = M::CoordType != Coord::Cart; + const auto ni2 = domain.mesh.n_active(in::x2); + const auto inv_n0 = ONE / m_params.template get("scales.n0"); + + // compute the density of the two species + if (tags & Inj::AssumeEmpty) { + if constexpr (M::Dim == Dim::_1D) { + Kokkos::deep_copy( + Kokkos::subview(domain.fields.bckp, Kokkos::ALL, std::make_pair(0, 1)), + ZERO); + } else if constexpr (M::Dim == Dim::_2D) { + Kokkos::deep_copy(Kokkos::subview(domain.fields.bckp, + Kokkos::ALL, + Kokkos::ALL, + std::make_pair(0, 1)), + ZERO); + } else if constexpr (M::Dim == Dim::_3D) { + Kokkos::deep_copy(Kokkos::subview(domain.fields.bckp, + Kokkos::ALL, + Kokkos::ALL, + Kokkos::ALL, + std::make_pair(0, 1)), + ZERO); } - - void AtmosphereParticlesIn(const dir::direction_t &direction, - domain_t &domain, - InjTags tags) - { - const auto [sign, dim, xg_min, xg_max] = get_atm_extent(direction); - - const auto x_surf = sign > 0 ? xg_min : xg_max; - const auto ds = m_params.template get( - "grid.boundaries.atmosphere.ds"); - const auto temp = m_params.template get( - "grid.boundaries.atmosphere.temperature"); - const auto height = m_params.template get( - "grid.boundaries.atmosphere.height"); - const auto species = - m_params.template get>( - "grid.boundaries.atmosphere.species"); - const auto nmax = m_params.template get( - "grid.boundaries.atmosphere.density"); - - Kokkos::deep_copy(domain.fields.bckp, ZERO); - auto scatter_bckp = Kokkos::Experimental::create_scatter_view( - domain.fields.bckp); - const auto use_weights = M::CoordType != Coord::Cart; - const auto ni2 = domain.mesh.n_active(in::x2); - const auto inv_n0 = ONE / m_params.template get("scales.n0"); - - // compute the density of the two species - if (tags & Inj::AssumeEmpty) - { - if constexpr (M::Dim == Dim::_1D) - { - Kokkos::deep_copy( - Kokkos::subview(domain.fields.bckp, Kokkos::ALL, std::make_pair(0, 1)), - ZERO); - } - else if constexpr (M::Dim == Dim::_2D) - { - Kokkos::deep_copy(Kokkos::subview(domain.fields.bckp, - Kokkos::ALL, - Kokkos::ALL, - std::make_pair(0, 1)), - ZERO); - } - else if constexpr (M::Dim == Dim::_3D) - { - Kokkos::deep_copy(Kokkos::subview(domain.fields.bckp, - Kokkos::ALL, - Kokkos::ALL, - Kokkos::ALL, - std::make_pair(0, 1)), - ZERO); - } + } else { + for (const auto& sp : + std::vector({ species.first, species.second })) { + auto& prtl_spec = domain.species[sp - 1]; + if (prtl_spec.npart() == 0) { + continue; } - else - { - for (const auto &sp : - std::vector({species.first, species.second})) - { - auto &prtl_spec = domain.species[sp - 1]; - if (prtl_spec.npart() == 0) - { - continue; - } - // clang-format off + // clang-format off Kokkos::parallel_for( "ComputeMoments", prtl_spec.rangeActiveParticles(), @@ -1199,265 +1130,230 @@ namespace ntt { use_weights, domain.mesh.metric, domain.mesh.flds_bc(), ni2, inv_n0, 0)); - // clang-format on - prtl_spec.set_unsorted(); - } - Kokkos::Experimental::contribute(domain.fields.bckp, scatter_bckp); - m_metadomain.SynchronizeFields(domain, Comm::Bckp, {0, 1}); - } + // clang-format on + prtl_spec.set_unsorted(); + } + Kokkos::Experimental::contribute(domain.fields.bckp, scatter_bckp); + m_metadomain.SynchronizeFields(domain, Comm::Bckp, { 0, 1 }); + } - if (dim == in::x1) - { - if (sign > 0) - { - const auto atm_injector = - arch::AtmosphereInjector{ - domain.mesh.metric, - domain.fields.bckp, - nmax, - height, - x_surf, - ds, - temp, - domain.random_pool, - species}; - arch::InjectNonUniform(m_params, - domain, - atm_injector, - nmax, - use_weights); - } - else - { - const auto atm_injector = - arch::AtmosphereInjector{ - domain.mesh.metric, - domain.fields.bckp, - nmax, - height, - x_surf, - ds, - temp, - domain.random_pool, - species}; - arch::InjectNonUniform(m_params, - domain, - atm_injector, - nmax, - use_weights); - } - } - else if (dim == in::x2) - { - if (sign > 0) - { - const auto atm_injector = - arch::AtmosphereInjector{ - domain.mesh.metric, - domain.fields.bckp, - nmax, - height, - x_surf, - ds, - temp, - domain.random_pool, - species}; - arch::InjectNonUniform(m_params, - domain, - atm_injector, - nmax, - use_weights); - } - else - { - const auto atm_injector = - arch::AtmosphereInjector{ - domain.mesh.metric, - domain.fields.bckp, - nmax, - height, - x_surf, - ds, - temp, - domain.random_pool, - species}; - arch::InjectNonUniform(m_params, - domain, - atm_injector, - nmax, - use_weights); - } - } - else if (dim == in::x3) - { - if (sign > 0) - { - const auto atm_injector = - arch::AtmosphereInjector{ - domain.mesh.metric, - domain.fields.bckp, - nmax, - height, - x_surf, - ds, - temp, - domain.random_pool, - species}; - arch::InjectNonUniform(m_params, - domain, - atm_injector, - nmax, - use_weights); - } - else - { - const auto atm_injector = - arch::AtmosphereInjector{ - domain.mesh.metric, - domain.fields.bckp, - nmax, - height, - x_surf, - ds, - temp, - domain.random_pool, - species}; - arch::InjectNonUniform(m_params, - domain, - atm_injector, - nmax, - use_weights); - } - } - else - { - raise::Error("Invalid dimension", HERE); - } - return; + if (dim == in::x1) { + if (sign > 0) { + const auto atm_injector = + arch::AtmosphereInjector { + domain.mesh.metric, + domain.fields.bckp, + nmax, + height, + x_surf, + ds, + temp, + domain.random_pool, + species + }; + arch::InjectNonUniform(m_params, + domain, + atm_injector, + nmax, + use_weights); + } else { + const auto atm_injector = + arch::AtmosphereInjector { + domain.mesh.metric, + domain.fields.bckp, + nmax, + height, + x_surf, + ds, + temp, + domain.random_pool, + species + }; + arch::InjectNonUniform(m_params, + domain, + atm_injector, + nmax, + use_weights); } + } else if (dim == in::x2) { + if (sign > 0) { + const auto atm_injector = + arch::AtmosphereInjector { + domain.mesh.metric, + domain.fields.bckp, + nmax, + height, + x_surf, + ds, + temp, + domain.random_pool, + species + }; + arch::InjectNonUniform(m_params, + domain, + atm_injector, + nmax, + use_weights); + } else { + const auto atm_injector = + arch::AtmosphereInjector { + domain.mesh.metric, + domain.fields.bckp, + nmax, + height, + x_surf, + ds, + temp, + domain.random_pool, + species + }; + arch::InjectNonUniform(m_params, + domain, + atm_injector, + nmax, + use_weights); + } + } else if (dim == in::x3) { + if (sign > 0) { + const auto atm_injector = + arch::AtmosphereInjector { + domain.mesh.metric, + domain.fields.bckp, + nmax, + height, + x_surf, + ds, + temp, + domain.random_pool, + species + }; + arch::InjectNonUniform(m_params, + domain, + atm_injector, + nmax, + use_weights); + } else { + const auto atm_injector = + arch::AtmosphereInjector { + domain.mesh.metric, + domain.fields.bckp, + nmax, + height, + x_surf, + ds, + temp, + domain.random_pool, + species + }; + arch::InjectNonUniform(m_params, + domain, + atm_injector, + nmax, + use_weights); + } + } else { + raise::Error("Invalid dimension", HERE); + } + return; + } - private: - /** - * @brief Get the buffer region of the atmosphere and the direction - * @param direction direction in which the atmosphere is applied - * @return tuple: [sign of the direction, the direction (as in::), the min and max extent - * @note xg_min and xg_max are the extents where the fields are set, not the atmosphere itself - * @note i.e. - * - * fields set particles injected - * ghost zone | | - * v v v - * |....|...........|*******************..... -> x1 - * ^ ^ - * xg_min xg_max - * | | | - * |<-- buffer -->|<-- atmosphere -->| - * - * in this case the function returns { -1, in::x1, xg_min, xg_max } - */ - auto get_atm_extent(dir::direction_t direction) const - -> std::tuple - { - const auto sign = direction.get_sign(); - const auto dim = direction.get_dim(); - const auto min_buff = m_params.template get( - "algorithms.current_filters") + - 2; - const auto buffer_ncells = min_buff > 5 ? min_buff : 5; - if (M::CoordType != Coord::Cart and (dim != in::x1 or sign > 0)) - { - raise::Error("For non-cartesian coordinates atmosphere BCs is " - "possible only in -x1 (@ rmin)", - HERE); - } - real_t xg_min{ZERO}, xg_max{ZERO}; - std::size_t ig_min, ig_max; - if (sign > 0) - { // + direction - ig_min = m_metadomain.mesh().n_active(dim) - buffer_ncells; - ig_max = m_metadomain.mesh().n_active(dim); - } - else - { // - direction - ig_min = 0; - ig_max = buffer_ncells; - } + private: + /** + * @brief Get the buffer region of the atmosphere and the direction + * @param direction direction in which the atmosphere is applied + * @return tuple: [sign of the direction, the direction (as in::), the min and max extent + * @note xg_min and xg_max are the extents where the fields are set, not the atmosphere itself + * @note i.e. + * + * fields set particles injected + * ghost zone | | + * v v v + * |....|...........|*******************..... -> x1 + * ^ ^ + * xg_min xg_max + * | | | + * |<-- buffer -->|<-- atmosphere -->| + * + * in this case the function returns { -1, in::x1, xg_min, xg_max } + */ + auto get_atm_extent(dir::direction_t direction) const + -> std::tuple { + const auto sign = direction.get_sign(); + const auto dim = direction.get_dim(); + const auto min_buff = m_params.template get( + "algorithms.current_filters") + + 2; + const auto buffer_ncells = min_buff > 5 ? min_buff : 5; + if (M::CoordType != Coord::Cart and (dim != in::x1 or sign > 0)) { + raise::Error("For non-cartesian coordinates atmosphere BCs is " + "possible only in -x1 (@ rmin)", + HERE); + } + real_t xg_min { ZERO }, xg_max { ZERO }; + std::size_t ig_min, ig_max; + if (sign > 0) { // + direction + ig_min = m_metadomain.mesh().n_active(dim) - buffer_ncells; + ig_max = m_metadomain.mesh().n_active(dim); + } else { // - direction + ig_min = 0; + ig_max = buffer_ncells; + } - if (dim == in::x1) - { - xg_min = m_metadomain.mesh().metric.template convert<1, Crd::Cd, Crd::Ph>( - static_cast(ig_min)); - xg_max = m_metadomain.mesh().metric.template convert<1, Crd::Cd, Crd::Ph>( - static_cast(ig_max)); - } - else if (dim == in::x2) - { - if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) - { - xg_min = m_metadomain.mesh().metric.template convert<2, Crd::Cd, Crd::Ph>( - static_cast(ig_min)); - xg_max = m_metadomain.mesh().metric.template convert<2, Crd::Cd, Crd::Ph>( - static_cast(ig_max)); - } - else - { - raise::Error("Invalid dimension", HERE); - } - } - else if (dim == in::x3) - { - if constexpr (M::Dim == Dim::_3D) - { - xg_min = m_metadomain.mesh().metric.template convert<3, Crd::Cd, Crd::Ph>( - static_cast(ig_min)); - xg_max = m_metadomain.mesh().metric.template convert<3, Crd::Cd, Crd::Ph>( - static_cast(ig_max)); - } - else - { - raise::Error("Invalid dimension", HERE); - } - } - else - { - raise::Error("Invalid dimension", HERE); - } - return {sign, dim, xg_min, xg_max}; + if (dim == in::x1) { + xg_min = m_metadomain.mesh().metric.template convert<1, Crd::Cd, Crd::Ph>( + static_cast(ig_min)); + xg_max = m_metadomain.mesh().metric.template convert<1, Crd::Cd, Crd::Ph>( + static_cast(ig_max)); + } else if (dim == in::x2) { + if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { + xg_min = m_metadomain.mesh().metric.template convert<2, Crd::Cd, Crd::Ph>( + static_cast(ig_min)); + xg_max = m_metadomain.mesh().metric.template convert<2, Crd::Cd, Crd::Ph>( + static_cast(ig_max)); + } else { + raise::Error("Invalid dimension", HERE); + } + } else if (dim == in::x3) { + if constexpr (M::Dim == Dim::_3D) { + xg_min = m_metadomain.mesh().metric.template convert<3, Crd::Cd, Crd::Ph>( + static_cast(ig_min)); + xg_max = m_metadomain.mesh().metric.template convert<3, Crd::Cd, Crd::Ph>( + static_cast(ig_max)); + } else { + raise::Error("Invalid dimension", HERE); } + } else { + raise::Error("Invalid dimension", HERE); + } + return { sign, dim, xg_min, xg_max }; + } - auto range_with_axis_BCs(const domain_t &domain) -> range_t - { - auto range = domain.mesh.rangeActiveCells(); - if constexpr (M::CoordType != Coord::Cart) - { - /** - * @brief taking one extra cell in the x2 direction if AXIS BCs - */ - if constexpr (M::Dim == Dim::_2D) - { - if (domain.mesh.flds_bc_in({0, +1}) == FldsBC::AXIS) - { - range = CreateRangePolicy( - {domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2)}, - {domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1}); - } - } - else if constexpr (M::Dim == Dim::_3D) - { - if (domain.mesh.flds_bc_in({0, +1, 0}) == FldsBC::AXIS) - { - range = CreateRangePolicy({domain.mesh.i_min(in::x1), - domain.mesh.i_min(in::x2), - domain.mesh.i_min(in::x3)}, - {domain.mesh.i_max(in::x1), - domain.mesh.i_max(in::x2) + 1, - domain.mesh.i_max(in::x3)}); - } - } + auto range_with_axis_BCs(const domain_t& domain) -> range_t { + auto range = domain.mesh.rangeActiveCells(); + if constexpr (M::CoordType != Coord::Cart) { + /** + * @brief taking one extra cell in the x2 direction if AXIS BCs + */ + if constexpr (M::Dim == Dim::_2D) { + if (domain.mesh.flds_bc_in({ 0, +1 }) == FldsBC::AXIS) { + range = CreateRangePolicy( + { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) }, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1 }); + } + } else if constexpr (M::Dim == Dim::_3D) { + if (domain.mesh.flds_bc_in({ 0, +1, 0 }) == FldsBC::AXIS) { + range = CreateRangePolicy({ domain.mesh.i_min(in::x1), + domain.mesh.i_min(in::x2), + domain.mesh.i_min(in::x3) }, + { domain.mesh.i_max(in::x1), + domain.mesh.i_max(in::x2) + 1, + domain.mesh.i_max(in::x3) }); } - return range; } - }; + } + return range; + } + }; } // namespace ntt diff --git a/src/global/enums.h b/src/global/enums.h index 2b3bf5936..32ef17157 100644 --- a/src/global/enums.h +++ b/src/global/enums.h @@ -9,7 +9,7 @@ * - enum ntt::PrtlBC // periodic, absorb, atmosphere, custom, * reflect, horizon, axis, sync * - enum ntt::FldsBC // periodic, match, fixed, atmosphere, - * custom, horizon, axis, sync + * custom, horizon, axis, conductor, sync * - enum ntt::PrtlPusher // boris, vay, photon, none * - enum ntt::Cooling // synchrotron, none * - enum ntt::FldsID // e, dive, d, divd, b, h, j, @@ -221,17 +221,20 @@ namespace ntt { CUSTOM = 5, HORIZON = 6, AXIS = 7, - SYNC = 8, // <- SYNC means synchronization with other domains - CONDUCTOR = 9 + CONDUCTOR = 8, + SYNC = 9 // <- SYNC means synchronization with other domains }; constexpr FldsBC(uint8_t c) : enums_hidden::BaseEnum { c } {} - static constexpr type variants[] = { PERIODIC, MATCH, FIXED, ATMOSPHERE, - CUSTOM, HORIZON, AXIS, SYNC, CONDUCTOR }; - static constexpr const char* lookup[] = { "periodic", "match", "fixed", - "atmosphere", "custom", "horizon", - "axis", "sync", "conductor"}; + static constexpr type variants[] = { + PERIODIC, MATCH, FIXED, ATMOSPHERE, CUSTOM, + HORIZON, AXIS, CONDUCTOR, SYNC, + }; + static constexpr const char* lookup[] = { + "periodic", "match", "fixed", "atmosphere", "custom", + "horizon", "axis", "conductor", "sync" + }; static constexpr std::size_t total = sizeof(variants) / sizeof(variants[0]); }; diff --git a/src/global/tests/enums.cpp b/src/global/tests/enums.cpp index 7785ec1a3..b26a2050f 100644 --- a/src/global/tests/enums.cpp +++ b/src/global/tests/enums.cpp @@ -61,8 +61,9 @@ auto main() -> int { enum_str_t all_simulation_engines = { "srpic", "grpic" }; enum_str_t all_particle_bcs = { "periodic", "absorb", "atmosphere", "custom", "reflect", "horizon", "axis", "sync" }; - enum_str_t all_fields_bcs = { "periodic", "match", "fixed", "atmosphere", - "custom", "horizon", "axis", "sync" }; + enum_str_t all_fields_bcs = { "periodic", "match", "fixed", + "atmosphere", "custom", "horizon", + "axis", "conductor", "sync" }; enum_str_t all_particle_pushers = { "boris", "vay", "photon", "none" }; enum_str_t all_coolings = { "synchrotron", "none" }; diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 720408d05..9d8e9c74e 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -5,6 +5,7 @@ * - kernel::bc::MatchBoundaries_kernel<> * - kernel::bc::AxisBoundaries_kernel<> * - kernel::bc::EnforcedBoundaries_kernel<> + * - kernel::bc::ConductorBoundaries_kernel<> * @namespaces: * - kernel::bc:: */ @@ -485,43 +486,40 @@ namespace kernel::bc { } }; - template + template struct ConductorBoundaries_kernel { - static_assert(M::is_metric, "M must be a metric class"); - static_assert(static_cast(o) < - static_cast(M::Dim), + static_assert(static_cast(o) < static_cast(D), "Invalid component index"); - static constexpr idx_t i = static_cast(o) + 1u; + // static constexpr idx_t i = static_cast(o) + 1u; - ndfield_t Fld; - const BCTags tags; + ndfield_t Fld; + const BCTags tags; - ConductorBoundaries_kernel(ndfield_t Fld, - BCTags tags) + ConductorBoundaries_kernel(ndfield_t Fld, BCTags tags) : Fld { Fld } , tags { tags } {} Inline void operator()(index_t i1) const { - if constexpr (M::Dim == Dim::_1D) { - - if constexpr (S == SimEngine::SRPIC) { - + if constexpr (D == Dim::_1D) { if (tags & BC::E) { - Fld((N_GHOSTS-1)-i1, em::ex1) = Fld(N_GHOSTS+i1, em::ex1); - Fld((N_GHOSTS-1)-i1, em::ex2) = -Fld(N_GHOSTS+i1+1, em::ex2); - Fld((N_GHOSTS-1)-i1, em::ex3) = -Fld(N_GHOSTS+i1+1, em::ex3); - } - - if (tags & BC::B) - { - Fld((N_GHOSTS-1)-i1, em::bx1) = -Fld(N_GHOSTS+i1+1, em::bx1); - Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); - Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); + if (i1 == 0) { + Fld(N_GHOSTS, em::ex2) = ZERO; + Fld(N_GHOSTS, em::ex3) = ZERO; + } else { + Fld(N_GHOSTS - i1, em::ex1) = Fld(N_GHOSTS + i1 - 1, em::ex1); + Fld(N_GHOSTS - i1, em::ex2) = -Fld(N_GHOSTS + i1, em::ex2); + Fld(N_GHOSTS - i1, em::ex3) = -Fld(N_GHOSTS + i1, em::ex3); + } } - } else { - // GRPIC - raise::KernelError(HERE, "1D GRPIC not implemented"); + if (tags & BC::B) { + if (i1 == 0) { + Fld(N_GHOSTS, em::bx1) = ZERO; + } else { + Fld(N_GHOSTS - i1, em::bx1) = -Fld(N_GHOSTS + i1, em::bx1); + Fld(N_GHOSTS - i1, em::bx2) = Fld(N_GHOSTS + i1 - 1, em::bx2); + Fld(N_GHOSTS - i1, em::bx3) = Fld(N_GHOSTS + i1 - 1, em::bx3); + } } } else { raise::KernelError( @@ -530,75 +528,70 @@ namespace kernel::bc { } } - Inline void operator()(index_t i1, index_t i2) const - { - if constexpr (M::Dim == Dim::_2D) - { - - if constexpr (S == SimEngine::SRPIC) - { - // SRPIC - if (tags & BC::E) - { - Fld((N_GHOSTS - 1) - i1, i2, em::ex1) = Fld(N_GHOSTS + i1, i2, em::ex1); - Fld((N_GHOSTS - 1) - i1, i2, em::ex2) = -Fld(N_GHOSTS + 1 + i1, i2, em::ex2); - Fld((N_GHOSTS - 1) - i1, i2, em::ex3) = -Fld(N_GHOSTS + 1 + i1, i2, em::ex3); + Inline void operator()(index_t i1, index_t i2) const { + if constexpr (D == Dim::_2D) { + if (tags & BC::E) { + if (i1 == 0) { + Fld(N_GHOSTS, i2, em::ex2) = ZERO; + Fld(N_GHOSTS, i2, em::ex3) = ZERO; + } else { + Fld(N_GHOSTS - i1, i2, em::ex1) = Fld(N_GHOSTS + i1 - 1, i2, em::ex1); + Fld(N_GHOSTS - i1, i2, em::ex2) = -Fld(N_GHOSTS + i1, i2, em::ex2); + Fld(N_GHOSTS - i1, i2, em::ex3) = -Fld(N_GHOSTS + i1, i2, em::ex3); } + } - if (tags & BC::B) - { - Fld((N_GHOSTS - 1) - i1, i2, em::bx1) = -Fld(N_GHOSTS + 1 + i1, i2, em::bx1); - Fld((N_GHOSTS - 1) - i1, i2, em::bx2) = Fld(N_GHOSTS + i1, i2, em::bx2); - Fld((N_GHOSTS - 1) - i1, i2, em::bx3) = Fld(N_GHOSTS + i1, i2, em::bx3); + if (tags & BC::B) { + if (i1 == 0) { + Fld(N_GHOSTS, i2, em::bx1) = ZERO; + } else { + Fld(N_GHOSTS - i1, i2, em::bx1) = -Fld(N_GHOSTS + i1, i2, em::bx1); + Fld(N_GHOSTS - i1, i2, em::bx2) = Fld(N_GHOSTS + i1 - 1, i2, em::bx2); + Fld(N_GHOSTS - i1, i2, em::bx3) = Fld(N_GHOSTS + i1 - 1, i2, em::bx3); } } - else - { - // GRPIC - raise::KernelError(HERE, "2D GRPIC not implemented"); - } - } - else - { + } else { raise::KernelError( - HERE, - "ConductorBoundaries_kernel: 2D implementation called for D != 2"); + HERE, + "ConductorBoundaries_kernel: 2D implementation called for D != 2"); } } - Inline void operator()(index_t i1, index_t i2, index_t i3) const - { - if constexpr (M::Dim == Dim::_3D) - { - - if constexpr (S == SimEngine::SRPIC) - { - // SRPIC - if (tags & BC::E) - { - Fld((N_GHOSTS - 1) - i1, i2, i3, em::ex1) = Fld(N_GHOSTS + i1, i2, i3, em::ex1); - Fld((N_GHOSTS - 1) - i1, i2, i3, em::ex2) = -Fld(N_GHOSTS + i1 + 1, i2, i3, em::ex2); - Fld((N_GHOSTS - 1) - i1, i2, i3, em::ex3) = -Fld(N_GHOSTS + i1 + 1, i2, i3, em::ex3); + Inline void operator()(index_t i1, index_t i2, index_t i3) const { + if constexpr (D == Dim::_3D) { + if (tags & BC::E) { + if (i1 == 0) { + Fld(N_GHOSTS, i2, i3, em::ex2) = ZERO; + Fld(N_GHOSTS, i2, i3, em::ex3) = ZERO; + } else { + Fld(N_GHOSTS - i1, i2, i3, em::ex1) = Fld(N_GHOSTS + i1 - 1, + i2, + i3, + em::ex1); + Fld(N_GHOSTS - i1, i2, i3, em::ex2) = -Fld(N_GHOSTS + i1, i2, i3, em::ex2); + Fld(N_GHOSTS - i1, i2, i3, em::ex3) = -Fld(N_GHOSTS + i1, i2, i3, em::ex3); } + } - if (tags & BC::B) - { - Fld((N_GHOSTS - 1) - i1, i2, i3, em::bx1) = -Fld(N_GHOSTS + i1 + 1, i2, i3, em::bx1); - Fld((N_GHOSTS - 1) - i1, i2, i3, em::bx2) = Fld(N_GHOSTS + i1, i2, i3, em::bx2); - Fld((N_GHOSTS - 1) - i1, i2, i3, em::bx3) = Fld(N_GHOSTS + i1, i2, i3, em::bx3); + if (tags & BC::B) { + if (i1 == 0) { + Fld(N_GHOSTS, i2, i3, em::bx1) = ZERO; + } else { + Fld(N_GHOSTS - i1, i2, i3, em::bx1) = -Fld(N_GHOSTS + i1, i2, i3, em::bx1); + Fld(N_GHOSTS - i1, i2, i3, em::bx2) = Fld(N_GHOSTS + i1 - 1, + i2, + i3, + em::bx2); + Fld(N_GHOSTS - i1, i2, i3, em::bx3) = Fld(N_GHOSTS + i1 - 1, + i2, + i3, + em::bx3); } } - else - { - // GRPIC - raise::KernelError(HERE, "3D GRPIC not implemented"); - } - } - else - { + } else { raise::KernelError( - HERE, - "ConductorBoundaries_kernel: 3D implementation called for D != 3"); + HERE, + "ConductorBoundaries_kernel: 3D implementation called for D != 3"); } } }; From 28a02f13191464de82b09fc6959390a08511e71f Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 7 Mar 2025 12:11:31 -0500 Subject: [PATCH 054/176] filters adapted for conducting BCs --- src/kernels/digital_filter.hpp | 132 +++++++++++++++++++++------------ 1 file changed, 86 insertions(+), 46 deletions(-) diff --git a/src/kernels/digital_filter.hpp b/src/kernels/digital_filter.hpp index 5d05fad2d..6a4599d35 100644 --- a/src/kernels/digital_filter.hpp +++ b/src/kernels/digital_filter.hpp @@ -17,9 +17,13 @@ #include "utils/error.h" #include "utils/numeric.h" -#define FILTER_IN_I1(ARR, COMP, I, J) \ +#define FILTER2D_IN_I1(ARR, COMP, I, J) \ INV_2*(ARR)((I), (J), (COMP)) + \ - INV_4*((ARR)((I)-1, (J), (COMP)) + (ARR)((I) + 1, (J), (COMP))) + INV_4*((ARR)((I) - 1, (J), (COMP)) + (ARR)((I) + 1, (J), (COMP))) + +#define FILTER2D_IN_I2(ARR, COMP, I, J) \ + INV_2*(ARR)((I), (J), (COMP)) + \ + INV_4*((ARR)((I), (J) - 1, (COMP)) + (ARR)((I), (J) + 1, (COMP))) namespace kernel { using namespace ntt; @@ -28,8 +32,9 @@ namespace kernel { class DigitalFilter_kernel { ndfield_t array; const ndfield_t buffer; - bool is_axis_i2min { false }, is_axis_i2max { false }; - static constexpr auto i2_min = N_GHOSTS; + const bool is_axis_i2min, is_axis_i2max; + const bool is_conductor_i1min; + static constexpr auto i1_min = N_GHOSTS, i2_min = N_GHOSTS; const std::size_t i2_max; public: @@ -39,20 +44,31 @@ namespace kernel { const boundaries_t& boundaries) : array { array } , buffer { buffer } - , i2_max { (short)D > 1 ? size_[1] + N_GHOSTS : 0 } { - if constexpr ((C != Coord::Cart) && (D != Dim::_1D)) { - raise::ErrorIf(boundaries.size() < 2, "boundaries defined incorrectly", HERE); - is_axis_i2min = (boundaries[1].first == FldsBC::AXIS); - is_axis_i2max = (boundaries[1].second == FldsBC::AXIS); - } - } + , is_axis_i2min { (D == Dim::_2D) and (boundaries[1].first == FldsBC::AXIS) } + , is_axis_i2max { (D == Dim::_2D) and (boundaries[1].second == FldsBC::AXIS) } + , is_conductor_i1min { boundaries[0].first == FldsBC::CONDUCTOR } + , i2_max { (short)D > 1 ? size_[1] + N_GHOSTS : 0 } {} Inline void operator()(index_t i1) const { if constexpr ((D == Dim::_1D) && (C == Coord::Cart)) { + if (is_conductor_i1min and i1 == i1_min) { + array(i1, cur::jx1) = (THREE * INV_4) * buffer(i1, cur::jx1) + + (INV_4)*buffer(i1 + 1, cur::jx1); + } else if (is_conductor_i1min and i1 == i1_min + 1) { + array(i1, cur::jx1) = INV_2 * buffer(i1, cur::jx1) + + INV_4 * (buffer(i1 - 1, cur::jx1) + + buffer(i1 + 1, cur::jx1)); + array(i1, cur::jx2) = (INV_2)*buffer(i1, cur::jx2) + + (INV_4)*buffer(i1 + 1, cur::jx2); + array(i1, cur::jx3) = (INV_2)*buffer(i1, cur::jx3) + + (INV_4)*buffer(i1 + 1, cur::jx3); + } else { #pragma unroll - for (const auto& comp : { cur::jx1, cur::jx2, cur::jx3 }) { - array(i1, comp) = INV_2 * buffer(i1, comp) + - INV_4 * (buffer(i1 - 1, comp) + buffer(i1 + 1, comp)); + for (const auto& comp : { cur::jx1, cur::jx2, cur::jx3 }) { + array(i1, comp) = INV_2 * buffer(i1, comp) + + INV_4 * + (buffer(i1 - 1, comp) + buffer(i1 + 1, comp)); + } } } else { raise::KernelError(HERE, "DigitalFilter_kernel: 1D implementation called for D != 1 or for non-Cartesian metric"); @@ -62,25 +78,49 @@ namespace kernel { Inline void operator()(index_t i1, index_t i2) const { if constexpr (D == Dim::_2D) { if constexpr (C == Coord::Cart) { + if (is_conductor_i1min and i1 == i1_min) { + array(i1, i2, cur::jx1) = + (THREE * INV_4) * (FILTER2D_IN_I2(buffer, cur::jx1, i1, i2)) + + (INV_4) * (FILTER2D_IN_I2(buffer, cur::jx1, i1 + 1, i2)); + } else if (is_conductor_i1min and i1 == i1_min + 1) { + array(i1, + i2, + cur::jx1) = INV_2 * (FILTER2D_IN_I2(buffer, cur::jx1, i1, i2)) + + INV_4 * + ((FILTER2D_IN_I2(buffer, cur::jx1, i1 - 1, i2)) + + (FILTER2D_IN_I2(buffer, cur::jx1, i1 + 1, i2))); + array(i1, + i2, + cur::jx2) = INV_2 * (FILTER2D_IN_I2(buffer, cur::jx2, i1, i2)) + + INV_4 * + (FILTER2D_IN_I2(buffer, cur::jx2, i1 + 1, i2)); + array(i1, + i2, + cur::jx3) = INV_2 * (FILTER2D_IN_I2(buffer, cur::jx3, i1, i2)) + + INV_4 * + (FILTER2D_IN_I2(buffer, cur::jx3, i1 + 1, i2)); + } else { #pragma unroll - for (const auto comp : { cur::jx1, cur::jx2, cur::jx3 }) { - array(i1, i2, comp) = INV_4 * buffer(i1, i2, comp) + - INV_8 * (buffer(i1 - 1, i2, comp) + - buffer(i1 + 1, i2, comp) + - buffer(i1, i2 - 1, comp) + - buffer(i1, i2 + 1, comp)) + - INV_16 * (buffer(i1 - 1, i2 - 1, comp) + - buffer(i1 + 1, i2 + 1, comp) + - buffer(i1 - 1, i2 + 1, comp) + - buffer(i1 + 1, i2 - 1, comp)); + for (const auto comp : { cur::jx1, cur::jx2, cur::jx3 }) { + array(i1, i2, comp) = INV_4 * buffer(i1, i2, comp) + + INV_8 * (buffer(i1 - 1, i2, comp) + + buffer(i1 + 1, i2, comp) + + buffer(i1, i2 - 1, comp) + + buffer(i1, i2 + 1, comp)) + + INV_16 * (buffer(i1 - 1, i2 - 1, comp) + + buffer(i1 + 1, i2 + 1, comp) + + buffer(i1 - 1, i2 + 1, comp) + + buffer(i1 + 1, i2 - 1, comp)); + } } } else { // spherical + // @TODO: get rid of temporary variables real_t cur_00, cur_0p1, cur_0m1; if (is_axis_i2min && (i2 == i2_min)) { /* --------------------------------- r, phi --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx1, i1, i2); - cur_0p1 = FILTER_IN_I1(buffer, cur::jx1, i1, i2 + 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2); + cur_0p1 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2 + 1); // ... filter in theta array(i1, i2, cur::jx1) = INV_2 * cur_00 + INV_2 * cur_0p1; @@ -88,58 +128,58 @@ namespace kernel { /* ---------------------------------- theta --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx2, i1, i2); - cur_0p1 = FILTER_IN_I1(buffer, cur::jx2, i1, i2 + 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2); + cur_0p1 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2 + 1); // ... filter in theta array(i1, i2, cur::jx2) = INV_4 * (cur_00 + cur_0p1); } else if (is_axis_i2min && (i2 == i2_min + 1)) { /* --------------------------------- r, phi --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx1, i1, i2); - cur_0p1 = FILTER_IN_I1(buffer, cur::jx1, i1, i2 + 1); - cur_0m1 = FILTER_IN_I1(buffer, cur::jx1, i1, i2 - 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2); + cur_0p1 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2 + 1); + cur_0m1 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2 - 1); // ... filter in theta array(i1, i2, cur::jx1) = INV_2 * cur_00 + INV_4 * (cur_0p1 + cur_0m1); // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx3, i1, i2); - cur_0p1 = FILTER_IN_I1(buffer, cur::jx3, i1, i2 + 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx3, i1, i2); + cur_0p1 = FILTER2D_IN_I1(buffer, cur::jx3, i1, i2 + 1); // ... filter in theta array(i1, i2, cur::jx3) = INV_2 * cur_00 + INV_4 * cur_0p1; /* ---------------------------------- theta --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx2, i1, i2); - cur_0p1 = FILTER_IN_I1(buffer, cur::jx2, i1, i2 + 1); - cur_0m1 = FILTER_IN_I1(buffer, cur::jx2, i1, i2 - 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2); + cur_0p1 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2 + 1); + cur_0m1 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2 - 1); // ... filter in theta array(i1, i2, cur::jx2) = INV_2 * cur_00 + INV_4 * (cur_0m1 + cur_0p1); } else if (is_axis_i2max && (i2 == i2_max - 1)) { /* --------------------------------- r, phi --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx1, i1, i2); - cur_0p1 = FILTER_IN_I1(buffer, cur::jx1, i1, i2 + 1); - cur_0m1 = FILTER_IN_I1(buffer, cur::jx1, i1, i2 - 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2); + cur_0p1 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2 + 1); + cur_0m1 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2 - 1); // ... filter in theta array(i1, i2, cur::jx1) = INV_2 * cur_00 + INV_4 * (cur_0m1 + cur_0p1); // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx3, i1, i2); - cur_0m1 = FILTER_IN_I1(buffer, cur::jx3, i1, i2 - 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx3, i1, i2); + cur_0m1 = FILTER2D_IN_I1(buffer, cur::jx3, i1, i2 - 1); // ... filter in theta array(i1, i2, cur::jx3) = INV_2 * cur_00 + INV_4 * cur_0m1; /* ---------------------------------- theta --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx2, i1, i2); - cur_0m1 = FILTER_IN_I1(buffer, cur::jx2, i1, i2 - 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2); + cur_0m1 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2 - 1); // ... filter in theta array(i1, i2, cur::jx2) = INV_4 * (cur_00 + cur_0m1); } else if (is_axis_i2max && (i2 == i2_max)) { /* --------------------------------- r, phi --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx1, i1, i2); - cur_0m1 = FILTER_IN_I1(buffer, cur::jx1, i1, i2 - 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2); + cur_0m1 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2 - 1); // ... filter in theta array(i1, i2, cur::jx1) = INV_2 * cur_00 + INV_2 * cur_0m1; @@ -210,6 +250,6 @@ namespace kernel { } // namespace kernel -#undef FILTER_IN_I1 +#undef FILTER2D_IN_I1 #endif // DIGITAL_FILTER_HPP From b800b615ef1e0c1388ae629856a7e5a86e9531ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 7 Mar 2025 20:26:49 -0600 Subject: [PATCH 055/176] revert to old pgen for testing --- setups/srpic/shocktest/pgen.hpp | 171 +++++++++++++++++++++++--------- 1 file changed, 125 insertions(+), 46 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 4f6decc76..ff54923d1 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -38,20 +38,15 @@ namespace user { // magnetic field components Inline auto bx1(const coord_t& x_ph) const -> real_t { - // return Bmag * math::cos(Btheta); - return ZERO; + return Bmag * math::cos(Btheta); } Inline auto bx2(const coord_t& x_ph) const -> real_t { - // return Bmag * math::sin(Btheta) * math::sin(Bphi); - return ZERO; + return Bmag * math::sin(Btheta) * math::sin(Bphi); } Inline auto bx3(const coord_t& x_ph) const -> real_t { - // return Bmag * math::sin(Btheta) * math::cos(Bphi); - // return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 - // + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); - return ZERO; + return Bmag * math::sin(Btheta) * math::cos(Bphi); } // electric field components @@ -60,15 +55,11 @@ namespace user { } Inline auto ex2(const coord_t& x_ph) const -> real_t { - // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - // return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 - // + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); - return ZERO; + return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); } Inline auto ex3(const coord_t& x_ph) const -> real_t { - // return Vx * Bmag * math::sin(Btheta) * math::sin(Bphi); - return ZERO; + return Vx * Bmag * math::sin(Btheta) * math::sin(Bphi); } private: @@ -102,16 +93,16 @@ namespace user { : arch::ProblemGenerator { p } , drift_ux { p.template get("setup.drift_ux") } , temperature { p.template get("setup.temperature") } - , x1arr_e { p.template get>("setup.x_e") } - , x2arr_e { p.template get>("setup.y_e") } - , ux1arr_e { p.template get>("setup.ux_e") } - , ux2arr_e { p.template get>("setup.uy_e") } - , ux3arr_e { p.template get>("setup.uz_e") } - , x1arr_i { p.template get>("setup.x_i") } - , x2arr_i { p.template get>("setup.y_i") } - , ux1arr_i { p.template get>("setup.ux_i") } - , ux2arr_i { p.template get>("setup.uy_i") } - , ux3arr_i { p.template get>("setup.uz_i") } + // , x1arr_e { p.template get>("setup.x_e") } + // , x2arr_e { p.template get>("setup.y_e") } + // , ux1arr_e { p.template get>("setup.ux_e") } + // , ux2arr_e { p.template get>("setup.uy_e") } + // , ux3arr_e { p.template get>("setup.uz_e") } + // , x1arr_i { p.template get>("setup.x_i") } + // , x2arr_i { p.template get>("setup.y_i") } + // , ux1arr_i { p.template get>("setup.ux_i") } + // , ux2arr_i { p.template get>("setup.uy_i") } + // , ux3arr_i { p.template get>("setup.uz_i") } , Btheta { p.template get("setup.Btheta", ZERO) } , Bmag { p.template get("setup.Bmag", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } @@ -135,28 +126,116 @@ namespace user { return init_flds; } - inline void InitPrtls(Domain& domain) { - arch::InjectGlobally(*metadomain, - domain, - 1, - { - { "x1", x1arr_e }, - { "x2", x2arr_e }, - { "ux1", ux1arr_e }, - { "ux2", ux1arr_e }, - { "ux3", ux3arr_e } - }); - arch::InjectGlobally(*metadomain, - domain, - 2, - { - { "x1", x1arr_i }, - { "x2", x2arr_i }, - { "ux1", ux1arr_i }, - { "ux2", ux1arr_i }, - { "ux3", ux3arr_i } - }); - } + // inline void InitPrtls(Domain& domain) { + // arch::InjectGlobally(*metadomain, + // domain, + // 1, + // { + // { "x1", x1arr_e }, + // { "x2", x2arr_e }, + // { "ux1", ux1arr_e }, + // { "ux2", ux1arr_e }, + // { "ux3", ux3arr_e } + // }); + // arch::InjectGlobally(*metadomain, + // domain, + // 2, + // { + // { "x1", x1arr_i }, + // { "x2", x2arr_i }, + // { "ux1", ux1arr_i }, + // { "ux2", ux1arr_i }, + // { "ux3", ux3arr_i } + // }); + // } + + inline void InitPrtls(Domain& domain) { + + // auto& species_e = domain.species[0]; + // auto& species_p = domain.species[1]; + auto& species_e = domain.species[0]; + auto& species_p = domain.species[1]; + + // array_t elec_ind("elec_ind"); + // array_t pos_ind("pos_ind"); + array_t elec_ind("elec_ind"); + array_t pos_ind("pos_ind"); + + auto offset_e = species_e.npart(); + auto offset_p = species_p.npart(); + + auto ux1_e = species_e.ux1; + auto ux2_e = species_e.ux2; + auto ux3_e = species_e.ux3; + auto i1_e = species_e.i1; + auto i2_e = species_e.i2; + auto dx1_e = species_e.dx1; + auto dx2_e = species_e.dx2; + auto phi_e = species_e.phi; + auto weight_e = species_e.weight; + auto tag_e = species_e.tag; + + auto ux1_p = species_p.ux1; + auto ux2_p = species_p.ux2; + auto ux3_p = species_p.ux3; + auto i1_p = species_p.i1; + auto i2_p = species_p.i2; + auto dx1_p = species_p.dx1; + auto dx2_p = species_p.dx2; + auto phi_p = species_p.phi; + auto weight_p = species_p.weight; + auto tag_p = species_p.tag; + + int nseed = 1; + + Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { + + // ToDo: fix this + auto i1_ = math::floor(10); + auto i2_ = math::floor(64); + auto dx1_ = HALF; + auto dx2_ = HALF; + + + auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); + auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); + + i1_e(elec_p + offset_e) = i1_; + dx1_e(elec_p + offset_e) = dx1_; + i2_e(elec_p + offset_e) = i2_; + dx2_e(elec_p + offset_e) = 2*dx2_; + // ux1_e(elec_p + offset_e) = -0.5; + // ux2_e(elec_p + offset_e) = 0.5; + ux1_e(elec_p + offset_e) = -drift_ux; + ux2_e(elec_p + offset_e) = ZERO; + ux3_e(elec_p + offset_e) = ZERO; + weight_e(elec_p + offset_e) = ONE; + tag_e(elec_p + offset_e) = ParticleTag::alive; + + i1_p(pos_p + offset_p) = i1_; + dx1_p(pos_p + offset_p) = dx1_; + i2_p(pos_p + offset_p) = i2_; + dx2_p(pos_p + offset_p) = -2*dx2_; + // ux1_p(pos_p + offset_p) = 0.5; + // ux2_p(pos_p + offset_p) = -0.5; + ux1_p(pos_p + offset_p) = -drift_ux; + ux2_p(pos_p + offset_p) = ZERO; + ux3_p(pos_p + offset_p) = ZERO; + weight_p(pos_p + offset_p) = ONE; + tag_p(pos_p + offset_p) = ParticleTag::alive; + + + }); + + + auto elec_ind_h = Kokkos::create_mirror(elec_ind); + Kokkos::deep_copy(elec_ind_h, elec_ind); + species_e.set_npart(offset_e + elec_ind_h()); + + auto pos_ind_h = Kokkos::create_mirror(pos_ind); + Kokkos::deep_copy(pos_ind_h, pos_ind); + species_p.set_npart(offset_p + pos_ind_h()); + } }; } // namespace user From 2b269f399782c7fab69c67ee589e4b274f9265f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 7 Mar 2025 21:26:16 -0600 Subject: [PATCH 056/176] enforce B0/E0 at boundary --- src/engines/srpic.hpp | 8 +++++--- src/kernels/fields_bcs.hpp | 33 +++++++++++++++++++++------------ 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index dd2f5ccec..891336309 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -889,11 +889,13 @@ namespace ntt { } else { raise::Error("Invalid dimension", HERE); } + + auto match_fields = m_pgen.MatchFields(time); Kokkos::parallel_for( - "MatchFields", + "ConductorFields", range, - kernel::bc::ConductorBoundaries_kernel(domain.fields.em, - tags)); + kernel::bc::ConductorBoundaries_kernel(domain.fields.em, + match_fields, tags)); // if constexpr (M::Dim == Dim::_1D) { // Kokkos::parallel_for( diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 9d8e9c74e..4d6be9380 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -486,25 +486,32 @@ namespace kernel::bc { } }; - template + template struct ConductorBoundaries_kernel { static_assert(static_cast(o) < static_cast(D), "Invalid component index"); - // static constexpr idx_t i = static_cast(o) + 1u; + static constexpr bool defines_ex2 = traits::has_method::value; + static constexpr bool defines_ex3 = traits::has_method::value; + static constexpr bool defines_bx1 = traits::has_method::value; + static_assert(( defines_ex2 or defines_ex3 or defines_bx1), + "none of the components of E or B are specified in PGEN"); ndfield_t Fld; + const I fset; const BCTags tags; - ConductorBoundaries_kernel(ndfield_t Fld, BCTags tags) + ConductorBoundaries_kernel(ndfield_t Fld, const I& fset, BCTags tags) : Fld { Fld } + , fset { fset } , tags { tags } {} Inline void operator()(index_t i1) const { if constexpr (D == Dim::_1D) { + coord_t x_Ph_H { ZERO }; if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, em::ex2) = ZERO; - Fld(N_GHOSTS, em::ex3) = ZERO; + Fld(N_GHOSTS, em::ex2) = fset.ex2(x_Ph_H); + Fld(N_GHOSTS, em::ex3) = fset.ex3(x_Ph_H); } else { Fld(N_GHOSTS - i1, em::ex1) = Fld(N_GHOSTS + i1 - 1, em::ex1); Fld(N_GHOSTS - i1, em::ex2) = -Fld(N_GHOSTS + i1, em::ex2); @@ -514,7 +521,7 @@ namespace kernel::bc { if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, em::bx1) = ZERO; + Fld(N_GHOSTS, em::bx1) = fset.bx1(x_Ph_H); } else { Fld(N_GHOSTS - i1, em::bx1) = -Fld(N_GHOSTS + i1, em::bx1); Fld(N_GHOSTS - i1, em::bx2) = Fld(N_GHOSTS + i1 - 1, em::bx2); @@ -530,10 +537,11 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2) const { if constexpr (D == Dim::_2D) { + coord_t x_Ph_H { ZERO }; if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, i2, em::ex2) = ZERO; - Fld(N_GHOSTS, i2, em::ex3) = ZERO; + Fld(N_GHOSTS, i2, em::ex2) = fset.ex2(x_Ph_H); + Fld(N_GHOSTS, i2, em::ex3) = fset.ex3(x_Ph_H); } else { Fld(N_GHOSTS - i1, i2, em::ex1) = Fld(N_GHOSTS + i1 - 1, i2, em::ex1); Fld(N_GHOSTS - i1, i2, em::ex2) = -Fld(N_GHOSTS + i1, i2, em::ex2); @@ -543,7 +551,7 @@ namespace kernel::bc { if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, i2, em::bx1) = ZERO; + Fld(N_GHOSTS, i2, em::bx1) = fset.bx1(x_Ph_H); } else { Fld(N_GHOSTS - i1, i2, em::bx1) = -Fld(N_GHOSTS + i1, i2, em::bx1); Fld(N_GHOSTS - i1, i2, em::bx2) = Fld(N_GHOSTS + i1 - 1, i2, em::bx2); @@ -559,10 +567,11 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2, index_t i3) const { if constexpr (D == Dim::_3D) { + coord_t x_Ph_H { ZERO }; if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, i2, i3, em::ex2) = ZERO; - Fld(N_GHOSTS, i2, i3, em::ex3) = ZERO; + Fld(N_GHOSTS, i2, i3, em::ex2) = fset.ex2(x_Ph_H); + Fld(N_GHOSTS, i2, i3, em::ex3) = fset.ex3(x_Ph_H); } else { Fld(N_GHOSTS - i1, i2, i3, em::ex1) = Fld(N_GHOSTS + i1 - 1, i2, @@ -575,7 +584,7 @@ namespace kernel::bc { if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, i2, i3, em::bx1) = ZERO; + Fld(N_GHOSTS, i2, i3, em::bx1) = fset.bx1(x_Ph_H); } else { Fld(N_GHOSTS - i1, i2, i3, em::bx1) = -Fld(N_GHOSTS + i1, i2, i3, em::bx1); Fld(N_GHOSTS - i1, i2, i3, em::bx2) = Fld(N_GHOSTS + i1 - 1, From 9b237323a2f43fea4f1ea1c25fe3d1466f8d3415 Mon Sep 17 00:00:00 2001 From: hayk Date: Sat, 8 Mar 2025 16:16:31 -0500 Subject: [PATCH 057/176] filter test fixed --- src/kernels/tests/digital_filter.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/kernels/tests/digital_filter.cpp b/src/kernels/tests/digital_filter.cpp index e0cc352f5..03a63753b 100644 --- a/src/kernels/tests/digital_filter.cpp +++ b/src/kernels/tests/digital_filter.cpp @@ -40,8 +40,13 @@ void testFilter(const std::vector& res, auto boundaries = boundaries_t {}; if constexpr (M::CoordType != Coord::Cart) { boundaries = { - {FldsBC::CUSTOM, FldsBC::CUSTOM}, - { FldsBC::AXIS, FldsBC::AXIS} + { FldsBC::CUSTOM, FldsBC::CUSTOM }, + { FldsBC::AXIS, FldsBC::AXIS } + }; + } else { + boundaries = { + { FldsBC::PERIODIC, FldsBC::PERIODIC }, + { FldsBC::PERIODIC, FldsBC::PERIODIC } }; } @@ -183,4 +188,4 @@ auto main(int argc, char* argv[]) -> int { } Kokkos::finalize(); return 0; -} \ No newline at end of file +} From 179fafee756f0a921e84f62f53e2c0ef27aaff75 Mon Sep 17 00:00:00 2001 From: hayk Date: Sun, 9 Mar 2025 01:59:57 -0500 Subject: [PATCH 058/176] removed extra kokkos flags --- cmake/kokkosConfig.cmake | 35 ----------------- legacy/tests/kernels-sr.cpp | 10 ++--- src/framework/containers/particles.cpp | 4 +- src/framework/domain/comm_mpi.hpp | 12 +++--- src/framework/domain/comm_nompi.hpp | 6 +-- src/global/arch/kokkos_aliases.cpp | 54 +++++++++++++------------- src/global/arch/kokkos_aliases.h | 18 ++++----- src/kernels/tests/prtls_to_phys.cpp | 45 ++++++++++----------- 8 files changed, 76 insertions(+), 108 deletions(-) diff --git a/cmake/kokkosConfig.cmake b/cmake/kokkosConfig.cmake index 63c32622d..8800f21a0 100644 --- a/cmake/kokkosConfig.cmake +++ b/cmake/kokkosConfig.cmake @@ -37,41 +37,6 @@ set(Kokkos_ENABLE_OPENMP ${default_KOKKOS_ENABLE_OPENMP} CACHE BOOL "Enable OpenMP") -# set memory space -if(${Kokkos_ENABLE_CUDA}) - add_compile_definitions(CUDA_ENABLED) - set(ACC_MEM_SPACE Kokkos::CudaSpace) -elseif(${Kokkos_ENABLE_HIP}) - add_compile_definitions(HIP_ENABLED) - set(ACC_MEM_SPACE Kokkos::HIPSpace) -else() - set(ACC_MEM_SPACE Kokkos::HostSpace) -endif() - -set(HOST_MEM_SPACE Kokkos::HostSpace) - -# set execution space -if(${Kokkos_ENABLE_CUDA}) - set(ACC_EXE_SPACE Kokkos::Cuda) -elseif(${Kokkos_ENABLE_HIP}) - set(ACC_EXE_SPACE Kokkos::HIP) -elseif(${Kokkos_ENABLE_OPENMP}) - set(ACC_EXE_SPACE Kokkos::OpenMP) -else() - set(ACC_EXE_SPACE Kokkos::Serial) -endif() - -if(${Kokkos_ENABLE_OPENMP}) - set(HOST_EXE_SPACE Kokkos::OpenMP) -else() - set(HOST_EXE_SPACE Kokkos::Serial) -endif() - -add_compile_options("-D AccelExeSpace=${ACC_EXE_SPACE}") -add_compile_options("-D AccelMemSpace=${ACC_MEM_SPACE}") -add_compile_options("-D HostExeSpace=${HOST_EXE_SPACE}") -add_compile_options("-D HostMemSpace=${HOST_MEM_SPACE}") - if(${BUILD_TESTING} STREQUAL "OFF") set(Kokkos_ENABLE_TESTS OFF diff --git a/legacy/tests/kernels-sr.cpp b/legacy/tests/kernels-sr.cpp index 59ce0646b..d765799e3 100644 --- a/legacy/tests/kernels-sr.cpp +++ b/legacy/tests/kernels-sr.cpp @@ -1,17 +1,17 @@ -#include "wrapper.h" - #include #include #include +#include "wrapper.h" + #include METRIC_HEADER #include PGEN_HEADER -#include "particle_macros.h" - #include "kernels/particle_pusher_sr.hpp" +#include "particle_macros.h" + template void put_value(ntt::array_t& arr, T value, int i) { auto arr_h = Kokkos::create_mirror_view(arr); @@ -221,4 +221,4 @@ auto main(int argc, char* argv[]) -> int { ntt::GlobalFinalize(); return 0; -} \ No newline at end of file +} diff --git a/src/framework/containers/particles.cpp b/src/framework/containers/particles.cpp index d78055824..d165f6233 100644 --- a/src/framework/containers/particles.cpp +++ b/src/framework/containers/particles.cpp @@ -220,14 +220,14 @@ namespace ntt { Kokkos::Experimental::fill( "TagAliveParticles", - AccelExeSpace(), + Kokkos::DefaultExecutionSpace(), Kokkos::subview(this_tag, std::make_pair(static_cast(0), n_alive)), ParticleTag::alive); Kokkos::Experimental::fill( "TagDeadParticles", - AccelExeSpace(), + Kokkos::DefaultExecutionSpace(), Kokkos::subview(this_tag, std::make_pair(n_alive, n_alive + n_dead)), ParticleTag::dead); diff --git a/src/framework/domain/comm_mpi.hpp b/src/framework/domain/comm_mpi.hpp index e5bc2d21e..d4f8b5651 100644 --- a/src/framework/domain/comm_mpi.hpp +++ b/src/framework/domain/comm_mpi.hpp @@ -83,7 +83,7 @@ namespace comm { (long int)(send_slice[0].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, comps.first }, { recv_slice[0].second, comps.second }), Lambda(index_t i1, index_t ci) { @@ -96,7 +96,7 @@ namespace comm { (long int)(send_slice[1].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, comps.first }, { recv_slice[0].second, recv_slice[1].second, comps.second }), Lambda(index_t i1, index_t i2, index_t ci) { @@ -111,7 +111,7 @@ namespace comm { (long int)(send_slice[2].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, recv_slice[2].first, @@ -239,7 +239,7 @@ namespace comm { const auto offset_c = comps.first; Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, comps.first }, { recv_slice[0].second, comps.second }), Lambda(index_t i1, index_t ci) { @@ -251,7 +251,7 @@ namespace comm { const auto offset_c = comps.first; Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, comps.first }, { recv_slice[0].second, recv_slice[1].second, comps.second }), Lambda(index_t i1, index_t i2, index_t ci) { @@ -266,7 +266,7 @@ namespace comm { const auto offset_c = comps.first; Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, recv_slice[2].first, diff --git a/src/framework/domain/comm_nompi.hpp b/src/framework/domain/comm_nompi.hpp index 197d336fa..b477ac176 100644 --- a/src/framework/domain/comm_nompi.hpp +++ b/src/framework/domain/comm_nompi.hpp @@ -70,7 +70,7 @@ namespace comm { (long int)(send_slice[0].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, comps.first }, { recv_slice[0].second, comps.second }), Lambda(index_t i1, index_t ci) { @@ -83,7 +83,7 @@ namespace comm { (long int)(send_slice[1].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, comps.first }, { recv_slice[0].second, recv_slice[1].second, comps.second }), Lambda(index_t i1, index_t i2, index_t ci) { @@ -98,7 +98,7 @@ namespace comm { (long int)(send_slice[2].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, recv_slice[2].first, diff --git a/src/global/arch/kokkos_aliases.cpp b/src/global/arch/kokkos_aliases.cpp index 6c15e3d52..73bc364d1 100644 --- a/src/global/arch/kokkos_aliases.cpp +++ b/src/global/arch/kokkos_aliases.cpp @@ -5,73 +5,75 @@ #include template <> -auto CreateRangePolicy( - const tuple_t& i1, - const tuple_t& i2) -> range_t { +auto CreateRangePolicy(const tuple_t& i1, + const tuple_t& i2) + -> range_t { index_t i1min = i1[0]; index_t i1max = i2[0]; - return Kokkos::RangePolicy(i1min, i1max); + return Kokkos::RangePolicy(i1min, i1max); } template <> -auto CreateRangePolicy( - const tuple_t& i1, - const tuple_t& i2) -> range_t { +auto CreateRangePolicy(const tuple_t& i1, + const tuple_t& i2) + -> range_t { index_t i1min = i1[0]; index_t i1max = i2[0]; index_t i2min = i1[1]; index_t i2max = i2[1]; - return Kokkos::MDRangePolicy, AccelExeSpace>({ i1min, i2min }, - { i1max, i2max }); + return Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( + { i1min, i2min }, + { i1max, i2max }); } template <> -auto CreateRangePolicy( - const tuple_t& i1, - const tuple_t& i2) -> range_t { +auto CreateRangePolicy(const tuple_t& i1, + const tuple_t& i2) + -> range_t { index_t i1min = i1[0]; index_t i1max = i2[0]; index_t i2min = i1[1]; index_t i2max = i2[1]; index_t i3min = i1[2]; index_t i3max = i2[2]; - return Kokkos::MDRangePolicy, AccelExeSpace>( + return Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { i1min, i2min, i3min }, { i1max, i2max, i3max }); } template <> -auto CreateRangePolicyOnHost( - const tuple_t& i1, - const tuple_t& i2) -> range_h_t { +auto CreateRangePolicyOnHost(const tuple_t& i1, + const tuple_t& i2) + -> range_h_t { index_t i1min = i1[0]; index_t i1max = i2[0]; - return Kokkos::RangePolicy(i1min, i1max); + return Kokkos::RangePolicy(i1min, i1max); } template <> -auto CreateRangePolicyOnHost( - const tuple_t& i1, - const tuple_t& i2) -> range_h_t { +auto CreateRangePolicyOnHost(const tuple_t& i1, + const tuple_t& i2) + -> range_h_t { index_t i1min = i1[0]; index_t i1max = i2[0]; index_t i2min = i1[1]; index_t i2max = i2[1]; - return Kokkos::MDRangePolicy, HostExeSpace>({ i1min, i2min }, - { i1max, i2max }); + return Kokkos::MDRangePolicy, Kokkos::DefaultHostExecutionSpace>( + { i1min, i2min }, + { i1max, i2max }); } template <> -auto CreateRangePolicyOnHost( - const tuple_t& i1, - const tuple_t& i2) -> range_h_t { +auto CreateRangePolicyOnHost(const tuple_t& i1, + const tuple_t& i2) + -> range_h_t { index_t i1min = i1[0]; index_t i1max = i2[0]; index_t i2min = i1[1]; index_t i2max = i2[1]; index_t i3min = i1[2]; index_t i3max = i2[2]; - return Kokkos::MDRangePolicy, HostExeSpace>( + return Kokkos::MDRangePolicy, Kokkos::DefaultHostExecutionSpace>( { i1min, i2min, i3min }, { i1max, i2max, i3max }); } diff --git a/src/global/arch/kokkos_aliases.h b/src/global/arch/kokkos_aliases.h index f9aac9685..c20738f11 100644 --- a/src/global/arch/kokkos_aliases.h +++ b/src/global/arch/kokkos_aliases.h @@ -34,7 +34,7 @@ namespace math = Kokkos; template -using array_t = Kokkos::View; +using array_t = Kokkos::View; // Array mirror alias of arbitrary type template @@ -174,17 +174,17 @@ namespace kokkos_aliases_hidden { template <> struct range_impl { - using type = Kokkos::RangePolicy; + using type = Kokkos::RangePolicy; }; template <> struct range_impl { - using type = Kokkos::MDRangePolicy, AccelExeSpace>; + using type = Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>; }; template <> struct range_impl { - using type = Kokkos::MDRangePolicy, AccelExeSpace>; + using type = Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>; }; } // namespace kokkos_aliases_hidden @@ -201,17 +201,17 @@ namespace kokkos_aliases_hidden { template <> struct range_h_impl { - using type = Kokkos::RangePolicy; + using type = Kokkos::RangePolicy; }; template <> struct range_h_impl { - using type = Kokkos::MDRangePolicy, HostExeSpace>; + using type = Kokkos::MDRangePolicy, Kokkos::DefaultHostExecutionSpace>; }; template <> struct range_h_impl { - using type = Kokkos::MDRangePolicy, HostExeSpace>; + using type = Kokkos::MDRangePolicy, Kokkos::DefaultHostExecutionSpace>; }; } // namespace kokkos_aliases_hidden @@ -242,8 +242,8 @@ auto CreateRangePolicyOnHost(const tuple_t&, const tuple_t&) -> range_h_t; // Random number pool/generator type alias -using random_number_pool_t = Kokkos::Random_XorShift1024_Pool; -using random_generator_t = typename random_number_pool_t::generator_type; +using random_number_pool_t = Kokkos::Random_XorShift1024_Pool; +using random_generator_t = typename random_number_pool_t::generator_type; // Random number generator functions template diff --git a/src/kernels/tests/prtls_to_phys.cpp b/src/kernels/tests/prtls_to_phys.cpp index 4719fe6a1..0ceb88cd1 100644 --- a/src/kernels/tests/prtls_to_phys.cpp +++ b/src/kernels/tests/prtls_to_phys.cpp @@ -177,28 +177,29 @@ void testPrtl2PhysSR(const std::vector& res, array_t buff_ux3 { "buff_ux3", nprtl / stride }; array_t buff_wei { "buff_wei", nprtl / stride }; - Kokkos::parallel_for("Init", - Kokkos::RangePolicy(0, nprtl / stride), - kernel::PrtlToPhys_kernel(stride, - buff_x1, - buff_x2, - buff_x3, - buff_ux1, - buff_ux2, - buff_ux3, - buff_wei, - i1, - i2, - i3, - dx1, - dx2, - dx3, - ux1, - ux2, - ux3, - phi, - weight, - metric)); + Kokkos::parallel_for( + "Init", + Kokkos::RangePolicy(0, nprtl / stride), + kernel::PrtlToPhys_kernel(stride, + buff_x1, + buff_x2, + buff_x3, + buff_ux1, + buff_ux2, + buff_ux3, + buff_wei, + i1, + i2, + i3, + dx1, + dx2, + dx3, + ux1, + ux2, + ux3, + phi, + weight, + metric)); Kokkos::parallel_for("Check", nprtl / stride, Checker(metric, From b6fe114ab719e45d58ebb6a4241cac3407df5040 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 10 Mar 2025 12:00:38 -0500 Subject: [PATCH 059/176] revert to zero at boundary --- src/engines/srpic.hpp | 28 +----------------------- src/kernels/fields_bcs.hpp | 44 ++++++++++++-------------------------- 2 files changed, 15 insertions(+), 57 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 891336309..96747e429 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -890,36 +890,10 @@ namespace ntt { raise::Error("Invalid dimension", HERE); } - auto match_fields = m_pgen.MatchFields(time); Kokkos::parallel_for( "ConductorFields", range, - kernel::bc::ConductorBoundaries_kernel(domain.fields.em, - match_fields, tags)); - - // if constexpr (M::Dim == Dim::_1D) { - // Kokkos::parallel_for( - // "MatchFields", - // CreateRangePolicy({ xi_min[0] }, { xi_max[0] }), - // kernel::bc::ConductorBoundaries_kernel(domain.fields.em, - // tags)); - // } else if constexpr (M::Dim == Dim::_2D) { - // Kokkos::parallel_for( - // "MatchFields", - // CreateRangePolicy({ xi_min[0], xi_min[1] }, - // { xi_max[0], xi_max[1] }), - // kernel::bc::ConductorBoundaries_kernel(domain.fields.em, - // tags)); - // } else if constexpr (M::Dim == Dim::_3D) { - // Kokkos::parallel_for( - // "MatchFields", - // CreateRangePolicy({ xi_min[0], xi_min[1], xi_min[2] }, - // { xi_max[0], xi_max[1], xi_max[2] }), - // kernel::bc::ConductorBoundaries_kernel(domain.fields.em, - // tags)); - // } else { - // raise::Error("Invalid dimension", HERE); - // } + kernel::bc::ConductorBoundaries_kernel(domain.fields.em, tags)); } } diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 4d6be9380..b1ee999d3 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -486,32 +486,24 @@ namespace kernel::bc { } }; - template + template struct ConductorBoundaries_kernel { static_assert(static_cast(o) < static_cast(D), "Invalid component index"); - static constexpr bool defines_ex2 = traits::has_method::value; - static constexpr bool defines_ex3 = traits::has_method::value; - static constexpr bool defines_bx1 = traits::has_method::value; - static_assert(( defines_ex2 or defines_ex3 or defines_bx1), - "none of the components of E or B are specified in PGEN"); ndfield_t Fld; - const I fset; const BCTags tags; - ConductorBoundaries_kernel(ndfield_t Fld, const I& fset, BCTags tags) + ConductorBoundaries_kernel(ndfield_t Fld, BCTags tags) : Fld { Fld } - , fset { fset } , tags { tags } {} Inline void operator()(index_t i1) const { if constexpr (D == Dim::_1D) { - coord_t x_Ph_H { ZERO }; if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, em::ex2) = fset.ex2(x_Ph_H); - Fld(N_GHOSTS, em::ex3) = fset.ex3(x_Ph_H); + Fld(N_GHOSTS, em::ex2) = ZERO; + Fld(N_GHOSTS, em::ex3) = ZERO; } else { Fld(N_GHOSTS - i1, em::ex1) = Fld(N_GHOSTS + i1 - 1, em::ex1); Fld(N_GHOSTS - i1, em::ex2) = -Fld(N_GHOSTS + i1, em::ex2); @@ -521,7 +513,7 @@ namespace kernel::bc { if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, em::bx1) = fset.bx1(x_Ph_H); + Fld(N_GHOSTS, em::bx1) = ZERO; } else { Fld(N_GHOSTS - i1, em::bx1) = -Fld(N_GHOSTS + i1, em::bx1); Fld(N_GHOSTS - i1, em::bx2) = Fld(N_GHOSTS + i1 - 1, em::bx2); @@ -537,11 +529,10 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2) const { if constexpr (D == Dim::_2D) { - coord_t x_Ph_H { ZERO }; if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, i2, em::ex2) = fset.ex2(x_Ph_H); - Fld(N_GHOSTS, i2, em::ex3) = fset.ex3(x_Ph_H); + Fld(N_GHOSTS, i2, em::ex2) = ZERO; + Fld(N_GHOSTS, i2, em::ex3) = ZERO; } else { Fld(N_GHOSTS - i1, i2, em::ex1) = Fld(N_GHOSTS + i1 - 1, i2, em::ex1); Fld(N_GHOSTS - i1, i2, em::ex2) = -Fld(N_GHOSTS + i1, i2, em::ex2); @@ -551,7 +542,7 @@ namespace kernel::bc { if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, i2, em::bx1) = fset.bx1(x_Ph_H); + Fld(N_GHOSTS, i2, em::bx1) = ZERO; } else { Fld(N_GHOSTS - i1, i2, em::bx1) = -Fld(N_GHOSTS + i1, i2, em::bx1); Fld(N_GHOSTS - i1, i2, em::bx2) = Fld(N_GHOSTS + i1 - 1, i2, em::bx2); @@ -567,16 +558,13 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2, index_t i3) const { if constexpr (D == Dim::_3D) { - coord_t x_Ph_H { ZERO }; if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, i2, i3, em::ex2) = fset.ex2(x_Ph_H); - Fld(N_GHOSTS, i2, i3, em::ex3) = fset.ex3(x_Ph_H); + Fld(N_GHOSTS, i2, i3, em::ex2) = ZERO; + Fld(N_GHOSTS, i2, i3, em::ex3) = ZERO; } else { Fld(N_GHOSTS - i1, i2, i3, em::ex1) = Fld(N_GHOSTS + i1 - 1, - i2, - i3, - em::ex1); + i2, i3, em::ex1); Fld(N_GHOSTS - i1, i2, i3, em::ex2) = -Fld(N_GHOSTS + i1, i2, i3, em::ex2); Fld(N_GHOSTS - i1, i2, i3, em::ex3) = -Fld(N_GHOSTS + i1, i2, i3, em::ex3); } @@ -584,17 +572,13 @@ namespace kernel::bc { if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, i2, i3, em::bx1) = fset.bx1(x_Ph_H); + Fld(N_GHOSTS, i2, i3, em::bx1) = ZERO; } else { Fld(N_GHOSTS - i1, i2, i3, em::bx1) = -Fld(N_GHOSTS + i1, i2, i3, em::bx1); Fld(N_GHOSTS - i1, i2, i3, em::bx2) = Fld(N_GHOSTS + i1 - 1, - i2, - i3, - em::bx2); + i2, i3, em::bx2); Fld(N_GHOSTS - i1, i2, i3, em::bx3) = Fld(N_GHOSTS + i1 - 1, - i2, - i3, - em::bx3); + i2, i3, em::bx3); } } } else { From 777d1fbff3cf66a7b0e5e49260cb1acb954d5761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Thu, 13 Mar 2025 10:00:56 -0500 Subject: [PATCH 060/176] cleanup of pgen --- setups/srpic/shock/pgen.hpp | 46 ------------------------------------- 1 file changed, 46 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 59c5590c9..b8f169521 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -106,52 +106,6 @@ namespace user { } } - - auto PerfectConductorFieldsConst(const bc_in&, const em& comp) const -> real_t{ - - // electric field components - if (comp == em::ex1) { - return ONE; - } else if (comp == em::ex2) { - return -ONE; - } else if (comp == em::ex3) { - return -ONE; } - // magentic field components - else if (comp == em::bx1) { - return -ONE; - } else if (comp == em::bx2) { - return ONE; - } else if (comp == em::bx3) { - return ONE;} - // should never be the case - else - { - return ZERO; - } - } - - // auto PerfectConductorCurrentsConst(const bc_in &, const cur &comp) const - // -> std::pair - // { - // // ToDo - // if (comp == cur::jx1) - // { - // return ZERO; - // } - // else if (comp == cur::jx2) - // { - // return ZERO; - // } - // else if (comp == cur::jx3) - // { - // return ZERO; - // } - // else - // { - // return ZERO; - // } - // } - auto MatchFields(real_t time) const -> InitFields { return init_flds; } From ef5c85409d3422f1cb71337166c719f8b5e76ed2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Thu, 13 Mar 2025 11:03:56 -0500 Subject: [PATCH 061/176] update example parameter file --- setups/srpic/shock/shock.toml | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/setups/srpic/shock/shock.toml b/setups/srpic/shock/shock.toml index 4ed3a2b9e..4e03721f7 100644 --- a/setups/srpic/shock/shock.toml +++ b/setups/srpic/shock/shock.toml @@ -1,22 +1,22 @@ [simulation] name = "shock" engine = "srpic" - runtime = 50.0 + runtime = 30000.0 [grid] - resolution = [2048, 128] - extent = [[0.0, 10.0], [-0.3125, 0.3125]] + resolution = [16384, 128] + extent = [[0.0, 8000.0], [-31.25, 31.25]] [grid.metric] metric = "minkowski" [grid.boundaries] - fields = [["ABSORB", "FIXED"], ["PERIODIC"]] + fields = [["CONDUCTOR", "MATCH"], ["PERIODIC"]] particles = [["REFLECT", "ABSORB"], ["PERIODIC"]] - + [scales] - larmor0 = 1e-2 - skindepth0 = 1e-2 + larmor0 = 100.0 + skindepth0 = 1.0 [algorithms] current_filters = 8 @@ -25,7 +25,7 @@ CFL = 0.5 [particles] - ppc0 = 16.0 + ppc0 = 8.0 [[particles.species]] label = "e-" @@ -34,21 +34,29 @@ maxnpart = 1e8 [[particles.species]] - label = "e+" + label = "p+" mass = 1.0 charge = 1.0 maxnpart = 1e8 [setup] drift_ux = 0.1 - temperature = 1e-3 + temperature = 1e-4 Bmag = 1.0 Btheta = 0.0 Bphi = 0.0 [output] - interval_time = 0.1 + interval_time = 500.0 format = "hdf5" - + [output.fields] - quantities = ["N_1", "N_2", "E", "B", "T0i_1", "T0i_2", "J"] + quantities = ["N_1", "N_2", "B", "E"] + + [output.particles] + enable = true + stride = 10 + + [output.spectra] + enable = false + From 84e047c0f741d3886cc7fcdf94c1addc36d6e0aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Wed, 19 Mar 2025 15:40:05 -0500 Subject: [PATCH 062/176] revert to cleaner injector for shocktest --- setups/srpic/shocktest/pgen.hpp | 152 +++++++------------------------- 1 file changed, 32 insertions(+), 120 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index ff54923d1..a7bcf5c3d 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -93,16 +93,16 @@ namespace user { : arch::ProblemGenerator { p } , drift_ux { p.template get("setup.drift_ux") } , temperature { p.template get("setup.temperature") } - // , x1arr_e { p.template get>("setup.x_e") } - // , x2arr_e { p.template get>("setup.y_e") } - // , ux1arr_e { p.template get>("setup.ux_e") } - // , ux2arr_e { p.template get>("setup.uy_e") } - // , ux3arr_e { p.template get>("setup.uz_e") } - // , x1arr_i { p.template get>("setup.x_i") } - // , x2arr_i { p.template get>("setup.y_i") } - // , ux1arr_i { p.template get>("setup.ux_i") } - // , ux2arr_i { p.template get>("setup.uy_i") } - // , ux3arr_i { p.template get>("setup.uz_i") } + , x1arr_e { p.template get>("setup.x_e") } + , x2arr_e { p.template get>("setup.y_e") } + , ux1arr_e { p.template get>("setup.ux_e") } + , ux2arr_e { p.template get>("setup.uy_e") } + , ux3arr_e { p.template get>("setup.uz_e") } + , x1arr_i { p.template get>("setup.x_i") } + , x2arr_i { p.template get>("setup.y_i") } + , ux1arr_i { p.template get>("setup.ux_i") } + , ux2arr_i { p.template get>("setup.uy_i") } + , ux3arr_i { p.template get>("setup.uz_i") } , Btheta { p.template get("setup.Btheta", ZERO) } , Bmag { p.template get("setup.Bmag", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } @@ -126,116 +126,28 @@ namespace user { return init_flds; } - // inline void InitPrtls(Domain& domain) { - // arch::InjectGlobally(*metadomain, - // domain, - // 1, - // { - // { "x1", x1arr_e }, - // { "x2", x2arr_e }, - // { "ux1", ux1arr_e }, - // { "ux2", ux1arr_e }, - // { "ux3", ux3arr_e } - // }); - // arch::InjectGlobally(*metadomain, - // domain, - // 2, - // { - // { "x1", x1arr_i }, - // { "x2", x2arr_i }, - // { "ux1", ux1arr_i }, - // { "ux2", ux1arr_i }, - // { "ux3", ux3arr_i } - // }); - // } - - inline void InitPrtls(Domain& domain) { - - // auto& species_e = domain.species[0]; - // auto& species_p = domain.species[1]; - auto& species_e = domain.species[0]; - auto& species_p = domain.species[1]; - - // array_t elec_ind("elec_ind"); - // array_t pos_ind("pos_ind"); - array_t elec_ind("elec_ind"); - array_t pos_ind("pos_ind"); - - auto offset_e = species_e.npart(); - auto offset_p = species_p.npart(); - - auto ux1_e = species_e.ux1; - auto ux2_e = species_e.ux2; - auto ux3_e = species_e.ux3; - auto i1_e = species_e.i1; - auto i2_e = species_e.i2; - auto dx1_e = species_e.dx1; - auto dx2_e = species_e.dx2; - auto phi_e = species_e.phi; - auto weight_e = species_e.weight; - auto tag_e = species_e.tag; - - auto ux1_p = species_p.ux1; - auto ux2_p = species_p.ux2; - auto ux3_p = species_p.ux3; - auto i1_p = species_p.i1; - auto i2_p = species_p.i2; - auto dx1_p = species_p.dx1; - auto dx2_p = species_p.dx2; - auto phi_p = species_p.phi; - auto weight_p = species_p.weight; - auto tag_p = species_p.tag; - - int nseed = 1; - - Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { - - // ToDo: fix this - auto i1_ = math::floor(10); - auto i2_ = math::floor(64); - auto dx1_ = HALF; - auto dx2_ = HALF; - - - auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); - auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); - - i1_e(elec_p + offset_e) = i1_; - dx1_e(elec_p + offset_e) = dx1_; - i2_e(elec_p + offset_e) = i2_; - dx2_e(elec_p + offset_e) = 2*dx2_; - // ux1_e(elec_p + offset_e) = -0.5; - // ux2_e(elec_p + offset_e) = 0.5; - ux1_e(elec_p + offset_e) = -drift_ux; - ux2_e(elec_p + offset_e) = ZERO; - ux3_e(elec_p + offset_e) = ZERO; - weight_e(elec_p + offset_e) = ONE; - tag_e(elec_p + offset_e) = ParticleTag::alive; - - i1_p(pos_p + offset_p) = i1_; - dx1_p(pos_p + offset_p) = dx1_; - i2_p(pos_p + offset_p) = i2_; - dx2_p(pos_p + offset_p) = -2*dx2_; - // ux1_p(pos_p + offset_p) = 0.5; - // ux2_p(pos_p + offset_p) = -0.5; - ux1_p(pos_p + offset_p) = -drift_ux; - ux2_p(pos_p + offset_p) = ZERO; - ux3_p(pos_p + offset_p) = ZERO; - weight_p(pos_p + offset_p) = ONE; - tag_p(pos_p + offset_p) = ParticleTag::alive; - - - }); - - - auto elec_ind_h = Kokkos::create_mirror(elec_ind); - Kokkos::deep_copy(elec_ind_h, elec_ind); - species_e.set_npart(offset_e + elec_ind_h()); - - auto pos_ind_h = Kokkos::create_mirror(pos_ind); - Kokkos::deep_copy(pos_ind_h, pos_ind); - species_p.set_npart(offset_p + pos_ind_h()); - } + inline void InitPrtls(Domain& domain) { + arch::InjectGlobally(*metadomain, + domain, + 1, + { + { "x1", x1arr_e }, + { "x2", x2arr_e }, + { "ux1", ux1arr_e }, + { "ux2", ux1arr_e }, + { "ux3", ux3arr_e } + }); + arch::InjectGlobally(*metadomain, + domain, + 2, + { + { "x1", x1arr_i }, + { "x2", x2arr_i }, + { "ux1", ux1arr_i }, + { "ux2", ux1arr_i }, + { "ux3", ux3arr_i } + }); + } }; } // namespace user From 69d4a0d180f3ae3b18600981e87a49b66929c495 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Thu, 20 Mar 2025 17:06:25 -0500 Subject: [PATCH 063/176] implemented new `arch::Piston` spatial distribution to set up shock partially filled with plasma --- setups/srpic/shock/pgen.hpp | 59 ++++++++++++++++++++++------------- setups/srpic/shock/shock.toml | 11 ++++--- src/archetypes/spatial_dist.h | 30 ++++++++++++++++++ 3 files changed, 74 insertions(+), 26 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index b8f169521..9bf51b866 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -79,7 +79,7 @@ namespace user { using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; - const real_t drift_ux, temperature; + const real_t drift_ux, temperature, filling_fraction; const real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -91,40 +91,57 @@ namespace user { , Bmag { p.template get("setup.Bmag", ZERO) } , Btheta { p.template get("setup.Btheta", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } - , init_flds { Bmag, Btheta, Bphi, drift_ux } {} + , init_flds { Bmag, Btheta, Bphi, drift_ux } + , filling_fraction { params.template get("setup.filling_fraction", 1.0) }{} inline PGen() {} - auto FixFieldsConst(const bc_in&, const em& comp) const - -> std::pair { - if (comp == em::ex2) { - return { init_flds.ex2({ ZERO }), true }; - } else if (comp == em::ex3) { - return { init_flds.ex3({ ZERO }), true }; - } else { - return { ZERO, false }; - } - } - auto MatchFields(real_t time) const -> InitFields { return init_flds; } inline void InitPrtls(Domain& local_domain) { + + // minimum and maximum position of particles + real_t xg_min = local_domain.mesh.extent(in::x1).first; + real_t xg_max = local_domain.mesh.extent(in::x1).second * filling_fraction; + + // define box to inject into + boundaries_t box; + // loop over all dimensions + for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { + // compute the range for the x-direction + if (d == static_cast(in::x1)) { + box.push_back({xg_min, xg_max}); + } else { + // inject into full range in other directions + box.push_back(Range::All); + } + } + + // spatial distribution of the particles + // -> hack to use the uniform distribution in NonUniformInjector + const auto spatial_dist = arch::Piston(local_domain.mesh.metric, xg_min, xg_max, in::x1); + + // energy distribution of the particles const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, local_domain.random_pool, temperature, -drift_ux, in::x1); - const auto injector = arch::UniformInjector( - energy_dist, - { 1, 2 }); - arch::InjectUniform>( - params, - local_domain, - injector, - 1.0); + const auto injector = arch::NonUniformInjector( + energy_dist, + spatial_dist, + {1, 2}); + + arch::InjectNonUniform>( + params, + local_domain, + injector, + 1.0, // target density + false, // no weights + box); } }; diff --git a/setups/srpic/shock/shock.toml b/setups/srpic/shock/shock.toml index 4e03721f7..f954345c2 100644 --- a/setups/srpic/shock/shock.toml +++ b/setups/srpic/shock/shock.toml @@ -40,11 +40,12 @@ maxnpart = 1e8 [setup] - drift_ux = 0.1 - temperature = 1e-4 - Bmag = 1.0 - Btheta = 0.0 - Bphi = 0.0 + drift_ux = 0.1 # speed towards the wall [c] + temperature = 1e-4 # temeperature of maxwell distribution [m_e c^2] + Bmag = 1.0 # magnetic field strength as fraction of magnetisation + Btheta = 0.0 # magnetic field angle in the plane + Bphi = 0.0 # magnetic field angle out of plane + filling_fraction = 1.0 # fraction of the shock piston filled with plasma [output] interval_time = 500.0 diff --git a/src/archetypes/spatial_dist.h b/src/archetypes/spatial_dist.h index be2836da2..098e1c5bb 100644 --- a/src/archetypes/spatial_dist.h +++ b/src/archetypes/spatial_dist.h @@ -4,6 +4,7 @@ * @implements * - arch::SpatialDistribution<> * - arch::Uniform<> : arch::SpatialDistribution<> + * - arch::Piston<> : arch::SpatialDistribution<> * - arch::Replenish<> : arch::SpatialDistribution<> * @namespace * - arch:: @@ -49,6 +50,35 @@ namespace arch { } }; + template + struct Piston : public arch::SpatialDistribution + { + Piston(const M &metric, real_t xmin, real_t xmax, in piston_direction = in::x1) + : arch::SpatialDistribution{metric} + , xmin {xmin} + , xmax {xmax} + , piston_direction {piston_direction} {} + + Inline auto operator()(const coord_t &x_Ph) const -> real_t override + { + // dimentsion to fill + const auto fill_dim = static_cast(piston_direction); + + if (x_Ph[fill_dim] < xmin || x_Ph[fill_dim] > xmax) + { + return 0.0; + } + else + { + return 1.0; + } + } + + private: + real_t xmin, xmax; + in piston_direction; + }; + template struct Replenish : public SpatialDistribution { using SpatialDistribution::metric; From e4d597968bc3411d20696ef3ca15890e06eeed7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Thu, 20 Mar 2025 17:43:25 -0500 Subject: [PATCH 064/176] fix inconsistency in return type --- src/archetypes/spatial_dist.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/archetypes/spatial_dist.h b/src/archetypes/spatial_dist.h index 098e1c5bb..3731922de 100644 --- a/src/archetypes/spatial_dist.h +++ b/src/archetypes/spatial_dist.h @@ -66,11 +66,11 @@ namespace arch { if (x_Ph[fill_dim] < xmin || x_Ph[fill_dim] > xmax) { - return 0.0; + return ZERO; } else { - return 1.0; + return ONE; } } From cbc20219b85a340644cc836be2cdb3a8d608f05b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Mar 2025 16:24:35 -0500 Subject: [PATCH 065/176] Added `MovingInjector` (wip) --- src/archetypes/particle_injector.h | 69 ++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/src/archetypes/particle_injector.h b/src/archetypes/particle_injector.h index cbcbbd389..4a6e72526 100644 --- a/src/archetypes/particle_injector.h +++ b/src/archetypes/particle_injector.h @@ -170,6 +170,75 @@ namespace arch { ~AtmosphereInjector() = default; }; + template + struct MovingInjector + { + struct TargetDensityProfile + { + const real_t nmax, xinj, xdrift; + + TargetDensityProfile(real_t xinj, real_t xdrift, real_t nmax) + : xinj{xinj}, xdrift{xdrift}, nmax{nmax} {} + + Inline auto operator()(const coord_t &x_Ph) const -> real_t + { + if constexpr ((O == in::x1) or + (O == in::x2 and (M::Dim == Dim::_2D or M::Dim == Dim::_3D)) or + (O == in::x3 and M::Dim == Dim::_3D)) + { + const auto xi = x_Ph[static_cast(O)]; + // + direction + if (xi < xinj - xdrift or xi >= xinj) + { + return ZERO; + } + else + { + if constexpr (M::CoordType == Coord::Cart) + { + return nmax; + } + else + { + raise::KernelError( + HERE, + "Moving injector in +x cannot be applied for non-cartesian"); + return ZERO; + } + } + } + else + { + raise::KernelError(HERE, "Wrong direction"); + return ZERO; + } + } + }; + using energy_dist_t = Maxwellian; + using spatial_dist_t = Replenish; + static_assert(M::is_metric, "M must be a metric class"); + static constexpr bool is_nonuniform_injector{true}; + static constexpr Dimension D{M::Dim}; + static constexpr Coord C{M::CoordType}; + + const energy_dist_t energy_dist; + const TargetDensityProfile target_density; + const spatial_dist_t spatial_dist; + const std::pair species; + + MovingInjector(const M &metric, + const ndfield_t &density, + const energy_dist_t &energy_dist, + real_t xinj, + real_t xdrift, + real_t nmax, + const std::pair &species) + : energy_dist{energy_dist}, + target_density{xinj, xdrift, nmax}, spatial_dist{metric, density, 0, target_density, nmax}, species{species} {} + + ~MovingInjector() = default; + }; + /** * @brief Injects uniform number density of particles everywhere in the domain * @param domain Domain object From 70ea9af6342afc63e25464369c68aac71dada3fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Mar 2025 16:25:10 -0500 Subject: [PATCH 066/176] Added `CustomPostStep` for shock pgen to continually inject particles (wip) --- setups/srpic/shock/pgen.hpp | 78 ++++++++++++++++++++++++++++++++++- setups/srpic/shock/shock.toml | 4 +- src/engines/engine_run.cpp | 2 +- 3 files changed, 79 insertions(+), 5 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 9bf51b866..7c0d750e3 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -79,7 +79,7 @@ namespace user { using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; - const real_t drift_ux, temperature, filling_fraction; + const real_t drift_ux, temperature, filling_fraction, injection_rate; const real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -92,7 +92,8 @@ namespace user { , Btheta { p.template get("setup.Btheta", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } , init_flds { Bmag, Btheta, Bphi, drift_ux } - , filling_fraction { params.template get("setup.filling_fraction", 1.0) }{} + , filling_fraction { params.template get("setup.filling_fraction", 1.0) } + , injection_rate { params.template get("setup.injection_rate", 1.0) } {} inline PGen() {} @@ -100,6 +101,11 @@ namespace user { return init_flds; } + // Inline auto TargetDensity(const coord_t &x_Ph) const -> real_t + // { + // return ONE; + // } + inline void InitPrtls(Domain& local_domain) { // minimum and maximum position of particles @@ -143,6 +149,74 @@ namespace user { false, // no weights box); } + + void CustomPostStep(std::size_t, long double time, real_t dt, Domain &domain) + { + /* + Moving injector for the particles + */ + + // energy distribution of the particles + const auto energy_dist = arch::Maxwellian(domain.mesh.metric, + domain.random_pool, + temperature, + -drift_ux, + in::x1); + + // minimum and maximum position of particles + // ToDo: Add time offset for start of movement? + real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction + + injection_rate * time - drift_ux * dt; + real_t xg_max = domain.mesh.extent(in::x1).second * filling_fraction + + injection_rate * time; + + + /* + I thought this option would be better, but I can't get it to work + */ + + // // define box to inject into + // boundaries_t box; + // // loop over all dimensions + // for (unsigned short d{0}; d < static_cast(M::Dim); ++d) + // { + // // compute the range for the x-direction + // if (d == static_cast(in::x1)) + // { + // box.push_back({xg_min, xg_max}); + // } else { + // // inject into full range in other directions + // box.push_back(Range::All); + // } + // } + + // const auto spatial_dist = arch::Replenish(domain.mesh.metric, + // domain.fields.bckp, + // box, + // TargetDensity, + // 1.0); + + // const auto injector = arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // {1, 2}); + + const auto injector = arch::MovingInjector{ + domain.mesh.metric, + domain.fields.bckp, + energy_dist, + xg_max, + xg_min, + 1.0, + {1, 2}}; + + arch::InjectNonUniform( + params, + domain, + injector, + 1.0, // target density + false); + } }; } // namespace user diff --git a/setups/srpic/shock/shock.toml b/setups/srpic/shock/shock.toml index f954345c2..772b452c9 100644 --- a/setups/srpic/shock/shock.toml +++ b/setups/srpic/shock/shock.toml @@ -45,10 +45,10 @@ Bmag = 1.0 # magnetic field strength as fraction of magnetisation Btheta = 0.0 # magnetic field angle in the plane Bphi = 0.0 # magnetic field angle out of plane - filling_fraction = 1.0 # fraction of the shock piston filled with plasma + filling_fraction = 0.1 # fraction of the shock piston filled with plasma [output] - interval_time = 500.0 + interval_time = 10.0 format = "hdf5" [output.fields] diff --git a/src/engines/engine_run.cpp b/src/engines/engine_run.cpp index 506fd121d..3ace709e9 100644 --- a/src/engines/engine_run.cpp +++ b/src/engines/engine_run.cpp @@ -52,7 +52,7 @@ namespace ntt { traits::has_method::value) { timers.start("Custom"); m_metadomain.runOnLocalDomains([&timers, this](auto& dom) { - m_pgen.CustomPostStep(step, time, dom); + m_pgen.CustomPostStep(step, time, dt, dom); }); timers.stop("Custom"); } From 972750763f79a15c5f4cb764b9e7f4901e394fe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Sat, 22 Mar 2025 11:42:48 -0500 Subject: [PATCH 067/176] bugfix --- src/archetypes/particle_injector.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/archetypes/particle_injector.h b/src/archetypes/particle_injector.h index 4a6e72526..967a77c75 100644 --- a/src/archetypes/particle_injector.h +++ b/src/archetypes/particle_injector.h @@ -188,7 +188,7 @@ namespace arch { { const auto xi = x_Ph[static_cast(O)]; // + direction - if (xi < xinj - xdrift or xi >= xinj) + if (xi < xdrift or xi >= xinj) { return ZERO; } From d7d12a31742c3303cda25a354fd724d99b9c3f5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Sat, 22 Mar 2025 11:56:32 -0500 Subject: [PATCH 068/176] added start time of injection and explicitly enforce injection region (wip) --- setups/srpic/shock/pgen.hpp | 102 +++++++++++++++++----------------- setups/srpic/shock/shock.toml | 2 + 2 files changed, 54 insertions(+), 50 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 7c0d750e3..04de58406 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -79,7 +79,8 @@ namespace user { using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; - const real_t drift_ux, temperature, filling_fraction, injection_rate; + const real_t drift_ux, temperature, filling_fraction, + injector_velocity, injection_start; const real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -93,7 +94,8 @@ namespace user { , Bphi { p.template get("setup.Bphi", ZERO) } , init_flds { Bmag, Btheta, Bphi, drift_ux } , filling_fraction { params.template get("setup.filling_fraction", 1.0) } - , injection_rate { params.template get("setup.injection_rate", 1.0) } {} + , injector_velocity { params.template get("setup.injector_velocity", 1.0) } + , injection_start { params.template get("setup.injection_start", 0.0) } {} inline PGen() {} @@ -164,58 +166,58 @@ namespace user { in::x1); // minimum and maximum position of particles - // ToDo: Add time offset for start of movement? - real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction + - injection_rate * time - drift_ux * dt; + real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction + + injector_velocity * (time - injection_start) + - drift_ux; // distance particles have moved in the last time step real_t xg_max = domain.mesh.extent(in::x1).second * filling_fraction + - injection_rate * time; + injector_velocity * (time - injection_start); + /* + I thought this option would be better, but I can't get it to work + */ - /* - I thought this option would be better, but I can't get it to work - */ - - // // define box to inject into - // boundaries_t box; - // // loop over all dimensions - // for (unsigned short d{0}; d < static_cast(M::Dim); ++d) - // { - // // compute the range for the x-direction - // if (d == static_cast(in::x1)) - // { - // box.push_back({xg_min, xg_max}); - // } else { - // // inject into full range in other directions - // box.push_back(Range::All); - // } - // } - - // const auto spatial_dist = arch::Replenish(domain.mesh.metric, - // domain.fields.bckp, - // box, - // TargetDensity, - // 1.0); - - // const auto injector = arch::NonUniformInjector( - // energy_dist, - // spatial_dist, - // {1, 2}); - - const auto injector = arch::MovingInjector{ - domain.mesh.metric, - domain.fields.bckp, - energy_dist, - xg_max, - xg_min, - 1.0, - {1, 2}}; + // define box to inject into + boundaries_t box; + // loop over all dimensions + for (unsigned short d{0}; d < static_cast(M::Dim); ++d) + { + // compute the range for the x-direction + if (d == static_cast(in::x1)) + { + box.push_back({xg_min, xg_max}); + } else { + // inject into full range in other directions + box.push_back(Range::All); + } + } - arch::InjectNonUniform( - params, - domain, - injector, - 1.0, // target density - false); + // const auto spatial_dist = arch::Replenish(domain.mesh.metric, + // domain.fields.bckp, + // box, + // TargetDensity, + // 1.0); + + // const auto injector = arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // {1, 2}); + + const auto injector = arch::MovingInjector{ + domain.mesh.metric, + domain.fields.bckp, + energy_dist, + xg_max, + xg_min, + 1.0, + {1, 2}}; + + arch::InjectNonUniform( + params, + domain, + injector, + 1.0, // target density + false, + box); } }; diff --git a/setups/srpic/shock/shock.toml b/setups/srpic/shock/shock.toml index 772b452c9..43da87d99 100644 --- a/setups/srpic/shock/shock.toml +++ b/setups/srpic/shock/shock.toml @@ -46,6 +46,8 @@ Btheta = 0.0 # magnetic field angle in the plane Bphi = 0.0 # magnetic field angle out of plane filling_fraction = 0.1 # fraction of the shock piston filled with plasma + injector_velocity = 1.0 # speed of injector [c] + injection_start = 0.0 # start time of moving injector [output] interval_time = 10.0 From 6cf0dc8e5eb44528d049b31111ff7ff81bb52bc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Sat, 22 Mar 2025 12:12:00 -0500 Subject: [PATCH 069/176] small bugfix --- setups/srpic/shock/pgen.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 04de58406..1c9c9a4d3 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -167,8 +167,8 @@ namespace user { // minimum and maximum position of particles real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction - + injector_velocity * (time - injection_start) - - drift_ux; // distance particles have moved in the last time step + + injector_velocity * (time - injection_start - dt) + - drift_ux * dt; // distance particles have moved in the last time step real_t xg_max = domain.mesh.extent(in::x1).second * filling_fraction + injector_velocity * (time - injection_start); From 22a0479855dad05a77b5d139e6c2d0c692dd2ead Mon Sep 17 00:00:00 2001 From: hayk Date: Mon, 24 Mar 2025 13:14:51 -0400 Subject: [PATCH 070/176] shock pgen changed --- setups/srpic/shock/pgen.hpp | 200 ++++++++++++++++++++-------------- src/archetypes/spatial_dist.h | 39 +++---- src/engines/engine_run.cpp | 2 +- 3 files changed, 134 insertions(+), 107 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 1c9c9a4d3..cc6f1b812 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -18,6 +18,16 @@ namespace user { using namespace ntt; + template + struct SpatialUniform : public arch::SpatialDistribution { + SpatialUniform(const M& metric) + : arch::SpatialDistribution { metric } {} + + Inline auto operator()(const coord_t&) const -> real_t override { + return ONE; + } + }; + template struct InitFields { /* @@ -79,8 +89,8 @@ namespace user { using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; - const real_t drift_ux, temperature, filling_fraction, - injector_velocity, injection_start; + const real_t drift_ux, temperature, filling_fraction, injector_velocity, + injection_start, dt; const real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -92,10 +102,14 @@ namespace user { , Bmag { p.template get("setup.Bmag", ZERO) } , Btheta { p.template get("setup.Btheta", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } - , init_flds { Bmag, Btheta, Bphi, drift_ux } - , filling_fraction { params.template get("setup.filling_fraction", 1.0) } - , injector_velocity { params.template get("setup.injector_velocity", 1.0) } - , injection_start { params.template get("setup.injection_start", 0.0) } {} + , init_flds { Bmag, Btheta, Bphi, drift_ux } + , filling_fraction { params.template get("setup.filling_fraction", + 1.0) } + , injector_velocity { params.template get( + "setup.injector_velocity", + 1.0) } + , injection_start { params.template get("setup.injection_start", 0.0) } + , dt { params.template get("algorithms.timestep.dt") } {} inline PGen() {} @@ -109,7 +123,7 @@ namespace user { // } inline void InitPrtls(Domain& local_domain) { - + // minimum and maximum position of particles real_t xg_min = local_domain.mesh.extent(in::x1).first; real_t xg_max = local_domain.mesh.extent(in::x1).second * filling_fraction; @@ -120,16 +134,19 @@ namespace user { for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { // compute the range for the x-direction if (d == static_cast(in::x1)) { - box.push_back({xg_min, xg_max}); + box.push_back({ xg_min, xg_max }); } else { // inject into full range in other directions box.push_back(Range::All); } } - // spatial distribution of the particles + // spatial distribution of the particles // -> hack to use the uniform distribution in NonUniformInjector - const auto spatial_dist = arch::Piston(local_domain.mesh.metric, xg_min, xg_max, in::x1); + const auto spatial_dist = arch::Piston(local_domain.mesh.metric, + xg_min, + xg_max, + in::x1); // energy distribution of the particles const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, @@ -139,85 +156,100 @@ namespace user { in::x1); const auto injector = arch::NonUniformInjector( - energy_dist, - spatial_dist, - {1, 2}); + energy_dist, + spatial_dist, + { 1, 2 }); arch::InjectNonUniform>( - params, - local_domain, - injector, - 1.0, // target density - false, // no weights - box); + params, + local_domain, + injector, + 1.0, // target density + false, // no weights + box); } - void CustomPostStep(std::size_t, long double time, real_t dt, Domain &domain) - { - /* - Moving injector for the particles - */ - - // energy distribution of the particles - const auto energy_dist = arch::Maxwellian(domain.mesh.metric, - domain.random_pool, - temperature, - -drift_ux, - in::x1); - - // minimum and maximum position of particles - real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction - + injector_velocity * (time - injection_start - dt) - - drift_ux * dt; // distance particles have moved in the last time step - real_t xg_max = domain.mesh.extent(in::x1).second * filling_fraction + - injector_velocity * (time - injection_start); - - /* - I thought this option would be better, but I can't get it to work - */ - - // define box to inject into - boundaries_t box; - // loop over all dimensions - for (unsigned short d{0}; d < static_cast(M::Dim); ++d) - { - // compute the range for the x-direction - if (d == static_cast(in::x1)) - { - box.push_back({xg_min, xg_max}); - } else { - // inject into full range in other directions - box.push_back(Range::All); - } - } + void CustomPostStep(std::size_t, long double time, Domain& domain) { + const auto energy_dist = arch::Maxwellian(domain.mesh.metric, + domain.random_pool, + temperature, + -drift_ux, + in::x1); + const auto spatial_dist = SpatialUniform(domain.mesh.metric); + const auto injector = arch::NonUniformInjector( + energy_dist, + spatial_dist, + { 1, 2 }); + + boundaries_t box; + + const auto x_init = domain.mesh.extent(in::x1).first + + filling_fraction * (domain.mesh.extent(in::x1).second - + domain.mesh.extent(in::x1).first); + + box[0].first = x_init + injector_velocity * (time - injection_start); + box[0].second = x_init + injector_velocity * (time - injection_start + dt) - + drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt; + for (auto d = 1u; d < M::Dim; ++d) { + box[d] = Range::All; + } - // const auto spatial_dist = arch::Replenish(domain.mesh.metric, - // domain.fields.bckp, - // box, - // TargetDensity, - // 1.0); - - // const auto injector = arch::NonUniformInjector( - // energy_dist, - // spatial_dist, - // {1, 2}); - - const auto injector = arch::MovingInjector{ - domain.mesh.metric, - domain.fields.bckp, - energy_dist, - xg_max, - xg_min, - 1.0, - {1, 2}}; - - arch::InjectNonUniform( - params, - domain, - injector, - 1.0, // target density - false, - box); + arch::InjectNonUniform(params, + domain, + injector, + ONE, + false, + box); + + // /* + // Moving injector for the particles + // */ + // + // // minimum and maximum position of particles + // real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction + + // injector_velocity * (time - injection_start - dt) - + // drift_ux * + // dt; // distance particles have moved in the last time step + // real_t xg_max = domain.mesh.extent(in::x1).second * filling_fraction + + // injector_velocity * (time - injection_start); + // + // /* + // I thought this option would be better, but I can't get it to work + // */ + // + // // define box to inject into + // // loop over all dimensions + // for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { + // // compute the range for the x-direction + // if (d == static_cast(in::x1)) { + // box.push_back({ xg_min, xg_max }); + // } else { + // // inject into full range in other directions + // box.push_back(Range::All); + // } + // } + + // const auto spatial_dist = arch::Replenish(domain.mesh.metric, + // domain.fields.bckp, + // box, + // TargetDensity, + // 1.0); + + // const auto injector = arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // {1, 2}); + + // const auto injector = arch::MovingInjector { + // domain.mesh.metric, + // domain.fields.bckp, + // energy_dist, + // xg_max, + // xg_min, + // 1.0, + // { 1, 2 } + // }; } }; diff --git a/src/archetypes/spatial_dist.h b/src/archetypes/spatial_dist.h index 3731922de..3555aaca5 100644 --- a/src/archetypes/spatial_dist.h +++ b/src/archetypes/spatial_dist.h @@ -51,32 +51,27 @@ namespace arch { }; template - struct Piston : public arch::SpatialDistribution - { - Piston(const M &metric, real_t xmin, real_t xmax, in piston_direction = in::x1) - : arch::SpatialDistribution{metric} - , xmin {xmin} - , xmax {xmax} - , piston_direction {piston_direction} {} - - Inline auto operator()(const coord_t &x_Ph) const -> real_t override - { - // dimentsion to fill - const auto fill_dim = static_cast(piston_direction); - - if (x_Ph[fill_dim] < xmin || x_Ph[fill_dim] > xmax) - { - return ZERO; - } - else - { - return ONE; - } + struct Piston : public arch::SpatialDistribution { + Piston(const M& metric, real_t xmin, real_t xmax, in piston_direction = in::x1) + : arch::SpatialDistribution { metric } + , xmin { xmin } + , xmax { xmax } + , piston_direction { piston_direction } {} + + Inline auto operator()(const coord_t& x_Ph) const -> real_t override { + // dimentsion to fill + const auto fill_dim = static_cast(piston_direction); + + if (x_Ph[fill_dim] < xmin || x_Ph[fill_dim] > xmax) { + return ZERO; + } else { + return ONE; } + } private: real_t xmin, xmax; - in piston_direction; + in piston_direction; }; template diff --git a/src/engines/engine_run.cpp b/src/engines/engine_run.cpp index 3ace709e9..506fd121d 100644 --- a/src/engines/engine_run.cpp +++ b/src/engines/engine_run.cpp @@ -52,7 +52,7 @@ namespace ntt { traits::has_method::value) { timers.start("Custom"); m_metadomain.runOnLocalDomains([&timers, this](auto& dom) { - m_pgen.CustomPostStep(step, time, dt, dom); + m_pgen.CustomPostStep(step, time, dom); }); timers.stop("Custom"); } From 13da0dda1a67d2fba6c7e6d7111965b75f0f0ce0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 24 Mar 2025 15:31:41 -0500 Subject: [PATCH 071/176] reduce resolution for faster test --- setups/srpic/shock/shock.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setups/srpic/shock/shock.toml b/setups/srpic/shock/shock.toml index 43da87d99..ca19a4078 100644 --- a/setups/srpic/shock/shock.toml +++ b/setups/srpic/shock/shock.toml @@ -4,8 +4,8 @@ runtime = 30000.0 [grid] - resolution = [16384, 128] - extent = [[0.0, 8000.0], [-31.25, 31.25]] + resolution = [4096, 128] + extent = [[0.0, 2000.0], [-31.25, 31.25]] [grid.metric] metric = "minkowski" From 54f083fe77d849e0c9658ab5714f10dd38915926 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 24 Mar 2025 15:31:52 -0500 Subject: [PATCH 072/176] cleanup and bugfix --- setups/srpic/shock/pgen.hpp | 195 +++++++++++++++++------------------- 1 file changed, 92 insertions(+), 103 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index cc6f1b812..2454c9037 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -18,16 +18,6 @@ namespace user { using namespace ntt; - template - struct SpatialUniform : public arch::SpatialDistribution { - SpatialUniform(const M& metric) - : arch::SpatialDistribution { metric } {} - - Inline auto operator()(const coord_t&) const -> real_t override { - return ONE; - } - }; - template struct InitFields { /* @@ -89,10 +79,13 @@ namespace user { using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; - const real_t drift_ux, temperature, filling_fraction, injector_velocity, - injection_start, dt; + // gas properties + const real_t drift_ux, temperature, filling_fraction; + // injector properties + const real_t injector_velocity, injection_start, dt; + const bool injection_on; - const real_t Btheta, Bphi, Bmag; + real_t Btheta, Bphi, Bmag; InitFields init_flds; inline PGen(const SimulationParams& p, const Metadomain& m) @@ -103,13 +96,11 @@ namespace user { , Btheta { p.template get("setup.Btheta", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } , init_flds { Bmag, Btheta, Bphi, drift_ux } - , filling_fraction { params.template get("setup.filling_fraction", - 1.0) } - , injector_velocity { params.template get( - "setup.injector_velocity", - 1.0) } - , injection_start { params.template get("setup.injection_start", 0.0) } - , dt { params.template get("algorithms.timestep.dt") } {} + , filling_fraction { p.template get("setup.filling_fraction", 1.0) } + , injector_velocity { p.template get("setup.injector_velocity", 1.0) } + , injection_start { p.template get("setup.injection_start", 0.0) } + , injection_on { p.template get("setup.continuous_injection", true) } + , dt { p.template get("algorithms.timestep.dt") } {} inline PGen() {} @@ -117,16 +108,13 @@ namespace user { return init_flds; } - // Inline auto TargetDensity(const coord_t &x_Ph) const -> real_t - // { - // return ONE; - // } - inline void InitPrtls(Domain& local_domain) { // minimum and maximum position of particles real_t xg_min = local_domain.mesh.extent(in::x1).first; - real_t xg_max = local_domain.mesh.extent(in::x1).second * filling_fraction; + real_t xg_max = local_domain.mesh.extent(in::x1).first + + filling_fraction * (local_domain.mesh.extent(in::x1).second - + local_domain.mesh.extent(in::x1).first); // define box to inject into boundaries_t box; @@ -170,86 +158,87 @@ namespace user { } void CustomPostStep(std::size_t, long double time, Domain& domain) { - const auto energy_dist = arch::Maxwellian(domain.mesh.metric, - domain.random_pool, - temperature, - -drift_ux, - in::x1); - const auto spatial_dist = SpatialUniform(domain.mesh.metric); - const auto injector = arch::NonUniformInjector( - energy_dist, - spatial_dist, - { 1, 2 }); - boundaries_t box; - - const auto x_init = domain.mesh.extent(in::x1).first + - filling_fraction * (domain.mesh.extent(in::x1).second - - domain.mesh.extent(in::x1).first); + // ToDo: more performant version? + if (injection_on) { + // same maxwell distribution as above + const auto energy_dist = arch::Maxwellian(domain.mesh.metric, + domain.random_pool, + temperature, + -drift_ux, + in::x1); + + // initial position of injector + const auto x_init = domain.mesh.extent(in::x1).first + + filling_fraction * (domain.mesh.extent(in::x1).second - + domain.mesh.extent(in::x1).first); + + // check if injector is supposed to start moving already + const auto dt_inj = time - injection_start > ZERO ? time - injection_start + : ZERO; + + // define box to inject into + boundaries_t box; + + // loop over all dimension + for (auto d = 0u; d < M::Dim; ++d) { + if (d == 0) { + box.push_back({ x_init + injector_velocity * (dt_inj)-drift_ux / + math::sqrt(1 + SQR(drift_ux)) * dt, + x_init + injector_velocity * (dt_inj + dt) }); + } else { + box.push_back(Range::All); + } + } - box[0].first = x_init + injector_velocity * (time - injection_start); - box[0].second = x_init + injector_velocity * (time - injection_start + dt) - - drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt; - for (auto d = 1u; d < M::Dim; ++d) { - box[d] = Range::All; + // spatial distribution of the particles + // -> hack to use the uniform distribution in NonUniformInjector + const auto spatial_dist = arch::Piston(domain.mesh.metric, + box[0].first, + box[0].second, + in::x1); + + // ToDo: extend Replenish to replace the current injector + const auto injector = + arch::NonUniformInjector( + energy_dist, + spatial_dist, + { 1, 2 }); + + // inject non-uniformly within the defined box + arch::InjectNonUniform(params, + domain, + injector, + ONE, + false, + box); + + /* + I thought this option would be better, but I can't get it to work + */ + + // const auto spatial_dist = arch::Replenish(domain.mesh.metric, + // domain.fields.bckp, + // box, + // TargetDensity, + // 1.0); + + // const auto injector = arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // {1, 2}); + + // const auto injector = arch::MovingInjector { + // domain.mesh.metric, + // domain.fields.bckp, + // energy_dist, + // box[0].first, + // box[0].second, + // 1.0, + // { 1, 2 } + // }; } - - arch::InjectNonUniform(params, - domain, - injector, - ONE, - false, - box); - - // /* - // Moving injector for the particles - // */ - // - // // minimum and maximum position of particles - // real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction + - // injector_velocity * (time - injection_start - dt) - - // drift_ux * - // dt; // distance particles have moved in the last time step - // real_t xg_max = domain.mesh.extent(in::x1).second * filling_fraction + - // injector_velocity * (time - injection_start); - // - // /* - // I thought this option would be better, but I can't get it to work - // */ - // - // // define box to inject into - // // loop over all dimensions - // for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { - // // compute the range for the x-direction - // if (d == static_cast(in::x1)) { - // box.push_back({ xg_min, xg_max }); - // } else { - // // inject into full range in other directions - // box.push_back(Range::All); - // } - // } - - // const auto spatial_dist = arch::Replenish(domain.mesh.metric, - // domain.fields.bckp, - // box, - // TargetDensity, - // 1.0); - - // const auto injector = arch::NonUniformInjector( - // energy_dist, - // spatial_dist, - // {1, 2}); - - // const auto injector = arch::MovingInjector { - // domain.mesh.metric, - // domain.fields.bckp, - // energy_dist, - // xg_max, - // xg_min, - // 1.0, - // { 1, 2 } - // }; } }; From 9f45d04b528ff15f850a86584a58287e52631748 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 24 Mar 2025 15:45:43 -0500 Subject: [PATCH 073/176] removed unnecessary parameter --- setups/srpic/shock/pgen.hpp | 156 +++++++++++++++++------------------- 1 file changed, 75 insertions(+), 81 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 2454c9037..30ab01538 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -83,8 +83,7 @@ namespace user { const real_t drift_ux, temperature, filling_fraction; // injector properties const real_t injector_velocity, injection_start, dt; - const bool injection_on; - + // magnetic field properties real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -99,7 +98,6 @@ namespace user { , filling_fraction { p.template get("setup.filling_fraction", 1.0) } , injector_velocity { p.template get("setup.injector_velocity", 1.0) } , injection_start { p.template get("setup.injection_start", 0.0) } - , injection_on { p.template get("setup.continuous_injection", true) } , dt { p.template get("algorithms.timestep.dt") } {} inline PGen() {} @@ -159,86 +157,82 @@ namespace user { void CustomPostStep(std::size_t, long double time, Domain& domain) { - // ToDo: more performant version? - if (injection_on) { - // same maxwell distribution as above - const auto energy_dist = arch::Maxwellian(domain.mesh.metric, - domain.random_pool, - temperature, - -drift_ux, - in::x1); - - // initial position of injector - const auto x_init = domain.mesh.extent(in::x1).first + - filling_fraction * (domain.mesh.extent(in::x1).second - - domain.mesh.extent(in::x1).first); - - // check if injector is supposed to start moving already - const auto dt_inj = time - injection_start > ZERO ? time - injection_start - : ZERO; - - // define box to inject into - boundaries_t box; - - // loop over all dimension - for (auto d = 0u; d < M::Dim; ++d) { - if (d == 0) { - box.push_back({ x_init + injector_velocity * (dt_inj)-drift_ux / - math::sqrt(1 + SQR(drift_ux)) * dt, - x_init + injector_velocity * (dt_inj + dt) }); - } else { - box.push_back(Range::All); - } - } + // same maxwell distribution as above + const auto energy_dist = arch::Maxwellian(domain.mesh.metric, + domain.random_pool, + temperature, + -drift_ux, + in::x1); + + // initial position of injector + const auto x_init = domain.mesh.extent(in::x1).first + + filling_fraction * (domain.mesh.extent(in::x1).second - + domain.mesh.extent(in::x1).first); - // spatial distribution of the particles - // -> hack to use the uniform distribution in NonUniformInjector - const auto spatial_dist = arch::Piston(domain.mesh.metric, - box[0].first, - box[0].second, - in::x1); - - // ToDo: extend Replenish to replace the current injector - const auto injector = - arch::NonUniformInjector( - energy_dist, - spatial_dist, - { 1, 2 }); - - // inject non-uniformly within the defined box - arch::InjectNonUniform(params, - domain, - injector, - ONE, - false, - box); - - /* - I thought this option would be better, but I can't get it to work - */ - - // const auto spatial_dist = arch::Replenish(domain.mesh.metric, - // domain.fields.bckp, - // box, - // TargetDensity, - // 1.0); - - // const auto injector = arch::NonUniformInjector( - // energy_dist, - // spatial_dist, - // {1, 2}); - - // const auto injector = arch::MovingInjector { - // domain.mesh.metric, - // domain.fields.bckp, - // energy_dist, - // box[0].first, - // box[0].second, - // 1.0, - // { 1, 2 } - // }; + // check if injector is supposed to start moving already + const auto dt_inj = time - injection_start > ZERO ? + time - injection_start : ZERO; + + // define box to inject into + boundaries_t box; + + // loop over all dimension + for (auto d = 0u; d < M::Dim; ++d) { + if (d == 0) { + box.push_back({ x_init + injector_velocity * dt_inj - + drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt, + x_init + injector_velocity * (dt_inj + dt) }); + } else { + box.push_back(Range::All); + } } + + // spatial distribution of the particles + // -> hack to use the uniform distribution in NonUniformInjector + const auto spatial_dist = arch::Piston(domain.mesh.metric, + box[0].first, + box[0].second, + in::x1); + + // ToDo: extend Replenish to replace the current injector + const auto injector = arch::NonUniformInjector( + energy_dist, + spatial_dist, + { 1, 2 }); + + // inject non-uniformly within the defined box + arch::InjectNonUniform(params, + domain, + injector, + ONE, + false, + box); + + /* + I thought this option would be better, but I can't get it to work + */ + + // const auto spatial_dist = arch::Replenish(domain.mesh.metric, + // domain.fields.bckp, + // box, + // TargetDensity, + // 1.0); + + // const auto injector = arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // {1, 2}); + + // const auto injector = arch::MovingInjector { + // domain.mesh.metric, + // domain.fields.bckp, + // energy_dist, + // box[0].first, + // box[0].second, + // 1.0, + // { 1, 2 } + // }; } }; From aa8b8e7f10edc6316476b37297d8b980bb5c79d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 24 Mar 2025 16:06:42 -0500 Subject: [PATCH 074/176] applied formatting to `MovingInjector` --- src/archetypes/particle_injector.h | 123 ++++++++++++++--------------- 1 file changed, 58 insertions(+), 65 deletions(-) diff --git a/src/archetypes/particle_injector.h b/src/archetypes/particle_injector.h index 967a77c75..818ab5bf5 100644 --- a/src/archetypes/particle_injector.h +++ b/src/archetypes/particle_injector.h @@ -171,72 +171,65 @@ namespace arch { }; template - struct MovingInjector - { - struct TargetDensityProfile - { - const real_t nmax, xinj, xdrift; - - TargetDensityProfile(real_t xinj, real_t xdrift, real_t nmax) - : xinj{xinj}, xdrift{xdrift}, nmax{nmax} {} - - Inline auto operator()(const coord_t &x_Ph) const -> real_t - { - if constexpr ((O == in::x1) or - (O == in::x2 and (M::Dim == Dim::_2D or M::Dim == Dim::_3D)) or - (O == in::x3 and M::Dim == Dim::_3D)) - { - const auto xi = x_Ph[static_cast(O)]; - // + direction - if (xi < xdrift or xi >= xinj) - { - return ZERO; - } - else - { - if constexpr (M::CoordType == Coord::Cart) - { - return nmax; - } - else - { - raise::KernelError( - HERE, - "Moving injector in +x cannot be applied for non-cartesian"); - return ZERO; - } - } - } - else - { - raise::KernelError(HERE, "Wrong direction"); - return ZERO; - } + struct MovingInjector { + struct TargetDensityProfile { + const real_t nmax, xinj, xdrift; + + TargetDensityProfile(real_t xinj, real_t xdrift, real_t nmax) + : xinj { xinj } + , xdrift { xdrift } + , nmax { nmax } {} + + Inline auto operator()(const coord_t& x_Ph) const -> real_t { + if constexpr ((O == in::x1) or + (O == in::x2 and (M::Dim == Dim::_2D or M::Dim == Dim::_3D)) or + (O == in::x3 and M::Dim == Dim::_3D)) { + const auto xi = x_Ph[static_cast(O)]; + // + direction + if (xi < xdrift or xi >= xinj) { + return ZERO; + } else { + if constexpr (M::CoordType == Coord::Cart) { + return nmax; + } else { + raise::KernelError( + HERE, + "Moving injector in +x cannot be applied for non-cartesian"); + return ZERO; } - }; - using energy_dist_t = Maxwellian; - using spatial_dist_t = Replenish; - static_assert(M::is_metric, "M must be a metric class"); - static constexpr bool is_nonuniform_injector{true}; - static constexpr Dimension D{M::Dim}; - static constexpr Coord C{M::CoordType}; - - const energy_dist_t energy_dist; - const TargetDensityProfile target_density; - const spatial_dist_t spatial_dist; - const std::pair species; - - MovingInjector(const M &metric, - const ndfield_t &density, - const energy_dist_t &energy_dist, - real_t xinj, - real_t xdrift, - real_t nmax, - const std::pair &species) - : energy_dist{energy_dist}, - target_density{xinj, xdrift, nmax}, spatial_dist{metric, density, 0, target_density, nmax}, species{species} {} - - ~MovingInjector() = default; + } + } else { + raise::KernelError(HERE, "Wrong direction"); + return ZERO; + } + } + }; + + using energy_dist_t = Maxwellian; + using spatial_dist_t = Replenish; + static_assert(M::is_metric, "M must be a metric class"); + static constexpr bool is_nonuniform_injector { true }; + static constexpr Dimension D { M::Dim }; + static constexpr Coord C { M::CoordType }; + + const energy_dist_t energy_dist; + const TargetDensityProfile target_density; + const spatial_dist_t spatial_dist; + const std::pair species; + + MovingInjector(const M& metric, + const ndfield_t& density, + const energy_dist_t& energy_dist, + real_t xinj, + real_t xdrift, + real_t nmax, + const std::pair& species) + : energy_dist { energy_dist } + , target_density { xinj, xdrift, nmax } + , spatial_dist { metric, density, 0, target_density, nmax } + , species { species } {} + + ~MovingInjector() = default; }; /** From d0fb4ca29ece804e9acfb3e0bc2d2438342b6267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 31 Mar 2025 11:03:57 -0500 Subject: [PATCH 075/176] first attempt at moving-window injection --- setups/srpic/shock/pgen.hpp | 123 +++++++++++++++++++++++++++++++++--- 1 file changed, 114 insertions(+), 9 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 30ab01538..9568767ff 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -83,6 +83,7 @@ namespace user { const real_t drift_ux, temperature, filling_fraction; // injector properties const real_t injector_velocity, injection_start, dt; + const int injection_frequency; // magnetic field properties real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -98,6 +99,7 @@ namespace user { , filling_fraction { p.template get("setup.filling_fraction", 1.0) } , injector_velocity { p.template get("setup.injector_velocity", 1.0) } , injection_start { p.template get("setup.injection_start", 0.0) } + , injection_frequency { p.template get("setup.injection_frequency", 100) } , dt { p.template get("algorithms.timestep.dt") } {} inline PGen() {} @@ -106,6 +108,25 @@ namespace user { return init_flds; } + auto ResetFields(const em& comp) const -> real_t { + if (comp == em::ex1) { + return init_flds.ex1({ ZERO }); + } else if (comp == em::ex2) { + return init_flds.ex2({ ZERO }); + } else if (comp == em::ex3) { + return init_flds.ex3({ ZERO }); + } else if (comp == em::bx1) { + return init_flds.bx1({ ZERO }); + } else if (comp == em::bx2) { + return init_flds.bx2({ ZERO }); + } else if (comp == em::bx3) { + return init_flds.bx3({ ZERO }); + } else { + raise::Error("Invalid component", HERE); + return ZERO; + } + } + inline void InitPrtls(Domain& local_domain) { // minimum and maximum position of particles @@ -155,14 +176,12 @@ namespace user { box); } - void CustomPostStep(std::size_t, long double time, Domain& domain) { + void CustomPostStep(std::size_t step, long double time, Domain& domain) { - // same maxwell distribution as above - const auto energy_dist = arch::Maxwellian(domain.mesh.metric, - domain.random_pool, - temperature, - -drift_ux, - in::x1); + // check if the injector should be active + if (step % injection_frequency != 0) { + return; + } // initial position of injector const auto x_init = domain.mesh.extent(in::x1).first + @@ -180,13 +199,99 @@ namespace user { for (auto d = 0u; d < M::Dim; ++d) { if (d == 0) { box.push_back({ x_init + injector_velocity * dt_inj - - drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt, + drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt - + 1.5 * injection_frequency * dt, x_init + injector_velocity * (dt_inj + dt) }); } else { box.push_back(Range::All); } } + // define indice range to reset fields + boundaries_t incl_ghosts; + for (auto d = 0; d < M::Dim; ++d) { + incl_ghosts.push_back({ true, true }); + } + const auto extent = domain.mesh.ExtentToRange(box, incl_ghosts); + tuple_t x_min { 0 }, x_max { 0 }; + for (auto d = 0; d < M::Dim; ++d) { + x_min[d] = extent[d].first; + x_max[d] = extent[d].second; + } + + // reset fields + std::vector comps = { em::ex1, em::ex2, em::ex3, + em::bx1, em::bx2, em::bx3 }; + + // loop over all components + for (const auto& comp : comps) { + auto value = ResetFields((em)comp); + + if constexpr (M::Dim == Dim::_1D) { + Kokkos::deep_copy(Kokkos::subview(domain.fields.em, + std::make_pair(x_min[0], x_max[0]), + comp), + value); + } else if constexpr (M::Dim == Dim::_2D) { + Kokkos::deep_copy(Kokkos::subview(domain.fields.em, + std::make_pair(x_min[0], x_max[0]), + std::make_pair(x_min[1], x_max[1]), + comp), + value); + } else if constexpr (M::Dim == Dim::_3D) { + Kokkos::deep_copy(Kokkos::subview(domain.fields.em, + std::make_pair(x_min[0], x_max[0]), + std::make_pair(x_min[1], x_max[1]), + std::make_pair(x_min[2], x_max[2]), + comp), + value); + } else { + raise::Error("Invalid dimension", HERE); + } + } + + /* + tag particles inside the injection zone as dead + */ + + // loop over particle species + for (std::size_t s { 0 }; s < 2; ++s) { + + // get particle properties + auto& species = domain.species[s]; + auto i1 = species.i1; + auto tag = species.tag; + + + // tag all particles with x > box[0].first as dead + Kokkos::parallel_for( + "RemoveParticles", + species.rangeActiveParticles(), + Lambda(index_t p) { + // check if the particle is already dead + if (tag(p) == ParticleTag::dead) { + return; + } + // select the x-coordinate index + auto x_i1 = i1(p); + // check if the particle is inside the box of new plasma + if (x_i1 > x_min[0]) { + tag(p) = ParticleTag::dead; + } + } + ); + } + + /* + Inject piston of fresh plasma + */ + + // same maxwell distribution as above + const auto energy_dist = arch::Maxwellian(domain.mesh.metric, + domain.random_pool, + temperature, + -drift_ux, + in::x1); // spatial distribution of the particles // -> hack to use the uniform distribution in NonUniformInjector const auto spatial_dist = arch::Piston(domain.mesh.metric, @@ -194,7 +299,7 @@ namespace user { box[0].second, in::x1); - // ToDo: extend Replenish to replace the current injector + // inject piston of fresh plasma const auto injector = arch::NonUniformInjector( energy_dist, spatial_dist, From 4bcb1a2ebf6ba5d100b1cc0961b2102bc305b8eb Mon Sep 17 00:00:00 2001 From: LudwigBoess Date: Mon, 31 Mar 2025 21:52:04 -0500 Subject: [PATCH 076/176] extended field reset box to the right to squash waves propagating into the empty box --- setups/srpic/shock/pgen.hpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 9568767ff..a25c3891a 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -190,17 +190,16 @@ namespace user { // check if injector is supposed to start moving already const auto dt_inj = time - injection_start > ZERO ? - time - injection_start : ZERO; + time - injection_start : ZERO; // define box to inject into boundaries_t box; - // loop over all dimension for (auto d = 0u; d < M::Dim; ++d) { if (d == 0) { box.push_back({ x_init + injector_velocity * dt_inj - drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt - - 1.5 * injection_frequency * dt, + injection_frequency * dt, x_init + injector_velocity * (dt_inj + dt) }); } else { box.push_back(Range::All); @@ -212,7 +211,9 @@ namespace user { for (auto d = 0; d < M::Dim; ++d) { incl_ghosts.push_back({ true, true }); } - const auto extent = domain.mesh.ExtentToRange(box, incl_ghosts); + auto fields_box = box; + fields_box[0].second += injection_frequency * dt; + const auto extent = domain.mesh.ExtentToRange(fields_box, incl_ghosts); tuple_t x_min { 0 }, x_max { 0 }; for (auto d = 0; d < M::Dim; ++d) { x_min[d] = extent[d].first; @@ -220,11 +221,13 @@ namespace user { } // reset fields - std::vector comps = { em::ex1, em::ex2, em::ex3, - em::bx1, em::bx2, em::bx3 }; + std::vector comps = { em::bx1, em::bx2, em::bx3, + em::ex1, em::ex2, em::ex3 }; // loop over all components for (const auto& comp : comps) { + + // get initial field value of component auto value = ResetFields((em)comp); if constexpr (M::Dim == Dim::_1D) { @@ -262,7 +265,6 @@ namespace user { auto i1 = species.i1; auto tag = species.tag; - // tag all particles with x > box[0].first as dead Kokkos::parallel_for( "RemoveParticles", From 420f36f86b41a8315bc200c0fadd09d4a98a857f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Tue, 1 Apr 2025 15:32:04 -0500 Subject: [PATCH 077/176] bugfix: added check to truncate the injection box at the end of the domain --- setups/srpic/shock/pgen.hpp | 48 ++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index a25c3891a..cf58b94c6 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -80,10 +80,10 @@ namespace user { using arch::ProblemGenerator::params; // gas properties - const real_t drift_ux, temperature, filling_fraction; + const real_t drift_ux, temperature, filling_fraction; // injector properties - const real_t injector_velocity, injection_start, dt; - const int injection_frequency; + const real_t injector_velocity, injection_start, dt; + const int injection_frequency; // magnetic field properties real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -189,18 +189,23 @@ namespace user { domain.mesh.extent(in::x1).first); // check if injector is supposed to start moving already - const auto dt_inj = time - injection_start > ZERO ? - time - injection_start : ZERO; + const auto dt_inj = time - injection_start > ZERO ? time - injection_start + : ZERO; + + // compute the position of the injector + auto xmax = x_init + injector_velocity * (dt_inj + dt); + if (xmax >= domain.mesh.extent(in::x1).second) { + xmax = domain.mesh.extent(in::x1).second; + } // define box to inject into boundaries_t box; // loop over all dimension for (auto d = 0u; d < M::Dim; ++d) { if (d == 0) { - box.push_back({ x_init + injector_velocity * dt_inj - - drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt - - injection_frequency * dt, - x_init + injector_velocity * (dt_inj + dt) }); + box.push_back({ xmax - drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt - + injection_frequency * dt, + xmax }); } else { box.push_back(Range::All); } @@ -212,7 +217,13 @@ namespace user { incl_ghosts.push_back({ true, true }); } auto fields_box = box; - fields_box[0].second += injection_frequency * dt; + // check if the box is still inside the domain + if (xmax + injection_frequency * dt < domain.mesh.extent(in::x1).second) { + fields_box[0].second += injection_frequency * dt; + } else { + // if right side of the box is outside of the domain -> truncate box + fields_box[0].second = domain.mesh.extent(in::x1).second; + } const auto extent = domain.mesh.ExtentToRange(fields_box, incl_ghosts); tuple_t x_min { 0 }, x_max { 0 }; for (auto d = 0; d < M::Dim; ++d) { @@ -221,7 +232,7 @@ namespace user { } // reset fields - std::vector comps = { em::bx1, em::bx2, em::bx3, + std::vector comps = { em::bx1, em::bx2, em::bx3, em::ex1, em::ex2, em::ex3 }; // loop over all components @@ -253,8 +264,8 @@ namespace user { } } - /* - tag particles inside the injection zone as dead + /* + tag particles inside the injection zone as dead */ // loop over particle species @@ -262,8 +273,8 @@ namespace user { // get particle properties auto& species = domain.species[s]; - auto i1 = species.i1; - auto tag = species.tag; + auto i1 = species.i1; + auto tag = species.tag; // tag all particles with x > box[0].first as dead Kokkos::parallel_for( @@ -277,14 +288,13 @@ namespace user { // select the x-coordinate index auto x_i1 = i1(p); // check if the particle is inside the box of new plasma - if (x_i1 > x_min[0]) { + if (x_i1 >= x_min[0]) { tag(p) = ParticleTag::dead; } - } - ); + }); } - /* + /* Inject piston of fresh plasma */ From df3fca1be70472e981340eba78c8dc0eeae80cd3 Mon Sep 17 00:00:00 2001 From: hayk Date: Tue, 1 Apr 2025 18:56:07 -0400 Subject: [PATCH 078/176] exespace/memspace -> kokkos default + cmake flags changed --- CMakeLists.txt | 23 +++--- cmake/MPIConfig.cmake | 4 -- cmake/adios2Config.cmake | 2 - cmake/defaults.cmake | 36 ---------- cmake/dependencies.cmake | 11 ++- cmake/kokkosConfig.cmake | 45 ------------ legacy/src/pic/particles/particle_pusher.hpp | 73 ++++++++++---------- legacy/tests/kernels-gr.cpp | 17 ++--- legacy/tests/kernels-sr.cpp | 17 ++--- src/CMakeLists.txt | 1 - src/checkpoint/tests/checkpoint-mpi.cpp | 4 +- src/checkpoint/tests/checkpoint-nompi.cpp | 4 +- src/checkpoint/writer.cpp | 6 +- src/engines/CMakeLists.txt | 1 - src/framework/CMakeLists.txt | 1 - src/framework/containers/particles.cpp | 4 +- src/framework/domain/comm_mpi.hpp | 12 ++-- src/framework/domain/comm_nompi.hpp | 6 +- src/framework/tests/CMakeLists.txt | 1 - src/global/arch/kokkos_aliases.cpp | 54 ++++++++------- src/global/arch/kokkos_aliases.h | 22 +++--- src/kernels/tests/prtls_to_phys.cpp | 47 +++++++------ 22 files changed, 158 insertions(+), 233 deletions(-) delete mode 100644 cmake/MPIConfig.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index dd22b9308..0409c4107 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -79,11 +79,10 @@ set(BUILD_TESTING CACHE BOOL "Build tests") # ------------------------ Third-party dependencies ------------------------ # -include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/kokkosConfig.cmake) include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/dependencies.cmake) -find_or_fetch_dependency(Kokkos FALSE) -find_or_fetch_dependency(plog TRUE) +find_or_fetch_dependency(Kokkos FALSE QUIET) +find_or_fetch_dependency(plog TRUE QUIET) set(DEPENDENCIES Kokkos::kokkos) include_directories(${plog_SRC}/include) @@ -92,14 +91,15 @@ set_precision(${precision}) # MPI if(${mpi}) - include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/MPIConfig.cmake) + find_or_fetch_dependency(MPI FALSE REQUIRED) + include_directories(${MPI_CXX_INCLUDE_PATH}) + add_compile_options("-D MPI_ENABLED") set(DEPENDENCIES ${DEPENDENCIES} MPI::MPI_CXX) endif() # Output if(${output}) - include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/adios2Config.cmake) - find_or_fetch_dependency(adios2 FALSE) + find_or_fetch_dependency(adios2 FALSE QUIET) if(NOT DEFINED ENV{HDF5_ROOT}) if(DEFINED ENV{CONDA_PREFIX}) execute_process(COMMAND bash -c "conda list | grep \"hdf5\" -q" @@ -110,7 +110,7 @@ if(${output}) endif() endif() find_package(HDF5 REQUIRED) - + add_compile_options("-D OUTPUT_ENABLED") if(${mpi}) set(DEPENDENCIES ${DEPENDENCIES} adios2::cxx11_mpi) else() @@ -120,6 +120,13 @@ endif() link_libraries(${DEPENDENCIES}) +get_cmake_property(all_vars VARIABLES) +foreach(_var ${all_vars}) + if(_var MATCHES "^Kokkos_.*") + message(STATUS "PRINTING ${_var}=${${_var}}") + endif() +endforeach() + if(TESTS) # ---------------------------------- Tests --------------------------------- # include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/tests.cmake) @@ -129,7 +136,7 @@ elseif(BENCHMARK) else() # ----------------------------------- GUI ---------------------------------- # if(${gui}) - find_or_fetch_dependency(nttiny FALSE) + find_or_fetch_dependency(nttiny FALSE QUIET) endif() # ------------------------------- Main source ------------------------------ # diff --git a/cmake/MPIConfig.cmake b/cmake/MPIConfig.cmake deleted file mode 100644 index d1bfeaab2..000000000 --- a/cmake/MPIConfig.cmake +++ /dev/null @@ -1,4 +0,0 @@ -find_package(MPI REQUIRED) -include_directories(${MPI_CXX_INCLUDE_PATH}) -add_compile_options("-D MPI_ENABLED") - diff --git a/cmake/adios2Config.cmake b/cmake/adios2Config.cmake index 5c480f3d8..7e0951949 100644 --- a/cmake/adios2Config.cmake +++ b/cmake/adios2Config.cmake @@ -23,5 +23,3 @@ set(ADIOS2_USE_MPI set(ADIOS2_USE_CUDA OFF CACHE BOOL "Use CUDA for ADIOS2") - -add_compile_options("-D OUTPUT_ENABLED") diff --git a/cmake/defaults.cmake b/cmake/defaults.cmake index 46b4609c5..78e8230c7 100644 --- a/cmake/defaults.cmake +++ b/cmake/defaults.cmake @@ -51,42 +51,6 @@ endif() set_property(CACHE default_gui PROPERTY TYPE BOOL) -if(DEFINED ENV{Kokkos_ENABLE_CUDA}) - set(default_KOKKOS_ENABLE_CUDA - $ENV{Kokkos_ENABLE_CUDA} - CACHE INTERNAL "Default flag for CUDA") -else() - set(default_KOKKOS_ENABLE_CUDA - OFF - CACHE INTERNAL "Default flag for CUDA") -endif() - -set_property(CACHE default_KOKKOS_ENABLE_CUDA PROPERTY TYPE BOOL) - -if(DEFINED ENV{Kokkos_ENABLE_HIP}) - set(default_KOKKOS_ENABLE_HIP - $ENV{Kokkos_ENABLE_HIP} - CACHE INTERNAL "Default flag for HIP") -else() - set(default_KOKKOS_ENABLE_HIP - OFF - CACHE INTERNAL "Default flag for HIP") -endif() - -set_property(CACHE default_KOKKOS_ENABLE_HIP PROPERTY TYPE BOOL) - -if(DEFINED ENV{Kokkos_ENABLE_OPENMP}) - set(default_KOKKOS_ENABLE_OPENMP - $ENV{Kokkos_ENABLE_OPENMP} - CACHE INTERNAL "Default flag for OpenMP") -else() - set(default_KOKKOS_ENABLE_OPENMP - OFF - CACHE INTERNAL "Default flag for OpenMP") -endif() - -set_property(CACHE default_KOKKOS_ENABLE_OPENMP PROPERTY TYPE BOOL) - if(DEFINED ENV{Entity_ENABLE_MPI}) set(default_mpi $ENV{Entity_ENABLE_MPI} diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index 06a3e6a1f..a6712feb2 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -34,12 +34,17 @@ function(check_internet_connection) endif() endfunction() -function(find_or_fetch_dependency package_name header_only) +function(find_or_fetch_dependency package_name header_only mode) if(NOT header_only) - find_package(${package_name} QUIET) + find_package(${package_name} ${mode}) endif() if(NOT ${package_name}_FOUND) + if (${package_name} STREQUAL "Kokkos") + include(${CMAKE_CURRENT_SOURCE_DIR}/kokkosConfig.cmake) + elseif(${package_name} STREQUAL "adios2") + include(${CMAKE_CURRENT_SOURCE_DIR}/adios2Config.cmake) + endif() if(DEFINED ${package_name}_REPOSITORY AND NOT FETCHCONTENT_FULLY_DISCONNECTED) # fetching package @@ -52,7 +57,7 @@ function(find_or_fetch_dependency package_name header_only) FetchContent_Declare( ${package_name} GIT_REPOSITORY ${${package_name}_REPOSITORY} - GIT_TAG 4.3.00) + GIT_TAG 4.5.01) else() FetchContent_Declare(${package_name} GIT_REPOSITORY ${${package_name}_REPOSITORY}) diff --git a/cmake/kokkosConfig.cmake b/cmake/kokkosConfig.cmake index 63c32622d..3d038cc19 100644 --- a/cmake/kokkosConfig.cmake +++ b/cmake/kokkosConfig.cmake @@ -27,51 +27,6 @@ else() CACHE BOOL "Kokkos debug bounds check") endif() -set(Kokkos_ENABLE_HIP - ${default_KOKKOS_ENABLE_HIP} - CACHE BOOL "Enable HIP") -set(Kokkos_ENABLE_CUDA - ${default_KOKKOS_ENABLE_CUDA} - CACHE BOOL "Enable CUDA") -set(Kokkos_ENABLE_OPENMP - ${default_KOKKOS_ENABLE_OPENMP} - CACHE BOOL "Enable OpenMP") - -# set memory space -if(${Kokkos_ENABLE_CUDA}) - add_compile_definitions(CUDA_ENABLED) - set(ACC_MEM_SPACE Kokkos::CudaSpace) -elseif(${Kokkos_ENABLE_HIP}) - add_compile_definitions(HIP_ENABLED) - set(ACC_MEM_SPACE Kokkos::HIPSpace) -else() - set(ACC_MEM_SPACE Kokkos::HostSpace) -endif() - -set(HOST_MEM_SPACE Kokkos::HostSpace) - -# set execution space -if(${Kokkos_ENABLE_CUDA}) - set(ACC_EXE_SPACE Kokkos::Cuda) -elseif(${Kokkos_ENABLE_HIP}) - set(ACC_EXE_SPACE Kokkos::HIP) -elseif(${Kokkos_ENABLE_OPENMP}) - set(ACC_EXE_SPACE Kokkos::OpenMP) -else() - set(ACC_EXE_SPACE Kokkos::Serial) -endif() - -if(${Kokkos_ENABLE_OPENMP}) - set(HOST_EXE_SPACE Kokkos::OpenMP) -else() - set(HOST_EXE_SPACE Kokkos::Serial) -endif() - -add_compile_options("-D AccelExeSpace=${ACC_EXE_SPACE}") -add_compile_options("-D AccelMemSpace=${ACC_MEM_SPACE}") -add_compile_options("-D HostExeSpace=${HOST_EXE_SPACE}") -add_compile_options("-D HostMemSpace=${HOST_MEM_SPACE}") - if(${BUILD_TESTING} STREQUAL "OFF") set(Kokkos_ENABLE_TESTS OFF diff --git a/legacy/src/pic/particles/particle_pusher.hpp b/legacy/src/pic/particles/particle_pusher.hpp index 4c0ec639a..7991a95a4 100644 --- a/legacy/src/pic/particles/particle_pusher.hpp +++ b/legacy/src/pic/particles/particle_pusher.hpp @@ -1,14 +1,13 @@ #ifndef PIC_PARTICLE_PUSHER_H #define PIC_PARTICLE_PUSHER_H -#include "wrapper.h" - -#include "pic.h" +#include "utils/qmath.h" #include "io/output.h" #include "meshblock/meshblock.h" #include "meshblock/particles.h" -#include "utils/qmath.h" +#include "pic.h" +#include "wrapper.h" #include METRIC_HEADER #include @@ -73,35 +72,34 @@ namespace ntt { real_t time, real_t coeff, real_t dt, - ProblemGenerator& pgen) : - EB { mblock.em }, - i1 { particles.i1 }, - i2 { particles.i2 }, - i3 { particles.i3 }, - i1_prev { particles.i1_prev }, - i2_prev { particles.i2_prev }, - i3_prev { particles.i3_prev }, - dx1 { particles.dx1 }, - dx2 { particles.dx2 }, - dx3 { particles.dx3 }, - dx1_prev { particles.dx1_prev }, - dx2_prev { particles.dx2_prev }, - dx3_prev { particles.dx3_prev }, - ux1 { particles.ux1 }, - ux2 { particles.ux2 }, - ux3 { particles.ux3 }, - phi { particles.phi }, - tag { particles.tag }, - metric { mblock.metric }, - time { time }, - coeff { coeff }, - dt { dt }, - ni1 { (int)mblock.Ni1() }, - ni2 { (int)mblock.Ni2() }, - ni3 { (int)mblock.Ni3() } + ProblemGenerator& pgen) + : EB { mblock.em } + , i1 { particles.i1 } + , i2 { particles.i2 } + , i3 { particles.i3 } + , i1_prev { particles.i1_prev } + , i2_prev { particles.i2_prev } + , i3_prev { particles.i3_prev } + , dx1 { particles.dx1 } + , dx2 { particles.dx2 } + , dx3 { particles.dx3 } + , dx1_prev { particles.dx1_prev } + , dx2_prev { particles.dx2_prev } + , dx3_prev { particles.dx3_prev } + , ux1 { particles.ux1 } + , ux2 { particles.ux2 } + , ux3 { particles.ux3 } + , phi { particles.phi } + , tag { particles.tag } + , metric { mblock.metric } + , time { time } + , coeff { coeff } + , dt { dt } + , ni1 { (int)mblock.Ni1() } + , ni2 { (int)mblock.Ni2() } + , ni3 { (int)mblock.Ni3() } #ifdef EXTERNAL_FORCE - , - pgen { pgen } + , pgen { pgen } #endif { (void)pgen; @@ -237,7 +235,7 @@ namespace ntt { const auto coeff = charge_ovr_mass * HALF * dt * params.B0(); Kokkos::parallel_for( "ParticlesPush", - Kokkos::RangePolicy(0, particles.npart()), + Kokkos::RangePolicy(0, particles.npart()), Pusher_kernel(mblock, particles, time, coeff, dt, pgen)); } @@ -638,9 +636,9 @@ namespace ntt { template template Inline void Pusher_kernel::get3VelCntrv(T, - index_t& p, + index_t& p, vec_t& xp, - vec_t& v) const { + vec_t& v) const { metric.v3_Cart2Cntrv(xp, { ux1(p), ux2(p), ux3(p) }, v); auto inv_energy { ONE / getEnergy(T {}, p) }; v[0] *= inv_energy; @@ -666,7 +664,8 @@ namespace ntt { } template <> - Inline void Pusher_kernel::getPrtlPos(index_t& p, coord_t& xp) const { + Inline void Pusher_kernel::getPrtlPos(index_t& p, + coord_t& xp) const { xp[0] = static_cast(i1(p)) + static_cast(dx1(p)); xp[1] = static_cast(i2(p)) + static_cast(dx2(p)); xp[2] = phi(p); @@ -1066,7 +1065,7 @@ namespace ntt { #else template Inline void Pusher_kernel::initForce(coord_t& xp, - vec_t& force_Cart) const { + vec_t& force_Cart) const { coord_t xp_Ph { ZERO }; coord_t xp_Code { ZERO }; for (short d { 0 }; d < static_cast(PrtlCoordD); ++d) { diff --git a/legacy/tests/kernels-gr.cpp b/legacy/tests/kernels-gr.cpp index 84a0c952b..6962f7c9f 100644 --- a/legacy/tests/kernels-gr.cpp +++ b/legacy/tests/kernels-gr.cpp @@ -1,16 +1,16 @@ -#include "wrapper.h" - #include #include #include -#include METRIC_HEADER +#include "wrapper.h" -#include "particle_macros.h" +#include METRIC_HEADER #include "kernels/particle_pusher_gr.hpp" +#include "particle_macros.h" + template void put_value(ntt::array_t& arr, T value, int i) { auto arr_h = Kokkos::create_mirror_view(arr); @@ -154,9 +154,10 @@ auto main(int argc, char* argv[]) -> int { static_cast(1.0e-5), 10, boundaries); - Kokkos::parallel_for("ParticlesPush", - Kokkos::RangePolicy(0, 1), - kernel); + Kokkos::parallel_for( + "ParticlesPush", + Kokkos::RangePolicy(0, 1), + kernel); auto [ra, tha] = get_physical_coord(0, i1, i2, dx1, dx2, metric); const real_t pha = get_value(phi, 0); @@ -207,4 +208,4 @@ auto main(int argc, char* argv[]) -> int { ntt::GlobalFinalize(); return 0; -} \ No newline at end of file +} diff --git a/legacy/tests/kernels-sr.cpp b/legacy/tests/kernels-sr.cpp index 59ce0646b..3f64122cd 100644 --- a/legacy/tests/kernels-sr.cpp +++ b/legacy/tests/kernels-sr.cpp @@ -1,17 +1,17 @@ -#include "wrapper.h" - #include #include #include +#include "wrapper.h" + #include METRIC_HEADER #include PGEN_HEADER -#include "particle_macros.h" - #include "kernels/particle_pusher_sr.hpp" +#include "particle_macros.h" + template void put_value(ntt::array_t& arr, T value, int i) { auto arr_h = Kokkos::create_mirror_view(arr); @@ -181,9 +181,10 @@ auto main(int argc, char* argv[]) -> int { ZERO, ZERO, ZERO); - Kokkos::parallel_for("ParticlesPush", - Kokkos::RangePolicy(0, 1), - kernel); + Kokkos::parallel_for( + "ParticlesPush", + Kokkos::RangePolicy(0, 1), + kernel); auto [xa, ya] = get_cartesian_coord(0, i1, i2, dx1, dx2, phi, metric); if (!ntt::AlmostEqual(xa, @@ -221,4 +222,4 @@ auto main(int argc, char* argv[]) -> int { ntt::GlobalFinalize(); return 0; -} \ No newline at end of file +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a41b84900..db2ab4c92 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -17,7 +17,6 @@ # # * kokkos [required] # * plog [required] -# * toml11 [required] # * ADIOS2 [optional] # * mpi [optional] # ------------------------------ diff --git a/src/checkpoint/tests/checkpoint-mpi.cpp b/src/checkpoint/tests/checkpoint-mpi.cpp index f97202ab1..bc6d6038a 100644 --- a/src/checkpoint/tests/checkpoint-mpi.cpp +++ b/src/checkpoint/tests/checkpoint-mpi.cpp @@ -144,8 +144,8 @@ auto main(int argc, char* argv[]) -> int { writer.saveField("em", field1); writer.saveField("em0", field2); - writer.savePerDomainVariable("s1_npart", 1, 0, npart1); - writer.savePerDomainVariable("s2_npart", 1, 0, npart2); + writer.savePerDomainVariable("s1_npart", 1, 0, npart1); + writer.savePerDomainVariable("s2_npart", 1, 0, npart2); writer.saveParticleQuantity("s1_i1", npart1_globtot, diff --git a/src/checkpoint/tests/checkpoint-nompi.cpp b/src/checkpoint/tests/checkpoint-nompi.cpp index 23dbd8871..be5e97e24 100644 --- a/src/checkpoint/tests/checkpoint-nompi.cpp +++ b/src/checkpoint/tests/checkpoint-nompi.cpp @@ -105,8 +105,8 @@ auto main(int argc, char* argv[]) -> int { writer.saveField("em", field1); writer.saveField("em0", field2); - writer.savePerDomainVariable("s1_npart", 1, 0, npart1); - writer.savePerDomainVariable("s2_npart", 1, 0, npart2); + writer.savePerDomainVariable("s1_npart", 1, 0, npart1); + writer.savePerDomainVariable("s2_npart", 1, 0, npart2); writer.saveParticleQuantity("s1_i1", npart1, 0, npart1, i1); writer.saveParticleQuantity("s1_ux1", npart1, 0, npart1, u1); diff --git a/src/checkpoint/writer.cpp b/src/checkpoint/writer.cpp index a70d8de09..fe77d3b56 100644 --- a/src/checkpoint/writer.cpp +++ b/src/checkpoint/writer.cpp @@ -274,9 +274,9 @@ namespace checkpoint { std::size_t, double); template void Writer::savePerDomainVariable(const std::string&, - std::size_t, - std::size_t, - npart_t); + std::size_t, + std::size_t, + npart_t); template void Writer::saveField(const std::string&, const ndfield_t&); diff --git a/src/engines/CMakeLists.txt b/src/engines/CMakeLists.txt index 6da2f4efd..9e51330ec 100644 --- a/src/engines/CMakeLists.txt +++ b/src/engines/CMakeLists.txt @@ -25,7 +25,6 @@ # # * kokkos [required] # * plog [required] -# * toml11 [required] # * adios2 [optional] # * hdf5 [optional] # * mpi [optional] diff --git a/src/framework/CMakeLists.txt b/src/framework/CMakeLists.txt index 8802f696b..d3b68084c 100644 --- a/src/framework/CMakeLists.txt +++ b/src/framework/CMakeLists.txt @@ -28,7 +28,6 @@ # # * kokkos [required] # * plog [required] -# * toml11 [required] # * ADIOS2 [optional] # * mpi [optional] # ------------------------------ diff --git a/src/framework/containers/particles.cpp b/src/framework/containers/particles.cpp index 048d57cde..9ec8810dc 100644 --- a/src/framework/containers/particles.cpp +++ b/src/framework/containers/particles.cpp @@ -218,13 +218,13 @@ namespace ntt { Kokkos::Experimental::fill( "TagAliveParticles", - AccelExeSpace(), + Kokkos::DefaultExecutionSpace(), Kokkos::subview(this_tag, std::make_pair(static_cast(0), n_alive)), ParticleTag::alive); Kokkos::Experimental::fill( "TagDeadParticles", - AccelExeSpace(), + Kokkos::DefaultExecutionSpace(), Kokkos::subview(this_tag, std::make_pair(n_alive, n_alive + n_dead)), ParticleTag::dead); diff --git a/src/framework/domain/comm_mpi.hpp b/src/framework/domain/comm_mpi.hpp index 8c6e532de..4f382be4a 100644 --- a/src/framework/domain/comm_mpi.hpp +++ b/src/framework/domain/comm_mpi.hpp @@ -83,7 +83,7 @@ namespace comm { (long int)(send_slice[0].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, comps.first }, { recv_slice[0].second, comps.second }), Lambda(index_t i1, index_t ci) { @@ -96,7 +96,7 @@ namespace comm { (long int)(send_slice[1].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, comps.first }, { recv_slice[0].second, recv_slice[1].second, comps.second }), Lambda(index_t i1, index_t i2, index_t ci) { @@ -111,7 +111,7 @@ namespace comm { (long int)(send_slice[2].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, recv_slice[2].first, @@ -239,7 +239,7 @@ namespace comm { const auto offset_c = comps.first; Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, comps.first }, { recv_slice[0].second, comps.second }), Lambda(index_t i1, index_t ci) { @@ -251,7 +251,7 @@ namespace comm { const auto offset_c = comps.first; Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, comps.first }, { recv_slice[0].second, recv_slice[1].second, comps.second }), Lambda(index_t i1, index_t i2, index_t ci) { @@ -266,7 +266,7 @@ namespace comm { const auto offset_c = comps.first; Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, recv_slice[2].first, diff --git a/src/framework/domain/comm_nompi.hpp b/src/framework/domain/comm_nompi.hpp index 197d336fa..b477ac176 100644 --- a/src/framework/domain/comm_nompi.hpp +++ b/src/framework/domain/comm_nompi.hpp @@ -70,7 +70,7 @@ namespace comm { (long int)(send_slice[0].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, comps.first }, { recv_slice[0].second, comps.second }), Lambda(index_t i1, index_t ci) { @@ -83,7 +83,7 @@ namespace comm { (long int)(send_slice[1].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, comps.first }, { recv_slice[0].second, recv_slice[1].second, comps.second }), Lambda(index_t i1, index_t i2, index_t ci) { @@ -98,7 +98,7 @@ namespace comm { (long int)(send_slice[2].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, recv_slice[2].first, diff --git a/src/framework/tests/CMakeLists.txt b/src/framework/tests/CMakeLists.txt index ce188e9f1..27c456610 100644 --- a/src/framework/tests/CMakeLists.txt +++ b/src/framework/tests/CMakeLists.txt @@ -5,7 +5,6 @@ # # * kokkos [required] # * plog [required] -# * toml11 [required] # * mpi [optional] # * adios2 [optional] # diff --git a/src/global/arch/kokkos_aliases.cpp b/src/global/arch/kokkos_aliases.cpp index d6622b0e5..25397af8e 100644 --- a/src/global/arch/kokkos_aliases.cpp +++ b/src/global/arch/kokkos_aliases.cpp @@ -9,73 +9,75 @@ auto CreateParticleRangePolicy(npart_t p1, npart_t p2) -> range_t { } template <> -auto CreateRangePolicy( - const tuple_t& i1, - const tuple_t& i2) -> range_t { +auto CreateRangePolicy(const tuple_t& i1, + const tuple_t& i2) + -> range_t { index_t i1min = i1[0]; index_t i1max = i2[0]; - return Kokkos::RangePolicy(i1min, i1max); + return Kokkos::RangePolicy(i1min, i1max); } template <> -auto CreateRangePolicy( - const tuple_t& i1, - const tuple_t& i2) -> range_t { +auto CreateRangePolicy(const tuple_t& i1, + const tuple_t& i2) + -> range_t { index_t i1min = i1[0]; index_t i1max = i2[0]; index_t i2min = i1[1]; index_t i2max = i2[1]; - return Kokkos::MDRangePolicy, AccelExeSpace>({ i1min, i2min }, - { i1max, i2max }); + return Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( + { i1min, i2min }, + { i1max, i2max }); } template <> -auto CreateRangePolicy( - const tuple_t& i1, - const tuple_t& i2) -> range_t { +auto CreateRangePolicy(const tuple_t& i1, + const tuple_t& i2) + -> range_t { index_t i1min = i1[0]; index_t i1max = i2[0]; index_t i2min = i1[1]; index_t i2max = i2[1]; index_t i3min = i1[2]; index_t i3max = i2[2]; - return Kokkos::MDRangePolicy, AccelExeSpace>( + return Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { i1min, i2min, i3min }, { i1max, i2max, i3max }); } template <> -auto CreateRangePolicyOnHost( - const tuple_t& i1, - const tuple_t& i2) -> range_h_t { +auto CreateRangePolicyOnHost(const tuple_t& i1, + const tuple_t& i2) + -> range_h_t { index_t i1min = i1[0]; index_t i1max = i2[0]; - return Kokkos::RangePolicy(i1min, i1max); + return Kokkos::RangePolicy(i1min, i1max); } template <> -auto CreateRangePolicyOnHost( - const tuple_t& i1, - const tuple_t& i2) -> range_h_t { +auto CreateRangePolicyOnHost(const tuple_t& i1, + const tuple_t& i2) + -> range_h_t { index_t i1min = i1[0]; index_t i1max = i2[0]; index_t i2min = i1[1]; index_t i2max = i2[1]; - return Kokkos::MDRangePolicy, HostExeSpace>({ i1min, i2min }, - { i1max, i2max }); + return Kokkos::MDRangePolicy, Kokkos::DefaultHostExecutionSpace>( + { i1min, i2min }, + { i1max, i2max }); } template <> -auto CreateRangePolicyOnHost( - const tuple_t& i1, - const tuple_t& i2) -> range_h_t { +auto CreateRangePolicyOnHost(const tuple_t& i1, + const tuple_t& i2) + -> range_h_t { index_t i1min = i1[0]; index_t i1max = i2[0]; index_t i2min = i1[1]; index_t i2max = i2[1]; index_t i3min = i1[2]; index_t i3max = i2[2]; - return Kokkos::MDRangePolicy, HostExeSpace>( + return Kokkos::MDRangePolicy, Kokkos::DefaultHostExecutionSpace>( { i1min, i2min, i3min }, { i1max, i2max, i3max }); } diff --git a/src/global/arch/kokkos_aliases.h b/src/global/arch/kokkos_aliases.h index e1a759bed..712fc6eff 100644 --- a/src/global/arch/kokkos_aliases.h +++ b/src/global/arch/kokkos_aliases.h @@ -34,7 +34,7 @@ namespace math = Kokkos; template -using array_t = Kokkos::View; +using array_t = Kokkos::View; // Array mirror alias of arbitrary type template @@ -174,17 +174,17 @@ namespace kokkos_aliases_hidden { template <> struct range_impl { - using type = Kokkos::RangePolicy; + using type = Kokkos::RangePolicy; }; template <> struct range_impl { - using type = Kokkos::MDRangePolicy, AccelExeSpace>; + using type = Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>; }; template <> struct range_impl { - using type = Kokkos::MDRangePolicy, AccelExeSpace>; + using type = Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>; }; } // namespace kokkos_aliases_hidden @@ -201,17 +201,17 @@ namespace kokkos_aliases_hidden { template <> struct range_h_impl { - using type = Kokkos::RangePolicy; + using type = Kokkos::RangePolicy; }; template <> struct range_h_impl { - using type = Kokkos::MDRangePolicy, HostExeSpace>; + using type = Kokkos::MDRangePolicy, Kokkos::DefaultHostExecutionSpace>; }; template <> struct range_h_impl { - using type = Kokkos::MDRangePolicy, HostExeSpace>; + using type = Kokkos::MDRangePolicy, Kokkos::DefaultHostExecutionSpace>; }; } // namespace kokkos_aliases_hidden @@ -234,8 +234,8 @@ auto CreateParticleRangePolicy(npart_t, npart_t) -> range_t; * @returns Kokkos::RangePolicy or Kokkos::MDRangePolicy in the accelerator execution space. */ template -auto CreateRangePolicy(const tuple_t&, - const tuple_t&) -> range_t; +auto CreateRangePolicy(const tuple_t&, const tuple_t&) + -> range_t; /** * @brief Function template for generating ND Kokkos range policy on the host. @@ -249,8 +249,8 @@ auto CreateRangePolicyOnHost(const tuple_t&, const tuple_t&) -> range_h_t; // Random number pool/generator type alias -using random_number_pool_t = Kokkos::Random_XorShift1024_Pool; -using random_generator_t = typename random_number_pool_t::generator_type; +using random_number_pool_t = Kokkos::Random_XorShift1024_Pool; +using random_generator_t = typename random_number_pool_t::generator_type; // Random number generator functions template diff --git a/src/kernels/tests/prtls_to_phys.cpp b/src/kernels/tests/prtls_to_phys.cpp index 4719fe6a1..962c21b5c 100644 --- a/src/kernels/tests/prtls_to_phys.cpp +++ b/src/kernels/tests/prtls_to_phys.cpp @@ -132,7 +132,7 @@ void testPrtl2PhysSR(const std::vector& res, extent = { ext[0], - {ZERO, constant::PI} + { ZERO, constant::PI } }; const M metric { res, extent, params }; @@ -177,28 +177,29 @@ void testPrtl2PhysSR(const std::vector& res, array_t buff_ux3 { "buff_ux3", nprtl / stride }; array_t buff_wei { "buff_wei", nprtl / stride }; - Kokkos::parallel_for("Init", - Kokkos::RangePolicy(0, nprtl / stride), - kernel::PrtlToPhys_kernel(stride, - buff_x1, - buff_x2, - buff_x3, - buff_ux1, - buff_ux2, - buff_ux3, - buff_wei, - i1, - i2, - i3, - dx1, - dx2, - dx3, - ux1, - ux2, - ux3, - phi, - weight, - metric)); + Kokkos::parallel_for( + "Init", + Kokkos::RangePolicy(0, nprtl / stride), + kernel::PrtlToPhys_kernel(stride, + buff_x1, + buff_x2, + buff_x3, + buff_ux1, + buff_ux2, + buff_ux3, + buff_wei, + i1, + i2, + i3, + dx1, + dx2, + dx3, + ux1, + ux2, + ux3, + phi, + weight, + metric)); Kokkos::parallel_for("Check", nprtl / stride, Checker(metric, From 18f02f64fc353752bc70d90d1d5ad31cc99e9d8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Tue, 1 Apr 2025 19:46:46 -0500 Subject: [PATCH 079/176] fix unit conversion bug in field reset --- setups/srpic/shock/pgen.hpp | 40 ++++++++----------------------------- 1 file changed, 8 insertions(+), 32 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index cf58b94c6..ad260bda0 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -9,6 +9,7 @@ #include "utils/numeric.h" #include "archetypes/energy_dist.h" +#include "archetypes/field_setter.h" #include "archetypes/particle_injector.h" #include "archetypes/problem_generator.h" #include "framework/domain/metadomain.h" @@ -231,38 +232,13 @@ namespace user { x_max[d] = extent[d].second; } - // reset fields - std::vector comps = { em::bx1, em::bx2, em::bx3, - em::ex1, em::ex2, em::ex3 }; - - // loop over all components - for (const auto& comp : comps) { - - // get initial field value of component - auto value = ResetFields((em)comp); - - if constexpr (M::Dim == Dim::_1D) { - Kokkos::deep_copy(Kokkos::subview(domain.fields.em, - std::make_pair(x_min[0], x_max[0]), - comp), - value); - } else if constexpr (M::Dim == Dim::_2D) { - Kokkos::deep_copy(Kokkos::subview(domain.fields.em, - std::make_pair(x_min[0], x_max[0]), - std::make_pair(x_min[1], x_max[1]), - comp), - value); - } else if constexpr (M::Dim == Dim::_3D) { - Kokkos::deep_copy(Kokkos::subview(domain.fields.em, - std::make_pair(x_min[0], x_max[0]), - std::make_pair(x_min[1], x_max[1]), - std::make_pair(x_min[2], x_max[2]), - comp), - value); - } else { - raise::Error("Invalid dimension", HERE); - } - } + Kokkos::parallel_for("ResetFields", + CreateRangePolicy(x_min, x_max), + arch::SetEMFields_kernel { + domain.fields.em, + init_flds, + domain.mesh.metric }); + /* tag particles inside the injection zone as dead From 3ccdd4db96ec7b248d21e7febb79d99c260031b8 Mon Sep 17 00:00:00 2001 From: hayk Date: Wed, 2 Apr 2025 01:27:21 -0400 Subject: [PATCH 080/176] improved report + fetch adios2 + rm hdf5 dependency --- .gitmodules | 1 - CMakeLists.txt | 17 -- cmake/adios2Config.cmake | 10 +- cmake/config.cmake | 9 +- cmake/defaults.cmake | 2 +- cmake/dependencies.cmake | 31 ++- cmake/report.cmake | 366 +++++++++------------------------ cmake/styling.cmake | 140 +++++++++++-- src/engines/CMakeLists.txt | 2 +- src/engines/engine_printer.cpp | 9 - 10 files changed, 259 insertions(+), 328 deletions(-) diff --git a/.gitmodules b/.gitmodules index e06c332fe..577d08ea6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,7 +4,6 @@ [submodule "extern/adios2"] path = extern/adios2 url = https://github.com/ornladios/ADIOS2.git - branch = master [submodule "extern/Kokkos"] path = extern/Kokkos url = https://github.com/kokkos/kokkos.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 0409c4107..22d1e64b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,16 +100,6 @@ endif() # Output if(${output}) find_or_fetch_dependency(adios2 FALSE QUIET) - if(NOT DEFINED ENV{HDF5_ROOT}) - if(DEFINED ENV{CONDA_PREFIX}) - execute_process(COMMAND bash -c "conda list | grep \"hdf5\" -q" - RESULT_VARIABLE HDF5_INSTALLED) - if(HDF5_INSTALLED EQUAL 0) - set(HDF5_ROOT $ENV{CONDA_PREFIX}) - endif() - endif() - endif() - find_package(HDF5 REQUIRED) add_compile_options("-D OUTPUT_ENABLED") if(${mpi}) set(DEPENDENCIES ${DEPENDENCIES} adios2::cxx11_mpi) @@ -120,13 +110,6 @@ endif() link_libraries(${DEPENDENCIES}) -get_cmake_property(all_vars VARIABLES) -foreach(_var ${all_vars}) - if(_var MATCHES "^Kokkos_.*") - message(STATUS "PRINTING ${_var}=${${_var}}") - endif() -endforeach() - if(TESTS) # ---------------------------------- Tests --------------------------------- # include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/tests.cmake) diff --git a/cmake/adios2Config.cmake b/cmake/adios2Config.cmake index 7e0951949..a4ce46179 100644 --- a/cmake/adios2Config.cmake +++ b/cmake/adios2Config.cmake @@ -12,14 +12,18 @@ set(ADIOS2_USE_Fortran CACHE BOOL "Use Fortran for ADIOS2") # Format/compression support -set(ADIOS2_USE_ZeroMQ - OFF - CACHE BOOL "Use ZeroMQ for ADIOS2") +set(ADIOS2_USE_HDF5 + ON + CACHE BOOL "Use HDF5 for ADIOS2") set(ADIOS2_USE_MPI ${mpi} CACHE BOOL "Use MPI for ADIOS2") +set(ADIOS2_USE_ZeroMQ + OFF + CACHE BOOL "Use ZeroMQ for ADIOS2") + set(ADIOS2_USE_CUDA OFF CACHE BOOL "Use CUDA for ADIOS2") diff --git a/cmake/config.cmake b/cmake/config.cmake index 58dd467e9..ab54717f1 100644 --- a/cmake/config.cmake +++ b/cmake/config.cmake @@ -34,13 +34,10 @@ function(set_problem_generator pgen_name) ) endif() set(PGEN - ${pgen_name} - PARENT_SCOPE) + ${pgen_name} PARENT_SCOPE) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/setups/${pgen_name}) set(PGEN_FOUND - TRUE - PARENT_SCOPE) + TRUE PARENT_SCOPE) set(problem_generators - ${PGEN_NAMES} - PARENT_SCOPE) + ${PGEN_NAMES} PARENT_SCOPE) endfunction() diff --git a/cmake/defaults.cmake b/cmake/defaults.cmake index 78e8230c7..2c72f5d7d 100644 --- a/cmake/defaults.cmake +++ b/cmake/defaults.cmake @@ -33,7 +33,7 @@ if(DEFINED ENV{Entity_ENABLE_OUTPUT}) CACHE INTERNAL "Default flag for output") else() set(default_output - OFF + ON CACHE INTERNAL "Default flag for output") endif() diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index a6712feb2..a31dfdea5 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -4,9 +4,10 @@ set(Kokkos_REPOSITORY set(plog_REPOSITORY https://github.com/SergiusTheBest/plog.git CACHE STRING "plog repository") +set (adios2_REPOSITORY + https://github.com/ornladios/ADIOS2.git + CACHE STRING "ADIOS2 repository") -# set (adios2_REPOSITORY https://github.com/ornladios/ADIOS2.git CACHE STRING -# "ADIOS2 repository") function(check_internet_connection) if(OFFLINE STREQUAL "ON") set(FETCHCONTENT_FULLY_DISCONNECTED @@ -41,9 +42,9 @@ function(find_or_fetch_dependency package_name header_only mode) if(NOT ${package_name}_FOUND) if (${package_name} STREQUAL "Kokkos") - include(${CMAKE_CURRENT_SOURCE_DIR}/kokkosConfig.cmake) + include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/kokkosConfig.cmake) elseif(${package_name} STREQUAL "adios2") - include(${CMAKE_CURRENT_SOURCE_DIR}/adios2Config.cmake) + include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/adios2Config.cmake) endif() if(DEFINED ${package_name}_REPOSITORY AND NOT FETCHCONTENT_FULLY_DISCONNECTED) @@ -70,6 +71,9 @@ function(find_or_fetch_dependency package_name header_only mode) set(${package_name}_SRC ${CMAKE_CURRENT_BINARY_DIR}/_deps/${lower_pckg_name}-src CACHE PATH "Path to ${package_name} src") + set(${package_name}_BUILD_DIR + ${CMAKE_CURRENT_BINARY_DIR}/_deps/${lower_pckg_name}-build + CACHE PATH "Path to ${package_name} build") set(${package_name}_FETCHED TRUE CACHE BOOL "Whether ${package_name} was fetched") @@ -96,7 +100,7 @@ function(find_or_fetch_dependency package_name header_only mode) endif() add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extern/${package_name} - extern/${package_name}) + extern/${package_name}) set(${package_name}_SRC ${CMAKE_CURRENT_SOURCE_DIR}/extern/${package_name} CACHE PATH "Path to ${package_name} src") @@ -132,7 +136,24 @@ function(find_or_fetch_dependency package_name header_only mode) ${Kokkos_VERSION} CACHE INTERNAL "Kokkos version") endif() + if(NOT DEFINED Kokkos_ARCH OR Kokkos_ARCH STREQUAL "" + OR NOT DEFINED Kokkos_DEVICES OR Kokkos_DEVICES STREQUAL "") + if(${Kokkos_FOUND}) + include(${Kokkos_DIR}/KokkosConfigCommon.cmake) + elseif(NOT ${Kokkos_BUILD_DIR} STREQUAL "") + include(${Kokkos_BUILD_DIR}/KokkosConfigCommon.cmake) + else() + message(STATUS "${Red}Kokkos_DIR and Kokkos_BUILD_DIR not set.${ColorReset}") + endif() + endif() + set(Kokkos_ARCH + ${Kokkos_ARCH} PARENT_SCOPE) + set(Kokkos_DEVICES + ${Kokkos_DEVICES} PARENT_SCOPE) endif() + set(${package_name}_FOUND ${${package_name}_FOUND} PARENT_SCOPE) + set(${package_name}_FETCHED ${${package_name}_FETCHED} PARENT_SCOPE) + set(${package_name}_BUILD_DIR ${${package_name}_BUILD_DIR} PARENT_SCOPE) endfunction() check_internet_connection() diff --git a/cmake/report.cmake b/cmake/report.cmake index 13dde63f7..60d846144 100644 --- a/cmake/report.cmake +++ b/cmake/report.cmake @@ -1,95 +1,3 @@ -function(PadTo Text Padding Target Result) - set(rt ${Text}) - string(FIND ${rt} "${Magenta}" mg_fnd) - - if(mg_fnd GREATER -1) - string(REGEX REPLACE "${Esc}\\[35m" "" rt ${rt}) - endif() - - string(LENGTH "${rt}" TextLength) - math(EXPR PaddingNeeded "${Target} - ${TextLength}") - set(rt ${Text}) - - if(PaddingNeeded GREATER 0) - foreach(i RANGE 0 ${PaddingNeeded}) - set(rt "${rt}${Padding}") - endforeach() - else() - set(${rt} "${rt}") - endif() - - set(${Result} - "${rt}" - PARENT_SCOPE) -endfunction() - -function( - PrintChoices - Label - Flag - Choices - Value - Default - Color - OutputString - Multiline - Padding) - list(LENGTH "${Choices}" nchoices) - set(rstring "") - set(counter 0) - - foreach(ch ${Choices}) - if(${counter} EQUAL 0) - set(rstring_i "- ${Label}") - - if(NOT "${Flag}" STREQUAL "") - set(rstring_i "${rstring_i} [${Magenta}${Flag}${ColorReset}]") - endif() - - set(rstring_i "${rstring_i}:") - padto("${rstring_i}" " " ${Padding} rstring_i) - else() - set(rstring_i "") - - if(NOT ${counter} EQUAL ${nchoices}) - if(${Multiline} EQUAL 1) - set(rstring_i "${rstring_i}\n") - padto("${rstring_i}" " " ${Padding} rstring_i) - else() - set(rstring_i "${rstring_i}/") - endif() - endif() - endif() - - if(${ch} STREQUAL ${Value}) - if(${ch} STREQUAL "ON") - set(col ${Green}) - elseif(${ch} STREQUAL "OFF") - set(col ${Red}) - else() - set(col ${Color}) - endif() - else() - set(col ${Dim}) - endif() - - if(${ch} STREQUAL ${Default}) - set(col ${Underline}${col}) - endif() - - set(rstring_i "${rstring_i}${col}${ch}${ColorReset}") - math(EXPR counter "${counter} + 1") - set(rstring "${rstring}${rstring_i}") - set(rstring_i "") - endforeach() - - set(${OutputString} - "${rstring}" - PARENT_SCOPE) -endfunction() - -set(ON_OFF_VALUES "ON" "OFF") - if(${PGEN_FOUND}) printchoices( "Problem generator" @@ -99,8 +7,7 @@ if(${PGEN_FOUND}) ${default_pgen} "${Blue}" PGEN_REPORT - 1 - 36) + 0) endif() printchoices( @@ -111,7 +18,6 @@ printchoices( ${default_precision} "${Blue}" PRECISION_REPORT - 1 36) printchoices( "Output" @@ -121,17 +27,6 @@ printchoices( ${default_output} "${Green}" OUTPUT_REPORT - 0 - 36) -printchoices( - "GUI" - "gui" - "${ON_OFF_VALUES}" - ${gui} - ${default_gui} - "${Green}" - GUI_REPORT - 0 36) printchoices( "MPI" @@ -141,8 +36,7 @@ printchoices( OFF "${Green}" MPI_REPORT - 0 - 42) + 36) printchoices( "Debug mode" "DEBUG" @@ -151,131 +45,7 @@ printchoices( OFF "${Green}" DEBUG_REPORT - 0 - 42) - -printchoices( - "CUDA" - "Kokkos_ENABLE_CUDA" - "${ON_OFF_VALUES}" - ${Kokkos_ENABLE_CUDA} - "OFF" - "${Green}" - CUDA_REPORT - 0 - 42) -printchoices( - "HIP" - "Kokkos_ENABLE_HIP" - "${ON_OFF_VALUES}" - ${Kokkos_ENABLE_HIP} - "OFF" - "${Green}" - HIP_REPORT - 0 - 42) -printchoices( - "OpenMP" - "Kokkos_ENABLE_OPENMP" - "${ON_OFF_VALUES}" - ${Kokkos_ENABLE_OPENMP} - "OFF" - "${Green}" - OPENMP_REPORT - 0 - 42) - -printchoices( - "C++ compiler" - "CMAKE_CXX_COMPILER" - "${CMAKE_CXX_COMPILER} v${CMAKE_CXX_COMPILER_VERSION}" - "${CMAKE_CXX_COMPILER} v${CMAKE_CXX_COMPILER_VERSION}" - "N/A" - "${ColorReset}" - CXX_COMPILER_REPORT - 0 - 42) - -printchoices( - "C compiler" - "CMAKE_C_COMPILER" - "${CMAKE_C_COMPILER} v${CMAKE_C_COMPILER_VERSION}" - "${CMAKE_C_COMPILER} v${CMAKE_C_COMPILER_VERSION}" - "N/A" - "${ColorReset}" - C_COMPILER_REPORT - 0 - 42) - -get_cmake_property(_variableNames VARIABLES) -foreach(_variableName ${_variableNames}) - string(REGEX MATCH "Kokkos_ARCH_*" _isMatched ${_variableName}) - if(_isMatched) - get_property( - isSet - CACHE ${_variableName} - PROPERTY VALUE) - if(isSet STREQUAL "ON") - string(REGEX REPLACE "Kokkos_ARCH_" "" ARCH ${_variableName}) - break() - endif() - endif() -endforeach() -printchoices( - "Architecture" - "Kokkos_ARCH_*" - "${ARCH}" - "${ARCH}" - "N/A" - "${ColorReset}" - ARCH_REPORT - 0 - 42) - -if(${Kokkos_ENABLE_CUDA}) - if("${CMAKE_CUDA_COMPILER}" STREQUAL "") - execute_process(COMMAND which nvcc OUTPUT_VARIABLE CUDACOMP) - else() - set(CUDACOMP ${CMAKE_CUDA_COMPILER}) - endif() - - string(STRIP ${CUDACOMP} CUDACOMP) - - message(STATUS "CUDA compiler: ${CUDACOMP}") - execute_process( - COMMAND - bash -c - "${CUDACOMP} --version | grep release | sed -e 's/.*release //' -e 's/,.*//'" - OUTPUT_VARIABLE CUDACOMP_VERSION - OUTPUT_STRIP_TRAILING_WHITESPACE) - - printchoices( - "CUDA compiler" - "CMAKE_CUDA_COMPILER" - "${CUDACOMP}" - "${CUDACOMP}" - "N/A" - "${ColorReset}" - CUDA_COMPILER_REPORT - 0 - 42) -endif() - -if(${Kokkos_ENABLE_HIP}) - execute_process( - COMMAND bash -c "hipcc --version | grep HIP | cut -d ':' -f 2 | tr -d ' '" - OUTPUT_VARIABLE ROCM_VERSION - OUTPUT_STRIP_TRAILING_WHITESPACE) -endif() - -set(DOT_SYMBOL "${ColorReset}.") -set(DOTTED_LINE_SYMBOL - "${ColorReset}. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " -) - -set(DASHED_LINE_SYMBOL - "${ColorReset}....................................................................... " -) + 36) if(NOT ${PROJECT_VERSION_TWEAK} EQUAL 0) set(VERSION_SYMBOL @@ -287,7 +57,7 @@ else() ) endif() -message( +set(REPORT_TEXT "${Blue} __ __ /\\ \\__ __/\\ \\__ __ ___\\ \\ _\\/\\_\\ \\ _\\ __ __ @@ -296,55 +66,107 @@ message( \\ \\____\\ \\_\\ \\_\\ \\__\\\\ \\_\\ \\__\\\\ \\____ \\/\\_\\ \\/____/\\/_/\\/_/\\/__/ \\/_/\\/__/ \\/___/ \\/_/ /\\___/ -Entity ${VERSION_SYMBOL}\t\t \\/__/") -message("${DASHED_LINE_SYMBOL} -Main configurations") +Entity ${VERSION_SYMBOL}\t\t \\/__/" +) +string(APPEND REPORT_TEXT + ${ColorReset} "\n" +) + +string(APPEND REPORT_TEXT + ${DASHED_LINE_SYMBOL} "\n" + "Configurations" "\n" +) if(${PGEN_FOUND}) - message(" ${PGEN_REPORT}") + string(APPEND REPORT_TEXT + " " ${PGEN_REPORT} "\n" + ) endif() -message(" ${PRECISION_REPORT}") -message(" ${OUTPUT_REPORT}") -message("${DASHED_LINE_SYMBOL}\nCompile configurations") +string(APPEND REPORT_TEXT + " " ${PRECISION_REPORT} "\n" + " " ${OUTPUT_REPORT} "\n" +) -if(NOT "${ARCH_REPORT}" STREQUAL "") - message(" ${ARCH_REPORT}") -endif() -message(" ${CUDA_REPORT}") -message(" ${HIP_REPORT}") -message(" ${OPENMP_REPORT}") +string(REPLACE ";" "+" Kokkos_ARCH "${Kokkos_ARCH}") +string(REPLACE ";" "+" Kokkos_DEVICES "${Kokkos_DEVICES}") -message(" ${C_COMPILER_REPORT}") +string(APPEND REPORT_TEXT + " - ARCH [${Magenta}Kokkos_ARCH_***${ColorReset}]: ${Kokkos_ARCH}" "\n" + " - DEVICES [${Magenta}Kokkos_ENABLE_***${ColorReset}]: ${Kokkos_DEVICES}" "\n" + " " ${MPI_REPORT} "\n" + " " ${DEBUG_REPORT} "\n" + ${DASHED_LINE_SYMBOL} "\n" + "Compilers & dependencies" "\n" +) -message(" ${CXX_COMPILER_REPORT}") +string(APPEND REPORT_TEXT + " - C compiler [${Magenta}CMAKE_C_COMPILER${ColorReset}]: v" ${CMAKE_C_COMPILER_VERSION} "\n" + " ${Dim}" ${CMAKE_C_COMPILER} "${ColorReset}\n" + " - C++ compiler [${Magenta}CMAKE_CXX_COMPILER${ColorReset}]: v" ${CMAKE_CXX_COMPILER_VERSION} "\n" + " ${Dim}" ${CMAKE_CXX_COMPILER} "${ColorReset}\n" +) -if(NOT "${CUDA_COMPILER_REPORT}" STREQUAL "") - message(" ${CUDA_COMPILER_REPORT}") +if(${Kokkos_DEVICES} MATCHES "CUDA") + if("${CMAKE_CUDA_COMPILER}" STREQUAL "") + execute_process(COMMAND which nvcc OUTPUT_VARIABLE CUDACOMP) + else() + set(CUDACOMP ${CMAKE_CUDA_COMPILER}) + endif() + string(STRIP ${CUDACOMP} CUDACOMP) + execute_process( + COMMAND + bash -c + "${CUDACOMP} --version | grep release | sed -e 's/.*release //' -e 's/,.*//'" + OUTPUT_VARIABLE CUDACOMP_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE) + string(APPEND REPORT_TEXT + " - CUDA compiler: v" ${CUDACOMP_VERSION} "\n" + " ${Dim}" ${CUDACOMP} "${ColorReset}\n" + ) +elseif(${Kokkos_DEVICES} MATCHES "HIP") + execute_process( + COMMAND bash -c "hipcc --version | grep HIP | cut -d ':' -f 2 | tr -d ' '" + OUTPUT_VARIABLE ROCM_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE) + string(APPEND REPORT_TEXT + " - ROCm: v" ${ROCM_VERSION} "\n" + ) endif() -message(" ${MPI_REPORT}") - -message(" ${DEBUG_REPORT}") - -message("${DASHED_LINE_SYMBOL}\nDependencies") - -if(NOT "${CUDACOMP_VERSION}" STREQUAL "") - message(" - CUDA:\tv${CUDACOMP_VERSION}") -elseif(NOT "${ROCM_VERSION}" STREQUAL "") - message(" - ROCm:\tv${ROCM_VERSION}") +string(APPEND REPORT_TEXT + " - Kokkos: v" ${Kokkos_VERSION} "\n" +) +if(${Kokkos_FOUND}) + string(APPEND REPORT_TEXT + " " ${Kokkos_DIR} "\n" + ) +else() + string(APPEND REPORT_TEXT + " " ${Kokkos_BUILD_DIR} "\n" + ) endif() -message(" - Kokkos:\tv${Kokkos_VERSION}") + if(${output}) - message(" - ADIOS2:\tv${adios2_VERSION}") -endif() -if(${HDF5_FOUND}) - message(" - HDF5:\tv${HDF5_VERSION}") + string(APPEND REPORT_TEXT + " - ADIOS2: v" ${adios2_VERSION} "\n" + ) + if(${adios2_FOUND}) + string(APPEND REPORT_TEXT + " " ${adios2_DIR} "\n" + ) + else() + string(APPEND REPORT_TEXT + " " ${adios2_BUILD_DIR} "\n" + ) + endif() endif() -message( - "${DASHED_LINE_SYMBOL} -Notes - ${Dim}: Set flags with `cmake ... -D ${Magenta}${ColorReset}${Dim}=`, the ${Underline}default${ColorReset}${Dim} value - : will be used unless the variable is explicitly set.${ColorReset} -") +string(APPEND REPORT_TEXT + ${DASHED_LINE_SYMBOL} "\n" + "Notes" "\n" + " ${Dim}: Set flags with `cmake ... -D ${Magenta}${ColorReset}${Dim}=`, the ${Underline}default${ColorReset}${Dim} value" "\n" + " : will be used unless the variable is explicitly set.${ColorReset}" "\n" +) + +message(${REPORT_TEXT}) diff --git a/cmake/styling.cmake b/cmake/styling.cmake index 70c448fff..0da4b3519 100644 --- a/cmake/styling.cmake +++ b/cmake/styling.cmake @@ -23,17 +23,131 @@ if(NOT WIN32) set(StrikeEnd "${Esc}[0m") endif() -# message("This is normal") message("${Red}This is Red${ColorReset}") -# message("${Green}This is Green${ColorReset}") message("${Yellow}This is -# Yellow${ColorReset}") message("${Blue}This is Blue${ColorReset}") -# message("${Magenta}This is Magenta${ColorReset}") message("${Cyan}This is -# Cyan${ColorReset}") message("${White}This is White${ColorReset}") -# message("${BoldRed}This is BoldRed${ColorReset}") message("${BoldGreen}This is -# BoldGreen${ColorReset}") message("${BoldYellow}This is -# BoldYellow${ColorReset}") message("${BoldBlue}This is BoldBlue${ColorReset}") -# message("${BoldMagenta}This is BoldMagenta${ColorReset}") -# message("${BoldCyan}This is BoldCyan${ColorReset}") message("${BoldWhite}This -# is BoldWhite\n\n${ColorReset}") - -# message() +set(DOTTED_LINE_SYMBOL + "${ColorReset}. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " +) +set(DASHED_LINE_SYMBOL + "${ColorReset}....................................................................... " +) + +set(ON_OFF_VALUES "ON" "OFF") + +function(PureLength Text Result) + set(rt ${Text}) + string(FIND ${rt} "${Magenta}" mg_fnd) + + if(mg_fnd GREATER -1) + string(REGEX REPLACE "${Esc}\\[35m" "" rt ${rt}) + endif() + + string(LENGTH "${rt}" TextLength) + set(${Result} + "${TextLength}" + PARENT_SCOPE) +endfunction() + +function(PadTo Text Padding Target Result) + purelength("${Text}" TextLength) + math(EXPR PaddingNeeded "${Target} - ${TextLength}") + set(rt ${Text}) + + if(PaddingNeeded GREATER 0) + foreach(i RANGE 0 ${PaddingNeeded}) + set(rt "${rt}${Padding}") + endforeach() + else() + set(rt "${rt}") + endif() + + set(${Result} + "${rt}" + PARENT_SCOPE) +endfunction() + +function( + PrintChoices + Label + Flag + Choices + Value + Default + Color + OutputString + Padding) + set(rstring "- ${Label}") + + if(NOT "${Flag}" STREQUAL "") + string(APPEND rstring " [${Magenta}${Flag}${ColorReset}]") + endif() + + string(APPEND rstring ":") + + if(${Padding} EQUAL 0) + list(LENGTH "${Choices}" nchoices) + math(EXPR lastchoice "${nchoices} - 1") + set(ncols 4) + math(EXPR lastcol "${ncols} - 1") + + set(longest 0) + foreach(ch IN LISTS Choices) + string(LENGTH ${ch} clen) + if(clen GREATER longest) + set(longest ${clen}) + endif() + endforeach() + + set(counter 0) + foreach(ch IN LISTS Choices) + if(${ch} STREQUAL ${Value}) + set(col ${Color}) + else() + set(col ${Dim}) + endif() + + if(${ch} STREQUAL ${Default}) + set(col ${Underline}${col}) + endif() + + string(LENGTH "${ch}" clen) + math(EXPR PaddingNeeded "${longest} - ${clen} + 4") + + if(counter EQUAL ${lastcol} AND NOT ${counter} EQUAL ${lastchoice}) + string(APPEND rstring "${col}~ ${ch}${ColorReset}") + else() + if(counter EQUAL 0) + string(APPEND rstring "\n ") + endif() + string(APPEND rstring "${col}~ ${ch}${ColorReset}") + foreach(i RANGE 0 ${PaddingNeeded}) + string(APPEND rstring " ") + endforeach() + endif() + + math(EXPR counter "(${counter} + 1) % ${ncols}") + endforeach() + else() + padto("${rstring}" " " ${Padding} rstring) + + set(new_choices ${Choices}) + foreach(ch IN LISTS new_choices) + string(REPLACE ${ch} "${Dim}${ch}${ColorReset}" new_choices "${new_choices}") + endforeach() + set(Choices ${new_choices}) + if(${Value} STREQUAL "ON") + set(col ${Green}) + elseif(${Value} STREQUAL "OFF") + set(col ${Red}) + else() + set(col ${Color}) + endif() + string(REPLACE ${Value} "${col}${Value}${ColorReset}" Choices "${Choices}") + string(REPLACE ${Default} "${Underline}${Default}${ColorReset}" Choices "${Choices}") + string(REPLACE ";" "/" Choices "${Choices}") + string(APPEND rstring "${Choices}") + endif() + + set(${OutputString} + "${rstring}" + PARENT_SCOPE) +endfunction() diff --git a/src/engines/CMakeLists.txt b/src/engines/CMakeLists.txt index 9e51330ec..d5b6b0664 100644 --- a/src/engines/CMakeLists.txt +++ b/src/engines/CMakeLists.txt @@ -38,7 +38,7 @@ add_library(ntt_engines ${SOURCES}) set(libs ntt_global ntt_framework ntt_metrics ntt_archetypes ntt_kernels ntt_pgen) if(${output}) - list(APPEND libs ntt_output hdf5::hdf5) + list(APPEND libs ntt_output) endif() add_dependencies(ntt_engines ${libs}) target_link_libraries(ntt_engines PUBLIC ${libs}) diff --git a/src/engines/engine_printer.cpp b/src/engines/engine_printer.cpp index f94715d09..eb8ff402d 100644 --- a/src/engines/engine_printer.cpp +++ b/src/engines/engine_printer.cpp @@ -20,7 +20,6 @@ #endif #if defined(OUTPUT_ENABLED) - #include #include #endif @@ -188,18 +187,11 @@ namespace ntt { KOKKOS_VERSION % 100); #if defined(OUTPUT_ENABLED) - unsigned h5_major, h5_minor, h5_release; - H5get_libversion(&h5_major, &h5_minor, &h5_release); - const std::string hdf5_version = fmt::format("%d.%d.%d", - h5_major, - h5_minor, - h5_release); const std::string adios2_version = fmt::format("%d.%d.%d", ADIOS2_VERSION / 10000, ADIOS2_VERSION / 100 % 100, ADIOS2_VERSION % 100); #else // not OUTPUT_ENABLED - const std::string hdf5_version = "OFF"; const std::string adios2_version = "OFF"; #endif @@ -217,7 +209,6 @@ namespace ntt { add_param(report, 4, "CXX", "%s [%s]", ccx.c_str(), cpp_standard.c_str()); add_param(report, 4, "CUDA", "%s", cuda_version.c_str()); add_param(report, 4, "MPI", "%s", mpi_version.c_str()); - add_param(report, 4, "HDF5", "%s", hdf5_version.c_str()); add_param(report, 4, "Kokkos", "%s", kokkos_version.c_str()); add_param(report, 4, "ADIOS2", "%s", adios2_version.c_str()); add_param(report, 4, "Precision", "%s", precision); From d1d999cc9209edf1c592988df8306796f807440e Mon Sep 17 00:00:00 2001 From: haykh Date: Thu, 3 Apr 2025 19:25:25 -0400 Subject: [PATCH 081/176] int type issues in mpi writer test --- src/output/tests/writer-mpi.cpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/output/tests/writer-mpi.cpp b/src/output/tests/writer-mpi.cpp index f6d3ee88a..cc9208889 100644 --- a/src/output/tests/writer-mpi.cpp +++ b/src/output/tests/writer-mpi.cpp @@ -61,8 +61,8 @@ auto main(int argc, char* argv[]) -> int { // write auto writer = out::Writer(); writer.init(&adios, "hdf5", "test", false); - writer.defineMeshLayout({ static_cast(mpi_size) * nx1 }, - { static_cast(mpi_rank) * nx1 }, + writer.defineMeshLayout({ static_cast(mpi_size) * nx1 }, + { static_cast(mpi_rank) * nx1 }, { nx1 }, { dwn1 }, false, @@ -89,9 +89,9 @@ auto main(int argc, char* argv[]) -> int { { // read adios2::IO io = adios.DeclareIO("read-test"); - io.SetEngine("hdf5"); - adios2::Engine reader = io.Open("test.h5", adios2::Mode::Read, MPI_COMM_SELF); - raise::ErrorIf(io.InquireAttribute("NGhosts").Data()[0] != 0, + io.SetEngine("HDF5"); + adios2::Engine reader = io.Open("test.h5", adios2::Mode::Read); + raise::ErrorIf(io.InquireAttribute("NGhosts").Data()[0] != 0, "NGhosts is not correct", HERE); raise::ErrorIf(io.InquireAttribute("Dimension").Data()[0] != 1, @@ -99,13 +99,13 @@ auto main(int argc, char* argv[]) -> int { HERE); for (std::size_t step = 0; reader.BeginStep() == adios2::StepStatus::OK; ++step) { - std::size_t step_read; - long double time_read; + timestep_t step_read; + simtime_t time_read; - reader.Get(io.InquireVariable("Step"), + reader.Get(io.InquireVariable("Step"), &step_read, adios2::Mode::Sync); - reader.Get(io.InquireVariable("Time"), + reader.Get(io.InquireVariable("Time"), &time_read, adios2::Mode::Sync); raise::ErrorIf(step_read != step, "Step is not correct", HERE); @@ -122,9 +122,9 @@ auto main(int argc, char* argv[]) -> int { const double l = l_offset; const double f = math::ceil(l / d) * d - l; - const auto first_cell = static_cast(f); - const auto l_size_dwn = static_cast(math::ceil((n - f) / d)); - const auto l_corner_dwn = static_cast(math::ceil(l / d)); + const auto first_cell = static_cast(f); + const auto l_size_dwn = static_cast(math::ceil((n - f) / d)); + const auto l_corner_dwn = static_cast(math::ceil(l / d)); array_t field_read {}; int cntr = 0; From 4708d81a3ce1ecf38e1584e80ad8ee1075cf21c0 Mon Sep 17 00:00:00 2001 From: haykh Date: Thu, 3 Apr 2025 20:33:03 -0400 Subject: [PATCH 082/176] cmake prints report with TESTS --- CMakeLists.txt | 3 +- cmake/report.cmake | 170 ++++++++++++++++++++++++++++---------------- cmake/styling.cmake | 31 +++++--- cmake/tests.cmake | 50 ++++++------- 4 files changed, 153 insertions(+), 101 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 22d1e64b5..4791ea55b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -125,5 +125,6 @@ else() # ------------------------------- Main source ------------------------------ # set_problem_generator(${pgen}) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src src) - include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/report.cmake) endif() + +include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/report.cmake) diff --git a/cmake/report.cmake b/cmake/report.cmake index 60d846144..21064cdf9 100644 --- a/cmake/report.cmake +++ b/cmake/report.cmake @@ -8,6 +8,24 @@ if(${PGEN_FOUND}) "${Blue}" PGEN_REPORT 0) +elseif(${TESTS}) + set(TEST_NAMES "") + foreach(test_dir IN LISTS TEST_DIRECTORIES) + get_property( + LOCAL_TEST_NAMES + DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${test_dir}/tests + PROPERTY TESTS) + list(APPEND TEST_NAMES ${LOCAL_TEST_NAMES}) + endforeach() + printchoices( + "Test cases" + "" + "${TEST_NAMES}" + "" + "${ColorReset}" + "" + TESTS_REPORT + 0) endif() printchoices( @@ -58,7 +76,7 @@ else() endif() set(REPORT_TEXT - "${Blue} __ __ + "${Blue} __ __ /\\ \\__ __/\\ \\__ __ ___\\ \\ _\\/\\_\\ \\ _\\ __ __ / __ \\ / __ \\ \\ \\/\\/\\ \\ \\ \\/ /\\ \\/\\ \\ @@ -66,46 +84,63 @@ set(REPORT_TEXT \\ \\____\\ \\_\\ \\_\\ \\__\\\\ \\_\\ \\__\\\\ \\____ \\/\\_\\ \\/____/\\/_/\\/_/\\/__/ \\/_/\\/__/ \\/___/ \\/_/ /\\___/ -Entity ${VERSION_SYMBOL}\t\t \\/__/" -) -string(APPEND REPORT_TEXT - ${ColorReset} "\n" -) +Entity ${VERSION_SYMBOL}\t\t \\/__/") +string(APPEND REPORT_TEXT ${ColorReset} "\n") -string(APPEND REPORT_TEXT - ${DASHED_LINE_SYMBOL} "\n" - "Configurations" "\n" -) +string(APPEND REPORT_TEXT ${DASHED_LINE_SYMBOL} "\n" "Configurations" "\n") if(${PGEN_FOUND}) - string(APPEND REPORT_TEXT - " " ${PGEN_REPORT} "\n" - ) + string(APPEND REPORT_TEXT " " ${PGEN_REPORT} "\n") +elseif(${TESTS}) + string(APPEND REPORT_TEXT " " ${TESTS_REPORT} "\n") endif() -string(APPEND REPORT_TEXT - " " ${PRECISION_REPORT} "\n" - " " ${OUTPUT_REPORT} "\n" -) +string( + APPEND + REPORT_TEXT + " " + ${PRECISION_REPORT} + "\n" + " " + ${OUTPUT_REPORT} + "\n") string(REPLACE ";" "+" Kokkos_ARCH "${Kokkos_ARCH}") string(REPLACE ";" "+" Kokkos_DEVICES "${Kokkos_DEVICES}") -string(APPEND REPORT_TEXT - " - ARCH [${Magenta}Kokkos_ARCH_***${ColorReset}]: ${Kokkos_ARCH}" "\n" - " - DEVICES [${Magenta}Kokkos_ENABLE_***${ColorReset}]: ${Kokkos_DEVICES}" "\n" - " " ${MPI_REPORT} "\n" - " " ${DEBUG_REPORT} "\n" - ${DASHED_LINE_SYMBOL} "\n" - "Compilers & dependencies" "\n" -) - -string(APPEND REPORT_TEXT - " - C compiler [${Magenta}CMAKE_C_COMPILER${ColorReset}]: v" ${CMAKE_C_COMPILER_VERSION} "\n" - " ${Dim}" ${CMAKE_C_COMPILER} "${ColorReset}\n" - " - C++ compiler [${Magenta}CMAKE_CXX_COMPILER${ColorReset}]: v" ${CMAKE_CXX_COMPILER_VERSION} "\n" - " ${Dim}" ${CMAKE_CXX_COMPILER} "${ColorReset}\n" -) +string( + APPEND + REPORT_TEXT + " - ARCH [${Magenta}Kokkos_ARCH_***${ColorReset}]: ${Kokkos_ARCH}" + "\n" + " - DEVICES [${Magenta}Kokkos_ENABLE_***${ColorReset}]: ${Kokkos_DEVICES}" + "\n" + " " + ${MPI_REPORT} + "\n" + " " + ${DEBUG_REPORT} + "\n" + ${DASHED_LINE_SYMBOL} + "\n" + "Compilers & dependencies" + "\n") + +string( + APPEND + REPORT_TEXT + " - C compiler [${Magenta}CMAKE_C_COMPILER${ColorReset}]: v" + ${CMAKE_C_COMPILER_VERSION} + "\n" + " ${Dim}" + ${CMAKE_C_COMPILER} + "${ColorReset}\n" + " - C++ compiler [${Magenta}CMAKE_CXX_COMPILER${ColorReset}]: v" + ${CMAKE_CXX_COMPILER_VERSION} + "\n" + " ${Dim}" + ${CMAKE_CXX_COMPILER} + "${ColorReset}\n") if(${Kokkos_DEVICES} MATCHES "CUDA") if("${CMAKE_CUDA_COMPILER}" STREQUAL "") @@ -120,53 +155,62 @@ if(${Kokkos_DEVICES} MATCHES "CUDA") "${CUDACOMP} --version | grep release | sed -e 's/.*release //' -e 's/,.*//'" OUTPUT_VARIABLE CUDACOMP_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) - string(APPEND REPORT_TEXT - " - CUDA compiler: v" ${CUDACOMP_VERSION} "\n" - " ${Dim}" ${CUDACOMP} "${ColorReset}\n" - ) + string( + APPEND + REPORT_TEXT + " - CUDA compiler: v" + ${CUDACOMP_VERSION} + "\n" + " ${Dim}" + ${CUDACOMP} + "${ColorReset}\n") elseif(${Kokkos_DEVICES} MATCHES "HIP") execute_process( COMMAND bash -c "hipcc --version | grep HIP | cut -d ':' -f 2 | tr -d ' '" OUTPUT_VARIABLE ROCM_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) - string(APPEND REPORT_TEXT - " - ROCm: v" ${ROCM_VERSION} "\n" - ) + string(APPEND REPORT_TEXT " - ROCm: v" ${ROCM_VERSION} "\n") endif() -string(APPEND REPORT_TEXT - " - Kokkos: v" ${Kokkos_VERSION} "\n" -) +string(APPEND REPORT_TEXT " - Kokkos: v" ${Kokkos_VERSION} "\n") if(${Kokkos_FOUND}) - string(APPEND REPORT_TEXT - " " ${Kokkos_DIR} "\n" - ) + string(APPEND REPORT_TEXT " " ${Dim}${Kokkos_DIR}${ColorReset} "\n") else() - string(APPEND REPORT_TEXT - " " ${Kokkos_BUILD_DIR} "\n" - ) + string(APPEND REPORT_TEXT " " ${Dim}${Kokkos_BUILD_DIR}${ColorReset} "\n") endif() if(${output}) - string(APPEND REPORT_TEXT - " - ADIOS2: v" ${adios2_VERSION} "\n" - ) + string(APPEND REPORT_TEXT " - ADIOS2: v" ${adios2_VERSION} "\n") if(${adios2_FOUND}) - string(APPEND REPORT_TEXT - " " ${adios2_DIR} "\n" - ) + string(APPEND REPORT_TEXT " " "${Dim}${adios2_DIR}${ColorReset}" "\n") else() - string(APPEND REPORT_TEXT - " " ${adios2_BUILD_DIR} "\n" - ) + string(APPEND REPORT_TEXT " " "${Dim}${adios2_BUILD_DIR}${ColorReset}" + "\n") endif() endif() -string(APPEND REPORT_TEXT - ${DASHED_LINE_SYMBOL} "\n" - "Notes" "\n" - " ${Dim}: Set flags with `cmake ... -D ${Magenta}${ColorReset}${Dim}=`, the ${Underline}default${ColorReset}${Dim} value" "\n" - " : will be used unless the variable is explicitly set.${ColorReset}" "\n" -) +string( + APPEND + REPORT_TEXT + ${DASHED_LINE_SYMBOL} + "\n" + "Notes" + "\n" + " ${Dim}: Set flags with `cmake ... -D ${Magenta}${ColorReset}${Dim}=`, the ${Underline}default${ColorReset}${Dim} value" + "\n" + " : will be used unless the variable is explicitly set.${ColorReset}") + +if(${TESTS}) + string( + APPEND + REPORT_TEXT + "\n" + " ${Dim}: Run the tests with the following command:" + "\n" + " : ctest --test-dir ${CMAKE_CURRENT_BINARY_DIR}${ColorReset}" + "\n") +endif() + +string(APPEND REPORT_TEXT "\n") message(${REPORT_TEXT}) diff --git a/cmake/styling.cmake b/cmake/styling.cmake index 0da4b3519..b9b72abcb 100644 --- a/cmake/styling.cmake +++ b/cmake/styling.cmake @@ -43,8 +43,8 @@ function(PureLength Text Result) string(LENGTH "${rt}" TextLength) set(${Result} - "${TextLength}" - PARENT_SCOPE) + "${TextLength}" + PARENT_SCOPE) endfunction() function(PadTo Text Padding Target Result) @@ -99,14 +99,20 @@ function( set(counter 0) foreach(ch IN LISTS Choices) - if(${ch} STREQUAL ${Value}) - set(col ${Color}) + if(NOT ${Value} STREQUAL "") + if(${ch} STREQUAL ${Value}) + set(col ${Color}) + else() + set(col ${Dim}) + endif() else() set(col ${Dim}) endif() - if(${ch} STREQUAL ${Default}) - set(col ${Underline}${col}) + if(NOT ${Default} STREQUAL "") + if(${ch} STREQUAL ${Default}) + set(col ${Underline}${col}) + endif() endif() string(LENGTH "${ch}" clen) @@ -131,7 +137,8 @@ function( set(new_choices ${Choices}) foreach(ch IN LISTS new_choices) - string(REPLACE ${ch} "${Dim}${ch}${ColorReset}" new_choices "${new_choices}") + string(REPLACE ${ch} "${Dim}${ch}${ColorReset}" new_choices + "${new_choices}") endforeach() set(Choices ${new_choices}) if(${Value} STREQUAL "ON") @@ -141,8 +148,14 @@ function( else() set(col ${Color}) endif() - string(REPLACE ${Value} "${col}${Value}${ColorReset}" Choices "${Choices}") - string(REPLACE ${Default} "${Underline}${Default}${ColorReset}" Choices "${Choices}") + if(NOT "${Value}" STREQUAL "") + string(REPLACE ${Value} "${col}${Value}${ColorReset}" Choices + "${Choices}") + endif() + if(NOT "${Default}" STREQUAL "") + string(REPLACE ${Default} "${Underline}${Default}${ColorReset}" Choices + "${Choices}") + endif() string(REPLACE ";" "/" Choices "${Choices}") string(APPEND rstring "${Choices}") endif() diff --git a/cmake/tests.cmake b/cmake/tests.cmake index ca8ee69c4..e2d92a343 100644 --- a/cmake/tests.cmake +++ b/cmake/tests.cmake @@ -13,32 +13,26 @@ if(${output}) add_subdirectory(${SRC_DIR}/checkpoint ${CMAKE_CURRENT_BINARY_DIR}/checkpoint) endif() -if(${mpi}) - # tests with mpi - if(${output}) - add_subdirectory(${SRC_DIR}/output/tests - ${CMAKE_CURRENT_BINARY_DIR}/output/tests) - add_subdirectory(${SRC_DIR}/checkpoint/tests - ${CMAKE_CURRENT_BINARY_DIR}/checkpoint/tests) - add_subdirectory(${SRC_DIR}/framework/tests - ${CMAKE_CURRENT_BINARY_DIR}/framework/tests) - endif() -else() - # tests without mpi - add_subdirectory(${SRC_DIR}/global/tests - ${CMAKE_CURRENT_BINARY_DIR}/global/tests) - add_subdirectory(${SRC_DIR}/metrics/tests - ${CMAKE_CURRENT_BINARY_DIR}/metrics/tests) - add_subdirectory(${SRC_DIR}/kernels/tests - ${CMAKE_CURRENT_BINARY_DIR}/kernels/tests) - add_subdirectory(${SRC_DIR}/archetypes/tests - ${CMAKE_CURRENT_BINARY_DIR}/archetypes/tests) - add_subdirectory(${SRC_DIR}/framework/tests - ${CMAKE_CURRENT_BINARY_DIR}/framework/tests) - if(${output}) - add_subdirectory(${SRC_DIR}/output/tests - ${CMAKE_CURRENT_BINARY_DIR}/output/tests) - add_subdirectory(${SRC_DIR}/checkpoint/tests - ${CMAKE_CURRENT_BINARY_DIR}/checkpoint/tests) - endif() +set(TEST_DIRECTORIES + "" + PARENT_SCOPE) + +if(NOT ${mpi}) + list(APPEND TEST_DIRECTORIES global) + list(APPEND TEST_DIRECTORIES metrics) + list(APPEND TEST_DIRECTORIES kernels) + list(APPEND TEST_DIRECTORIES archetypes) + list(APPEND TEST_DIRECTORIES framework) +elseif(${mpi} AND ${output}) + list(APPEND TEST_DIRECTORIES framework) endif() + +if(${output}) + list(APPEND TEST_DIRECTORIES output) + list(APPEND TEST_DIRECTORIES checkpoint) +endif() + +foreach(test_dir IN LISTS TEST_DIRECTORIES) + add_subdirectory(${SRC_DIR}/${test_dir}/tests + ${CMAKE_CURRENT_BINARY_DIR}/${test_dir}/tests) +endforeach() From 4d7528783d501b653bd227ad842fa012f2be0bde Mon Sep 17 00:00:00 2001 From: haykh Date: Thu, 3 Apr 2025 20:33:26 -0400 Subject: [PATCH 083/176] type+unused warnings fixed --- src/framework/containers/particles.cpp | 7 +++---- src/kernels/fields_bcs.hpp | 28 +++++++++++++++++++------- src/metrics/tests/ks-qks.cpp | 1 + src/output/tests/writer-nompi.cpp | 3 ++- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/framework/containers/particles.cpp b/src/framework/containers/particles.cpp index 9ec8810dc..2f59a004d 100644 --- a/src/framework/containers/particles.cpp +++ b/src/framework/containers/particles.cpp @@ -84,7 +84,7 @@ namespace ntt { rangeActiveParticles(), Lambda(index_t p) { auto npptag_acc = npptag_scat.access(); - if (this_tag(p) < 0 || this_tag(p) >= num_tags) { + if (this_tag(p) < 0 || this_tag(p) >= static_cast(num_tags)) { raise::KernelError(HERE, "Invalid tag value"); } npptag_acc(this_tag(p)) += 1; @@ -144,9 +144,8 @@ namespace ntt { template void Particles::RemoveDead() { - const auto n_part = npart(); - npart_t n_alive = 0, n_dead = 0; - auto& this_tag = tag; + npart_t n_alive = 0, n_dead = 0; + auto& this_tag = tag; Kokkos::parallel_reduce( "CountDeadAlive", diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index dbb47f42c..ecdba8eb1 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -562,6 +562,7 @@ namespace kernel::bc { const I fset; const M metric; const ncells_t i_edge; + const BCTags tags; EnforcedBoundaries_kernel(ndfield_t& Fld, const I& fset, @@ -571,7 +572,8 @@ namespace kernel::bc { : Fld { Fld } , fset { fset } , metric { metric } - , i_edge { i_edge + N_GHOSTS } {} + , i_edge { i_edge + N_GHOSTS } + , tags { tags } {} Inline void operator()(index_t i1) const { if constexpr (D == Dim::_1D) { @@ -580,8 +582,12 @@ namespace kernel::bc { coord_t x_Ph_H { ZERO }; metric.template convert({ i1_ }, x_Ph_0); metric.template convert({ i1_ + HALF }, x_Ph_H); - bool setEx1 = defines_ex1, setEx2 = defines_ex2, setEx3 = defines_ex3, - setBx1 = defines_bx1, setBx2 = defines_bx2, setBx3 = defines_bx3; + bool setEx1 = defines_ex1 and (tags & BC::E), + setEx2 = defines_ex2 and (tags & BC::E), + setEx3 = defines_ex3 and (tags & BC::E), + setBx1 = defines_bx1 and (tags & BC::B), + setBx2 = defines_bx2 and (tags & BC::B), + setBx3 = defines_bx3 and (tags & BC::B); if constexpr (O == in::x1) { // x1 -- normal // x2,x3 -- tangential @@ -657,8 +663,12 @@ namespace kernel::bc { metric.template convert({ i1_ + HALF, i2_ }, x_Ph_H0); metric.template convert({ i1_ + HALF, i2_ + HALF }, x_Ph_HH); - bool setEx1 = defines_ex1, setEx2 = defines_ex2, setEx3 = defines_ex3, - setBx1 = defines_bx1, setBx2 = defines_bx2, setBx3 = defines_bx3; + bool setEx1 = defines_ex1 and (tags & BC::E), + setEx2 = defines_ex2 and (tags & BC::E), + setEx3 = defines_ex3 and (tags & BC::E), + setBx1 = defines_bx1 and (tags & BC::B), + setBx2 = defines_bx2 and (tags & BC::B), + setBx3 = defines_bx3 and (tags & BC::B); if constexpr (O == in::x1) { // x1 -- normal // x2,x3 -- tangential @@ -757,8 +767,12 @@ namespace kernel::bc { x_Ph_H0H); metric.template convert({ i1_, i2_ + HALF, i3_ + HALF }, x_Ph_0HH); - bool setEx1 = defines_ex1, setEx2 = defines_ex2, setEx3 = defines_ex3, - setBx1 = defines_bx1, setBx2 = defines_bx2, setBx3 = defines_bx3; + bool setEx1 = defines_ex1 and (tags & BC::E), + setEx2 = defines_ex2 and (tags & BC::E), + setEx3 = defines_ex3 and (tags & BC::E), + setBx1 = defines_bx1 and (tags & BC::B), + setBx2 = defines_bx2 and (tags & BC::B), + setBx3 = defines_bx3 and (tags & BC::B); if constexpr (O == in::x1) { // x1 -- normal // x2,x3 -- tangential diff --git a/src/metrics/tests/ks-qks.cpp b/src/metrics/tests/ks-qks.cpp index bed051e16..167f564ee 100644 --- a/src/metrics/tests/ks-qks.cpp +++ b/src/metrics/tests/ks-qks.cpp @@ -27,6 +27,7 @@ Inline auto equal(const vec_t& a, const auto eps = epsilon * acc; for (unsigned short d = 0; d < D; ++d) { if (not cmp::AlmostEqual(a[d], b[d], eps)) { + printf("%s: %.12e : %.12e\n", msg, a[d], b[d]); return false; } } diff --git a/src/output/tests/writer-nompi.cpp b/src/output/tests/writer-nompi.cpp index b063539ca..593f37f92 100644 --- a/src/output/tests/writer-nompi.cpp +++ b/src/output/tests/writer-nompi.cpp @@ -23,7 +23,8 @@ void cleanup() { } #define CEILDIV(a, b) \ - (static_cast(math::ceil(static_cast(a) / static_cast(b)))) + (static_cast( \ + math::ceil(static_cast(a) / static_cast(b)))) auto main(int argc, char* argv[]) -> int { Kokkos::initialize(argc, argv); From 518071b1d3f0fcb30d5bcbef3cede5f6dc334a4b Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 4 Apr 2025 02:43:00 -0400 Subject: [PATCH 084/176] warnings in cmake + nullptr --- CMakeLists.txt | 13 ++++----- cmake/benchmark.cmake | 2 ++ cmake/config.cmake | 17 +++++++----- cmake/defaults.cmake | 2 ++ cmake/dependencies.cmake | 41 ++++++++++++++++++----------- cmake/kokkosConfig.cmake | 2 ++ cmake/report.cmake | 26 ++++++++++-------- cmake/styling.cmake | 12 ++++----- cmake/tests.cmake | 4 +-- src/CMakeLists.txt | 1 + src/archetypes/CMakeLists.txt | 2 +- src/archetypes/tests/CMakeLists.txt | 1 + src/checkpoint/CMakeLists.txt | 1 + src/checkpoint/tests/CMakeLists.txt | 1 + src/engines/CMakeLists.txt | 1 + src/framework/CMakeLists.txt | 1 + src/framework/parameters.cpp | 10 +++---- src/framework/tests/CMakeLists.txt | 1 + src/global/CMakeLists.txt | 1 + src/global/tests/CMakeLists.txt | 1 + src/kernels/CMakeLists.txt | 2 +- src/kernels/tests/CMakeLists.txt | 1 + src/metrics/CMakeLists.txt | 2 +- src/metrics/tests/CMakeLists.txt | 2 +- src/output/CMakeLists.txt | 1 + src/output/tests/CMakeLists.txt | 1 + 26 files changed, 92 insertions(+), 57 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4791ea55b..f83e6637c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,3 +1,5 @@ +# cmake-lint: disable=C0103,C0111,E1120,R0913,R0915 + cmake_minimum_required(VERSION 3.16) cmake_policy(SET CMP0110 NEW) @@ -8,10 +10,10 @@ project( VERSION 1.2.0 LANGUAGES CXX C) add_compile_options("-D ENTITY_VERSION=\"${PROJECT_VERSION}\"") +set(hash_cmd "git diff --quiet src/ && echo $(git rev-parse HEAD) ") +string(APPEND hash_cmd "|| echo $(git rev-parse HEAD)-mod") execute_process( - COMMAND - bash -c - "git diff --quiet src/ && echo $(git rev-parse HEAD) || echo $(git rev-parse HEAD)-mod" + COMMAND bash -c ${hash_cmd} WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" OUTPUT_VARIABLE GIT_HASH ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) @@ -55,9 +57,8 @@ if(${DEBUG} STREQUAL "OFF") set(CMAKE_BUILD_TYPE Release CACHE STRING "CMake build type") - set(CMAKE_CXX_FLAGS - "${CMAKE_CXX_FLAGS} -DNDEBUG -Wno-unused-local-typedefs -Wno-unknown-cuda-version" - ) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DNDEBUG " + "-Wno-unused-local-typedefs -Wno-unknown-cuda-version") else() set(CMAKE_BUILD_TYPE Debug diff --git a/cmake/benchmark.cmake b/cmake/benchmark.cmake index d2e8ca47c..39b075716 100644 --- a/cmake/benchmark.cmake +++ b/cmake/benchmark.cmake @@ -1,3 +1,5 @@ +# cmake-lint: disable=C0103 + set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src) add_subdirectory(${SRC_DIR}/global ${CMAKE_CURRENT_BINARY_DIR}/global) diff --git a/cmake/config.cmake b/cmake/config.cmake index ab54717f1..7214812cd 100644 --- a/cmake/config.cmake +++ b/cmake/config.cmake @@ -1,3 +1,5 @@ +# cmake-lint: disable=C0103 + # -------------------------------- Precision ------------------------------- # function(set_precision precision_name) list(FIND precisions ${precision_name} PRECISION_FOUND) @@ -28,16 +30,17 @@ function(set_problem_generator pgen_name) endforeach() list(FIND PGEN_NAMES ${pgen_name} PGEN_FOUND) if(NOT ${pgen_name} STREQUAL "." AND ${PGEN_FOUND} EQUAL -1) - message( - FATAL_ERROR - "Invalid problem generator: ${pgen_name}\nValid options are: ${PGEN_NAMES}" - ) + message(FATAL_ERROR "Invalid problem generator: " + "${pgen_name}\nValid options are: ${PGEN_NAMES}") endif() set(PGEN - ${pgen_name} PARENT_SCOPE) + ${pgen_name} + PARENT_SCOPE) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/setups/${pgen_name}) set(PGEN_FOUND - TRUE PARENT_SCOPE) + TRUE + PARENT_SCOPE) set(problem_generators - ${PGEN_NAMES} PARENT_SCOPE) + ${PGEN_NAMES} + PARENT_SCOPE) endfunction() diff --git a/cmake/defaults.cmake b/cmake/defaults.cmake index 2c72f5d7d..30e605a5c 100644 --- a/cmake/defaults.cmake +++ b/cmake/defaults.cmake @@ -1,3 +1,5 @@ +# cmake-lint: disable=C0103 + # ----------------------------- Defaults ---------------------------------- # if(DEFINED ENV{Entity_ENABLE_DEBUG}) set(default_debug diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index a31dfdea5..a21ea00e8 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -1,10 +1,12 @@ +# cmake-lint: disable=C0103,C0111,R0915,R0912 + set(Kokkos_REPOSITORY https://github.com/kokkos/kokkos.git CACHE STRING "Kokkos repository") set(plog_REPOSITORY https://github.com/SergiusTheBest/plog.git CACHE STRING "plog repository") -set (adios2_REPOSITORY +set(adios2_REPOSITORY https://github.com/ornladios/ADIOS2.git CACHE STRING "ADIOS2 repository") @@ -41,7 +43,7 @@ function(find_or_fetch_dependency package_name header_only mode) endif() if(NOT ${package_name}_FOUND) - if (${package_name} STREQUAL "Kokkos") + if(${package_name} STREQUAL "Kokkos") include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/kokkosConfig.cmake) elseif(${package_name} STREQUAL "adios2") include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/adios2Config.cmake) @@ -49,10 +51,8 @@ function(find_or_fetch_dependency package_name header_only mode) if(DEFINED ${package_name}_REPOSITORY AND NOT FETCHCONTENT_FULLY_DISCONNECTED) # fetching package - message( - STATUS - "${Blue}${package_name} not found. Fetching from ${${package_name}_REPOSITORY}${ColorReset}" - ) + message(STATUS "${Blue}${package_name} not found. " + "Fetching from ${${package_name}_REPOSITORY}${ColorReset}") include(FetchContent) if(${package_name} STREQUAL "Kokkos") FetchContent_Declare( @@ -100,7 +100,7 @@ function(find_or_fetch_dependency package_name header_only mode) endif() add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extern/${package_name} - extern/${package_name}) + extern/${package_name}) set(${package_name}_SRC ${CMAKE_CURRENT_SOURCE_DIR}/extern/${package_name} CACHE PATH "Path to ${package_name} src") @@ -136,24 +136,35 @@ function(find_or_fetch_dependency package_name header_only mode) ${Kokkos_VERSION} CACHE INTERNAL "Kokkos version") endif() - if(NOT DEFINED Kokkos_ARCH OR Kokkos_ARCH STREQUAL "" - OR NOT DEFINED Kokkos_DEVICES OR Kokkos_DEVICES STREQUAL "") + if(NOT DEFINED Kokkos_ARCH + OR Kokkos_ARCH STREQUAL "" + OR NOT DEFINED Kokkos_DEVICES + OR Kokkos_DEVICES STREQUAL "") if(${Kokkos_FOUND}) include(${Kokkos_DIR}/KokkosConfigCommon.cmake) elseif(NOT ${Kokkos_BUILD_DIR} STREQUAL "") include(${Kokkos_BUILD_DIR}/KokkosConfigCommon.cmake) else() - message(STATUS "${Red}Kokkos_DIR and Kokkos_BUILD_DIR not set.${ColorReset}") + message( + STATUS "${Red}Kokkos_DIR and Kokkos_BUILD_DIR not set.${ColorReset}") endif() endif() set(Kokkos_ARCH - ${Kokkos_ARCH} PARENT_SCOPE) + ${Kokkos_ARCH} + PARENT_SCOPE) set(Kokkos_DEVICES - ${Kokkos_DEVICES} PARENT_SCOPE) + ${Kokkos_DEVICES} + PARENT_SCOPE) endif() - set(${package_name}_FOUND ${${package_name}_FOUND} PARENT_SCOPE) - set(${package_name}_FETCHED ${${package_name}_FETCHED} PARENT_SCOPE) - set(${package_name}_BUILD_DIR ${${package_name}_BUILD_DIR} PARENT_SCOPE) + set(${package_name}_FOUND + ${${package_name}_FOUND} + PARENT_SCOPE) + set(${package_name}_FETCHED + ${${package_name}_FETCHED} + PARENT_SCOPE) + set(${package_name}_BUILD_DIR + ${${package_name}_BUILD_DIR} + PARENT_SCOPE) endfunction() check_internet_connection() diff --git a/cmake/kokkosConfig.cmake b/cmake/kokkosConfig.cmake index 3d038cc19..f1a1cf207 100644 --- a/cmake/kokkosConfig.cmake +++ b/cmake/kokkosConfig.cmake @@ -1,3 +1,5 @@ +# cmake-lint: disable=C0103 + # ----------------------------- Kokkos settings ---------------------------- # if(${DEBUG} STREQUAL "OFF") set(Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION diff --git a/cmake/report.cmake b/cmake/report.cmake index 21064cdf9..5a38b0dd5 100644 --- a/cmake/report.cmake +++ b/cmake/report.cmake @@ -66,13 +66,13 @@ printchoices( 36) if(NOT ${PROJECT_VERSION_TWEAK} EQUAL 0) - set(VERSION_SYMBOL - "v${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}-rc${PROJECT_VERSION_TWEAK}" - ) + set(VERSION_SYMBOL "v${PROJECT_VERSION_MAJOR}." "${PROJECT_VERSION_MINOR}.") + string(APPEND VERSION_SYMBOL + "${PROJECT_VERSION_PATCH}-rc${PROJECT_VERSION_TWEAK}") else() - set(VERSION_SYMBOL - "v${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH} " - ) + set(VERSION_SYMBOL "v${PROJECT_VERSION_MAJOR}.") + string(APPEND VERSION_SYMBOL + "${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH} ") endif() set(REPORT_TEXT @@ -149,12 +149,13 @@ if(${Kokkos_DEVICES} MATCHES "CUDA") set(CUDACOMP ${CMAKE_CUDA_COMPILER}) endif() string(STRIP ${CUDACOMP} CUDACOMP) + set(cmd "${CUDACOMP} --version |") + string(APPEND cmd " grep release | sed -e 's/.*release //' -e 's/,.*//'") execute_process( - COMMAND - bash -c - "${CUDACOMP} --version | grep release | sed -e 's/.*release //' -e 's/,.*//'" + COMMAND bash -c ${cmd} OUTPUT_VARIABLE CUDACOMP_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) + message(STATUS "CUDACOMP: ${CUDACOMP_VERSION}") string( APPEND REPORT_TEXT @@ -165,8 +166,9 @@ if(${Kokkos_DEVICES} MATCHES "CUDA") ${CUDACOMP} "${ColorReset}\n") elseif(${Kokkos_DEVICES} MATCHES "HIP") + set(cmd "hipcc --version | grep HIP | cut -d ':' -f 2 | tr -d ' '") execute_process( - COMMAND bash -c "hipcc --version | grep HIP | cut -d ':' -f 2 | tr -d ' '" + COMMAND bash -c ${cmd} OUTPUT_VARIABLE ROCM_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) string(APPEND REPORT_TEXT " - ROCm: v" ${ROCM_VERSION} "\n") @@ -196,7 +198,9 @@ string( "\n" "Notes" "\n" - " ${Dim}: Set flags with `cmake ... -D ${Magenta}${ColorReset}${Dim}=`, the ${Underline}default${ColorReset}${Dim} value" + " ${Dim}: Set flags with `cmake ... -D " + "${Magenta}${ColorReset}${Dim}=`, " + "the ${Underline}default${ColorReset}${Dim} value" "\n" " : will be used unless the variable is explicitly set.${ColorReset}") diff --git a/cmake/styling.cmake b/cmake/styling.cmake index b9b72abcb..878cb44a4 100644 --- a/cmake/styling.cmake +++ b/cmake/styling.cmake @@ -1,3 +1,5 @@ +# cmake-lint: disable=C0103,C0301,C0111,E1120,R0913,R0915 + if(NOT WIN32) string(ASCII 27 Esc) set(ColorReset "${Esc}[m") @@ -23,13 +25,11 @@ if(NOT WIN32) set(StrikeEnd "${Esc}[0m") endif() -set(DOTTED_LINE_SYMBOL - "${ColorReset}. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " -) +set(DOTTED_LINE_SYMBOL "${ColorReset}. . . . . . . . . . . . . . . .") +string(APPEND DOTTED_LINE_SYMBOL " . . . . . . . . . . . . . . . . . . . . ") -set(DASHED_LINE_SYMBOL - "${ColorReset}....................................................................... " -) +set(DASHED_LINE_SYMBOL "${ColorReset}.................................") +string(APPEND DASHED_LINE_SYMBOL "...................................... ") set(ON_OFF_VALUES "ON" "OFF") diff --git a/cmake/tests.cmake b/cmake/tests.cmake index e2d92a343..0e108d365 100644 --- a/cmake/tests.cmake +++ b/cmake/tests.cmake @@ -13,9 +13,7 @@ if(${output}) add_subdirectory(${SRC_DIR}/checkpoint ${CMAKE_CURRENT_BINARY_DIR}/checkpoint) endif() -set(TEST_DIRECTORIES - "" - PARENT_SCOPE) +set(TEST_DIRECTORIES "") if(NOT ${mpi}) list(APPEND TEST_DIRECTORIES global) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index db2ab4c92..f9d921df0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103 # ------------------------------ # @defines: entity [STATIC/SHARED] # diff --git a/src/archetypes/CMakeLists.txt b/src/archetypes/CMakeLists.txt index 8e2f325af..93f8baaaa 100644 --- a/src/archetypes/CMakeLists.txt +++ b/src/archetypes/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103 # ------------------------------ # @defines: ntt_archetypes [INTERFACE] # @@ -24,4 +25,3 @@ target_link_libraries(ntt_archetypes INTERFACE ${libs}) target_include_directories(ntt_archetypes INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/../) - diff --git a/src/archetypes/tests/CMakeLists.txt b/src/archetypes/tests/CMakeLists.txt index 694a6b4f9..9419847c5 100644 --- a/src/archetypes/tests/CMakeLists.txt +++ b/src/archetypes/tests/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103,C0111 # ------------------------------ # @brief: Generates tests for the `ntt_archetypes` module # diff --git a/src/checkpoint/CMakeLists.txt b/src/checkpoint/CMakeLists.txt index fa641bfb5..096aad690 100644 --- a/src/checkpoint/CMakeLists.txt +++ b/src/checkpoint/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103 # ------------------------------ # @defines: ntt_checkpoint [STATIC/SHARED] # diff --git a/src/checkpoint/tests/CMakeLists.txt b/src/checkpoint/tests/CMakeLists.txt index 54400652e..cbfd63aa9 100644 --- a/src/checkpoint/tests/CMakeLists.txt +++ b/src/checkpoint/tests/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103,C0111 # ------------------------------ # @brief: Generates tests for the `ntt_checkpoint` module # diff --git a/src/engines/CMakeLists.txt b/src/engines/CMakeLists.txt index d5b6b0664..4cef18630 100644 --- a/src/engines/CMakeLists.txt +++ b/src/engines/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103 # ------------------------------ # @defines: ntt_engines [STATIC/SHARED] # diff --git a/src/framework/CMakeLists.txt b/src/framework/CMakeLists.txt index d3b68084c..4c407fb0c 100644 --- a/src/framework/CMakeLists.txt +++ b/src/framework/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103 # ------------------------------ # @defines: ntt_framework [STATIC/SHARED] # diff --git a/src/framework/parameters.cpp b/src/framework/parameters.cpp index e3bddfd70..5848b1c01 100644 --- a/src/framework/parameters.cpp +++ b/src/framework/parameters.cpp @@ -31,10 +31,10 @@ namespace ntt { template - auto get_dx0_V0( - const std::vector& resolution, - const boundaries_t& extent, - const std::map& params) -> std::pair { + auto get_dx0_V0(const std::vector& resolution, + const boundaries_t& extent, + const std::map& params) + -> std::pair { const auto metric = M(resolution, extent, params); const auto dx0 = metric.dxMin(); coord_t x_corner { ZERO }; @@ -839,7 +839,7 @@ namespace ntt { void SimulationParams::setSetupParams(const toml::value& toml_data) { /* [setup] -------------------------------------------------------------- */ - const auto& setup = toml::find_or(toml_data, "setup", toml::table {}); + const auto setup = toml::find_or(toml_data, "setup", toml::table {}); for (const auto& [key, val] : setup) { if (val.is_boolean()) { set("setup." + key, (bool)(val.as_boolean())); diff --git a/src/framework/tests/CMakeLists.txt b/src/framework/tests/CMakeLists.txt index 27c456610..92e327d80 100644 --- a/src/framework/tests/CMakeLists.txt +++ b/src/framework/tests/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103,C0111 # ------------------------------ # @brief: Generates tests for the `ntt_framework` module # diff --git a/src/global/CMakeLists.txt b/src/global/CMakeLists.txt index 97946f059..7546b6a98 100644 --- a/src/global/CMakeLists.txt +++ b/src/global/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103 # ------------------------------ # @defines: ntt_global [STATIC/SHARED] # diff --git a/src/global/tests/CMakeLists.txt b/src/global/tests/CMakeLists.txt index e30da20a0..c206f85b0 100644 --- a/src/global/tests/CMakeLists.txt +++ b/src/global/tests/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103,C0111 # ------------------------------ # @brief: Generates tests for the `ntt_global` module # diff --git a/src/kernels/CMakeLists.txt b/src/kernels/CMakeLists.txt index c8a1f409f..60eda24cb 100644 --- a/src/kernels/CMakeLists.txt +++ b/src/kernels/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103 # ------------------------------ # @defines: ntt_kernels [INTERFACE] # @@ -24,4 +25,3 @@ target_link_libraries(ntt_kernels INTERFACE ${libs}) target_include_directories(ntt_kernels INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/../) - diff --git a/src/kernels/tests/CMakeLists.txt b/src/kernels/tests/CMakeLists.txt index a41ea43ef..7579eb6d3 100644 --- a/src/kernels/tests/CMakeLists.txt +++ b/src/kernels/tests/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103,C0111 # ------------------------------ # @brief: Generates tests for the `ntt_kernels` module # diff --git a/src/metrics/CMakeLists.txt b/src/metrics/CMakeLists.txt index e053bb61c..0bb5b977c 100644 --- a/src/metrics/CMakeLists.txt +++ b/src/metrics/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103 # ------------------------------ # @defines: ntt_metrics [INTERFACE] # @@ -22,4 +23,3 @@ target_link_libraries(ntt_metrics INTERFACE ${libs}) target_include_directories(ntt_metrics INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/../) - diff --git a/src/metrics/tests/CMakeLists.txt b/src/metrics/tests/CMakeLists.txt index c997ab079..0d661c318 100644 --- a/src/metrics/tests/CMakeLists.txt +++ b/src/metrics/tests/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103,C0111 # ------------------------------ # @brief: Generates tests for the `ntt_metrics` module # @@ -28,4 +29,3 @@ gen_test(coord_trans) gen_test(sph-qsph) gen_test(ks-qks) gen_test(sr-cart-sph) - diff --git a/src/output/CMakeLists.txt b/src/output/CMakeLists.txt index 81333e9ff..8a2ea0f16 100644 --- a/src/output/CMakeLists.txt +++ b/src/output/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103 # ------------------------------ # @defines: ntt_output [STATIC/SHARED] # diff --git a/src/output/tests/CMakeLists.txt b/src/output/tests/CMakeLists.txt index afc7950c4..835bb532f 100644 --- a/src/output/tests/CMakeLists.txt +++ b/src/output/tests/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103,C0111 # ------------------------------ # @brief: Generates tests for the `ntt_output` module # From 195de49511ae1b7e7233338903095ab5af57a50b Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 4 Apr 2025 03:12:26 -0400 Subject: [PATCH 085/176] log level controlled from input --- input.example.toml | 6 ++++++ src/framework/parameters.cpp | 2 ++ src/framework/simulation.cpp | 7 ++++++- src/global/defaults.h | 3 ++- src/global/utils/formatting.h | 2 +- src/global/utils/plog.h | 14 +++++++++++--- 6 files changed, 28 insertions(+), 6 deletions(-) diff --git a/input.example.toml b/input.example.toml index a49067811..982e4d1f2 100644 --- a/input.example.toml +++ b/input.example.toml @@ -478,3 +478,9 @@ # @type: bool # @default: true colored_stdout = "" + # Specify the log level: + # @type: string + # @valid: "VERBOSE", "WARNING", "ERROR" + # @default: "VERBOSE" + # @note: VERBOSE prints all messages, WARNING prints only warnings and errors, ERROR prints only errors + log_level = "" diff --git a/src/framework/parameters.cpp b/src/framework/parameters.cpp index 5848b1c01..556d9f547 100644 --- a/src/framework/parameters.cpp +++ b/src/framework/parameters.cpp @@ -601,6 +601,8 @@ namespace ntt { toml::find_or(toml_data, "diagnostics", "blocking_timers", false)); set("diagnostics.colored_stdout", toml::find_or(toml_data, "diagnostics", "colored_stdout", false)); + set("diagnostics.log_level", + toml::find_or(toml_data, "diagnostics", "log_level", defaults::diag::log_level)); /* inferred variables --------------------------------------------------- */ // fields/particle boundaries diff --git a/src/framework/simulation.cpp b/src/framework/simulation.cpp index bea50ff09..a36d5cf52 100644 --- a/src/framework/simulation.cpp +++ b/src/framework/simulation.cpp @@ -32,7 +32,12 @@ namespace ntt { const auto raw_params = toml::parse(inputfname); const auto sim_name = toml::find(raw_params, "simulation", "name"); - logger::initPlog(sim_name); + const auto log_level = toml::find_or(raw_params, + "diagnostics", + "log_level", + defaults::diag::log_level); + logger::initPlog(sim_name, + log_level); m_requested_engine = SimEngine::pick( fmt::toLower(toml::find(raw_params, "simulation", "engine")).c_str()); diff --git a/src/global/defaults.h b/src/global/defaults.h index b81369022..d17647ccd 100644 --- a/src/global/defaults.h +++ b/src/global/defaults.h @@ -68,7 +68,8 @@ namespace ntt::defaults { } // namespace checkpoint namespace diag { - const timestep_t interval = 1; + const timestep_t interval = 1; + const std::string log_level = "VERBOSE"; } // namespace diag namespace gca { diff --git a/src/global/utils/formatting.h b/src/global/utils/formatting.h index 8dc7b6ba8..c8c236cab 100644 --- a/src/global/utils/formatting.h +++ b/src/global/utils/formatting.h @@ -171,7 +171,7 @@ namespace fmt { std::size_t cntr { 0 }; for (auto i { 0u }; i < columns.size(); ++i) { const auto anch { static_cast(anchors[i] < 0 ? -anchors[i] - : anchors[i]) }; + : anchors[i]) }; const auto leftalign { anchors[i] <= 0 }; const auto cmn { columns[i] }; const auto cmn_len { strlenUTF8(cmn) }; diff --git a/src/global/utils/plog.h b/src/global/utils/plog.h index 03dc19319..2422c7cf9 100644 --- a/src/global/utils/plog.h +++ b/src/global/utils/plog.h @@ -13,6 +13,8 @@ #ifndef GLOBAL_UTILS_PLOG_H #define GLOBAL_UTILS_PLOG_H +#include "utils/formatting.h" + #include #include #include @@ -57,7 +59,7 @@ namespace plog { namespace logger { template - inline void initPlog(const std::string& fname) { + inline void initPlog(const std::string& fname, const std::string& log_level) { // setup logging const auto logfile_name = fname + ".log"; const auto infofile_name = fname + ".info"; @@ -77,7 +79,13 @@ namespace logger { infofile_name.c_str()); static plog::RollingFileAppender errfileAppender( errfile_name.c_str()); - plog::init(plog::verbose, &logfileAppender); + auto log_severity = plog::verbose; + if (fmt::toLower(log_level) == "WARNING") { + log_severity = plog::warning; + } else if (fmt::toLower(log_level) == "ERROR") { + log_severity = plog::error; + } + plog::init(log_severity, &logfileAppender); plog::init(plog::verbose, &infofileAppender); plog::init(plog::verbose, &errfileAppender); @@ -93,4 +101,4 @@ namespace logger { } // namespace logger -#endif // GLOBAL_UTILS_PLOG_H \ No newline at end of file +#endif // GLOBAL_UTILS_PLOG_H From e96a2d38d8bace74a06d170bb3254fadc23b3c3a Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 4 Apr 2025 15:40:17 -0400 Subject: [PATCH 086/176] conductor in all directions --- src/engines/srpic.hpp | 58 ++++++-- src/kernels/fields_bcs.hpp | 288 ++++++++++++++++++++++++++++++------- 2 files changed, 290 insertions(+), 56 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 96747e429..a1228acda 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -854,9 +854,6 @@ namespace ntt { } else { const auto sign = direction.get_sign(); const auto dim = direction.get_dim(); - raise::ErrorIf(dim != in::x1, - "Perfect conductor BCs only implemented for x1", - HERE); std::vector xi_min, xi_max; @@ -866,7 +863,7 @@ namespace ntt { const auto dd = all_dirs[d]; if (dim == dd) { xi_min.push_back(0); - xi_max.push_back(N_GHOSTS + 1); + xi_max.push_back((sign < 0) ? (N_GHOSTS + 1) : N_GHOSTS); } else { xi_min.push_back(0); xi_max.push_back(domain.mesh.n_all(dd)); @@ -890,10 +887,55 @@ namespace ntt { raise::Error("Invalid dimension", HERE); } - Kokkos::parallel_for( - "ConductorFields", - range, - kernel::bc::ConductorBoundaries_kernel(domain.fields.em, tags)); + if (dim == in::x1) { + if (sign > 0) { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } else { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } + } else if (dim == in::x2) { + if (sign > 0) { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } else { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } + } else { + if (sign > 0) { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } else { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } + } } } diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index b1ee999d3..a69193d42 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -486,38 +486,52 @@ namespace kernel::bc { } }; - template + template struct ConductorBoundaries_kernel { static_assert(static_cast(o) < static_cast(D), "Invalid component index"); - ndfield_t Fld; - const BCTags tags; + ndfield_t Fld; + const BCTags tags; + const std::size_t i_edge; - ConductorBoundaries_kernel(ndfield_t Fld, BCTags tags) + ConductorBoundaries_kernel(ndfield_t Fld, std::size_t i_edge, BCTags tags) : Fld { Fld } + , i_edge { i_edge } , tags { tags } {} Inline void operator()(index_t i1) const { if constexpr (D == Dim::_1D) { if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, em::ex2) = ZERO; - Fld(N_GHOSTS, em::ex3) = ZERO; + Fld(i_edge, em::ex2) = ZERO; + Fld(i_edge, em::ex3) = ZERO; } else { - Fld(N_GHOSTS - i1, em::ex1) = Fld(N_GHOSTS + i1 - 1, em::ex1); - Fld(N_GHOSTS - i1, em::ex2) = -Fld(N_GHOSTS + i1, em::ex2); - Fld(N_GHOSTS - i1, em::ex3) = -Fld(N_GHOSTS + i1, em::ex3); + if constexpr (not P) { + Fld(i_edge - i1, em::ex1) = Fld(i_edge + i1 - 1, em::ex1); + Fld(i_edge - i1, em::ex2) = -Fld(i_edge + i1, em::ex2); + Fld(i_edge - i1, em::ex3) = -Fld(i_edge + i1, em::ex3); + } else { + Fld(i_edge + i1 - 1, em::ex1) = Fld(i_edge - i1, em::ex1); + Fld(i_edge + i1, em::ex2) = -Fld(i_edge - i1, em::ex2); + Fld(i_edge + i1, em::ex3) = -Fld(i_edge - i1, em::ex3); + } } } if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, em::bx1) = ZERO; + Fld(i_edge, em::bx1) = ZERO; } else { - Fld(N_GHOSTS - i1, em::bx1) = -Fld(N_GHOSTS + i1, em::bx1); - Fld(N_GHOSTS - i1, em::bx2) = Fld(N_GHOSTS + i1 - 1, em::bx2); - Fld(N_GHOSTS - i1, em::bx3) = Fld(N_GHOSTS + i1 - 1, em::bx3); + if constexpr (not P) { + Fld(i_edge - i1, em::bx1) = -Fld(i_edge + i1, em::bx1); + Fld(i_edge - i1, em::bx2) = Fld(i_edge + i1 - 1, em::bx2); + Fld(i_edge - i1, em::bx3) = Fld(i_edge + i1 - 1, em::bx3); + } else { + Fld(i_edge + i1, em::bx1) = -Fld(i_edge - i1, em::bx1); + Fld(i_edge + i1 - 1, em::bx2) = Fld(i_edge - i1, em::bx2); + Fld(i_edge + i1 - 1, em::bx3) = Fld(i_edge - i1, em::bx3); + } } } } else { @@ -529,24 +543,71 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2) const { if constexpr (D == Dim::_2D) { - if (tags & BC::E) { - if (i1 == 0) { - Fld(N_GHOSTS, i2, em::ex2) = ZERO; - Fld(N_GHOSTS, i2, em::ex3) = ZERO; - } else { - Fld(N_GHOSTS - i1, i2, em::ex1) = Fld(N_GHOSTS + i1 - 1, i2, em::ex1); - Fld(N_GHOSTS - i1, i2, em::ex2) = -Fld(N_GHOSTS + i1, i2, em::ex2); - Fld(N_GHOSTS - i1, i2, em::ex3) = -Fld(N_GHOSTS + i1, i2, em::ex3); + if constexpr (o == in::x1) { + if (tags & BC::E) { + if (i1 == 0) { + Fld(i_edge, i2, em::ex2) = ZERO; + Fld(i_edge, i2, em::ex3) = ZERO; + } else { + if constexpr (not P) { + Fld(i_edge - i1, i2, em::ex1) = Fld(i_edge + i1 - 1, i2, em::ex1); + Fld(i_edge - i1, i2, em::ex2) = -Fld(i_edge + i1, i2, em::ex2); + Fld(i_edge - i1, i2, em::ex3) = -Fld(i_edge + i1, i2, em::ex3); + } else { + Fld(i_edge + i1 - 1, i2, em::ex1) = Fld(i_edge - i1, i2, em::ex1); + Fld(i_edge + i1, i2, em::ex2) = -Fld(i_edge - i1, i2, em::ex2); + Fld(i_edge + i1, i2, em::ex3) = -Fld(i_edge - i1, i2, em::ex3); + } + } } - } - if (tags & BC::B) { - if (i1 == 0) { - Fld(N_GHOSTS, i2, em::bx1) = ZERO; - } else { - Fld(N_GHOSTS - i1, i2, em::bx1) = -Fld(N_GHOSTS + i1, i2, em::bx1); - Fld(N_GHOSTS - i1, i2, em::bx2) = Fld(N_GHOSTS + i1 - 1, i2, em::bx2); - Fld(N_GHOSTS - i1, i2, em::bx3) = Fld(N_GHOSTS + i1 - 1, i2, em::bx3); + if (tags & BC::B) { + if (i1 == 0) { + Fld(i_edge, i2, em::bx1) = ZERO; + } else { + if constexpr (not P) { + Fld(i_edge - i1, i2, em::bx1) = -Fld(i_edge + i1, i2, em::bx1); + Fld(i_edge - i1, i2, em::bx2) = Fld(i_edge + i1 - 1, i2, em::bx2); + Fld(i_edge - i1, i2, em::bx3) = Fld(i_edge + i1 - 1, i2, em::bx3); + } else { + Fld(i_edge + i1, i2, em::bx1) = -Fld(i_edge - i1, i2, em::bx1); + Fld(i_edge + i1 - 1, i2, em::bx2) = Fld(i_edge - i1, i2, em::bx2); + Fld(i_edge + i1 - 1, i2, em::bx3) = Fld(i_edge - i1, i2, em::bx3); + } + } + } + } else { + if (tags & BC::E) { + if (i2 == 0) { + Fld(i1, i_edge, em::ex1) = ZERO; + Fld(i1, i_edge, em::ex3) = ZERO; + } else { + if constexpr (not P) { + Fld(i1, i_edge - i2, em::ex1) = -Fld(i1, i_edge + i2, em::ex1); + Fld(i1, i_edge - i2, em::ex2) = Fld(i1, i_edge + i2 - 1, em::ex2); + Fld(i1, i_edge - i2, em::ex3) = -Fld(i1, i_edge + i2, em::ex3); + } else { + Fld(i1, i_edge + i2, em::ex1) = -Fld(i1, i_edge - i2, em::ex1); + Fld(i1, i_edge + i2 - 1, em::ex2) = Fld(i1, i_edge - i2, em::ex2); + Fld(i1, i_edge + i2, em::ex3) = -Fld(i1, i_edge - i2, em::ex3); + } + } + } + + if (tags & BC::B) { + if (i2 == 0) { + Fld(i1, i_edge, em::bx2) = ZERO; + } else { + if constexpr (not P) { + Fld(i1, i_edge - i2, em::bx1) = Fld(i1, i_edge + i2 - 1, em::bx1); + Fld(i1, i_edge - i2, em::bx2) = -Fld(i1, i_edge + i2, em::bx2); + Fld(i1, i_edge - i2, em::bx3) = Fld(i1, i_edge + i2 - 1, em::bx3); + } else { + Fld(i1, i_edge + i2 - 1, em::bx1) = Fld(i1, i_edge - i2, em::bx1); + Fld(i1, i_edge + i2, em::bx2) = -Fld(i1, i_edge - i2, em::bx2); + Fld(i1, i_edge + i2 - 1, em::bx3) = Fld(i1, i_edge - i2, em::bx3); + } + } } } } else { @@ -558,27 +619,158 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2, index_t i3) const { if constexpr (D == Dim::_3D) { - if (tags & BC::E) { - if (i1 == 0) { - Fld(N_GHOSTS, i2, i3, em::ex2) = ZERO; - Fld(N_GHOSTS, i2, i3, em::ex3) = ZERO; - } else { - Fld(N_GHOSTS - i1, i2, i3, em::ex1) = Fld(N_GHOSTS + i1 - 1, - i2, i3, em::ex1); - Fld(N_GHOSTS - i1, i2, i3, em::ex2) = -Fld(N_GHOSTS + i1, i2, i3, em::ex2); - Fld(N_GHOSTS - i1, i2, i3, em::ex3) = -Fld(N_GHOSTS + i1, i2, i3, em::ex3); + if constexpr (o == in::x1) { + if (tags & BC::E) { + if (i1 == 0) { + Fld(i_edge, i2, i3, em::ex2) = ZERO; + Fld(i_edge, i2, i3, em::ex3) = ZERO; + } else { + if constexpr (not P) { + Fld(i_edge - i1, i2, i3, em::ex1) = Fld(i_edge + i1 - 1, + i2, + i3, + em::ex1); + Fld(i_edge - i1, i2, i3, em::ex2) = -Fld(i_edge + i1, i2, i3, em::ex2); + Fld(i_edge - i1, i2, i3, em::ex3) = -Fld(i_edge + i1, i2, i3, em::ex3); + } else { + Fld(i_edge + i1 - 1, i2, i3, em::ex1) = Fld(i_edge - i1, + i2, + i3, + em::ex1); + Fld(i_edge + i1, i2, i3, em::ex2) = -Fld(i_edge - i1, i2, i3, em::ex2); + Fld(i_edge + i1, i2, i3, em::ex3) = -Fld(i_edge - i1, i2, i3, em::ex3); + } + } } - } - if (tags & BC::B) { - if (i1 == 0) { - Fld(N_GHOSTS, i2, i3, em::bx1) = ZERO; - } else { - Fld(N_GHOSTS - i1, i2, i3, em::bx1) = -Fld(N_GHOSTS + i1, i2, i3, em::bx1); - Fld(N_GHOSTS - i1, i2, i3, em::bx2) = Fld(N_GHOSTS + i1 - 1, - i2, i3, em::bx2); - Fld(N_GHOSTS - i1, i2, i3, em::bx3) = Fld(N_GHOSTS + i1 - 1, - i2, i3, em::bx3); + if (tags & BC::B) { + if (i1 == 0) { + Fld(i_edge, i2, i3, em::bx1) = ZERO; + } else { + if constexpr (not P) { + Fld(i_edge - i1, i2, i3, em::bx1) = -Fld(i_edge + i1, i2, i3, em::bx1); + Fld(i_edge - i1, i2, i3, em::bx2) = Fld(i_edge + i1 - 1, + i2, + i3, + em::bx2); + Fld(i_edge - i1, i2, i3, em::bx3) = Fld(i_edge + i1 - 1, + i2, + i3, + em::bx3); + } else { + Fld(i_edge + i1, i2, i3, em::bx1) = -Fld(i_edge - i1, i2, i3, em::bx1); + Fld(i_edge + i1 - 1, i2, i3, em::bx2) = Fld(i_edge - i1, + i2, + i3, + em::bx2); + Fld(i_edge + i1 - 1, i2, i3, em::bx3) = Fld(i_edge - i1, + i2, + i3, + em::bx3); + } + } + } + } else if (o == in::x2) { + if (tags & BC::E) { + if (i2 == 0) { + Fld(i1, i_edge, i3, em::ex1) = ZERO; + Fld(i1, i_edge, i3, em::ex3) = ZERO; + } else { + if constexpr (not P) { + Fld(i1, i_edge - i2, i3, em::ex1) = -Fld(i1, i_edge + i2, i3, em::ex1); + Fld(i1, i_edge - i2, i3, em::ex2) = Fld(i1, + i_edge + i2 - 1, + i3, + em::ex2); + Fld(i1, i_edge - i2, i3, em::ex3) = -Fld(i1, i_edge + i2, i3, em::ex3); + } else { + Fld(i1, i_edge + i2, i3, em::ex1) = -Fld(i1, i_edge - i2, i3, em::ex1); + Fld(i1, i_edge + i2 - 1, i3, em::ex2) = Fld(i1, + i_edge - i2, + i3, + em::ex2); + Fld(i1, i_edge + i2, i3, em::ex3) = -Fld(i1, i_edge - i2, i3, em::ex3); + } + } + } + + if (tags & BC::B) { + if (i2 == 0) { + Fld(i1, i_edge, i3, em::bx2) = ZERO; + } else { + if constexpr (not P) { + Fld(i1, i_edge - i2, i3, em::bx1) = Fld(i1, + i_edge + i2 - 1, + i3, + em::bx1); + Fld(i1, i_edge - i2, i3, em::bx2) = -Fld(i1, i_edge + i2, i3, em::bx2); + Fld(i1, i_edge - i2, i3, em::bx3) = Fld(i1, + i_edge + i2 - 1, + i3, + em::bx3); + } else { + Fld(i1, i_edge + i2 - 1, i3, em::bx1) = Fld(i1, + i_edge - i2, + i3, + em::bx1); + Fld(i1, i_edge + i2, i3, em::bx2) = -Fld(i1, i_edge - i2, i3, em::bx2); + Fld(i1, i_edge + i2 - 1, i3, em::bx3) = Fld(i1, + i_edge - i2, + i3, + em::bx3); + } + } + } + } else { + if (tags & BC::E) { + if (i3 == 0) { + Fld(i1, i2, i_edge, em::ex1) = ZERO; + Fld(i1, i2, i_edge, em::ex2) = ZERO; + } else { + if constexpr (not P) { + Fld(i1, i2, i_edge - i3, em::ex1) = -Fld(i1, i2, i_edge + i3, em::ex1); + Fld(i1, i2, i_edge - i3, em::ex2) = -Fld(i1, i2, i_edge + i3, em::ex2); + Fld(i1, i2, i_edge - i3, em::ex3) = Fld(i1, + i2, + i_edge + i3 - 1, + em::ex3); + } else { + Fld(i1, i2, i_edge + i3, em::ex1) = -Fld(i1, i2, i_edge - i3, em::ex1); + Fld(i1, i2, i_edge + i3, em::ex2) = -Fld(i1, i2, i_edge - i3, em::ex2); + Fld(i1, i2, i_edge + i3 - 1, em::ex3) = Fld(i1, + i2, + i_edge - i3, + em::ex3); + } + } + } + + if (tags & BC::B) { + if (i3 == 0) { + Fld(i1, i2, i_edge, em::bx3) = ZERO; + } else { + if constexpr (not P) { + Fld(i1, i2, i_edge - i3, em::bx1) = Fld(i1, + i2, + i_edge + i3 - 1, + em::bx1); + Fld(i1, i2, i_edge - i3, em::bx2) = Fld(i1, + i2, + i_edge + i3 - 1, + em::bx2); + Fld(i1, i2, i_edge - i3, em::bx3) = -Fld(i1, i2, i_edge + i3, em::bx3); + } else { + Fld(i1, i2, i_edge + i3 - 1, em::bx1) = Fld(i1, + i2, + i_edge - i3, + em::bx1); + Fld(i1, i2, i_edge + i3 - 1, em::bx2) = Fld(i1, + i2, + i_edge - i3, + em::bx2); + Fld(i1, i2, i_edge + i3, em::bx3) = -Fld(i1, i2, i_edge - i3, em::bx3); + } + } } } } else { From af420129db2c35dd87b9e3ad0ea3a5249a305495 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Feb 2025 10:43:05 -0600 Subject: [PATCH 087/176] prep for conductor boundaries --- src/global/arch/traits.h | 12 ++++++++++++ src/global/enums.h | 7 ++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/global/arch/traits.h b/src/global/arch/traits.h index 4cde4fca5..65cc63cf8 100644 --- a/src/global/arch/traits.h +++ b/src/global/arch/traits.h @@ -112,6 +112,18 @@ namespace traits { template using fix_fields_const_t = decltype(&T::FixFieldsConst); + template + using perfect_conductor_fields_t = decltype(&T::PerfectConductorFields); + + template + using perfect_conductor_fields_const_t = decltype(&T::PerfectConductorFieldsConst); + + template + using perfect_conductor_currents_t = decltype(&T::PerfectConductorCurrents); + + template + using perfect_conductor_currents_const_t = decltype(&T::PerfectConductorCurrentsConst); + template using custom_fields_t = decltype(&T::CustomFields); diff --git a/src/global/enums.h b/src/global/enums.h index 3afa0497a..8407d0b66 100644 --- a/src/global/enums.h +++ b/src/global/enums.h @@ -222,15 +222,16 @@ namespace ntt { HORIZON = 6, AXIS = 7, SYNC = 8, // <- SYNC means synchronization with other domains + CONDUCTOR = 9 }; constexpr FldsBC(uint8_t c) : enums_hidden::BaseEnum { c } {} - static constexpr type variants[] = { PERIODIC, MATCH, FIXED, ATMOSPHERE, - CUSTOM, HORIZON, AXIS, SYNC }; + static constexpr type variants[] = { PERIODIC, MATCH, FIXED, ATMOSPHERE, + CUSTOM, HORIZON, AXIS, SYNC, CONDUCTOR }; static constexpr const char* lookup[] = { "periodic", "match", "fixed", "atmosphere", "custom", "horizon", - "axis", "sync" }; + "axis", "sync", "conductor"}; static constexpr std::size_t total = sizeof(variants) / sizeof(variants[0]); }; From 7c6ddf46887ee75b87a291a4444c07a06da8691e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Feb 2025 10:43:32 -0600 Subject: [PATCH 088/176] first stubborn attempt at conductor boundaries (broken) --- setups/srpic/shock/pgen.hpp | 46 +++++++++++++ src/engines/srpic.hpp | 133 ++++++++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index b8f169521..59c5590c9 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -106,6 +106,52 @@ namespace user { } } + + auto PerfectConductorFieldsConst(const bc_in&, const em& comp) const -> real_t{ + + // electric field components + if (comp == em::ex1) { + return ONE; + } else if (comp == em::ex2) { + return -ONE; + } else if (comp == em::ex3) { + return -ONE; } + // magentic field components + else if (comp == em::bx1) { + return -ONE; + } else if (comp == em::bx2) { + return ONE; + } else if (comp == em::bx3) { + return ONE;} + // should never be the case + else + { + return ZERO; + } + } + + // auto PerfectConductorCurrentsConst(const bc_in &, const cur &comp) const + // -> std::pair + // { + // // ToDo + // if (comp == cur::jx1) + // { + // return ZERO; + // } + // else if (comp == cur::jx2) + // { + // return ZERO; + // } + // else if (comp == cur::jx3) + // { + // return ZERO; + // } + // else + // { + // return ZERO; + // } + // } + auto MatchFields(real_t time) const -> InitFields { return init_flds; } diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 0a9cc311b..c5b48310e 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -596,6 +596,10 @@ namespace ntt { if (domain.mesh.flds_bc_in(direction) == FldsBC::FIXED) { FixedFieldsIn(direction, domain, tags); } + } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::CONDUCTOR) { + if (domain.mesh.flds_bc_in(direction) == FldsBC::CONDUCTOR) { + PerfectConductorFieldsIn(direction, domain, tags); + } } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::CUSTOM) { if (domain.mesh.flds_bc_in(direction) == FldsBC::CUSTOM) { CustomFieldsIn(direction, domain, tags); @@ -834,6 +838,135 @@ namespace ntt { } } + void PerfectConductorFieldsIn(dir::direction_t direction, + domain_t& domain, + BCTags tags) { + /** + * perfect conductor field boundaries + */ + const auto sign = direction.get_sign(); + const auto dim = direction.get_dim(); + raise::ErrorIf(dim != in::x1 and M::CoordType != Coord::Cart, + "Perfect conductor BCs only implemented for x1 in " + "non-cartesian coordinates", + HERE); + + // magnetic and electron field components + em normal_b_comp, tang_b_comp1, tang_b_comp2, + normal_e_comp, tang_e_comp1, tang_e_comp2; + + // current components + // cur normal_j_comp, tang_j_comp1, tang_j_comp2; + + if (dim == in::x1) { + normal_b_comp = em::bx1; + tang_b_comp1 = em::bx2; + tang_b_comp2 = em::bx3; + + normal_e_comp = em::ex1; + tang_e_comp1 = em::ex2; + tang_e_comp2 = em::ex3; + } else if (dim == in::x2) { + normal_b_comp = em::bx2; + tang_b_comp1 = em::bx1; + tang_b_comp2 = em::bx3; + + normal_e_comp = em::ex2; + tang_e_comp1 = em::ex1; + tang_e_comp2 = em::ex3; + } else if (dim == in::x3) { + normal_b_comp = em::bx3; + tang_b_comp1 = em::bx1; + tang_b_comp2 = em::bx2; + + normal_e_comp = em::ex3; + tang_e_comp1 = em::ex1; + tang_e_comp2 = em::ex2; + } else { + raise::Error("Invalid dimension", HERE); + } + + std::vector origin_xi_min, origin_xi_max, + target_xi_min, target_xi_max; + const std::vector all_dirs { in::x1, in::x2, in::x3 }; + + for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { + const auto dd = all_dirs[d]; + if (dim == dd) { + // origin: right side of boundary + origin_xi_min.push_back(N_GHOSTS+1); + origin_xi_max.push_back(2*N_GHOSTS); + // target: left side of boundary + target_xi_min.push_back(0); + target_xi_max.push_back(N_GHOSTS); + + } else { + origin_xi_min.push_back(0); + origin_xi_max.push_back(domain.mesh.n_all(dd)); + + target_xi_min.push_back(0); + target_xi_max.push_back(domain.mesh.n_all(dd)); + } + } + raise::ErrorIf(target_xi_min.size() != origin_xi_min.size() or + origin_xi_min.size() != static_cast(M::Dim), + "Invalid range size", + HERE); + + std::vector comps; + if (tags & BC::E) { + comps.push_back(normal_e_comp); + comps.push_back(tang_e_comp1); + comps.push_back(tang_e_comp2); + } + if (tags & BC::B) { + comps.push_back(normal_b_comp); + comps.push_back(tang_b_comp1); + comps.push_back(tang_b_comp2); + } + + // ToDo: smarter loop/views + auto EB = domain.fields.em; + + // loop over all components + for (const auto& comp : comps) { + + // store sign of component behind boundary + auto new_sign = m_pgen.PerfectConductorFieldsConst( + (bc_in)(sign * ((short)dim + 1)), + (em)comp); + // to do: Kokkos::parallel_for + for (int i = 0; i < N_GHOSTS; i++) + { + if constexpr (M::Dim == Dim::_1D) { + // multiply with correct sign + EB(target_xi_min[0]+i, comp) = new_sign * EB(origin_xi_max[0]-i, comp); + + } else if constexpr (M::Dim == Dim::_2D) { + for (int j = 0; j < domain.mesh.n_all(in::x2); j++) + { + EB(target_xi_min[0]+i, j, comp) = + new_sign * EB(origin_xi_max[0]-i, j, comp); + } + } else if constexpr (M::Dim == Dim::_3D) { + for (int j = 0; j < domain.mesh.n_all(in::x2); j++) + { + for (int k = 0; k < domain.mesh.n_all(in::x3); k++) + { + EB(target_xi_min[0]+i, j, k, comp) = + new_sign * EB(origin_xi_max[0]-i, j, k, comp); + } + } + } else { + raise::Error("Invalid dimension", HERE); + } + } + + // ToDo: set zero at boundary + } + } + + void AtmosphereFieldsIn(dir::direction_t direction, domain_t& domain, BCTags tags) { From 9333b76b33cc08dbcc5427331ec37b202bda963f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Feb 2025 16:41:20 -0600 Subject: [PATCH 089/176] first attempt at ConductorBoundaries_kernel --- src/kernels/fields_bcs.hpp | 107 +++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index dbb47f42c..cadc9cfda 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -485,6 +485,113 @@ namespace kernel::bc { } }; + template + struct ConductorBoundaries_kernel { + static_assert(M::is_metric, "M must be a metric class"); + static_assert(static_cast(o) < + static_cast(M::Dim), + "Invalid component index"); + static constexpr idx_t i = static_cast(o) + 1u; + + ndfield_t Fld; + const I fset; + const M metric; + const BCTags tags; + + ConductorBoundaries_kernel(ndfield_t Fld, + BCTags tags) + : Fld { Fld } + , tags { tags } {} + + Inline void operator()(index_t i1) const { + if constexpr (M::Dim == Dim::_1D) { + + if constexpr (S == SimEngine::SRPIC) { + + if (tags & BC::E) { + Fld((N_GHOSTS-1)-i1, em::ex1) = Fld(N_GHOSTS+i1, em::ex1); + Fld((N_GHOSTS-1)-i1, em::ex2) = -Fld(N_GHOSTS+i1, em::ex2); + Fld((N_GHOSTS-1)-i1, em::ex3) = -Fld(N_GHOSTS+i1, em::ex3); + } + + if (tags & BC::B) + { + Fld((N_GHOSTS-1)-i1, em::bx1) = -Fld(N_GHOSTS+i1, em::bx1); + Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); + Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); + } + + } else { + // GRPIC + raise::KernelError(HERE, "1D GRPIC not implemented"); + } + } else { + raise::KernelError( + HERE, + "ConductorBoundaries_kernel: 1D implementation called for D != 1"); + } + } + + Inline void operator()(index_t i1, index_t i2) const { + if constexpr (M::Dim == Dim::_2D) { + + if constexpr (S == SimEngine::SRPIC) { + // SRPIC + if (tags & BC::E) { + Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); + Fld((N_GHOSTS-1)-i1, i2, em::ex2) = -Fld(N_GHOSTS+i1, i2, em::ex2); + Fld((N_GHOSTS-1)-i1, i2, em::ex3) = -Fld(N_GHOSTS+i1, i2, em::ex3); + } + + if (tags & BC::B) + { + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = -Fld(N_GHOSTS+i1, i2, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); + } + } else { + // GRPIC + raise::KernelError(HERE, "2D GRPIC not implemented"); + } + } else { + raise::KernelError( + HERE, + "ConductorBoundaries_kernel: 2D implementation called for D != 2"); + } + } + + Inline void operator()(index_t i1, index_t i2, index_t i3) const { + if constexpr (M::Dim == Dim::_3D) { + const auto i1_ = COORD(i1); + const auto i2_ = COORD(i2); + const auto i3_ = COORD(i3); + + if constexpr (S == SimEngine::SRPIC) { + // SRPIC + if (tags & BC::E) { + Fld((N_GHOSTS-1)-i1, i2, i3, em::ex1) = Fld(N_GHOSTS+i1, i2, i3, em::ex1); + Fld((N_GHOSTS-1)-i1, i2, i3, em::ex2) = -Fld(N_GHOSTS+i1, i2, i3, em::ex2); + Fld((N_GHOSTS-1)-i1, i2, i3, em::ex3) = -Fld(N_GHOSTS+i1, i2, i3, em::ex3); + } + + if (tags & BC::B) + { + Fld((N_GHOSTS-1)-i1, i2, i3, em::bx1) = -Fld(N_GHOSTS+i1, i2, i3, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, i3, em::bx2) = Fld(N_GHOSTS+i1, i2, i3, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, i3, em::bx3) = Fld(N_GHOSTS+i1, i2, i3, em::bx3); + } + } else { + // GRPIC + raise::KernelError(HERE, "3D GRPIC not implemented"); + } + } else { + raise::KernelError( + HERE, + "ConductorBoundaries_kernel: 3D implementation called for D != 3"); + } + } + }; + /* * @tparam D: Dimension * @tparam P: Positive/Negative direction From 156507077a91473816ba5b3503c2f4b09bd20a15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Feb 2025 16:55:21 -0600 Subject: [PATCH 090/176] bugfix --- src/kernels/fields_bcs.hpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index cadc9cfda..95c32d894 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -485,7 +485,7 @@ namespace kernel::bc { } }; - template + template struct ConductorBoundaries_kernel { static_assert(M::is_metric, "M must be a metric class"); static_assert(static_cast(o) < @@ -494,7 +494,6 @@ namespace kernel::bc { static constexpr idx_t i = static_cast(o) + 1u; ndfield_t Fld; - const I fset; const M metric; const BCTags tags; @@ -562,13 +561,10 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2, index_t i3) const { if constexpr (M::Dim == Dim::_3D) { - const auto i1_ = COORD(i1); - const auto i2_ = COORD(i2); - const auto i3_ = COORD(i3); if constexpr (S == SimEngine::SRPIC) { // SRPIC - if (tags & BC::E) { + if (tags & BC::E) { Fld((N_GHOSTS-1)-i1, i2, i3, em::ex1) = Fld(N_GHOSTS+i1, i2, i3, em::ex1); Fld((N_GHOSTS-1)-i1, i2, i3, em::ex2) = -Fld(N_GHOSTS+i1, i2, i3, em::ex2); Fld((N_GHOSTS-1)-i1, i2, i3, em::ex3) = -Fld(N_GHOSTS+i1, i2, i3, em::ex3); From 6a136fbda06f66448bd2e29791cb46ca18b98948 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Feb 2025 17:02:04 -0600 Subject: [PATCH 091/176] first attempt at Kokkos::parallel_for loop (broken) --- src/engines/srpic.hpp | 117 ++++++------------------------------------ 1 file changed, 17 insertions(+), 100 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index c5b48310e..0a2ae360b 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -851,122 +851,39 @@ namespace ntt { "non-cartesian coordinates", HERE); - // magnetic and electron field components - em normal_b_comp, tang_b_comp1, tang_b_comp2, - normal_e_comp, tang_e_comp1, tang_e_comp2; - // current components - // cur normal_j_comp, tang_j_comp1, tang_j_comp2; + std::vector xi_min, xi_max; - if (dim == in::x1) { - normal_b_comp = em::bx1; - tang_b_comp1 = em::bx2; - tang_b_comp2 = em::bx3; - - normal_e_comp = em::ex1; - tang_e_comp1 = em::ex2; - tang_e_comp2 = em::ex3; - } else if (dim == in::x2) { - normal_b_comp = em::bx2; - tang_b_comp1 = em::bx1; - tang_b_comp2 = em::bx3; - - normal_e_comp = em::ex2; - tang_e_comp1 = em::ex1; - tang_e_comp2 = em::ex3; - } else if (dim == in::x3) { - normal_b_comp = em::bx3; - tang_b_comp1 = em::bx1; - tang_b_comp2 = em::bx2; - - normal_e_comp = em::ex3; - tang_e_comp1 = em::ex1; - tang_e_comp2 = em::ex2; - } else { - raise::Error("Invalid dimension", HERE); - } - - std::vector origin_xi_min, origin_xi_max, - target_xi_min, target_xi_max; const std::vector all_dirs { in::x1, in::x2, in::x3 }; for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { const auto dd = all_dirs[d]; if (dim == dd) { - // origin: right side of boundary - origin_xi_min.push_back(N_GHOSTS+1); - origin_xi_max.push_back(2*N_GHOSTS); - // target: left side of boundary - target_xi_min.push_back(0); - target_xi_max.push_back(N_GHOSTS); - + xi_min.push_back(0); + xi_max.push_back(N_GHOSTS); } else { - origin_xi_min.push_back(0); - origin_xi_max.push_back(domain.mesh.n_all(dd)); - - target_xi_min.push_back(0); - target_xi_max.push_back(domain.mesh.n_all(dd)); + xi_min.push_back(0); + xi_max.push_back(domain.mesh.n_all(dd)); } } - raise::ErrorIf(target_xi_min.size() != origin_xi_min.size() or - origin_xi_min.size() != static_cast(M::Dim), + raise::ErrorIf(xi_min.size() != xi_max.size() or + xi_min.size() != static_cast(M::Dim), "Invalid range size", HERE); - std::vector comps; - if (tags & BC::E) { - comps.push_back(normal_e_comp); - comps.push_back(tang_e_comp1); - comps.push_back(tang_e_comp2); - } - if (tags & BC::B) { - comps.push_back(normal_b_comp); - comps.push_back(tang_b_comp1); - comps.push_back(tang_b_comp2); - } - - // ToDo: smarter loop/views - auto EB = domain.fields.em; - - // loop over all components - for (const auto& comp : comps) { - - // store sign of component behind boundary - auto new_sign = m_pgen.PerfectConductorFieldsConst( - (bc_in)(sign * ((short)dim + 1)), - (em)comp); - // to do: Kokkos::parallel_for - for (int i = 0; i < N_GHOSTS; i++) - { - if constexpr (M::Dim == Dim::_1D) { - // multiply with correct sign - EB(target_xi_min[0]+i, comp) = new_sign * EB(origin_xi_max[0]-i, comp); - - } else if constexpr (M::Dim == Dim::_2D) { - for (int j = 0; j < domain.mesh.n_all(in::x2); j++) - { - EB(target_xi_min[0]+i, j, comp) = - new_sign * EB(origin_xi_max[0]-i, j, comp); - } - } else if constexpr (M::Dim == Dim::_3D) { - for (int j = 0; j < domain.mesh.n_all(in::x2); j++) - { - for (int k = 0; k < domain.mesh.n_all(in::x3); k++) - { - EB(target_xi_min[0]+i, j, k, comp) = - new_sign * EB(origin_xi_max[0]-i, j, k, comp); - } - } - } else { - raise::Error("Invalid dimension", HERE); - } - } - - // ToDo: set zero at boundary + if (dim == in::x1) + { + Kokkos::parallel_for( + "MatchFields", + CreateRangePolicy(xi_min, xi_max), + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); } + + } - void AtmosphereFieldsIn(dir::direction_t direction, domain_t& domain, BCTags tags) { From 861e78345702dd606f9a16161ed78306fd5834ea Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 21 Feb 2025 22:12:56 -0500 Subject: [PATCH 092/176] Ongoing. --- src/engines/srpic.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 0a2ae360b..a317aa7ea 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -871,11 +871,11 @@ namespace ntt { "Invalid range size", HERE); - if (dim == in::x1) + if constexpr (M::Dim == Dim::_1D) { { Kokkos::parallel_for( "MatchFields", - CreateRangePolicy(xi_min, xi_max), + CreateRangePolicy(xi_min[0], xi_max[0]), kernel::bc::ConductorBoundaries_kernel( domain.fields.em, tags)); From cd81a5b467bee1ea8379cf90764a78af51ec3006 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 21 Feb 2025 22:16:54 -0500 Subject: [PATCH 093/176] Ongoing. --- src/engines/srpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index a317aa7ea..9d015aa7e 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -871,7 +871,7 @@ namespace ntt { "Invalid range size", HERE); - if constexpr (M::Dim == Dim::_1D) { + if constexpr (M::Dim == Dim::_1D) { Kokkos::parallel_for( "MatchFields", From aec0d414a13ee09f92afc6c701782a265b8c8faf Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 21 Feb 2025 22:19:01 -0500 Subject: [PATCH 094/176] Ongoing. --- src/engines/srpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 9d015aa7e..c997f026a 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -875,7 +875,7 @@ namespace ntt { { Kokkos::parallel_for( "MatchFields", - CreateRangePolicy(xi_min[0], xi_max[0]), + CreateRangePolicy( { xi_min[0] } , { xi_max[0] } ), kernel::bc::ConductorBoundaries_kernel( domain.fields.em, tags)); From 1c7105bb47d3dbcc9eb66d9ea30920d7627fde40 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 21 Feb 2025 22:35:06 -0500 Subject: [PATCH 095/176] Ongoing. --- src/engines/srpic.hpp | 2 +- src/kernels/fields_bcs.hpp | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index c997f026a..4bc16d273 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -876,7 +876,7 @@ namespace ntt { Kokkos::parallel_for( "MatchFields", CreateRangePolicy( { xi_min[0] } , { xi_max[0] } ), - kernel::bc::ConductorBoundaries_kernel( + kernel::bc::ConductorBoundaries_kernel( domain.fields.em, tags)); } diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 95c32d894..bb851b45f 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -494,7 +494,6 @@ namespace kernel::bc { static constexpr idx_t i = static_cast(o) + 1u; ndfield_t Fld; - const M metric; const BCTags tags; ConductorBoundaries_kernel(ndfield_t Fld, From de66f25dc33795fcf7412bce377176e2863d5898 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 21 Feb 2025 22:35:54 -0500 Subject: [PATCH 096/176] Ongoing. --- src/engines/srpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 4bc16d273..c997f026a 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -876,7 +876,7 @@ namespace ntt { Kokkos::parallel_for( "MatchFields", CreateRangePolicy( { xi_min[0] } , { xi_max[0] } ), - kernel::bc::ConductorBoundaries_kernel( + kernel::bc::ConductorBoundaries_kernel( domain.fields.em, tags)); } From 2f8a01e374665c89d89b7e8deea6e8b235deac32 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 11:16:58 -0500 Subject: [PATCH 097/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 179 ++++++++++++++++++++++++++++++ setups/srpic/shocktest/shock.py | 75 +++++++++++++ setups/srpic/shocktest/shock.toml | 56 ++++++++++ 3 files changed, 310 insertions(+) create mode 100644 setups/srpic/shocktest/pgen.hpp create mode 100644 setups/srpic/shocktest/shock.py create mode 100644 setups/srpic/shocktest/shock.toml diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp new file mode 100644 index 000000000..83f19df52 --- /dev/null +++ b/setups/srpic/shocktest/pgen.hpp @@ -0,0 +1,179 @@ +#ifndef PROBLEM_GENERATOR_H +#define PROBLEM_GENERATOR_H + +#include "enums.h" +#include "global.h" + +#include "arch/traits.h" +#include "utils/error.h" +#include "utils/numeric.h" + +#include "archetypes/energy_dist.h" +#include "archetypes/particle_injector.h" +#include "archetypes/problem_generator.h" +#include "framework/domain/metadomain.h" + +#include + +namespace user { + using namespace ntt; + + template + struct InitFields { + /* + Sets up magnetic and electric field components for the simulation. + Must satisfy E = -v x B for Lorentz Force to be zero. + + @param bmag: magnetic field scaling + @param btheta: magnetic field polar angle + @param bphi: magnetic field azimuthal angle + @param drift_ux: drift velocity in the x direction + */ + InitFields(real_t bmag, real_t btheta, real_t bphi, real_t drift_ux) + : Bmag { bmag } + , Btheta { btheta * static_cast(convert::deg2rad) } + , Bphi { bphi * static_cast(convert::deg2rad) } + , Vx { drift_ux } {} + + // magnetic field components + Inline auto bx1(const coord_t& x_Ph) const -> real_t { + return x_Ph[0]; + } + + Inline auto bx2(const coord_t&) const -> real_t { + return x_Ph[0]; + } + + Inline auto bx3(const coord_t&) const -> real_t { + return x_Ph[0]; + } + + // electric field components + Inline auto ex1(const coord_t&) const -> real_t { + return x_Ph[0]; + } + + Inline auto ex2(const coord_t&) const -> real_t { + return x_Ph[0]; + } + + Inline auto ex3(const coord_t&) const -> real_t { + return x_Ph[0]; + } + + private: + const real_t Btheta, Bphi, Vx, Bmag; + }; + + template + struct PGen : public arch::ProblemGenerator { + // compatibility traits for the problem generator + static constexpr auto engines { traits::compatible_with::value }; + static constexpr auto metrics { traits::compatible_with::value }; + static constexpr auto dimensions { + traits::compatible_with::value + }; + + // for easy access to variables in the child class + using arch::ProblemGenerator::D; + using arch::ProblemGenerator::C; + using arch::ProblemGenerator::params; + + const real_t drift_ux, temperature; + + const real_t Btheta, Bphi, Bmag; + InitFields init_flds; + + inline PGen(const SimulationParams& p, const Metadomain& m) + : arch::ProblemGenerator { p } + , drift_ux { p.template get("setup.drift_ux") } + , temperature { p.template get("setup.temperature") } + , Bmag { p.template get("setup.Bmag", ZERO) } + , Btheta { p.template get("setup.Btheta", ZERO) } + , Bphi { p.template get("setup.Bphi", ZERO) } + , init_flds { Bmag, Btheta, Bphi, drift_ux } {} + + inline PGen() {} + + auto FixFieldsConst(const bc_in&, const em& comp) const + -> std::pair { + if (comp == em::ex2) { + return { init_flds.ex2({ ZERO }), true }; + } else if (comp == em::ex3) { + return { init_flds.ex3({ ZERO }), true }; + } else { + return { ZERO, false }; + } + } + + + auto PerfectConductorFieldsConst(const bc_in&, const em& comp) const -> real_t{ + + // electric field components + if (comp == em::ex1) { + return ONE; + } else if (comp == em::ex2) { + return -ONE; + } else if (comp == em::ex3) { + return -ONE; } + // magentic field components + else if (comp == em::bx1) { + return -ONE; + } else if (comp == em::bx2) { + return ONE; + } else if (comp == em::bx3) { + return ONE;} + // should never be the case + else + { + return ZERO; + } + } + + // auto PerfectConductorCurrentsConst(const bc_in &, const cur &comp) const + // -> std::pair + // { + // // ToDo + // if (comp == cur::jx1) + // { + // return ZERO; + // } + // else if (comp == cur::jx2) + // { + // return ZERO; + // } + // else if (comp == cur::jx3) + // { + // return ZERO; + // } + // else + // { + // return ZERO; + // } + // } + + auto MatchFields(real_t time) const -> InitFields { + return init_flds; + } + + inline void InitPrtls(Domain& local_domain) { + const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, + local_domain.random_pool, + temperature, + -drift_ux, + in::x1); + + const auto injector = arch::UniformInjector( + energy_dist, + { 1, 2 }); + // arch::InjectUniform>( + // params, + // local_domain, + // injector, + // 1.0); + } + }; + +} // namespace user + +#endif diff --git a/setups/srpic/shocktest/shock.py b/setups/srpic/shocktest/shock.py new file mode 100644 index 000000000..dc1565572 --- /dev/null +++ b/setups/srpic/shocktest/shock.py @@ -0,0 +1,75 @@ +import nt2.read as nt2r +import matplotlib.pyplot as plt +import matplotlib as mpl + +data = nt2r.Data("shock.h5") + + +def frame(ti, f): + quantities = [ + { + "name": "density", + "compute": lambda f: f.N_2 + f.N_1, + "cmap": "inferno", + "norm": mpl.colors.Normalize(0, 5), + }, + { + "name": r"$E_x$", + "compute": lambda f: f.Ex, + "cmap": "RdBu_r", + "norm": mpl.colors.Normalize(-0.05, 0.05), + }, + { + "name": r"$E_y$", + "compute": lambda f: f.Ey, + "cmap": "RdBu_r", + "norm": mpl.colors.Normalize(-0.05, 0.05), + }, + { + "name": r"$E_z$", + "compute": lambda f: f.Ez, + "cmap": "RdBu_r", + "norm": mpl.colors.Normalize(-0.05, 0.05), + }, + { + "name": r"$B_x$", + "compute": lambda f: f.Bx, + "cmap": "BrBG", + "norm": mpl.colors.Normalize(-0.05, 0.05), + }, + { + "name": r"$B_y$", + "compute": lambda f: f.By, + "cmap": "BrBG", + "norm": mpl.colors.Normalize(-0.05, 0.05), + }, + { + "name": r"$B_z$", + "compute": lambda f: f.Bz, + "cmap": "BrBG", + "norm": mpl.colors.Normalize(-0.05, 0.05), + }, + ] + fig = plt.figure(figsize=(12, 5.5), dpi=300) + gs = fig.add_gridspec(len(quantities), 1, hspace=0.02) + axs = [fig.add_subplot(gs[i]) for i in range(len(quantities))] + + for ax, q in zip(axs, quantities): + q["compute"](f.isel(t=ti)).plot( + ax=ax, + cmap=q["cmap"], + norm=q["norm"], + cbar_kwargs={"label": q["name"], "shrink": 0.8, "aspect": 10, "pad": 0.005}, + ) + for i, ax in enumerate(axs): + ax.set(aspect=1) + if i != 0: + ax.set(title=None) + if i != len(axs) - 1: + ax.set( + xticks=[], + xticklabels=[], + xlabel=None, + title=ax.get_title().split(",")[0], + ) + return fig diff --git a/setups/srpic/shocktest/shock.toml b/setups/srpic/shocktest/shock.toml new file mode 100644 index 000000000..fdbc44465 --- /dev/null +++ b/setups/srpic/shocktest/shock.toml @@ -0,0 +1,56 @@ +[simulation] + name = "shock" + engine = "srpic" + runtime = 50.0 + +[grid] + resolution = [2048, 128] + extent = [[0.0, 10.0], [-0.3125, 0.3125]] + + [grid.metric] + metric = "minkowski" + + [grid.boundaries] + fields = [["CONDUCTOR", "FIXED"], ["PERIODIC"]] + particles = [["REFLECT", "ABSORB"], ["PERIODIC"]] + +[scales] + larmor0 = 1e-2 + skindepth0 = 1e-2 + +[algorithms] + current_filters = 8 + fieldsolver = "false" + deposit = "false" + + [algorithms.timestep] + CFL = 0.5 + +[particles] + ppc0 = 16.0 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = -1.0 + maxnpart = 1e8 + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = 1.0 + maxnpart = 1e8 + +[setup] + drift_ux = 0.1 + temperature = 1e-3 + Bmag = 1.0 + Btheta = 0.0 + Bphi = 0.0 + +[output] + interval = 1 + format = "hdf5" + + [output.fields] + quantities = ["N_1", "N_2", "E", "B", "T0i_1", "T0i_2", "J"] From 2c7019b58b1d717e4a126a75382274be29aa63a4 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 11:20:28 -0500 Subject: [PATCH 098/176] Ongoing. --- setups/srpic/shocktest/shock.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setups/srpic/shocktest/shock.toml b/setups/srpic/shocktest/shock.toml index fdbc44465..a77f5c2e9 100644 --- a/setups/srpic/shocktest/shock.toml +++ b/setups/srpic/shocktest/shock.toml @@ -54,3 +54,6 @@ [output.fields] quantities = ["N_1", "N_2", "E", "B", "T0i_1", "T0i_2", "J"] + + [output.debug] + ghosts = true From c0414671c6986b015f589c5832dd83fa7432a1c0 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 14:01:06 -0500 Subject: [PATCH 099/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 83f19df52..7724893a1 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -40,24 +40,24 @@ namespace user { return x_Ph[0]; } - Inline auto bx2(const coord_t&) const -> real_t { + Inline auto bx2(const coord_t& x_Ph) const -> real_t { return x_Ph[0]; } - Inline auto bx3(const coord_t&) const -> real_t { + Inline auto bx3(const coord_t& x_Ph) const -> real_t { return x_Ph[0]; } // electric field components - Inline auto ex1(const coord_t&) const -> real_t { + Inline auto ex1(const coord_t& x_Ph) const -> real_t { return x_Ph[0]; } - Inline auto ex2(const coord_t&) const -> real_t { + Inline auto ex2(const coord_t& x_Ph) const -> real_t { return x_Ph[0]; } - Inline auto ex3(const coord_t&) const -> real_t { + Inline auto ex3(const coord_t& x_Ph) const -> real_t { return x_Ph[0]; } From b85346ebc97ff7a5f59e17248187e8df71704528 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 17:15:38 -0500 Subject: [PATCH 100/176] Ongoing. --- src/kernels/fields_bcs.hpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index bb851b45f..e8993907c 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,9 +543,12 @@ namespace kernel::bc { if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = -Fld(N_GHOSTS+i1, i2, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); + // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = -Fld(N_GHOSTS+i1, i2, em::bx1); + // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); + // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; } } else { // GRPIC From b51a8bb69e885b2281e1b563254fd293289a7e8a Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 17:21:07 -0500 Subject: [PATCH 101/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 7724893a1..0dc91d3e2 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -41,7 +41,7 @@ namespace user { } Inline auto bx2(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return -x_Ph[0]; } Inline auto bx3(const coord_t& x_Ph) const -> real_t { From f94e0fb79c268a20bfdba1c5d4161142ba5ce17a Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 18:50:58 -0500 Subject: [PATCH 102/176] Ongoing. --- src/engines/srpic.hpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index c997f026a..86b85de32 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -880,6 +880,16 @@ namespace ntt { domain.fields.em, tags)); } + + if constexpr (M::Dim == Dim::_2D) + { + Kokkos::parallel_for( + "MatchFields", + CreateRangePolicy( { xi_min[0], xi_min[1] } , { xi_max[0], xi_max[1] } ), + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } } From 7536825ef6942ff41e85c01abfe9ce7db19c1947 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 19:05:19 -0500 Subject: [PATCH 103/176] Ongoing. --- src/kernels/fields_bcs.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index e8993907c..c85115f12 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,12 +543,12 @@ namespace kernel::bc { if (tags & BC::B) { - // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = -Fld(N_GHOSTS+i1, i2, em::bx1); - // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); - // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = Fld(N_GHOSTS+i1, i2, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); + // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; + // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; + // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; } } else { // GRPIC From f313624b9171651d79eb888435106edfba028000 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 19:10:18 -0500 Subject: [PATCH 104/176] Ongoing. --- src/kernels/fields_bcs.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index c85115f12..da6e88dbd 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,9 +543,9 @@ namespace kernel::bc { if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = Fld(N_GHOSTS+i1, i2, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; From dd6056edef8e71ed274ea78879028d03a07ec4d7 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 20:01:27 -0500 Subject: [PATCH 105/176] Ongoing. --- src/kernels/fields_bcs.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index da6e88dbd..58e5af4b4 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,9 +543,9 @@ namespace kernel::bc { if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); + Fld((N_GHOSTS)-i1, i2, em::bx1) = - Fld(N_GHOSTS+1+i1, i2, em::bx1); + Fld((N_GHOSTS)-i1, i2, em::bx2) = - Fld(N_GHOSTS+1+i1, i2, em::bx2); + Fld((N_GHOSTS)-i1, i2, em::bx3) = - Fld(N_GHOSTS+1+i1, i2, em::bx3); // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; From 67a6a191ec16043c14e5079da2942b2f431e446b Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 20:54:37 -0500 Subject: [PATCH 106/176] Ongoing. --- src/kernels/fields_bcs.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 58e5af4b4..da6e88dbd 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,9 +543,9 @@ namespace kernel::bc { if (tags & BC::B) { - Fld((N_GHOSTS)-i1, i2, em::bx1) = - Fld(N_GHOSTS+1+i1, i2, em::bx1); - Fld((N_GHOSTS)-i1, i2, em::bx2) = - Fld(N_GHOSTS+1+i1, i2, em::bx2); - Fld((N_GHOSTS)-i1, i2, em::bx3) = - Fld(N_GHOSTS+1+i1, i2, em::bx3); + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; From 679d0374d52ea2a1d56cffb4fe8d72514ed1c4de Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 21:02:54 -0500 Subject: [PATCH 107/176] Ongoing. --- src/kernels/fields_bcs.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index da6e88dbd..31cc55268 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,12 +543,12 @@ namespace kernel::bc { if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); - // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; - // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; - // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; + // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); + // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); + // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; } } else { // GRPIC From 80043ef77450790525d694ffe204b826fabf3f0a Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 21:13:50 -0500 Subject: [PATCH 108/176] Ongoing. --- src/kernels/fields_bcs.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 31cc55268..0c6d0782c 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -543,12 +543,12 @@ namespace kernel::bc { if (tags & BC::B) { - // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); - // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); - // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+1+i1, i2, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); + // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; + // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; + // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; } } else { // GRPIC From aff570eec4c46872be891aa87a02d972d0235973 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 21:43:02 -0500 Subject: [PATCH 109/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 0dc91d3e2..685521f09 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -45,7 +45,7 @@ namespace user { } Inline auto bx3(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return ONE; } // electric field components From 6d4eb13bed081dd157c0527e65ef3461572f7ec9 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 22:06:25 -0500 Subject: [PATCH 110/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 685521f09..0dc91d3e2 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -45,7 +45,7 @@ namespace user { } Inline auto bx3(const coord_t& x_Ph) const -> real_t { - return ONE; + return x_Ph[0]; } // electric field components From c3aa545c58fb323651a03999298131fd28fc94b0 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 22:16:17 -0500 Subject: [PATCH 111/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 0dc91d3e2..f8bf67dab 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -37,7 +37,7 @@ namespace user { // magnetic field components Inline auto bx1(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return ZERO; } Inline auto bx2(const coord_t& x_Ph) const -> real_t { @@ -50,15 +50,15 @@ namespace user { // electric field components Inline auto ex1(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return ZERO; } Inline auto ex2(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return ZERO; } Inline auto ex3(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return ZERO; } private: From f5d9bec2182ded70cee21ec9db214f74c2f75951 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 22:34:56 -0500 Subject: [PATCH 112/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index f8bf67dab..39ce8ea03 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -45,7 +45,7 @@ namespace user { } Inline auto bx3(const coord_t& x_Ph) const -> real_t { - return x_Ph[0]; + return -x_Ph[0]; } // electric field components From 2c2d595947601bba212ea37432f911ff304089e2 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Sat, 22 Feb 2025 22:58:19 -0500 Subject: [PATCH 113/176] Ongoing. --- src/kernels/fields_bcs.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 0c6d0782c..7f9a054a5 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -536,9 +536,9 @@ namespace kernel::bc { if constexpr (S == SimEngine::SRPIC) { // SRPIC if (tags & BC::E) { - Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); - Fld((N_GHOSTS-1)-i1, i2, em::ex2) = -Fld(N_GHOSTS+i1, i2, em::ex2); - Fld((N_GHOSTS-1)-i1, i2, em::ex3) = -Fld(N_GHOSTS+i1, i2, em::ex3); + Fld((N_GHOSTS-1)-i1, i2, em::ex1) = - Fld(N_GHOSTS+i1, i2, em::ex1); + Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+1+i1, i2, em::ex2); + Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+1+i1, i2, em::ex3); } if (tags & BC::B) From 54b02223075e7bb01204c31e76967e96a9aed182 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Tue, 25 Feb 2025 23:51:36 -0500 Subject: [PATCH 114/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 39ce8ea03..2cb66539f 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -36,16 +36,16 @@ namespace user { , Vx { drift_ux } {} // magnetic field components - Inline auto bx1(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto bx1(const coord_t&) const -> real_t { + return Bmag * math::cos(Btheta); } - Inline auto bx2(const coord_t& x_Ph) const -> real_t { - return -x_Ph[0]; + Inline auto bx2(const coord_t&) const -> real_t { + return Bmag * math::sin(Btheta) * math::sin(Bphi); } - Inline auto bx3(const coord_t& x_Ph) const -> real_t { - return -x_Ph[0]; + Inline auto bx3(const coord_t&) const -> real_t { + return Bmag * math::sin(Btheta) * math::cos(Bphi); } // electric field components From e023e37ff798d0838d09d343b9fde596e1bd2056 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Tue, 25 Feb 2025 23:59:44 -0500 Subject: [PATCH 115/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 2cb66539f..b77583ccb 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -49,16 +49,16 @@ namespace user { } // electric field components - Inline auto ex1(const coord_t& x_Ph) const -> real_t { + Inline auto ex1(const coord_t&) const -> real_t { return ZERO; } - Inline auto ex2(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto ex2(const coord_t&) const -> real_t { + return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); } - Inline auto ex3(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto ex3(const coord_t&) const -> real_t { + return Vx * Bmag * math::sin(Btheta) * math::sin(Bphi); } private: From 42549067c3744a0c43aef401c5c09f821ed0b344 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Thu, 27 Feb 2025 11:42:46 -0500 Subject: [PATCH 116/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index b77583ccb..f92086528 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -36,29 +36,34 @@ namespace user { , Vx { drift_ux } {} // magnetic field components - Inline auto bx1(const coord_t&) const -> real_t { - return Bmag * math::cos(Btheta); + Inline auto bx1(const coord_t& x_ph) const -> real_t { + // return Bmag * math::cos(Btheta); + return ZERO; } - Inline auto bx2(const coord_t&) const -> real_t { - return Bmag * math::sin(Btheta) * math::sin(Bphi); + Inline auto bx2(const coord_t& x_ph) const -> real_t { + // return Bmag * math::sin(Btheta) * math::sin(Bphi); + return ZERO; } - Inline auto bx3(const coord_t&) const -> real_t { - return Bmag * math::sin(Btheta) * math::cos(Bphi); + Inline auto bx3(const coord_t& x_ph) const -> real_t { + // return Bmag * math::sin(Btheta) * math::cos(Bphi); + return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x))*math::tanh(20.*(-0.5 + x)))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); } // electric field components - Inline auto ex1(const coord_t&) const -> real_t { + Inline auto ex1(const coord_t& x_ph) const -> real_t { return ZERO; } - Inline auto ex2(const coord_t&) const -> real_t { - return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); + Inline auto ex2(const coord_t& x_ph) const -> real_t { + // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); + return ZERO + 0.01 * (ONE - math::tanh(20.*(-1.5 + x))*math::tanh(20.*(-0.5 + x)))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); } - Inline auto ex3(const coord_t&) const -> real_t { - return Vx * Bmag * math::sin(Btheta) * math::sin(Bphi); + Inline auto ex3(const coord_t& x_ph) const -> real_t { + // return Vx * Bmag * math::sin(Btheta) * math::sin(Bphi); + return ZERO; } private: From 20ae9a4f73aef85be0c8ce588d17cb1e562ed8e3 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Thu, 27 Feb 2025 12:01:42 -0500 Subject: [PATCH 117/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index f92086528..c43688d02 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -48,7 +48,7 @@ namespace user { Inline auto bx3(const coord_t& x_ph) const -> real_t { // return Bmag * math::sin(Btheta) * math::cos(Bphi); - return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x))*math::tanh(20.*(-0.5 + x)))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); } // electric field components @@ -58,7 +58,7 @@ namespace user { Inline auto ex2(const coord_t& x_ph) const -> real_t { // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - return ZERO + 0.01 * (ONE - math::tanh(20.*(-1.5 + x))*math::tanh(20.*(-0.5 + x)))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ZERO + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); } Inline auto ex3(const coord_t& x_ph) const -> real_t { From 9facba71628d0b8295b06710ac8d6519c606652d Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Thu, 27 Feb 2025 13:13:12 -0500 Subject: [PATCH 118/176] Ongoing. --- src/kernels/fields_bcs.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 7f9a054a5..77497114c 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -536,7 +536,7 @@ namespace kernel::bc { if constexpr (S == SimEngine::SRPIC) { // SRPIC if (tags & BC::E) { - Fld((N_GHOSTS-1)-i1, i2, em::ex1) = - Fld(N_GHOSTS+i1, i2, em::ex1); + Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+1+i1, i2, em::ex2); Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+1+i1, i2, em::ex3); } @@ -544,8 +544,8 @@ namespace kernel::bc { if (tags & BC::B) { Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+1+i1, i2, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = - Fld(N_GHOSTS+i1, i2, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = - Fld(N_GHOSTS+i1, i2, em::bx3); + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; From 8f0e72c1bacd22ba022645bd8de1052ed6c8af92 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Thu, 27 Feb 2025 13:18:43 -0500 Subject: [PATCH 119/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index c43688d02..10d4c6b59 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -58,7 +58,7 @@ namespace user { Inline auto ex2(const coord_t& x_ph) const -> real_t { // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - return ZERO + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); } Inline auto ex3(const coord_t& x_ph) const -> real_t { From cedf47a613ee4eb2e5edb31ccdbba9d36c54cad5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Thu, 27 Feb 2025 18:04:22 -0600 Subject: [PATCH 120/176] first attempt at single particle injection --- setups/srpic/shocktest/pgen.hpp | 164 ++++++++++++++++++++------------ src/kernels/fields_bcs.hpp | 27 +++--- 2 files changed, 116 insertions(+), 75 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 10d4c6b59..4d4b1cb37 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -111,72 +111,116 @@ namespace user { } } - - auto PerfectConductorFieldsConst(const bc_in&, const em& comp) const -> real_t{ - - // electric field components - if (comp == em::ex1) { - return ONE; - } else if (comp == em::ex2) { - return -ONE; - } else if (comp == em::ex3) { - return -ONE; } - // magentic field components - else if (comp == em::bx1) { - return -ONE; - } else if (comp == em::bx2) { - return ONE; - } else if (comp == em::bx3) { - return ONE;} - // should never be the case - else - { - return ZERO; - } - } - - // auto PerfectConductorCurrentsConst(const bc_in &, const cur &comp) const - // -> std::pair - // { - // // ToDo - // if (comp == cur::jx1) - // { - // return ZERO; - // } - // else if (comp == cur::jx2) - // { - // return ZERO; - // } - // else if (comp == cur::jx3) - // { - // return ZERO; - // } - // else - // { - // return ZERO; - // } - // } auto MatchFields(real_t time) const -> InitFields { return init_flds; } - inline void InitPrtls(Domain& local_domain) { - const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, - local_domain.random_pool, - temperature, - -drift_ux, - in::x1); - - const auto injector = arch::UniformInjector( - energy_dist, - { 1, 2 }); - // arch::InjectUniform>( - // params, - // local_domain, - // injector, - // 1.0); + inline void InitPrtls(Domain& domain) { + + auto& species_e = domain.species[0]; + auto& species_p = domain.species[1]; + auto metric = domain.mesh.metric; + auto m = domain.mesh.metric; + + array_t elec_ind("elec_ind"); + array_t pos_ind("pos_ind"); + + auto offset_e = species_e.npart(); + auto offset_p = species_p.npart(); + + auto ux1_e = species_e.ux1; + auto ux2_e = species_e.ux2; + auto ux3_e = species_e.ux3; + auto i1_e = species_e.i1; + auto i2_e = species_e.i2; + auto dx1_e = species_e.dx1; + auto dx2_e = species_e.dx2; + auto phi_e = species_e.phi; + auto weight_e = species_e.weight; + auto tag_e = species_e.tag; + + auto ux1_p = species_p.ux1; + auto ux2_p = species_p.ux2; + auto ux3_p = species_p.ux3; + auto i1_p = species_p.i1; + auto i2_p = species_p.i2; + auto dx1_p = species_p.dx1; + auto dx2_p = species_p.dx2; + auto phi_p = species_p.phi; + auto weight_p = species_p.weight; + auto tag_p = species_p.tag; + + int nseed = 1; + auto dseed = HALF * constant::PI / static_cast(nseed); + + Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { + + // ToDo: fix this + auto i1_ = ONE; + auto i2_ = ONE; + auto dx1_ = ONE - HALF; + auto dx2_ = ONE - HALF; + + + auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); + auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); + + i1_e(elec_p + offset_e) = i1_; + dx1_e(elec_p + offset_e) = dx1_; + i2_e(elec_p + offset_e) = i2_; + dx2_e(elec_p + offset_e) = dx2_; + phi_e(elec_p + offset_e) = ZERO; + ux1_e(elec_p + offset_e) = -drift_ux; + ux2_e(elec_p + offset_e) = -drift_ux; + ux3_e(elec_p + offset_e) = ZERO; + weight_e(elec_p + offset_e) = ONE; + tag_e(elec_p + offset_e) = ParticleTag::alive; + + i1_p(pos_p + offset_p) = i1_; + dx1_p(pos_p + offset_p) = dx1_; + i2_p(pos_p + offset_p) = i2_; + dx2_p(pos_p + offset_p) = dx2_; + phi_p(pos_p + offset_p) = ZERO; + ux1_p(pos_p + offset_p) = -drift_ux; + ux2_p(pos_p + offset_p) = drift_ux; + ux3_p(pos_p + offset_p) = ZERO; + weight_p(pos_p + offset_p) = ONE; + tag_p(pos_p + offset_p) = ParticleTag::alive; + + + }); + + + auto elec_ind_h = Kokkos::create_mirror(elec_ind); + Kokkos::deep_copy(elec_ind_h, elec_ind); + species_e.set_npart(offset_e + elec_ind_h()); + + auto pos_ind_h = Kokkos::create_mirror(pos_ind); + Kokkos::deep_copy(pos_ind_h, pos_ind); + species_p.set_npart(offset_p + pos_ind_h()); + + } + + + // inline void InitPrtls(Domain& local_domain) { + // const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, + // local_domain.random_pool, + // temperature, + // -drift_ux, + // in::x1); + + // const auto injector = arch::UniformInjector( + // energy_dist, + // { 1, 2 }); + // arch::InjectUniform>( + // params, + // local_domain, + // injector, + // 1.0); + // } + }; } // namespace user diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 77497114c..e3fe4f2ba 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -506,19 +506,19 @@ namespace kernel::bc { if constexpr (S == SimEngine::SRPIC) { - if (tags & BC::E) { - Fld((N_GHOSTS-1)-i1, em::ex1) = Fld(N_GHOSTS+i1, em::ex1); - Fld((N_GHOSTS-1)-i1, em::ex2) = -Fld(N_GHOSTS+i1, em::ex2); - Fld((N_GHOSTS-1)-i1, em::ex3) = -Fld(N_GHOSTS+i1, em::ex3); - } - - if (tags & BC::B) - { - Fld((N_GHOSTS-1)-i1, em::bx1) = -Fld(N_GHOSTS+i1, em::bx1); - Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); - Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); - } + if (tags & BC::E) + { + Fld((N_GHOSTS - 1) - i1, em::ex1) = Fld(N_GHOSTS + i1, em::ex1); + Fld((N_GHOSTS - 1) - i1, em::ex2) = -Fld(N_GHOSTS + 1 + i1, em::ex2); + Fld((N_GHOSTS - 1) - i1, em::ex3) = -Fld(N_GHOSTS + 1 + i1, em::ex3); + } + if (tags & BC::B) + { + Fld((N_GHOSTS - 1) - i1, em::bx1) = -Fld(N_GHOSTS + 1 + i1, em::bx1); + Fld((N_GHOSTS - 1) - i1, em::bx2) = Fld(N_GHOSTS + i1, em::bx2); + Fld((N_GHOSTS - 1) - i1, em::bx3) = Fld(N_GHOSTS + i1, em::bx3); + } } else { // GRPIC raise::KernelError(HERE, "1D GRPIC not implemented"); @@ -546,9 +546,6 @@ namespace kernel::bc { Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+1+i1, i2, em::bx1); Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); - // Fld((N_GHOSTS-1)-i1, i2, em::bx1) = 1.0; - // Fld((N_GHOSTS-1)-i1, i2, em::bx2) = 2.0; - // Fld((N_GHOSTS-1)-i1, i2, em::bx3) = 3.0; } } else { // GRPIC From 77dc646323348d720c9e8348c458aba78808b58e Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 07:44:22 -0500 Subject: [PATCH 121/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 4d4b1cb37..14c2006e9 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -48,7 +48,8 @@ namespace user { Inline auto bx3(const coord_t& x_ph) const -> real_t { // return Bmag * math::sin(Btheta) * math::cos(Bphi); - return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + // return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ZERO; } // electric field components @@ -58,7 +59,8 @@ namespace user { Inline auto ex2(const coord_t& x_ph) const -> real_t { // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + // return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ZERO; } Inline auto ex3(const coord_t& x_ph) const -> real_t { @@ -120,8 +122,6 @@ namespace user { auto& species_e = domain.species[0]; auto& species_p = domain.species[1]; - auto metric = domain.mesh.metric; - auto m = domain.mesh.metric; array_t elec_ind("elec_ind"); array_t pos_ind("pos_ind"); @@ -152,15 +152,14 @@ namespace user { auto tag_p = species_p.tag; int nseed = 1; - auto dseed = HALF * constant::PI / static_cast(nseed); Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { // ToDo: fix this auto i1_ = ONE; auto i2_ = ONE; - auto dx1_ = ONE - HALF; - auto dx2_ = ONE - HALF; + auto dx1_ = HALF; + auto dx2_ = HALF; auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); From afc0fe976718d89ca5e8b7dbbfd630410564ebe1 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 07:47:32 -0500 Subject: [PATCH 122/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 14c2006e9..85d08214a 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -156,8 +156,8 @@ namespace user { Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { // ToDo: fix this - auto i1_ = ONE; - auto i2_ = ONE; + auto i1_ = 100; + auto i2_ = 100; auto dx1_ = HALF; auto dx2_ = HALF; From c6a01f8546df922a91afccd662075e704a2436e8 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 09:42:48 -0500 Subject: [PATCH 123/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 90 ++++++++++++++++----------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 85d08214a..b682cd621 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -153,51 +153,51 @@ namespace user { int nseed = 1; - Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { - - // ToDo: fix this - auto i1_ = 100; - auto i2_ = 100; - auto dx1_ = HALF; - auto dx2_ = HALF; - - - auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); - auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); - - i1_e(elec_p + offset_e) = i1_; - dx1_e(elec_p + offset_e) = dx1_; - i2_e(elec_p + offset_e) = i2_; - dx2_e(elec_p + offset_e) = dx2_; - phi_e(elec_p + offset_e) = ZERO; - ux1_e(elec_p + offset_e) = -drift_ux; - ux2_e(elec_p + offset_e) = -drift_ux; - ux3_e(elec_p + offset_e) = ZERO; - weight_e(elec_p + offset_e) = ONE; - tag_e(elec_p + offset_e) = ParticleTag::alive; - - i1_p(pos_p + offset_p) = i1_; - dx1_p(pos_p + offset_p) = dx1_; - i2_p(pos_p + offset_p) = i2_; - dx2_p(pos_p + offset_p) = dx2_; - phi_p(pos_p + offset_p) = ZERO; - ux1_p(pos_p + offset_p) = -drift_ux; - ux2_p(pos_p + offset_p) = drift_ux; - ux3_p(pos_p + offset_p) = ZERO; - weight_p(pos_p + offset_p) = ONE; - tag_p(pos_p + offset_p) = ParticleTag::alive; - - - }); - - - auto elec_ind_h = Kokkos::create_mirror(elec_ind); - Kokkos::deep_copy(elec_ind_h, elec_ind); - species_e.set_npart(offset_e + elec_ind_h()); - - auto pos_ind_h = Kokkos::create_mirror(pos_ind); - Kokkos::deep_copy(pos_ind_h, pos_ind); - species_p.set_npart(offset_p + pos_ind_h()); + // Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { + + // // ToDo: fix this + // auto i1_ = 100; + // auto i2_ = 100; + // auto dx1_ = HALF; + // auto dx2_ = HALF; + + + // auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); + // auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); + + // i1_e(elec_p + offset_e) = i1_; + // dx1_e(elec_p + offset_e) = dx1_; + // i2_e(elec_p + offset_e) = i2_; + // dx2_e(elec_p + offset_e) = dx2_; + // phi_e(elec_p + offset_e) = ZERO; + // ux1_e(elec_p + offset_e) = -drift_ux; + // ux2_e(elec_p + offset_e) = -drift_ux; + // ux3_e(elec_p + offset_e) = ZERO; + // weight_e(elec_p + offset_e) = ONE; + // tag_e(elec_p + offset_e) = ParticleTag::alive; + + // i1_p(pos_p + offset_p) = i1_; + // dx1_p(pos_p + offset_p) = dx1_; + // i2_p(pos_p + offset_p) = i2_; + // dx2_p(pos_p + offset_p) = dx2_; + // phi_p(pos_p + offset_p) = ZERO; + // ux1_p(pos_p + offset_p) = -drift_ux; + // ux2_p(pos_p + offset_p) = drift_ux; + // ux3_p(pos_p + offset_p) = ZERO; + // weight_p(pos_p + offset_p) = ONE; + // tag_p(pos_p + offset_p) = ParticleTag::alive; + + + // }); + + + // auto elec_ind_h = Kokkos::create_mirror(elec_ind); + // Kokkos::deep_copy(elec_ind_h, elec_ind); + // species_e.set_npart(offset_e + elec_ind_h()); + + // auto pos_ind_h = Kokkos::create_mirror(pos_ind); + // Kokkos::deep_copy(pos_ind_h, pos_ind); + // species_p.set_npart(offset_p + pos_ind_h()); } From 4f3d261f6bb7e40544f97bddc6753059ddeac54b Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 09:51:22 -0500 Subject: [PATCH 124/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 70 ++++++++++++++++----------------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index b682cd621..f74e45080 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -151,53 +151,51 @@ namespace user { auto weight_p = species_p.weight; auto tag_p = species_p.tag; - int nseed = 1; + int nseed = 10; - // Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { + Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { - // // ToDo: fix this - // auto i1_ = 100; - // auto i2_ = 100; - // auto dx1_ = HALF; - // auto dx2_ = HALF; + // ToDo: fix this + auto i1_ = math::floor(100); + auto i2_ = math::floor(64); + auto dx1_ = HALF; + auto dx2_ = HALF; - // auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); - // auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); + auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); + auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); - // i1_e(elec_p + offset_e) = i1_; - // dx1_e(elec_p + offset_e) = dx1_; - // i2_e(elec_p + offset_e) = i2_; - // dx2_e(elec_p + offset_e) = dx2_; - // phi_e(elec_p + offset_e) = ZERO; - // ux1_e(elec_p + offset_e) = -drift_ux; - // ux2_e(elec_p + offset_e) = -drift_ux; - // ux3_e(elec_p + offset_e) = ZERO; - // weight_e(elec_p + offset_e) = ONE; - // tag_e(elec_p + offset_e) = ParticleTag::alive; + i1_e(elec_p + offset_e) = i1_; + dx1_e(elec_p + offset_e) = dx1_; + i2_e(elec_p + offset_e) = i2_; + dx2_e(elec_p + offset_e) = dx2_; + ux1_e(elec_p + offset_e) = ZERO; + ux2_e(elec_p + offset_e) = ZERO; + ux3_e(elec_p + offset_e) = ZERO; + weight_e(elec_p + offset_e) = ONE; + tag_e(elec_p + offset_e) = ParticleTag::alive; - // i1_p(pos_p + offset_p) = i1_; - // dx1_p(pos_p + offset_p) = dx1_; - // i2_p(pos_p + offset_p) = i2_; - // dx2_p(pos_p + offset_p) = dx2_; - // phi_p(pos_p + offset_p) = ZERO; - // ux1_p(pos_p + offset_p) = -drift_ux; - // ux2_p(pos_p + offset_p) = drift_ux; - // ux3_p(pos_p + offset_p) = ZERO; - // weight_p(pos_p + offset_p) = ONE; - // tag_p(pos_p + offset_p) = ParticleTag::alive; + i1_p(pos_p + offset_p) = i1_; + dx1_p(pos_p + offset_p) = dx1_; + i2_p(pos_p + offset_p) = i2_; + dx2_p(pos_p + offset_p) = dx2_; + ux1_p(pos_p + offset_p) = ZERO; + ux2_p(pos_p + offset_p) = ZERO; + ux3_p(pos_p + offset_p) = ZERO; + weight_p(pos_p + offset_p) = ONE; + tag_p(pos_p + offset_p) = ParticleTag::alive; - // }); + }); - // auto elec_ind_h = Kokkos::create_mirror(elec_ind); - // Kokkos::deep_copy(elec_ind_h, elec_ind); - // species_e.set_npart(offset_e + elec_ind_h()); + auto elec_ind_h = Kokkos::create_mirror(elec_ind); + Kokkos::deep_copy(elec_ind_h, elec_ind); + species_e.set_npart(offset_e + elec_ind_h()); - // auto pos_ind_h = Kokkos::create_mirror(pos_ind); - // Kokkos::deep_copy(pos_ind_h, pos_ind); - // species_p.set_npart(offset_p + pos_ind_h()); + auto pos_ind_h = Kokkos::create_mirror(pos_ind); + Kokkos::deep_copy(pos_ind_h, pos_ind); + species_p.set_npart(offset_p + pos_ind_h()); } From bec35fd8c64cbb39a78252410646a566ec9a5118 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 09:56:29 -0500 Subject: [PATCH 125/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index f74e45080..a835b7990 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -151,7 +151,7 @@ namespace user { auto weight_p = species_p.weight; auto tag_p = species_p.tag; - int nseed = 10; + int nseed = 1; Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { @@ -169,8 +169,8 @@ namespace user { dx1_e(elec_p + offset_e) = dx1_; i2_e(elec_p + offset_e) = i2_; dx2_e(elec_p + offset_e) = dx2_; - ux1_e(elec_p + offset_e) = ZERO; - ux2_e(elec_p + offset_e) = ZERO; + ux1_e(elec_p + offset_e) = -drift_ux; + ux2_e(elec_p + offset_e) = drift_ux; ux3_e(elec_p + offset_e) = ZERO; weight_e(elec_p + offset_e) = ONE; tag_e(elec_p + offset_e) = ParticleTag::alive; @@ -179,8 +179,8 @@ namespace user { dx1_p(pos_p + offset_p) = dx1_; i2_p(pos_p + offset_p) = i2_; dx2_p(pos_p + offset_p) = dx2_; - ux1_p(pos_p + offset_p) = ZERO; - ux2_p(pos_p + offset_p) = ZERO; + ux1_p(pos_p + offset_p) = -drift_ux; + ux2_p(pos_p + offset_p) = drift_ux; ux3_p(pos_p + offset_p) = ZERO; weight_p(pos_p + offset_p) = ONE; tag_p(pos_p + offset_p) = ParticleTag::alive; From 400feb03cd2482abae63bf20ba3fe00916ab553b Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 09:58:00 -0500 Subject: [PATCH 126/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index a835b7990..9711646e3 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -169,8 +169,8 @@ namespace user { dx1_e(elec_p + offset_e) = dx1_; i2_e(elec_p + offset_e) = i2_; dx2_e(elec_p + offset_e) = dx2_; - ux1_e(elec_p + offset_e) = -drift_ux; - ux2_e(elec_p + offset_e) = drift_ux; + ux1_e(elec_p + offset_e) = -0.5; + ux2_e(elec_p + offset_e) = 0.5; ux3_e(elec_p + offset_e) = ZERO; weight_e(elec_p + offset_e) = ONE; tag_e(elec_p + offset_e) = ParticleTag::alive; @@ -179,8 +179,8 @@ namespace user { dx1_p(pos_p + offset_p) = dx1_; i2_p(pos_p + offset_p) = i2_; dx2_p(pos_p + offset_p) = dx2_; - ux1_p(pos_p + offset_p) = -drift_ux; - ux2_p(pos_p + offset_p) = drift_ux; + ux1_p(pos_p + offset_p) = -0.5; + ux2_p(pos_p + offset_p) = 0.5; ux3_p(pos_p + offset_p) = ZERO; weight_p(pos_p + offset_p) = ONE; tag_p(pos_p + offset_p) = ParticleTag::alive; From 906d2766a8a21e24b438967151f1f2115f701a7c Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 11:05:00 -0500 Subject: [PATCH 127/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 9711646e3..4cf89ac69 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -156,7 +156,7 @@ namespace user { Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { // ToDo: fix this - auto i1_ = math::floor(100); + auto i1_ = math::floor(10); auto i2_ = math::floor(64); auto dx1_ = HALF; auto dx2_ = HALF; From 8bf2718953707cab4241aa152cc1b1734beb9659 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 11:18:31 -0500 Subject: [PATCH 128/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 4cf89ac69..a1ce9a1a9 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -179,8 +179,8 @@ namespace user { dx1_p(pos_p + offset_p) = dx1_; i2_p(pos_p + offset_p) = i2_; dx2_p(pos_p + offset_p) = dx2_; - ux1_p(pos_p + offset_p) = -0.5; - ux2_p(pos_p + offset_p) = 0.5; + ux1_p(pos_p + offset_p) = 0.5; + ux2_p(pos_p + offset_p) = -0.5; ux3_p(pos_p + offset_p) = ZERO; weight_p(pos_p + offset_p) = ONE; tag_p(pos_p + offset_p) = ParticleTag::alive; From 5534983abaf90b99e7f076ac7ad80815e78cd67f Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 13:20:21 -0500 Subject: [PATCH 129/176] Ongoing. --- src/kernels/fields_bcs.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index e3fe4f2ba..6f157c4aa 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -537,13 +537,13 @@ namespace kernel::bc { // SRPIC if (tags & BC::E) { Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); - Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+1+i1, i2, em::ex2); - Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+1+i1, i2, em::ex3); + Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+i1, i2, em::ex2); + Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+i1, i2, em::ex3); } if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+1+i1, i2, em::bx1); + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); } From 1a60a7aa64c26fd14984788c68268b7ca9ce46d0 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 13:34:59 -0500 Subject: [PATCH 130/176] Ongoing. --- src/kernels/fields_bcs.hpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 6f157c4aa..12dbe45ed 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -536,16 +536,19 @@ namespace kernel::bc { if constexpr (S == SimEngine::SRPIC) { // SRPIC if (tags & BC::E) { - Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); - Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+i1, i2, em::ex2); - Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+i1, i2, em::ex3); + // Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); + // Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+1+i1, i2, em::ex2); + // Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+1+i1, i2, em::ex3); + Fld((N_GHOSTS-1)-i1, i2, em::ex1) = ZERO; + Fld((N_GHOSTS-1)-i1, i2, em::ex2) = ZERO; + Fld((N_GHOSTS-1)-i1, i2, em::ex3) = ZERO; } if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = - Fld(N_GHOSTS+i1, i2, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = Fld(N_GHOSTS+i1, i2, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = Fld(N_GHOSTS+i1, i2, em::bx3); + Fld((N_GHOSTS-1)-i1, i2, em::bx1) = ZERO; + Fld((N_GHOSTS-1)-i1, i2, em::bx2) = ZERO; + Fld((N_GHOSTS-1)-i1, i2, em::bx3) = ZERO; } } else { // GRPIC From d50abcdbef8bcf9949aa5c7564745f21f6e231e6 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 13:36:55 -0500 Subject: [PATCH 131/176] Ongoing. --- src/kernels/fields_bcs.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 12dbe45ed..def4668c6 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -542,6 +542,8 @@ namespace kernel::bc { Fld((N_GHOSTS-1)-i1, i2, em::ex1) = ZERO; Fld((N_GHOSTS-1)-i1, i2, em::ex2) = ZERO; Fld((N_GHOSTS-1)-i1, i2, em::ex3) = ZERO; + Fld((N_GHOSTS), i2, em::ex2) = ZERO; + Fld((N_GHOSTS), i2, em::ex3) = ZERO; } if (tags & BC::B) From fbdfb920b2df113f5c380ad55f31e30bf4d21d93 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Fri, 28 Feb 2025 14:53:48 -0500 Subject: [PATCH 132/176] Ongoing. --- setups/srpic/shocktest/pgen.hpp | 156 ++++++++++++++++---------------- 1 file changed, 78 insertions(+), 78 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index a1ce9a1a9..1b8926c67 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -48,8 +48,8 @@ namespace user { Inline auto bx3(const coord_t& x_ph) const -> real_t { // return Bmag * math::sin(Btheta) * math::cos(Bphi); - // return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); - return ZERO; + return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + // return ZERO; } // electric field components @@ -59,8 +59,8 @@ namespace user { Inline auto ex2(const coord_t& x_ph) const -> real_t { // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - // return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); - return ZERO; + return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + // return ZERO; } Inline auto ex3(const coord_t& x_ph) const -> real_t { @@ -120,82 +120,82 @@ namespace user { inline void InitPrtls(Domain& domain) { - auto& species_e = domain.species[0]; - auto& species_p = domain.species[1]; + // auto& species_e = domain.species[0]; + // auto& species_p = domain.species[1]; - array_t elec_ind("elec_ind"); - array_t pos_ind("pos_ind"); + // array_t elec_ind("elec_ind"); + // array_t pos_ind("pos_ind"); - auto offset_e = species_e.npart(); - auto offset_p = species_p.npart(); - - auto ux1_e = species_e.ux1; - auto ux2_e = species_e.ux2; - auto ux3_e = species_e.ux3; - auto i1_e = species_e.i1; - auto i2_e = species_e.i2; - auto dx1_e = species_e.dx1; - auto dx2_e = species_e.dx2; - auto phi_e = species_e.phi; - auto weight_e = species_e.weight; - auto tag_e = species_e.tag; - - auto ux1_p = species_p.ux1; - auto ux2_p = species_p.ux2; - auto ux3_p = species_p.ux3; - auto i1_p = species_p.i1; - auto i2_p = species_p.i2; - auto dx1_p = species_p.dx1; - auto dx2_p = species_p.dx2; - auto phi_p = species_p.phi; - auto weight_p = species_p.weight; - auto tag_p = species_p.tag; - - int nseed = 1; - - Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { - - // ToDo: fix this - auto i1_ = math::floor(10); - auto i2_ = math::floor(64); - auto dx1_ = HALF; - auto dx2_ = HALF; - - - auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); - auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); - - i1_e(elec_p + offset_e) = i1_; - dx1_e(elec_p + offset_e) = dx1_; - i2_e(elec_p + offset_e) = i2_; - dx2_e(elec_p + offset_e) = dx2_; - ux1_e(elec_p + offset_e) = -0.5; - ux2_e(elec_p + offset_e) = 0.5; - ux3_e(elec_p + offset_e) = ZERO; - weight_e(elec_p + offset_e) = ONE; - tag_e(elec_p + offset_e) = ParticleTag::alive; - - i1_p(pos_p + offset_p) = i1_; - dx1_p(pos_p + offset_p) = dx1_; - i2_p(pos_p + offset_p) = i2_; - dx2_p(pos_p + offset_p) = dx2_; - ux1_p(pos_p + offset_p) = 0.5; - ux2_p(pos_p + offset_p) = -0.5; - ux3_p(pos_p + offset_p) = ZERO; - weight_p(pos_p + offset_p) = ONE; - tag_p(pos_p + offset_p) = ParticleTag::alive; - - - }); - - - auto elec_ind_h = Kokkos::create_mirror(elec_ind); - Kokkos::deep_copy(elec_ind_h, elec_ind); - species_e.set_npart(offset_e + elec_ind_h()); - - auto pos_ind_h = Kokkos::create_mirror(pos_ind); - Kokkos::deep_copy(pos_ind_h, pos_ind); - species_p.set_npart(offset_p + pos_ind_h()); + // auto offset_e = species_e.npart(); + // auto offset_p = species_p.npart(); + + // auto ux1_e = species_e.ux1; + // auto ux2_e = species_e.ux2; + // auto ux3_e = species_e.ux3; + // auto i1_e = species_e.i1; + // auto i2_e = species_e.i2; + // auto dx1_e = species_e.dx1; + // auto dx2_e = species_e.dx2; + // auto phi_e = species_e.phi; + // auto weight_e = species_e.weight; + // auto tag_e = species_e.tag; + + // auto ux1_p = species_p.ux1; + // auto ux2_p = species_p.ux2; + // auto ux3_p = species_p.ux3; + // auto i1_p = species_p.i1; + // auto i2_p = species_p.i2; + // auto dx1_p = species_p.dx1; + // auto dx2_p = species_p.dx2; + // auto phi_p = species_p.phi; + // auto weight_p = species_p.weight; + // auto tag_p = species_p.tag; + + // int nseed = 1; + + // Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { + + // // ToDo: fix this + // auto i1_ = math::floor(10); + // auto i2_ = math::floor(64); + // auto dx1_ = HALF; + // auto dx2_ = HALF; + + + // auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); + // auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); + + // i1_e(elec_p + offset_e) = i1_; + // dx1_e(elec_p + offset_e) = dx1_; + // i2_e(elec_p + offset_e) = i2_; + // dx2_e(elec_p + offset_e) = dx2_; + // ux1_e(elec_p + offset_e) = -0.5; + // ux2_e(elec_p + offset_e) = 0.5; + // ux3_e(elec_p + offset_e) = ZERO; + // weight_e(elec_p + offset_e) = ONE; + // tag_e(elec_p + offset_e) = ParticleTag::alive; + + // i1_p(pos_p + offset_p) = i1_; + // dx1_p(pos_p + offset_p) = dx1_; + // i2_p(pos_p + offset_p) = i2_; + // dx2_p(pos_p + offset_p) = dx2_; + // ux1_p(pos_p + offset_p) = 0.5; + // ux2_p(pos_p + offset_p) = -0.5; + // ux3_p(pos_p + offset_p) = ZERO; + // weight_p(pos_p + offset_p) = ONE; + // tag_p(pos_p + offset_p) = ParticleTag::alive; + + + // }); + + + // auto elec_ind_h = Kokkos::create_mirror(elec_ind); + // Kokkos::deep_copy(elec_ind_h, elec_ind); + // species_e.set_npart(offset_e + elec_ind_h()); + + // auto pos_ind_h = Kokkos::create_mirror(pos_ind); + // Kokkos::deep_copy(pos_ind_h, pos_ind); + // species_p.set_npart(offset_p + pos_ind_h()); } From b7bed425e6ba920ae46cf63fd1cc6a55cc94e894 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 3 Mar 2025 13:11:03 -0600 Subject: [PATCH 133/176] cleanup of conductor BCs --- src/kernels/fields_bcs.hpp | 101 ++++++++++++++++++++----------------- 1 file changed, 56 insertions(+), 45 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index def4668c6..7a499210f 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -506,19 +506,19 @@ namespace kernel::bc { if constexpr (S == SimEngine::SRPIC) { - if (tags & BC::E) - { - Fld((N_GHOSTS - 1) - i1, em::ex1) = Fld(N_GHOSTS + i1, em::ex1); - Fld((N_GHOSTS - 1) - i1, em::ex2) = -Fld(N_GHOSTS + 1 + i1, em::ex2); - Fld((N_GHOSTS - 1) - i1, em::ex3) = -Fld(N_GHOSTS + 1 + i1, em::ex3); - } + if (tags & BC::E) { + Fld((N_GHOSTS-1)-i1, em::ex1) = Fld(N_GHOSTS+i1, em::ex1); + Fld((N_GHOSTS-1)-i1, em::ex2) = -Fld(N_GHOSTS+i1+1, em::ex2); + Fld((N_GHOSTS-1)-i1, em::ex3) = -Fld(N_GHOSTS+i1+1, em::ex3); + } + + if (tags & BC::B) + { + Fld((N_GHOSTS-1)-i1, em::bx1) = -Fld(N_GHOSTS+i1+1, em::bx1); + Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); + Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); + } - if (tags & BC::B) - { - Fld((N_GHOSTS - 1) - i1, em::bx1) = -Fld(N_GHOSTS + 1 + i1, em::bx1); - Fld((N_GHOSTS - 1) - i1, em::bx2) = Fld(N_GHOSTS + i1, em::bx2); - Fld((N_GHOSTS - 1) - i1, em::bx3) = Fld(N_GHOSTS + i1, em::bx3); - } } else { // GRPIC raise::KernelError(HERE, "1D GRPIC not implemented"); @@ -530,64 +530,75 @@ namespace kernel::bc { } } - Inline void operator()(index_t i1, index_t i2) const { - if constexpr (M::Dim == Dim::_2D) { + Inline void operator()(index_t i1, index_t i2) const + { + if constexpr (M::Dim == Dim::_2D) + { - if constexpr (S == SimEngine::SRPIC) { + if constexpr (S == SimEngine::SRPIC) + { // SRPIC - if (tags & BC::E) { - // Fld((N_GHOSTS-1)-i1, i2, em::ex1) = Fld(N_GHOSTS+i1, i2, em::ex1); - // Fld((N_GHOSTS-1)-i1, i2, em::ex2) = - Fld(N_GHOSTS+1+i1, i2, em::ex2); - // Fld((N_GHOSTS-1)-i1, i2, em::ex3) = - Fld(N_GHOSTS+1+i1, i2, em::ex3); - Fld((N_GHOSTS-1)-i1, i2, em::ex1) = ZERO; - Fld((N_GHOSTS-1)-i1, i2, em::ex2) = ZERO; - Fld((N_GHOSTS-1)-i1, i2, em::ex3) = ZERO; - Fld((N_GHOSTS), i2, em::ex2) = ZERO; - Fld((N_GHOSTS), i2, em::ex3) = ZERO; + if (tags & BC::E) + { + Fld((N_GHOSTS - 1) - i1, i2, em::ex1) = Fld(N_GHOSTS + i1, i2, em::ex1); + Fld((N_GHOSTS - 1) - i1, i2, em::ex2) = -Fld(N_GHOSTS + 1 + i1, i2, em::ex2); + Fld((N_GHOSTS - 1) - i1, i2, em::ex3) = -Fld(N_GHOSTS + 1 + i1, i2, em::ex3); } if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, em::bx1) = ZERO; - Fld((N_GHOSTS-1)-i1, i2, em::bx2) = ZERO; - Fld((N_GHOSTS-1)-i1, i2, em::bx3) = ZERO; + Fld((N_GHOSTS - 1) - i1, i2, em::bx1) = -Fld(N_GHOSTS + 1 + i1, i2, em::bx1); + Fld((N_GHOSTS - 1) - i1, i2, em::bx2) = Fld(N_GHOSTS + i1, i2, em::bx2); + Fld((N_GHOSTS - 1) - i1, i2, em::bx3) = Fld(N_GHOSTS + i1, i2, em::bx3); } - } else { + } + else + { // GRPIC raise::KernelError(HERE, "2D GRPIC not implemented"); } - } else { + } + else + { raise::KernelError( - HERE, - "ConductorBoundaries_kernel: 2D implementation called for D != 2"); + HERE, + "ConductorBoundaries_kernel: 2D implementation called for D != 2"); } } - Inline void operator()(index_t i1, index_t i2, index_t i3) const { - if constexpr (M::Dim == Dim::_3D) { + Inline void operator()(index_t i1, index_t i2, index_t i3) const + { + if constexpr (M::Dim == Dim::_3D) + { - if constexpr (S == SimEngine::SRPIC) { + if constexpr (S == SimEngine::SRPIC) + { // SRPIC - if (tags & BC::E) { - Fld((N_GHOSTS-1)-i1, i2, i3, em::ex1) = Fld(N_GHOSTS+i1, i2, i3, em::ex1); - Fld((N_GHOSTS-1)-i1, i2, i3, em::ex2) = -Fld(N_GHOSTS+i1, i2, i3, em::ex2); - Fld((N_GHOSTS-1)-i1, i2, i3, em::ex3) = -Fld(N_GHOSTS+i1, i2, i3, em::ex3); + if (tags & BC::E) + { + Fld((N_GHOSTS - 1) - i1, i2, i3, em::ex1) = Fld(N_GHOSTS + i1, i2, i3, em::ex1); + Fld((N_GHOSTS - 1) - i1, i2, i3, em::ex2) = -Fld(N_GHOSTS + i1 + 1, i2, i3, em::ex2); + Fld((N_GHOSTS - 1) - i1, i2, i3, em::ex3) = -Fld(N_GHOSTS + i1 + 1, i2, i3, em::ex3); } if (tags & BC::B) { - Fld((N_GHOSTS-1)-i1, i2, i3, em::bx1) = -Fld(N_GHOSTS+i1, i2, i3, em::bx1); - Fld((N_GHOSTS-1)-i1, i2, i3, em::bx2) = Fld(N_GHOSTS+i1, i2, i3, em::bx2); - Fld((N_GHOSTS-1)-i1, i2, i3, em::bx3) = Fld(N_GHOSTS+i1, i2, i3, em::bx3); + Fld((N_GHOSTS - 1) - i1, i2, i3, em::bx1) = -Fld(N_GHOSTS + i1 + 1, i2, i3, em::bx1); + Fld((N_GHOSTS - 1) - i1, i2, i3, em::bx2) = Fld(N_GHOSTS + i1, i2, i3, em::bx2); + Fld((N_GHOSTS - 1) - i1, i2, i3, em::bx3) = Fld(N_GHOSTS + i1, i2, i3, em::bx3); } - } else { + } + else + { // GRPIC raise::KernelError(HERE, "3D GRPIC not implemented"); } - } else { + } + else + { raise::KernelError( - HERE, - "ConductorBoundaries_kernel: 3D implementation called for D != 3"); + HERE, + "ConductorBoundaries_kernel: 3D implementation called for D != 3"); } } }; From 624991803f2d9a520fdb0cd3770806d18f4b9dd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 3 Mar 2025 13:11:50 -0600 Subject: [PATCH 134/176] add 3D case and set correct boundary cells to zero --- src/engines/srpic.hpp | 137 ++++++++++++++++++++++++++++-------------- 1 file changed, 91 insertions(+), 46 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index 86b85de32..b1fca46da 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -839,59 +839,104 @@ namespace ntt { } void PerfectConductorFieldsIn(dir::direction_t direction, - domain_t& domain, - BCTags tags) { + domain_t& domain, + BCTags tags) { /** * perfect conductor field boundaries */ - const auto sign = direction.get_sign(); - const auto dim = direction.get_dim(); - raise::ErrorIf(dim != in::x1 and M::CoordType != Coord::Cart, - "Perfect conductor BCs only implemented for x1 in " - "non-cartesian coordinates", - HERE); + if constexpr (M::CoordType != Coord::Cart) { + (void)direction; + (void)domain; + (void)tags; + raise::Error( + "Perfect conductor BCs only applicable to cartesian coordinates", + HERE); + } else { + const auto sign = direction.get_sign(); + const auto dim = direction.get_dim(); + std::vector xi_min, xi_max; - std::vector xi_min, xi_max; + const std::vector all_dirs { in::x1, in::x2, in::x3 }; - const std::vector all_dirs { in::x1, in::x2, in::x3 }; - - for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { - const auto dd = all_dirs[d]; - if (dim == dd) { + for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { + const auto dd = all_dirs[d]; + if (dim == dd) { xi_min.push_back(0); - xi_max.push_back(N_GHOSTS); - } else { - xi_min.push_back(0); - xi_max.push_back(domain.mesh.n_all(dd)); + xi_max.push_back((sign < 0) ? (N_GHOSTS + 1) : N_GHOSTS); + } else { + xi_min.push_back(0); + xi_max.push_back(domain.mesh.n_all(dd)); + } } - } - raise::ErrorIf(xi_min.size() != xi_max.size() or - xi_min.size() != static_cast(M::Dim), - "Invalid range size", - HERE); + raise::ErrorIf(xi_min.size() != xi_max.size() or + xi_min.size() != static_cast(M::Dim), + "Invalid range size", + HERE); - if constexpr (M::Dim == Dim::_1D) - { - Kokkos::parallel_for( - "MatchFields", - CreateRangePolicy( { xi_min[0] } , { xi_max[0] } ), - kernel::bc::ConductorBoundaries_kernel( - domain.fields.em, - tags)); - } + range_t range; + if constexpr (M::Dim == Dim::_1D) { + range = CreateRangePolicy({ xi_min[0] }, { xi_max[0] }); + } else if constexpr (M::Dim == Dim::_2D) { + range = CreateRangePolicy({ xi_min[0], xi_min[1] }, + { xi_max[0], xi_max[1] }); + } else if constexpr (M::Dim == Dim::_3D) { + range = CreateRangePolicy({ xi_min[0], xi_min[1], xi_min[2] }, + { xi_max[0], xi_max[1], xi_max[2] }); + } else { + raise::Error("Invalid dimension", HERE); + } - if constexpr (M::Dim == Dim::_2D) - { - Kokkos::parallel_for( - "MatchFields", - CreateRangePolicy( { xi_min[0], xi_min[1] } , { xi_max[0], xi_max[1] } ), - kernel::bc::ConductorBoundaries_kernel( - domain.fields.em, - tags)); + if (dim == in::x1) { + if (sign > 0) { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } else { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } + } else if (dim == in::x2) { + if (sign > 0) { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } else { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } + } else { + if (sign > 0) { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } else { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + tags)); + } + } } - - } void AtmosphereFieldsIn(dir::direction_t direction, @@ -922,15 +967,15 @@ namespace ntt { return; } const auto intersect_range = domain.mesh.ExtentToRange(box, incl_ghosts); - tuple_t range_min { 0 }; - tuple_t range_max { 0 }; + tuple_t range_min { 0 }; + tuple_t range_max { 0 }; for (unsigned short d { 0 }; d < M::Dim; ++d) { range_min[d] = intersect_range[d].first; range_max[d] = intersect_range[d].second; } - auto atm_fields = m_pgen.AtmFields(time); - ncells_t il_edge; + auto atm_fields = m_pgen.AtmFields(time); + std::size_t il_edge; if (sign > 0) { il_edge = range_min[dd] - N_GHOSTS; } else { From 268f35afd100d05d34603cfa9cdd8d292e8d0380 Mon Sep 17 00:00:00 2001 From: jmahlmann Date: Tue, 4 Mar 2025 13:22:38 -0600 Subject: [PATCH 135/176] Ongoing --- setups/srpic/shocktest/pgen.hpp | 156 ++++++++++++++++---------------- 1 file changed, 78 insertions(+), 78 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 1b8926c67..a1ce9a1a9 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -48,8 +48,8 @@ namespace user { Inline auto bx3(const coord_t& x_ph) const -> real_t { // return Bmag * math::sin(Btheta) * math::cos(Bphi); - return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); - // return ZERO; + // return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ZERO; } // electric field components @@ -59,8 +59,8 @@ namespace user { Inline auto ex2(const coord_t& x_ph) const -> real_t { // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); - // return ZERO; + // return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + return ZERO; } Inline auto ex3(const coord_t& x_ph) const -> real_t { @@ -120,82 +120,82 @@ namespace user { inline void InitPrtls(Domain& domain) { - // auto& species_e = domain.species[0]; - // auto& species_p = domain.species[1]; + auto& species_e = domain.species[0]; + auto& species_p = domain.species[1]; - // array_t elec_ind("elec_ind"); - // array_t pos_ind("pos_ind"); + array_t elec_ind("elec_ind"); + array_t pos_ind("pos_ind"); - // auto offset_e = species_e.npart(); - // auto offset_p = species_p.npart(); - - // auto ux1_e = species_e.ux1; - // auto ux2_e = species_e.ux2; - // auto ux3_e = species_e.ux3; - // auto i1_e = species_e.i1; - // auto i2_e = species_e.i2; - // auto dx1_e = species_e.dx1; - // auto dx2_e = species_e.dx2; - // auto phi_e = species_e.phi; - // auto weight_e = species_e.weight; - // auto tag_e = species_e.tag; - - // auto ux1_p = species_p.ux1; - // auto ux2_p = species_p.ux2; - // auto ux3_p = species_p.ux3; - // auto i1_p = species_p.i1; - // auto i2_p = species_p.i2; - // auto dx1_p = species_p.dx1; - // auto dx2_p = species_p.dx2; - // auto phi_p = species_p.phi; - // auto weight_p = species_p.weight; - // auto tag_p = species_p.tag; - - // int nseed = 1; - - // Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { - - // // ToDo: fix this - // auto i1_ = math::floor(10); - // auto i2_ = math::floor(64); - // auto dx1_ = HALF; - // auto dx2_ = HALF; - - - // auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); - // auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); - - // i1_e(elec_p + offset_e) = i1_; - // dx1_e(elec_p + offset_e) = dx1_; - // i2_e(elec_p + offset_e) = i2_; - // dx2_e(elec_p + offset_e) = dx2_; - // ux1_e(elec_p + offset_e) = -0.5; - // ux2_e(elec_p + offset_e) = 0.5; - // ux3_e(elec_p + offset_e) = ZERO; - // weight_e(elec_p + offset_e) = ONE; - // tag_e(elec_p + offset_e) = ParticleTag::alive; - - // i1_p(pos_p + offset_p) = i1_; - // dx1_p(pos_p + offset_p) = dx1_; - // i2_p(pos_p + offset_p) = i2_; - // dx2_p(pos_p + offset_p) = dx2_; - // ux1_p(pos_p + offset_p) = 0.5; - // ux2_p(pos_p + offset_p) = -0.5; - // ux3_p(pos_p + offset_p) = ZERO; - // weight_p(pos_p + offset_p) = ONE; - // tag_p(pos_p + offset_p) = ParticleTag::alive; - - - // }); - - - // auto elec_ind_h = Kokkos::create_mirror(elec_ind); - // Kokkos::deep_copy(elec_ind_h, elec_ind); - // species_e.set_npart(offset_e + elec_ind_h()); - - // auto pos_ind_h = Kokkos::create_mirror(pos_ind); - // Kokkos::deep_copy(pos_ind_h, pos_ind); - // species_p.set_npart(offset_p + pos_ind_h()); + auto offset_e = species_e.npart(); + auto offset_p = species_p.npart(); + + auto ux1_e = species_e.ux1; + auto ux2_e = species_e.ux2; + auto ux3_e = species_e.ux3; + auto i1_e = species_e.i1; + auto i2_e = species_e.i2; + auto dx1_e = species_e.dx1; + auto dx2_e = species_e.dx2; + auto phi_e = species_e.phi; + auto weight_e = species_e.weight; + auto tag_e = species_e.tag; + + auto ux1_p = species_p.ux1; + auto ux2_p = species_p.ux2; + auto ux3_p = species_p.ux3; + auto i1_p = species_p.i1; + auto i2_p = species_p.i2; + auto dx1_p = species_p.dx1; + auto dx2_p = species_p.dx2; + auto phi_p = species_p.phi; + auto weight_p = species_p.weight; + auto tag_p = species_p.tag; + + int nseed = 1; + + Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { + + // ToDo: fix this + auto i1_ = math::floor(10); + auto i2_ = math::floor(64); + auto dx1_ = HALF; + auto dx2_ = HALF; + + + auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); + auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); + + i1_e(elec_p + offset_e) = i1_; + dx1_e(elec_p + offset_e) = dx1_; + i2_e(elec_p + offset_e) = i2_; + dx2_e(elec_p + offset_e) = dx2_; + ux1_e(elec_p + offset_e) = -0.5; + ux2_e(elec_p + offset_e) = 0.5; + ux3_e(elec_p + offset_e) = ZERO; + weight_e(elec_p + offset_e) = ONE; + tag_e(elec_p + offset_e) = ParticleTag::alive; + + i1_p(pos_p + offset_p) = i1_; + dx1_p(pos_p + offset_p) = dx1_; + i2_p(pos_p + offset_p) = i2_; + dx2_p(pos_p + offset_p) = dx2_; + ux1_p(pos_p + offset_p) = 0.5; + ux2_p(pos_p + offset_p) = -0.5; + ux3_p(pos_p + offset_p) = ZERO; + weight_p(pos_p + offset_p) = ONE; + tag_p(pos_p + offset_p) = ParticleTag::alive; + + + }); + + + auto elec_ind_h = Kokkos::create_mirror(elec_ind); + Kokkos::deep_copy(elec_ind_h, elec_ind); + species_e.set_npart(offset_e + elec_ind_h()); + + auto pos_ind_h = Kokkos::create_mirror(pos_ind); + Kokkos::deep_copy(pos_ind_h, pos_ind); + species_p.set_npart(offset_p + pos_ind_h()); } From c4325e70b9bf4b15004f2b0b56b54dea58badf3e Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 7 Mar 2025 11:30:47 -0500 Subject: [PATCH 136/176] minor reformatting of conductor BC --- input.example.toml | 2 +- setups/srpic/shocktest/pgen.hpp | 149 +++++++++------------------- setups/srpic/shocktest/shock.toml | 26 +++-- src/engines/srpic.hpp | 52 ++++++++++ src/global/enums.h | 19 ++-- src/global/tests/enums.cpp | 5 +- src/kernels/fields_bcs.hpp | 157 ++++++++++++++---------------- 7 files changed, 204 insertions(+), 206 deletions(-) diff --git a/input.example.toml b/input.example.toml index a49067811..c71049b84 100644 --- a/input.example.toml +++ b/input.example.toml @@ -90,7 +90,7 @@ # Boundary conditions for fields: # @required # @type: 1/2/3-size array of string tuples, each of size 1 or 2 - # @valid: "PERIODIC", "MATCH", "FIXED", "ATMOSPHERE", "CUSTOM", "HORIZON" + # @valid: "PERIODIC", "MATCH", "FIXED", "ATMOSPHERE", "CUSTOM", "HORIZON", "CONDUCTOR" # @example: [["CUSTOM", "MATCH"]] (for 2D spherical [[rmin, rmax]]) # @note: When periodic in any of the directions, you should only set one value: [..., ["PERIODIC"], ...] # @note: In spherical, bondaries in theta/phi are set automatically (only specify bc @ [rmin, rmax]): [["ATMOSPHERE", "MATCH"]] diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index a1ce9a1a9..4f6decc76 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -14,6 +14,7 @@ #include "framework/domain/metadomain.h" #include +#include namespace user { using namespace ntt; @@ -48,7 +49,8 @@ namespace user { Inline auto bx3(const coord_t& x_ph) const -> real_t { // return Bmag * math::sin(Btheta) * math::cos(Bphi); - // return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + // return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + // + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); return ZERO; } @@ -59,7 +61,8 @@ namespace user { Inline auto ex2(const coord_t& x_ph) const -> real_t { // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - // return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); + // return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 + // + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); return ZERO; } @@ -88,17 +91,32 @@ namespace user { const real_t drift_ux, temperature; - const real_t Btheta, Bphi, Bmag; - InitFields init_flds; + const std::vector x1arr_e, x2arr_e, ux1arr_e, ux2arr_e, ux3arr_e; + const std::vector x1arr_i, x2arr_i, ux1arr_i, ux2arr_i, ux3arr_i; + + const real_t Btheta, Bphi, Bmag; + InitFields init_flds; + const Metadomain* metadomain; inline PGen(const SimulationParams& p, const Metadomain& m) : arch::ProblemGenerator { p } , drift_ux { p.template get("setup.drift_ux") } , temperature { p.template get("setup.temperature") } - , Bmag { p.template get("setup.Bmag", ZERO) } + , x1arr_e { p.template get>("setup.x_e") } + , x2arr_e { p.template get>("setup.y_e") } + , ux1arr_e { p.template get>("setup.ux_e") } + , ux2arr_e { p.template get>("setup.uy_e") } + , ux3arr_e { p.template get>("setup.uz_e") } + , x1arr_i { p.template get>("setup.x_i") } + , x2arr_i { p.template get>("setup.y_i") } + , ux1arr_i { p.template get>("setup.ux_i") } + , ux2arr_i { p.template get>("setup.uy_i") } + , ux3arr_i { p.template get>("setup.uz_i") } , Btheta { p.template get("setup.Btheta", ZERO) } + , Bmag { p.template get("setup.Bmag", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } - , init_flds { Bmag, Btheta, Bphi, drift_ux } {} + , init_flds { Bmag, Btheta, Bphi, drift_ux } + , metadomain { &m } {} inline PGen() {} @@ -113,111 +131,32 @@ namespace user { } } - auto MatchFields(real_t time) const -> InitFields { return init_flds; } inline void InitPrtls(Domain& domain) { - - auto& species_e = domain.species[0]; - auto& species_p = domain.species[1]; - - array_t elec_ind("elec_ind"); - array_t pos_ind("pos_ind"); - - auto offset_e = species_e.npart(); - auto offset_p = species_p.npart(); - - auto ux1_e = species_e.ux1; - auto ux2_e = species_e.ux2; - auto ux3_e = species_e.ux3; - auto i1_e = species_e.i1; - auto i2_e = species_e.i2; - auto dx1_e = species_e.dx1; - auto dx2_e = species_e.dx2; - auto phi_e = species_e.phi; - auto weight_e = species_e.weight; - auto tag_e = species_e.tag; - - auto ux1_p = species_p.ux1; - auto ux2_p = species_p.ux2; - auto ux3_p = species_p.ux3; - auto i1_p = species_p.i1; - auto i2_p = species_p.i2; - auto dx1_p = species_p.dx1; - auto dx2_p = species_p.dx2; - auto phi_p = species_p.phi; - auto weight_p = species_p.weight; - auto tag_p = species_p.tag; - - int nseed = 1; - - Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { - - // ToDo: fix this - auto i1_ = math::floor(10); - auto i2_ = math::floor(64); - auto dx1_ = HALF; - auto dx2_ = HALF; - - - auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); - auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); - - i1_e(elec_p + offset_e) = i1_; - dx1_e(elec_p + offset_e) = dx1_; - i2_e(elec_p + offset_e) = i2_; - dx2_e(elec_p + offset_e) = dx2_; - ux1_e(elec_p + offset_e) = -0.5; - ux2_e(elec_p + offset_e) = 0.5; - ux3_e(elec_p + offset_e) = ZERO; - weight_e(elec_p + offset_e) = ONE; - tag_e(elec_p + offset_e) = ParticleTag::alive; - - i1_p(pos_p + offset_p) = i1_; - dx1_p(pos_p + offset_p) = dx1_; - i2_p(pos_p + offset_p) = i2_; - dx2_p(pos_p + offset_p) = dx2_; - ux1_p(pos_p + offset_p) = 0.5; - ux2_p(pos_p + offset_p) = -0.5; - ux3_p(pos_p + offset_p) = ZERO; - weight_p(pos_p + offset_p) = ONE; - tag_p(pos_p + offset_p) = ParticleTag::alive; - - - }); - - - auto elec_ind_h = Kokkos::create_mirror(elec_ind); - Kokkos::deep_copy(elec_ind_h, elec_ind); - species_e.set_npart(offset_e + elec_ind_h()); - - auto pos_ind_h = Kokkos::create_mirror(pos_ind); - Kokkos::deep_copy(pos_ind_h, pos_ind); - species_p.set_npart(offset_p + pos_ind_h()); - - + arch::InjectGlobally(*metadomain, + domain, + 1, + { + { "x1", x1arr_e }, + { "x2", x2arr_e }, + { "ux1", ux1arr_e }, + { "ux2", ux1arr_e }, + { "ux3", ux3arr_e } + }); + arch::InjectGlobally(*metadomain, + domain, + 2, + { + { "x1", x1arr_i }, + { "x2", x2arr_i }, + { "ux1", ux1arr_i }, + { "ux2", ux1arr_i }, + { "ux3", ux3arr_i } + }); } - - - // inline void InitPrtls(Domain& local_domain) { - // const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, - // local_domain.random_pool, - // temperature, - // -drift_ux, - // in::x1); - - // const auto injector = arch::UniformInjector( - // energy_dist, - // { 1, 2 }); - // arch::InjectUniform>( - // params, - // local_domain, - // injector, - // 1.0); - // } - }; } // namespace user diff --git a/setups/srpic/shocktest/shock.toml b/setups/srpic/shocktest/shock.toml index a77f5c2e9..6c0c9a3a0 100644 --- a/setups/srpic/shocktest/shock.toml +++ b/setups/srpic/shocktest/shock.toml @@ -11,7 +11,7 @@ metric = "minkowski" [grid.boundaries] - fields = [["CONDUCTOR", "FIXED"], ["PERIODIC"]] + fields = [["CONDUCTOR", "FIXED"], ["PERIODIC"]] particles = [["REFLECT", "ABSORB"], ["PERIODIC"]] [scales] @@ -20,8 +20,8 @@ [algorithms] current_filters = 8 - fieldsolver = "false" - deposit = "false" + fieldsolver = "false" + deposit = "false" [algorithms.timestep] CFL = 0.5 @@ -44,13 +44,23 @@ [setup] drift_ux = 0.1 temperature = 1e-3 - Bmag = 1.0 - Btheta = 0.0 - Bphi = 0.0 + Bmag = 1.0 + Btheta = 0.0 + Bphi = 0.0 + x_e = [0.05] + y_e = [0.0] + ux_e = [-0.01] + uy_e = [0.01] + uz_e = [0.001] + x_i = [0.05] + y_i = [0.0] + ux_i = [0.01] + uy_i = [-0.01] + uz_i = [-0.001] [output] - interval = 1 - format = "hdf5" + interval = 1 + format = "hdf5" [output.fields] quantities = ["N_1", "N_2", "E", "B", "T0i_1", "T0i_2", "J"] diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index b1fca46da..f75989739 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -854,6 +854,19 @@ namespace ntt { } else { const auto sign = direction.get_sign(); const auto dim = direction.get_dim(); +<<<<<<< HEAD +||||||| parent of bd723f39 (minor reformatting of conductor BC) + const auto sign = direction.get_sign(); + const auto dim = direction.get_dim(); + raise::ErrorIf(dim != in::x1 and M::CoordType != Coord::Cart, + "Perfect conductor BCs only implemented for x1 in " + "non-cartesian coordinates", + HERE); +======= + raise::ErrorIf(dim != in::x1, + "Perfect conductor BCs only implemented for x1", + HERE); +>>>>>>> bd723f39 (minor reformatting of conductor BC) std::vector xi_min, xi_max; @@ -863,7 +876,13 @@ namespace ntt { const auto dd = all_dirs[d]; if (dim == dd) { xi_min.push_back(0); +<<<<<<< HEAD xi_max.push_back((sign < 0) ? (N_GHOSTS + 1) : N_GHOSTS); +||||||| parent of bd723f39 (minor reformatting of conductor BC) + xi_max.push_back(N_GHOSTS); +======= + xi_max.push_back(N_GHOSTS + 1); +>>>>>>> bd723f39 (minor reformatting of conductor BC) } else { xi_min.push_back(0); xi_max.push_back(domain.mesh.n_all(dd)); @@ -886,6 +905,7 @@ namespace ntt { } else { raise::Error("Invalid dimension", HERE); } +<<<<<<< HEAD if (dim == in::x1) { if (sign > 0) { @@ -936,6 +956,38 @@ namespace ntt { tags)); } } +||||||| parent of bd723f39 (minor reformatting of conductor BC) +======= + Kokkos::parallel_for( + "MatchFields", + range, + kernel::bc::ConductorBoundaries_kernel(domain.fields.em, + tags)); + + // if constexpr (M::Dim == Dim::_1D) { + // Kokkos::parallel_for( + // "MatchFields", + // CreateRangePolicy({ xi_min[0] }, { xi_max[0] }), + // kernel::bc::ConductorBoundaries_kernel(domain.fields.em, + // tags)); + // } else if constexpr (M::Dim == Dim::_2D) { + // Kokkos::parallel_for( + // "MatchFields", + // CreateRangePolicy({ xi_min[0], xi_min[1] }, + // { xi_max[0], xi_max[1] }), + // kernel::bc::ConductorBoundaries_kernel(domain.fields.em, + // tags)); + // } else if constexpr (M::Dim == Dim::_3D) { + // Kokkos::parallel_for( + // "MatchFields", + // CreateRangePolicy({ xi_min[0], xi_min[1], xi_min[2] }, + // { xi_max[0], xi_max[1], xi_max[2] }), + // kernel::bc::ConductorBoundaries_kernel(domain.fields.em, + // tags)); + // } else { + // raise::Error("Invalid dimension", HERE); + // } +>>>>>>> bd723f39 (minor reformatting of conductor BC) } } diff --git a/src/global/enums.h b/src/global/enums.h index 8407d0b66..d80297d8d 100644 --- a/src/global/enums.h +++ b/src/global/enums.h @@ -9,7 +9,7 @@ * - enum ntt::PrtlBC // periodic, absorb, atmosphere, custom, * reflect, horizon, axis, sync * - enum ntt::FldsBC // periodic, match, fixed, atmosphere, - * custom, horizon, axis, sync + * custom, horizon, axis, conductor, sync * - enum ntt::PrtlPusher // boris, vay, photon, none * - enum ntt::Cooling // synchrotron, none * - enum ntt::FldsID // e, dive, d, divd, b, h, j, @@ -221,17 +221,20 @@ namespace ntt { CUSTOM = 5, HORIZON = 6, AXIS = 7, - SYNC = 8, // <- SYNC means synchronization with other domains - CONDUCTOR = 9 + CONDUCTOR = 8, + SYNC = 9 // <- SYNC means synchronization with other domains }; constexpr FldsBC(uint8_t c) : enums_hidden::BaseEnum { c } {} - static constexpr type variants[] = { PERIODIC, MATCH, FIXED, ATMOSPHERE, - CUSTOM, HORIZON, AXIS, SYNC, CONDUCTOR }; - static constexpr const char* lookup[] = { "periodic", "match", "fixed", - "atmosphere", "custom", "horizon", - "axis", "sync", "conductor"}; + static constexpr type variants[] = { + PERIODIC, MATCH, FIXED, ATMOSPHERE, CUSTOM, + HORIZON, AXIS, CONDUCTOR, SYNC, + }; + static constexpr const char* lookup[] = { + "periodic", "match", "fixed", "atmosphere", "custom", + "horizon", "axis", "conductor", "sync" + }; static constexpr std::size_t total = sizeof(variants) / sizeof(variants[0]); }; diff --git a/src/global/tests/enums.cpp b/src/global/tests/enums.cpp index 673efaf34..ebc074b72 100644 --- a/src/global/tests/enums.cpp +++ b/src/global/tests/enums.cpp @@ -61,8 +61,9 @@ auto main() -> int { enum_str_t all_simulation_engines = { "srpic", "grpic" }; enum_str_t all_particle_bcs = { "periodic", "absorb", "atmosphere", "custom", "reflect", "horizon", "axis", "sync" }; - enum_str_t all_fields_bcs = { "periodic", "match", "fixed", "atmosphere", - "custom", "horizon", "axis", "sync" }; + enum_str_t all_fields_bcs = { "periodic", "match", "fixed", + "atmosphere", "custom", "horizon", + "axis", "conductor", "sync" }; enum_str_t all_particle_pushers = { "boris", "vay", "photon", "none" }; enum_str_t all_coolings = { "synchrotron", "none" }; diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 7a499210f..901f8e02a 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -5,6 +5,7 @@ * - kernel::bc::MatchBoundaries_kernel<> * - kernel::bc::AxisBoundaries_kernel<> * - kernel::bc::EnforcedBoundaries_kernel<> + * - kernel::bc::ConductorBoundaries_kernel<> * @namespaces: * - kernel::bc:: */ @@ -485,43 +486,40 @@ namespace kernel::bc { } }; - template + template struct ConductorBoundaries_kernel { - static_assert(M::is_metric, "M must be a metric class"); - static_assert(static_cast(o) < - static_cast(M::Dim), + static_assert(static_cast(o) < static_cast(D), "Invalid component index"); - static constexpr idx_t i = static_cast(o) + 1u; + // static constexpr idx_t i = static_cast(o) + 1u; - ndfield_t Fld; - const BCTags tags; + ndfield_t Fld; + const BCTags tags; - ConductorBoundaries_kernel(ndfield_t Fld, - BCTags tags) + ConductorBoundaries_kernel(ndfield_t Fld, BCTags tags) : Fld { Fld } , tags { tags } {} Inline void operator()(index_t i1) const { - if constexpr (M::Dim == Dim::_1D) { - - if constexpr (S == SimEngine::SRPIC) { - + if constexpr (D == Dim::_1D) { if (tags & BC::E) { - Fld((N_GHOSTS-1)-i1, em::ex1) = Fld(N_GHOSTS+i1, em::ex1); - Fld((N_GHOSTS-1)-i1, em::ex2) = -Fld(N_GHOSTS+i1+1, em::ex2); - Fld((N_GHOSTS-1)-i1, em::ex3) = -Fld(N_GHOSTS+i1+1, em::ex3); - } - - if (tags & BC::B) - { - Fld((N_GHOSTS-1)-i1, em::bx1) = -Fld(N_GHOSTS+i1+1, em::bx1); - Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); - Fld((N_GHOSTS-1)-i1, em::bx1) = Fld(N_GHOSTS+i1, em::bx1); + if (i1 == 0) { + Fld(N_GHOSTS, em::ex2) = ZERO; + Fld(N_GHOSTS, em::ex3) = ZERO; + } else { + Fld(N_GHOSTS - i1, em::ex1) = Fld(N_GHOSTS + i1 - 1, em::ex1); + Fld(N_GHOSTS - i1, em::ex2) = -Fld(N_GHOSTS + i1, em::ex2); + Fld(N_GHOSTS - i1, em::ex3) = -Fld(N_GHOSTS + i1, em::ex3); + } } - } else { - // GRPIC - raise::KernelError(HERE, "1D GRPIC not implemented"); + if (tags & BC::B) { + if (i1 == 0) { + Fld(N_GHOSTS, em::bx1) = ZERO; + } else { + Fld(N_GHOSTS - i1, em::bx1) = -Fld(N_GHOSTS + i1, em::bx1); + Fld(N_GHOSTS - i1, em::bx2) = Fld(N_GHOSTS + i1 - 1, em::bx2); + Fld(N_GHOSTS - i1, em::bx3) = Fld(N_GHOSTS + i1 - 1, em::bx3); + } } } else { raise::KernelError( @@ -530,75 +528,70 @@ namespace kernel::bc { } } - Inline void operator()(index_t i1, index_t i2) const - { - if constexpr (M::Dim == Dim::_2D) - { - - if constexpr (S == SimEngine::SRPIC) - { - // SRPIC - if (tags & BC::E) - { - Fld((N_GHOSTS - 1) - i1, i2, em::ex1) = Fld(N_GHOSTS + i1, i2, em::ex1); - Fld((N_GHOSTS - 1) - i1, i2, em::ex2) = -Fld(N_GHOSTS + 1 + i1, i2, em::ex2); - Fld((N_GHOSTS - 1) - i1, i2, em::ex3) = -Fld(N_GHOSTS + 1 + i1, i2, em::ex3); + Inline void operator()(index_t i1, index_t i2) const { + if constexpr (D == Dim::_2D) { + if (tags & BC::E) { + if (i1 == 0) { + Fld(N_GHOSTS, i2, em::ex2) = ZERO; + Fld(N_GHOSTS, i2, em::ex3) = ZERO; + } else { + Fld(N_GHOSTS - i1, i2, em::ex1) = Fld(N_GHOSTS + i1 - 1, i2, em::ex1); + Fld(N_GHOSTS - i1, i2, em::ex2) = -Fld(N_GHOSTS + i1, i2, em::ex2); + Fld(N_GHOSTS - i1, i2, em::ex3) = -Fld(N_GHOSTS + i1, i2, em::ex3); } + } - if (tags & BC::B) - { - Fld((N_GHOSTS - 1) - i1, i2, em::bx1) = -Fld(N_GHOSTS + 1 + i1, i2, em::bx1); - Fld((N_GHOSTS - 1) - i1, i2, em::bx2) = Fld(N_GHOSTS + i1, i2, em::bx2); - Fld((N_GHOSTS - 1) - i1, i2, em::bx3) = Fld(N_GHOSTS + i1, i2, em::bx3); + if (tags & BC::B) { + if (i1 == 0) { + Fld(N_GHOSTS, i2, em::bx1) = ZERO; + } else { + Fld(N_GHOSTS - i1, i2, em::bx1) = -Fld(N_GHOSTS + i1, i2, em::bx1); + Fld(N_GHOSTS - i1, i2, em::bx2) = Fld(N_GHOSTS + i1 - 1, i2, em::bx2); + Fld(N_GHOSTS - i1, i2, em::bx3) = Fld(N_GHOSTS + i1 - 1, i2, em::bx3); } } - else - { - // GRPIC - raise::KernelError(HERE, "2D GRPIC not implemented"); - } - } - else - { + } else { raise::KernelError( - HERE, - "ConductorBoundaries_kernel: 2D implementation called for D != 2"); + HERE, + "ConductorBoundaries_kernel: 2D implementation called for D != 2"); } } - Inline void operator()(index_t i1, index_t i2, index_t i3) const - { - if constexpr (M::Dim == Dim::_3D) - { - - if constexpr (S == SimEngine::SRPIC) - { - // SRPIC - if (tags & BC::E) - { - Fld((N_GHOSTS - 1) - i1, i2, i3, em::ex1) = Fld(N_GHOSTS + i1, i2, i3, em::ex1); - Fld((N_GHOSTS - 1) - i1, i2, i3, em::ex2) = -Fld(N_GHOSTS + i1 + 1, i2, i3, em::ex2); - Fld((N_GHOSTS - 1) - i1, i2, i3, em::ex3) = -Fld(N_GHOSTS + i1 + 1, i2, i3, em::ex3); + Inline void operator()(index_t i1, index_t i2, index_t i3) const { + if constexpr (D == Dim::_3D) { + if (tags & BC::E) { + if (i1 == 0) { + Fld(N_GHOSTS, i2, i3, em::ex2) = ZERO; + Fld(N_GHOSTS, i2, i3, em::ex3) = ZERO; + } else { + Fld(N_GHOSTS - i1, i2, i3, em::ex1) = Fld(N_GHOSTS + i1 - 1, + i2, + i3, + em::ex1); + Fld(N_GHOSTS - i1, i2, i3, em::ex2) = -Fld(N_GHOSTS + i1, i2, i3, em::ex2); + Fld(N_GHOSTS - i1, i2, i3, em::ex3) = -Fld(N_GHOSTS + i1, i2, i3, em::ex3); } + } - if (tags & BC::B) - { - Fld((N_GHOSTS - 1) - i1, i2, i3, em::bx1) = -Fld(N_GHOSTS + i1 + 1, i2, i3, em::bx1); - Fld((N_GHOSTS - 1) - i1, i2, i3, em::bx2) = Fld(N_GHOSTS + i1, i2, i3, em::bx2); - Fld((N_GHOSTS - 1) - i1, i2, i3, em::bx3) = Fld(N_GHOSTS + i1, i2, i3, em::bx3); + if (tags & BC::B) { + if (i1 == 0) { + Fld(N_GHOSTS, i2, i3, em::bx1) = ZERO; + } else { + Fld(N_GHOSTS - i1, i2, i3, em::bx1) = -Fld(N_GHOSTS + i1, i2, i3, em::bx1); + Fld(N_GHOSTS - i1, i2, i3, em::bx2) = Fld(N_GHOSTS + i1 - 1, + i2, + i3, + em::bx2); + Fld(N_GHOSTS - i1, i2, i3, em::bx3) = Fld(N_GHOSTS + i1 - 1, + i2, + i3, + em::bx3); } } - else - { - // GRPIC - raise::KernelError(HERE, "3D GRPIC not implemented"); - } - } - else - { + } else { raise::KernelError( - HERE, - "ConductorBoundaries_kernel: 3D implementation called for D != 3"); + HERE, + "ConductorBoundaries_kernel: 3D implementation called for D != 3"); } } }; From 6e66aa27097095bea992672762b20a7a9093f865 Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 7 Mar 2025 12:11:31 -0500 Subject: [PATCH 137/176] filters adapted for conducting BCs --- src/kernels/digital_filter.hpp | 129 ++++++++++++++++++++++----------- 1 file changed, 86 insertions(+), 43 deletions(-) diff --git a/src/kernels/digital_filter.hpp b/src/kernels/digital_filter.hpp index f01eb0fb8..0ec6dcdbc 100644 --- a/src/kernels/digital_filter.hpp +++ b/src/kernels/digital_filter.hpp @@ -17,10 +17,14 @@ #include "utils/error.h" #include "utils/numeric.h" -#define FILTER_IN_I1(ARR, COMP, I, J) \ +#define FILTER2D_IN_I1(ARR, COMP, I, J) \ INV_2*(ARR)((I), (J), (COMP)) + \ INV_4*((ARR)((I) - 1, (J), (COMP)) + (ARR)((I) + 1, (J), (COMP))) +#define FILTER2D_IN_I2(ARR, COMP, I, J) \ + INV_2*(ARR)((I), (J), (COMP)) + \ + INV_4*((ARR)((I), (J) - 1, (COMP)) + (ARR)((I), (J) + 1, (COMP))) + namespace kernel { using namespace ntt; @@ -31,6 +35,10 @@ namespace kernel { bool is_axis_i2min { false }, is_axis_i2max { false }; static constexpr auto i2_min = N_GHOSTS; const ncells_t i2_max; + const bool is_axis_i2min, is_axis_i2max; + const bool is_conductor_i1min; + static constexpr auto i1_min = N_GHOSTS, i2_min = N_GHOSTS; + const std::size_t i2_max; public: DigitalFilter_kernel(ndfield_t& array, @@ -39,20 +47,31 @@ namespace kernel { const boundaries_t& boundaries) : array { array } , buffer { buffer } - , i2_max { (short)D > 1 ? size_[1] + N_GHOSTS : 0 } { - if constexpr ((C != Coord::Cart) && (D != Dim::_1D)) { - raise::ErrorIf(boundaries.size() < 2, "boundaries defined incorrectly", HERE); - is_axis_i2min = (boundaries[1].first == FldsBC::AXIS); - is_axis_i2max = (boundaries[1].second == FldsBC::AXIS); - } - } + , is_axis_i2min { (D == Dim::_2D) and (boundaries[1].first == FldsBC::AXIS) } + , is_axis_i2max { (D == Dim::_2D) and (boundaries[1].second == FldsBC::AXIS) } + , is_conductor_i1min { boundaries[0].first == FldsBC::CONDUCTOR } + , i2_max { (short)D > 1 ? size_[1] + N_GHOSTS : 0 } {} Inline void operator()(index_t i1) const { if constexpr ((D == Dim::_1D) && (C == Coord::Cart)) { + if (is_conductor_i1min and i1 == i1_min) { + array(i1, cur::jx1) = (THREE * INV_4) * buffer(i1, cur::jx1) + + (INV_4)*buffer(i1 + 1, cur::jx1); + } else if (is_conductor_i1min and i1 == i1_min + 1) { + array(i1, cur::jx1) = INV_2 * buffer(i1, cur::jx1) + + INV_4 * (buffer(i1 - 1, cur::jx1) + + buffer(i1 + 1, cur::jx1)); + array(i1, cur::jx2) = (INV_2)*buffer(i1, cur::jx2) + + (INV_4)*buffer(i1 + 1, cur::jx2); + array(i1, cur::jx3) = (INV_2)*buffer(i1, cur::jx3) + + (INV_4)*buffer(i1 + 1, cur::jx3); + } else { #pragma unroll - for (const auto& comp : { cur::jx1, cur::jx2, cur::jx3 }) { - array(i1, comp) = INV_2 * buffer(i1, comp) + - INV_4 * (buffer(i1 - 1, comp) + buffer(i1 + 1, comp)); + for (const auto& comp : { cur::jx1, cur::jx2, cur::jx3 }) { + array(i1, comp) = INV_2 * buffer(i1, comp) + + INV_4 * + (buffer(i1 - 1, comp) + buffer(i1 + 1, comp)); + } } } else { raise::KernelError(HERE, "DigitalFilter_kernel: 1D implementation called for D != 1 or for non-Cartesian metric"); @@ -62,25 +81,49 @@ namespace kernel { Inline void operator()(index_t i1, index_t i2) const { if constexpr (D == Dim::_2D) { if constexpr (C == Coord::Cart) { + if (is_conductor_i1min and i1 == i1_min) { + array(i1, i2, cur::jx1) = + (THREE * INV_4) * (FILTER2D_IN_I2(buffer, cur::jx1, i1, i2)) + + (INV_4) * (FILTER2D_IN_I2(buffer, cur::jx1, i1 + 1, i2)); + } else if (is_conductor_i1min and i1 == i1_min + 1) { + array(i1, + i2, + cur::jx1) = INV_2 * (FILTER2D_IN_I2(buffer, cur::jx1, i1, i2)) + + INV_4 * + ((FILTER2D_IN_I2(buffer, cur::jx1, i1 - 1, i2)) + + (FILTER2D_IN_I2(buffer, cur::jx1, i1 + 1, i2))); + array(i1, + i2, + cur::jx2) = INV_2 * (FILTER2D_IN_I2(buffer, cur::jx2, i1, i2)) + + INV_4 * + (FILTER2D_IN_I2(buffer, cur::jx2, i1 + 1, i2)); + array(i1, + i2, + cur::jx3) = INV_2 * (FILTER2D_IN_I2(buffer, cur::jx3, i1, i2)) + + INV_4 * + (FILTER2D_IN_I2(buffer, cur::jx3, i1 + 1, i2)); + } else { #pragma unroll - for (const auto comp : { cur::jx1, cur::jx2, cur::jx3 }) { - array(i1, i2, comp) = INV_4 * buffer(i1, i2, comp) + - INV_8 * (buffer(i1 - 1, i2, comp) + - buffer(i1 + 1, i2, comp) + - buffer(i1, i2 - 1, comp) + - buffer(i1, i2 + 1, comp)) + - INV_16 * (buffer(i1 - 1, i2 - 1, comp) + - buffer(i1 + 1, i2 + 1, comp) + - buffer(i1 - 1, i2 + 1, comp) + - buffer(i1 + 1, i2 - 1, comp)); + for (const auto comp : { cur::jx1, cur::jx2, cur::jx3 }) { + array(i1, i2, comp) = INV_4 * buffer(i1, i2, comp) + + INV_8 * (buffer(i1 - 1, i2, comp) + + buffer(i1 + 1, i2, comp) + + buffer(i1, i2 - 1, comp) + + buffer(i1, i2 + 1, comp)) + + INV_16 * (buffer(i1 - 1, i2 - 1, comp) + + buffer(i1 + 1, i2 + 1, comp) + + buffer(i1 - 1, i2 + 1, comp) + + buffer(i1 + 1, i2 - 1, comp)); + } } } else { // spherical + // @TODO: get rid of temporary variables real_t cur_00, cur_0p1, cur_0m1; if (is_axis_i2min && (i2 == i2_min)) { /* --------------------------------- r, phi --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx1, i1, i2); - cur_0p1 = FILTER_IN_I1(buffer, cur::jx1, i1, i2 + 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2); + cur_0p1 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2 + 1); // ... filter in theta array(i1, i2, cur::jx1) = INV_2 * cur_00 + INV_2 * cur_0p1; @@ -88,58 +131,58 @@ namespace kernel { /* ---------------------------------- theta --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx2, i1, i2); - cur_0p1 = FILTER_IN_I1(buffer, cur::jx2, i1, i2 + 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2); + cur_0p1 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2 + 1); // ... filter in theta array(i1, i2, cur::jx2) = INV_4 * (cur_00 + cur_0p1); } else if (is_axis_i2min && (i2 == i2_min + 1)) { /* --------------------------------- r, phi --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx1, i1, i2); - cur_0p1 = FILTER_IN_I1(buffer, cur::jx1, i1, i2 + 1); - cur_0m1 = FILTER_IN_I1(buffer, cur::jx1, i1, i2 - 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2); + cur_0p1 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2 + 1); + cur_0m1 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2 - 1); // ... filter in theta array(i1, i2, cur::jx1) = INV_2 * cur_00 + INV_4 * (cur_0p1 + cur_0m1); // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx3, i1, i2); - cur_0p1 = FILTER_IN_I1(buffer, cur::jx3, i1, i2 + 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx3, i1, i2); + cur_0p1 = FILTER2D_IN_I1(buffer, cur::jx3, i1, i2 + 1); // ... filter in theta array(i1, i2, cur::jx3) = INV_2 * cur_00 + INV_4 * cur_0p1; /* ---------------------------------- theta --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx2, i1, i2); - cur_0p1 = FILTER_IN_I1(buffer, cur::jx2, i1, i2 + 1); - cur_0m1 = FILTER_IN_I1(buffer, cur::jx2, i1, i2 - 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2); + cur_0p1 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2 + 1); + cur_0m1 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2 - 1); // ... filter in theta array(i1, i2, cur::jx2) = INV_2 * cur_00 + INV_4 * (cur_0m1 + cur_0p1); } else if (is_axis_i2max && (i2 == i2_max - 1)) { /* --------------------------------- r, phi --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx1, i1, i2); - cur_0p1 = FILTER_IN_I1(buffer, cur::jx1, i1, i2 + 1); - cur_0m1 = FILTER_IN_I1(buffer, cur::jx1, i1, i2 - 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2); + cur_0p1 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2 + 1); + cur_0m1 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2 - 1); // ... filter in theta array(i1, i2, cur::jx1) = INV_2 * cur_00 + INV_4 * (cur_0m1 + cur_0p1); // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx3, i1, i2); - cur_0m1 = FILTER_IN_I1(buffer, cur::jx3, i1, i2 - 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx3, i1, i2); + cur_0m1 = FILTER2D_IN_I1(buffer, cur::jx3, i1, i2 - 1); // ... filter in theta array(i1, i2, cur::jx3) = INV_2 * cur_00 + INV_4 * cur_0m1; /* ---------------------------------- theta --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx2, i1, i2); - cur_0m1 = FILTER_IN_I1(buffer, cur::jx2, i1, i2 - 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2); + cur_0m1 = FILTER2D_IN_I1(buffer, cur::jx2, i1, i2 - 1); // ... filter in theta array(i1, i2, cur::jx2) = INV_4 * (cur_00 + cur_0m1); } else if (is_axis_i2max && (i2 == i2_max)) { /* --------------------------------- r, phi --------------------------------- */ // ... filter in r - cur_00 = FILTER_IN_I1(buffer, cur::jx1, i1, i2); - cur_0m1 = FILTER_IN_I1(buffer, cur::jx1, i1, i2 - 1); + cur_00 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2); + cur_0m1 = FILTER2D_IN_I1(buffer, cur::jx1, i1, i2 - 1); // ... filter in theta array(i1, i2, cur::jx1) = INV_2 * cur_00 + INV_2 * cur_0m1; @@ -210,6 +253,6 @@ namespace kernel { } // namespace kernel -#undef FILTER_IN_I1 +#undef FILTER2D_IN_I1 #endif // DIGITAL_FILTER_HPP From 81d51f99f1c7d1b1b293ddbeda575f03c45e5aeb Mon Sep 17 00:00:00 2001 From: hayk Date: Sat, 8 Mar 2025 16:16:31 -0500 Subject: [PATCH 138/176] filter test fixed --- src/kernels/tests/digital_filter.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/kernels/tests/digital_filter.cpp b/src/kernels/tests/digital_filter.cpp index e0cc352f5..03a63753b 100644 --- a/src/kernels/tests/digital_filter.cpp +++ b/src/kernels/tests/digital_filter.cpp @@ -40,8 +40,13 @@ void testFilter(const std::vector& res, auto boundaries = boundaries_t {}; if constexpr (M::CoordType != Coord::Cart) { boundaries = { - {FldsBC::CUSTOM, FldsBC::CUSTOM}, - { FldsBC::AXIS, FldsBC::AXIS} + { FldsBC::CUSTOM, FldsBC::CUSTOM }, + { FldsBC::AXIS, FldsBC::AXIS } + }; + } else { + boundaries = { + { FldsBC::PERIODIC, FldsBC::PERIODIC }, + { FldsBC::PERIODIC, FldsBC::PERIODIC } }; } @@ -183,4 +188,4 @@ auto main(int argc, char* argv[]) -> int { } Kokkos::finalize(); return 0; -} \ No newline at end of file +} From 8bb8428204469b52823347b3ab84aed2254475b3 Mon Sep 17 00:00:00 2001 From: hayk Date: Sun, 9 Mar 2025 01:59:57 -0500 Subject: [PATCH 139/176] removed extra kokkos flags --- cmake/kokkosConfig.cmake | 35 ----------------- legacy/tests/kernels-sr.cpp | 10 ++--- src/framework/containers/particles.cpp | 2 +- src/framework/domain/comm_mpi.hpp | 12 +++--- src/framework/domain/comm_nompi.hpp | 6 +-- src/global/arch/kokkos_aliases.cpp | 54 +++++++++++++------------- src/global/arch/kokkos_aliases.h | 18 ++++----- src/kernels/tests/prtls_to_phys.cpp | 45 ++++++++++----------- 8 files changed, 75 insertions(+), 107 deletions(-) diff --git a/cmake/kokkosConfig.cmake b/cmake/kokkosConfig.cmake index 63c32622d..8800f21a0 100644 --- a/cmake/kokkosConfig.cmake +++ b/cmake/kokkosConfig.cmake @@ -37,41 +37,6 @@ set(Kokkos_ENABLE_OPENMP ${default_KOKKOS_ENABLE_OPENMP} CACHE BOOL "Enable OpenMP") -# set memory space -if(${Kokkos_ENABLE_CUDA}) - add_compile_definitions(CUDA_ENABLED) - set(ACC_MEM_SPACE Kokkos::CudaSpace) -elseif(${Kokkos_ENABLE_HIP}) - add_compile_definitions(HIP_ENABLED) - set(ACC_MEM_SPACE Kokkos::HIPSpace) -else() - set(ACC_MEM_SPACE Kokkos::HostSpace) -endif() - -set(HOST_MEM_SPACE Kokkos::HostSpace) - -# set execution space -if(${Kokkos_ENABLE_CUDA}) - set(ACC_EXE_SPACE Kokkos::Cuda) -elseif(${Kokkos_ENABLE_HIP}) - set(ACC_EXE_SPACE Kokkos::HIP) -elseif(${Kokkos_ENABLE_OPENMP}) - set(ACC_EXE_SPACE Kokkos::OpenMP) -else() - set(ACC_EXE_SPACE Kokkos::Serial) -endif() - -if(${Kokkos_ENABLE_OPENMP}) - set(HOST_EXE_SPACE Kokkos::OpenMP) -else() - set(HOST_EXE_SPACE Kokkos::Serial) -endif() - -add_compile_options("-D AccelExeSpace=${ACC_EXE_SPACE}") -add_compile_options("-D AccelMemSpace=${ACC_MEM_SPACE}") -add_compile_options("-D HostExeSpace=${HOST_EXE_SPACE}") -add_compile_options("-D HostMemSpace=${HOST_MEM_SPACE}") - if(${BUILD_TESTING} STREQUAL "OFF") set(Kokkos_ENABLE_TESTS OFF diff --git a/legacy/tests/kernels-sr.cpp b/legacy/tests/kernels-sr.cpp index 59ce0646b..d765799e3 100644 --- a/legacy/tests/kernels-sr.cpp +++ b/legacy/tests/kernels-sr.cpp @@ -1,17 +1,17 @@ -#include "wrapper.h" - #include #include #include +#include "wrapper.h" + #include METRIC_HEADER #include PGEN_HEADER -#include "particle_macros.h" - #include "kernels/particle_pusher_sr.hpp" +#include "particle_macros.h" + template void put_value(ntt::array_t& arr, T value, int i) { auto arr_h = Kokkos::create_mirror_view(arr); @@ -221,4 +221,4 @@ auto main(int argc, char* argv[]) -> int { ntt::GlobalFinalize(); return 0; -} \ No newline at end of file +} diff --git a/src/framework/containers/particles.cpp b/src/framework/containers/particles.cpp index 048d57cde..634a34862 100644 --- a/src/framework/containers/particles.cpp +++ b/src/framework/containers/particles.cpp @@ -224,7 +224,7 @@ namespace ntt { Kokkos::Experimental::fill( "TagDeadParticles", - AccelExeSpace(), + Kokkos::DefaultExecutionSpace(), Kokkos::subview(this_tag, std::make_pair(n_alive, n_alive + n_dead)), ParticleTag::dead); diff --git a/src/framework/domain/comm_mpi.hpp b/src/framework/domain/comm_mpi.hpp index 8c6e532de..4f382be4a 100644 --- a/src/framework/domain/comm_mpi.hpp +++ b/src/framework/domain/comm_mpi.hpp @@ -83,7 +83,7 @@ namespace comm { (long int)(send_slice[0].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, comps.first }, { recv_slice[0].second, comps.second }), Lambda(index_t i1, index_t ci) { @@ -96,7 +96,7 @@ namespace comm { (long int)(send_slice[1].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, comps.first }, { recv_slice[0].second, recv_slice[1].second, comps.second }), Lambda(index_t i1, index_t i2, index_t ci) { @@ -111,7 +111,7 @@ namespace comm { (long int)(send_slice[2].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, recv_slice[2].first, @@ -239,7 +239,7 @@ namespace comm { const auto offset_c = comps.first; Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, comps.first }, { recv_slice[0].second, comps.second }), Lambda(index_t i1, index_t ci) { @@ -251,7 +251,7 @@ namespace comm { const auto offset_c = comps.first; Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, comps.first }, { recv_slice[0].second, recv_slice[1].second, comps.second }), Lambda(index_t i1, index_t i2, index_t ci) { @@ -266,7 +266,7 @@ namespace comm { const auto offset_c = comps.first; Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, recv_slice[2].first, diff --git a/src/framework/domain/comm_nompi.hpp b/src/framework/domain/comm_nompi.hpp index 197d336fa..b477ac176 100644 --- a/src/framework/domain/comm_nompi.hpp +++ b/src/framework/domain/comm_nompi.hpp @@ -70,7 +70,7 @@ namespace comm { (long int)(send_slice[0].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, comps.first }, { recv_slice[0].second, comps.second }), Lambda(index_t i1, index_t ci) { @@ -83,7 +83,7 @@ namespace comm { (long int)(send_slice[1].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, comps.first }, { recv_slice[0].second, recv_slice[1].second, comps.second }), Lambda(index_t i1, index_t i2, index_t ci) { @@ -98,7 +98,7 @@ namespace comm { (long int)(send_slice[2].first); Kokkos::parallel_for( "CommunicateField-extract", - Kokkos::MDRangePolicy, AccelExeSpace>( + Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { recv_slice[0].first, recv_slice[1].first, recv_slice[2].first, diff --git a/src/global/arch/kokkos_aliases.cpp b/src/global/arch/kokkos_aliases.cpp index d6622b0e5..25397af8e 100644 --- a/src/global/arch/kokkos_aliases.cpp +++ b/src/global/arch/kokkos_aliases.cpp @@ -9,73 +9,75 @@ auto CreateParticleRangePolicy(npart_t p1, npart_t p2) -> range_t { } template <> -auto CreateRangePolicy( - const tuple_t& i1, - const tuple_t& i2) -> range_t { +auto CreateRangePolicy(const tuple_t& i1, + const tuple_t& i2) + -> range_t { index_t i1min = i1[0]; index_t i1max = i2[0]; - return Kokkos::RangePolicy(i1min, i1max); + return Kokkos::RangePolicy(i1min, i1max); } template <> -auto CreateRangePolicy( - const tuple_t& i1, - const tuple_t& i2) -> range_t { +auto CreateRangePolicy(const tuple_t& i1, + const tuple_t& i2) + -> range_t { index_t i1min = i1[0]; index_t i1max = i2[0]; index_t i2min = i1[1]; index_t i2max = i2[1]; - return Kokkos::MDRangePolicy, AccelExeSpace>({ i1min, i2min }, - { i1max, i2max }); + return Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( + { i1min, i2min }, + { i1max, i2max }); } template <> -auto CreateRangePolicy( - const tuple_t& i1, - const tuple_t& i2) -> range_t { +auto CreateRangePolicy(const tuple_t& i1, + const tuple_t& i2) + -> range_t { index_t i1min = i1[0]; index_t i1max = i2[0]; index_t i2min = i1[1]; index_t i2max = i2[1]; index_t i3min = i1[2]; index_t i3max = i2[2]; - return Kokkos::MDRangePolicy, AccelExeSpace>( + return Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>( { i1min, i2min, i3min }, { i1max, i2max, i3max }); } template <> -auto CreateRangePolicyOnHost( - const tuple_t& i1, - const tuple_t& i2) -> range_h_t { +auto CreateRangePolicyOnHost(const tuple_t& i1, + const tuple_t& i2) + -> range_h_t { index_t i1min = i1[0]; index_t i1max = i2[0]; - return Kokkos::RangePolicy(i1min, i1max); + return Kokkos::RangePolicy(i1min, i1max); } template <> -auto CreateRangePolicyOnHost( - const tuple_t& i1, - const tuple_t& i2) -> range_h_t { +auto CreateRangePolicyOnHost(const tuple_t& i1, + const tuple_t& i2) + -> range_h_t { index_t i1min = i1[0]; index_t i1max = i2[0]; index_t i2min = i1[1]; index_t i2max = i2[1]; - return Kokkos::MDRangePolicy, HostExeSpace>({ i1min, i2min }, - { i1max, i2max }); + return Kokkos::MDRangePolicy, Kokkos::DefaultHostExecutionSpace>( + { i1min, i2min }, + { i1max, i2max }); } template <> -auto CreateRangePolicyOnHost( - const tuple_t& i1, - const tuple_t& i2) -> range_h_t { +auto CreateRangePolicyOnHost(const tuple_t& i1, + const tuple_t& i2) + -> range_h_t { index_t i1min = i1[0]; index_t i1max = i2[0]; index_t i2min = i1[1]; index_t i2max = i2[1]; index_t i3min = i1[2]; index_t i3max = i2[2]; - return Kokkos::MDRangePolicy, HostExeSpace>( + return Kokkos::MDRangePolicy, Kokkos::DefaultHostExecutionSpace>( { i1min, i2min, i3min }, { i1max, i2max, i3max }); } diff --git a/src/global/arch/kokkos_aliases.h b/src/global/arch/kokkos_aliases.h index e1a759bed..adb0b6451 100644 --- a/src/global/arch/kokkos_aliases.h +++ b/src/global/arch/kokkos_aliases.h @@ -34,7 +34,7 @@ namespace math = Kokkos; template -using array_t = Kokkos::View; +using array_t = Kokkos::View; // Array mirror alias of arbitrary type template @@ -174,17 +174,17 @@ namespace kokkos_aliases_hidden { template <> struct range_impl { - using type = Kokkos::RangePolicy; + using type = Kokkos::RangePolicy; }; template <> struct range_impl { - using type = Kokkos::MDRangePolicy, AccelExeSpace>; + using type = Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>; }; template <> struct range_impl { - using type = Kokkos::MDRangePolicy, AccelExeSpace>; + using type = Kokkos::MDRangePolicy, Kokkos::DefaultExecutionSpace>; }; } // namespace kokkos_aliases_hidden @@ -201,17 +201,17 @@ namespace kokkos_aliases_hidden { template <> struct range_h_impl { - using type = Kokkos::RangePolicy; + using type = Kokkos::RangePolicy; }; template <> struct range_h_impl { - using type = Kokkos::MDRangePolicy, HostExeSpace>; + using type = Kokkos::MDRangePolicy, Kokkos::DefaultHostExecutionSpace>; }; template <> struct range_h_impl { - using type = Kokkos::MDRangePolicy, HostExeSpace>; + using type = Kokkos::MDRangePolicy, Kokkos::DefaultHostExecutionSpace>; }; } // namespace kokkos_aliases_hidden @@ -249,8 +249,8 @@ auto CreateRangePolicyOnHost(const tuple_t&, const tuple_t&) -> range_h_t; // Random number pool/generator type alias -using random_number_pool_t = Kokkos::Random_XorShift1024_Pool; -using random_generator_t = typename random_number_pool_t::generator_type; +using random_number_pool_t = Kokkos::Random_XorShift1024_Pool; +using random_generator_t = typename random_number_pool_t::generator_type; // Random number generator functions template diff --git a/src/kernels/tests/prtls_to_phys.cpp b/src/kernels/tests/prtls_to_phys.cpp index 4719fe6a1..0ceb88cd1 100644 --- a/src/kernels/tests/prtls_to_phys.cpp +++ b/src/kernels/tests/prtls_to_phys.cpp @@ -177,28 +177,29 @@ void testPrtl2PhysSR(const std::vector& res, array_t buff_ux3 { "buff_ux3", nprtl / stride }; array_t buff_wei { "buff_wei", nprtl / stride }; - Kokkos::parallel_for("Init", - Kokkos::RangePolicy(0, nprtl / stride), - kernel::PrtlToPhys_kernel(stride, - buff_x1, - buff_x2, - buff_x3, - buff_ux1, - buff_ux2, - buff_ux3, - buff_wei, - i1, - i2, - i3, - dx1, - dx2, - dx3, - ux1, - ux2, - ux3, - phi, - weight, - metric)); + Kokkos::parallel_for( + "Init", + Kokkos::RangePolicy(0, nprtl / stride), + kernel::PrtlToPhys_kernel(stride, + buff_x1, + buff_x2, + buff_x3, + buff_ux1, + buff_ux2, + buff_ux3, + buff_wei, + i1, + i2, + i3, + dx1, + dx2, + dx3, + ux1, + ux2, + ux3, + phi, + weight, + metric)); Kokkos::parallel_for("Check", nprtl / stride, Checker(metric, From b167d373c568f8e66597bac3b97a6d68061f345a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 7 Mar 2025 20:26:49 -0600 Subject: [PATCH 140/176] revert to old pgen for testing --- setups/srpic/shocktest/pgen.hpp | 171 +++++++++++++++++++++++--------- 1 file changed, 125 insertions(+), 46 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index 4f6decc76..ff54923d1 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -38,20 +38,15 @@ namespace user { // magnetic field components Inline auto bx1(const coord_t& x_ph) const -> real_t { - // return Bmag * math::cos(Btheta); - return ZERO; + return Bmag * math::cos(Btheta); } Inline auto bx2(const coord_t& x_ph) const -> real_t { - // return Bmag * math::sin(Btheta) * math::sin(Bphi); - return ZERO; + return Bmag * math::sin(Btheta) * math::sin(Bphi); } Inline auto bx3(const coord_t& x_ph) const -> real_t { - // return Bmag * math::sin(Btheta) * math::cos(Bphi); - // return ONE + 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 - // + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); - return ZERO; + return Bmag * math::sin(Btheta) * math::cos(Bphi); } // electric field components @@ -60,15 +55,11 @@ namespace user { } Inline auto ex2(const coord_t& x_ph) const -> real_t { - // return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); - // return ZERO - 0.01 * (ONE - math::tanh(20.*(-1.5 + x_ph[0]))*math::tanh(20.*(-0.5 - // + x_ph[0])))/2.0 * math::sin(4.0 * constant::PI * x_ph[0]); - return ZERO; + return -Vx * Bmag * math::sin(Btheta) * math::cos(Bphi); } Inline auto ex3(const coord_t& x_ph) const -> real_t { - // return Vx * Bmag * math::sin(Btheta) * math::sin(Bphi); - return ZERO; + return Vx * Bmag * math::sin(Btheta) * math::sin(Bphi); } private: @@ -102,16 +93,16 @@ namespace user { : arch::ProblemGenerator { p } , drift_ux { p.template get("setup.drift_ux") } , temperature { p.template get("setup.temperature") } - , x1arr_e { p.template get>("setup.x_e") } - , x2arr_e { p.template get>("setup.y_e") } - , ux1arr_e { p.template get>("setup.ux_e") } - , ux2arr_e { p.template get>("setup.uy_e") } - , ux3arr_e { p.template get>("setup.uz_e") } - , x1arr_i { p.template get>("setup.x_i") } - , x2arr_i { p.template get>("setup.y_i") } - , ux1arr_i { p.template get>("setup.ux_i") } - , ux2arr_i { p.template get>("setup.uy_i") } - , ux3arr_i { p.template get>("setup.uz_i") } + // , x1arr_e { p.template get>("setup.x_e") } + // , x2arr_e { p.template get>("setup.y_e") } + // , ux1arr_e { p.template get>("setup.ux_e") } + // , ux2arr_e { p.template get>("setup.uy_e") } + // , ux3arr_e { p.template get>("setup.uz_e") } + // , x1arr_i { p.template get>("setup.x_i") } + // , x2arr_i { p.template get>("setup.y_i") } + // , ux1arr_i { p.template get>("setup.ux_i") } + // , ux2arr_i { p.template get>("setup.uy_i") } + // , ux3arr_i { p.template get>("setup.uz_i") } , Btheta { p.template get("setup.Btheta", ZERO) } , Bmag { p.template get("setup.Bmag", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } @@ -135,28 +126,116 @@ namespace user { return init_flds; } - inline void InitPrtls(Domain& domain) { - arch::InjectGlobally(*metadomain, - domain, - 1, - { - { "x1", x1arr_e }, - { "x2", x2arr_e }, - { "ux1", ux1arr_e }, - { "ux2", ux1arr_e }, - { "ux3", ux3arr_e } - }); - arch::InjectGlobally(*metadomain, - domain, - 2, - { - { "x1", x1arr_i }, - { "x2", x2arr_i }, - { "ux1", ux1arr_i }, - { "ux2", ux1arr_i }, - { "ux3", ux3arr_i } - }); - } + // inline void InitPrtls(Domain& domain) { + // arch::InjectGlobally(*metadomain, + // domain, + // 1, + // { + // { "x1", x1arr_e }, + // { "x2", x2arr_e }, + // { "ux1", ux1arr_e }, + // { "ux2", ux1arr_e }, + // { "ux3", ux3arr_e } + // }); + // arch::InjectGlobally(*metadomain, + // domain, + // 2, + // { + // { "x1", x1arr_i }, + // { "x2", x2arr_i }, + // { "ux1", ux1arr_i }, + // { "ux2", ux1arr_i }, + // { "ux3", ux3arr_i } + // }); + // } + + inline void InitPrtls(Domain& domain) { + + // auto& species_e = domain.species[0]; + // auto& species_p = domain.species[1]; + auto& species_e = domain.species[0]; + auto& species_p = domain.species[1]; + + // array_t elec_ind("elec_ind"); + // array_t pos_ind("pos_ind"); + array_t elec_ind("elec_ind"); + array_t pos_ind("pos_ind"); + + auto offset_e = species_e.npart(); + auto offset_p = species_p.npart(); + + auto ux1_e = species_e.ux1; + auto ux2_e = species_e.ux2; + auto ux3_e = species_e.ux3; + auto i1_e = species_e.i1; + auto i2_e = species_e.i2; + auto dx1_e = species_e.dx1; + auto dx2_e = species_e.dx2; + auto phi_e = species_e.phi; + auto weight_e = species_e.weight; + auto tag_e = species_e.tag; + + auto ux1_p = species_p.ux1; + auto ux2_p = species_p.ux2; + auto ux3_p = species_p.ux3; + auto i1_p = species_p.i1; + auto i2_p = species_p.i2; + auto dx1_p = species_p.dx1; + auto dx2_p = species_p.dx2; + auto phi_p = species_p.phi; + auto weight_p = species_p.weight; + auto tag_p = species_p.tag; + + int nseed = 1; + + Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { + + // ToDo: fix this + auto i1_ = math::floor(10); + auto i2_ = math::floor(64); + auto dx1_ = HALF; + auto dx2_ = HALF; + + + auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); + auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); + + i1_e(elec_p + offset_e) = i1_; + dx1_e(elec_p + offset_e) = dx1_; + i2_e(elec_p + offset_e) = i2_; + dx2_e(elec_p + offset_e) = 2*dx2_; + // ux1_e(elec_p + offset_e) = -0.5; + // ux2_e(elec_p + offset_e) = 0.5; + ux1_e(elec_p + offset_e) = -drift_ux; + ux2_e(elec_p + offset_e) = ZERO; + ux3_e(elec_p + offset_e) = ZERO; + weight_e(elec_p + offset_e) = ONE; + tag_e(elec_p + offset_e) = ParticleTag::alive; + + i1_p(pos_p + offset_p) = i1_; + dx1_p(pos_p + offset_p) = dx1_; + i2_p(pos_p + offset_p) = i2_; + dx2_p(pos_p + offset_p) = -2*dx2_; + // ux1_p(pos_p + offset_p) = 0.5; + // ux2_p(pos_p + offset_p) = -0.5; + ux1_p(pos_p + offset_p) = -drift_ux; + ux2_p(pos_p + offset_p) = ZERO; + ux3_p(pos_p + offset_p) = ZERO; + weight_p(pos_p + offset_p) = ONE; + tag_p(pos_p + offset_p) = ParticleTag::alive; + + + }); + + + auto elec_ind_h = Kokkos::create_mirror(elec_ind); + Kokkos::deep_copy(elec_ind_h, elec_ind); + species_e.set_npart(offset_e + elec_ind_h()); + + auto pos_ind_h = Kokkos::create_mirror(pos_ind); + Kokkos::deep_copy(pos_ind_h, pos_ind); + species_p.set_npart(offset_p + pos_ind_h()); + } }; } // namespace user From ec88a4fa8f0e16118cfd08bdcab9dd80546a15f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 7 Mar 2025 21:26:16 -0600 Subject: [PATCH 141/176] enforce B0/E0 at boundary --- src/engines/srpic.hpp | 52 -------------------------------------- src/kernels/fields_bcs.hpp | 33 +++++++++++++++--------- 2 files changed, 21 insertions(+), 64 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index f75989739..b1fca46da 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -854,19 +854,6 @@ namespace ntt { } else { const auto sign = direction.get_sign(); const auto dim = direction.get_dim(); -<<<<<<< HEAD -||||||| parent of bd723f39 (minor reformatting of conductor BC) - const auto sign = direction.get_sign(); - const auto dim = direction.get_dim(); - raise::ErrorIf(dim != in::x1 and M::CoordType != Coord::Cart, - "Perfect conductor BCs only implemented for x1 in " - "non-cartesian coordinates", - HERE); -======= - raise::ErrorIf(dim != in::x1, - "Perfect conductor BCs only implemented for x1", - HERE); ->>>>>>> bd723f39 (minor reformatting of conductor BC) std::vector xi_min, xi_max; @@ -876,13 +863,7 @@ namespace ntt { const auto dd = all_dirs[d]; if (dim == dd) { xi_min.push_back(0); -<<<<<<< HEAD xi_max.push_back((sign < 0) ? (N_GHOSTS + 1) : N_GHOSTS); -||||||| parent of bd723f39 (minor reformatting of conductor BC) - xi_max.push_back(N_GHOSTS); -======= - xi_max.push_back(N_GHOSTS + 1); ->>>>>>> bd723f39 (minor reformatting of conductor BC) } else { xi_min.push_back(0); xi_max.push_back(domain.mesh.n_all(dd)); @@ -905,7 +886,6 @@ namespace ntt { } else { raise::Error("Invalid dimension", HERE); } -<<<<<<< HEAD if (dim == in::x1) { if (sign > 0) { @@ -956,38 +936,6 @@ namespace ntt { tags)); } } -||||||| parent of bd723f39 (minor reformatting of conductor BC) -======= - Kokkos::parallel_for( - "MatchFields", - range, - kernel::bc::ConductorBoundaries_kernel(domain.fields.em, - tags)); - - // if constexpr (M::Dim == Dim::_1D) { - // Kokkos::parallel_for( - // "MatchFields", - // CreateRangePolicy({ xi_min[0] }, { xi_max[0] }), - // kernel::bc::ConductorBoundaries_kernel(domain.fields.em, - // tags)); - // } else if constexpr (M::Dim == Dim::_2D) { - // Kokkos::parallel_for( - // "MatchFields", - // CreateRangePolicy({ xi_min[0], xi_min[1] }, - // { xi_max[0], xi_max[1] }), - // kernel::bc::ConductorBoundaries_kernel(domain.fields.em, - // tags)); - // } else if constexpr (M::Dim == Dim::_3D) { - // Kokkos::parallel_for( - // "MatchFields", - // CreateRangePolicy({ xi_min[0], xi_min[1], xi_min[2] }, - // { xi_max[0], xi_max[1], xi_max[2] }), - // kernel::bc::ConductorBoundaries_kernel(domain.fields.em, - // tags)); - // } else { - // raise::Error("Invalid dimension", HERE); - // } ->>>>>>> bd723f39 (minor reformatting of conductor BC) } } diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 901f8e02a..dd926580d 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -486,25 +486,32 @@ namespace kernel::bc { } }; - template + template struct ConductorBoundaries_kernel { static_assert(static_cast(o) < static_cast(D), "Invalid component index"); - // static constexpr idx_t i = static_cast(o) + 1u; + static constexpr bool defines_ex2 = traits::has_method::value; + static constexpr bool defines_ex3 = traits::has_method::value; + static constexpr bool defines_bx1 = traits::has_method::value; + static_assert(( defines_ex2 or defines_ex3 or defines_bx1), + "none of the components of E or B are specified in PGEN"); ndfield_t Fld; + const I fset; const BCTags tags; - ConductorBoundaries_kernel(ndfield_t Fld, BCTags tags) + ConductorBoundaries_kernel(ndfield_t Fld, const I& fset, BCTags tags) : Fld { Fld } + , fset { fset } , tags { tags } {} Inline void operator()(index_t i1) const { if constexpr (D == Dim::_1D) { + coord_t x_Ph_H { ZERO }; if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, em::ex2) = ZERO; - Fld(N_GHOSTS, em::ex3) = ZERO; + Fld(N_GHOSTS, em::ex2) = fset.ex2(x_Ph_H); + Fld(N_GHOSTS, em::ex3) = fset.ex3(x_Ph_H); } else { Fld(N_GHOSTS - i1, em::ex1) = Fld(N_GHOSTS + i1 - 1, em::ex1); Fld(N_GHOSTS - i1, em::ex2) = -Fld(N_GHOSTS + i1, em::ex2); @@ -514,7 +521,7 @@ namespace kernel::bc { if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, em::bx1) = ZERO; + Fld(N_GHOSTS, em::bx1) = fset.bx1(x_Ph_H); } else { Fld(N_GHOSTS - i1, em::bx1) = -Fld(N_GHOSTS + i1, em::bx1); Fld(N_GHOSTS - i1, em::bx2) = Fld(N_GHOSTS + i1 - 1, em::bx2); @@ -530,10 +537,11 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2) const { if constexpr (D == Dim::_2D) { + coord_t x_Ph_H { ZERO }; if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, i2, em::ex2) = ZERO; - Fld(N_GHOSTS, i2, em::ex3) = ZERO; + Fld(N_GHOSTS, i2, em::ex2) = fset.ex2(x_Ph_H); + Fld(N_GHOSTS, i2, em::ex3) = fset.ex3(x_Ph_H); } else { Fld(N_GHOSTS - i1, i2, em::ex1) = Fld(N_GHOSTS + i1 - 1, i2, em::ex1); Fld(N_GHOSTS - i1, i2, em::ex2) = -Fld(N_GHOSTS + i1, i2, em::ex2); @@ -543,7 +551,7 @@ namespace kernel::bc { if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, i2, em::bx1) = ZERO; + Fld(N_GHOSTS, i2, em::bx1) = fset.bx1(x_Ph_H); } else { Fld(N_GHOSTS - i1, i2, em::bx1) = -Fld(N_GHOSTS + i1, i2, em::bx1); Fld(N_GHOSTS - i1, i2, em::bx2) = Fld(N_GHOSTS + i1 - 1, i2, em::bx2); @@ -559,10 +567,11 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2, index_t i3) const { if constexpr (D == Dim::_3D) { + coord_t x_Ph_H { ZERO }; if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, i2, i3, em::ex2) = ZERO; - Fld(N_GHOSTS, i2, i3, em::ex3) = ZERO; + Fld(N_GHOSTS, i2, i3, em::ex2) = fset.ex2(x_Ph_H); + Fld(N_GHOSTS, i2, i3, em::ex3) = fset.ex3(x_Ph_H); } else { Fld(N_GHOSTS - i1, i2, i3, em::ex1) = Fld(N_GHOSTS + i1 - 1, i2, @@ -575,7 +584,7 @@ namespace kernel::bc { if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, i2, i3, em::bx1) = ZERO; + Fld(N_GHOSTS, i2, i3, em::bx1) = fset.bx1(x_Ph_H); } else { Fld(N_GHOSTS - i1, i2, i3, em::bx1) = -Fld(N_GHOSTS + i1, i2, i3, em::bx1); Fld(N_GHOSTS - i1, i2, i3, em::bx2) = Fld(N_GHOSTS + i1 - 1, From 680b950db0a8bdaf88d75c405296024daf116790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 10 Mar 2025 12:00:38 -0500 Subject: [PATCH 142/176] revert to zero at boundary --- src/kernels/fields_bcs.hpp | 44 ++++++++++++-------------------------- 1 file changed, 14 insertions(+), 30 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index dd926580d..8ea6b72d8 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -486,32 +486,24 @@ namespace kernel::bc { } }; - template + template struct ConductorBoundaries_kernel { static_assert(static_cast(o) < static_cast(D), "Invalid component index"); - static constexpr bool defines_ex2 = traits::has_method::value; - static constexpr bool defines_ex3 = traits::has_method::value; - static constexpr bool defines_bx1 = traits::has_method::value; - static_assert(( defines_ex2 or defines_ex3 or defines_bx1), - "none of the components of E or B are specified in PGEN"); ndfield_t Fld; - const I fset; const BCTags tags; - ConductorBoundaries_kernel(ndfield_t Fld, const I& fset, BCTags tags) + ConductorBoundaries_kernel(ndfield_t Fld, BCTags tags) : Fld { Fld } - , fset { fset } , tags { tags } {} Inline void operator()(index_t i1) const { if constexpr (D == Dim::_1D) { - coord_t x_Ph_H { ZERO }; if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, em::ex2) = fset.ex2(x_Ph_H); - Fld(N_GHOSTS, em::ex3) = fset.ex3(x_Ph_H); + Fld(N_GHOSTS, em::ex2) = ZERO; + Fld(N_GHOSTS, em::ex3) = ZERO; } else { Fld(N_GHOSTS - i1, em::ex1) = Fld(N_GHOSTS + i1 - 1, em::ex1); Fld(N_GHOSTS - i1, em::ex2) = -Fld(N_GHOSTS + i1, em::ex2); @@ -521,7 +513,7 @@ namespace kernel::bc { if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, em::bx1) = fset.bx1(x_Ph_H); + Fld(N_GHOSTS, em::bx1) = ZERO; } else { Fld(N_GHOSTS - i1, em::bx1) = -Fld(N_GHOSTS + i1, em::bx1); Fld(N_GHOSTS - i1, em::bx2) = Fld(N_GHOSTS + i1 - 1, em::bx2); @@ -537,11 +529,10 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2) const { if constexpr (D == Dim::_2D) { - coord_t x_Ph_H { ZERO }; if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, i2, em::ex2) = fset.ex2(x_Ph_H); - Fld(N_GHOSTS, i2, em::ex3) = fset.ex3(x_Ph_H); + Fld(N_GHOSTS, i2, em::ex2) = ZERO; + Fld(N_GHOSTS, i2, em::ex3) = ZERO; } else { Fld(N_GHOSTS - i1, i2, em::ex1) = Fld(N_GHOSTS + i1 - 1, i2, em::ex1); Fld(N_GHOSTS - i1, i2, em::ex2) = -Fld(N_GHOSTS + i1, i2, em::ex2); @@ -551,7 +542,7 @@ namespace kernel::bc { if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, i2, em::bx1) = fset.bx1(x_Ph_H); + Fld(N_GHOSTS, i2, em::bx1) = ZERO; } else { Fld(N_GHOSTS - i1, i2, em::bx1) = -Fld(N_GHOSTS + i1, i2, em::bx1); Fld(N_GHOSTS - i1, i2, em::bx2) = Fld(N_GHOSTS + i1 - 1, i2, em::bx2); @@ -567,16 +558,13 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2, index_t i3) const { if constexpr (D == Dim::_3D) { - coord_t x_Ph_H { ZERO }; if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, i2, i3, em::ex2) = fset.ex2(x_Ph_H); - Fld(N_GHOSTS, i2, i3, em::ex3) = fset.ex3(x_Ph_H); + Fld(N_GHOSTS, i2, i3, em::ex2) = ZERO; + Fld(N_GHOSTS, i2, i3, em::ex3) = ZERO; } else { Fld(N_GHOSTS - i1, i2, i3, em::ex1) = Fld(N_GHOSTS + i1 - 1, - i2, - i3, - em::ex1); + i2, i3, em::ex1); Fld(N_GHOSTS - i1, i2, i3, em::ex2) = -Fld(N_GHOSTS + i1, i2, i3, em::ex2); Fld(N_GHOSTS - i1, i2, i3, em::ex3) = -Fld(N_GHOSTS + i1, i2, i3, em::ex3); } @@ -584,17 +572,13 @@ namespace kernel::bc { if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, i2, i3, em::bx1) = fset.bx1(x_Ph_H); + Fld(N_GHOSTS, i2, i3, em::bx1) = ZERO; } else { Fld(N_GHOSTS - i1, i2, i3, em::bx1) = -Fld(N_GHOSTS + i1, i2, i3, em::bx1); Fld(N_GHOSTS - i1, i2, i3, em::bx2) = Fld(N_GHOSTS + i1 - 1, - i2, - i3, - em::bx2); + i2, i3, em::bx2); Fld(N_GHOSTS - i1, i2, i3, em::bx3) = Fld(N_GHOSTS + i1 - 1, - i2, - i3, - em::bx3); + i2, i3, em::bx3); } } } else { From de24703779b97f928042853abfcbbf211666b08a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Thu, 13 Mar 2025 10:00:56 -0500 Subject: [PATCH 143/176] cleanup of pgen --- setups/srpic/shock/pgen.hpp | 46 ------------------------------------- 1 file changed, 46 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 59c5590c9..b8f169521 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -106,52 +106,6 @@ namespace user { } } - - auto PerfectConductorFieldsConst(const bc_in&, const em& comp) const -> real_t{ - - // electric field components - if (comp == em::ex1) { - return ONE; - } else if (comp == em::ex2) { - return -ONE; - } else if (comp == em::ex3) { - return -ONE; } - // magentic field components - else if (comp == em::bx1) { - return -ONE; - } else if (comp == em::bx2) { - return ONE; - } else if (comp == em::bx3) { - return ONE;} - // should never be the case - else - { - return ZERO; - } - } - - // auto PerfectConductorCurrentsConst(const bc_in &, const cur &comp) const - // -> std::pair - // { - // // ToDo - // if (comp == cur::jx1) - // { - // return ZERO; - // } - // else if (comp == cur::jx2) - // { - // return ZERO; - // } - // else if (comp == cur::jx3) - // { - // return ZERO; - // } - // else - // { - // return ZERO; - // } - // } - auto MatchFields(real_t time) const -> InitFields { return init_flds; } From a41bbe2b726d2d785f0e48ccd82686f1d20aba77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Thu, 13 Mar 2025 11:03:56 -0500 Subject: [PATCH 144/176] update example parameter file --- setups/srpic/shock/shock.toml | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/setups/srpic/shock/shock.toml b/setups/srpic/shock/shock.toml index 4ed3a2b9e..4e03721f7 100644 --- a/setups/srpic/shock/shock.toml +++ b/setups/srpic/shock/shock.toml @@ -1,22 +1,22 @@ [simulation] name = "shock" engine = "srpic" - runtime = 50.0 + runtime = 30000.0 [grid] - resolution = [2048, 128] - extent = [[0.0, 10.0], [-0.3125, 0.3125]] + resolution = [16384, 128] + extent = [[0.0, 8000.0], [-31.25, 31.25]] [grid.metric] metric = "minkowski" [grid.boundaries] - fields = [["ABSORB", "FIXED"], ["PERIODIC"]] + fields = [["CONDUCTOR", "MATCH"], ["PERIODIC"]] particles = [["REFLECT", "ABSORB"], ["PERIODIC"]] - + [scales] - larmor0 = 1e-2 - skindepth0 = 1e-2 + larmor0 = 100.0 + skindepth0 = 1.0 [algorithms] current_filters = 8 @@ -25,7 +25,7 @@ CFL = 0.5 [particles] - ppc0 = 16.0 + ppc0 = 8.0 [[particles.species]] label = "e-" @@ -34,21 +34,29 @@ maxnpart = 1e8 [[particles.species]] - label = "e+" + label = "p+" mass = 1.0 charge = 1.0 maxnpart = 1e8 [setup] drift_ux = 0.1 - temperature = 1e-3 + temperature = 1e-4 Bmag = 1.0 Btheta = 0.0 Bphi = 0.0 [output] - interval_time = 0.1 + interval_time = 500.0 format = "hdf5" - + [output.fields] - quantities = ["N_1", "N_2", "E", "B", "T0i_1", "T0i_2", "J"] + quantities = ["N_1", "N_2", "B", "E"] + + [output.particles] + enable = true + stride = 10 + + [output.spectra] + enable = false + From 5d9fe735066c5a0e3cb33d2b67c215475f1a02fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Wed, 19 Mar 2025 15:40:05 -0500 Subject: [PATCH 145/176] revert to cleaner injector for shocktest --- setups/srpic/shocktest/pgen.hpp | 152 +++++++------------------------- 1 file changed, 32 insertions(+), 120 deletions(-) diff --git a/setups/srpic/shocktest/pgen.hpp b/setups/srpic/shocktest/pgen.hpp index ff54923d1..a7bcf5c3d 100644 --- a/setups/srpic/shocktest/pgen.hpp +++ b/setups/srpic/shocktest/pgen.hpp @@ -93,16 +93,16 @@ namespace user { : arch::ProblemGenerator { p } , drift_ux { p.template get("setup.drift_ux") } , temperature { p.template get("setup.temperature") } - // , x1arr_e { p.template get>("setup.x_e") } - // , x2arr_e { p.template get>("setup.y_e") } - // , ux1arr_e { p.template get>("setup.ux_e") } - // , ux2arr_e { p.template get>("setup.uy_e") } - // , ux3arr_e { p.template get>("setup.uz_e") } - // , x1arr_i { p.template get>("setup.x_i") } - // , x2arr_i { p.template get>("setup.y_i") } - // , ux1arr_i { p.template get>("setup.ux_i") } - // , ux2arr_i { p.template get>("setup.uy_i") } - // , ux3arr_i { p.template get>("setup.uz_i") } + , x1arr_e { p.template get>("setup.x_e") } + , x2arr_e { p.template get>("setup.y_e") } + , ux1arr_e { p.template get>("setup.ux_e") } + , ux2arr_e { p.template get>("setup.uy_e") } + , ux3arr_e { p.template get>("setup.uz_e") } + , x1arr_i { p.template get>("setup.x_i") } + , x2arr_i { p.template get>("setup.y_i") } + , ux1arr_i { p.template get>("setup.ux_i") } + , ux2arr_i { p.template get>("setup.uy_i") } + , ux3arr_i { p.template get>("setup.uz_i") } , Btheta { p.template get("setup.Btheta", ZERO) } , Bmag { p.template get("setup.Bmag", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } @@ -126,116 +126,28 @@ namespace user { return init_flds; } - // inline void InitPrtls(Domain& domain) { - // arch::InjectGlobally(*metadomain, - // domain, - // 1, - // { - // { "x1", x1arr_e }, - // { "x2", x2arr_e }, - // { "ux1", ux1arr_e }, - // { "ux2", ux1arr_e }, - // { "ux3", ux3arr_e } - // }); - // arch::InjectGlobally(*metadomain, - // domain, - // 2, - // { - // { "x1", x1arr_i }, - // { "x2", x2arr_i }, - // { "ux1", ux1arr_i }, - // { "ux2", ux1arr_i }, - // { "ux3", ux3arr_i } - // }); - // } - - inline void InitPrtls(Domain& domain) { - - // auto& species_e = domain.species[0]; - // auto& species_p = domain.species[1]; - auto& species_e = domain.species[0]; - auto& species_p = domain.species[1]; - - // array_t elec_ind("elec_ind"); - // array_t pos_ind("pos_ind"); - array_t elec_ind("elec_ind"); - array_t pos_ind("pos_ind"); - - auto offset_e = species_e.npart(); - auto offset_p = species_p.npart(); - - auto ux1_e = species_e.ux1; - auto ux2_e = species_e.ux2; - auto ux3_e = species_e.ux3; - auto i1_e = species_e.i1; - auto i2_e = species_e.i2; - auto dx1_e = species_e.dx1; - auto dx2_e = species_e.dx2; - auto phi_e = species_e.phi; - auto weight_e = species_e.weight; - auto tag_e = species_e.tag; - - auto ux1_p = species_p.ux1; - auto ux2_p = species_p.ux2; - auto ux3_p = species_p.ux3; - auto i1_p = species_p.i1; - auto i2_p = species_p.i2; - auto dx1_p = species_p.dx1; - auto dx2_p = species_p.dx2; - auto phi_p = species_p.phi; - auto weight_p = species_p.weight; - auto tag_p = species_p.tag; - - int nseed = 1; - - Kokkos::parallel_for("init_particles", nseed, KOKKOS_LAMBDA(const int& s) { - - // ToDo: fix this - auto i1_ = math::floor(10); - auto i2_ = math::floor(64); - auto dx1_ = HALF; - auto dx2_ = HALF; - - - auto elec_p = Kokkos::atomic_fetch_add(&elec_ind(), 1); - auto pos_p = Kokkos::atomic_fetch_add(&pos_ind(), 1); - - i1_e(elec_p + offset_e) = i1_; - dx1_e(elec_p + offset_e) = dx1_; - i2_e(elec_p + offset_e) = i2_; - dx2_e(elec_p + offset_e) = 2*dx2_; - // ux1_e(elec_p + offset_e) = -0.5; - // ux2_e(elec_p + offset_e) = 0.5; - ux1_e(elec_p + offset_e) = -drift_ux; - ux2_e(elec_p + offset_e) = ZERO; - ux3_e(elec_p + offset_e) = ZERO; - weight_e(elec_p + offset_e) = ONE; - tag_e(elec_p + offset_e) = ParticleTag::alive; - - i1_p(pos_p + offset_p) = i1_; - dx1_p(pos_p + offset_p) = dx1_; - i2_p(pos_p + offset_p) = i2_; - dx2_p(pos_p + offset_p) = -2*dx2_; - // ux1_p(pos_p + offset_p) = 0.5; - // ux2_p(pos_p + offset_p) = -0.5; - ux1_p(pos_p + offset_p) = -drift_ux; - ux2_p(pos_p + offset_p) = ZERO; - ux3_p(pos_p + offset_p) = ZERO; - weight_p(pos_p + offset_p) = ONE; - tag_p(pos_p + offset_p) = ParticleTag::alive; - - - }); - - - auto elec_ind_h = Kokkos::create_mirror(elec_ind); - Kokkos::deep_copy(elec_ind_h, elec_ind); - species_e.set_npart(offset_e + elec_ind_h()); - - auto pos_ind_h = Kokkos::create_mirror(pos_ind); - Kokkos::deep_copy(pos_ind_h, pos_ind); - species_p.set_npart(offset_p + pos_ind_h()); - } + inline void InitPrtls(Domain& domain) { + arch::InjectGlobally(*metadomain, + domain, + 1, + { + { "x1", x1arr_e }, + { "x2", x2arr_e }, + { "ux1", ux1arr_e }, + { "ux2", ux1arr_e }, + { "ux3", ux3arr_e } + }); + arch::InjectGlobally(*metadomain, + domain, + 2, + { + { "x1", x1arr_i }, + { "x2", x2arr_i }, + { "ux1", ux1arr_i }, + { "ux2", ux1arr_i }, + { "ux3", ux3arr_i } + }); + } }; } // namespace user From 6d43018085d3923ab7aae5d25b9917d2e690321f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Thu, 20 Mar 2025 17:06:25 -0500 Subject: [PATCH 146/176] implemented new `arch::Piston` spatial distribution to set up shock partially filled with plasma --- setups/srpic/shock/pgen.hpp | 59 ++++++++++++++++++++++------------- setups/srpic/shock/shock.toml | 11 ++++--- src/archetypes/spatial_dist.h | 30 ++++++++++++++++++ 3 files changed, 74 insertions(+), 26 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index b8f169521..9bf51b866 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -79,7 +79,7 @@ namespace user { using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; - const real_t drift_ux, temperature; + const real_t drift_ux, temperature, filling_fraction; const real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -91,40 +91,57 @@ namespace user { , Bmag { p.template get("setup.Bmag", ZERO) } , Btheta { p.template get("setup.Btheta", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } - , init_flds { Bmag, Btheta, Bphi, drift_ux } {} + , init_flds { Bmag, Btheta, Bphi, drift_ux } + , filling_fraction { params.template get("setup.filling_fraction", 1.0) }{} inline PGen() {} - auto FixFieldsConst(const bc_in&, const em& comp) const - -> std::pair { - if (comp == em::ex2) { - return { init_flds.ex2({ ZERO }), true }; - } else if (comp == em::ex3) { - return { init_flds.ex3({ ZERO }), true }; - } else { - return { ZERO, false }; - } - } - auto MatchFields(real_t time) const -> InitFields { return init_flds; } inline void InitPrtls(Domain& local_domain) { + + // minimum and maximum position of particles + real_t xg_min = local_domain.mesh.extent(in::x1).first; + real_t xg_max = local_domain.mesh.extent(in::x1).second * filling_fraction; + + // define box to inject into + boundaries_t box; + // loop over all dimensions + for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { + // compute the range for the x-direction + if (d == static_cast(in::x1)) { + box.push_back({xg_min, xg_max}); + } else { + // inject into full range in other directions + box.push_back(Range::All); + } + } + + // spatial distribution of the particles + // -> hack to use the uniform distribution in NonUniformInjector + const auto spatial_dist = arch::Piston(local_domain.mesh.metric, xg_min, xg_max, in::x1); + + // energy distribution of the particles const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, local_domain.random_pool, temperature, -drift_ux, in::x1); - const auto injector = arch::UniformInjector( - energy_dist, - { 1, 2 }); - arch::InjectUniform>( - params, - local_domain, - injector, - 1.0); + const auto injector = arch::NonUniformInjector( + energy_dist, + spatial_dist, + {1, 2}); + + arch::InjectNonUniform>( + params, + local_domain, + injector, + 1.0, // target density + false, // no weights + box); } }; diff --git a/setups/srpic/shock/shock.toml b/setups/srpic/shock/shock.toml index 4e03721f7..f954345c2 100644 --- a/setups/srpic/shock/shock.toml +++ b/setups/srpic/shock/shock.toml @@ -40,11 +40,12 @@ maxnpart = 1e8 [setup] - drift_ux = 0.1 - temperature = 1e-4 - Bmag = 1.0 - Btheta = 0.0 - Bphi = 0.0 + drift_ux = 0.1 # speed towards the wall [c] + temperature = 1e-4 # temeperature of maxwell distribution [m_e c^2] + Bmag = 1.0 # magnetic field strength as fraction of magnetisation + Btheta = 0.0 # magnetic field angle in the plane + Bphi = 0.0 # magnetic field angle out of plane + filling_fraction = 1.0 # fraction of the shock piston filled with plasma [output] interval_time = 500.0 diff --git a/src/archetypes/spatial_dist.h b/src/archetypes/spatial_dist.h index d036c0166..2407d98cd 100644 --- a/src/archetypes/spatial_dist.h +++ b/src/archetypes/spatial_dist.h @@ -4,6 +4,7 @@ * @implements * - arch::SpatialDistribution<> * - arch::Uniform<> : arch::SpatialDistribution<> + * - arch::Piston<> : arch::SpatialDistribution<> * - arch::Replenish<> : arch::SpatialDistribution<> * @namespace * - arch:: @@ -49,6 +50,35 @@ namespace arch { } }; + template + struct Piston : public arch::SpatialDistribution + { + Piston(const M &metric, real_t xmin, real_t xmax, in piston_direction = in::x1) + : arch::SpatialDistribution{metric} + , xmin {xmin} + , xmax {xmax} + , piston_direction {piston_direction} {} + + Inline auto operator()(const coord_t &x_Ph) const -> real_t override + { + // dimentsion to fill + const auto fill_dim = static_cast(piston_direction); + + if (x_Ph[fill_dim] < xmin || x_Ph[fill_dim] > xmax) + { + return 0.0; + } + else + { + return 1.0; + } + } + + private: + real_t xmin, xmax; + in piston_direction; + }; + template struct Replenish : public SpatialDistribution { using SpatialDistribution::metric; From 4f5b1d87e3b6f5321e3c51e53bfc1de6bd45dc6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Thu, 20 Mar 2025 17:43:25 -0500 Subject: [PATCH 147/176] fix inconsistency in return type --- src/archetypes/spatial_dist.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/archetypes/spatial_dist.h b/src/archetypes/spatial_dist.h index 2407d98cd..2edbf5b1d 100644 --- a/src/archetypes/spatial_dist.h +++ b/src/archetypes/spatial_dist.h @@ -66,11 +66,11 @@ namespace arch { if (x_Ph[fill_dim] < xmin || x_Ph[fill_dim] > xmax) { - return 0.0; + return ZERO; } else { - return 1.0; + return ONE; } } From 3e9c4acc7236a4c912e507708657e491f34c25fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Mar 2025 16:24:35 -0500 Subject: [PATCH 148/176] Added `MovingInjector` (wip) --- src/archetypes/particle_injector.h | 69 ++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/src/archetypes/particle_injector.h b/src/archetypes/particle_injector.h index 3884e6ced..daa41506c 100644 --- a/src/archetypes/particle_injector.h +++ b/src/archetypes/particle_injector.h @@ -170,6 +170,75 @@ namespace arch { ~AtmosphereInjector() = default; }; + template + struct MovingInjector + { + struct TargetDensityProfile + { + const real_t nmax, xinj, xdrift; + + TargetDensityProfile(real_t xinj, real_t xdrift, real_t nmax) + : xinj{xinj}, xdrift{xdrift}, nmax{nmax} {} + + Inline auto operator()(const coord_t &x_Ph) const -> real_t + { + if constexpr ((O == in::x1) or + (O == in::x2 and (M::Dim == Dim::_2D or M::Dim == Dim::_3D)) or + (O == in::x3 and M::Dim == Dim::_3D)) + { + const auto xi = x_Ph[static_cast(O)]; + // + direction + if (xi < xinj - xdrift or xi >= xinj) + { + return ZERO; + } + else + { + if constexpr (M::CoordType == Coord::Cart) + { + return nmax; + } + else + { + raise::KernelError( + HERE, + "Moving injector in +x cannot be applied for non-cartesian"); + return ZERO; + } + } + } + else + { + raise::KernelError(HERE, "Wrong direction"); + return ZERO; + } + } + }; + using energy_dist_t = Maxwellian; + using spatial_dist_t = Replenish; + static_assert(M::is_metric, "M must be a metric class"); + static constexpr bool is_nonuniform_injector{true}; + static constexpr Dimension D{M::Dim}; + static constexpr Coord C{M::CoordType}; + + const energy_dist_t energy_dist; + const TargetDensityProfile target_density; + const spatial_dist_t spatial_dist; + const std::pair species; + + MovingInjector(const M &metric, + const ndfield_t &density, + const energy_dist_t &energy_dist, + real_t xinj, + real_t xdrift, + real_t nmax, + const std::pair &species) + : energy_dist{energy_dist}, + target_density{xinj, xdrift, nmax}, spatial_dist{metric, density, 0, target_density, nmax}, species{species} {} + + ~MovingInjector() = default; + }; + /** * @brief Injects uniform number density of particles everywhere in the domain * @param domain Domain object From e724b506f2acef8869c1a92bb476ff4c7065ac1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Fri, 21 Mar 2025 16:25:10 -0500 Subject: [PATCH 149/176] Added `CustomPostStep` for shock pgen to continually inject particles (wip) --- setups/srpic/shock/pgen.hpp | 78 ++++++++++++++++++++++++++++++++++- setups/srpic/shock/shock.toml | 4 +- src/engines/engine_run.cpp | 2 +- 3 files changed, 79 insertions(+), 5 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 9bf51b866..7c0d750e3 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -79,7 +79,7 @@ namespace user { using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; - const real_t drift_ux, temperature, filling_fraction; + const real_t drift_ux, temperature, filling_fraction, injection_rate; const real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -92,7 +92,8 @@ namespace user { , Btheta { p.template get("setup.Btheta", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } , init_flds { Bmag, Btheta, Bphi, drift_ux } - , filling_fraction { params.template get("setup.filling_fraction", 1.0) }{} + , filling_fraction { params.template get("setup.filling_fraction", 1.0) } + , injection_rate { params.template get("setup.injection_rate", 1.0) } {} inline PGen() {} @@ -100,6 +101,11 @@ namespace user { return init_flds; } + // Inline auto TargetDensity(const coord_t &x_Ph) const -> real_t + // { + // return ONE; + // } + inline void InitPrtls(Domain& local_domain) { // minimum and maximum position of particles @@ -143,6 +149,74 @@ namespace user { false, // no weights box); } + + void CustomPostStep(std::size_t, long double time, real_t dt, Domain &domain) + { + /* + Moving injector for the particles + */ + + // energy distribution of the particles + const auto energy_dist = arch::Maxwellian(domain.mesh.metric, + domain.random_pool, + temperature, + -drift_ux, + in::x1); + + // minimum and maximum position of particles + // ToDo: Add time offset for start of movement? + real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction + + injection_rate * time - drift_ux * dt; + real_t xg_max = domain.mesh.extent(in::x1).second * filling_fraction + + injection_rate * time; + + + /* + I thought this option would be better, but I can't get it to work + */ + + // // define box to inject into + // boundaries_t box; + // // loop over all dimensions + // for (unsigned short d{0}; d < static_cast(M::Dim); ++d) + // { + // // compute the range for the x-direction + // if (d == static_cast(in::x1)) + // { + // box.push_back({xg_min, xg_max}); + // } else { + // // inject into full range in other directions + // box.push_back(Range::All); + // } + // } + + // const auto spatial_dist = arch::Replenish(domain.mesh.metric, + // domain.fields.bckp, + // box, + // TargetDensity, + // 1.0); + + // const auto injector = arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // {1, 2}); + + const auto injector = arch::MovingInjector{ + domain.mesh.metric, + domain.fields.bckp, + energy_dist, + xg_max, + xg_min, + 1.0, + {1, 2}}; + + arch::InjectNonUniform( + params, + domain, + injector, + 1.0, // target density + false); + } }; } // namespace user diff --git a/setups/srpic/shock/shock.toml b/setups/srpic/shock/shock.toml index f954345c2..772b452c9 100644 --- a/setups/srpic/shock/shock.toml +++ b/setups/srpic/shock/shock.toml @@ -45,10 +45,10 @@ Bmag = 1.0 # magnetic field strength as fraction of magnetisation Btheta = 0.0 # magnetic field angle in the plane Bphi = 0.0 # magnetic field angle out of plane - filling_fraction = 1.0 # fraction of the shock piston filled with plasma + filling_fraction = 0.1 # fraction of the shock piston filled with plasma [output] - interval_time = 500.0 + interval_time = 10.0 format = "hdf5" [output.fields] diff --git a/src/engines/engine_run.cpp b/src/engines/engine_run.cpp index 2d4b0d5ed..a3ae21ccb 100644 --- a/src/engines/engine_run.cpp +++ b/src/engines/engine_run.cpp @@ -52,7 +52,7 @@ namespace ntt { traits::has_method::value) { timers.start("Custom"); m_metadomain.runOnLocalDomains([&timers, this](auto& dom) { - m_pgen.CustomPostStep(step, time, dom); + m_pgen.CustomPostStep(step, time, dt, dom); }); timers.stop("Custom"); } From 236060b4b9b092c5e568b5691a76ac2f76772fce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Sat, 22 Mar 2025 11:42:48 -0500 Subject: [PATCH 150/176] bugfix --- src/archetypes/particle_injector.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/archetypes/particle_injector.h b/src/archetypes/particle_injector.h index daa41506c..676cf2857 100644 --- a/src/archetypes/particle_injector.h +++ b/src/archetypes/particle_injector.h @@ -188,7 +188,7 @@ namespace arch { { const auto xi = x_Ph[static_cast(O)]; // + direction - if (xi < xinj - xdrift or xi >= xinj) + if (xi < xdrift or xi >= xinj) { return ZERO; } From 2391bee2aad7d3d22427fe3481675aab9ebf4d3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Sat, 22 Mar 2025 11:56:32 -0500 Subject: [PATCH 151/176] added start time of injection and explicitly enforce injection region (wip) --- setups/srpic/shock/pgen.hpp | 102 +++++++++++++++++----------------- setups/srpic/shock/shock.toml | 2 + 2 files changed, 54 insertions(+), 50 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 7c0d750e3..04de58406 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -79,7 +79,8 @@ namespace user { using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; - const real_t drift_ux, temperature, filling_fraction, injection_rate; + const real_t drift_ux, temperature, filling_fraction, + injector_velocity, injection_start; const real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -93,7 +94,8 @@ namespace user { , Bphi { p.template get("setup.Bphi", ZERO) } , init_flds { Bmag, Btheta, Bphi, drift_ux } , filling_fraction { params.template get("setup.filling_fraction", 1.0) } - , injection_rate { params.template get("setup.injection_rate", 1.0) } {} + , injector_velocity { params.template get("setup.injector_velocity", 1.0) } + , injection_start { params.template get("setup.injection_start", 0.0) } {} inline PGen() {} @@ -164,58 +166,58 @@ namespace user { in::x1); // minimum and maximum position of particles - // ToDo: Add time offset for start of movement? - real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction + - injection_rate * time - drift_ux * dt; + real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction + + injector_velocity * (time - injection_start) + - drift_ux; // distance particles have moved in the last time step real_t xg_max = domain.mesh.extent(in::x1).second * filling_fraction + - injection_rate * time; + injector_velocity * (time - injection_start); + /* + I thought this option would be better, but I can't get it to work + */ - /* - I thought this option would be better, but I can't get it to work - */ - - // // define box to inject into - // boundaries_t box; - // // loop over all dimensions - // for (unsigned short d{0}; d < static_cast(M::Dim); ++d) - // { - // // compute the range for the x-direction - // if (d == static_cast(in::x1)) - // { - // box.push_back({xg_min, xg_max}); - // } else { - // // inject into full range in other directions - // box.push_back(Range::All); - // } - // } - - // const auto spatial_dist = arch::Replenish(domain.mesh.metric, - // domain.fields.bckp, - // box, - // TargetDensity, - // 1.0); - - // const auto injector = arch::NonUniformInjector( - // energy_dist, - // spatial_dist, - // {1, 2}); - - const auto injector = arch::MovingInjector{ - domain.mesh.metric, - domain.fields.bckp, - energy_dist, - xg_max, - xg_min, - 1.0, - {1, 2}}; + // define box to inject into + boundaries_t box; + // loop over all dimensions + for (unsigned short d{0}; d < static_cast(M::Dim); ++d) + { + // compute the range for the x-direction + if (d == static_cast(in::x1)) + { + box.push_back({xg_min, xg_max}); + } else { + // inject into full range in other directions + box.push_back(Range::All); + } + } - arch::InjectNonUniform( - params, - domain, - injector, - 1.0, // target density - false); + // const auto spatial_dist = arch::Replenish(domain.mesh.metric, + // domain.fields.bckp, + // box, + // TargetDensity, + // 1.0); + + // const auto injector = arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // {1, 2}); + + const auto injector = arch::MovingInjector{ + domain.mesh.metric, + domain.fields.bckp, + energy_dist, + xg_max, + xg_min, + 1.0, + {1, 2}}; + + arch::InjectNonUniform( + params, + domain, + injector, + 1.0, // target density + false, + box); } }; diff --git a/setups/srpic/shock/shock.toml b/setups/srpic/shock/shock.toml index 772b452c9..43da87d99 100644 --- a/setups/srpic/shock/shock.toml +++ b/setups/srpic/shock/shock.toml @@ -46,6 +46,8 @@ Btheta = 0.0 # magnetic field angle in the plane Bphi = 0.0 # magnetic field angle out of plane filling_fraction = 0.1 # fraction of the shock piston filled with plasma + injector_velocity = 1.0 # speed of injector [c] + injection_start = 0.0 # start time of moving injector [output] interval_time = 10.0 From d67f6c746e360b6e27aa7aabdb97d314e9c5d467 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Sat, 22 Mar 2025 12:12:00 -0500 Subject: [PATCH 152/176] small bugfix --- setups/srpic/shock/pgen.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 04de58406..1c9c9a4d3 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -167,8 +167,8 @@ namespace user { // minimum and maximum position of particles real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction - + injector_velocity * (time - injection_start) - - drift_ux; // distance particles have moved in the last time step + + injector_velocity * (time - injection_start - dt) + - drift_ux * dt; // distance particles have moved in the last time step real_t xg_max = domain.mesh.extent(in::x1).second * filling_fraction + injector_velocity * (time - injection_start); From bb8e30810d140863dcbbad1def9ce1b1ae5da202 Mon Sep 17 00:00:00 2001 From: hayk Date: Mon, 24 Mar 2025 13:14:51 -0400 Subject: [PATCH 153/176] shock pgen changed --- setups/srpic/shock/pgen.hpp | 200 ++++++++++++++++++++-------------- src/archetypes/spatial_dist.h | 39 +++---- src/engines/engine_run.cpp | 2 +- 3 files changed, 134 insertions(+), 107 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 1c9c9a4d3..cc6f1b812 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -18,6 +18,16 @@ namespace user { using namespace ntt; + template + struct SpatialUniform : public arch::SpatialDistribution { + SpatialUniform(const M& metric) + : arch::SpatialDistribution { metric } {} + + Inline auto operator()(const coord_t&) const -> real_t override { + return ONE; + } + }; + template struct InitFields { /* @@ -79,8 +89,8 @@ namespace user { using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; - const real_t drift_ux, temperature, filling_fraction, - injector_velocity, injection_start; + const real_t drift_ux, temperature, filling_fraction, injector_velocity, + injection_start, dt; const real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -92,10 +102,14 @@ namespace user { , Bmag { p.template get("setup.Bmag", ZERO) } , Btheta { p.template get("setup.Btheta", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } - , init_flds { Bmag, Btheta, Bphi, drift_ux } - , filling_fraction { params.template get("setup.filling_fraction", 1.0) } - , injector_velocity { params.template get("setup.injector_velocity", 1.0) } - , injection_start { params.template get("setup.injection_start", 0.0) } {} + , init_flds { Bmag, Btheta, Bphi, drift_ux } + , filling_fraction { params.template get("setup.filling_fraction", + 1.0) } + , injector_velocity { params.template get( + "setup.injector_velocity", + 1.0) } + , injection_start { params.template get("setup.injection_start", 0.0) } + , dt { params.template get("algorithms.timestep.dt") } {} inline PGen() {} @@ -109,7 +123,7 @@ namespace user { // } inline void InitPrtls(Domain& local_domain) { - + // minimum and maximum position of particles real_t xg_min = local_domain.mesh.extent(in::x1).first; real_t xg_max = local_domain.mesh.extent(in::x1).second * filling_fraction; @@ -120,16 +134,19 @@ namespace user { for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { // compute the range for the x-direction if (d == static_cast(in::x1)) { - box.push_back({xg_min, xg_max}); + box.push_back({ xg_min, xg_max }); } else { // inject into full range in other directions box.push_back(Range::All); } } - // spatial distribution of the particles + // spatial distribution of the particles // -> hack to use the uniform distribution in NonUniformInjector - const auto spatial_dist = arch::Piston(local_domain.mesh.metric, xg_min, xg_max, in::x1); + const auto spatial_dist = arch::Piston(local_domain.mesh.metric, + xg_min, + xg_max, + in::x1); // energy distribution of the particles const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, @@ -139,85 +156,100 @@ namespace user { in::x1); const auto injector = arch::NonUniformInjector( - energy_dist, - spatial_dist, - {1, 2}); + energy_dist, + spatial_dist, + { 1, 2 }); arch::InjectNonUniform>( - params, - local_domain, - injector, - 1.0, // target density - false, // no weights - box); + params, + local_domain, + injector, + 1.0, // target density + false, // no weights + box); } - void CustomPostStep(std::size_t, long double time, real_t dt, Domain &domain) - { - /* - Moving injector for the particles - */ - - // energy distribution of the particles - const auto energy_dist = arch::Maxwellian(domain.mesh.metric, - domain.random_pool, - temperature, - -drift_ux, - in::x1); - - // minimum and maximum position of particles - real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction - + injector_velocity * (time - injection_start - dt) - - drift_ux * dt; // distance particles have moved in the last time step - real_t xg_max = domain.mesh.extent(in::x1).second * filling_fraction + - injector_velocity * (time - injection_start); - - /* - I thought this option would be better, but I can't get it to work - */ - - // define box to inject into - boundaries_t box; - // loop over all dimensions - for (unsigned short d{0}; d < static_cast(M::Dim); ++d) - { - // compute the range for the x-direction - if (d == static_cast(in::x1)) - { - box.push_back({xg_min, xg_max}); - } else { - // inject into full range in other directions - box.push_back(Range::All); - } - } + void CustomPostStep(std::size_t, long double time, Domain& domain) { + const auto energy_dist = arch::Maxwellian(domain.mesh.metric, + domain.random_pool, + temperature, + -drift_ux, + in::x1); + const auto spatial_dist = SpatialUniform(domain.mesh.metric); + const auto injector = arch::NonUniformInjector( + energy_dist, + spatial_dist, + { 1, 2 }); + + boundaries_t box; + + const auto x_init = domain.mesh.extent(in::x1).first + + filling_fraction * (domain.mesh.extent(in::x1).second - + domain.mesh.extent(in::x1).first); + + box[0].first = x_init + injector_velocity * (time - injection_start); + box[0].second = x_init + injector_velocity * (time - injection_start + dt) - + drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt; + for (auto d = 1u; d < M::Dim; ++d) { + box[d] = Range::All; + } - // const auto spatial_dist = arch::Replenish(domain.mesh.metric, - // domain.fields.bckp, - // box, - // TargetDensity, - // 1.0); - - // const auto injector = arch::NonUniformInjector( - // energy_dist, - // spatial_dist, - // {1, 2}); - - const auto injector = arch::MovingInjector{ - domain.mesh.metric, - domain.fields.bckp, - energy_dist, - xg_max, - xg_min, - 1.0, - {1, 2}}; - - arch::InjectNonUniform( - params, - domain, - injector, - 1.0, // target density - false, - box); + arch::InjectNonUniform(params, + domain, + injector, + ONE, + false, + box); + + // /* + // Moving injector for the particles + // */ + // + // // minimum and maximum position of particles + // real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction + + // injector_velocity * (time - injection_start - dt) - + // drift_ux * + // dt; // distance particles have moved in the last time step + // real_t xg_max = domain.mesh.extent(in::x1).second * filling_fraction + + // injector_velocity * (time - injection_start); + // + // /* + // I thought this option would be better, but I can't get it to work + // */ + // + // // define box to inject into + // // loop over all dimensions + // for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { + // // compute the range for the x-direction + // if (d == static_cast(in::x1)) { + // box.push_back({ xg_min, xg_max }); + // } else { + // // inject into full range in other directions + // box.push_back(Range::All); + // } + // } + + // const auto spatial_dist = arch::Replenish(domain.mesh.metric, + // domain.fields.bckp, + // box, + // TargetDensity, + // 1.0); + + // const auto injector = arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // {1, 2}); + + // const auto injector = arch::MovingInjector { + // domain.mesh.metric, + // domain.fields.bckp, + // energy_dist, + // xg_max, + // xg_min, + // 1.0, + // { 1, 2 } + // }; } }; diff --git a/src/archetypes/spatial_dist.h b/src/archetypes/spatial_dist.h index 2edbf5b1d..ad9404ea3 100644 --- a/src/archetypes/spatial_dist.h +++ b/src/archetypes/spatial_dist.h @@ -51,32 +51,27 @@ namespace arch { }; template - struct Piston : public arch::SpatialDistribution - { - Piston(const M &metric, real_t xmin, real_t xmax, in piston_direction = in::x1) - : arch::SpatialDistribution{metric} - , xmin {xmin} - , xmax {xmax} - , piston_direction {piston_direction} {} - - Inline auto operator()(const coord_t &x_Ph) const -> real_t override - { - // dimentsion to fill - const auto fill_dim = static_cast(piston_direction); - - if (x_Ph[fill_dim] < xmin || x_Ph[fill_dim] > xmax) - { - return ZERO; - } - else - { - return ONE; - } + struct Piston : public arch::SpatialDistribution { + Piston(const M& metric, real_t xmin, real_t xmax, in piston_direction = in::x1) + : arch::SpatialDistribution { metric } + , xmin { xmin } + , xmax { xmax } + , piston_direction { piston_direction } {} + + Inline auto operator()(const coord_t& x_Ph) const -> real_t override { + // dimentsion to fill + const auto fill_dim = static_cast(piston_direction); + + if (x_Ph[fill_dim] < xmin || x_Ph[fill_dim] > xmax) { + return ZERO; + } else { + return ONE; } + } private: real_t xmin, xmax; - in piston_direction; + in piston_direction; }; template diff --git a/src/engines/engine_run.cpp b/src/engines/engine_run.cpp index a3ae21ccb..2d4b0d5ed 100644 --- a/src/engines/engine_run.cpp +++ b/src/engines/engine_run.cpp @@ -52,7 +52,7 @@ namespace ntt { traits::has_method::value) { timers.start("Custom"); m_metadomain.runOnLocalDomains([&timers, this](auto& dom) { - m_pgen.CustomPostStep(step, time, dt, dom); + m_pgen.CustomPostStep(step, time, dom); }); timers.stop("Custom"); } From 31ef1000e7e814e06cd027b79104bc8be6cdba9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 24 Mar 2025 15:31:41 -0500 Subject: [PATCH 154/176] reduce resolution for faster test --- setups/srpic/shock/shock.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setups/srpic/shock/shock.toml b/setups/srpic/shock/shock.toml index 43da87d99..ca19a4078 100644 --- a/setups/srpic/shock/shock.toml +++ b/setups/srpic/shock/shock.toml @@ -4,8 +4,8 @@ runtime = 30000.0 [grid] - resolution = [16384, 128] - extent = [[0.0, 8000.0], [-31.25, 31.25]] + resolution = [4096, 128] + extent = [[0.0, 2000.0], [-31.25, 31.25]] [grid.metric] metric = "minkowski" From b32468754eaf799e94da7d3a531a2926afc9d5c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 24 Mar 2025 15:31:52 -0500 Subject: [PATCH 155/176] cleanup and bugfix --- setups/srpic/shock/pgen.hpp | 195 +++++++++++++++++------------------- 1 file changed, 92 insertions(+), 103 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index cc6f1b812..2454c9037 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -18,16 +18,6 @@ namespace user { using namespace ntt; - template - struct SpatialUniform : public arch::SpatialDistribution { - SpatialUniform(const M& metric) - : arch::SpatialDistribution { metric } {} - - Inline auto operator()(const coord_t&) const -> real_t override { - return ONE; - } - }; - template struct InitFields { /* @@ -89,10 +79,13 @@ namespace user { using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; - const real_t drift_ux, temperature, filling_fraction, injector_velocity, - injection_start, dt; + // gas properties + const real_t drift_ux, temperature, filling_fraction; + // injector properties + const real_t injector_velocity, injection_start, dt; + const bool injection_on; - const real_t Btheta, Bphi, Bmag; + real_t Btheta, Bphi, Bmag; InitFields init_flds; inline PGen(const SimulationParams& p, const Metadomain& m) @@ -103,13 +96,11 @@ namespace user { , Btheta { p.template get("setup.Btheta", ZERO) } , Bphi { p.template get("setup.Bphi", ZERO) } , init_flds { Bmag, Btheta, Bphi, drift_ux } - , filling_fraction { params.template get("setup.filling_fraction", - 1.0) } - , injector_velocity { params.template get( - "setup.injector_velocity", - 1.0) } - , injection_start { params.template get("setup.injection_start", 0.0) } - , dt { params.template get("algorithms.timestep.dt") } {} + , filling_fraction { p.template get("setup.filling_fraction", 1.0) } + , injector_velocity { p.template get("setup.injector_velocity", 1.0) } + , injection_start { p.template get("setup.injection_start", 0.0) } + , injection_on { p.template get("setup.continuous_injection", true) } + , dt { p.template get("algorithms.timestep.dt") } {} inline PGen() {} @@ -117,16 +108,13 @@ namespace user { return init_flds; } - // Inline auto TargetDensity(const coord_t &x_Ph) const -> real_t - // { - // return ONE; - // } - inline void InitPrtls(Domain& local_domain) { // minimum and maximum position of particles real_t xg_min = local_domain.mesh.extent(in::x1).first; - real_t xg_max = local_domain.mesh.extent(in::x1).second * filling_fraction; + real_t xg_max = local_domain.mesh.extent(in::x1).first + + filling_fraction * (local_domain.mesh.extent(in::x1).second - + local_domain.mesh.extent(in::x1).first); // define box to inject into boundaries_t box; @@ -170,86 +158,87 @@ namespace user { } void CustomPostStep(std::size_t, long double time, Domain& domain) { - const auto energy_dist = arch::Maxwellian(domain.mesh.metric, - domain.random_pool, - temperature, - -drift_ux, - in::x1); - const auto spatial_dist = SpatialUniform(domain.mesh.metric); - const auto injector = arch::NonUniformInjector( - energy_dist, - spatial_dist, - { 1, 2 }); - boundaries_t box; - - const auto x_init = domain.mesh.extent(in::x1).first + - filling_fraction * (domain.mesh.extent(in::x1).second - - domain.mesh.extent(in::x1).first); + // ToDo: more performant version? + if (injection_on) { + // same maxwell distribution as above + const auto energy_dist = arch::Maxwellian(domain.mesh.metric, + domain.random_pool, + temperature, + -drift_ux, + in::x1); + + // initial position of injector + const auto x_init = domain.mesh.extent(in::x1).first + + filling_fraction * (domain.mesh.extent(in::x1).second - + domain.mesh.extent(in::x1).first); + + // check if injector is supposed to start moving already + const auto dt_inj = time - injection_start > ZERO ? time - injection_start + : ZERO; + + // define box to inject into + boundaries_t box; + + // loop over all dimension + for (auto d = 0u; d < M::Dim; ++d) { + if (d == 0) { + box.push_back({ x_init + injector_velocity * (dt_inj)-drift_ux / + math::sqrt(1 + SQR(drift_ux)) * dt, + x_init + injector_velocity * (dt_inj + dt) }); + } else { + box.push_back(Range::All); + } + } - box[0].first = x_init + injector_velocity * (time - injection_start); - box[0].second = x_init + injector_velocity * (time - injection_start + dt) - - drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt; - for (auto d = 1u; d < M::Dim; ++d) { - box[d] = Range::All; + // spatial distribution of the particles + // -> hack to use the uniform distribution in NonUniformInjector + const auto spatial_dist = arch::Piston(domain.mesh.metric, + box[0].first, + box[0].second, + in::x1); + + // ToDo: extend Replenish to replace the current injector + const auto injector = + arch::NonUniformInjector( + energy_dist, + spatial_dist, + { 1, 2 }); + + // inject non-uniformly within the defined box + arch::InjectNonUniform(params, + domain, + injector, + ONE, + false, + box); + + /* + I thought this option would be better, but I can't get it to work + */ + + // const auto spatial_dist = arch::Replenish(domain.mesh.metric, + // domain.fields.bckp, + // box, + // TargetDensity, + // 1.0); + + // const auto injector = arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // {1, 2}); + + // const auto injector = arch::MovingInjector { + // domain.mesh.metric, + // domain.fields.bckp, + // energy_dist, + // box[0].first, + // box[0].second, + // 1.0, + // { 1, 2 } + // }; } - - arch::InjectNonUniform(params, - domain, - injector, - ONE, - false, - box); - - // /* - // Moving injector for the particles - // */ - // - // // minimum and maximum position of particles - // real_t xg_min = domain.mesh.extent(in::x1).second * filling_fraction + - // injector_velocity * (time - injection_start - dt) - - // drift_ux * - // dt; // distance particles have moved in the last time step - // real_t xg_max = domain.mesh.extent(in::x1).second * filling_fraction + - // injector_velocity * (time - injection_start); - // - // /* - // I thought this option would be better, but I can't get it to work - // */ - // - // // define box to inject into - // // loop over all dimensions - // for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { - // // compute the range for the x-direction - // if (d == static_cast(in::x1)) { - // box.push_back({ xg_min, xg_max }); - // } else { - // // inject into full range in other directions - // box.push_back(Range::All); - // } - // } - - // const auto spatial_dist = arch::Replenish(domain.mesh.metric, - // domain.fields.bckp, - // box, - // TargetDensity, - // 1.0); - - // const auto injector = arch::NonUniformInjector( - // energy_dist, - // spatial_dist, - // {1, 2}); - - // const auto injector = arch::MovingInjector { - // domain.mesh.metric, - // domain.fields.bckp, - // energy_dist, - // xg_max, - // xg_min, - // 1.0, - // { 1, 2 } - // }; } }; From 2d1a45436cd677560cd93bab56c255eea251db43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 24 Mar 2025 15:45:43 -0500 Subject: [PATCH 156/176] removed unnecessary parameter --- setups/srpic/shock/pgen.hpp | 156 +++++++++++++++++------------------- 1 file changed, 75 insertions(+), 81 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 2454c9037..30ab01538 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -83,8 +83,7 @@ namespace user { const real_t drift_ux, temperature, filling_fraction; // injector properties const real_t injector_velocity, injection_start, dt; - const bool injection_on; - + // magnetic field properties real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -99,7 +98,6 @@ namespace user { , filling_fraction { p.template get("setup.filling_fraction", 1.0) } , injector_velocity { p.template get("setup.injector_velocity", 1.0) } , injection_start { p.template get("setup.injection_start", 0.0) } - , injection_on { p.template get("setup.continuous_injection", true) } , dt { p.template get("algorithms.timestep.dt") } {} inline PGen() {} @@ -159,86 +157,82 @@ namespace user { void CustomPostStep(std::size_t, long double time, Domain& domain) { - // ToDo: more performant version? - if (injection_on) { - // same maxwell distribution as above - const auto energy_dist = arch::Maxwellian(domain.mesh.metric, - domain.random_pool, - temperature, - -drift_ux, - in::x1); - - // initial position of injector - const auto x_init = domain.mesh.extent(in::x1).first + - filling_fraction * (domain.mesh.extent(in::x1).second - - domain.mesh.extent(in::x1).first); - - // check if injector is supposed to start moving already - const auto dt_inj = time - injection_start > ZERO ? time - injection_start - : ZERO; - - // define box to inject into - boundaries_t box; - - // loop over all dimension - for (auto d = 0u; d < M::Dim; ++d) { - if (d == 0) { - box.push_back({ x_init + injector_velocity * (dt_inj)-drift_ux / - math::sqrt(1 + SQR(drift_ux)) * dt, - x_init + injector_velocity * (dt_inj + dt) }); - } else { - box.push_back(Range::All); - } - } + // same maxwell distribution as above + const auto energy_dist = arch::Maxwellian(domain.mesh.metric, + domain.random_pool, + temperature, + -drift_ux, + in::x1); + + // initial position of injector + const auto x_init = domain.mesh.extent(in::x1).first + + filling_fraction * (domain.mesh.extent(in::x1).second - + domain.mesh.extent(in::x1).first); - // spatial distribution of the particles - // -> hack to use the uniform distribution in NonUniformInjector - const auto spatial_dist = arch::Piston(domain.mesh.metric, - box[0].first, - box[0].second, - in::x1); - - // ToDo: extend Replenish to replace the current injector - const auto injector = - arch::NonUniformInjector( - energy_dist, - spatial_dist, - { 1, 2 }); - - // inject non-uniformly within the defined box - arch::InjectNonUniform(params, - domain, - injector, - ONE, - false, - box); - - /* - I thought this option would be better, but I can't get it to work - */ - - // const auto spatial_dist = arch::Replenish(domain.mesh.metric, - // domain.fields.bckp, - // box, - // TargetDensity, - // 1.0); - - // const auto injector = arch::NonUniformInjector( - // energy_dist, - // spatial_dist, - // {1, 2}); - - // const auto injector = arch::MovingInjector { - // domain.mesh.metric, - // domain.fields.bckp, - // energy_dist, - // box[0].first, - // box[0].second, - // 1.0, - // { 1, 2 } - // }; + // check if injector is supposed to start moving already + const auto dt_inj = time - injection_start > ZERO ? + time - injection_start : ZERO; + + // define box to inject into + boundaries_t box; + + // loop over all dimension + for (auto d = 0u; d < M::Dim; ++d) { + if (d == 0) { + box.push_back({ x_init + injector_velocity * dt_inj - + drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt, + x_init + injector_velocity * (dt_inj + dt) }); + } else { + box.push_back(Range::All); + } } + + // spatial distribution of the particles + // -> hack to use the uniform distribution in NonUniformInjector + const auto spatial_dist = arch::Piston(domain.mesh.metric, + box[0].first, + box[0].second, + in::x1); + + // ToDo: extend Replenish to replace the current injector + const auto injector = arch::NonUniformInjector( + energy_dist, + spatial_dist, + { 1, 2 }); + + // inject non-uniformly within the defined box + arch::InjectNonUniform(params, + domain, + injector, + ONE, + false, + box); + + /* + I thought this option would be better, but I can't get it to work + */ + + // const auto spatial_dist = arch::Replenish(domain.mesh.metric, + // domain.fields.bckp, + // box, + // TargetDensity, + // 1.0); + + // const auto injector = arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // {1, 2}); + + // const auto injector = arch::MovingInjector { + // domain.mesh.metric, + // domain.fields.bckp, + // energy_dist, + // box[0].first, + // box[0].second, + // 1.0, + // { 1, 2 } + // }; } }; From b3ee77b9714c2e83fb1b3be6caf397cf30255fbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 24 Mar 2025 16:06:42 -0500 Subject: [PATCH 157/176] applied formatting to `MovingInjector` --- src/archetypes/particle_injector.h | 123 ++++++++++++++--------------- 1 file changed, 58 insertions(+), 65 deletions(-) diff --git a/src/archetypes/particle_injector.h b/src/archetypes/particle_injector.h index 676cf2857..62b9249c3 100644 --- a/src/archetypes/particle_injector.h +++ b/src/archetypes/particle_injector.h @@ -171,72 +171,65 @@ namespace arch { }; template - struct MovingInjector - { - struct TargetDensityProfile - { - const real_t nmax, xinj, xdrift; - - TargetDensityProfile(real_t xinj, real_t xdrift, real_t nmax) - : xinj{xinj}, xdrift{xdrift}, nmax{nmax} {} - - Inline auto operator()(const coord_t &x_Ph) const -> real_t - { - if constexpr ((O == in::x1) or - (O == in::x2 and (M::Dim == Dim::_2D or M::Dim == Dim::_3D)) or - (O == in::x3 and M::Dim == Dim::_3D)) - { - const auto xi = x_Ph[static_cast(O)]; - // + direction - if (xi < xdrift or xi >= xinj) - { - return ZERO; - } - else - { - if constexpr (M::CoordType == Coord::Cart) - { - return nmax; - } - else - { - raise::KernelError( - HERE, - "Moving injector in +x cannot be applied for non-cartesian"); - return ZERO; - } - } - } - else - { - raise::KernelError(HERE, "Wrong direction"); - return ZERO; - } + struct MovingInjector { + struct TargetDensityProfile { + const real_t nmax, xinj, xdrift; + + TargetDensityProfile(real_t xinj, real_t xdrift, real_t nmax) + : xinj { xinj } + , xdrift { xdrift } + , nmax { nmax } {} + + Inline auto operator()(const coord_t& x_Ph) const -> real_t { + if constexpr ((O == in::x1) or + (O == in::x2 and (M::Dim == Dim::_2D or M::Dim == Dim::_3D)) or + (O == in::x3 and M::Dim == Dim::_3D)) { + const auto xi = x_Ph[static_cast(O)]; + // + direction + if (xi < xdrift or xi >= xinj) { + return ZERO; + } else { + if constexpr (M::CoordType == Coord::Cart) { + return nmax; + } else { + raise::KernelError( + HERE, + "Moving injector in +x cannot be applied for non-cartesian"); + return ZERO; } - }; - using energy_dist_t = Maxwellian; - using spatial_dist_t = Replenish; - static_assert(M::is_metric, "M must be a metric class"); - static constexpr bool is_nonuniform_injector{true}; - static constexpr Dimension D{M::Dim}; - static constexpr Coord C{M::CoordType}; - - const energy_dist_t energy_dist; - const TargetDensityProfile target_density; - const spatial_dist_t spatial_dist; - const std::pair species; - - MovingInjector(const M &metric, - const ndfield_t &density, - const energy_dist_t &energy_dist, - real_t xinj, - real_t xdrift, - real_t nmax, - const std::pair &species) - : energy_dist{energy_dist}, - target_density{xinj, xdrift, nmax}, spatial_dist{metric, density, 0, target_density, nmax}, species{species} {} - - ~MovingInjector() = default; + } + } else { + raise::KernelError(HERE, "Wrong direction"); + return ZERO; + } + } + }; + + using energy_dist_t = Maxwellian; + using spatial_dist_t = Replenish; + static_assert(M::is_metric, "M must be a metric class"); + static constexpr bool is_nonuniform_injector { true }; + static constexpr Dimension D { M::Dim }; + static constexpr Coord C { M::CoordType }; + + const energy_dist_t energy_dist; + const TargetDensityProfile target_density; + const spatial_dist_t spatial_dist; + const std::pair species; + + MovingInjector(const M& metric, + const ndfield_t& density, + const energy_dist_t& energy_dist, + real_t xinj, + real_t xdrift, + real_t nmax, + const std::pair& species) + : energy_dist { energy_dist } + , target_density { xinj, xdrift, nmax } + , spatial_dist { metric, density, 0, target_density, nmax } + , species { species } {} + + ~MovingInjector() = default; }; /** From 8d61ce85a70cc3fed3b4a57db2f84a4c7bccad45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Mon, 31 Mar 2025 11:03:57 -0500 Subject: [PATCH 158/176] first attempt at moving-window injection --- setups/srpic/shock/pgen.hpp | 123 +++++++++++++++++++++++++++++++++--- 1 file changed, 114 insertions(+), 9 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 30ab01538..9568767ff 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -83,6 +83,7 @@ namespace user { const real_t drift_ux, temperature, filling_fraction; // injector properties const real_t injector_velocity, injection_start, dt; + const int injection_frequency; // magnetic field properties real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -98,6 +99,7 @@ namespace user { , filling_fraction { p.template get("setup.filling_fraction", 1.0) } , injector_velocity { p.template get("setup.injector_velocity", 1.0) } , injection_start { p.template get("setup.injection_start", 0.0) } + , injection_frequency { p.template get("setup.injection_frequency", 100) } , dt { p.template get("algorithms.timestep.dt") } {} inline PGen() {} @@ -106,6 +108,25 @@ namespace user { return init_flds; } + auto ResetFields(const em& comp) const -> real_t { + if (comp == em::ex1) { + return init_flds.ex1({ ZERO }); + } else if (comp == em::ex2) { + return init_flds.ex2({ ZERO }); + } else if (comp == em::ex3) { + return init_flds.ex3({ ZERO }); + } else if (comp == em::bx1) { + return init_flds.bx1({ ZERO }); + } else if (comp == em::bx2) { + return init_flds.bx2({ ZERO }); + } else if (comp == em::bx3) { + return init_flds.bx3({ ZERO }); + } else { + raise::Error("Invalid component", HERE); + return ZERO; + } + } + inline void InitPrtls(Domain& local_domain) { // minimum and maximum position of particles @@ -155,14 +176,12 @@ namespace user { box); } - void CustomPostStep(std::size_t, long double time, Domain& domain) { + void CustomPostStep(std::size_t step, long double time, Domain& domain) { - // same maxwell distribution as above - const auto energy_dist = arch::Maxwellian(domain.mesh.metric, - domain.random_pool, - temperature, - -drift_ux, - in::x1); + // check if the injector should be active + if (step % injection_frequency != 0) { + return; + } // initial position of injector const auto x_init = domain.mesh.extent(in::x1).first + @@ -180,13 +199,99 @@ namespace user { for (auto d = 0u; d < M::Dim; ++d) { if (d == 0) { box.push_back({ x_init + injector_velocity * dt_inj - - drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt, + drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt - + 1.5 * injection_frequency * dt, x_init + injector_velocity * (dt_inj + dt) }); } else { box.push_back(Range::All); } } + // define indice range to reset fields + boundaries_t incl_ghosts; + for (auto d = 0; d < M::Dim; ++d) { + incl_ghosts.push_back({ true, true }); + } + const auto extent = domain.mesh.ExtentToRange(box, incl_ghosts); + tuple_t x_min { 0 }, x_max { 0 }; + for (auto d = 0; d < M::Dim; ++d) { + x_min[d] = extent[d].first; + x_max[d] = extent[d].second; + } + + // reset fields + std::vector comps = { em::ex1, em::ex2, em::ex3, + em::bx1, em::bx2, em::bx3 }; + + // loop over all components + for (const auto& comp : comps) { + auto value = ResetFields((em)comp); + + if constexpr (M::Dim == Dim::_1D) { + Kokkos::deep_copy(Kokkos::subview(domain.fields.em, + std::make_pair(x_min[0], x_max[0]), + comp), + value); + } else if constexpr (M::Dim == Dim::_2D) { + Kokkos::deep_copy(Kokkos::subview(domain.fields.em, + std::make_pair(x_min[0], x_max[0]), + std::make_pair(x_min[1], x_max[1]), + comp), + value); + } else if constexpr (M::Dim == Dim::_3D) { + Kokkos::deep_copy(Kokkos::subview(domain.fields.em, + std::make_pair(x_min[0], x_max[0]), + std::make_pair(x_min[1], x_max[1]), + std::make_pair(x_min[2], x_max[2]), + comp), + value); + } else { + raise::Error("Invalid dimension", HERE); + } + } + + /* + tag particles inside the injection zone as dead + */ + + // loop over particle species + for (std::size_t s { 0 }; s < 2; ++s) { + + // get particle properties + auto& species = domain.species[s]; + auto i1 = species.i1; + auto tag = species.tag; + + + // tag all particles with x > box[0].first as dead + Kokkos::parallel_for( + "RemoveParticles", + species.rangeActiveParticles(), + Lambda(index_t p) { + // check if the particle is already dead + if (tag(p) == ParticleTag::dead) { + return; + } + // select the x-coordinate index + auto x_i1 = i1(p); + // check if the particle is inside the box of new plasma + if (x_i1 > x_min[0]) { + tag(p) = ParticleTag::dead; + } + } + ); + } + + /* + Inject piston of fresh plasma + */ + + // same maxwell distribution as above + const auto energy_dist = arch::Maxwellian(domain.mesh.metric, + domain.random_pool, + temperature, + -drift_ux, + in::x1); // spatial distribution of the particles // -> hack to use the uniform distribution in NonUniformInjector const auto spatial_dist = arch::Piston(domain.mesh.metric, @@ -194,7 +299,7 @@ namespace user { box[0].second, in::x1); - // ToDo: extend Replenish to replace the current injector + // inject piston of fresh plasma const auto injector = arch::NonUniformInjector( energy_dist, spatial_dist, From b2c738af48dce1d73b7021ae3fbf401c96984817 Mon Sep 17 00:00:00 2001 From: LudwigBoess Date: Mon, 31 Mar 2025 21:52:04 -0500 Subject: [PATCH 159/176] extended field reset box to the right to squash waves propagating into the empty box --- setups/srpic/shock/pgen.hpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index 9568767ff..a25c3891a 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -190,17 +190,16 @@ namespace user { // check if injector is supposed to start moving already const auto dt_inj = time - injection_start > ZERO ? - time - injection_start : ZERO; + time - injection_start : ZERO; // define box to inject into boundaries_t box; - // loop over all dimension for (auto d = 0u; d < M::Dim; ++d) { if (d == 0) { box.push_back({ x_init + injector_velocity * dt_inj - drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt - - 1.5 * injection_frequency * dt, + injection_frequency * dt, x_init + injector_velocity * (dt_inj + dt) }); } else { box.push_back(Range::All); @@ -212,7 +211,9 @@ namespace user { for (auto d = 0; d < M::Dim; ++d) { incl_ghosts.push_back({ true, true }); } - const auto extent = domain.mesh.ExtentToRange(box, incl_ghosts); + auto fields_box = box; + fields_box[0].second += injection_frequency * dt; + const auto extent = domain.mesh.ExtentToRange(fields_box, incl_ghosts); tuple_t x_min { 0 }, x_max { 0 }; for (auto d = 0; d < M::Dim; ++d) { x_min[d] = extent[d].first; @@ -220,11 +221,13 @@ namespace user { } // reset fields - std::vector comps = { em::ex1, em::ex2, em::ex3, - em::bx1, em::bx2, em::bx3 }; + std::vector comps = { em::bx1, em::bx2, em::bx3, + em::ex1, em::ex2, em::ex3 }; // loop over all components for (const auto& comp : comps) { + + // get initial field value of component auto value = ResetFields((em)comp); if constexpr (M::Dim == Dim::_1D) { @@ -262,7 +265,6 @@ namespace user { auto i1 = species.i1; auto tag = species.tag; - // tag all particles with x > box[0].first as dead Kokkos::parallel_for( "RemoveParticles", From fb1692875ad050ba73dee745d0800dd356d9f8bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Tue, 1 Apr 2025 15:32:04 -0500 Subject: [PATCH 160/176] bugfix: added check to truncate the injection box at the end of the domain --- setups/srpic/shock/pgen.hpp | 48 ++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index a25c3891a..cf58b94c6 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -80,10 +80,10 @@ namespace user { using arch::ProblemGenerator::params; // gas properties - const real_t drift_ux, temperature, filling_fraction; + const real_t drift_ux, temperature, filling_fraction; // injector properties - const real_t injector_velocity, injection_start, dt; - const int injection_frequency; + const real_t injector_velocity, injection_start, dt; + const int injection_frequency; // magnetic field properties real_t Btheta, Bphi, Bmag; InitFields init_flds; @@ -189,18 +189,23 @@ namespace user { domain.mesh.extent(in::x1).first); // check if injector is supposed to start moving already - const auto dt_inj = time - injection_start > ZERO ? - time - injection_start : ZERO; + const auto dt_inj = time - injection_start > ZERO ? time - injection_start + : ZERO; + + // compute the position of the injector + auto xmax = x_init + injector_velocity * (dt_inj + dt); + if (xmax >= domain.mesh.extent(in::x1).second) { + xmax = domain.mesh.extent(in::x1).second; + } // define box to inject into boundaries_t box; // loop over all dimension for (auto d = 0u; d < M::Dim; ++d) { if (d == 0) { - box.push_back({ x_init + injector_velocity * dt_inj - - drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt - - injection_frequency * dt, - x_init + injector_velocity * (dt_inj + dt) }); + box.push_back({ xmax - drift_ux / math::sqrt(1 + SQR(drift_ux)) * dt - + injection_frequency * dt, + xmax }); } else { box.push_back(Range::All); } @@ -212,7 +217,13 @@ namespace user { incl_ghosts.push_back({ true, true }); } auto fields_box = box; - fields_box[0].second += injection_frequency * dt; + // check if the box is still inside the domain + if (xmax + injection_frequency * dt < domain.mesh.extent(in::x1).second) { + fields_box[0].second += injection_frequency * dt; + } else { + // if right side of the box is outside of the domain -> truncate box + fields_box[0].second = domain.mesh.extent(in::x1).second; + } const auto extent = domain.mesh.ExtentToRange(fields_box, incl_ghosts); tuple_t x_min { 0 }, x_max { 0 }; for (auto d = 0; d < M::Dim; ++d) { @@ -221,7 +232,7 @@ namespace user { } // reset fields - std::vector comps = { em::bx1, em::bx2, em::bx3, + std::vector comps = { em::bx1, em::bx2, em::bx3, em::ex1, em::ex2, em::ex3 }; // loop over all components @@ -253,8 +264,8 @@ namespace user { } } - /* - tag particles inside the injection zone as dead + /* + tag particles inside the injection zone as dead */ // loop over particle species @@ -262,8 +273,8 @@ namespace user { // get particle properties auto& species = domain.species[s]; - auto i1 = species.i1; - auto tag = species.tag; + auto i1 = species.i1; + auto tag = species.tag; // tag all particles with x > box[0].first as dead Kokkos::parallel_for( @@ -277,14 +288,13 @@ namespace user { // select the x-coordinate index auto x_i1 = i1(p); // check if the particle is inside the box of new plasma - if (x_i1 > x_min[0]) { + if (x_i1 >= x_min[0]) { tag(p) = ParticleTag::dead; } - } - ); + }); } - /* + /* Inject piston of fresh plasma */ From 884a1571f2c12f096fe966b81bf066294e499f03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludwig=20B=C3=B6ss?= Date: Tue, 1 Apr 2025 19:46:46 -0500 Subject: [PATCH 161/176] fix unit conversion bug in field reset --- setups/srpic/shock/pgen.hpp | 40 ++++++++----------------------------- 1 file changed, 8 insertions(+), 32 deletions(-) diff --git a/setups/srpic/shock/pgen.hpp b/setups/srpic/shock/pgen.hpp index cf58b94c6..ad260bda0 100644 --- a/setups/srpic/shock/pgen.hpp +++ b/setups/srpic/shock/pgen.hpp @@ -9,6 +9,7 @@ #include "utils/numeric.h" #include "archetypes/energy_dist.h" +#include "archetypes/field_setter.h" #include "archetypes/particle_injector.h" #include "archetypes/problem_generator.h" #include "framework/domain/metadomain.h" @@ -231,38 +232,13 @@ namespace user { x_max[d] = extent[d].second; } - // reset fields - std::vector comps = { em::bx1, em::bx2, em::bx3, - em::ex1, em::ex2, em::ex3 }; - - // loop over all components - for (const auto& comp : comps) { - - // get initial field value of component - auto value = ResetFields((em)comp); - - if constexpr (M::Dim == Dim::_1D) { - Kokkos::deep_copy(Kokkos::subview(domain.fields.em, - std::make_pair(x_min[0], x_max[0]), - comp), - value); - } else if constexpr (M::Dim == Dim::_2D) { - Kokkos::deep_copy(Kokkos::subview(domain.fields.em, - std::make_pair(x_min[0], x_max[0]), - std::make_pair(x_min[1], x_max[1]), - comp), - value); - } else if constexpr (M::Dim == Dim::_3D) { - Kokkos::deep_copy(Kokkos::subview(domain.fields.em, - std::make_pair(x_min[0], x_max[0]), - std::make_pair(x_min[1], x_max[1]), - std::make_pair(x_min[2], x_max[2]), - comp), - value); - } else { - raise::Error("Invalid dimension", HERE); - } - } + Kokkos::parallel_for("ResetFields", + CreateRangePolicy(x_min, x_max), + arch::SetEMFields_kernel { + domain.fields.em, + init_flds, + domain.mesh.metric }); + /* tag particles inside the injection zone as dead From 3f4c5659a95c2d30af22530a8198676de6d44e9c Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 4 Apr 2025 15:40:17 -0400 Subject: [PATCH 162/176] conductor in all directions --- src/kernels/fields_bcs.hpp | 288 ++++++++++++++++++++++++++++++------- 1 file changed, 240 insertions(+), 48 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 8ea6b72d8..970ab559e 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -486,38 +486,52 @@ namespace kernel::bc { } }; - template + template struct ConductorBoundaries_kernel { static_assert(static_cast(o) < static_cast(D), "Invalid component index"); - ndfield_t Fld; - const BCTags tags; + ndfield_t Fld; + const BCTags tags; + const std::size_t i_edge; - ConductorBoundaries_kernel(ndfield_t Fld, BCTags tags) + ConductorBoundaries_kernel(ndfield_t Fld, std::size_t i_edge, BCTags tags) : Fld { Fld } + , i_edge { i_edge } , tags { tags } {} Inline void operator()(index_t i1) const { if constexpr (D == Dim::_1D) { if (tags & BC::E) { if (i1 == 0) { - Fld(N_GHOSTS, em::ex2) = ZERO; - Fld(N_GHOSTS, em::ex3) = ZERO; + Fld(i_edge, em::ex2) = ZERO; + Fld(i_edge, em::ex3) = ZERO; } else { - Fld(N_GHOSTS - i1, em::ex1) = Fld(N_GHOSTS + i1 - 1, em::ex1); - Fld(N_GHOSTS - i1, em::ex2) = -Fld(N_GHOSTS + i1, em::ex2); - Fld(N_GHOSTS - i1, em::ex3) = -Fld(N_GHOSTS + i1, em::ex3); + if constexpr (not P) { + Fld(i_edge - i1, em::ex1) = Fld(i_edge + i1 - 1, em::ex1); + Fld(i_edge - i1, em::ex2) = -Fld(i_edge + i1, em::ex2); + Fld(i_edge - i1, em::ex3) = -Fld(i_edge + i1, em::ex3); + } else { + Fld(i_edge + i1 - 1, em::ex1) = Fld(i_edge - i1, em::ex1); + Fld(i_edge + i1, em::ex2) = -Fld(i_edge - i1, em::ex2); + Fld(i_edge + i1, em::ex3) = -Fld(i_edge - i1, em::ex3); + } } } if (tags & BC::B) { if (i1 == 0) { - Fld(N_GHOSTS, em::bx1) = ZERO; + Fld(i_edge, em::bx1) = ZERO; } else { - Fld(N_GHOSTS - i1, em::bx1) = -Fld(N_GHOSTS + i1, em::bx1); - Fld(N_GHOSTS - i1, em::bx2) = Fld(N_GHOSTS + i1 - 1, em::bx2); - Fld(N_GHOSTS - i1, em::bx3) = Fld(N_GHOSTS + i1 - 1, em::bx3); + if constexpr (not P) { + Fld(i_edge - i1, em::bx1) = -Fld(i_edge + i1, em::bx1); + Fld(i_edge - i1, em::bx2) = Fld(i_edge + i1 - 1, em::bx2); + Fld(i_edge - i1, em::bx3) = Fld(i_edge + i1 - 1, em::bx3); + } else { + Fld(i_edge + i1, em::bx1) = -Fld(i_edge - i1, em::bx1); + Fld(i_edge + i1 - 1, em::bx2) = Fld(i_edge - i1, em::bx2); + Fld(i_edge + i1 - 1, em::bx3) = Fld(i_edge - i1, em::bx3); + } } } } else { @@ -529,24 +543,71 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2) const { if constexpr (D == Dim::_2D) { - if (tags & BC::E) { - if (i1 == 0) { - Fld(N_GHOSTS, i2, em::ex2) = ZERO; - Fld(N_GHOSTS, i2, em::ex3) = ZERO; - } else { - Fld(N_GHOSTS - i1, i2, em::ex1) = Fld(N_GHOSTS + i1 - 1, i2, em::ex1); - Fld(N_GHOSTS - i1, i2, em::ex2) = -Fld(N_GHOSTS + i1, i2, em::ex2); - Fld(N_GHOSTS - i1, i2, em::ex3) = -Fld(N_GHOSTS + i1, i2, em::ex3); + if constexpr (o == in::x1) { + if (tags & BC::E) { + if (i1 == 0) { + Fld(i_edge, i2, em::ex2) = ZERO; + Fld(i_edge, i2, em::ex3) = ZERO; + } else { + if constexpr (not P) { + Fld(i_edge - i1, i2, em::ex1) = Fld(i_edge + i1 - 1, i2, em::ex1); + Fld(i_edge - i1, i2, em::ex2) = -Fld(i_edge + i1, i2, em::ex2); + Fld(i_edge - i1, i2, em::ex3) = -Fld(i_edge + i1, i2, em::ex3); + } else { + Fld(i_edge + i1 - 1, i2, em::ex1) = Fld(i_edge - i1, i2, em::ex1); + Fld(i_edge + i1, i2, em::ex2) = -Fld(i_edge - i1, i2, em::ex2); + Fld(i_edge + i1, i2, em::ex3) = -Fld(i_edge - i1, i2, em::ex3); + } + } } - } - if (tags & BC::B) { - if (i1 == 0) { - Fld(N_GHOSTS, i2, em::bx1) = ZERO; - } else { - Fld(N_GHOSTS - i1, i2, em::bx1) = -Fld(N_GHOSTS + i1, i2, em::bx1); - Fld(N_GHOSTS - i1, i2, em::bx2) = Fld(N_GHOSTS + i1 - 1, i2, em::bx2); - Fld(N_GHOSTS - i1, i2, em::bx3) = Fld(N_GHOSTS + i1 - 1, i2, em::bx3); + if (tags & BC::B) { + if (i1 == 0) { + Fld(i_edge, i2, em::bx1) = ZERO; + } else { + if constexpr (not P) { + Fld(i_edge - i1, i2, em::bx1) = -Fld(i_edge + i1, i2, em::bx1); + Fld(i_edge - i1, i2, em::bx2) = Fld(i_edge + i1 - 1, i2, em::bx2); + Fld(i_edge - i1, i2, em::bx3) = Fld(i_edge + i1 - 1, i2, em::bx3); + } else { + Fld(i_edge + i1, i2, em::bx1) = -Fld(i_edge - i1, i2, em::bx1); + Fld(i_edge + i1 - 1, i2, em::bx2) = Fld(i_edge - i1, i2, em::bx2); + Fld(i_edge + i1 - 1, i2, em::bx3) = Fld(i_edge - i1, i2, em::bx3); + } + } + } + } else { + if (tags & BC::E) { + if (i2 == 0) { + Fld(i1, i_edge, em::ex1) = ZERO; + Fld(i1, i_edge, em::ex3) = ZERO; + } else { + if constexpr (not P) { + Fld(i1, i_edge - i2, em::ex1) = -Fld(i1, i_edge + i2, em::ex1); + Fld(i1, i_edge - i2, em::ex2) = Fld(i1, i_edge + i2 - 1, em::ex2); + Fld(i1, i_edge - i2, em::ex3) = -Fld(i1, i_edge + i2, em::ex3); + } else { + Fld(i1, i_edge + i2, em::ex1) = -Fld(i1, i_edge - i2, em::ex1); + Fld(i1, i_edge + i2 - 1, em::ex2) = Fld(i1, i_edge - i2, em::ex2); + Fld(i1, i_edge + i2, em::ex3) = -Fld(i1, i_edge - i2, em::ex3); + } + } + } + + if (tags & BC::B) { + if (i2 == 0) { + Fld(i1, i_edge, em::bx2) = ZERO; + } else { + if constexpr (not P) { + Fld(i1, i_edge - i2, em::bx1) = Fld(i1, i_edge + i2 - 1, em::bx1); + Fld(i1, i_edge - i2, em::bx2) = -Fld(i1, i_edge + i2, em::bx2); + Fld(i1, i_edge - i2, em::bx3) = Fld(i1, i_edge + i2 - 1, em::bx3); + } else { + Fld(i1, i_edge + i2 - 1, em::bx1) = Fld(i1, i_edge - i2, em::bx1); + Fld(i1, i_edge + i2, em::bx2) = -Fld(i1, i_edge - i2, em::bx2); + Fld(i1, i_edge + i2 - 1, em::bx3) = Fld(i1, i_edge - i2, em::bx3); + } + } } } } else { @@ -558,27 +619,158 @@ namespace kernel::bc { Inline void operator()(index_t i1, index_t i2, index_t i3) const { if constexpr (D == Dim::_3D) { - if (tags & BC::E) { - if (i1 == 0) { - Fld(N_GHOSTS, i2, i3, em::ex2) = ZERO; - Fld(N_GHOSTS, i2, i3, em::ex3) = ZERO; - } else { - Fld(N_GHOSTS - i1, i2, i3, em::ex1) = Fld(N_GHOSTS + i1 - 1, - i2, i3, em::ex1); - Fld(N_GHOSTS - i1, i2, i3, em::ex2) = -Fld(N_GHOSTS + i1, i2, i3, em::ex2); - Fld(N_GHOSTS - i1, i2, i3, em::ex3) = -Fld(N_GHOSTS + i1, i2, i3, em::ex3); + if constexpr (o == in::x1) { + if (tags & BC::E) { + if (i1 == 0) { + Fld(i_edge, i2, i3, em::ex2) = ZERO; + Fld(i_edge, i2, i3, em::ex3) = ZERO; + } else { + if constexpr (not P) { + Fld(i_edge - i1, i2, i3, em::ex1) = Fld(i_edge + i1 - 1, + i2, + i3, + em::ex1); + Fld(i_edge - i1, i2, i3, em::ex2) = -Fld(i_edge + i1, i2, i3, em::ex2); + Fld(i_edge - i1, i2, i3, em::ex3) = -Fld(i_edge + i1, i2, i3, em::ex3); + } else { + Fld(i_edge + i1 - 1, i2, i3, em::ex1) = Fld(i_edge - i1, + i2, + i3, + em::ex1); + Fld(i_edge + i1, i2, i3, em::ex2) = -Fld(i_edge - i1, i2, i3, em::ex2); + Fld(i_edge + i1, i2, i3, em::ex3) = -Fld(i_edge - i1, i2, i3, em::ex3); + } + } } - } - if (tags & BC::B) { - if (i1 == 0) { - Fld(N_GHOSTS, i2, i3, em::bx1) = ZERO; - } else { - Fld(N_GHOSTS - i1, i2, i3, em::bx1) = -Fld(N_GHOSTS + i1, i2, i3, em::bx1); - Fld(N_GHOSTS - i1, i2, i3, em::bx2) = Fld(N_GHOSTS + i1 - 1, - i2, i3, em::bx2); - Fld(N_GHOSTS - i1, i2, i3, em::bx3) = Fld(N_GHOSTS + i1 - 1, - i2, i3, em::bx3); + if (tags & BC::B) { + if (i1 == 0) { + Fld(i_edge, i2, i3, em::bx1) = ZERO; + } else { + if constexpr (not P) { + Fld(i_edge - i1, i2, i3, em::bx1) = -Fld(i_edge + i1, i2, i3, em::bx1); + Fld(i_edge - i1, i2, i3, em::bx2) = Fld(i_edge + i1 - 1, + i2, + i3, + em::bx2); + Fld(i_edge - i1, i2, i3, em::bx3) = Fld(i_edge + i1 - 1, + i2, + i3, + em::bx3); + } else { + Fld(i_edge + i1, i2, i3, em::bx1) = -Fld(i_edge - i1, i2, i3, em::bx1); + Fld(i_edge + i1 - 1, i2, i3, em::bx2) = Fld(i_edge - i1, + i2, + i3, + em::bx2); + Fld(i_edge + i1 - 1, i2, i3, em::bx3) = Fld(i_edge - i1, + i2, + i3, + em::bx3); + } + } + } + } else if (o == in::x2) { + if (tags & BC::E) { + if (i2 == 0) { + Fld(i1, i_edge, i3, em::ex1) = ZERO; + Fld(i1, i_edge, i3, em::ex3) = ZERO; + } else { + if constexpr (not P) { + Fld(i1, i_edge - i2, i3, em::ex1) = -Fld(i1, i_edge + i2, i3, em::ex1); + Fld(i1, i_edge - i2, i3, em::ex2) = Fld(i1, + i_edge + i2 - 1, + i3, + em::ex2); + Fld(i1, i_edge - i2, i3, em::ex3) = -Fld(i1, i_edge + i2, i3, em::ex3); + } else { + Fld(i1, i_edge + i2, i3, em::ex1) = -Fld(i1, i_edge - i2, i3, em::ex1); + Fld(i1, i_edge + i2 - 1, i3, em::ex2) = Fld(i1, + i_edge - i2, + i3, + em::ex2); + Fld(i1, i_edge + i2, i3, em::ex3) = -Fld(i1, i_edge - i2, i3, em::ex3); + } + } + } + + if (tags & BC::B) { + if (i2 == 0) { + Fld(i1, i_edge, i3, em::bx2) = ZERO; + } else { + if constexpr (not P) { + Fld(i1, i_edge - i2, i3, em::bx1) = Fld(i1, + i_edge + i2 - 1, + i3, + em::bx1); + Fld(i1, i_edge - i2, i3, em::bx2) = -Fld(i1, i_edge + i2, i3, em::bx2); + Fld(i1, i_edge - i2, i3, em::bx3) = Fld(i1, + i_edge + i2 - 1, + i3, + em::bx3); + } else { + Fld(i1, i_edge + i2 - 1, i3, em::bx1) = Fld(i1, + i_edge - i2, + i3, + em::bx1); + Fld(i1, i_edge + i2, i3, em::bx2) = -Fld(i1, i_edge - i2, i3, em::bx2); + Fld(i1, i_edge + i2 - 1, i3, em::bx3) = Fld(i1, + i_edge - i2, + i3, + em::bx3); + } + } + } + } else { + if (tags & BC::E) { + if (i3 == 0) { + Fld(i1, i2, i_edge, em::ex1) = ZERO; + Fld(i1, i2, i_edge, em::ex2) = ZERO; + } else { + if constexpr (not P) { + Fld(i1, i2, i_edge - i3, em::ex1) = -Fld(i1, i2, i_edge + i3, em::ex1); + Fld(i1, i2, i_edge - i3, em::ex2) = -Fld(i1, i2, i_edge + i3, em::ex2); + Fld(i1, i2, i_edge - i3, em::ex3) = Fld(i1, + i2, + i_edge + i3 - 1, + em::ex3); + } else { + Fld(i1, i2, i_edge + i3, em::ex1) = -Fld(i1, i2, i_edge - i3, em::ex1); + Fld(i1, i2, i_edge + i3, em::ex2) = -Fld(i1, i2, i_edge - i3, em::ex2); + Fld(i1, i2, i_edge + i3 - 1, em::ex3) = Fld(i1, + i2, + i_edge - i3, + em::ex3); + } + } + } + + if (tags & BC::B) { + if (i3 == 0) { + Fld(i1, i2, i_edge, em::bx3) = ZERO; + } else { + if constexpr (not P) { + Fld(i1, i2, i_edge - i3, em::bx1) = Fld(i1, + i2, + i_edge + i3 - 1, + em::bx1); + Fld(i1, i2, i_edge - i3, em::bx2) = Fld(i1, + i2, + i_edge + i3 - 1, + em::bx2); + Fld(i1, i2, i_edge - i3, em::bx3) = -Fld(i1, i2, i_edge + i3, em::bx3); + } else { + Fld(i1, i2, i_edge + i3 - 1, em::bx1) = Fld(i1, + i2, + i_edge - i3, + em::bx1); + Fld(i1, i2, i_edge + i3 - 1, em::bx2) = Fld(i1, + i2, + i_edge - i3, + em::bx2); + Fld(i1, i2, i_edge + i3, em::bx3) = -Fld(i1, i2, i_edge - i3, em::bx3); + } + } } } } else { From 6ae91c91fc8176b01964589caffaab8e8225cb73 Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 4 Apr 2025 15:44:54 -0400 Subject: [PATCH 163/176] minor formatting --- src/engines/srpic.hpp | 72 ++++++++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 31 deletions(-) diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp index b1fca46da..3684fb573 100644 --- a/src/engines/srpic.hpp +++ b/src/engines/srpic.hpp @@ -651,7 +651,7 @@ namespace ntt { tuple_t range_min { 0 }; tuple_t range_max { 0 }; - for (unsigned short d { 0 }; d < M::Dim; ++d) { + for (auto d { 0u }; d < M::Dim; ++d) { range_min[d] = intersect_range[d].first; range_max[d] = intersect_range[d].second; } @@ -708,28 +708,32 @@ namespace ntt { /** * axis boundaries */ - raise::ErrorIf(M::CoordType == Coord::Cart, - "Invalid coordinate type for axis BCs", - HERE); - raise::ErrorIf(direction.get_dim() != in::x2, - "Invalid axis direction, should be x2", - HERE); - const auto i2_min = domain.mesh.i_min(in::x2); - const auto i2_max = domain.mesh.i_max(in::x2); - if (direction.get_sign() < 0) { - Kokkos::parallel_for( - "AxisBCFields", - domain.mesh.n_all(in::x1), - kernel::bc::AxisBoundaries_kernel(domain.fields.em, - i2_min, - tags)); + if constexpr (M::CoordType != Coord::Cart) { + raise::ErrorIf(direction.get_dim() != in::x2, + "Invalid axis direction, should be x2", + HERE); + const auto i2_min = domain.mesh.i_min(in::x2); + const auto i2_max = domain.mesh.i_max(in::x2); + if (direction.get_sign() < 0) { + Kokkos::parallel_for( + "AxisBCFields", + domain.mesh.n_all(in::x1), + kernel::bc::AxisBoundaries_kernel(domain.fields.em, + i2_min, + tags)); + } else { + Kokkos::parallel_for( + "AxisBCFields", + domain.mesh.n_all(in::x1), + kernel::bc::AxisBoundaries_kernel(domain.fields.em, + i2_max, + tags)); + } } else { - Kokkos::parallel_for( - "AxisBCFields", - domain.mesh.n_all(in::x1), - kernel::bc::AxisBoundaries_kernel(domain.fields.em, - i2_max, - tags)); + (void)direction; + (void)domain; + (void)tags; + raise::Error("Invalid coordinate type for axis BCs", HERE); } } @@ -834,6 +838,9 @@ namespace ntt { } } } else { + (void)direction; + (void)domain; + (void)tags; raise::Error("Fixed fields not present (both const and non-const)", HERE); } } @@ -844,14 +851,7 @@ namespace ntt { /** * perfect conductor field boundaries */ - if constexpr (M::CoordType != Coord::Cart) { - (void)direction; - (void)domain; - (void)tags; - raise::Error( - "Perfect conductor BCs only applicable to cartesian coordinates", - HERE); - } else { + if constexpr (M::CoordType == Coord::Cart) { const auto sign = direction.get_sign(); const auto dim = direction.get_dim(); @@ -859,7 +859,7 @@ namespace ntt { const std::vector all_dirs { in::x1, in::x2, in::x3 }; - for (unsigned short d { 0 }; d < static_cast(M::Dim); ++d) { + for (auto d { 0u }; d < static_cast(M::Dim); ++d) { const auto dd = all_dirs[d]; if (dim == dd) { xi_min.push_back(0); @@ -936,6 +936,13 @@ namespace ntt { tags)); } } + } else { + (void)direction; + (void)domain; + (void)tags; + raise::Error( + "Perfect conductor BCs only applicable to cartesian coordinates", + HERE); } } @@ -1060,6 +1067,9 @@ namespace ntt { raise::Error("Invalid dimension", HERE); } } else { + (void)direction; + (void)domain; + (void)tags; raise::Error("Atm fields not implemented in PGEN for atmosphere BCs", HERE); } } From a31c9c72dd36e6fec2e5428c81eb1e18bb6cbc93 Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 4 Apr 2025 16:50:03 -0400 Subject: [PATCH 164/176] rm TODOs --- src/framework/domain/comm_mpi.hpp | 1 - src/framework/domain/grid.cpp | 1 - src/framework/domain/metadomain.cpp | 1 - src/framework/simulation.cpp | 2 -- 4 files changed, 5 deletions(-) diff --git a/src/framework/domain/comm_mpi.hpp b/src/framework/domain/comm_mpi.hpp index 4f382be4a..159502e7a 100644 --- a/src/framework/domain/comm_mpi.hpp +++ b/src/framework/domain/comm_mpi.hpp @@ -220,7 +220,6 @@ namespace comm { if (recv_rank >= 0) { - // !TODO: perhaps directly recv to the fld? if (not additive) { if constexpr (D == Dim::_1D) { Kokkos::deep_copy(Kokkos::subview(fld, recv_slice[0], comps), recv_fld); diff --git a/src/framework/domain/grid.cpp b/src/framework/domain/grid.cpp index baa23fb5c..c022184b1 100644 --- a/src/framework/domain/grid.cpp +++ b/src/framework/domain/grid.cpp @@ -85,7 +85,6 @@ namespace ntt { return CreateRangePolicy(imin, imax); } - // !TODO: too ugly, implement a better solution (combine with device) template auto Grid::rangeCellsOnHost( const box_region_t& region) const -> range_h_t { diff --git a/src/framework/domain/metadomain.cpp b/src/framework/domain/metadomain.cpp index 10e3e4fb0..eec6c8fa0 100644 --- a/src/framework/domain/metadomain.cpp +++ b/src/framework/domain/metadomain.cpp @@ -234,7 +234,6 @@ namespace ntt { template void Metadomain::redefineBoundaries() { - // !TODO: not setting CommBC for now for (unsigned int idx { 0 }; idx < g_ndomains; ++idx) { // offset of the subdomain[idx] auto& current_domain = g_subdomains[idx]; diff --git a/src/framework/simulation.cpp b/src/framework/simulation.cpp index bea50ff09..560539c25 100644 --- a/src/framework/simulation.cpp +++ b/src/framework/simulation.cpp @@ -48,8 +48,6 @@ namespace ntt { HERE); m_requested_dimension = static_cast(res.size()); - // !TODO: when mixing checkpoint metadata with input, - // ... need to properly take care of the diffs m_params.setRawData(raw_params); timestep_t checkpoint_step = 0; if (is_resuming) { From 58a6214d728ad779b6809754856a7b286a223749 Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 4 Apr 2025 16:50:21 -0400 Subject: [PATCH 165/176] conductor filters in full 3D --- src/kernels/digital_filter.hpp | 225 +++++++++++++++++++++++++++------ 1 file changed, 185 insertions(+), 40 deletions(-) diff --git a/src/kernels/digital_filter.hpp b/src/kernels/digital_filter.hpp index 0ec6dcdbc..4f1385c16 100644 --- a/src/kernels/digital_filter.hpp +++ b/src/kernels/digital_filter.hpp @@ -25,6 +25,33 @@ INV_2*(ARR)((I), (J), (COMP)) + \ INV_4*((ARR)((I), (J) - 1, (COMP)) + (ARR)((I), (J) + 1, (COMP))) +#define FILTER3D_IN_I1_I2(ARR, COMP, I, J, K) \ + INV_4*(ARR)(I, J, K, (COMP)) + \ + INV_8*((ARR)((I) - 1, (J), (K), (COMP)) + (ARR)((I) + 1, (J), (K), (COMP)) + \ + (ARR)((I), (J) - 1, (K), (COMP)) + (ARR)((I), (J) + 1, (K), (COMP))) + \ + INV_16*((ARR)((I) - 1, (J) - 1, (K), (COMP)) + \ + (ARR)((I) + 1, (J) + 1, (K), (COMP)) + \ + (ARR)((I) - 1, (J) + 1, (K), (COMP)) + \ + (ARR)((I) + 1, (J) - 1, (K), (COMP))) + +#define FILTER3D_IN_I2_I3(ARR, COMP, I, J, K) \ + INV_4*(ARR)(I, J, K, (COMP)) + \ + INV_8*((ARR)((I), (J) - 1, (K), (COMP)) + (ARR)((I), (J) + 1, (K), (COMP)) + \ + (ARR)((I), (J), (K) - 1, (COMP)) + (ARR)((I), (J), (K) + 1, (COMP))) + \ + INV_16*((ARR)((I), (J) - 1, (K) - 1, (COMP)) + \ + (ARR)((I), (J) + 1, (K) + 1, (COMP)) + \ + (ARR)((I), (J) - 1, (K) + 1, (COMP)) + \ + (ARR)((I), (J) + 1, (K) - 1, (COMP))) + +#define FILTER3D_IN_I1_I3(ARR, COMP, I, J, K) \ + INV_4*(ARR)(I, J, K, (COMP)) + \ + INV_8*((ARR)((I) - 1, (J), (K), (COMP)) + (ARR)((I) + 1, (J), (K), (COMP)) + \ + (ARR)((I), (J), (K) - 1, (COMP)) + (ARR)((I), (J), (K) + 1, (COMP))) + \ + INV_16*((ARR)((I) - 1, (J), (K) - 1, (COMP)) + \ + (ARR)((I) + 1, (J), (K) + 1, (COMP)) + \ + (ARR)((I) - 1, (J), (K) + 1, (COMP)) + \ + (ARR)((I) + 1, (J), (K) - 1, (COMP))) + namespace kernel { using namespace ntt; @@ -36,9 +63,14 @@ namespace kernel { static constexpr auto i2_min = N_GHOSTS; const ncells_t i2_max; const bool is_axis_i2min, is_axis_i2max; - const bool is_conductor_i1min; - static constexpr auto i1_min = N_GHOSTS, i2_min = N_GHOSTS; - const std::size_t i2_max; + const bool is_conductor_i1min, is_conductor_i1max; + const bool is_conductor_i2min, is_conductor_i2max; + const bool is_conductor_i3min, is_conductor_i3max; + static constexpr auto i1_min = N_GHOSTS, i2_min = N_GHOSTS, i3_min = N_GHOSTS; + const std::size_t i1_max, i2_max, i3_max; + + // @TODO: Current implementation might have issues + // ... at the corners between two conductors public: DigitalFilter_kernel(ndfield_t& array, @@ -50,21 +82,40 @@ namespace kernel { , is_axis_i2min { (D == Dim::_2D) and (boundaries[1].first == FldsBC::AXIS) } , is_axis_i2max { (D == Dim::_2D) and (boundaries[1].second == FldsBC::AXIS) } , is_conductor_i1min { boundaries[0].first == FldsBC::CONDUCTOR } - , i2_max { (short)D > 1 ? size_[1] + N_GHOSTS : 0 } {} + , is_conductor_i1max { boundaries[0].second == FldsBC::CONDUCTOR } + , is_conductor_i2min { (short)D > 1 + ? (boundaries[1].first == FldsBC::CONDUCTOR) + : false } + , is_conductor_i2max { (short)D > 1 + ? (boundaries[1].second == FldsBC::CONDUCTOR) + : false } + , is_conductor_i3min { (short)D > 2 + ? (boundaries[2].first == FldsBC::CONDUCTOR) + : false } + , is_conductor_i3max { (short)D > 2 + ? (boundaries[2].second == FldsBC::CONDUCTOR) + : false } + , i1_max { size_[0] + N_GHOSTS } + , i2_max { (short)D > 1 ? (size_[1] + N_GHOSTS) : 0 } + , i3_max { (short)D > 2 ? (size_[2] + N_GHOSTS) : 0 } {} Inline void operator()(index_t i1) const { if constexpr ((D == Dim::_1D) && (C == Coord::Cart)) { - if (is_conductor_i1min and i1 == i1_min) { + if ((is_conductor_i1min and i1 == i1_min) or + (is_conductor_i1max and i1 == i1_max - 1)) { + const auto i1side = is_conductor_i1min ? (i1 + 1) : (i1 - 1); array(i1, cur::jx1) = (THREE * INV_4) * buffer(i1, cur::jx1) + - (INV_4)*buffer(i1 + 1, cur::jx1); - } else if (is_conductor_i1min and i1 == i1_min + 1) { + (INV_4)*buffer(i1side, cur::jx1); + } else if ((is_conductor_i1min and i1 == i1_min + 1) or + (is_conductor_i1max and i1 == i1_max - 2)) { + const auto i1side = is_conductor_i1min ? (i1 + 1) : (i1 - 1); array(i1, cur::jx1) = INV_2 * buffer(i1, cur::jx1) + INV_4 * (buffer(i1 - 1, cur::jx1) + buffer(i1 + 1, cur::jx1)); array(i1, cur::jx2) = (INV_2)*buffer(i1, cur::jx2) + - (INV_4)*buffer(i1 + 1, cur::jx2); + (INV_4)*buffer(i1side, cur::jx2); array(i1, cur::jx3) = (INV_2)*buffer(i1, cur::jx3) + - (INV_4)*buffer(i1 + 1, cur::jx3); + (INV_4)*buffer(i1side, cur::jx3); } else { #pragma unroll for (const auto& comp : { cur::jx1, cur::jx2, cur::jx3 }) { @@ -81,11 +132,15 @@ namespace kernel { Inline void operator()(index_t i1, index_t i2) const { if constexpr (D == Dim::_2D) { if constexpr (C == Coord::Cart) { - if (is_conductor_i1min and i1 == i1_min) { + if ((is_conductor_i1min and i1 == i1_min) or + (is_conductor_i1max and i1 == i1_max - 1)) { + const auto i1side = is_conductor_i1min ? (i1 + 1) : (i1 - 1); array(i1, i2, cur::jx1) = (THREE * INV_4) * (FILTER2D_IN_I2(buffer, cur::jx1, i1, i2)) + - (INV_4) * (FILTER2D_IN_I2(buffer, cur::jx1, i1 + 1, i2)); - } else if (is_conductor_i1min and i1 == i1_min + 1) { + (INV_4) * (FILTER2D_IN_I2(buffer, cur::jx1, i1side, i2)); + } else if ((is_conductor_i1min and i1 == i1_min + 1) or + (is_conductor_i1max and i1 == i1_max - 2)) { + const auto i1side = is_conductor_i1min ? (i1 + 1) : (i1 - 1); array(i1, i2, cur::jx1) = INV_2 * (FILTER2D_IN_I2(buffer, cur::jx1, i1, i2)) + @@ -96,12 +151,37 @@ namespace kernel { i2, cur::jx2) = INV_2 * (FILTER2D_IN_I2(buffer, cur::jx2, i1, i2)) + INV_4 * - (FILTER2D_IN_I2(buffer, cur::jx2, i1 + 1, i2)); + (FILTER2D_IN_I2(buffer, cur::jx2, i1side, i2)); array(i1, i2, cur::jx3) = INV_2 * (FILTER2D_IN_I2(buffer, cur::jx3, i1, i2)) + INV_4 * - (FILTER2D_IN_I2(buffer, cur::jx3, i1 + 1, i2)); + (FILTER2D_IN_I2(buffer, cur::jx3, i1side, i2)); + } else if ((is_conductor_i2min and i2 == i2_min) or + (is_conductor_i2max and i2 == i2_max - 1)) { + const auto i2side = is_conductor_i2min ? (i2 + 1) : (i2 - 1); + array(i1, i2, cur::jx2) = + (THREE * INV_4) * (FILTER2D_IN_I1(buffer, cur::jx2, i1, i2)) + + (INV_4) * (FILTER2D_IN_I1(buffer, cur::jx2, i1, i2side)); + } else if ((is_conductor_i2min and i2 == i2_min + 1) or + (is_conductor_i2max and i2 == i2_max - 2)) { + const auto i2side = is_conductor_i2min ? (i2 + 1) : (i2 - 1); + array(i1, + i2, + cur::jx1) = INV_2 * (FILTER2D_IN_I1(buffer, cur::jx1, i1, i2)) + + INV_4 * + (FILTER2D_IN_I1(buffer, cur::jx1, i1, i2side)); + array(i1, + i2, + cur::jx2) = INV_2 * (FILTER2D_IN_I1(buffer, cur::jx2, i1, i2)) + + INV_4 * + ((FILTER2D_IN_I1(buffer, cur::jx2, i1, i2 - 1)) + + (FILTER2D_IN_I1(buffer, cur::jx2, i1, i2 + 1))); + array(i1, + i2, + cur::jx3) = INV_2 * (FILTER2D_IN_I1(buffer, cur::jx3, i1, i2)) + + INV_4 * + (FILTER2D_IN_I1(buffer, cur::jx3, i1, i2side)); } else { #pragma unroll for (const auto comp : { cur::jx1, cur::jx2, cur::jx3 }) { @@ -212,33 +292,93 @@ namespace kernel { Inline void operator()(index_t i1, index_t i2, index_t i3) const { if constexpr (D == Dim::_3D) { if constexpr (C == Coord::Cart) { + if ((is_conductor_i1min and i1 == i1_min) or + (is_conductor_i1max and i1 == i1_max - 1)) { + const auto i1side = is_conductor_i1min ? (i1 + 1) : (i1 - 1); + array(i1, i2, i3, cur::jx1) = + (THREE * INV_4) * (FILTER3D_IN_I2_I3(buffer, cur::jx1, i1, i2, i3)) + + (INV_4) * (FILTER3D_IN_I2_I3(buffer, cur::jx1, i1side, i2, i3)); + } else if ((is_conductor_i1min and i1 == i1_min + 1) or + (is_conductor_i1max and i1 == i1_max - 2)) { + const auto i1side = is_conductor_i1min ? (i1 + 1) : (i1 - 1); + array(i1, i2, i3, cur::jx1) = + INV_2 * (FILTER3D_IN_I2_I3(buffer, cur::jx1, i1, i2, i3)) + + INV_4 * ((FILTER3D_IN_I2_I3(buffer, cur::jx1, i1 - 1, i2, i3)) + + (FILTER3D_IN_I2_I3(buffer, cur::jx1, i1 + 1, i2, i3))); + array(i1, i2, i3, cur::jx2) = + INV_2 * (FILTER3D_IN_I2_I3(buffer, cur::jx2, i1, i2, i3)) + + INV_4 * (FILTER3D_IN_I2_I3(buffer, cur::jx2, i1side, i2, i3)); + array(i1, i2, i3, cur::jx3) = + INV_2 * (FILTER3D_IN_I2_I3(buffer, cur::jx3, i1, i2, i3)) + + INV_4 * (FILTER3D_IN_I2_I3(buffer, cur::jx3, i1side, i2, i3)); + } else if ((is_conductor_i2min and i2 == i2_min) or + (is_conductor_i2max and i2 == i2_max - 1)) { + const auto i2side = is_conductor_i2min ? (i2 + 1) : (i2 - 1); + array(i1, i2, i3, cur::jx2) = + (THREE * INV_4) * (FILTER3D_IN_I1_I3(buffer, cur::jx2, i1, i2, i3)) + + (INV_4) * (FILTER3D_IN_I1_I3(buffer, cur::jx2, i1, i2side, i3)); + } else if ((is_conductor_i2min and i2 == i2_min + 1) or + (is_conductor_i2max and i2 == i2_max - 2)) { + const auto i2side = is_conductor_i2min ? (i2 + 1) : (i2 - 1); + array(i1, i2, i3, cur::jx1) = + INV_2 * (FILTER3D_IN_I1_I3(buffer, cur::jx1, i1, i2, i3)) + + INV_4 * (FILTER3D_IN_I1_I3(buffer, cur::jx1, i1, i2side, i3)); + array(i1, i2, i3, cur::jx2) = + INV_2 * (FILTER3D_IN_I1_I3(buffer, cur::jx2, i1, i2, i3)) + + INV_4 * ((FILTER3D_IN_I1_I3(buffer, cur::jx2, i1, i2 - 1, i3)) + + (FILTER3D_IN_I1_I3(buffer, cur::jx2, i1, i2 + 1, i3))); + array(i1, i2, i3, cur::jx3) = + INV_2 * (FILTER3D_IN_I1_I3(buffer, cur::jx3, i1, i2, i3)) + + INV_4 * (FILTER3D_IN_I1_I3(buffer, cur::jx3, i1, i2side, i3)); + } else if ((is_conductor_i3min and i3 == i3_min) or + (is_conductor_i3max and i3 == i3_max - 1)) { + const auto i3side = is_conductor_i3min ? (i3 + 1) : (i3 - 1); + array(i1, i2, i3, cur::jx3) = + (THREE * INV_4) * (FILTER3D_IN_I1_I2(buffer, cur::jx3, i1, i2, i3)) + + (INV_4) * (FILTER3D_IN_I1_I2(buffer, cur::jx3, i1, i2, i3side)); + } else if ((is_conductor_i3min and i3 == i3_min + 1) or + (is_conductor_i3max and i3 == i3_max - 2)) { + const auto i3side = is_conductor_i3min ? (i3 + 1) : (i3 - 1); + array(i1, i2, i3, cur::jx1) = + INV_2 * (FILTER3D_IN_I1_I2(buffer, cur::jx1, i1, i2, i3)) + + INV_4 * (FILTER3D_IN_I1_I2(buffer, cur::jx1, i1, i2, i3side)); + array(i1, i2, i3, cur::jx2) = + INV_2 * (FILTER3D_IN_I1_I2(buffer, cur::jx2, i1, i2, i3)) + + INV_4 * (FILTER3D_IN_I1_I2(buffer, cur::jx2, i1, i2, i3side)); + array(i1, i2, i3, cur::jx3) = + INV_2 * (FILTER3D_IN_I1_I2(buffer, cur::jx3, i1, i2, i3)) + + INV_4 * ((FILTER3D_IN_I1_I2(buffer, cur::jx3, i1, i2, i3 - 1)) + + (FILTER3D_IN_I1_I2(buffer, cur::jx3, i1, i2, i3 + 1))); + } else { #pragma unroll - for (auto& comp : { cur::jx1, cur::jx2, cur::jx3 }) { - array(i1, i2, i3, comp) = - INV_8 * buffer(i1, i2, i3, comp) + - INV_16 * - (buffer(i1 - 1, i2, i3, comp) + buffer(i1 + 1, i2, i3, comp) + - buffer(i1, i2 - 1, i3, comp) + buffer(i1, i2 + 1, i3, comp) + - buffer(i1, i2, i3 - 1, comp) + buffer(i1, i2, i3 + 1, comp)) + - INV_32 * - (buffer(i1 - 1, i2 - 1, i3, comp) + - buffer(i1 + 1, i2 + 1, i3, comp) + - buffer(i1 - 1, i2 + 1, i3, comp) + - buffer(i1 + 1, i2 - 1, i3, comp) + - buffer(i1, i2 - 1, i3 - 1, comp) + - buffer(i1, i2 + 1, i3 + 1, comp) + buffer(i1, i2, i3 - 1, comp) + - buffer(i1, i2, i3 + 1, comp) + buffer(i1 - 1, i2, i3 - 1, comp) + - buffer(i1 + 1, i2, i3 + 1, comp) + - buffer(i1 - 1, i2, i3 + 1, comp) + - buffer(i1 + 1, i2, i3 - 1, comp)) + - INV_64 * (buffer(i1 - 1, i2 - 1, i3 - 1, comp) + - buffer(i1 + 1, i2 + 1, i3 + 1, comp) + - buffer(i1 - 1, i2 + 1, i3 + 1, comp) + - buffer(i1 + 1, i2 - 1, i3 - 1, comp) + - buffer(i1 - 1, i2 - 1, i3 + 1, comp) + - buffer(i1 + 1, i2 + 1, i3 - 1, comp) + - buffer(i1 - 1, i2 + 1, i3 - 1, comp) + - buffer(i1 + 1, i2 - 1, i3 + 1, comp)); + for (auto& comp : { cur::jx1, cur::jx2, cur::jx3 }) { + array(i1, i2, i3, comp) = + INV_8 * buffer(i1, i2, i3, comp) + + INV_16 * + (buffer(i1 - 1, i2, i3, comp) + buffer(i1 + 1, i2, i3, comp) + + buffer(i1, i2 - 1, i3, comp) + buffer(i1, i2 + 1, i3, comp) + + buffer(i1, i2, i3 - 1, comp) + buffer(i1, i2, i3 + 1, comp)) + + INV_32 * + (buffer(i1 - 1, i2 - 1, i3, comp) + + buffer(i1 + 1, i2 + 1, i3, comp) + + buffer(i1 - 1, i2 + 1, i3, comp) + + buffer(i1 + 1, i2 - 1, i3, comp) + + buffer(i1, i2 - 1, i3 - 1, comp) + + buffer(i1, i2 + 1, i3 + 1, comp) + + buffer(i1, i2, i3 - 1, comp) + buffer(i1, i2, i3 + 1, comp) + + buffer(i1 - 1, i2, i3 - 1, comp) + + buffer(i1 + 1, i2, i3 + 1, comp) + + buffer(i1 - 1, i2, i3 + 1, comp) + + buffer(i1 + 1, i2, i3 - 1, comp)) + + INV_64 * (buffer(i1 - 1, i2 - 1, i3 - 1, comp) + + buffer(i1 + 1, i2 + 1, i3 + 1, comp) + + buffer(i1 - 1, i2 + 1, i3 + 1, comp) + + buffer(i1 + 1, i2 - 1, i3 - 1, comp) + + buffer(i1 - 1, i2 - 1, i3 + 1, comp) + + buffer(i1 + 1, i2 + 1, i3 - 1, comp) + + buffer(i1 - 1, i2 + 1, i3 - 1, comp) + + buffer(i1 + 1, i2 - 1, i3 + 1, comp)); + } } } else { raise::KernelNotImplementedError(HERE); @@ -253,6 +393,11 @@ namespace kernel { } // namespace kernel +#undef FILTER3D_IN_I1_I3 +#undef FILTER3D_IN_I2_I3 +#undef FILTER3D_IN_I1_I2 + +#undef FILTER2D_IN_I2 #undef FILTER2D_IN_I1 #endif // DIGITAL_FILTER_HPP From 0303acf48d29e64faeb982b0106fc37f08db2d0c Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 4 Apr 2025 17:26:12 -0400 Subject: [PATCH 166/176] rebase to 1.2.0rc --- src/checkpoint/tests/checkpoint-nompi.cpp | 4 ++-- src/framework/containers/particles.cpp | 2 +- src/kernels/digital_filter.hpp | 5 +---- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/checkpoint/tests/checkpoint-nompi.cpp b/src/checkpoint/tests/checkpoint-nompi.cpp index 23dbd8871..be5e97e24 100644 --- a/src/checkpoint/tests/checkpoint-nompi.cpp +++ b/src/checkpoint/tests/checkpoint-nompi.cpp @@ -105,8 +105,8 @@ auto main(int argc, char* argv[]) -> int { writer.saveField("em", field1); writer.saveField("em0", field2); - writer.savePerDomainVariable("s1_npart", 1, 0, npart1); - writer.savePerDomainVariable("s2_npart", 1, 0, npart2); + writer.savePerDomainVariable("s1_npart", 1, 0, npart1); + writer.savePerDomainVariable("s2_npart", 1, 0, npart2); writer.saveParticleQuantity("s1_i1", npart1, 0, npart1, i1); writer.saveParticleQuantity("s1_ux1", npart1, 0, npart1, u1); diff --git a/src/framework/containers/particles.cpp b/src/framework/containers/particles.cpp index 634a34862..9ec8810dc 100644 --- a/src/framework/containers/particles.cpp +++ b/src/framework/containers/particles.cpp @@ -218,7 +218,7 @@ namespace ntt { Kokkos::Experimental::fill( "TagAliveParticles", - AccelExeSpace(), + Kokkos::DefaultExecutionSpace(), Kokkos::subview(this_tag, std::make_pair(static_cast(0), n_alive)), ParticleTag::alive); diff --git a/src/kernels/digital_filter.hpp b/src/kernels/digital_filter.hpp index 4f1385c16..5ac60327d 100644 --- a/src/kernels/digital_filter.hpp +++ b/src/kernels/digital_filter.hpp @@ -59,15 +59,12 @@ namespace kernel { class DigitalFilter_kernel { ndfield_t array; const ndfield_t buffer; - bool is_axis_i2min { false }, is_axis_i2max { false }; - static constexpr auto i2_min = N_GHOSTS; - const ncells_t i2_max; const bool is_axis_i2min, is_axis_i2max; const bool is_conductor_i1min, is_conductor_i1max; const bool is_conductor_i2min, is_conductor_i2max; const bool is_conductor_i3min, is_conductor_i3max; static constexpr auto i1_min = N_GHOSTS, i2_min = N_GHOSTS, i3_min = N_GHOSTS; - const std::size_t i1_max, i2_max, i3_max; + const ncells_t i1_max, i2_max, i3_max; // @TODO: Current implementation might have issues // ... at the corners between two conductors From dd8a35b9ccf419bccbfdbb0c6e7f7a7890364b96 Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 4 Apr 2025 17:35:17 -0400 Subject: [PATCH 167/176] (RUNTEST) From cb34e94bb928c2e7b8df3cb8d191d3628377eefc Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 4 Apr 2025 17:50:03 -0400 Subject: [PATCH 168/176] mpi tests fixed (RUNTEST) --- src/checkpoint/tests/checkpoint-mpi.cpp | 4 ++-- src/output/tests/writer-mpi.cpp | 24 ++++++++++++------------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/checkpoint/tests/checkpoint-mpi.cpp b/src/checkpoint/tests/checkpoint-mpi.cpp index f97202ab1..bc6d6038a 100644 --- a/src/checkpoint/tests/checkpoint-mpi.cpp +++ b/src/checkpoint/tests/checkpoint-mpi.cpp @@ -144,8 +144,8 @@ auto main(int argc, char* argv[]) -> int { writer.saveField("em", field1); writer.saveField("em0", field2); - writer.savePerDomainVariable("s1_npart", 1, 0, npart1); - writer.savePerDomainVariable("s2_npart", 1, 0, npart2); + writer.savePerDomainVariable("s1_npart", 1, 0, npart1); + writer.savePerDomainVariable("s2_npart", 1, 0, npart2); writer.saveParticleQuantity("s1_i1", npart1_globtot, diff --git a/src/output/tests/writer-mpi.cpp b/src/output/tests/writer-mpi.cpp index f6d3ee88a..cc9208889 100644 --- a/src/output/tests/writer-mpi.cpp +++ b/src/output/tests/writer-mpi.cpp @@ -61,8 +61,8 @@ auto main(int argc, char* argv[]) -> int { // write auto writer = out::Writer(); writer.init(&adios, "hdf5", "test", false); - writer.defineMeshLayout({ static_cast(mpi_size) * nx1 }, - { static_cast(mpi_rank) * nx1 }, + writer.defineMeshLayout({ static_cast(mpi_size) * nx1 }, + { static_cast(mpi_rank) * nx1 }, { nx1 }, { dwn1 }, false, @@ -89,9 +89,9 @@ auto main(int argc, char* argv[]) -> int { { // read adios2::IO io = adios.DeclareIO("read-test"); - io.SetEngine("hdf5"); - adios2::Engine reader = io.Open("test.h5", adios2::Mode::Read, MPI_COMM_SELF); - raise::ErrorIf(io.InquireAttribute("NGhosts").Data()[0] != 0, + io.SetEngine("HDF5"); + adios2::Engine reader = io.Open("test.h5", adios2::Mode::Read); + raise::ErrorIf(io.InquireAttribute("NGhosts").Data()[0] != 0, "NGhosts is not correct", HERE); raise::ErrorIf(io.InquireAttribute("Dimension").Data()[0] != 1, @@ -99,13 +99,13 @@ auto main(int argc, char* argv[]) -> int { HERE); for (std::size_t step = 0; reader.BeginStep() == adios2::StepStatus::OK; ++step) { - std::size_t step_read; - long double time_read; + timestep_t step_read; + simtime_t time_read; - reader.Get(io.InquireVariable("Step"), + reader.Get(io.InquireVariable("Step"), &step_read, adios2::Mode::Sync); - reader.Get(io.InquireVariable("Time"), + reader.Get(io.InquireVariable("Time"), &time_read, adios2::Mode::Sync); raise::ErrorIf(step_read != step, "Step is not correct", HERE); @@ -122,9 +122,9 @@ auto main(int argc, char* argv[]) -> int { const double l = l_offset; const double f = math::ceil(l / d) * d - l; - const auto first_cell = static_cast(f); - const auto l_size_dwn = static_cast(math::ceil((n - f) / d)); - const auto l_corner_dwn = static_cast(math::ceil(l / d)); + const auto first_cell = static_cast(f); + const auto l_size_dwn = static_cast(math::ceil((n - f) / d)); + const auto l_corner_dwn = static_cast(math::ceil(l / d)); array_t field_read {}; int cntr = 0; From 0bc510f7c03c3897ec29249f3c8486005e3af125 Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 4 Apr 2025 17:56:58 -0400 Subject: [PATCH 169/176] added non-mpi cpu test action (RUNTEST) --- .github/workflows/actions.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index f60ee9061..a5c546328 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -27,9 +27,14 @@ jobs: matrix: device: [cpu, amd-gpu, nvidia-gpu] precision: [double, single] + mpi: [serial, parallel] exclude: # my AMD GPU doesn't support fp64 atomics : ( - device: amd-gpu precision: double + - device: amd-gpu + mpi: parallel + - device: nvidia-gpu + mpi: parallel runs-on: [self-hosted, "${{ matrix.device }}"] steps: - name: Checkout @@ -47,7 +52,7 @@ jobs: fi elif [ "${{ matrix.device }}" = "amd-gpu" ]; then FLAGS="-D Kokkos_ENABLE_HIP=ON -D Kokkos_ARCH_AMD_GFX1100=ON" - elif [ "${{ matrix.device }}" = "cpu" ]; then + elif [ "${{ matrix.mpi }}" = "parallel" ]; then FLAGS="-D mpi=ON" fi cmake -B build -D TESTS=ON -D output=ON -D precision=${{ matrix.precision }} $FLAGS From 6643066318a7174c707bada0dd26c7073df559c3 Mon Sep 17 00:00:00 2001 From: hayk Date: Tue, 1 Apr 2025 18:56:07 -0400 Subject: [PATCH 170/176] exespace/memspace -> kokkos default + cmake flags changed --- CMakeLists.txt | 23 +++--- cmake/MPIConfig.cmake | 4 -- cmake/adios2Config.cmake | 2 - cmake/defaults.cmake | 36 ---------- cmake/dependencies.cmake | 11 ++- cmake/kokkosConfig.cmake | 10 --- legacy/src/pic/particles/particle_pusher.hpp | 73 ++++++++++---------- legacy/tests/kernels-gr.cpp | 17 ++--- legacy/tests/kernels-sr.cpp | 7 +- src/CMakeLists.txt | 1 - src/checkpoint/writer.cpp | 6 +- src/engines/CMakeLists.txt | 1 - src/framework/CMakeLists.txt | 1 - src/framework/tests/CMakeLists.txt | 1 - src/global/arch/kokkos_aliases.h | 4 +- src/kernels/tests/prtls_to_phys.cpp | 2 +- 16 files changed, 78 insertions(+), 121 deletions(-) delete mode 100644 cmake/MPIConfig.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index dd22b9308..0409c4107 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -79,11 +79,10 @@ set(BUILD_TESTING CACHE BOOL "Build tests") # ------------------------ Third-party dependencies ------------------------ # -include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/kokkosConfig.cmake) include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/dependencies.cmake) -find_or_fetch_dependency(Kokkos FALSE) -find_or_fetch_dependency(plog TRUE) +find_or_fetch_dependency(Kokkos FALSE QUIET) +find_or_fetch_dependency(plog TRUE QUIET) set(DEPENDENCIES Kokkos::kokkos) include_directories(${plog_SRC}/include) @@ -92,14 +91,15 @@ set_precision(${precision}) # MPI if(${mpi}) - include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/MPIConfig.cmake) + find_or_fetch_dependency(MPI FALSE REQUIRED) + include_directories(${MPI_CXX_INCLUDE_PATH}) + add_compile_options("-D MPI_ENABLED") set(DEPENDENCIES ${DEPENDENCIES} MPI::MPI_CXX) endif() # Output if(${output}) - include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/adios2Config.cmake) - find_or_fetch_dependency(adios2 FALSE) + find_or_fetch_dependency(adios2 FALSE QUIET) if(NOT DEFINED ENV{HDF5_ROOT}) if(DEFINED ENV{CONDA_PREFIX}) execute_process(COMMAND bash -c "conda list | grep \"hdf5\" -q" @@ -110,7 +110,7 @@ if(${output}) endif() endif() find_package(HDF5 REQUIRED) - + add_compile_options("-D OUTPUT_ENABLED") if(${mpi}) set(DEPENDENCIES ${DEPENDENCIES} adios2::cxx11_mpi) else() @@ -120,6 +120,13 @@ endif() link_libraries(${DEPENDENCIES}) +get_cmake_property(all_vars VARIABLES) +foreach(_var ${all_vars}) + if(_var MATCHES "^Kokkos_.*") + message(STATUS "PRINTING ${_var}=${${_var}}") + endif() +endforeach() + if(TESTS) # ---------------------------------- Tests --------------------------------- # include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/tests.cmake) @@ -129,7 +136,7 @@ elseif(BENCHMARK) else() # ----------------------------------- GUI ---------------------------------- # if(${gui}) - find_or_fetch_dependency(nttiny FALSE) + find_or_fetch_dependency(nttiny FALSE QUIET) endif() # ------------------------------- Main source ------------------------------ # diff --git a/cmake/MPIConfig.cmake b/cmake/MPIConfig.cmake deleted file mode 100644 index d1bfeaab2..000000000 --- a/cmake/MPIConfig.cmake +++ /dev/null @@ -1,4 +0,0 @@ -find_package(MPI REQUIRED) -include_directories(${MPI_CXX_INCLUDE_PATH}) -add_compile_options("-D MPI_ENABLED") - diff --git a/cmake/adios2Config.cmake b/cmake/adios2Config.cmake index 5c480f3d8..7e0951949 100644 --- a/cmake/adios2Config.cmake +++ b/cmake/adios2Config.cmake @@ -23,5 +23,3 @@ set(ADIOS2_USE_MPI set(ADIOS2_USE_CUDA OFF CACHE BOOL "Use CUDA for ADIOS2") - -add_compile_options("-D OUTPUT_ENABLED") diff --git a/cmake/defaults.cmake b/cmake/defaults.cmake index 46b4609c5..78e8230c7 100644 --- a/cmake/defaults.cmake +++ b/cmake/defaults.cmake @@ -51,42 +51,6 @@ endif() set_property(CACHE default_gui PROPERTY TYPE BOOL) -if(DEFINED ENV{Kokkos_ENABLE_CUDA}) - set(default_KOKKOS_ENABLE_CUDA - $ENV{Kokkos_ENABLE_CUDA} - CACHE INTERNAL "Default flag for CUDA") -else() - set(default_KOKKOS_ENABLE_CUDA - OFF - CACHE INTERNAL "Default flag for CUDA") -endif() - -set_property(CACHE default_KOKKOS_ENABLE_CUDA PROPERTY TYPE BOOL) - -if(DEFINED ENV{Kokkos_ENABLE_HIP}) - set(default_KOKKOS_ENABLE_HIP - $ENV{Kokkos_ENABLE_HIP} - CACHE INTERNAL "Default flag for HIP") -else() - set(default_KOKKOS_ENABLE_HIP - OFF - CACHE INTERNAL "Default flag for HIP") -endif() - -set_property(CACHE default_KOKKOS_ENABLE_HIP PROPERTY TYPE BOOL) - -if(DEFINED ENV{Kokkos_ENABLE_OPENMP}) - set(default_KOKKOS_ENABLE_OPENMP - $ENV{Kokkos_ENABLE_OPENMP} - CACHE INTERNAL "Default flag for OpenMP") -else() - set(default_KOKKOS_ENABLE_OPENMP - OFF - CACHE INTERNAL "Default flag for OpenMP") -endif() - -set_property(CACHE default_KOKKOS_ENABLE_OPENMP PROPERTY TYPE BOOL) - if(DEFINED ENV{Entity_ENABLE_MPI}) set(default_mpi $ENV{Entity_ENABLE_MPI} diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index 06a3e6a1f..a6712feb2 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -34,12 +34,17 @@ function(check_internet_connection) endif() endfunction() -function(find_or_fetch_dependency package_name header_only) +function(find_or_fetch_dependency package_name header_only mode) if(NOT header_only) - find_package(${package_name} QUIET) + find_package(${package_name} ${mode}) endif() if(NOT ${package_name}_FOUND) + if (${package_name} STREQUAL "Kokkos") + include(${CMAKE_CURRENT_SOURCE_DIR}/kokkosConfig.cmake) + elseif(${package_name} STREQUAL "adios2") + include(${CMAKE_CURRENT_SOURCE_DIR}/adios2Config.cmake) + endif() if(DEFINED ${package_name}_REPOSITORY AND NOT FETCHCONTENT_FULLY_DISCONNECTED) # fetching package @@ -52,7 +57,7 @@ function(find_or_fetch_dependency package_name header_only) FetchContent_Declare( ${package_name} GIT_REPOSITORY ${${package_name}_REPOSITORY} - GIT_TAG 4.3.00) + GIT_TAG 4.5.01) else() FetchContent_Declare(${package_name} GIT_REPOSITORY ${${package_name}_REPOSITORY}) diff --git a/cmake/kokkosConfig.cmake b/cmake/kokkosConfig.cmake index 8800f21a0..3d038cc19 100644 --- a/cmake/kokkosConfig.cmake +++ b/cmake/kokkosConfig.cmake @@ -27,16 +27,6 @@ else() CACHE BOOL "Kokkos debug bounds check") endif() -set(Kokkos_ENABLE_HIP - ${default_KOKKOS_ENABLE_HIP} - CACHE BOOL "Enable HIP") -set(Kokkos_ENABLE_CUDA - ${default_KOKKOS_ENABLE_CUDA} - CACHE BOOL "Enable CUDA") -set(Kokkos_ENABLE_OPENMP - ${default_KOKKOS_ENABLE_OPENMP} - CACHE BOOL "Enable OpenMP") - if(${BUILD_TESTING} STREQUAL "OFF") set(Kokkos_ENABLE_TESTS OFF diff --git a/legacy/src/pic/particles/particle_pusher.hpp b/legacy/src/pic/particles/particle_pusher.hpp index 4c0ec639a..7991a95a4 100644 --- a/legacy/src/pic/particles/particle_pusher.hpp +++ b/legacy/src/pic/particles/particle_pusher.hpp @@ -1,14 +1,13 @@ #ifndef PIC_PARTICLE_PUSHER_H #define PIC_PARTICLE_PUSHER_H -#include "wrapper.h" - -#include "pic.h" +#include "utils/qmath.h" #include "io/output.h" #include "meshblock/meshblock.h" #include "meshblock/particles.h" -#include "utils/qmath.h" +#include "pic.h" +#include "wrapper.h" #include METRIC_HEADER #include @@ -73,35 +72,34 @@ namespace ntt { real_t time, real_t coeff, real_t dt, - ProblemGenerator& pgen) : - EB { mblock.em }, - i1 { particles.i1 }, - i2 { particles.i2 }, - i3 { particles.i3 }, - i1_prev { particles.i1_prev }, - i2_prev { particles.i2_prev }, - i3_prev { particles.i3_prev }, - dx1 { particles.dx1 }, - dx2 { particles.dx2 }, - dx3 { particles.dx3 }, - dx1_prev { particles.dx1_prev }, - dx2_prev { particles.dx2_prev }, - dx3_prev { particles.dx3_prev }, - ux1 { particles.ux1 }, - ux2 { particles.ux2 }, - ux3 { particles.ux3 }, - phi { particles.phi }, - tag { particles.tag }, - metric { mblock.metric }, - time { time }, - coeff { coeff }, - dt { dt }, - ni1 { (int)mblock.Ni1() }, - ni2 { (int)mblock.Ni2() }, - ni3 { (int)mblock.Ni3() } + ProblemGenerator& pgen) + : EB { mblock.em } + , i1 { particles.i1 } + , i2 { particles.i2 } + , i3 { particles.i3 } + , i1_prev { particles.i1_prev } + , i2_prev { particles.i2_prev } + , i3_prev { particles.i3_prev } + , dx1 { particles.dx1 } + , dx2 { particles.dx2 } + , dx3 { particles.dx3 } + , dx1_prev { particles.dx1_prev } + , dx2_prev { particles.dx2_prev } + , dx3_prev { particles.dx3_prev } + , ux1 { particles.ux1 } + , ux2 { particles.ux2 } + , ux3 { particles.ux3 } + , phi { particles.phi } + , tag { particles.tag } + , metric { mblock.metric } + , time { time } + , coeff { coeff } + , dt { dt } + , ni1 { (int)mblock.Ni1() } + , ni2 { (int)mblock.Ni2() } + , ni3 { (int)mblock.Ni3() } #ifdef EXTERNAL_FORCE - , - pgen { pgen } + , pgen { pgen } #endif { (void)pgen; @@ -237,7 +235,7 @@ namespace ntt { const auto coeff = charge_ovr_mass * HALF * dt * params.B0(); Kokkos::parallel_for( "ParticlesPush", - Kokkos::RangePolicy(0, particles.npart()), + Kokkos::RangePolicy(0, particles.npart()), Pusher_kernel(mblock, particles, time, coeff, dt, pgen)); } @@ -638,9 +636,9 @@ namespace ntt { template template Inline void Pusher_kernel::get3VelCntrv(T, - index_t& p, + index_t& p, vec_t& xp, - vec_t& v) const { + vec_t& v) const { metric.v3_Cart2Cntrv(xp, { ux1(p), ux2(p), ux3(p) }, v); auto inv_energy { ONE / getEnergy(T {}, p) }; v[0] *= inv_energy; @@ -666,7 +664,8 @@ namespace ntt { } template <> - Inline void Pusher_kernel::getPrtlPos(index_t& p, coord_t& xp) const { + Inline void Pusher_kernel::getPrtlPos(index_t& p, + coord_t& xp) const { xp[0] = static_cast(i1(p)) + static_cast(dx1(p)); xp[1] = static_cast(i2(p)) + static_cast(dx2(p)); xp[2] = phi(p); @@ -1066,7 +1065,7 @@ namespace ntt { #else template Inline void Pusher_kernel::initForce(coord_t& xp, - vec_t& force_Cart) const { + vec_t& force_Cart) const { coord_t xp_Ph { ZERO }; coord_t xp_Code { ZERO }; for (short d { 0 }; d < static_cast(PrtlCoordD); ++d) { diff --git a/legacy/tests/kernels-gr.cpp b/legacy/tests/kernels-gr.cpp index 84a0c952b..6962f7c9f 100644 --- a/legacy/tests/kernels-gr.cpp +++ b/legacy/tests/kernels-gr.cpp @@ -1,16 +1,16 @@ -#include "wrapper.h" - #include #include #include -#include METRIC_HEADER +#include "wrapper.h" -#include "particle_macros.h" +#include METRIC_HEADER #include "kernels/particle_pusher_gr.hpp" +#include "particle_macros.h" + template void put_value(ntt::array_t& arr, T value, int i) { auto arr_h = Kokkos::create_mirror_view(arr); @@ -154,9 +154,10 @@ auto main(int argc, char* argv[]) -> int { static_cast(1.0e-5), 10, boundaries); - Kokkos::parallel_for("ParticlesPush", - Kokkos::RangePolicy(0, 1), - kernel); + Kokkos::parallel_for( + "ParticlesPush", + Kokkos::RangePolicy(0, 1), + kernel); auto [ra, tha] = get_physical_coord(0, i1, i2, dx1, dx2, metric); const real_t pha = get_value(phi, 0); @@ -207,4 +208,4 @@ auto main(int argc, char* argv[]) -> int { ntt::GlobalFinalize(); return 0; -} \ No newline at end of file +} diff --git a/legacy/tests/kernels-sr.cpp b/legacy/tests/kernels-sr.cpp index d765799e3..3f64122cd 100644 --- a/legacy/tests/kernels-sr.cpp +++ b/legacy/tests/kernels-sr.cpp @@ -181,9 +181,10 @@ auto main(int argc, char* argv[]) -> int { ZERO, ZERO, ZERO); - Kokkos::parallel_for("ParticlesPush", - Kokkos::RangePolicy(0, 1), - kernel); + Kokkos::parallel_for( + "ParticlesPush", + Kokkos::RangePolicy(0, 1), + kernel); auto [xa, ya] = get_cartesian_coord(0, i1, i2, dx1, dx2, phi, metric); if (!ntt::AlmostEqual(xa, diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a41b84900..db2ab4c92 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -17,7 +17,6 @@ # # * kokkos [required] # * plog [required] -# * toml11 [required] # * ADIOS2 [optional] # * mpi [optional] # ------------------------------ diff --git a/src/checkpoint/writer.cpp b/src/checkpoint/writer.cpp index a70d8de09..fe77d3b56 100644 --- a/src/checkpoint/writer.cpp +++ b/src/checkpoint/writer.cpp @@ -274,9 +274,9 @@ namespace checkpoint { std::size_t, double); template void Writer::savePerDomainVariable(const std::string&, - std::size_t, - std::size_t, - npart_t); + std::size_t, + std::size_t, + npart_t); template void Writer::saveField(const std::string&, const ndfield_t&); diff --git a/src/engines/CMakeLists.txt b/src/engines/CMakeLists.txt index 6da2f4efd..9e51330ec 100644 --- a/src/engines/CMakeLists.txt +++ b/src/engines/CMakeLists.txt @@ -25,7 +25,6 @@ # # * kokkos [required] # * plog [required] -# * toml11 [required] # * adios2 [optional] # * hdf5 [optional] # * mpi [optional] diff --git a/src/framework/CMakeLists.txt b/src/framework/CMakeLists.txt index 8802f696b..d3b68084c 100644 --- a/src/framework/CMakeLists.txt +++ b/src/framework/CMakeLists.txt @@ -28,7 +28,6 @@ # # * kokkos [required] # * plog [required] -# * toml11 [required] # * ADIOS2 [optional] # * mpi [optional] # ------------------------------ diff --git a/src/framework/tests/CMakeLists.txt b/src/framework/tests/CMakeLists.txt index ce188e9f1..27c456610 100644 --- a/src/framework/tests/CMakeLists.txt +++ b/src/framework/tests/CMakeLists.txt @@ -5,7 +5,6 @@ # # * kokkos [required] # * plog [required] -# * toml11 [required] # * mpi [optional] # * adios2 [optional] # diff --git a/src/global/arch/kokkos_aliases.h b/src/global/arch/kokkos_aliases.h index adb0b6451..712fc6eff 100644 --- a/src/global/arch/kokkos_aliases.h +++ b/src/global/arch/kokkos_aliases.h @@ -234,8 +234,8 @@ auto CreateParticleRangePolicy(npart_t, npart_t) -> range_t; * @returns Kokkos::RangePolicy or Kokkos::MDRangePolicy in the accelerator execution space. */ template -auto CreateRangePolicy(const tuple_t&, - const tuple_t&) -> range_t; +auto CreateRangePolicy(const tuple_t&, const tuple_t&) + -> range_t; /** * @brief Function template for generating ND Kokkos range policy on the host. diff --git a/src/kernels/tests/prtls_to_phys.cpp b/src/kernels/tests/prtls_to_phys.cpp index 0ceb88cd1..962c21b5c 100644 --- a/src/kernels/tests/prtls_to_phys.cpp +++ b/src/kernels/tests/prtls_to_phys.cpp @@ -132,7 +132,7 @@ void testPrtl2PhysSR(const std::vector& res, extent = { ext[0], - {ZERO, constant::PI} + { ZERO, constant::PI } }; const M metric { res, extent, params }; From 908c08aaa8bec4de1ebbb22d3764307a06683164 Mon Sep 17 00:00:00 2001 From: hayk Date: Wed, 2 Apr 2025 01:27:21 -0400 Subject: [PATCH 171/176] improved report + fetch adios2 + rm hdf5 dependency --- .gitmodules | 1 - CMakeLists.txt | 17 -- cmake/adios2Config.cmake | 10 +- cmake/config.cmake | 9 +- cmake/defaults.cmake | 2 +- cmake/dependencies.cmake | 31 ++- cmake/report.cmake | 366 +++++++++------------------------ cmake/styling.cmake | 140 +++++++++++-- src/engines/CMakeLists.txt | 2 +- src/engines/engine_printer.cpp | 9 - 10 files changed, 259 insertions(+), 328 deletions(-) diff --git a/.gitmodules b/.gitmodules index e06c332fe..577d08ea6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,7 +4,6 @@ [submodule "extern/adios2"] path = extern/adios2 url = https://github.com/ornladios/ADIOS2.git - branch = master [submodule "extern/Kokkos"] path = extern/Kokkos url = https://github.com/kokkos/kokkos.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 0409c4107..22d1e64b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,16 +100,6 @@ endif() # Output if(${output}) find_or_fetch_dependency(adios2 FALSE QUIET) - if(NOT DEFINED ENV{HDF5_ROOT}) - if(DEFINED ENV{CONDA_PREFIX}) - execute_process(COMMAND bash -c "conda list | grep \"hdf5\" -q" - RESULT_VARIABLE HDF5_INSTALLED) - if(HDF5_INSTALLED EQUAL 0) - set(HDF5_ROOT $ENV{CONDA_PREFIX}) - endif() - endif() - endif() - find_package(HDF5 REQUIRED) add_compile_options("-D OUTPUT_ENABLED") if(${mpi}) set(DEPENDENCIES ${DEPENDENCIES} adios2::cxx11_mpi) @@ -120,13 +110,6 @@ endif() link_libraries(${DEPENDENCIES}) -get_cmake_property(all_vars VARIABLES) -foreach(_var ${all_vars}) - if(_var MATCHES "^Kokkos_.*") - message(STATUS "PRINTING ${_var}=${${_var}}") - endif() -endforeach() - if(TESTS) # ---------------------------------- Tests --------------------------------- # include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/tests.cmake) diff --git a/cmake/adios2Config.cmake b/cmake/adios2Config.cmake index 7e0951949..a4ce46179 100644 --- a/cmake/adios2Config.cmake +++ b/cmake/adios2Config.cmake @@ -12,14 +12,18 @@ set(ADIOS2_USE_Fortran CACHE BOOL "Use Fortran for ADIOS2") # Format/compression support -set(ADIOS2_USE_ZeroMQ - OFF - CACHE BOOL "Use ZeroMQ for ADIOS2") +set(ADIOS2_USE_HDF5 + ON + CACHE BOOL "Use HDF5 for ADIOS2") set(ADIOS2_USE_MPI ${mpi} CACHE BOOL "Use MPI for ADIOS2") +set(ADIOS2_USE_ZeroMQ + OFF + CACHE BOOL "Use ZeroMQ for ADIOS2") + set(ADIOS2_USE_CUDA OFF CACHE BOOL "Use CUDA for ADIOS2") diff --git a/cmake/config.cmake b/cmake/config.cmake index 58dd467e9..ab54717f1 100644 --- a/cmake/config.cmake +++ b/cmake/config.cmake @@ -34,13 +34,10 @@ function(set_problem_generator pgen_name) ) endif() set(PGEN - ${pgen_name} - PARENT_SCOPE) + ${pgen_name} PARENT_SCOPE) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/setups/${pgen_name}) set(PGEN_FOUND - TRUE - PARENT_SCOPE) + TRUE PARENT_SCOPE) set(problem_generators - ${PGEN_NAMES} - PARENT_SCOPE) + ${PGEN_NAMES} PARENT_SCOPE) endfunction() diff --git a/cmake/defaults.cmake b/cmake/defaults.cmake index 78e8230c7..2c72f5d7d 100644 --- a/cmake/defaults.cmake +++ b/cmake/defaults.cmake @@ -33,7 +33,7 @@ if(DEFINED ENV{Entity_ENABLE_OUTPUT}) CACHE INTERNAL "Default flag for output") else() set(default_output - OFF + ON CACHE INTERNAL "Default flag for output") endif() diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index a6712feb2..a31dfdea5 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -4,9 +4,10 @@ set(Kokkos_REPOSITORY set(plog_REPOSITORY https://github.com/SergiusTheBest/plog.git CACHE STRING "plog repository") +set (adios2_REPOSITORY + https://github.com/ornladios/ADIOS2.git + CACHE STRING "ADIOS2 repository") -# set (adios2_REPOSITORY https://github.com/ornladios/ADIOS2.git CACHE STRING -# "ADIOS2 repository") function(check_internet_connection) if(OFFLINE STREQUAL "ON") set(FETCHCONTENT_FULLY_DISCONNECTED @@ -41,9 +42,9 @@ function(find_or_fetch_dependency package_name header_only mode) if(NOT ${package_name}_FOUND) if (${package_name} STREQUAL "Kokkos") - include(${CMAKE_CURRENT_SOURCE_DIR}/kokkosConfig.cmake) + include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/kokkosConfig.cmake) elseif(${package_name} STREQUAL "adios2") - include(${CMAKE_CURRENT_SOURCE_DIR}/adios2Config.cmake) + include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/adios2Config.cmake) endif() if(DEFINED ${package_name}_REPOSITORY AND NOT FETCHCONTENT_FULLY_DISCONNECTED) @@ -70,6 +71,9 @@ function(find_or_fetch_dependency package_name header_only mode) set(${package_name}_SRC ${CMAKE_CURRENT_BINARY_DIR}/_deps/${lower_pckg_name}-src CACHE PATH "Path to ${package_name} src") + set(${package_name}_BUILD_DIR + ${CMAKE_CURRENT_BINARY_DIR}/_deps/${lower_pckg_name}-build + CACHE PATH "Path to ${package_name} build") set(${package_name}_FETCHED TRUE CACHE BOOL "Whether ${package_name} was fetched") @@ -96,7 +100,7 @@ function(find_or_fetch_dependency package_name header_only mode) endif() add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extern/${package_name} - extern/${package_name}) + extern/${package_name}) set(${package_name}_SRC ${CMAKE_CURRENT_SOURCE_DIR}/extern/${package_name} CACHE PATH "Path to ${package_name} src") @@ -132,7 +136,24 @@ function(find_or_fetch_dependency package_name header_only mode) ${Kokkos_VERSION} CACHE INTERNAL "Kokkos version") endif() + if(NOT DEFINED Kokkos_ARCH OR Kokkos_ARCH STREQUAL "" + OR NOT DEFINED Kokkos_DEVICES OR Kokkos_DEVICES STREQUAL "") + if(${Kokkos_FOUND}) + include(${Kokkos_DIR}/KokkosConfigCommon.cmake) + elseif(NOT ${Kokkos_BUILD_DIR} STREQUAL "") + include(${Kokkos_BUILD_DIR}/KokkosConfigCommon.cmake) + else() + message(STATUS "${Red}Kokkos_DIR and Kokkos_BUILD_DIR not set.${ColorReset}") + endif() + endif() + set(Kokkos_ARCH + ${Kokkos_ARCH} PARENT_SCOPE) + set(Kokkos_DEVICES + ${Kokkos_DEVICES} PARENT_SCOPE) endif() + set(${package_name}_FOUND ${${package_name}_FOUND} PARENT_SCOPE) + set(${package_name}_FETCHED ${${package_name}_FETCHED} PARENT_SCOPE) + set(${package_name}_BUILD_DIR ${${package_name}_BUILD_DIR} PARENT_SCOPE) endfunction() check_internet_connection() diff --git a/cmake/report.cmake b/cmake/report.cmake index 13dde63f7..60d846144 100644 --- a/cmake/report.cmake +++ b/cmake/report.cmake @@ -1,95 +1,3 @@ -function(PadTo Text Padding Target Result) - set(rt ${Text}) - string(FIND ${rt} "${Magenta}" mg_fnd) - - if(mg_fnd GREATER -1) - string(REGEX REPLACE "${Esc}\\[35m" "" rt ${rt}) - endif() - - string(LENGTH "${rt}" TextLength) - math(EXPR PaddingNeeded "${Target} - ${TextLength}") - set(rt ${Text}) - - if(PaddingNeeded GREATER 0) - foreach(i RANGE 0 ${PaddingNeeded}) - set(rt "${rt}${Padding}") - endforeach() - else() - set(${rt} "${rt}") - endif() - - set(${Result} - "${rt}" - PARENT_SCOPE) -endfunction() - -function( - PrintChoices - Label - Flag - Choices - Value - Default - Color - OutputString - Multiline - Padding) - list(LENGTH "${Choices}" nchoices) - set(rstring "") - set(counter 0) - - foreach(ch ${Choices}) - if(${counter} EQUAL 0) - set(rstring_i "- ${Label}") - - if(NOT "${Flag}" STREQUAL "") - set(rstring_i "${rstring_i} [${Magenta}${Flag}${ColorReset}]") - endif() - - set(rstring_i "${rstring_i}:") - padto("${rstring_i}" " " ${Padding} rstring_i) - else() - set(rstring_i "") - - if(NOT ${counter} EQUAL ${nchoices}) - if(${Multiline} EQUAL 1) - set(rstring_i "${rstring_i}\n") - padto("${rstring_i}" " " ${Padding} rstring_i) - else() - set(rstring_i "${rstring_i}/") - endif() - endif() - endif() - - if(${ch} STREQUAL ${Value}) - if(${ch} STREQUAL "ON") - set(col ${Green}) - elseif(${ch} STREQUAL "OFF") - set(col ${Red}) - else() - set(col ${Color}) - endif() - else() - set(col ${Dim}) - endif() - - if(${ch} STREQUAL ${Default}) - set(col ${Underline}${col}) - endif() - - set(rstring_i "${rstring_i}${col}${ch}${ColorReset}") - math(EXPR counter "${counter} + 1") - set(rstring "${rstring}${rstring_i}") - set(rstring_i "") - endforeach() - - set(${OutputString} - "${rstring}" - PARENT_SCOPE) -endfunction() - -set(ON_OFF_VALUES "ON" "OFF") - if(${PGEN_FOUND}) printchoices( "Problem generator" @@ -99,8 +7,7 @@ if(${PGEN_FOUND}) ${default_pgen} "${Blue}" PGEN_REPORT - 1 - 36) + 0) endif() printchoices( @@ -111,7 +18,6 @@ printchoices( ${default_precision} "${Blue}" PRECISION_REPORT - 1 36) printchoices( "Output" @@ -121,17 +27,6 @@ printchoices( ${default_output} "${Green}" OUTPUT_REPORT - 0 - 36) -printchoices( - "GUI" - "gui" - "${ON_OFF_VALUES}" - ${gui} - ${default_gui} - "${Green}" - GUI_REPORT - 0 36) printchoices( "MPI" @@ -141,8 +36,7 @@ printchoices( OFF "${Green}" MPI_REPORT - 0 - 42) + 36) printchoices( "Debug mode" "DEBUG" @@ -151,131 +45,7 @@ printchoices( OFF "${Green}" DEBUG_REPORT - 0 - 42) - -printchoices( - "CUDA" - "Kokkos_ENABLE_CUDA" - "${ON_OFF_VALUES}" - ${Kokkos_ENABLE_CUDA} - "OFF" - "${Green}" - CUDA_REPORT - 0 - 42) -printchoices( - "HIP" - "Kokkos_ENABLE_HIP" - "${ON_OFF_VALUES}" - ${Kokkos_ENABLE_HIP} - "OFF" - "${Green}" - HIP_REPORT - 0 - 42) -printchoices( - "OpenMP" - "Kokkos_ENABLE_OPENMP" - "${ON_OFF_VALUES}" - ${Kokkos_ENABLE_OPENMP} - "OFF" - "${Green}" - OPENMP_REPORT - 0 - 42) - -printchoices( - "C++ compiler" - "CMAKE_CXX_COMPILER" - "${CMAKE_CXX_COMPILER} v${CMAKE_CXX_COMPILER_VERSION}" - "${CMAKE_CXX_COMPILER} v${CMAKE_CXX_COMPILER_VERSION}" - "N/A" - "${ColorReset}" - CXX_COMPILER_REPORT - 0 - 42) - -printchoices( - "C compiler" - "CMAKE_C_COMPILER" - "${CMAKE_C_COMPILER} v${CMAKE_C_COMPILER_VERSION}" - "${CMAKE_C_COMPILER} v${CMAKE_C_COMPILER_VERSION}" - "N/A" - "${ColorReset}" - C_COMPILER_REPORT - 0 - 42) - -get_cmake_property(_variableNames VARIABLES) -foreach(_variableName ${_variableNames}) - string(REGEX MATCH "Kokkos_ARCH_*" _isMatched ${_variableName}) - if(_isMatched) - get_property( - isSet - CACHE ${_variableName} - PROPERTY VALUE) - if(isSet STREQUAL "ON") - string(REGEX REPLACE "Kokkos_ARCH_" "" ARCH ${_variableName}) - break() - endif() - endif() -endforeach() -printchoices( - "Architecture" - "Kokkos_ARCH_*" - "${ARCH}" - "${ARCH}" - "N/A" - "${ColorReset}" - ARCH_REPORT - 0 - 42) - -if(${Kokkos_ENABLE_CUDA}) - if("${CMAKE_CUDA_COMPILER}" STREQUAL "") - execute_process(COMMAND which nvcc OUTPUT_VARIABLE CUDACOMP) - else() - set(CUDACOMP ${CMAKE_CUDA_COMPILER}) - endif() - - string(STRIP ${CUDACOMP} CUDACOMP) - - message(STATUS "CUDA compiler: ${CUDACOMP}") - execute_process( - COMMAND - bash -c - "${CUDACOMP} --version | grep release | sed -e 's/.*release //' -e 's/,.*//'" - OUTPUT_VARIABLE CUDACOMP_VERSION - OUTPUT_STRIP_TRAILING_WHITESPACE) - - printchoices( - "CUDA compiler" - "CMAKE_CUDA_COMPILER" - "${CUDACOMP}" - "${CUDACOMP}" - "N/A" - "${ColorReset}" - CUDA_COMPILER_REPORT - 0 - 42) -endif() - -if(${Kokkos_ENABLE_HIP}) - execute_process( - COMMAND bash -c "hipcc --version | grep HIP | cut -d ':' -f 2 | tr -d ' '" - OUTPUT_VARIABLE ROCM_VERSION - OUTPUT_STRIP_TRAILING_WHITESPACE) -endif() - -set(DOT_SYMBOL "${ColorReset}.") -set(DOTTED_LINE_SYMBOL - "${ColorReset}. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " -) - -set(DASHED_LINE_SYMBOL - "${ColorReset}....................................................................... " -) + 36) if(NOT ${PROJECT_VERSION_TWEAK} EQUAL 0) set(VERSION_SYMBOL @@ -287,7 +57,7 @@ else() ) endif() -message( +set(REPORT_TEXT "${Blue} __ __ /\\ \\__ __/\\ \\__ __ ___\\ \\ _\\/\\_\\ \\ _\\ __ __ @@ -296,55 +66,107 @@ message( \\ \\____\\ \\_\\ \\_\\ \\__\\\\ \\_\\ \\__\\\\ \\____ \\/\\_\\ \\/____/\\/_/\\/_/\\/__/ \\/_/\\/__/ \\/___/ \\/_/ /\\___/ -Entity ${VERSION_SYMBOL}\t\t \\/__/") -message("${DASHED_LINE_SYMBOL} -Main configurations") +Entity ${VERSION_SYMBOL}\t\t \\/__/" +) +string(APPEND REPORT_TEXT + ${ColorReset} "\n" +) + +string(APPEND REPORT_TEXT + ${DASHED_LINE_SYMBOL} "\n" + "Configurations" "\n" +) if(${PGEN_FOUND}) - message(" ${PGEN_REPORT}") + string(APPEND REPORT_TEXT + " " ${PGEN_REPORT} "\n" + ) endif() -message(" ${PRECISION_REPORT}") -message(" ${OUTPUT_REPORT}") -message("${DASHED_LINE_SYMBOL}\nCompile configurations") +string(APPEND REPORT_TEXT + " " ${PRECISION_REPORT} "\n" + " " ${OUTPUT_REPORT} "\n" +) -if(NOT "${ARCH_REPORT}" STREQUAL "") - message(" ${ARCH_REPORT}") -endif() -message(" ${CUDA_REPORT}") -message(" ${HIP_REPORT}") -message(" ${OPENMP_REPORT}") +string(REPLACE ";" "+" Kokkos_ARCH "${Kokkos_ARCH}") +string(REPLACE ";" "+" Kokkos_DEVICES "${Kokkos_DEVICES}") -message(" ${C_COMPILER_REPORT}") +string(APPEND REPORT_TEXT + " - ARCH [${Magenta}Kokkos_ARCH_***${ColorReset}]: ${Kokkos_ARCH}" "\n" + " - DEVICES [${Magenta}Kokkos_ENABLE_***${ColorReset}]: ${Kokkos_DEVICES}" "\n" + " " ${MPI_REPORT} "\n" + " " ${DEBUG_REPORT} "\n" + ${DASHED_LINE_SYMBOL} "\n" + "Compilers & dependencies" "\n" +) -message(" ${CXX_COMPILER_REPORT}") +string(APPEND REPORT_TEXT + " - C compiler [${Magenta}CMAKE_C_COMPILER${ColorReset}]: v" ${CMAKE_C_COMPILER_VERSION} "\n" + " ${Dim}" ${CMAKE_C_COMPILER} "${ColorReset}\n" + " - C++ compiler [${Magenta}CMAKE_CXX_COMPILER${ColorReset}]: v" ${CMAKE_CXX_COMPILER_VERSION} "\n" + " ${Dim}" ${CMAKE_CXX_COMPILER} "${ColorReset}\n" +) -if(NOT "${CUDA_COMPILER_REPORT}" STREQUAL "") - message(" ${CUDA_COMPILER_REPORT}") +if(${Kokkos_DEVICES} MATCHES "CUDA") + if("${CMAKE_CUDA_COMPILER}" STREQUAL "") + execute_process(COMMAND which nvcc OUTPUT_VARIABLE CUDACOMP) + else() + set(CUDACOMP ${CMAKE_CUDA_COMPILER}) + endif() + string(STRIP ${CUDACOMP} CUDACOMP) + execute_process( + COMMAND + bash -c + "${CUDACOMP} --version | grep release | sed -e 's/.*release //' -e 's/,.*//'" + OUTPUT_VARIABLE CUDACOMP_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE) + string(APPEND REPORT_TEXT + " - CUDA compiler: v" ${CUDACOMP_VERSION} "\n" + " ${Dim}" ${CUDACOMP} "${ColorReset}\n" + ) +elseif(${Kokkos_DEVICES} MATCHES "HIP") + execute_process( + COMMAND bash -c "hipcc --version | grep HIP | cut -d ':' -f 2 | tr -d ' '" + OUTPUT_VARIABLE ROCM_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE) + string(APPEND REPORT_TEXT + " - ROCm: v" ${ROCM_VERSION} "\n" + ) endif() -message(" ${MPI_REPORT}") - -message(" ${DEBUG_REPORT}") - -message("${DASHED_LINE_SYMBOL}\nDependencies") - -if(NOT "${CUDACOMP_VERSION}" STREQUAL "") - message(" - CUDA:\tv${CUDACOMP_VERSION}") -elseif(NOT "${ROCM_VERSION}" STREQUAL "") - message(" - ROCm:\tv${ROCM_VERSION}") +string(APPEND REPORT_TEXT + " - Kokkos: v" ${Kokkos_VERSION} "\n" +) +if(${Kokkos_FOUND}) + string(APPEND REPORT_TEXT + " " ${Kokkos_DIR} "\n" + ) +else() + string(APPEND REPORT_TEXT + " " ${Kokkos_BUILD_DIR} "\n" + ) endif() -message(" - Kokkos:\tv${Kokkos_VERSION}") + if(${output}) - message(" - ADIOS2:\tv${adios2_VERSION}") -endif() -if(${HDF5_FOUND}) - message(" - HDF5:\tv${HDF5_VERSION}") + string(APPEND REPORT_TEXT + " - ADIOS2: v" ${adios2_VERSION} "\n" + ) + if(${adios2_FOUND}) + string(APPEND REPORT_TEXT + " " ${adios2_DIR} "\n" + ) + else() + string(APPEND REPORT_TEXT + " " ${adios2_BUILD_DIR} "\n" + ) + endif() endif() -message( - "${DASHED_LINE_SYMBOL} -Notes - ${Dim}: Set flags with `cmake ... -D ${Magenta}${ColorReset}${Dim}=`, the ${Underline}default${ColorReset}${Dim} value - : will be used unless the variable is explicitly set.${ColorReset} -") +string(APPEND REPORT_TEXT + ${DASHED_LINE_SYMBOL} "\n" + "Notes" "\n" + " ${Dim}: Set flags with `cmake ... -D ${Magenta}${ColorReset}${Dim}=`, the ${Underline}default${ColorReset}${Dim} value" "\n" + " : will be used unless the variable is explicitly set.${ColorReset}" "\n" +) + +message(${REPORT_TEXT}) diff --git a/cmake/styling.cmake b/cmake/styling.cmake index 70c448fff..0da4b3519 100644 --- a/cmake/styling.cmake +++ b/cmake/styling.cmake @@ -23,17 +23,131 @@ if(NOT WIN32) set(StrikeEnd "${Esc}[0m") endif() -# message("This is normal") message("${Red}This is Red${ColorReset}") -# message("${Green}This is Green${ColorReset}") message("${Yellow}This is -# Yellow${ColorReset}") message("${Blue}This is Blue${ColorReset}") -# message("${Magenta}This is Magenta${ColorReset}") message("${Cyan}This is -# Cyan${ColorReset}") message("${White}This is White${ColorReset}") -# message("${BoldRed}This is BoldRed${ColorReset}") message("${BoldGreen}This is -# BoldGreen${ColorReset}") message("${BoldYellow}This is -# BoldYellow${ColorReset}") message("${BoldBlue}This is BoldBlue${ColorReset}") -# message("${BoldMagenta}This is BoldMagenta${ColorReset}") -# message("${BoldCyan}This is BoldCyan${ColorReset}") message("${BoldWhite}This -# is BoldWhite\n\n${ColorReset}") - -# message() +set(DOTTED_LINE_SYMBOL + "${ColorReset}. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " +) +set(DASHED_LINE_SYMBOL + "${ColorReset}....................................................................... " +) + +set(ON_OFF_VALUES "ON" "OFF") + +function(PureLength Text Result) + set(rt ${Text}) + string(FIND ${rt} "${Magenta}" mg_fnd) + + if(mg_fnd GREATER -1) + string(REGEX REPLACE "${Esc}\\[35m" "" rt ${rt}) + endif() + + string(LENGTH "${rt}" TextLength) + set(${Result} + "${TextLength}" + PARENT_SCOPE) +endfunction() + +function(PadTo Text Padding Target Result) + purelength("${Text}" TextLength) + math(EXPR PaddingNeeded "${Target} - ${TextLength}") + set(rt ${Text}) + + if(PaddingNeeded GREATER 0) + foreach(i RANGE 0 ${PaddingNeeded}) + set(rt "${rt}${Padding}") + endforeach() + else() + set(rt "${rt}") + endif() + + set(${Result} + "${rt}" + PARENT_SCOPE) +endfunction() + +function( + PrintChoices + Label + Flag + Choices + Value + Default + Color + OutputString + Padding) + set(rstring "- ${Label}") + + if(NOT "${Flag}" STREQUAL "") + string(APPEND rstring " [${Magenta}${Flag}${ColorReset}]") + endif() + + string(APPEND rstring ":") + + if(${Padding} EQUAL 0) + list(LENGTH "${Choices}" nchoices) + math(EXPR lastchoice "${nchoices} - 1") + set(ncols 4) + math(EXPR lastcol "${ncols} - 1") + + set(longest 0) + foreach(ch IN LISTS Choices) + string(LENGTH ${ch} clen) + if(clen GREATER longest) + set(longest ${clen}) + endif() + endforeach() + + set(counter 0) + foreach(ch IN LISTS Choices) + if(${ch} STREQUAL ${Value}) + set(col ${Color}) + else() + set(col ${Dim}) + endif() + + if(${ch} STREQUAL ${Default}) + set(col ${Underline}${col}) + endif() + + string(LENGTH "${ch}" clen) + math(EXPR PaddingNeeded "${longest} - ${clen} + 4") + + if(counter EQUAL ${lastcol} AND NOT ${counter} EQUAL ${lastchoice}) + string(APPEND rstring "${col}~ ${ch}${ColorReset}") + else() + if(counter EQUAL 0) + string(APPEND rstring "\n ") + endif() + string(APPEND rstring "${col}~ ${ch}${ColorReset}") + foreach(i RANGE 0 ${PaddingNeeded}) + string(APPEND rstring " ") + endforeach() + endif() + + math(EXPR counter "(${counter} + 1) % ${ncols}") + endforeach() + else() + padto("${rstring}" " " ${Padding} rstring) + + set(new_choices ${Choices}) + foreach(ch IN LISTS new_choices) + string(REPLACE ${ch} "${Dim}${ch}${ColorReset}" new_choices "${new_choices}") + endforeach() + set(Choices ${new_choices}) + if(${Value} STREQUAL "ON") + set(col ${Green}) + elseif(${Value} STREQUAL "OFF") + set(col ${Red}) + else() + set(col ${Color}) + endif() + string(REPLACE ${Value} "${col}${Value}${ColorReset}" Choices "${Choices}") + string(REPLACE ${Default} "${Underline}${Default}${ColorReset}" Choices "${Choices}") + string(REPLACE ";" "/" Choices "${Choices}") + string(APPEND rstring "${Choices}") + endif() + + set(${OutputString} + "${rstring}" + PARENT_SCOPE) +endfunction() diff --git a/src/engines/CMakeLists.txt b/src/engines/CMakeLists.txt index 9e51330ec..d5b6b0664 100644 --- a/src/engines/CMakeLists.txt +++ b/src/engines/CMakeLists.txt @@ -38,7 +38,7 @@ add_library(ntt_engines ${SOURCES}) set(libs ntt_global ntt_framework ntt_metrics ntt_archetypes ntt_kernels ntt_pgen) if(${output}) - list(APPEND libs ntt_output hdf5::hdf5) + list(APPEND libs ntt_output) endif() add_dependencies(ntt_engines ${libs}) target_link_libraries(ntt_engines PUBLIC ${libs}) diff --git a/src/engines/engine_printer.cpp b/src/engines/engine_printer.cpp index f94715d09..eb8ff402d 100644 --- a/src/engines/engine_printer.cpp +++ b/src/engines/engine_printer.cpp @@ -20,7 +20,6 @@ #endif #if defined(OUTPUT_ENABLED) - #include #include #endif @@ -188,18 +187,11 @@ namespace ntt { KOKKOS_VERSION % 100); #if defined(OUTPUT_ENABLED) - unsigned h5_major, h5_minor, h5_release; - H5get_libversion(&h5_major, &h5_minor, &h5_release); - const std::string hdf5_version = fmt::format("%d.%d.%d", - h5_major, - h5_minor, - h5_release); const std::string adios2_version = fmt::format("%d.%d.%d", ADIOS2_VERSION / 10000, ADIOS2_VERSION / 100 % 100, ADIOS2_VERSION % 100); #else // not OUTPUT_ENABLED - const std::string hdf5_version = "OFF"; const std::string adios2_version = "OFF"; #endif @@ -217,7 +209,6 @@ namespace ntt { add_param(report, 4, "CXX", "%s [%s]", ccx.c_str(), cpp_standard.c_str()); add_param(report, 4, "CUDA", "%s", cuda_version.c_str()); add_param(report, 4, "MPI", "%s", mpi_version.c_str()); - add_param(report, 4, "HDF5", "%s", hdf5_version.c_str()); add_param(report, 4, "Kokkos", "%s", kokkos_version.c_str()); add_param(report, 4, "ADIOS2", "%s", adios2_version.c_str()); add_param(report, 4, "Precision", "%s", precision); From 9e22301345e5bc865231b5d75c8d1114409c1067 Mon Sep 17 00:00:00 2001 From: haykh Date: Thu, 3 Apr 2025 20:33:03 -0400 Subject: [PATCH 172/176] cmake prints report with TESTS --- CMakeLists.txt | 3 +- cmake/report.cmake | 170 ++++++++++++++++++++++++++++---------------- cmake/styling.cmake | 31 +++++--- cmake/tests.cmake | 50 ++++++------- 4 files changed, 153 insertions(+), 101 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 22d1e64b5..4791ea55b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -125,5 +125,6 @@ else() # ------------------------------- Main source ------------------------------ # set_problem_generator(${pgen}) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src src) - include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/report.cmake) endif() + +include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/report.cmake) diff --git a/cmake/report.cmake b/cmake/report.cmake index 60d846144..21064cdf9 100644 --- a/cmake/report.cmake +++ b/cmake/report.cmake @@ -8,6 +8,24 @@ if(${PGEN_FOUND}) "${Blue}" PGEN_REPORT 0) +elseif(${TESTS}) + set(TEST_NAMES "") + foreach(test_dir IN LISTS TEST_DIRECTORIES) + get_property( + LOCAL_TEST_NAMES + DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${test_dir}/tests + PROPERTY TESTS) + list(APPEND TEST_NAMES ${LOCAL_TEST_NAMES}) + endforeach() + printchoices( + "Test cases" + "" + "${TEST_NAMES}" + "" + "${ColorReset}" + "" + TESTS_REPORT + 0) endif() printchoices( @@ -58,7 +76,7 @@ else() endif() set(REPORT_TEXT - "${Blue} __ __ + "${Blue} __ __ /\\ \\__ __/\\ \\__ __ ___\\ \\ _\\/\\_\\ \\ _\\ __ __ / __ \\ / __ \\ \\ \\/\\/\\ \\ \\ \\/ /\\ \\/\\ \\ @@ -66,46 +84,63 @@ set(REPORT_TEXT \\ \\____\\ \\_\\ \\_\\ \\__\\\\ \\_\\ \\__\\\\ \\____ \\/\\_\\ \\/____/\\/_/\\/_/\\/__/ \\/_/\\/__/ \\/___/ \\/_/ /\\___/ -Entity ${VERSION_SYMBOL}\t\t \\/__/" -) -string(APPEND REPORT_TEXT - ${ColorReset} "\n" -) +Entity ${VERSION_SYMBOL}\t\t \\/__/") +string(APPEND REPORT_TEXT ${ColorReset} "\n") -string(APPEND REPORT_TEXT - ${DASHED_LINE_SYMBOL} "\n" - "Configurations" "\n" -) +string(APPEND REPORT_TEXT ${DASHED_LINE_SYMBOL} "\n" "Configurations" "\n") if(${PGEN_FOUND}) - string(APPEND REPORT_TEXT - " " ${PGEN_REPORT} "\n" - ) + string(APPEND REPORT_TEXT " " ${PGEN_REPORT} "\n") +elseif(${TESTS}) + string(APPEND REPORT_TEXT " " ${TESTS_REPORT} "\n") endif() -string(APPEND REPORT_TEXT - " " ${PRECISION_REPORT} "\n" - " " ${OUTPUT_REPORT} "\n" -) +string( + APPEND + REPORT_TEXT + " " + ${PRECISION_REPORT} + "\n" + " " + ${OUTPUT_REPORT} + "\n") string(REPLACE ";" "+" Kokkos_ARCH "${Kokkos_ARCH}") string(REPLACE ";" "+" Kokkos_DEVICES "${Kokkos_DEVICES}") -string(APPEND REPORT_TEXT - " - ARCH [${Magenta}Kokkos_ARCH_***${ColorReset}]: ${Kokkos_ARCH}" "\n" - " - DEVICES [${Magenta}Kokkos_ENABLE_***${ColorReset}]: ${Kokkos_DEVICES}" "\n" - " " ${MPI_REPORT} "\n" - " " ${DEBUG_REPORT} "\n" - ${DASHED_LINE_SYMBOL} "\n" - "Compilers & dependencies" "\n" -) - -string(APPEND REPORT_TEXT - " - C compiler [${Magenta}CMAKE_C_COMPILER${ColorReset}]: v" ${CMAKE_C_COMPILER_VERSION} "\n" - " ${Dim}" ${CMAKE_C_COMPILER} "${ColorReset}\n" - " - C++ compiler [${Magenta}CMAKE_CXX_COMPILER${ColorReset}]: v" ${CMAKE_CXX_COMPILER_VERSION} "\n" - " ${Dim}" ${CMAKE_CXX_COMPILER} "${ColorReset}\n" -) +string( + APPEND + REPORT_TEXT + " - ARCH [${Magenta}Kokkos_ARCH_***${ColorReset}]: ${Kokkos_ARCH}" + "\n" + " - DEVICES [${Magenta}Kokkos_ENABLE_***${ColorReset}]: ${Kokkos_DEVICES}" + "\n" + " " + ${MPI_REPORT} + "\n" + " " + ${DEBUG_REPORT} + "\n" + ${DASHED_LINE_SYMBOL} + "\n" + "Compilers & dependencies" + "\n") + +string( + APPEND + REPORT_TEXT + " - C compiler [${Magenta}CMAKE_C_COMPILER${ColorReset}]: v" + ${CMAKE_C_COMPILER_VERSION} + "\n" + " ${Dim}" + ${CMAKE_C_COMPILER} + "${ColorReset}\n" + " - C++ compiler [${Magenta}CMAKE_CXX_COMPILER${ColorReset}]: v" + ${CMAKE_CXX_COMPILER_VERSION} + "\n" + " ${Dim}" + ${CMAKE_CXX_COMPILER} + "${ColorReset}\n") if(${Kokkos_DEVICES} MATCHES "CUDA") if("${CMAKE_CUDA_COMPILER}" STREQUAL "") @@ -120,53 +155,62 @@ if(${Kokkos_DEVICES} MATCHES "CUDA") "${CUDACOMP} --version | grep release | sed -e 's/.*release //' -e 's/,.*//'" OUTPUT_VARIABLE CUDACOMP_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) - string(APPEND REPORT_TEXT - " - CUDA compiler: v" ${CUDACOMP_VERSION} "\n" - " ${Dim}" ${CUDACOMP} "${ColorReset}\n" - ) + string( + APPEND + REPORT_TEXT + " - CUDA compiler: v" + ${CUDACOMP_VERSION} + "\n" + " ${Dim}" + ${CUDACOMP} + "${ColorReset}\n") elseif(${Kokkos_DEVICES} MATCHES "HIP") execute_process( COMMAND bash -c "hipcc --version | grep HIP | cut -d ':' -f 2 | tr -d ' '" OUTPUT_VARIABLE ROCM_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) - string(APPEND REPORT_TEXT - " - ROCm: v" ${ROCM_VERSION} "\n" - ) + string(APPEND REPORT_TEXT " - ROCm: v" ${ROCM_VERSION} "\n") endif() -string(APPEND REPORT_TEXT - " - Kokkos: v" ${Kokkos_VERSION} "\n" -) +string(APPEND REPORT_TEXT " - Kokkos: v" ${Kokkos_VERSION} "\n") if(${Kokkos_FOUND}) - string(APPEND REPORT_TEXT - " " ${Kokkos_DIR} "\n" - ) + string(APPEND REPORT_TEXT " " ${Dim}${Kokkos_DIR}${ColorReset} "\n") else() - string(APPEND REPORT_TEXT - " " ${Kokkos_BUILD_DIR} "\n" - ) + string(APPEND REPORT_TEXT " " ${Dim}${Kokkos_BUILD_DIR}${ColorReset} "\n") endif() if(${output}) - string(APPEND REPORT_TEXT - " - ADIOS2: v" ${adios2_VERSION} "\n" - ) + string(APPEND REPORT_TEXT " - ADIOS2: v" ${adios2_VERSION} "\n") if(${adios2_FOUND}) - string(APPEND REPORT_TEXT - " " ${adios2_DIR} "\n" - ) + string(APPEND REPORT_TEXT " " "${Dim}${adios2_DIR}${ColorReset}" "\n") else() - string(APPEND REPORT_TEXT - " " ${adios2_BUILD_DIR} "\n" - ) + string(APPEND REPORT_TEXT " " "${Dim}${adios2_BUILD_DIR}${ColorReset}" + "\n") endif() endif() -string(APPEND REPORT_TEXT - ${DASHED_LINE_SYMBOL} "\n" - "Notes" "\n" - " ${Dim}: Set flags with `cmake ... -D ${Magenta}${ColorReset}${Dim}=`, the ${Underline}default${ColorReset}${Dim} value" "\n" - " : will be used unless the variable is explicitly set.${ColorReset}" "\n" -) +string( + APPEND + REPORT_TEXT + ${DASHED_LINE_SYMBOL} + "\n" + "Notes" + "\n" + " ${Dim}: Set flags with `cmake ... -D ${Magenta}${ColorReset}${Dim}=`, the ${Underline}default${ColorReset}${Dim} value" + "\n" + " : will be used unless the variable is explicitly set.${ColorReset}") + +if(${TESTS}) + string( + APPEND + REPORT_TEXT + "\n" + " ${Dim}: Run the tests with the following command:" + "\n" + " : ctest --test-dir ${CMAKE_CURRENT_BINARY_DIR}${ColorReset}" + "\n") +endif() + +string(APPEND REPORT_TEXT "\n") message(${REPORT_TEXT}) diff --git a/cmake/styling.cmake b/cmake/styling.cmake index 0da4b3519..b9b72abcb 100644 --- a/cmake/styling.cmake +++ b/cmake/styling.cmake @@ -43,8 +43,8 @@ function(PureLength Text Result) string(LENGTH "${rt}" TextLength) set(${Result} - "${TextLength}" - PARENT_SCOPE) + "${TextLength}" + PARENT_SCOPE) endfunction() function(PadTo Text Padding Target Result) @@ -99,14 +99,20 @@ function( set(counter 0) foreach(ch IN LISTS Choices) - if(${ch} STREQUAL ${Value}) - set(col ${Color}) + if(NOT ${Value} STREQUAL "") + if(${ch} STREQUAL ${Value}) + set(col ${Color}) + else() + set(col ${Dim}) + endif() else() set(col ${Dim}) endif() - if(${ch} STREQUAL ${Default}) - set(col ${Underline}${col}) + if(NOT ${Default} STREQUAL "") + if(${ch} STREQUAL ${Default}) + set(col ${Underline}${col}) + endif() endif() string(LENGTH "${ch}" clen) @@ -131,7 +137,8 @@ function( set(new_choices ${Choices}) foreach(ch IN LISTS new_choices) - string(REPLACE ${ch} "${Dim}${ch}${ColorReset}" new_choices "${new_choices}") + string(REPLACE ${ch} "${Dim}${ch}${ColorReset}" new_choices + "${new_choices}") endforeach() set(Choices ${new_choices}) if(${Value} STREQUAL "ON") @@ -141,8 +148,14 @@ function( else() set(col ${Color}) endif() - string(REPLACE ${Value} "${col}${Value}${ColorReset}" Choices "${Choices}") - string(REPLACE ${Default} "${Underline}${Default}${ColorReset}" Choices "${Choices}") + if(NOT "${Value}" STREQUAL "") + string(REPLACE ${Value} "${col}${Value}${ColorReset}" Choices + "${Choices}") + endif() + if(NOT "${Default}" STREQUAL "") + string(REPLACE ${Default} "${Underline}${Default}${ColorReset}" Choices + "${Choices}") + endif() string(REPLACE ";" "/" Choices "${Choices}") string(APPEND rstring "${Choices}") endif() diff --git a/cmake/tests.cmake b/cmake/tests.cmake index ca8ee69c4..e2d92a343 100644 --- a/cmake/tests.cmake +++ b/cmake/tests.cmake @@ -13,32 +13,26 @@ if(${output}) add_subdirectory(${SRC_DIR}/checkpoint ${CMAKE_CURRENT_BINARY_DIR}/checkpoint) endif() -if(${mpi}) - # tests with mpi - if(${output}) - add_subdirectory(${SRC_DIR}/output/tests - ${CMAKE_CURRENT_BINARY_DIR}/output/tests) - add_subdirectory(${SRC_DIR}/checkpoint/tests - ${CMAKE_CURRENT_BINARY_DIR}/checkpoint/tests) - add_subdirectory(${SRC_DIR}/framework/tests - ${CMAKE_CURRENT_BINARY_DIR}/framework/tests) - endif() -else() - # tests without mpi - add_subdirectory(${SRC_DIR}/global/tests - ${CMAKE_CURRENT_BINARY_DIR}/global/tests) - add_subdirectory(${SRC_DIR}/metrics/tests - ${CMAKE_CURRENT_BINARY_DIR}/metrics/tests) - add_subdirectory(${SRC_DIR}/kernels/tests - ${CMAKE_CURRENT_BINARY_DIR}/kernels/tests) - add_subdirectory(${SRC_DIR}/archetypes/tests - ${CMAKE_CURRENT_BINARY_DIR}/archetypes/tests) - add_subdirectory(${SRC_DIR}/framework/tests - ${CMAKE_CURRENT_BINARY_DIR}/framework/tests) - if(${output}) - add_subdirectory(${SRC_DIR}/output/tests - ${CMAKE_CURRENT_BINARY_DIR}/output/tests) - add_subdirectory(${SRC_DIR}/checkpoint/tests - ${CMAKE_CURRENT_BINARY_DIR}/checkpoint/tests) - endif() +set(TEST_DIRECTORIES + "" + PARENT_SCOPE) + +if(NOT ${mpi}) + list(APPEND TEST_DIRECTORIES global) + list(APPEND TEST_DIRECTORIES metrics) + list(APPEND TEST_DIRECTORIES kernels) + list(APPEND TEST_DIRECTORIES archetypes) + list(APPEND TEST_DIRECTORIES framework) +elseif(${mpi} AND ${output}) + list(APPEND TEST_DIRECTORIES framework) endif() + +if(${output}) + list(APPEND TEST_DIRECTORIES output) + list(APPEND TEST_DIRECTORIES checkpoint) +endif() + +foreach(test_dir IN LISTS TEST_DIRECTORIES) + add_subdirectory(${SRC_DIR}/${test_dir}/tests + ${CMAKE_CURRENT_BINARY_DIR}/${test_dir}/tests) +endforeach() From 004c6e953dee1286c9d363924b02f9245916eda7 Mon Sep 17 00:00:00 2001 From: haykh Date: Thu, 3 Apr 2025 20:33:26 -0400 Subject: [PATCH 173/176] type+unused warnings fixed --- src/framework/containers/particles.cpp | 7 +++---- src/kernels/fields_bcs.hpp | 28 +++++++++++++++++++------- src/metrics/tests/ks-qks.cpp | 1 + src/output/tests/writer-nompi.cpp | 3 ++- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/framework/containers/particles.cpp b/src/framework/containers/particles.cpp index 9ec8810dc..2f59a004d 100644 --- a/src/framework/containers/particles.cpp +++ b/src/framework/containers/particles.cpp @@ -84,7 +84,7 @@ namespace ntt { rangeActiveParticles(), Lambda(index_t p) { auto npptag_acc = npptag_scat.access(); - if (this_tag(p) < 0 || this_tag(p) >= num_tags) { + if (this_tag(p) < 0 || this_tag(p) >= static_cast(num_tags)) { raise::KernelError(HERE, "Invalid tag value"); } npptag_acc(this_tag(p)) += 1; @@ -144,9 +144,8 @@ namespace ntt { template void Particles::RemoveDead() { - const auto n_part = npart(); - npart_t n_alive = 0, n_dead = 0; - auto& this_tag = tag; + npart_t n_alive = 0, n_dead = 0; + auto& this_tag = tag; Kokkos::parallel_reduce( "CountDeadAlive", diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 970ab559e..782200e29 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -858,6 +858,7 @@ namespace kernel::bc { const I fset; const M metric; const ncells_t i_edge; + const BCTags tags; EnforcedBoundaries_kernel(ndfield_t& Fld, const I& fset, @@ -867,7 +868,8 @@ namespace kernel::bc { : Fld { Fld } , fset { fset } , metric { metric } - , i_edge { i_edge + N_GHOSTS } {} + , i_edge { i_edge + N_GHOSTS } + , tags { tags } {} Inline void operator()(index_t i1) const { if constexpr (D == Dim::_1D) { @@ -876,8 +878,12 @@ namespace kernel::bc { coord_t x_Ph_H { ZERO }; metric.template convert({ i1_ }, x_Ph_0); metric.template convert({ i1_ + HALF }, x_Ph_H); - bool setEx1 = defines_ex1, setEx2 = defines_ex2, setEx3 = defines_ex3, - setBx1 = defines_bx1, setBx2 = defines_bx2, setBx3 = defines_bx3; + bool setEx1 = defines_ex1 and (tags & BC::E), + setEx2 = defines_ex2 and (tags & BC::E), + setEx3 = defines_ex3 and (tags & BC::E), + setBx1 = defines_bx1 and (tags & BC::B), + setBx2 = defines_bx2 and (tags & BC::B), + setBx3 = defines_bx3 and (tags & BC::B); if constexpr (O == in::x1) { // x1 -- normal // x2,x3 -- tangential @@ -953,8 +959,12 @@ namespace kernel::bc { metric.template convert({ i1_ + HALF, i2_ }, x_Ph_H0); metric.template convert({ i1_ + HALF, i2_ + HALF }, x_Ph_HH); - bool setEx1 = defines_ex1, setEx2 = defines_ex2, setEx3 = defines_ex3, - setBx1 = defines_bx1, setBx2 = defines_bx2, setBx3 = defines_bx3; + bool setEx1 = defines_ex1 and (tags & BC::E), + setEx2 = defines_ex2 and (tags & BC::E), + setEx3 = defines_ex3 and (tags & BC::E), + setBx1 = defines_bx1 and (tags & BC::B), + setBx2 = defines_bx2 and (tags & BC::B), + setBx3 = defines_bx3 and (tags & BC::B); if constexpr (O == in::x1) { // x1 -- normal // x2,x3 -- tangential @@ -1053,8 +1063,12 @@ namespace kernel::bc { x_Ph_H0H); metric.template convert({ i1_, i2_ + HALF, i3_ + HALF }, x_Ph_0HH); - bool setEx1 = defines_ex1, setEx2 = defines_ex2, setEx3 = defines_ex3, - setBx1 = defines_bx1, setBx2 = defines_bx2, setBx3 = defines_bx3; + bool setEx1 = defines_ex1 and (tags & BC::E), + setEx2 = defines_ex2 and (tags & BC::E), + setEx3 = defines_ex3 and (tags & BC::E), + setBx1 = defines_bx1 and (tags & BC::B), + setBx2 = defines_bx2 and (tags & BC::B), + setBx3 = defines_bx3 and (tags & BC::B); if constexpr (O == in::x1) { // x1 -- normal // x2,x3 -- tangential diff --git a/src/metrics/tests/ks-qks.cpp b/src/metrics/tests/ks-qks.cpp index bed051e16..167f564ee 100644 --- a/src/metrics/tests/ks-qks.cpp +++ b/src/metrics/tests/ks-qks.cpp @@ -27,6 +27,7 @@ Inline auto equal(const vec_t& a, const auto eps = epsilon * acc; for (unsigned short d = 0; d < D; ++d) { if (not cmp::AlmostEqual(a[d], b[d], eps)) { + printf("%s: %.12e : %.12e\n", msg, a[d], b[d]); return false; } } diff --git a/src/output/tests/writer-nompi.cpp b/src/output/tests/writer-nompi.cpp index b063539ca..593f37f92 100644 --- a/src/output/tests/writer-nompi.cpp +++ b/src/output/tests/writer-nompi.cpp @@ -23,7 +23,8 @@ void cleanup() { } #define CEILDIV(a, b) \ - (static_cast(math::ceil(static_cast(a) / static_cast(b)))) + (static_cast( \ + math::ceil(static_cast(a) / static_cast(b)))) auto main(int argc, char* argv[]) -> int { Kokkos::initialize(argc, argv); From b539a91fb5b53766d3a3e620b509802c8fc1d9bb Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 4 Apr 2025 02:43:00 -0400 Subject: [PATCH 174/176] warnings in cmake + nullptr --- CMakeLists.txt | 13 ++++----- cmake/benchmark.cmake | 2 ++ cmake/config.cmake | 17 +++++++----- cmake/defaults.cmake | 2 ++ cmake/dependencies.cmake | 41 ++++++++++++++++++----------- cmake/kokkosConfig.cmake | 2 ++ cmake/report.cmake | 26 ++++++++++-------- cmake/styling.cmake | 12 ++++----- cmake/tests.cmake | 4 +-- src/CMakeLists.txt | 1 + src/archetypes/CMakeLists.txt | 2 +- src/archetypes/tests/CMakeLists.txt | 1 + src/checkpoint/CMakeLists.txt | 1 + src/checkpoint/tests/CMakeLists.txt | 1 + src/engines/CMakeLists.txt | 1 + src/framework/CMakeLists.txt | 1 + src/framework/parameters.cpp | 10 +++---- src/framework/tests/CMakeLists.txt | 1 + src/global/CMakeLists.txt | 1 + src/global/tests/CMakeLists.txt | 1 + src/kernels/CMakeLists.txt | 2 +- src/kernels/tests/CMakeLists.txt | 1 + src/metrics/CMakeLists.txt | 2 +- src/metrics/tests/CMakeLists.txt | 2 +- src/output/CMakeLists.txt | 1 + src/output/tests/CMakeLists.txt | 1 + 26 files changed, 92 insertions(+), 57 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4791ea55b..f83e6637c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,3 +1,5 @@ +# cmake-lint: disable=C0103,C0111,E1120,R0913,R0915 + cmake_minimum_required(VERSION 3.16) cmake_policy(SET CMP0110 NEW) @@ -8,10 +10,10 @@ project( VERSION 1.2.0 LANGUAGES CXX C) add_compile_options("-D ENTITY_VERSION=\"${PROJECT_VERSION}\"") +set(hash_cmd "git diff --quiet src/ && echo $(git rev-parse HEAD) ") +string(APPEND hash_cmd "|| echo $(git rev-parse HEAD)-mod") execute_process( - COMMAND - bash -c - "git diff --quiet src/ && echo $(git rev-parse HEAD) || echo $(git rev-parse HEAD)-mod" + COMMAND bash -c ${hash_cmd} WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" OUTPUT_VARIABLE GIT_HASH ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) @@ -55,9 +57,8 @@ if(${DEBUG} STREQUAL "OFF") set(CMAKE_BUILD_TYPE Release CACHE STRING "CMake build type") - set(CMAKE_CXX_FLAGS - "${CMAKE_CXX_FLAGS} -DNDEBUG -Wno-unused-local-typedefs -Wno-unknown-cuda-version" - ) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DNDEBUG " + "-Wno-unused-local-typedefs -Wno-unknown-cuda-version") else() set(CMAKE_BUILD_TYPE Debug diff --git a/cmake/benchmark.cmake b/cmake/benchmark.cmake index d2e8ca47c..39b075716 100644 --- a/cmake/benchmark.cmake +++ b/cmake/benchmark.cmake @@ -1,3 +1,5 @@ +# cmake-lint: disable=C0103 + set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src) add_subdirectory(${SRC_DIR}/global ${CMAKE_CURRENT_BINARY_DIR}/global) diff --git a/cmake/config.cmake b/cmake/config.cmake index ab54717f1..7214812cd 100644 --- a/cmake/config.cmake +++ b/cmake/config.cmake @@ -1,3 +1,5 @@ +# cmake-lint: disable=C0103 + # -------------------------------- Precision ------------------------------- # function(set_precision precision_name) list(FIND precisions ${precision_name} PRECISION_FOUND) @@ -28,16 +30,17 @@ function(set_problem_generator pgen_name) endforeach() list(FIND PGEN_NAMES ${pgen_name} PGEN_FOUND) if(NOT ${pgen_name} STREQUAL "." AND ${PGEN_FOUND} EQUAL -1) - message( - FATAL_ERROR - "Invalid problem generator: ${pgen_name}\nValid options are: ${PGEN_NAMES}" - ) + message(FATAL_ERROR "Invalid problem generator: " + "${pgen_name}\nValid options are: ${PGEN_NAMES}") endif() set(PGEN - ${pgen_name} PARENT_SCOPE) + ${pgen_name} + PARENT_SCOPE) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/setups/${pgen_name}) set(PGEN_FOUND - TRUE PARENT_SCOPE) + TRUE + PARENT_SCOPE) set(problem_generators - ${PGEN_NAMES} PARENT_SCOPE) + ${PGEN_NAMES} + PARENT_SCOPE) endfunction() diff --git a/cmake/defaults.cmake b/cmake/defaults.cmake index 2c72f5d7d..30e605a5c 100644 --- a/cmake/defaults.cmake +++ b/cmake/defaults.cmake @@ -1,3 +1,5 @@ +# cmake-lint: disable=C0103 + # ----------------------------- Defaults ---------------------------------- # if(DEFINED ENV{Entity_ENABLE_DEBUG}) set(default_debug diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index a31dfdea5..a21ea00e8 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -1,10 +1,12 @@ +# cmake-lint: disable=C0103,C0111,R0915,R0912 + set(Kokkos_REPOSITORY https://github.com/kokkos/kokkos.git CACHE STRING "Kokkos repository") set(plog_REPOSITORY https://github.com/SergiusTheBest/plog.git CACHE STRING "plog repository") -set (adios2_REPOSITORY +set(adios2_REPOSITORY https://github.com/ornladios/ADIOS2.git CACHE STRING "ADIOS2 repository") @@ -41,7 +43,7 @@ function(find_or_fetch_dependency package_name header_only mode) endif() if(NOT ${package_name}_FOUND) - if (${package_name} STREQUAL "Kokkos") + if(${package_name} STREQUAL "Kokkos") include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/kokkosConfig.cmake) elseif(${package_name} STREQUAL "adios2") include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/adios2Config.cmake) @@ -49,10 +51,8 @@ function(find_or_fetch_dependency package_name header_only mode) if(DEFINED ${package_name}_REPOSITORY AND NOT FETCHCONTENT_FULLY_DISCONNECTED) # fetching package - message( - STATUS - "${Blue}${package_name} not found. Fetching from ${${package_name}_REPOSITORY}${ColorReset}" - ) + message(STATUS "${Blue}${package_name} not found. " + "Fetching from ${${package_name}_REPOSITORY}${ColorReset}") include(FetchContent) if(${package_name} STREQUAL "Kokkos") FetchContent_Declare( @@ -100,7 +100,7 @@ function(find_or_fetch_dependency package_name header_only mode) endif() add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extern/${package_name} - extern/${package_name}) + extern/${package_name}) set(${package_name}_SRC ${CMAKE_CURRENT_SOURCE_DIR}/extern/${package_name} CACHE PATH "Path to ${package_name} src") @@ -136,24 +136,35 @@ function(find_or_fetch_dependency package_name header_only mode) ${Kokkos_VERSION} CACHE INTERNAL "Kokkos version") endif() - if(NOT DEFINED Kokkos_ARCH OR Kokkos_ARCH STREQUAL "" - OR NOT DEFINED Kokkos_DEVICES OR Kokkos_DEVICES STREQUAL "") + if(NOT DEFINED Kokkos_ARCH + OR Kokkos_ARCH STREQUAL "" + OR NOT DEFINED Kokkos_DEVICES + OR Kokkos_DEVICES STREQUAL "") if(${Kokkos_FOUND}) include(${Kokkos_DIR}/KokkosConfigCommon.cmake) elseif(NOT ${Kokkos_BUILD_DIR} STREQUAL "") include(${Kokkos_BUILD_DIR}/KokkosConfigCommon.cmake) else() - message(STATUS "${Red}Kokkos_DIR and Kokkos_BUILD_DIR not set.${ColorReset}") + message( + STATUS "${Red}Kokkos_DIR and Kokkos_BUILD_DIR not set.${ColorReset}") endif() endif() set(Kokkos_ARCH - ${Kokkos_ARCH} PARENT_SCOPE) + ${Kokkos_ARCH} + PARENT_SCOPE) set(Kokkos_DEVICES - ${Kokkos_DEVICES} PARENT_SCOPE) + ${Kokkos_DEVICES} + PARENT_SCOPE) endif() - set(${package_name}_FOUND ${${package_name}_FOUND} PARENT_SCOPE) - set(${package_name}_FETCHED ${${package_name}_FETCHED} PARENT_SCOPE) - set(${package_name}_BUILD_DIR ${${package_name}_BUILD_DIR} PARENT_SCOPE) + set(${package_name}_FOUND + ${${package_name}_FOUND} + PARENT_SCOPE) + set(${package_name}_FETCHED + ${${package_name}_FETCHED} + PARENT_SCOPE) + set(${package_name}_BUILD_DIR + ${${package_name}_BUILD_DIR} + PARENT_SCOPE) endfunction() check_internet_connection() diff --git a/cmake/kokkosConfig.cmake b/cmake/kokkosConfig.cmake index 3d038cc19..f1a1cf207 100644 --- a/cmake/kokkosConfig.cmake +++ b/cmake/kokkosConfig.cmake @@ -1,3 +1,5 @@ +# cmake-lint: disable=C0103 + # ----------------------------- Kokkos settings ---------------------------- # if(${DEBUG} STREQUAL "OFF") set(Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION diff --git a/cmake/report.cmake b/cmake/report.cmake index 21064cdf9..5a38b0dd5 100644 --- a/cmake/report.cmake +++ b/cmake/report.cmake @@ -66,13 +66,13 @@ printchoices( 36) if(NOT ${PROJECT_VERSION_TWEAK} EQUAL 0) - set(VERSION_SYMBOL - "v${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}-rc${PROJECT_VERSION_TWEAK}" - ) + set(VERSION_SYMBOL "v${PROJECT_VERSION_MAJOR}." "${PROJECT_VERSION_MINOR}.") + string(APPEND VERSION_SYMBOL + "${PROJECT_VERSION_PATCH}-rc${PROJECT_VERSION_TWEAK}") else() - set(VERSION_SYMBOL - "v${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH} " - ) + set(VERSION_SYMBOL "v${PROJECT_VERSION_MAJOR}.") + string(APPEND VERSION_SYMBOL + "${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH} ") endif() set(REPORT_TEXT @@ -149,12 +149,13 @@ if(${Kokkos_DEVICES} MATCHES "CUDA") set(CUDACOMP ${CMAKE_CUDA_COMPILER}) endif() string(STRIP ${CUDACOMP} CUDACOMP) + set(cmd "${CUDACOMP} --version |") + string(APPEND cmd " grep release | sed -e 's/.*release //' -e 's/,.*//'") execute_process( - COMMAND - bash -c - "${CUDACOMP} --version | grep release | sed -e 's/.*release //' -e 's/,.*//'" + COMMAND bash -c ${cmd} OUTPUT_VARIABLE CUDACOMP_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) + message(STATUS "CUDACOMP: ${CUDACOMP_VERSION}") string( APPEND REPORT_TEXT @@ -165,8 +166,9 @@ if(${Kokkos_DEVICES} MATCHES "CUDA") ${CUDACOMP} "${ColorReset}\n") elseif(${Kokkos_DEVICES} MATCHES "HIP") + set(cmd "hipcc --version | grep HIP | cut -d ':' -f 2 | tr -d ' '") execute_process( - COMMAND bash -c "hipcc --version | grep HIP | cut -d ':' -f 2 | tr -d ' '" + COMMAND bash -c ${cmd} OUTPUT_VARIABLE ROCM_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) string(APPEND REPORT_TEXT " - ROCm: v" ${ROCM_VERSION} "\n") @@ -196,7 +198,9 @@ string( "\n" "Notes" "\n" - " ${Dim}: Set flags with `cmake ... -D ${Magenta}${ColorReset}${Dim}=`, the ${Underline}default${ColorReset}${Dim} value" + " ${Dim}: Set flags with `cmake ... -D " + "${Magenta}${ColorReset}${Dim}=`, " + "the ${Underline}default${ColorReset}${Dim} value" "\n" " : will be used unless the variable is explicitly set.${ColorReset}") diff --git a/cmake/styling.cmake b/cmake/styling.cmake index b9b72abcb..878cb44a4 100644 --- a/cmake/styling.cmake +++ b/cmake/styling.cmake @@ -1,3 +1,5 @@ +# cmake-lint: disable=C0103,C0301,C0111,E1120,R0913,R0915 + if(NOT WIN32) string(ASCII 27 Esc) set(ColorReset "${Esc}[m") @@ -23,13 +25,11 @@ if(NOT WIN32) set(StrikeEnd "${Esc}[0m") endif() -set(DOTTED_LINE_SYMBOL - "${ColorReset}. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " -) +set(DOTTED_LINE_SYMBOL "${ColorReset}. . . . . . . . . . . . . . . .") +string(APPEND DOTTED_LINE_SYMBOL " . . . . . . . . . . . . . . . . . . . . ") -set(DASHED_LINE_SYMBOL - "${ColorReset}....................................................................... " -) +set(DASHED_LINE_SYMBOL "${ColorReset}.................................") +string(APPEND DASHED_LINE_SYMBOL "...................................... ") set(ON_OFF_VALUES "ON" "OFF") diff --git a/cmake/tests.cmake b/cmake/tests.cmake index e2d92a343..0e108d365 100644 --- a/cmake/tests.cmake +++ b/cmake/tests.cmake @@ -13,9 +13,7 @@ if(${output}) add_subdirectory(${SRC_DIR}/checkpoint ${CMAKE_CURRENT_BINARY_DIR}/checkpoint) endif() -set(TEST_DIRECTORIES - "" - PARENT_SCOPE) +set(TEST_DIRECTORIES "") if(NOT ${mpi}) list(APPEND TEST_DIRECTORIES global) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index db2ab4c92..f9d921df0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103 # ------------------------------ # @defines: entity [STATIC/SHARED] # diff --git a/src/archetypes/CMakeLists.txt b/src/archetypes/CMakeLists.txt index 8e2f325af..93f8baaaa 100644 --- a/src/archetypes/CMakeLists.txt +++ b/src/archetypes/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103 # ------------------------------ # @defines: ntt_archetypes [INTERFACE] # @@ -24,4 +25,3 @@ target_link_libraries(ntt_archetypes INTERFACE ${libs}) target_include_directories(ntt_archetypes INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/../) - diff --git a/src/archetypes/tests/CMakeLists.txt b/src/archetypes/tests/CMakeLists.txt index 694a6b4f9..9419847c5 100644 --- a/src/archetypes/tests/CMakeLists.txt +++ b/src/archetypes/tests/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103,C0111 # ------------------------------ # @brief: Generates tests for the `ntt_archetypes` module # diff --git a/src/checkpoint/CMakeLists.txt b/src/checkpoint/CMakeLists.txt index fa641bfb5..096aad690 100644 --- a/src/checkpoint/CMakeLists.txt +++ b/src/checkpoint/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103 # ------------------------------ # @defines: ntt_checkpoint [STATIC/SHARED] # diff --git a/src/checkpoint/tests/CMakeLists.txt b/src/checkpoint/tests/CMakeLists.txt index 54400652e..cbfd63aa9 100644 --- a/src/checkpoint/tests/CMakeLists.txt +++ b/src/checkpoint/tests/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103,C0111 # ------------------------------ # @brief: Generates tests for the `ntt_checkpoint` module # diff --git a/src/engines/CMakeLists.txt b/src/engines/CMakeLists.txt index d5b6b0664..4cef18630 100644 --- a/src/engines/CMakeLists.txt +++ b/src/engines/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103 # ------------------------------ # @defines: ntt_engines [STATIC/SHARED] # diff --git a/src/framework/CMakeLists.txt b/src/framework/CMakeLists.txt index d3b68084c..4c407fb0c 100644 --- a/src/framework/CMakeLists.txt +++ b/src/framework/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103 # ------------------------------ # @defines: ntt_framework [STATIC/SHARED] # diff --git a/src/framework/parameters.cpp b/src/framework/parameters.cpp index e3bddfd70..5848b1c01 100644 --- a/src/framework/parameters.cpp +++ b/src/framework/parameters.cpp @@ -31,10 +31,10 @@ namespace ntt { template - auto get_dx0_V0( - const std::vector& resolution, - const boundaries_t& extent, - const std::map& params) -> std::pair { + auto get_dx0_V0(const std::vector& resolution, + const boundaries_t& extent, + const std::map& params) + -> std::pair { const auto metric = M(resolution, extent, params); const auto dx0 = metric.dxMin(); coord_t x_corner { ZERO }; @@ -839,7 +839,7 @@ namespace ntt { void SimulationParams::setSetupParams(const toml::value& toml_data) { /* [setup] -------------------------------------------------------------- */ - const auto& setup = toml::find_or(toml_data, "setup", toml::table {}); + const auto setup = toml::find_or(toml_data, "setup", toml::table {}); for (const auto& [key, val] : setup) { if (val.is_boolean()) { set("setup." + key, (bool)(val.as_boolean())); diff --git a/src/framework/tests/CMakeLists.txt b/src/framework/tests/CMakeLists.txt index 27c456610..92e327d80 100644 --- a/src/framework/tests/CMakeLists.txt +++ b/src/framework/tests/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103,C0111 # ------------------------------ # @brief: Generates tests for the `ntt_framework` module # diff --git a/src/global/CMakeLists.txt b/src/global/CMakeLists.txt index 97946f059..7546b6a98 100644 --- a/src/global/CMakeLists.txt +++ b/src/global/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103 # ------------------------------ # @defines: ntt_global [STATIC/SHARED] # diff --git a/src/global/tests/CMakeLists.txt b/src/global/tests/CMakeLists.txt index e30da20a0..c206f85b0 100644 --- a/src/global/tests/CMakeLists.txt +++ b/src/global/tests/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103,C0111 # ------------------------------ # @brief: Generates tests for the `ntt_global` module # diff --git a/src/kernels/CMakeLists.txt b/src/kernels/CMakeLists.txt index c8a1f409f..60eda24cb 100644 --- a/src/kernels/CMakeLists.txt +++ b/src/kernels/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103 # ------------------------------ # @defines: ntt_kernels [INTERFACE] # @@ -24,4 +25,3 @@ target_link_libraries(ntt_kernels INTERFACE ${libs}) target_include_directories(ntt_kernels INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/../) - diff --git a/src/kernels/tests/CMakeLists.txt b/src/kernels/tests/CMakeLists.txt index a41ea43ef..7579eb6d3 100644 --- a/src/kernels/tests/CMakeLists.txt +++ b/src/kernels/tests/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103,C0111 # ------------------------------ # @brief: Generates tests for the `ntt_kernels` module # diff --git a/src/metrics/CMakeLists.txt b/src/metrics/CMakeLists.txt index e053bb61c..0bb5b977c 100644 --- a/src/metrics/CMakeLists.txt +++ b/src/metrics/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103 # ------------------------------ # @defines: ntt_metrics [INTERFACE] # @@ -22,4 +23,3 @@ target_link_libraries(ntt_metrics INTERFACE ${libs}) target_include_directories(ntt_metrics INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/../) - diff --git a/src/metrics/tests/CMakeLists.txt b/src/metrics/tests/CMakeLists.txt index c997ab079..0d661c318 100644 --- a/src/metrics/tests/CMakeLists.txt +++ b/src/metrics/tests/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103,C0111 # ------------------------------ # @brief: Generates tests for the `ntt_metrics` module # @@ -28,4 +29,3 @@ gen_test(coord_trans) gen_test(sph-qsph) gen_test(ks-qks) gen_test(sr-cart-sph) - diff --git a/src/output/CMakeLists.txt b/src/output/CMakeLists.txt index 81333e9ff..8a2ea0f16 100644 --- a/src/output/CMakeLists.txt +++ b/src/output/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103 # ------------------------------ # @defines: ntt_output [STATIC/SHARED] # diff --git a/src/output/tests/CMakeLists.txt b/src/output/tests/CMakeLists.txt index afc7950c4..835bb532f 100644 --- a/src/output/tests/CMakeLists.txt +++ b/src/output/tests/CMakeLists.txt @@ -1,3 +1,4 @@ +# cmake-lint: disable=C0103,C0111 # ------------------------------ # @brief: Generates tests for the `ntt_output` module # From ec7ff77c6cb3b1fd93701047acd67fbca9425048 Mon Sep 17 00:00:00 2001 From: hayk Date: Fri, 4 Apr 2025 03:12:26 -0400 Subject: [PATCH 175/176] log level controlled from input --- input.example.toml | 6 ++++++ src/framework/parameters.cpp | 2 ++ src/framework/simulation.cpp | 7 ++++++- src/global/defaults.h | 3 ++- src/global/utils/formatting.h | 2 +- src/global/utils/plog.h | 14 +++++++++++--- 6 files changed, 28 insertions(+), 6 deletions(-) diff --git a/input.example.toml b/input.example.toml index c71049b84..f519a2799 100644 --- a/input.example.toml +++ b/input.example.toml @@ -478,3 +478,9 @@ # @type: bool # @default: true colored_stdout = "" + # Specify the log level: + # @type: string + # @valid: "VERBOSE", "WARNING", "ERROR" + # @default: "VERBOSE" + # @note: VERBOSE prints all messages, WARNING prints only warnings and errors, ERROR prints only errors + log_level = "" diff --git a/src/framework/parameters.cpp b/src/framework/parameters.cpp index 5848b1c01..556d9f547 100644 --- a/src/framework/parameters.cpp +++ b/src/framework/parameters.cpp @@ -601,6 +601,8 @@ namespace ntt { toml::find_or(toml_data, "diagnostics", "blocking_timers", false)); set("diagnostics.colored_stdout", toml::find_or(toml_data, "diagnostics", "colored_stdout", false)); + set("diagnostics.log_level", + toml::find_or(toml_data, "diagnostics", "log_level", defaults::diag::log_level)); /* inferred variables --------------------------------------------------- */ // fields/particle boundaries diff --git a/src/framework/simulation.cpp b/src/framework/simulation.cpp index 560539c25..6865140c9 100644 --- a/src/framework/simulation.cpp +++ b/src/framework/simulation.cpp @@ -32,7 +32,12 @@ namespace ntt { const auto raw_params = toml::parse(inputfname); const auto sim_name = toml::find(raw_params, "simulation", "name"); - logger::initPlog(sim_name); + const auto log_level = toml::find_or(raw_params, + "diagnostics", + "log_level", + defaults::diag::log_level); + logger::initPlog(sim_name, + log_level); m_requested_engine = SimEngine::pick( fmt::toLower(toml::find(raw_params, "simulation", "engine")).c_str()); diff --git a/src/global/defaults.h b/src/global/defaults.h index b81369022..d17647ccd 100644 --- a/src/global/defaults.h +++ b/src/global/defaults.h @@ -68,7 +68,8 @@ namespace ntt::defaults { } // namespace checkpoint namespace diag { - const timestep_t interval = 1; + const timestep_t interval = 1; + const std::string log_level = "VERBOSE"; } // namespace diag namespace gca { diff --git a/src/global/utils/formatting.h b/src/global/utils/formatting.h index 8dc7b6ba8..c8c236cab 100644 --- a/src/global/utils/formatting.h +++ b/src/global/utils/formatting.h @@ -171,7 +171,7 @@ namespace fmt { std::size_t cntr { 0 }; for (auto i { 0u }; i < columns.size(); ++i) { const auto anch { static_cast(anchors[i] < 0 ? -anchors[i] - : anchors[i]) }; + : anchors[i]) }; const auto leftalign { anchors[i] <= 0 }; const auto cmn { columns[i] }; const auto cmn_len { strlenUTF8(cmn) }; diff --git a/src/global/utils/plog.h b/src/global/utils/plog.h index 03dc19319..2422c7cf9 100644 --- a/src/global/utils/plog.h +++ b/src/global/utils/plog.h @@ -13,6 +13,8 @@ #ifndef GLOBAL_UTILS_PLOG_H #define GLOBAL_UTILS_PLOG_H +#include "utils/formatting.h" + #include #include #include @@ -57,7 +59,7 @@ namespace plog { namespace logger { template - inline void initPlog(const std::string& fname) { + inline void initPlog(const std::string& fname, const std::string& log_level) { // setup logging const auto logfile_name = fname + ".log"; const auto infofile_name = fname + ".info"; @@ -77,7 +79,13 @@ namespace logger { infofile_name.c_str()); static plog::RollingFileAppender errfileAppender( errfile_name.c_str()); - plog::init(plog::verbose, &logfileAppender); + auto log_severity = plog::verbose; + if (fmt::toLower(log_level) == "WARNING") { + log_severity = plog::warning; + } else if (fmt::toLower(log_level) == "ERROR") { + log_severity = plog::error; + } + plog::init(log_severity, &logfileAppender); plog::init(plog::verbose, &infofileAppender); plog::init(plog::verbose, &errfileAppender); @@ -93,4 +101,4 @@ namespace logger { } // namespace logger -#endif // GLOBAL_UTILS_PLOG_H \ No newline at end of file +#endif // GLOBAL_UTILS_PLOG_H From 8850ab9d05b641378b1c1699dea1f0376cb671e4 Mon Sep 17 00:00:00 2001 From: hayk Date: Sat, 5 Apr 2025 06:58:40 -0400 Subject: [PATCH 176/176] warnings fixed --- src/framework/domain/comm_mpi.hpp | 6 ++---- src/framework/domain/communications.cpp | 21 ++++++++++----------- src/framework/domain/metadomain.cpp | 2 +- src/global/utils/timer.cpp | 2 +- src/output/tests/writer-mpi.cpp | 4 ++-- 5 files changed, 16 insertions(+), 19 deletions(-) diff --git a/src/framework/domain/comm_mpi.hpp b/src/framework/domain/comm_mpi.hpp index 159502e7a..d9783bdb6 100644 --- a/src/framework/domain/comm_mpi.hpp +++ b/src/framework/domain/comm_mpi.hpp @@ -335,10 +335,8 @@ namespace comm { const unsigned short NPLDS = species.npld(); // buffers to store recv data - const auto npart_alive = npptag_vec[ParticleTag::alive]; - const auto npart_dead = npptag_vec[ParticleTag::dead]; - const auto npart_send = outgoing_indices.extent(0) - npart_dead; - const auto npart_recv = std::accumulate(npptag_recv_vec.begin(), + const auto npart_dead = npptag_vec[ParticleTag::dead]; + const auto npart_recv = std::accumulate(npptag_recv_vec.begin(), npptag_recv_vec.end(), static_cast(0)); array_t recv_buff_int { "recv_buff_int", npart_recv * NINTS }; diff --git a/src/framework/domain/communications.cpp b/src/framework/domain/communications.cpp index 946756f49..cf7e04974 100644 --- a/src/framework/domain/communications.cpp +++ b/src/framework/domain/communications.cpp @@ -36,10 +36,10 @@ namespace ntt { using comm_params_t = std::pair>; template - auto GetSendRecvRanks( - Metadomain* metadomain, - Domain& domain, - dir::direction_t direction) -> std::pair { + auto GetSendRecvRanks(Metadomain* metadomain, + Domain& domain, + dir::direction_t direction) + -> std::pair { Domain* send_to_nghbr_ptr = nullptr; Domain* recv_from_nghbr_ptr = nullptr; // set pointers to the correct send/recv domains @@ -119,11 +119,11 @@ namespace ntt { } template - auto GetSendRecvParams( - Metadomain* metadomain, - Domain& domain, - dir::direction_t direction, - bool synchronize) -> std::pair { + auto GetSendRecvParams(Metadomain* metadomain, + Domain& domain, + dir::direction_t direction, + bool synchronize) + -> std::pair { const auto [send_indrank, recv_indrank] = GetSendRecvRanks(metadomain, domain, direction); const auto [send_ind, send_rank] = send_indrank; @@ -506,8 +506,7 @@ namespace ntt { const auto npart_dead = npptag_vec[ParticleTag::dead]; const auto npart_alive = npptag_vec[ParticleTag::alive]; - const auto npart = species.npart(); - const auto npart_holes = npart - npart_alive; + const auto npart = species.npart(); // # of particles to receive per each tag (direction) std::vector npptag_recv_vec(ntags - 2, 0); diff --git a/src/framework/domain/metadomain.cpp b/src/framework/domain/metadomain.cpp index eec6c8fa0..8952d417d 100644 --- a/src/framework/domain/metadomain.cpp +++ b/src/framework/domain/metadomain.cpp @@ -46,7 +46,7 @@ namespace ntt { #if defined(MPI_ENABLED) MPI_Comm_size(MPI_COMM_WORLD, &g_mpi_size); MPI_Comm_rank(MPI_COMM_WORLD, &g_mpi_rank); - raise::ErrorIf(global_ndomains != g_mpi_size, + raise::ErrorIf(global_ndomains != (unsigned int)g_mpi_size, "Exactly 1 domain per MPI rank is allowed", HERE); #endif diff --git a/src/global/utils/timer.cpp b/src/global/utils/timer.cpp index bc133b4b7..3cca1b365 100644 --- a/src/global/utils/timer.cpp +++ b/src/global/utils/timer.cpp @@ -68,7 +68,7 @@ namespace timer { return {}; } std::vector all_totals(size, 0.0); - for (auto i { 0u }; i < size; ++i) { + for (auto i { 0 }; i < size; ++i) { for (auto& [name, timer] : m_timers) { if (std::find(ignore_in_tot.begin(), ignore_in_tot.end(), name) == ignore_in_tot.end()) { diff --git a/src/output/tests/writer-mpi.cpp b/src/output/tests/writer-mpi.cpp index cc9208889..af8a38ef1 100644 --- a/src/output/tests/writer-mpi.cpp +++ b/src/output/tests/writer-mpi.cpp @@ -22,7 +22,8 @@ void cleanup() { } #define CEILDIV(a, b) \ - (static_cast(math::ceil(static_cast(a) / static_cast(b)))) + (static_cast( \ + math::ceil(static_cast(a) / static_cast(b)))) auto main(int argc, char* argv[]) -> int { Kokkos::initialize(argc, argv); @@ -115,7 +116,6 @@ auto main(int argc, char* argv[]) -> int { const auto l_size = nx1; const auto l_offset = nx1 * mpi_rank; - const auto g_size = nx1 * mpi_size; const double n = l_size; const double d = dwn1;