From 1df0cce978d8da8e00a1f0c7d3154a8cb091585a Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova Date: Tue, 24 Sep 2024 12:20:06 -0400 Subject: [PATCH 001/234] compilable example for grpic --- setups/grpic/pgen_grpic_example.hpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/setups/grpic/pgen_grpic_example.hpp b/setups/grpic/pgen_grpic_example.hpp index f553ae849..3374a12d7 100644 --- a/setups/grpic/pgen_grpic_example.hpp +++ b/setups/grpic/pgen_grpic_example.hpp @@ -12,7 +12,7 @@ namespace user { using namespace ntt; template - struct PGen : public ProblemGenerator { + struct PGen : public arch::ProblemGenerator { // compatibility traits for the problem generator static constexpr auto engines { traits::compatible_with::value }; static constexpr auto metrics { @@ -21,13 +21,12 @@ namespace user { static constexpr auto dimensions { traits::compatible_with::value }; // for easy access to variables in the child class - using ProblemGenerator::D; - using ProblemGenerator::C; - using ProblemGenerator::params; - using ProblemGenerator::domain; + using arch::ProblemGenerator::D; + using arch::ProblemGenerator::C; + using arch::ProblemGenerator::params; inline PGen(SimulationParams& p, const Metadomain&) : - ProblemGenerator(p) {} + arch::ProblemGenerator(p) {} inline PGen() {} }; From 0c81d9b3ddebb3c64afdf13782d565d28fbbda25 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 24 Sep 2024 12:48:33 -0400 Subject: [PATCH 002/234] starting a test pgen --- setups/grpic/wald/pgen.hpp | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 setups/grpic/wald/pgen.hpp diff --git a/setups/grpic/wald/pgen.hpp b/setups/grpic/wald/pgen.hpp new file mode 100644 index 000000000..3374a12d7 --- /dev/null +++ b/setups/grpic/wald/pgen.hpp @@ -0,0 +1,36 @@ +#ifndef PROBLEM_GENERATOR_H +#define PROBLEM_GENERATOR_H + +#include "enums.h" +#include "global.h" + +#include "arch/traits.h" + +#include "archetypes/problem_generator.h" + +namespace user { + using namespace ntt; + + template + struct PGen : public arch::ProblemGenerator { + // compatibility traits for the problem generator + static constexpr auto engines { traits::compatible_with::value }; + static constexpr auto metrics { + traits::compatible_with::value + }; + static constexpr auto dimensions { traits::compatible_with::value }; + + // for easy access to variables in the child class + using arch::ProblemGenerator::D; + using arch::ProblemGenerator::C; + using arch::ProblemGenerator::params; + + inline PGen(SimulationParams& p, const Metadomain&) : + arch::ProblemGenerator(p) {} + + inline PGen() {} + }; + +} // namespace user + +#endif From fc126af8cf4dca5419b346d31bbdb58de5085d0f Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 24 Sep 2024 13:46:21 -0400 Subject: [PATCH 003/234] engines/grpic: added relevant libraries --- src/engines/grpic.hpp | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 148c1c5c5..5ed9fe57d 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -13,13 +13,37 @@ #define ENGINES_GRPIC_GRPIC_H #include "enums.h" +#include "global.h" +#include "arch/kokkos_aliases.h" +#include "arch/traits.h" +#include "utils/log.h" +#include "utils/numeric.h" #include "utils/timer.h" +#include "archetypes/particle_injector.h" #include "framework/domain/domain.h" +#include "framework/parameters.h" #include "engines/engine.hpp" +#include "kernels/ampere_gr.hpp" +#include "kernels/aux_fields_gr.hpp" +#include "kernels/currents_deposit.hpp" +#include "kernels/digital_filter.hpp" +#include "kernels/faraday_gr.hpp" +#include "kernels/fields_bcs.hpp" +#include "kernels/particle_moments.hpp" +#include "kernels/particle_pusher_gr.hpp" + +#include "pgen.hpp" + +#include +#include + +#include +#include + namespace ntt { template @@ -27,6 +51,7 @@ namespace ntt { using Engine::m_params; using Engine::m_metadomain; + public: static constexpr auto S { SimEngine::SRPIC }; From c7fc87a85bc45d010dc1dbe9a00d5dd9310bc3e2 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 24 Sep 2024 13:58:39 -0400 Subject: [PATCH 004/234] kernels/ampere_gr: fixed dublicate ErrorIF --- src/kernels/ampere_gr.hpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/kernels/ampere_gr.hpp b/src/kernels/ampere_gr.hpp index 5af0fa4ef..045eb1729 100644 --- a/src/kernels/ampere_gr.hpp +++ b/src/kernels/ampere_gr.hpp @@ -57,9 +57,6 @@ namespace kernel::gr { , coeff { coeff } { if constexpr ((D == Dim::_2D) || (D == Dim::_3D)) { raise::ErrorIf(boundaries.size() < 2, "boundaries defined incorrectly", HERE); - raise::ErrorIf(boundaries[1].size() < 2, - "boundaries defined incorrectly", - HERE); is_axis_i2min = (boundaries[1].first == FldsBC::AXIS); is_axis_i2max = (boundaries[1].second == FldsBC::AXIS); } From d106f45dc6a4b18a40df73a66fc6efe726ad18a9 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 24 Sep 2024 14:21:40 -0400 Subject: [PATCH 005/234] engines/grpic: minor syntax adustments to match engines/srpic --- src/engines/grpic.hpp | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 5ed9fe57d..687074a9e 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -4,7 +4,7 @@ * @implements * - ntt::GRPICEngine<> : ntt::Engine<> * @cpp: - * - srpic.cpp + * - grpic.cpp * @namespaces: * - ntt:: */ @@ -48,19 +48,32 @@ namespace ntt { template class GRPICEngine : public Engine { - using Engine::m_params; - using Engine::m_metadomain; - + using base_t = Engine; + using pgen_t = user::PGen; + using domain_t = Domain; + // constexprs + using base_t::pgen_is_ok; + // contents + using base_t::m_metadomain; + using base_t::m_params; + using base_t::m_pgen; + // methods + using base_t::init; + // variables + using base_t::dt; + using base_t::max_steps; + using base_t::runtime; + using base_t::step; + using base_t::time; public: - static constexpr auto S { SimEngine::SRPIC }; + static constexpr auto S { SimEngine::GRPIC }; - GRPICEngine(SimulationParams& params) - : Engine { params } {} + GRPICEngine(SimulationParams& params) : base_t { params } {} ~GRPICEngine() = default; - void step_forward(timer::Timers&, Domain&) override {} + void step_forward(timer::Timers& timers, domain_t& dom) override {} }; } // namespace ntt From 70fa507a00ca1f7fb488fcb9ebef46452fe9cfc8 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 24 Sep 2024 17:06:01 -0400 Subject: [PATCH 006/234] engines/grpic: added initialisation of fields and boundary conditions --- src/engines/grpic.hpp | 184 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 182 insertions(+), 2 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 687074a9e..67091f9fb 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -73,9 +73,189 @@ namespace ntt { ~GRPICEngine() = default; - void step_forward(timer::Timers& timers, domain_t& dom) override {} - }; + void step_forward(timer::Timers& timers, domain_t& dom) override { + const auto fieldsolver_enabled = m_params.template get( + "algorithms.toggles.fieldsolver"); + const auto deposit_enabled = m_params.template get( + "algorithms.toggles.deposit"); + const auto sort_interval = m_params.template get( + "particles.sort_interval"); + + if (step == 0) { + // communicate fields and apply BCs on the first timestep + /** + * Initially: em0::B -- + * em0::D -- + * em::B at -1/2 + * em::D at -1/2 + * + * cur0::J -- + * cur::J -- + * + * aux::E -- + * aux::H -- + * + * x_prtl at -1/2 + * u_prtl at -1/2 + */ + + /** + * em0::D, em::D, em0::B, em::B <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0 | Comm::D | Comm::D0); + FieldBoundaries(dom, BC::B | BC::D); + + } + } + + void FieldBoundaries(domain_t& domain, BCTags tags) { + for (auto& direction : dir::Directions::orth) { + if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::ABSORB) { + AbsorbFieldsIn(direction, domain, tags); + } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::AXIS) { + if (domain.mesh.flds_bc_in(direction) == FldsBC::AXIS) { + AxisFieldsIn(direction, domain, tags); + } + } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::CUSTOM) { + if (domain.mesh.flds_bc_in(direction) == FldsBC::CUSTOM) { + CustomFieldsIn(direction, domain, tags); + } + } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { + raise::Error("HORIZON BCs only applicable for GR", HERE); + } + } // loop over directions + } + void AbsorbFieldsIn(dir::direction_t direction, + domain_t& domain, + BCTags tags) { + /** + * absorbing boundaries + */ + const auto ds = m_params.template get( + "grid.boundaries.absorb.ds"); + const auto dim = direction.get_dim(); + real_t xg_min, xg_max, xg_edge; + auto sign = direction.get_sign(); + if (sign > 0) { // + direction + xg_max = m_metadomain.mesh().extent(dim).second; + xg_min = xg_max - ds; + xg_edge = xg_max; + } else { // - direction + xg_min = m_metadomain.mesh().extent(dim).first; + xg_max = xg_min + ds; + xg_edge = xg_min; + } + boundaries_t box; + boundaries_t incl_ghosts; + for (unsigned short d { 0 }; d < M::Dim; ++d) { + if (d == static_cast(dim)) { + box.push_back({ xg_min, xg_max }); + if (sign > 0) { + incl_ghosts.push_back({ false, true }); + } else { + incl_ghosts.push_back({ true, false }); + } + } else { + box.push_back(Range::All); + incl_ghosts.push_back({ true, true }); + } + } + if (not domain.mesh.Intersects(box)) { + return; + } + const auto intersect_range = domain.mesh.ExtentToRange(box, incl_ghosts); + tuple_t range_min { 0 }; + tuple_t range_max { 0 }; + + for (unsigned short d { 0 }; d < M::Dim; ++d) { + range_min[d] = intersect_range[d].first; + range_max[d] = intersect_range[d].second; + } + if (dim == in::x1) { + Kokkos::parallel_for( + "AbsorbFields", + CreateRangePolicy(range_min, range_max), + kernel::AbsorbBoundaries_kernel(domain.fields.em, + domain.mesh.metric, + xg_edge, + ds, + tags)); + } else if (dim == in::x2) { + if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { + Kokkos::parallel_for( + "AbsorbFields", + CreateRangePolicy(range_min, range_max), + kernel::AbsorbBoundaries_kernel(domain.fields.em, + domain.mesh.metric, + xg_edge, + ds, + tags)); + } else { + raise::Error("Invalid dimension", HERE); + } + } else if (dim == in::x3) { + if constexpr (M::Dim == Dim::_3D) { + Kokkos::parallel_for( + "AbsorbFields", + CreateRangePolicy(range_min, range_max), + kernel::AbsorbBoundaries_kernel(domain.fields.em, + domain.mesh.metric, + xg_edge, + ds, + tags)); + } else { + raise::Error("Invalid dimension", HERE); + } + } + } + + void AxisFieldsIn(dir::direction_t direction, + domain_t& domain, + BCTags tags) { + /** + * axis boundaries + */ + raise::ErrorIf(M::CoordType == Coord::Cart, + "Invalid coordinate type for axis BCs", + HERE); + raise::ErrorIf(direction.get_dim() != in::x2, + "Invalid axis direction, should be x2", + HERE); + const auto i2_min = domain.mesh.i_min(in::x2); + const auto i2_max = domain.mesh.i_max(in::x2); + if (direction.get_sign() < 0) { + Kokkos::parallel_for( + "AxisBCFields", + domain.mesh.n_all(in::x1), + kernel::AxisBoundaries_kernel(domain.fields.em, i2_min, tags)); + } else { + Kokkos::parallel_for( + "AxisBCFields", + domain.mesh.n_all(in::x1), + kernel::AxisBoundaries_kernel(domain.fields.em, i2_max, tags)); + } + } + + void CustomFieldsIn(dir::direction_t direction, + domain_t& domain, + BCTags tags) { + (void)direction; + (void)domain; + (void)tags; + raise::Error("Custom boundaries not implemented", HERE); + // if constexpr ( + // traits::has_member::value) { + // const auto [box, custom_fields] = m_pgen.CustomFields(time); + // if (domain.mesh.Intersects(box)) { + // } + // + // } else { + // raise::Error("Custom boundaries not implemented", HERE); + // } + } + + }; } // namespace ntt #endif // ENGINES_GRPIC_GRPIC_H From 53e68610719c257928f9c7d39d1fabe258e54d9c Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 25 Sep 2024 12:55:12 -0400 Subject: [PATCH 007/234] engines/grpic: added cope of em field into em0 --- src/engines/grpic.hpp | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 67091f9fb..a84a0ed5f 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -105,6 +105,13 @@ namespace ntt { m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0 | Comm::D | Comm::D0); FieldBoundaries(dom, BC::B | BC::D); + /** + * em0::B <- em::B + * em0::D <- em::D + * + * Now: em0::B & em0::D at -1/2 + */ + CopyFields(dom); } } @@ -255,6 +262,22 @@ namespace ntt { // } } + /** + * @brief Swaps em and em0 fields, cur and cur0 currents. + */ + // void SwapFields() { + // auto& mblock = this->meshblock; + // std::swap(mblock.em, mblock.em0); + // std::swap(mblock.cur, mblock.cur0); + // } + + /** + * @brief Copies em fields into em0 + */ + void CopyFields(domain_t& domain) { + Kokkos::deep_copy(domain.fields.em0, domain.fields.em); + } + }; } // namespace ntt From 011e02416f1650268bf5c8cc712d9761f192a5a8 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 25 Sep 2024 15:00:21 -0400 Subject: [PATCH 008/234] engines/grpic: added computation of aux fields --- src/engines/grpic.hpp | 82 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 81 insertions(+), 1 deletion(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index a84a0ed5f..afc452948 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -46,6 +46,15 @@ namespace ntt { + enum class gr_getE { + D0_B, + D_B0 + }; + enum class gr_getH { + D_B0, + D0_B0 + }; + template class GRPICEngine : public Engine { using base_t = Engine; @@ -112,6 +121,15 @@ namespace ntt { * Now: em0::B & em0::D at -1/2 */ CopyFields(dom); + + /** + * aux::E <- alpha * em::D + beta x em0::B + * aux::H <- alpha * em::B0 - beta x em::D + * + * Now: aux::E & aux::H at -1/2 + */ + ComputeAuxE(dom, gr_getE::D0_B); + ComputeAuxH(dom, gr_getH::D_B0); } } @@ -274,10 +292,72 @@ namespace ntt { /** * @brief Copies em fields into em0 */ - void CopyFields(domain_t& domain) { + void CopyFields(domain_t& domain) { Kokkos::deep_copy(domain.fields.em0, domain.fields.em); } + void ComputeAuxE(domain_t& domain, const gr_getE& g) { + auto range = range_with_axis_BCs(domain); + if (g == gr_getE::D0_B) { + Kokkos::parallel_for("ComputeAuxE", + range, + kernel::gr::ComputeAuxE_kernel(domain.fields.em0, + domain.fields.em, + domain.fields.em, + domain.mesh.metric)); + } else if (g == gr_getE::D_B0) { + Kokkos::parallel_for("ComputeAuxE", + range, + kernel::gr::ComputeAuxE_kernel(domain.fields.em, + domain.fields.em0, + domain.fields.em, + domain.mesh.metric)); + } else { + raise::Error("Wrong option for `g`", HERE); + } + } + + void ComputeAuxH(domain_t& domain, const gr_getH& g) { + auto range = range_with_axis_BCs(domain); + if (g == gr_getH::D_B0) { + Kokkos::parallel_for("ComputeAuxH", + range, + kernel::gr::ComputeAuxH_kernel(domain.fields.em, + domain.fields.em0, + domain.fields.em, + domain.mesh.metric)); + } else if (g == gr_getH::D0_B0) { + Kokkos::parallel_for("ComputeAuxH", + range, + kernel::gr::ComputeAuxH_kernel(domain.fields.em0, + domain.fields.em0, + domain.fields.em, + domain.mesh.metric)); + } else { + raise::Error("Wrong option for `g`", HERE); + } + } + + auto range_with_axis_BCs(const domain_t& domain) -> range_t { + auto range = domain.mesh.rangeActiveCells(); + if constexpr (M::CoordType != Coord::Cart) { + /** + * @brief taking one extra cell in the x1 and x2 directions if AXIS BCs + */ + if constexpr (M::Dim == Dim::_2D) { + if (domain.mesh.flds_bc_in({ 0, +1 }) == FldsBC::AXIS) { + range = CreateRangePolicy( + { domain.mesh.i_min(in::x1) - 1, domain.mesh.i_min(in::x2) }, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1 }); + } + } else if constexpr (M::Dim == Dim::_3D) { + raise::Error("Invalid dimension", HERE); + } + } + return range; + } + + }; } // namespace ntt From cdf69dbae9298eac92c1b362ee4dcccd7befdb00 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 25 Sep 2024 16:21:32 -0400 Subject: [PATCH 009/234] engines/grpis: fixed "aux" for passing to aux kernels, added Faraday function --- src/engines/grpic.hpp | 52 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index afc452948..4bdd56fce 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -54,6 +54,15 @@ namespace ntt { D_B0, D0_B0 }; + enum class gr_faraday { + aux, + main + }; + enum class gr_ampere { + init, + aux, + main + }; template class GRPICEngine : public Engine { @@ -130,9 +139,17 @@ namespace ntt { */ ComputeAuxE(dom, gr_getE::D0_B); ComputeAuxH(dom, gr_getH::D_B0); + + /** + * aux::E, aux::H <- boundary conditions + */ + // ?? aux field boundaries ?? + + Faraday(dom, gr_faraday::aux, HALF); } } +/* algorithm substeps --------------------------------------------------- */ void FieldBoundaries(domain_t& domain, BCTags tags) { for (auto& direction : dir::Directions::orth) { if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::ABSORB) { @@ -324,14 +341,14 @@ namespace ntt { range, kernel::gr::ComputeAuxH_kernel(domain.fields.em, domain.fields.em0, - domain.fields.em, + domain.fields.aux, domain.mesh.metric)); } else if (g == gr_getH::D0_B0) { Kokkos::parallel_for("ComputeAuxH", range, kernel::gr::ComputeAuxH_kernel(domain.fields.em0, domain.fields.em0, - domain.fields.em, + domain.fields.aux, domain.mesh.metric)); } else { raise::Error("Wrong option for `g`", HERE); @@ -357,6 +374,37 @@ namespace ntt { return range; } + void Faraday(domain_t& domain, const gr_faraday& g, real_t fraction = ONE) { + logger::Checkpoint("Launching Faraday kernel", HERE); + const auto dT = fraction * + m_params.template get( + "algorithms.timestep.correction") * + dt; + if (g == gr_faraday::aux) { + Kokkos::parallel_for("Faraday", + domain.mesh.rangeActiveCells(), + kernel::gr::Faraday_kernel(domain.fields.em0, + domain.fields.em0, + domain.fields.aux, + domain.mesh.metric, + dT, + domain.mesh.i_max(in::x2), + domain.mesh.flds_bc())); + } else if (g == gr_faraday::main) { + Kokkos::parallel_for("Faraday", + domain.mesh.rangeActiveCells(), + kernel::gr::Faraday_kernel(domain.fields.em0, + domain.fields.em0, + domain.fields.aux, + domain.mesh.metric, + dT, + domain.mesh.i_max(in::x2), + domain.mesh.flds_bc())); + + } else { + raise::Error("Wrong option for `g`", HERE); + } + } }; } // namespace ntt From e96b41cd57c034dab9b4d3794bcd22306e101976 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 25 Sep 2024 16:55:27 -0400 Subject: [PATCH 010/234] minor: added a comment on Faraday step --- src/engines/grpic.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 4bdd56fce..2d670a89c 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -145,6 +145,11 @@ namespace ntt { */ // ?? aux field boundaries ?? + /** + * em0::B <- (em0::B) <- -curl aux::E + * + * Now: em0::B at 0 + */ Faraday(dom, gr_faraday::aux, HALF); } } From ee954128e89b2323f13d874bfae36c96ddf450ec Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 25 Sep 2024 17:00:41 -0400 Subject: [PATCH 011/234] engines/grpic: bc for B&B0 --- src/engines/grpic.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 2d670a89c..afe22eb47 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -151,6 +151,12 @@ namespace ntt { * Now: em0::B at 0 */ Faraday(dom, gr_faraday::aux, HALF); + + /** + * em0::B, em::B <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); + FieldBoundaries(dom, BC::B); } } From 25b4e766088df14039ebe9dcff8b6dcd3c970c98 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 25 Sep 2024 17:21:12 -0400 Subject: [PATCH 012/234] engine/grpic: fixed passing ni2 to faraday kernel. prep for ampere kernel --- src/engines/grpic.hpp | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index afe22eb47..d55d0a6b2 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -157,6 +157,13 @@ namespace ntt { */ m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); FieldBoundaries(dom, BC::B); + + /** + * em::D <- (em0::D) <- curl aux::H + * + * Now: em::D at 0 + */ + Ampere(dom, gr_ampere::init, HALF); } } @@ -399,7 +406,7 @@ namespace ntt { domain.fields.aux, domain.mesh.metric, dT, - domain.mesh.i_max(in::x2), + domain.mesh.n_active(in::x2), domain.mesh.flds_bc())); } else if (g == gr_faraday::main) { Kokkos::parallel_for("Faraday", @@ -409,13 +416,37 @@ namespace ntt { domain.fields.aux, domain.mesh.metric, dT, - domain.mesh.i_max(in::x2), + domain.mesh.n_active(in::x2), domain.mesh.flds_bc())); } else { raise::Error("Wrong option for `g`", HERE); } } + + void Ampere(domain_t& domain, const gr_ampere& g, real_t fraction = ONE) { + logger::Checkpoint("Launching Ampere kernel", HERE); + const auto dT = fraction * + m_params.template get( + "algorithms.timestep.correction") * + dt; + auto range = CreateRangePolicy( + { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) + 1}, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2)}); + auto range_pole = CreateRangePolicy( + { domain.mesh.i_min(in::x1)}, + { domain.mesh.i_max(in::x1)}); + const auto ni2 = domain.mesh.n_active(in::x2); + Kokkos::parallel_for("Ampere", + range, + kernel::gr::Ampere_kernel(domain.fields.em, + domain.fields.em, + domain.fields.em, + domain.mesh.metric, + dT, + ni2, + domain.mesh.flds_bc())); + } }; } // namespace ntt From 9c04fe27c936f37a238706fe36d22f590a50cae1 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 25 Sep 2024 17:51:47 -0400 Subject: [PATCH 013/234] engines/grpic: added ampere and subsequent BC --- src/engines/grpic.hpp | 142 +++++++++++++++++++++++++----------------- 1 file changed, 85 insertions(+), 57 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index d55d0a6b2..3306c1060 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -26,7 +26,6 @@ #include "framework/parameters.h" #include "engines/engine.hpp" - #include "kernels/ampere_gr.hpp" #include "kernels/aux_fields_gr.hpp" #include "kernels/currents_deposit.hpp" @@ -35,7 +34,6 @@ #include "kernels/fields_bcs.hpp" #include "kernels/particle_moments.hpp" #include "kernels/particle_pusher_gr.hpp" - #include "pgen.hpp" #include @@ -102,25 +100,26 @@ namespace ntt { if (step == 0) { // communicate fields and apply BCs on the first timestep /** - * Initially: em0::B -- - * em0::D -- - * em::B at -1/2 - * em::D at -1/2 - * - * cur0::J -- - * cur::J -- - * - * aux::E -- - * aux::H -- - * - * x_prtl at -1/2 - * u_prtl at -1/2 - */ + * Initially: em0::B -- + * em0::D -- + * em::B at -1/2 + * em::D at -1/2 + * + * cur0::J -- + * cur::J -- + * + * aux::E -- + * aux::H -- + * + * x_prtl at -1/2 + * u_prtl at -1/2 + */ /** - * em0::D, em::D, em0::B, em::B <- boundary conditions - */ - m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0 | Comm::D | Comm::D0); + * em0::D, em::D, em0::B, em::B <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, + Comm::B | Comm::B0 | Comm::D | Comm::D0); FieldBoundaries(dom, BC::B | BC::D); /** @@ -167,7 +166,7 @@ namespace ntt { } } -/* algorithm substeps --------------------------------------------------- */ + /* algorithm substeps --------------------------------------------------- */ void FieldBoundaries(domain_t& domain, BCTags tags) { for (auto& direction : dir::Directions::orth) { if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::ABSORB) { @@ -361,7 +360,7 @@ namespace ntt { domain.fields.em0, domain.fields.aux, domain.mesh.metric)); - } else if (g == gr_getH::D0_B0) { + } else if (g == gr_getH::D0_B0) { Kokkos::parallel_for("ComputeAuxH", range, kernel::gr::ComputeAuxH_kernel(domain.fields.em0, @@ -399,31 +398,33 @@ namespace ntt { "algorithms.timestep.correction") * dt; if (g == gr_faraday::aux) { - Kokkos::parallel_for("Faraday", - domain.mesh.rangeActiveCells(), - kernel::gr::Faraday_kernel(domain.fields.em0, - domain.fields.em0, - domain.fields.aux, - domain.mesh.metric, - dT, - domain.mesh.n_active(in::x2), - domain.mesh.flds_bc())); + Kokkos::parallel_for( + "Faraday", + domain.mesh.rangeActiveCells(), + kernel::gr::Faraday_kernel(domain.fields.em0, + domain.fields.em0, + domain.fields.aux, + domain.mesh.metric, + dT, + domain.mesh.n_active(in::x2), + domain.mesh.flds_bc())); } else if (g == gr_faraday::main) { - Kokkos::parallel_for("Faraday", - domain.mesh.rangeActiveCells(), - kernel::gr::Faraday_kernel(domain.fields.em0, - domain.fields.em0, - domain.fields.aux, - domain.mesh.metric, - dT, - domain.mesh.n_active(in::x2), - domain.mesh.flds_bc())); + Kokkos::parallel_for( + "Faraday", + domain.mesh.rangeActiveCells(), + kernel::gr::Faraday_kernel(domain.fields.em0, + domain.fields.em0, + domain.fields.aux, + domain.mesh.metric, + dT, + domain.mesh.n_active(in::x2), + domain.mesh.flds_bc())); } else { - raise::Error("Wrong option for `g`", HERE); - } + raise::Error("Wrong option for `g`", HERE); + } } - + void Ampere(domain_t& domain, const gr_ampere& g, real_t fraction = ONE) { logger::Checkpoint("Launching Ampere kernel", HERE); const auto dT = fraction * @@ -431,23 +432,50 @@ namespace ntt { "algorithms.timestep.correction") * dt; auto range = CreateRangePolicy( - { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) + 1}, - { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2)}); - auto range_pole = CreateRangePolicy( - { domain.mesh.i_min(in::x1)}, - { domain.mesh.i_max(in::x1)}); + { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) + 1 }, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) }); + auto range_pole = CreateRangePolicy({ domain.mesh.i_min(in::x1) }, + { domain.mesh.i_max(in::x1) }); const auto ni2 = domain.mesh.n_active(in::x2); - Kokkos::parallel_for("Ampere", - range, - kernel::gr::Ampere_kernel(domain.fields.em, - domain.fields.em, - domain.fields.em, - domain.mesh.metric, - dT, - ni2, - domain.mesh.flds_bc())); - } + if (g == gr_ampere::aux) { + // First push, updates D0 with J. + Kokkos::parallel_for("Ampere-1", + range, + kernel::gr::Ampere_kernel(domain.fields.em, // has to be zeros + domain.fields.em0, + domain.fields.aux, + domain.mesh.metric, + dT, + ni2, + domain.mesh.flds_bc())); + } else if (g == gr_ampere::main) { + // Second push, updates D with J0 but assigns it to D0. + Kokkos::parallel_for("Ampere-2", + range, + kernel::gr::Ampere_kernel(domain.fields.em, + domain.fields.em0, + domain.fields.aux, + domain.mesh.metric, + dT, + ni2, + domain.mesh.flds_bc())); + } else if (g == gr_ampere::init) { + // Second push, updates D with J0 and assigns it to D. + Kokkos::parallel_for("Ampere-3", + range, + kernel::gr::Ampere_kernel(domain.fields.em, //has to be zeros + domain.fields.em, + domain.fields.aux, + domain.mesh.metric, + dT, + ni2, + domain.mesh.flds_bc())); + } else { + raise::Error("Wrong option for `g`", HERE); + } + + } }; } // namespace ntt From f14b6430e94e7b97aca9bf67d41eebabf61f32ca Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Thu, 26 Sep 2024 15:25:56 -0400 Subject: [PATCH 014/234] engine/grpic: added everything for initial step, except BC for aux --- src/engines/grpic.hpp | 102 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 97 insertions(+), 5 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 3306c1060..d23a966db 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -163,6 +163,99 @@ namespace ntt { * Now: em::D at 0 */ Ampere(dom, gr_ampere::init, HALF); + + /** + * em0::D, em::D <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + FieldBoundaries(dom, BC::D); + + /** + * aux::E <- alpha * em::D + beta x em0::B + * aux::H <- alpha * em0::B - beta x em::D + * + * Now: aux::E & aux::H at 0 + */ + ComputeAuxE(dom, gr_getE::D_B0); + ComputeAuxH(dom, gr_getH::D_B0); + + /** + * aux::E, aux::H <- boundary conditions + */ + // ?? aux field boundaries ?? + + // !ADD: GR -- particles? + + /** + * em0::B <- (em::B) <- -curl aux::E + * + * Now: em0::B at 1/2 + */ + Faraday(dom, gr_faraday::main, ONE); + /** + * em0::B, em::B <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); + FieldBoundaries(dom, BC::B); + + /** + * em0::D <- (em0::D) <- curl aux::H + * + * Now: em0::D at 1/2 + */ + Ampere(dom, gr_ampere::aux, ONE); + /** + * em0::D, em::D <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + FieldBoundaries(dom, BC::D); + + /** + * aux::H <- alpha * em0::B - beta x em0::D + * + * Now: aux::H at 1/2 + */ + ComputeAuxH(dom, gr_getH::D0_B0); + /** + * aux::H <- boundary conditions + */ + // ?? aux field boundaries ?? + + /** + * em0::D <- (em::D) <- curl aux::H + * + * Now: em0::D at 1 + * em::D at 0 + */ + Ampere(dom, gr_ampere::main, ONE); + /** + * em0::D, em::D <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + FieldBoundaries(dom, BC::D); + + /** + * em::D <-> em0::D + * em::B <-> em0::B + * em::J <-> em0::J + */ + SwapFields(dom); + /** + * Finally: em0::B at -1/2 + * em0::D at 0 + * em::B at 1/2 + * em::D at 1 + * + * cur0::J -- + * cur::J -- + * + * aux::E -- + * aux::H -- + * + * x_prtl at 1 + * u_prtl at 1/2 + */ + } } @@ -317,11 +410,10 @@ namespace ntt { /** * @brief Swaps em and em0 fields, cur and cur0 currents. */ - // void SwapFields() { - // auto& mblock = this->meshblock; - // std::swap(mblock.em, mblock.em0); - // std::swap(mblock.cur, mblock.cur0); - // } + void SwapFields(domain_t& domain) { + std::swap(domain.fields.em, domain.fields.em0); + std::swap(domain.fields.cur, domain.fields.cur0); + } /** * @brief Copies em fields into em0 From 910e535a88687c24c9917d59767fe0e91183033f Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 27 Sep 2024 13:26:17 -0400 Subject: [PATCH 015/234] engines/grpis: minor update to first aux step for consistency --- src/engines/grpic.hpp | 51 +++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index d23a966db..7306e08ab 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -118,8 +118,7 @@ namespace ntt { /** * em0::D, em::D, em0::B, em::B <- boundary conditions */ - m_metadomain.CommunicateFields(dom, - Comm::B | Comm::B0 | Comm::D | Comm::D0); + m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0 | Comm::D | Comm::D0); FieldBoundaries(dom, BC::B | BC::D); /** @@ -136,7 +135,7 @@ namespace ntt { * * Now: aux::E & aux::H at -1/2 */ - ComputeAuxE(dom, gr_getE::D0_B); + ComputeAuxE(dom, gr_getE::D_B0); ComputeAuxH(dom, gr_getH::D_B0); /** @@ -257,6 +256,20 @@ namespace ntt { */ } + + if (fieldsolver_enabled) { + + } + + { + + + } + + if (fieldsolver_enabled) { + + } + } /* algorithm substeps --------------------------------------------------- */ @@ -466,19 +479,17 @@ namespace ntt { auto range_with_axis_BCs(const domain_t& domain) -> range_t { auto range = domain.mesh.rangeActiveCells(); - if constexpr (M::CoordType != Coord::Cart) { - /** - * @brief taking one extra cell in the x1 and x2 directions if AXIS BCs - */ - if constexpr (M::Dim == Dim::_2D) { - if (domain.mesh.flds_bc_in({ 0, +1 }) == FldsBC::AXIS) { - range = CreateRangePolicy( - { domain.mesh.i_min(in::x1) - 1, domain.mesh.i_min(in::x2) }, - { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1 }); - } - } else if constexpr (M::Dim == Dim::_3D) { - raise::Error("Invalid dimension", HERE); + /** + * @brief taking one extra cell in the x1 and x2 directions if AXIS BCs + */ + if constexpr (M::Dim == Dim::_2D) { + if (domain.mesh.flds_bc_in({ 0, +1 }) == FldsBC::AXIS) { + range = CreateRangePolicy( + { domain.mesh.i_min(in::x1) - 1, domain.mesh.i_min(in::x2) }, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1 }); } + } else if constexpr (M::Dim == Dim::_3D) { + raise::Error("Invalid dimension", HERE); } return range; } @@ -524,17 +535,15 @@ namespace ntt { "algorithms.timestep.correction") * dt; auto range = CreateRangePolicy( - { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) + 1 }, - { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) }); - auto range_pole = CreateRangePolicy({ domain.mesh.i_min(in::x1) }, - { domain.mesh.i_max(in::x1) }); + { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2)}, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1}); const auto ni2 = domain.mesh.n_active(in::x2); if (g == gr_ampere::aux) { // First push, updates D0 with J. Kokkos::parallel_for("Ampere-1", range, - kernel::gr::Ampere_kernel(domain.fields.em, // has to be zeros + kernel::gr::Ampere_kernel(domain.fields.em, domain.fields.em0, domain.fields.aux, domain.mesh.metric, @@ -556,7 +565,7 @@ namespace ntt { // Second push, updates D with J0 and assigns it to D. Kokkos::parallel_for("Ampere-3", range, - kernel::gr::Ampere_kernel(domain.fields.em, //has to be zeros + kernel::gr::Ampere_kernel(domain.fields.em, domain.fields.em, domain.fields.aux, domain.mesh.metric, From faf471b60f40fab90cbb8b64b3767f6199e370f7 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 27 Sep 2024 13:28:35 -0400 Subject: [PATCH 016/234] engines/grpic: fixed ComputeAuxE function --- src/engines/grpic.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 7306e08ab..7c3d13fbe 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -442,14 +442,14 @@ namespace ntt { range, kernel::gr::ComputeAuxE_kernel(domain.fields.em0, domain.fields.em, - domain.fields.em, + domain.fields.aux, domain.mesh.metric)); } else if (g == gr_getE::D_B0) { Kokkos::parallel_for("ComputeAuxE", range, kernel::gr::ComputeAuxE_kernel(domain.fields.em, domain.fields.em0, - domain.fields.em, + domain.fields.aux, domain.mesh.metric)); } else { raise::Error("Wrong option for `g`", HERE); From dfe51f4fd1d08dc1abcfce866040b77bb5991be2 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 27 Sep 2024 13:30:28 -0400 Subject: [PATCH 017/234] engines/grpic: fixed Faraday function --- src/engines/grpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 7c3d13fbe..de7f04752 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -515,7 +515,7 @@ namespace ntt { Kokkos::parallel_for( "Faraday", domain.mesh.rangeActiveCells(), - kernel::gr::Faraday_kernel(domain.fields.em0, + kernel::gr::Faraday_kernel(domain.fields.em, domain.fields.em0, domain.fields.aux, domain.mesh.metric, From e57cbf8fa2e787f39e93ae10c7ccb1ef95cf6bb6 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 27 Sep 2024 13:32:17 -0400 Subject: [PATCH 018/234] engines/grpis: fixed Ampere function --- src/engines/grpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index de7f04752..6a7e23540 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -543,7 +543,7 @@ namespace ntt { // First push, updates D0 with J. Kokkos::parallel_for("Ampere-1", range, - kernel::gr::Ampere_kernel(domain.fields.em, + kernel::gr::Ampere_kernel(domain.fields.em0, domain.fields.em0, domain.fields.aux, domain.mesh.metric, From eeac7ff3474c10083550604639246c9cf7f2317b Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 27 Sep 2024 13:45:25 -0400 Subject: [PATCH 019/234] engines/grpic: added a class for field_bc to accomodate passing em, em0, aux to BC kernels --- src/engines/grpic.hpp | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 6a7e23540..14ecffb28 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -61,6 +61,11 @@ namespace ntt { aux, main }; + enum class gr_bc { + main, + main0, + aux + }; template class GRPICEngine : public Engine { @@ -273,17 +278,17 @@ namespace ntt { } /* algorithm substeps --------------------------------------------------- */ - void FieldBoundaries(domain_t& domain, BCTags tags) { + void FieldBoundaries(domain_t& domain, BCTags tags, const gr_bc& g) { for (auto& direction : dir::Directions::orth) { if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::ABSORB) { - AbsorbFieldsIn(direction, domain, tags); + AbsorbFieldsIn(direction, domain, tags, g); } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::AXIS) { if (domain.mesh.flds_bc_in(direction) == FldsBC::AXIS) { - AxisFieldsIn(direction, domain, tags); + AxisFieldsIn(direction, domain, tags, g); } } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::CUSTOM) { if (domain.mesh.flds_bc_in(direction) == FldsBC::CUSTOM) { - CustomFieldsIn(direction, domain, tags); + CustomFieldsIn(direction, domain, tags, g); } } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { raise::Error("HORIZON BCs only applicable for GR", HERE); @@ -293,7 +298,8 @@ namespace ntt { void AbsorbFieldsIn(dir::direction_t direction, domain_t& domain, - BCTags tags) { + BCTags tags + const gr_getE& g) { /** * absorbing boundaries */ @@ -377,7 +383,8 @@ namespace ntt { void AxisFieldsIn(dir::direction_t direction, domain_t& domain, - BCTags tags) { + BCTags tags + const gr_getE& g) { /** * axis boundaries */ @@ -404,7 +411,8 @@ namespace ntt { void CustomFieldsIn(dir::direction_t direction, domain_t& domain, - BCTags tags) { + BCTags tags + const gr_getE& g) { (void)direction; (void)domain; (void)tags; From c84140031d34576445e4471cb3a6c954cfc1abbe Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 27 Sep 2024 14:06:50 -0400 Subject: [PATCH 020/234] engines/grpic: cut out 3d from absorb BCs --- src/engines/grpic.hpp | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 14ecffb28..5c5a54770 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -353,7 +353,7 @@ namespace ntt { ds, tags)); } else if (dim == in::x2) { - if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { + if constexpr (M::Dim == Dim::_2D) { Kokkos::parallel_for( "AbsorbFields", CreateRangePolicy(range_min, range_max), @@ -365,19 +365,8 @@ namespace ntt { } else { raise::Error("Invalid dimension", HERE); } - } else if (dim == in::x3) { - if constexpr (M::Dim == Dim::_3D) { - Kokkos::parallel_for( - "AbsorbFields", - CreateRangePolicy(range_min, range_max), - kernel::AbsorbBoundaries_kernel(domain.fields.em, - domain.mesh.metric, - xg_edge, - ds, - tags)); - } else { + } else { raise::Error("Invalid dimension", HERE); - } } } From 03b05593729e7689844f82ba14a9bc925ce0e26f Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 27 Sep 2024 14:10:10 -0400 Subject: [PATCH 021/234] minot: fix syntax --- src/engines/grpic.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 5c5a54770..9032448ad 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -298,8 +298,8 @@ namespace ntt { void AbsorbFieldsIn(dir::direction_t direction, domain_t& domain, - BCTags tags - const gr_getE& g) { + BCTags tags, + const gr_bc& g) { /** * absorbing boundaries */ @@ -372,8 +372,8 @@ namespace ntt { void AxisFieldsIn(dir::direction_t direction, domain_t& domain, - BCTags tags - const gr_getE& g) { + BCTags tags, + const gr_bc& g) { /** * axis boundaries */ @@ -400,8 +400,8 @@ namespace ntt { void CustomFieldsIn(dir::direction_t direction, domain_t& domain, - BCTags tags - const gr_getE& g) { + BCTags tags, + const gr_bc& g) { (void)direction; (void)domain; (void)tags; From cbe2fee224701675b5753d426e42e8332ed7c0d9 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 27 Sep 2024 14:10:44 -0400 Subject: [PATCH 022/234] minor: cut out options from gr_bc class --- src/engines/grpic.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 9032448ad..982e76ba9 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -63,7 +63,6 @@ namespace ntt { }; enum class gr_bc { main, - main0, aux }; From 67ec72092caf3eefacfaf13c72ed4d587c5bf422 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 27 Sep 2024 14:12:48 -0400 Subject: [PATCH 023/234] minor: cut out gr_bc from most bc functions since only horizon bc needs aux fields; em and em0 are always set together --- src/engines/grpic.hpp | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 982e76ba9..4fc809a08 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -280,14 +280,14 @@ namespace ntt { void FieldBoundaries(domain_t& domain, BCTags tags, const gr_bc& g) { for (auto& direction : dir::Directions::orth) { if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::ABSORB) { - AbsorbFieldsIn(direction, domain, tags, g); + AbsorbFieldsIn(direction, domain, tags); } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::AXIS) { if (domain.mesh.flds_bc_in(direction) == FldsBC::AXIS) { - AxisFieldsIn(direction, domain, tags, g); + AxisFieldsIn(direction, domain, tags); } } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::CUSTOM) { if (domain.mesh.flds_bc_in(direction) == FldsBC::CUSTOM) { - CustomFieldsIn(direction, domain, tags, g); + CustomFieldsIn(direction, domain, tags); } } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { raise::Error("HORIZON BCs only applicable for GR", HERE); @@ -297,8 +297,7 @@ namespace ntt { void AbsorbFieldsIn(dir::direction_t direction, domain_t& domain, - BCTags tags, - const gr_bc& g) { + BCTags tags) { /** * absorbing boundaries */ @@ -371,8 +370,7 @@ namespace ntt { void AxisFieldsIn(dir::direction_t direction, domain_t& domain, - BCTags tags, - const gr_bc& g) { + BCTags tags) { /** * axis boundaries */ From 34a42a4b8b8adabeab4774339e94ffb8d05afe62 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 27 Sep 2024 14:15:03 -0400 Subject: [PATCH 024/234] engines/grpic: added gr_bc into step=0 functions --- src/engines/grpic.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 4fc809a08..5a8fb3987 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -123,7 +123,7 @@ namespace ntt { * em0::D, em::D, em0::B, em::B <- boundary conditions */ m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0 | Comm::D | Comm::D0); - FieldBoundaries(dom, BC::B | BC::D); + FieldBoundaries(dom, BC::B | BC::D, gr_bc::main); /** * em0::B <- em::B @@ -158,7 +158,7 @@ namespace ntt { * em0::B, em::B <- boundary conditions */ m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); - FieldBoundaries(dom, BC::B); + FieldBoundaries(dom, BC::B, gr_bc::main); /** * em::D <- (em0::D) <- curl aux::H @@ -171,7 +171,7 @@ namespace ntt { * em0::D, em::D <- boundary conditions */ m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); - FieldBoundaries(dom, BC::D); + FieldBoundaries(dom, BC::D, gr_bc::main); /** * aux::E <- alpha * em::D + beta x em0::B @@ -199,7 +199,7 @@ namespace ntt { * em0::B, em::B <- boundary conditions */ m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); - FieldBoundaries(dom, BC::B); + FieldBoundaries(dom, BC::B, gr_bc::main); /** * em0::D <- (em0::D) <- curl aux::H @@ -211,7 +211,7 @@ namespace ntt { * em0::D, em::D <- boundary conditions */ m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); - FieldBoundaries(dom, BC::D); + FieldBoundaries(dom, BC::D, gr_bc::main); /** * aux::H <- alpha * em0::B - beta x em0::D @@ -235,7 +235,7 @@ namespace ntt { * em0::D, em::D <- boundary conditions */ m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); - FieldBoundaries(dom, BC::D); + FieldBoundaries(dom, BC::D, gr_bc::main); /** * em::D <-> em0::D From a3d5db85e50087b8412b8b09f7c26ab6a27b0e87 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 27 Sep 2024 14:22:49 -0400 Subject: [PATCH 025/234] engines/grpic: added BC for em0 --- src/engines/grpic.hpp | 56 +++++++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 5a8fb3987..cdce0c593 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -278,21 +278,29 @@ namespace ntt { /* algorithm substeps --------------------------------------------------- */ void FieldBoundaries(domain_t& domain, BCTags tags, const gr_bc& g) { - for (auto& direction : dir::Directions::orth) { - if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::ABSORB) { - AbsorbFieldsIn(direction, domain, tags); - } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::AXIS) { - if (domain.mesh.flds_bc_in(direction) == FldsBC::AXIS) { - AxisFieldsIn(direction, domain, tags); + if (g == gr_bc::main) { + for (auto& direction : dir::Directions::orth) { + if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::ABSORB) { + AbsorbFieldsIn(direction, domain, tags); + } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::AXIS) { + if (domain.mesh.flds_bc_in(direction) == FldsBC::AXIS) { + AxisFieldsIn(direction, domain, tags); + } + } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::CUSTOM) { + if (domain.mesh.flds_bc_in(direction) == FldsBC::CUSTOM) { + CustomFieldsIn(direction, domain, tags, g); + } + } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { + raise::Error("HORIZON BCs only applicable for GR", HERE); } - } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::CUSTOM) { - if (domain.mesh.flds_bc_in(direction) == FldsBC::CUSTOM) { - CustomFieldsIn(direction, domain, tags); + } // loop over directions + } else if (g == gr_bc::aux) { + for (auto& direction : dir::Directions::orth) { + if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { + raise::Error("HORIZON BCs only applicable for GR", HERE); } - } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { - raise::Error("HORIZON BCs only applicable for GR", HERE); } - } // loop over directions + } } void AbsorbFieldsIn(dir::direction_t direction, @@ -350,6 +358,14 @@ namespace ntt { xg_edge, ds, tags)); + Kokkos::parallel_for( + "AbsorbFields", + CreateRangePolicy(range_min, range_max), + kernel::AbsorbBoundaries_kernel(domain.fields.em0, + domain.mesh.metric, + xg_edge, + ds, + tags)); } else if (dim == in::x2) { if constexpr (M::Dim == Dim::_2D) { Kokkos::parallel_for( @@ -360,6 +376,14 @@ namespace ntt { xg_edge, ds, tags)); + Kokkos::parallel_for( + "AbsorbFields", + CreateRangePolicy(range_min, range_max), + kernel::AbsorbBoundaries_kernel(domain.fields.em0, + domain.mesh.metric, + xg_edge, + ds, + tags)); } else { raise::Error("Invalid dimension", HERE); } @@ -387,11 +411,19 @@ namespace ntt { "AxisBCFields", domain.mesh.n_all(in::x1), kernel::AxisBoundaries_kernel(domain.fields.em, i2_min, tags)); + Kokkos::parallel_for( + "AxisBCFields", + domain.mesh.n_all(in::x1), + kernel::AxisBoundaries_kernel(domain.fields.em0, i2_min, tags)); } else { Kokkos::parallel_for( "AxisBCFields", domain.mesh.n_all(in::x1), kernel::AxisBoundaries_kernel(domain.fields.em, i2_max, tags)); + Kokkos::parallel_for( + "AxisBCFields", + domain.mesh.n_all(in::x1), + kernel::AxisBoundaries_kernel(domain.fields.em0, i2_max, tags)); } } From 313850bb0a20fd78dc690686eb2d1946d8154d2d Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 27 Sep 2024 16:07:17 -0400 Subject: [PATCH 026/234] added a kernel for OpenBoundaries --- src/kernels/fields_bcs.hpp | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index e617010b4..5f1e6255e 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -489,6 +489,40 @@ namespace kernel { } }; +template + struct OpenBoundaries_kernel { + ndfield_t Fld; + const std::size_t i1_min; + const bool setE, setB; + + OpenBoundaries_kernel(ndfield_t Fld, BCTags tags) + : Fld { Fld } + , i1_min { i1_min } + , setE { tags & BC::Ex1 or tags & BC::Ex2 or tags & BC::Ex3 } + , setB { tags & BC::Bx1 or tags & BC::Bx2 or tags & BC::Bx3 } {} + + Inline void operator()(index_t i2) const { + if constexpr (M::Dim == Dim::_2D) { + if (setE) { + Fld(i1_min - 1, i2, em::ex1) = Fld(i1_min , i2, em::ex1); + Fld(i1_min , i2, em::ex2) = Fld(i1_min + 1, i2, em::ex2); + Fld(i1_min - 1, i2, em::ex2) = Fld(i1_min , i2, em::ex2); + Fld(i1_min , i2, em::ex3) = Fld(i1_min + 1, i2, em::ex3); + Fld(i1_min - 1, i2, em::ex3) = Fld(i1_min , i2, em::ex3); + } else if (setB) { + Fld(i1_min , i2, em::bx1) = Fld(i1_min + 1, i2, em::bx1); + Fld(i1_min - 1, i2, em::bx1) = Fld(i1_min , i2, em::bx1); + Fld(i1_min - 1, i2, em::bx2) = Fld(i1_min , i2, em::bx2); + Fld(i1_min - 1, i2, em::bx3) = Fld(i1_min , i2, em::bx3); + } + } else { + raise::KernelError( + HERE, + "AbsorbFields_kernel: 2D implementation called for D != 2"); + } + } + }; + } // namespace kernel #endif // KERNELS_FIELDS_BCS_HPP From 0d05349d207231bc6ebc775378a75f394fa7270d Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Mon, 30 Sep 2024 15:48:22 -0400 Subject: [PATCH 027/234] engines/grpic: implemented function to call for OpenBoundaries_kernel kernels/fields_bcs: fixed parameter passing in OpenBoundaries_kernel --- src/engines/grpic.hpp | 35 +++++++++++++++++++++++++++++++++-- src/kernels/fields_bcs.hpp | 2 +- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index cdce0c593..54ffd3872 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -291,13 +291,13 @@ namespace ntt { CustomFieldsIn(direction, domain, tags, g); } } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { - raise::Error("HORIZON BCs only applicable for GR", HERE); + OpenFieldsIn(direction, domain, tags); } } // loop over directions } else if (g == gr_bc::aux) { for (auto& direction : dir::Directions::orth) { if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { - raise::Error("HORIZON BCs only applicable for GR", HERE); + OpenFieldsIn(direction, domain, tags); } } } @@ -392,6 +392,37 @@ namespace ntt { } } + void OpenFieldsIn(dir::direction_t direction, + domain_t& domain, + BCTags tags, + const gr_bc& g) { + /** + * open boundaries + */ + raise::ErrorIf(M::CoordType == Coord::Cart, + "Invalid coordinate type for axis BCs", + HERE); + raise::ErrorIf(direction.get_dim() != in::x1, + "Invalid axis direction, should be x2", + HERE); + const auto i1_min = domain.mesh.i_min(in::x1); + if (g == gr_bc::main) { + Kokkos::parallel_for( + "OpenBCFields", + domain.mesh.n_all(in::x1), + kernel::OpenBoundaries_kernel(domain.fields.em, i1_min, tags)); + Kokkos::parallel_for( + "OpenBCFields", + domain.mesh.n_all(in::x1), + kernel::AxisBoundaries_kernel(domain.fields.em0, i1_min, tags)); + } else if (g == gr_bc::aux) { + Kokkos::parallel_for( + "OpenBCFields", + domain.mesh.n_all(in::x1), + kernel::OpenBoundaries_kernel(domain.fields.aux, i1_min, tags)); + } + } + void AxisFieldsIn(dir::direction_t direction, domain_t& domain, BCTags tags) { diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 5f1e6255e..5a50b6daf 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -495,7 +495,7 @@ template const std::size_t i1_min; const bool setE, setB; - OpenBoundaries_kernel(ndfield_t Fld, BCTags tags) + OpenBoundaries_kernel(ndfield_t Fld, std::size_t i1_min, BCTags tags) : Fld { Fld } , i1_min { i1_min } , setE { tags & BC::Ex1 or tags & BC::Ex2 or tags & BC::Ex3 } From 9c8efaa0e8cf676e461a11a09378f21bdb23291a Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Mon, 30 Sep 2024 15:54:30 -0400 Subject: [PATCH 028/234] enginesgrpic: added aux field boundaries calls to initial step --- src/engines/grpic.hpp | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 54ffd3872..6da68dc18 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -145,7 +145,7 @@ namespace ntt { /** * aux::E, aux::H <- boundary conditions */ - // ?? aux field boundaries ?? + FieldBoundaries(dom, BC::B | BC::D, gr_bc::aux); /** * em0::B <- (em0::B) <- -curl aux::E @@ -185,7 +185,7 @@ namespace ntt { /** * aux::E, aux::H <- boundary conditions */ - // ?? aux field boundaries ?? + FieldBoundaries(dom, BC::B | BC::D, gr_bc::aux); // !ADD: GR -- particles? @@ -222,7 +222,7 @@ namespace ntt { /** * aux::H <- boundary conditions */ - // ?? aux field boundaries ?? + FieldBoundaries(dom, BC::B, gr_bc::aux); /** * em0::D <- (em::D) <- curl aux::H @@ -258,7 +258,6 @@ namespace ntt { * x_prtl at 1 * u_prtl at 1/2 */ - } if (fieldsolver_enabled) { @@ -291,13 +290,13 @@ namespace ntt { CustomFieldsIn(direction, domain, tags, g); } } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { - OpenFieldsIn(direction, domain, tags); + OpenFieldsIn(direction, domain, tags, g); } } // loop over directions } else if (g == gr_bc::aux) { for (auto& direction : dir::Directions::orth) { if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { - OpenFieldsIn(direction, domain, tags); + OpenFieldsIn(direction, domain, tags, g); } } } @@ -409,16 +408,16 @@ namespace ntt { if (g == gr_bc::main) { Kokkos::parallel_for( "OpenBCFields", - domain.mesh.n_all(in::x1), + domain.mesh.n_all(in::x2), kernel::OpenBoundaries_kernel(domain.fields.em, i1_min, tags)); Kokkos::parallel_for( "OpenBCFields", - domain.mesh.n_all(in::x1), - kernel::AxisBoundaries_kernel(domain.fields.em0, i1_min, tags)); + domain.mesh.n_all(in::x2), + kernel::OpenBoundaries_kernel(domain.fields.em0, i1_min, tags)); } else if (g == gr_bc::aux) { Kokkos::parallel_for( "OpenBCFields", - domain.mesh.n_all(in::x1), + domain.mesh.n_all(in::x2), kernel::OpenBoundaries_kernel(domain.fields.aux, i1_min, tags)); } } From 2569aec93f3dbe42fbe4bee1d0772f55f4a07ac7 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Mon, 30 Sep 2024 16:16:18 -0400 Subject: [PATCH 029/234] added TimeAverageDB() and kernel for it in kernels/aux --- src/engines/grpic.hpp | 34 ++++++++++++++++++++++++++++++- src/kernels/aux_fields_gr.hpp | 38 +++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 6da68dc18..a75742be2 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -260,8 +260,31 @@ namespace ntt { */ } - if (fieldsolver_enabled) { + /** + * Initially: em0::B at n-3/2 + * em0::D at n-1 + * em::B at n-1/2 + * em::D at n + * + * cur0::J -- + * cur::J at n-1/2 + * + * aux::E -- + * aux::H -- + * + * x_prtl at n + * u_prtl at n-1/2 + */ + if (fieldsolver_enabled) { + /** + * em0::D <- (em0::D + em::D) / 2 + * em0::B <- (em0::B + em::B) / 2 + * + * Now: em0::D at n-1/2 + * em0::B at n-1 + */ + TimeAverageDB(dom); } { @@ -633,6 +656,15 @@ namespace ntt { } } + + void TimeAverageDB(domain_t& domain) { + auto range = range_with_axis_BCs(domain); + Kokkos::parallel_for("TimeAverageDB", + range, + kernel::gr::TimeAverageDB_kernel(domain.fields.em, + domain.fields.em0, + domain.mesh.metric)); + } }; } // namespace ntt diff --git a/src/kernels/aux_fields_gr.hpp b/src/kernels/aux_fields_gr.hpp index 5744c3092..4f87ba1f2 100644 --- a/src/kernels/aux_fields_gr.hpp +++ b/src/kernels/aux_fields_gr.hpp @@ -230,6 +230,44 @@ namespace kernel::gr { } } }; + + /** + * @brief Kernel for computing time average of B and D + * @tparam M Metric + */ + template + class TimeAverageDB_kernel { + static_assert(M::is_metric, "M must be a metric class"); + static constexpr auto D = M::Dim; + + const ndfield_t BDf; + ndfield_t BDf0; + const M metric; + + public: + TimeAverageDB_kernel(const ndfield_t& BDf, + const ndfield_t& BDf0, + const M& metric) + : BDf { BDf } + , BDf0 { BDf0 } + , metric { metric } {} + + Inline void operator()(index_t i1, index_t i2) const { + if constexpr (D == Dim::_2D) { + BDf0(i1, i2, em::bx1) = HALF * (BDf0(i1, i2, em::bx1) + BDf(i1, i2, em::bx1)); + BDf0(i1, i2, em::bx2) = HALF * (BDf0(i1, i2, em::bx2) + BDf(i1, i2, em::bx2)); + BDf0(i1, i2, em::bx3) = HALF * (BDf0(i1, i2, em::bx3) + BDf(i1, i2, em::bx3)); + BDf0(i1, i2, em::ex1) = HALF * (BDf0(i1, i2, em::bx1) + BDf(i1, i2, em::ex1)); + BDf0(i1, i2, em::ex2) = HALF * (BDf0(i1, i2, em::bx2) + BDf(i1, i2, em::ex2)); + BDf0(i1, i2, em::ex3) = HALF * (BDf0(i1, i2, em::bx3) + BDf(i1, i2, em::ex3)); + } else { + raise::KernelError( + HERE, + "ComputeAuxH_kernel: 2D implementation called for D != 2"); + } + } + + }; } // namespace kernel::gr #endif // KERNELS_AUX_FIELDS_GR_HPP From 3cac38afd72002ffbcefd533b6ecbfa16f6a6eea Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Mon, 30 Sep 2024 16:26:17 -0400 Subject: [PATCH 030/234] engines/grpic: first half of the main step for field solver --- src/engines/grpic.hpp | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index a75742be2..152b55e66 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -285,15 +285,53 @@ namespace ntt { * em0::B at n-1 */ TimeAverageDB(dom); + /** + * aux::E <- alpha * em0::D + beta x em::B + * + * Now: aux::E at n-1/2 + */ + ComputeAuxE(dom, gr_getE::D0_B); + /** + * aux::E <- boundary conditions + */ + FieldBoundaries(dom, BC::D, gr_bc::aux); + /** + * em0::B <- (em0::B) <- -curl aux::E + * + * Now: em0::B at n + */ + Faraday(dom, gr_faraday::aux, ONE); + + + /** + * em0::B, em::B <- boundary conditions + */ + FieldBoundaries(dom, BC::B, gr_bc::main); + m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); + + + /** + * aux::H <- alpha * em0::B - beta x em::D + * + * Now: aux::H at n + */ + ComputeAuxH(dom, gr_getH::D_B0); + /** + * aux::H <- boundary conditions + */ + FieldBoundaries(dom, BC::B, gr_bc::aux); } { - + /** + * particle pusher goes here + * + */ } if (fieldsolver_enabled) { - + } } From 386958f46d23341462a45457f1398cf5120f0c73 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Mon, 30 Sep 2024 16:36:45 -0400 Subject: [PATCH 031/234] added TimeAverageJ() and its kernel in kernels/aux --- src/engines/grpic.hpp | 11 ++++++++++- src/kernels/aux_fields_gr.hpp | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 152b55e66..3a25f5ad2 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -331,7 +331,7 @@ namespace ntt { } if (fieldsolver_enabled) { - + TimeAverageJ(dom); } } @@ -703,6 +703,15 @@ namespace ntt { domain.fields.em0, domain.mesh.metric)); } + + void TimeAverageJ(domain_t& domain) { + auto range = range_with_axis_BCs(domain); + Kokkos::parallel_for("TimeAverageJ", + range, + kernel::gr::TimeAverageJ_kernel(domain.fields.cur, + domain.fields.cur0, + domain.mesh.metric)); + } }; } // namespace ntt diff --git a/src/kernels/aux_fields_gr.hpp b/src/kernels/aux_fields_gr.hpp index 4f87ba1f2..b56a819f9 100644 --- a/src/kernels/aux_fields_gr.hpp +++ b/src/kernels/aux_fields_gr.hpp @@ -266,7 +266,40 @@ namespace kernel::gr { "ComputeAuxH_kernel: 2D implementation called for D != 2"); } } + }; + /** + * @brief Kernel for computing time average of J + * @tparam M Metric + */ + template + class TimeAverageJ_kernel { + static_assert(M::is_metric, "M must be a metric class"); + static constexpr auto D = M::Dim; + + const ndfield_t Jf; + ndfield_t Jf0; + const M metric; + + public: + TimeAverageJ_kernel(const ndfield_t& Jf, + const ndfield_t& Jf0, + const M& metric) + : Jf { Jf } + , Jf0 { Jf0 } + , metric { metric } {} + + Inline void operator()(index_t i1, index_t i2) const { + if constexpr (D == Dim::_2D) { + Jf0(i1, i2, cur::jx1) = HALF * (Jf0(i1, i2, cur::jx1) + Jf(i1, i2, cur::jx1)); + Jf0(i1, i2, cur::jx2) = HALF * (Jf0(i1, i2, cur::jx2) + Jf(i1, i2, cur::jx2)); + Jf0(i1, i2, cur::jx3) = HALF * (Jf0(i1, i2, cur::jx3) + Jf(i1, i2, cur::jx3)); + } else { + raise::KernelError( + HERE, + "ComputeAuxH_kernel: 2D implementation called for D != 2"); + } + } }; } // namespace kernel::gr From 139d179fa5633d27e580214d8652233dedfc4490 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Mon, 30 Sep 2024 16:50:56 -0400 Subject: [PATCH 032/234] engines/grpic: added all steps relevant to field solver --- src/engines/grpic.hpp | 109 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 108 insertions(+), 1 deletion(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 3a25f5ad2..c35c3f44c 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -331,9 +331,116 @@ namespace ntt { } if (fieldsolver_enabled) { + /** + * cur::J <- (cur0::J + cur::J) / 2 + * + * Now: cur::J at n + */ TimeAverageJ(dom); - } + + /** + * aux::Е <- alpha * em::D + beta x em0::B + * + * Now: aux::Е at n + */ + ComputeAuxE(dom, gr_getE::D_B0); + /** + * aux::Е <- boundary conditions + */ + FieldBoundaries(dom, BC::D, gr_bc::aux); + /** + * em0::B <- (em::B) <- -curl aux::E + * + * Now: em0::B at n+1/2 + * em::B at n-1/2 + */ + Faraday(dom, gr_faraday::main, ONE); + + /** + * em0::B, em::B <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); + FieldBoundaries(dom, BC::B, gr_bc::main); + + + /** + * em0::D <- (em0::D) <- curl aux::H + * + * Now: em0::D at n+1/2 + */ + Ampere(dom, gr_ampere::aux, ONE); + + + if (deposit_enabled) { + /** + * em0::D <- (em0::D) <- cur::J + * + * Now: em0::D at n+1/2 + */ + // AmpereCurrents(gr_ampere::aux); + } + + /** + * em0::D, em::D <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + FieldBoundaries(dom, BC::D, gr_bc::main); + + /** + * aux::H <- alpha * em0::B - beta x em0::D + * + * Now: aux::H at n+1/2 + */ + ComputeAuxH(dom, gr_getH::D0_B0); + /** + * aux::H <- boundary conditions + */ + FieldBoundaries(dom, BC::B, gr_bc::aux); + /** + * em0::D <- (em::D) <- curl aux::H + * + * Now: em0::D at n+1 + * em::D at n + */ + Ampere(dom, gr_ampere::main, ONE); + if (deposit_enabled) { + /** + * em0::D <- (em0::D) <- cur0::J + * + * Now: em0::D at n+1 + */ + // AmpereCurrents(gr_ampere::main); + } + /** + * em::D <-> em0::D + * em::B <-> em0::B + * cur::J <-> cur0::J + */ + SwapFields(dom); + + + /** + * em0::D, em::D <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + FieldBoundaries(dom, BC::D, gr_bc::main); + } + /** + * Finally: em0::B at n-1/2 + * em0::D at n + * em::B at n+1/2 + * em::D at n+1 + * + * cur0::J (at n) + * cur::J at n+1/2 + * + * aux::E (at n+1/2) + * aux::H (at n) + * + * x_prtl at n+1 + * u_prtl at n+1/2 + */ } /* algorithm substeps --------------------------------------------------- */ From 52fb48be7905c635a3996af90de66895e99e3ba6 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Mon, 30 Sep 2024 17:09:21 -0400 Subject: [PATCH 033/234] engines/grpic: added function for Ampere currents --- src/engines/grpic.hpp | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index c35c3f44c..60c00b1d9 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -799,6 +799,40 @@ namespace ntt { } else { raise::Error("Wrong option for `g`", HERE); } + } + + void AmpereCurrents(domain_t& domain, const gr_ampere& g) { + logger::Checkpoint("Launching Ampere kernel for adding currents", HERE); + const auto q0 = m_params.template get("scales.q0"); + const auto n0 = m_params.template get("scales.n0"); + const auto B0 = m_params.template get("scales.B0"); + const auto coeff = -dt * q0 * n0 / B0; + auto range = range_with_axis_BCs(domain); + const auto ni2 = domain.mesh.n_active(in::x2); + + if (g == gr_ampere::aux) { + // Updates D0 with J. + Kokkos::parallel_for("Ampere-1", + range, + kernel::gr::CurrentsAmpere_kernel(domain.fields.em0, + domain.fields.cur, + domain.mesh.metric, + coeff, + ni2, + domain.mesh.flds_bc())); + } else if (g == gr_ampere::main) { + // Updates D0 with J0. + Kokkos::parallel_for("Ampere-2", + range, + kernel::gr::CurrentsAmpere_kernel(domain.fields.em0, + domain.fields.cur0, + domain.mesh.metric, + coeff, + ni2, + domain.mesh.flds_bc())); + } else { + raise::Error("Wrong option for `g`", HERE); + } } From 781bd42769cc2a6379ee0e0bc74343bc1e0cbceb Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Mon, 30 Sep 2024 17:14:21 -0400 Subject: [PATCH 034/234] see below -- added call from the time step --- src/engines/grpic.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 60c00b1d9..d19b87fe4 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -377,7 +377,7 @@ namespace ntt { * * Now: em0::D at n+1/2 */ - // AmpereCurrents(gr_ampere::aux); + AmpereCurrents(dom, gr_ampere::aux); } /** @@ -410,7 +410,7 @@ namespace ntt { * * Now: em0::D at n+1 */ - // AmpereCurrents(gr_ampere::main); + AmpereCurrents(dom, gr_ampere::main); } /** * em::D <-> em0::D From 3615f79af7080536ebc173db5503d0aa211671b1 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 1 Oct 2024 12:16:48 -0400 Subject: [PATCH 035/234] engines/grpic: started ParticlePush --- src/engines/grpic.hpp | 154 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 152 insertions(+), 2 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index d19b87fe4..9926756d2 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -324,10 +324,44 @@ namespace ntt { { /** - * particle pusher goes here - * + * x_prtl, u_prtl <- em::D, em0::B + * + * Now: x_prtl at n + 1, u_prtl at n + 1/2 */ + timers.start("ParticlePusher"); + ParticlePush(dom); + timers.stop("ParticlePusher"); + + /** + * cur0::J <- current deposition + * + * Now: cur0::J at n+1/2 + */ + if (deposit_enabled) { + timers.start("CurrentDeposit"); + Kokkos::deep_copy(dom.fields.cur, ZERO); + CurrentsDeposit(dom); + timers.stop("CurrentDeposit"); + + timers.start("Communications"); + m_metadomain.SynchronizeFields(dom, Comm::J); + m_metadomain.CommunicateFields(dom, Comm::J); + timers.stop("Communications"); + + // timers.start("FieldBoundaries"); + // CurrentsBoundaryConditions(); + // timers.stop("FieldBoundaries"); + + timers.start("CurrentFiltering"); + CurrentsFilter(dom); + timers.stop("CurrentFiltering"); + } + timers.start("Communications"); + if ((sort_interval > 0) and (step % sort_interval == 0)) { + m_metadomain.CommunicateParticles(dom, &timers); + } + timers.stop("Communications"); } if (fieldsolver_enabled) { @@ -853,6 +887,122 @@ namespace ntt { domain.fields.cur0, domain.mesh.metric)); } + + void CurrentsDeposit(domain_t& domain) { + auto scatter_cur = Kokkos::Experimental::create_scatter_view( + domain.fields.cur); + for (auto& species : domain.species) { + logger::Checkpoint( + fmt::format("Launching currents deposit kernel for %d [%s] : %lu %f", + species.index(), + species.label().c_str(), + species.npart(), + (double)species.charge()), + HERE); + if (species.npart() == 0 || cmp::AlmostZero(species.charge())) { + continue; + } + Kokkos::parallel_for("CurrentsDeposit", + species.rangeActiveParticles(), + kernel::DepositCurrents_kernel( + scatter_cur, + species.i1, + species.i2, + species.i3, + species.i1_prev, + species.i2_prev, + species.i3_prev, + species.dx1, + species.dx2, + species.dx3, + species.dx1_prev, + species.dx2_prev, + species.dx3_prev, + species.ux1, + species.ux2, + species.ux3, + species.phi, + species.weight, + species.tag, + domain.mesh.metric, + (real_t)(species.charge()), + dt)); + } + Kokkos::Experimental::contribute(domain.fields.cur, scatter_cur); + } + + void CurrentsFilter(domain_t& domain) { + logger::Checkpoint("Launching currents filtering kernels", HERE); + auto range = range_with_axis_BCs(domain); + const auto nfilter = m_params.template get( + "algorithms.current_filters"); + tuple_t size; + if constexpr (M::Dim == Dim::_1D || M::Dim == Dim::_2D || M::Dim == Dim::_3D) { + size[0] = domain.mesh.n_active(in::x1); + } + if constexpr (M::Dim == Dim::_2D || M::Dim == Dim::_3D) { + size[1] = domain.mesh.n_active(in::x2); + } + if constexpr (M::Dim == Dim::_3D) { + size[2] = domain.mesh.n_active(in::x3); + } + // !TODO: this needs to be done more efficiently + for (unsigned short i = 0; i < nfilter; ++i) { + Kokkos::deep_copy(domain.fields.buff, domain.fields.cur); + Kokkos::parallel_for("CurrentsFilter", + range, + kernel::DigitalFilter_kernel( + domain.fields.cur, + domain.fields.buff, + size, + domain.mesh.flds_bc())); + m_metadomain.CommunicateFields(domain, Comm::J); + } + } + + void ParticlePush(domain_t& domain) { + for (auto& species : domain.species) { + species.set_unsorted(); + logger::Checkpoint( + fmt::format("Launching particle pusher kernel for %d [%s] : %lu", + species.index(), + species.label().c_str(), + species.npart()), + HERE); + if (species.npart() == 0) { + continue; + } + const auto q_ovr_m = species.mass() > ZERO + ? species.charge() / species.mass() + : ZERO; + // coeff = q / m (dt / 2) omegaB0 + const auto coeff = q_ovr_m * HALF * dt * + m_params.template get("scales.omegaB0"); + // clang-format off + Kokkos::parallel_for( + "ParticlePusher", + species.rangeActiveParticles(), + kernel::gr::Pusher_kernel( + domain.fields.em, + domain.fields.em0, + species.i1, species.i2, species.i3, + species.i1_prev, species.i2_prev, species.i3_prev, + species.dx1, species.dx2, species.dx3, + species.dx1_prev, species.dx2_prev, species.dx3_prev, + species.ux1, species.ux2, species.ux3, + species.phi, species.tag, + domain.mesh.metric, + coeff, dt, + domain.mesh.n_active(in::x1), + domain.mesh.n_active(in::x2), + domain.mesh.n_active(in::x3), + m_params.template get("algorithms.gr.pusher_eps"), m_params.template get("algorithms.gr.pusher_niter"), + domain.mesh.prtl_bc() + )); + // clang-format on + } + } + }; } // namespace ntt From a3eba30cbac537f83a9e1679f73076eaea2cef3c Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 1 Oct 2024 12:21:25 -0400 Subject: [PATCH 036/234] see below --- src/engines/grpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 9926756d2..b524fa19b 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -329,7 +329,7 @@ namespace ntt { * Now: x_prtl at n + 1, u_prtl at n + 1/2 */ timers.start("ParticlePusher"); - ParticlePush(dom); + // ParticlePush(dom); timers.stop("ParticlePusher"); /** From 1c6242482125f7797f3b15e14d641284ef173492 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 1 Oct 2024 12:32:47 -0400 Subject: [PATCH 037/234] engines/grpic: added timers --- src/engines/grpic.hpp | 65 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index b524fa19b..1c5d96593 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -277,6 +277,7 @@ namespace ntt { */ if (fieldsolver_enabled) { + timers.start("FieldSolver"); /** * em0::D <- (em0::D + em::D) / 2 * em0::B <- (em0::B + em::B) / 2 @@ -291,35 +292,49 @@ namespace ntt { * Now: aux::E at n-1/2 */ ComputeAuxE(dom, gr_getE::D0_B); + timers.stop("FieldSolver"); + + timers.start("FieldBoundaries"); /** * aux::E <- boundary conditions */ FieldBoundaries(dom, BC::D, gr_bc::aux); + timers.stop("FieldBoundaries"); + + timers.start("FieldSolver"); /** * em0::B <- (em0::B) <- -curl aux::E * * Now: em0::B at n */ Faraday(dom, gr_faraday::aux, ONE); - + timers.stop("FieldSolver"); /** * em0::B, em::B <- boundary conditions */ + timers.start("FieldBoundaries"); FieldBoundaries(dom, BC::B, gr_bc::main); + timers.stop("FieldBoundaries"); + timers.start("Communications"); m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); + timers.stop("Communications"); - + timers.start("FieldSolver"); /** * aux::H <- alpha * em0::B - beta x em::D * * Now: aux::H at n */ ComputeAuxH(dom, gr_getH::D_B0); + timers.stop("FieldSolver"); + + timers.start("FieldBoundaries"); /** * aux::H <- boundary conditions */ FieldBoundaries(dom, BC::B, gr_bc::aux); + timers.stop("FieldBoundaries"); } { @@ -365,23 +380,29 @@ namespace ntt { } if (fieldsolver_enabled) { + timers.start("FieldSolver"); /** * cur::J <- (cur0::J + cur::J) / 2 * * Now: cur::J at n */ TimeAverageJ(dom); - /** * aux::Е <- alpha * em::D + beta x em0::B * * Now: aux::Е at n */ ComputeAuxE(dom, gr_getE::D_B0); + timers.stop("FieldSolver"); + + timers.start("FieldBoundaries"); /** * aux::Е <- boundary conditions */ FieldBoundaries(dom, BC::D, gr_bc::aux); + timers.stop("FieldBoundaries"); + + timers.start("FieldSolver"); /** * em0::B <- (em::B) <- -curl aux::E * @@ -389,48 +410,65 @@ namespace ntt { * em::B at n-1/2 */ Faraday(dom, gr_faraday::main, ONE); - + timers.stop("FieldSolver"); + /** * em0::B, em::B <- boundary conditions */ + timers.start("Communications"); m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); + timers.stop("Communications"); + timers.start("FieldBoundaries"); FieldBoundaries(dom, BC::B, gr_bc::main); + timers.stop("FieldBoundaries"); - + timers.start("FieldSolver"); /** * em0::D <- (em0::D) <- curl aux::H * * Now: em0::D at n+1/2 */ Ampere(dom, gr_ampere::aux, ONE); - + timers.stop("FieldSolver"); if (deposit_enabled) { + timers.start("FieldSolver"); /** * em0::D <- (em0::D) <- cur::J * * Now: em0::D at n+1/2 */ AmpereCurrents(dom, gr_ampere::aux); + timers.stop("FieldSolver"); } - + /** * em0::D, em::D <- boundary conditions */ + timers.start("Communications"); m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + timers.stop("Communications"); + timers.start("FieldBoundaries"); FieldBoundaries(dom, BC::D, gr_bc::main); - + timers.stop("FieldBoundaries"); + timers.start("FieldSolver"); /** * aux::H <- alpha * em0::B - beta x em0::D * * Now: aux::H at n+1/2 */ ComputeAuxH(dom, gr_getH::D0_B0); + timers.stop("FieldSolver"); + + timers.start("FieldBoundaries"); /** * aux::H <- boundary conditions */ FieldBoundaries(dom, BC::B, gr_bc::aux); + timers.stop("FieldBoundaries"); + + timers.start("FieldSolver"); /** * em0::D <- (em::D) <- curl aux::H * @@ -438,27 +476,36 @@ namespace ntt { * em::D at n */ Ampere(dom, gr_ampere::main, ONE); + timers.stop("FieldSolver"); + if (deposit_enabled) { + timers.start("FieldSolver"); /** * em0::D <- (em0::D) <- cur0::J * * Now: em0::D at n+1 */ AmpereCurrents(dom, gr_ampere::main); + timers.stop("FieldSolver"); } + timers.start("FieldSolver"); /** * em::D <-> em0::D * em::B <-> em0::B * cur::J <-> cur0::J */ SwapFields(dom); - + timers.stop("FieldSolver"); /** * em0::D, em::D <- boundary conditions */ + timers.start("Communications"); m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + timers.stop("Communications"); + timers.start("FieldBoundaries"); FieldBoundaries(dom, BC::D, gr_bc::main); + timers.stop("FieldBoundaries"); } /** * Finally: em0::B at n-1/2 From ec795dbb6a25070ac40cd0abead8ff2d5984c775 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 1 Oct 2024 14:00:57 -0400 Subject: [PATCH 038/234] parameters: changed to requiring n=2 for GRPIC --- src/framework/parameters.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/framework/parameters.cpp b/src/framework/parameters.cpp index 20c7e83d8..e22470b1b 100644 --- a/src/framework/parameters.cpp +++ b/src/framework/parameters.cpp @@ -587,10 +587,10 @@ namespace ntt { prtl_bc_enum.push_back({ PrtlBC::PERIODIC, PrtlBC::PERIODIC }); } } else { - raise::ErrorIf(flds_bc[0].size() != 1, + raise::ErrorIf(flds_bc[0].size() != 2, "invalid `grid.boundaries.fields`", HERE); - raise::ErrorIf(prtl_bc[0].size() != 1, + raise::ErrorIf(prtl_bc[0].size() != 2, "invalid `grid.boundaries.particles`", HERE); flds_bc_enum.push_back( From bcc87d3a1378e20cedff7fe3dbce160dfa6867f1 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 1 Oct 2024 15:27:17 -0400 Subject: [PATCH 039/234] minor: fixed description for qkerr metric for h_tilde --- src/metrics/qkerr_schild.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/metrics/qkerr_schild.h b/src/metrics/qkerr_schild.h index d531b8b3b..85507c6e1 100644 --- a/src/metrics/qkerr_schild.h +++ b/src/metrics/qkerr_schild.h @@ -265,7 +265,7 @@ namespace metric { } /** - * sqrt(det(h_ij)) + * sqrt(det(h_ij)) divided by sin(theta). * @param x coordinate array in code units */ Inline auto sqrt_det_h_tilde(const coord_t& x) const -> real_t { From 4393c2101041f3d91c6da58afbab38560b6af4d1 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 1 Oct 2024 17:37:59 -0400 Subject: [PATCH 040/234] parameters: restored !=1 --- src/framework/parameters.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/framework/parameters.cpp b/src/framework/parameters.cpp index e22470b1b..20c7e83d8 100644 --- a/src/framework/parameters.cpp +++ b/src/framework/parameters.cpp @@ -587,10 +587,10 @@ namespace ntt { prtl_bc_enum.push_back({ PrtlBC::PERIODIC, PrtlBC::PERIODIC }); } } else { - raise::ErrorIf(flds_bc[0].size() != 2, + raise::ErrorIf(flds_bc[0].size() != 1, "invalid `grid.boundaries.fields`", HERE); - raise::ErrorIf(prtl_bc[0].size() != 2, + raise::ErrorIf(prtl_bc[0].size() != 1, "invalid `grid.boundaries.particles`", HERE); flds_bc_enum.push_back( From 31582dc4672a763b2ba3f7ca78b9c58692a6888a Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 2 Oct 2024 16:42:43 -0400 Subject: [PATCH 041/234] archetypes/field_setter: fixed wrong conversion of theta and call for d() [instead of b] in 3D grpic --- src/archetypes/field_setter.h | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/archetypes/field_setter.h b/src/archetypes/field_setter.h index 281c28df6..171e9f8a8 100644 --- a/src/archetypes/field_setter.h +++ b/src/archetypes/field_setter.h @@ -202,13 +202,13 @@ namespace arch { const real_t x1_0 { metric.template convert<1, Crd::Cd, Crd::Ph>(i1_) }; const real_t x1_H { metric.template convert<1, Crd::Cd, Crd::Ph>( i1_ + HALF) }; - const real_t x2_0 { metric.template convert<1, Crd::Cd, Crd::Ph>(i2_) }; - const real_t x2_H { metric.template convert<1, Crd::Cd, Crd::Ph>( + const real_t x2_0 { metric.template convert<2, Crd::Cd, Crd::Ph>(i2_) }; + const real_t x2_H { metric.template convert<2, Crd::Cd, Crd::Ph>( i2_ + HALF) }; { // bx1 - vec_t b_PU { finit.dx1({ x1_0, x2_H }), - finit.dx2({ x1_0, x2_H }), - finit.dx3({ x1_0, x2_H }) }; + vec_t b_PU { finit.bx1({ x1_0, x2_H }), + finit.bx2({ x1_0, x2_H }), + finit.bx3({ x1_0, x2_H }) }; vec_t b_U { ZERO }; metric.template transform({ i1_, i2_ + HALF }, b_PU, @@ -216,9 +216,9 @@ namespace arch { EM(i1, i2, em::bx1) = b_U[0]; } { // bx2 - vec_t b_PU { finit.dx1({ x1_H, x2_0 }), - finit.dx2({ x1_H, x2_0 }), - finit.dx3({ x1_H, x2_0 }) }; + vec_t b_PU { finit.bx1({ x1_H, x2_0 }), + finit.bx2({ x1_H, x2_0 }), + finit.bx3({ x1_H, x2_0 }) }; vec_t b_U { ZERO }; metric.template transform({ i1_ + HALF, i2_ }, b_PU, @@ -226,9 +226,9 @@ namespace arch { EM(i1, i2, em::bx2) = b_U[1]; } { // bx3 - vec_t b_PU { finit.dx1({ x1_H, x2_H }), - finit.dx2({ x1_H, x2_H }), - finit.dx3({ x1_H, x2_H }) }; + vec_t b_PU { finit.bx1({ x1_H, x2_H }), + finit.bx2({ x1_H, x2_H }), + finit.bx3({ x1_H, x2_H }) }; vec_t b_U { ZERO }; metric.template transform({ i1_ + HALF, i2_ + HALF }, b_PU, From 09014694cc7a72601a5414996c683776d57043e4 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:14:11 -0400 Subject: [PATCH 042/234] kernels/fields_bc: fixed if statement for open BC --- src/kernels/fields_bcs.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 5a50b6daf..d02bfc485 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -509,7 +509,7 @@ template Fld(i1_min - 1, i2, em::ex2) = Fld(i1_min , i2, em::ex2); Fld(i1_min , i2, em::ex3) = Fld(i1_min + 1, i2, em::ex3); Fld(i1_min - 1, i2, em::ex3) = Fld(i1_min , i2, em::ex3); - } else if (setB) { + } if (setB) { Fld(i1_min , i2, em::bx1) = Fld(i1_min + 1, i2, em::bx1); Fld(i1_min - 1, i2, em::bx1) = Fld(i1_min , i2, em::bx1); Fld(i1_min - 1, i2, em::bx2) = Fld(i1_min , i2, em::bx2); From f67d40ff8fc37de9d88a3ffedf690ad326894baa Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:46:55 -0400 Subject: [PATCH 043/234] kernels/fields_bc: added a separate kernel for aux fields --- src/kernels/fields_bcs.hpp | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index d02bfc485..441793ac0 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -523,6 +523,37 @@ template } }; +template + struct OpenBoundariesAux_kernel { + ndfield_t Fld; + const std::size_t i1_min; + const bool setE, setB; + + OpenBoundariesAux_kernel(ndfield_t Fld, std::size_t i1_min, BCTags tags) + : Fld { Fld } + , i1_min { i1_min } + , setE { tags & BC::Ex1 or tags & BC::Ex2 or tags & BC::Ex3 } + , setB { tags & BC::Bx1 or tags & BC::Bx2 or tags & BC::Bx3 } {} + + Inline void operator()(index_t i2) const { + if constexpr (M::Dim == Dim::_2D) { + if (setE) { + Fld(i1_min - 1, i2, em::ex1) = Fld(i1_min , i2, em::ex1); + Fld(i1_min - 1, i2, em::ex2) = Fld(i1_min , i2, em::ex2); + Fld(i1_min - 1, i2, em::ex3) = Fld(i1_min , i2, em::ex3); + } if (setB) { + Fld(i1_min - 1, i2, em::bx1) = Fld(i1_min , i2, em::bx1); + Fld(i1_min - 1, i2, em::bx2) = Fld(i1_min , i2, em::bx2); + Fld(i1_min - 1, i2, em::bx3) = Fld(i1_min , i2, em::bx3); + } + } else { + raise::KernelError( + HERE, + "AbsorbFields_kernel: 2D implementation called for D != 2"); + } + } + }; + } // namespace kernel #endif // KERNELS_FIELDS_BCS_HPP From fd4c61fe4e7aa91db0716848b1a20f80a2ddab8c Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:47:57 -0400 Subject: [PATCH 044/234] see below --- src/engines/grpic.hpp | 30 ++++++------------------------ 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 1c5d96593..9c71fe475 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -614,27 +614,6 @@ namespace ntt { xg_edge, ds, tags)); - } else if (dim == in::x2) { - if constexpr (M::Dim == Dim::_2D) { - Kokkos::parallel_for( - "AbsorbFields", - CreateRangePolicy(range_min, range_max), - kernel::AbsorbBoundaries_kernel(domain.fields.em, - domain.mesh.metric, - xg_edge, - ds, - tags)); - Kokkos::parallel_for( - "AbsorbFields", - CreateRangePolicy(range_min, range_max), - kernel::AbsorbBoundaries_kernel(domain.fields.em0, - domain.mesh.metric, - xg_edge, - ds, - tags)); - } else { - raise::Error("Invalid dimension", HERE); - } } else { raise::Error("Invalid dimension", HERE); } @@ -654,10 +633,13 @@ namespace ntt { "Invalid axis direction, should be x2", HERE); const auto i1_min = domain.mesh.i_min(in::x1); + auto range = CreateRangePolicy( + {domain.mesh.i_min(in::x2)}, + {domain.mesh.i_max(in::x2) + 1}); if (g == gr_bc::main) { Kokkos::parallel_for( "OpenBCFields", - domain.mesh.n_all(in::x2), + range, kernel::OpenBoundaries_kernel(domain.fields.em, i1_min, tags)); Kokkos::parallel_for( "OpenBCFields", @@ -666,8 +648,8 @@ namespace ntt { } else if (g == gr_bc::aux) { Kokkos::parallel_for( "OpenBCFields", - domain.mesh.n_all(in::x2), - kernel::OpenBoundaries_kernel(domain.fields.aux, i1_min, tags)); + range, + kernel::OpenBoundariesAux_kernel(domain.fields.aux, i1_min, tags)); } } From bbb272977a0520bb92c679ff791ee89ca3457698 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:51:13 -0400 Subject: [PATCH 045/234] engines/grpic: also changed range for axis BC in this commit and commit below --- src/engines/grpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 9c71fe475..5df9a314a 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -643,7 +643,7 @@ namespace ntt { kernel::OpenBoundaries_kernel(domain.fields.em, i1_min, tags)); Kokkos::parallel_for( "OpenBCFields", - domain.mesh.n_all(in::x2), + range, kernel::OpenBoundaries_kernel(domain.fields.em0, i1_min, tags)); } else if (g == gr_bc::aux) { Kokkos::parallel_for( From 8ef8b7f4c40c401b41a2c61311f403666f4b5ba4 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 2 Oct 2024 18:04:58 -0400 Subject: [PATCH 046/234] engines/grpic: changed range for axis BC --- src/engines/grpic.hpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 5df9a314a..74d06a3f9 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -667,23 +667,26 @@ namespace ntt { HERE); const auto i2_min = domain.mesh.i_min(in::x2); const auto i2_max = domain.mesh.i_max(in::x2); + auto range = CreateRangePolicy( + {domain.mesh.i_min(in::x1) - 1}, + {domain.mesh.i_max(in::x1)}); if (direction.get_sign() < 0) { Kokkos::parallel_for( "AxisBCFields", - domain.mesh.n_all(in::x1), + range, kernel::AxisBoundaries_kernel(domain.fields.em, i2_min, tags)); Kokkos::parallel_for( "AxisBCFields", - domain.mesh.n_all(in::x1), + range, kernel::AxisBoundaries_kernel(domain.fields.em0, i2_min, tags)); } else { Kokkos::parallel_for( "AxisBCFields", - domain.mesh.n_all(in::x1), + range, kernel::AxisBoundaries_kernel(domain.fields.em, i2_max, tags)); Kokkos::parallel_for( "AxisBCFields", - domain.mesh.n_all(in::x1), + range, kernel::AxisBoundaries_kernel(domain.fields.em0, i2_max, tags)); } } From 67ad01a88664ef0740ffd4d139a07af5eb889369 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Thu, 3 Oct 2024 18:30:39 -0400 Subject: [PATCH 047/234] updated pgen for wald --- setups/grpic/wald/pgen.hpp | 47 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/setups/grpic/wald/pgen.hpp b/setups/grpic/wald/pgen.hpp index 3374a12d7..efb9c78e7 100644 --- a/setups/grpic/wald/pgen.hpp +++ b/setups/grpic/wald/pgen.hpp @@ -4,13 +4,51 @@ #include "enums.h" #include "global.h" +#include "arch/kokkos_aliases.h" #include "arch/traits.h" #include "archetypes/problem_generator.h" +#include "framework/domain/metadomain.h" namespace user { using namespace ntt; + template + struct InitFields { + InitFields() {} + + Inline auto VerticalPotential(const coord_t& x_Ph) const -> real_t { + return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); + } + + Inline auto bx1(const coord_t& x_Ph) const -> real_t { + return x_Ph[0] * math::cos(x_Ph[1]); + } + + Inline auto bx2(const coord_t& x_Ph) const -> real_t { + return -x_Ph[0] * math::sin(x_Ph[1]); + } + + Inline auto bx3(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx1(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx2(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx3(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + private: + // const real_t Bsurf, Rstar; + }; + template struct PGen : public arch::ProblemGenerator { // compatibility traits for the problem generator @@ -24,13 +62,18 @@ namespace user { using arch::ProblemGenerator::D; using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; + + InitFields init_flds; - inline PGen(SimulationParams& p, const Metadomain&) : - arch::ProblemGenerator(p) {} + inline PGen(SimulationParams& p, const Metadomain& m) : + arch::ProblemGenerator(p) + // , init_flds { } + {} inline PGen() {} }; + } // namespace user #endif From 731559e2a4539ebe1f254ac3beae45ca5e81b142 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Thu, 3 Oct 2024 19:12:38 -0400 Subject: [PATCH 048/234] engines/grpic: adjusted ranges for all kernels --- src/engines/grpic.hpp | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 74d06a3f9..40e743766 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -674,20 +674,20 @@ namespace ntt { Kokkos::parallel_for( "AxisBCFields", range, - kernel::AxisBoundaries_kernel(domain.fields.em, i2_min, tags)); + kernel::AxisBoundariesGR_kernel(domain.fields.em, i2_min, tags)); Kokkos::parallel_for( "AxisBCFields", range, - kernel::AxisBoundaries_kernel(domain.fields.em0, i2_min, tags)); + kernel::AxisBoundariesGR_kernel(domain.fields.em0, i2_min, tags)); } else { Kokkos::parallel_for( "AxisBCFields", range, - kernel::AxisBoundaries_kernel(domain.fields.em, i2_max, tags)); + kernel::AxisBoundariesGR_kernel(domain.fields.em, i2_max, tags)); Kokkos::parallel_for( "AxisBCFields", range, - kernel::AxisBoundaries_kernel(domain.fields.em0, i2_max, tags)); + kernel::AxisBoundariesGR_kernel(domain.fields.em0, i2_max, tags)); } } @@ -825,8 +825,8 @@ namespace ntt { "algorithms.timestep.correction") * dt; auto range = CreateRangePolicy( - { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2)}, - { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1}); + { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) + 1}, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2)}); const auto ni2 = domain.mesh.n_active(in::x2); if (g == gr_ampere::aux) { @@ -873,7 +873,10 @@ namespace ntt { const auto n0 = m_params.template get("scales.n0"); const auto B0 = m_params.template get("scales.B0"); const auto coeff = -dt * q0 * n0 / B0; - auto range = range_with_axis_BCs(domain); + // auto range = range_with_axis_BCs(domain); + auto range = CreateRangePolicy( + { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) + 1}, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2)}); const auto ni2 = domain.mesh.n_active(in::x2); if (g == gr_ampere::aux) { @@ -903,18 +906,16 @@ namespace ntt { } void TimeAverageDB(domain_t& domain) { - auto range = range_with_axis_BCs(domain); Kokkos::parallel_for("TimeAverageDB", - range, + domain.mesh.rangeActiveCells(), kernel::gr::TimeAverageDB_kernel(domain.fields.em, domain.fields.em0, domain.mesh.metric)); } void TimeAverageJ(domain_t& domain) { - auto range = range_with_axis_BCs(domain); Kokkos::parallel_for("TimeAverageJ", - range, + domain.mesh.rangeActiveCells(), kernel::gr::TimeAverageJ_kernel(domain.fields.cur, domain.fields.cur0, domain.mesh.metric)); From 4ea3c9c0e442a032709c0b9b0216147d9f8e09f6 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Thu, 3 Oct 2024 19:13:28 -0400 Subject: [PATCH 049/234] minor changes to kernels/fields_bc gr-specific --- src/kernels/fields_bcs.hpp | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 441793ac0..fd6ba542f 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -216,6 +216,29 @@ namespace kernel { } }; + template + struct AxisBoundariesGR_kernel { + ndfield_t Fld; + const std::size_t i_edge; + const bool setE, setB; + + AxisBoundariesGR_kernel(ndfield_t Fld, std::size_t i_edge, BCTags tags) + : Fld { Fld } + , i_edge { i_edge } + , setE { tags & BC::Ex1 or tags & BC::Ex2 or tags & BC::Ex3 } + , setB { tags & BC::Bx1 or tags & BC::Bx2 or tags & BC::Bx3 } {} + + Inline void operator()(index_t i1) const { + if constexpr (D == Dim::_2D) { + if (setB) { + Fld(i1, i_edge, em::bx2) = ZERO; + } + } else { + raise::KernelError(HERE, "AxisBoundaries_kernel: D != 2"); + } + } + }; + template struct AtmosphereBoundaries_kernel { static constexpr Dimension D = M::Dim; From 9b1f59f706dfaed2ba539a817b56197eedd5d3e3 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 4 Oct 2024 14:50:26 -0400 Subject: [PATCH 050/234] wald: vertical potential setup --- setups/grpic/wald/pgen.hpp | 44 +++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/setups/grpic/wald/pgen.hpp b/setups/grpic/wald/pgen.hpp index efb9c78e7..a5f8f503e 100644 --- a/setups/grpic/wald/pgen.hpp +++ b/setups/grpic/wald/pgen.hpp @@ -13,20 +13,40 @@ namespace user { using namespace ntt; - template + template struct InitFields { - InitFields() {} + InitFields(M metric_) : metric { metric_ } {} - Inline auto VerticalPotential(const coord_t& x_Ph) const -> real_t { + Inline auto VerticalPotential(const coord_t& x_Cd) const -> real_t { + coord_t x_Ph { ZERO }; + metric.template convert(x_Cd, x_Ph); return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); } Inline auto bx1(const coord_t& x_Ph) const -> real_t { - return x_Ph[0] * math::cos(x_Ph[1]); + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0]; + x0m[1] = xi[1] - HALF; + x0p[0] = xi[0]; + x0p[1] = xi[1] + HALF; + + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + return (VerticalPotential(x0p) - VerticalPotential(x0m)) * inv_sqrt_detH_ijP; } Inline auto bx2(const coord_t& x_Ph) const -> real_t { - return -x_Ph[0] * math::sin(x_Ph[1]); + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0] + HALF - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF + HALF; + x0p[1] = xi[1]; + + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + return -(VerticalPotential(x0p) - VerticalPotential(x0m)) * inv_sqrt_detH_ijP; } Inline auto bx3(const coord_t& x_Ph) const -> real_t { @@ -46,7 +66,7 @@ namespace user { } private: - // const real_t Bsurf, Rstar; + const M metric; }; template @@ -62,18 +82,16 @@ namespace user { using arch::ProblemGenerator::D; using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; - - InitFields init_flds; - inline PGen(SimulationParams& p, const Metadomain& m) : - arch::ProblemGenerator(p) - // , init_flds { } - {} + InitFields init_flds; + + inline PGen(SimulationParams& p, const Metadomain& m) + : arch::ProblemGenerator(p) + , init_flds { m.mesh().metric } {} inline PGen() {} }; - } // namespace user #endif From fca070fe6cd7f21c981cf5fbf745eef4c2e7f37a Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 4 Oct 2024 18:25:02 -0400 Subject: [PATCH 051/234] kernel/ampere_gr: bug for theta=0 --- src/kernels/ampere_gr.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/kernels/ampere_gr.hpp b/src/kernels/ampere_gr.hpp index 045eb1729..46b950796 100644 --- a/src/kernels/ampere_gr.hpp +++ b/src/kernels/ampere_gr.hpp @@ -74,6 +74,8 @@ namespace kernel::gr { if ((i2 == i2min) && is_axis_i2min) { // theta = 0 const real_t inv_polar_area_pH { ONE / metric.polar_area(i1_ + HALF) }; + const real_t inv_sqrt_detH_0pH { ONE / + metric.sqrt_det_h({ i1_, HALF }) }; Dout(i1, i2, em::dx1) = Din(i1, i2, em::dx1) + inv_polar_area_pH * coeff * H(i1, i2, em::hx3); Dout(i1, i2, em::dx2) = Din(i1, i2, em::dx2) + From 2a2a12f755ef333c66cd77fd4093b0f766bc3d14 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 4 Oct 2024 18:47:18 -0400 Subject: [PATCH 052/234] kernels/faraday_gr: bug at i2min --- src/kernels/faraday_gr.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/kernels/faraday_gr.hpp b/src/kernels/faraday_gr.hpp index 19eede5f2..5f6e5590b 100644 --- a/src/kernels/faraday_gr.hpp +++ b/src/kernels/faraday_gr.hpp @@ -73,7 +73,9 @@ namespace kernel::gr { Bout(i1, i2, em::bx1) = Bin(i1, i2, em::bx1) + coeff * inv_sqrt_detH_0pH * (E(i1, i2, em::ex3) - E(i1, i2 + 1, em::ex3)); - if ((i2 != i2min) || !is_axis_i2min) { + if ((i2 == i2min) && is_axis_i2min) { + Bout(i1, i2, em::bx2) = ZERO; + } else { const real_t inv_sqrt_detH_pH0 { ONE / metric.sqrt_det_h( { i1_ + HALF, i2_ }) }; Bout(i1, i2, em::bx2) = Bin(i1, i2, em::bx2) + From 2acf70bd850f1f59b858a4d05cc5b5230919f06e Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 4 Oct 2024 19:03:04 -0400 Subject: [PATCH 053/234] kernels/aux: bugs in time averaging --- src/kernels/aux_fields_gr.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/kernels/aux_fields_gr.hpp b/src/kernels/aux_fields_gr.hpp index b56a819f9..c739cffbd 100644 --- a/src/kernels/aux_fields_gr.hpp +++ b/src/kernels/aux_fields_gr.hpp @@ -257,9 +257,9 @@ namespace kernel::gr { BDf0(i1, i2, em::bx1) = HALF * (BDf0(i1, i2, em::bx1) + BDf(i1, i2, em::bx1)); BDf0(i1, i2, em::bx2) = HALF * (BDf0(i1, i2, em::bx2) + BDf(i1, i2, em::bx2)); BDf0(i1, i2, em::bx3) = HALF * (BDf0(i1, i2, em::bx3) + BDf(i1, i2, em::bx3)); - BDf0(i1, i2, em::ex1) = HALF * (BDf0(i1, i2, em::bx1) + BDf(i1, i2, em::ex1)); - BDf0(i1, i2, em::ex2) = HALF * (BDf0(i1, i2, em::bx2) + BDf(i1, i2, em::ex2)); - BDf0(i1, i2, em::ex3) = HALF * (BDf0(i1, i2, em::bx3) + BDf(i1, i2, em::ex3)); + BDf0(i1, i2, em::ex1) = HALF * (BDf0(i1, i2, em::ex1) + BDf(i1, i2, em::ex1)); + BDf0(i1, i2, em::ex2) = HALF * (BDf0(i1, i2, em::ex2) + BDf(i1, i2, em::ex2)); + BDf0(i1, i2, em::ex3) = HALF * (BDf0(i1, i2, em::ex3) + BDf(i1, i2, em::ex3)); } else { raise::KernelError( HERE, @@ -291,9 +291,9 @@ namespace kernel::gr { Inline void operator()(index_t i1, index_t i2) const { if constexpr (D == Dim::_2D) { - Jf0(i1, i2, cur::jx1) = HALF * (Jf0(i1, i2, cur::jx1) + Jf(i1, i2, cur::jx1)); - Jf0(i1, i2, cur::jx2) = HALF * (Jf0(i1, i2, cur::jx2) + Jf(i1, i2, cur::jx2)); - Jf0(i1, i2, cur::jx3) = HALF * (Jf0(i1, i2, cur::jx3) + Jf(i1, i2, cur::jx3)); + Jf(i1, i2, cur::jx1) = HALF * (Jf0(i1, i2, cur::jx1) + Jf(i1, i2, cur::jx1)); + Jf(i1, i2, cur::jx2) = HALF * (Jf0(i1, i2, cur::jx2) + Jf(i1, i2, cur::jx2)); + Jf(i1, i2, cur::jx3) = HALF * (Jf0(i1, i2, cur::jx3) + Jf(i1, i2, cur::jx3)); } else { raise::KernelError( HERE, From 1656da322b73545c5433576671112b5a2ce02257 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 4 Oct 2024 20:13:52 -0400 Subject: [PATCH 054/234] metric: kerr-schild a=0 minor fixes --- src/metrics/kerr_schild_0.h | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/metrics/kerr_schild_0.h b/src/metrics/kerr_schild_0.h index 70689f4f0..31080ed4c 100644 --- a/src/metrics/kerr_schild_0.h +++ b/src/metrics/kerr_schild_0.h @@ -36,6 +36,7 @@ namespace metric { private: const real_t dr, dtheta, dphi; const real_t dr_inv, dtheta_inv, dphi_inv; + const real_t a, rg_, rh_; public: static constexpr const char* Label { "kerr_schild_0" }; @@ -57,6 +58,9 @@ namespace metric { boundaries_t ext, const std::map& = {}) : MetricBase { res, ext } + , a { ZERO } + , rg_ { ONE } + , rh_ { TWO } , dr { (x1_max - x1_min) / nx1 } , dtheta { (x2_max - x2_min) / nx2 } , dphi { (x3_max - x3_min) / nx3 } @@ -70,17 +74,17 @@ namespace metric { [[nodiscard]] Inline auto spin() const -> const real_t& { - return ZERO; + return a; } [[nodiscard]] Inline auto rhorizon() const -> const real_t& { - return ZERO; + return rh_; } [[nodiscard]] Inline auto rg() const -> const real_t& { - return ZERO; + return rg_; } /** From c51f80c92edf5d7a031e6d5846772d8d97077825 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 18 Oct 2024 17:15:06 -0400 Subject: [PATCH 055/234] minor kernels/aux_gr --- src/kernels/aux_fields_gr.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kernels/aux_fields_gr.hpp b/src/kernels/aux_fields_gr.hpp index c739cffbd..0983b6b7e 100644 --- a/src/kernels/aux_fields_gr.hpp +++ b/src/kernels/aux_fields_gr.hpp @@ -277,8 +277,8 @@ namespace kernel::gr { static_assert(M::is_metric, "M must be a metric class"); static constexpr auto D = M::Dim; - const ndfield_t Jf; - ndfield_t Jf0; + ndfield_t Jf; + const ndfield_t Jf0; const M metric; public: From be2ec2bd64e5e3dae61635bca8369104018cf060 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 18 Oct 2024 17:16:06 -0400 Subject: [PATCH 056/234] monor style --- src/kernels/fields_bcs.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index fd6ba542f..c980d1e77 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -561,13 +561,13 @@ template Inline void operator()(index_t i2) const { if constexpr (M::Dim == Dim::_2D) { if (setE) { - Fld(i1_min - 1, i2, em::ex1) = Fld(i1_min , i2, em::ex1); - Fld(i1_min - 1, i2, em::ex2) = Fld(i1_min , i2, em::ex2); - Fld(i1_min - 1, i2, em::ex3) = Fld(i1_min , i2, em::ex3); + Fld(i1_min - 1, i2, em::ex1) = Fld(i1_min, i2, em::ex1); + Fld(i1_min - 1, i2, em::ex2) = Fld(i1_min, i2, em::ex2); + Fld(i1_min - 1, i2, em::ex3) = Fld(i1_min, i2, em::ex3); } if (setB) { - Fld(i1_min - 1, i2, em::bx1) = Fld(i1_min , i2, em::bx1); - Fld(i1_min - 1, i2, em::bx2) = Fld(i1_min , i2, em::bx2); - Fld(i1_min - 1, i2, em::bx3) = Fld(i1_min , i2, em::bx3); + Fld(i1_min - 1, i2, em::bx1) = Fld(i1_min, i2, em::bx1); + Fld(i1_min - 1, i2, em::bx2) = Fld(i1_min, i2, em::bx2); + Fld(i1_min - 1, i2, em::bx3) = Fld(i1_min, i2, em::bx3); } } else { raise::KernelError( From 50dfdb35a913eff798ce1b9c26ef05813fb1f41a Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 18 Oct 2024 17:17:40 -0400 Subject: [PATCH 057/234] engines/grpic: parameters for field boundaries --- src/engines/grpic.hpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 40e743766..0c21c0728 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -123,7 +123,7 @@ namespace ntt { * em0::D, em::D, em0::B, em::B <- boundary conditions */ m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0 | Comm::D | Comm::D0); - FieldBoundaries(dom, BC::B | BC::D, gr_bc::main); + FieldBoundaries(dom, BC::B | BC::E, gr_bc::main); /** * em0::B <- em::B @@ -145,7 +145,7 @@ namespace ntt { /** * aux::E, aux::H <- boundary conditions */ - FieldBoundaries(dom, BC::B | BC::D, gr_bc::aux); + FieldBoundaries(dom, BC::B | BC::E, gr_bc::aux); /** * em0::B <- (em0::B) <- -curl aux::E @@ -171,7 +171,7 @@ namespace ntt { * em0::D, em::D <- boundary conditions */ m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); - FieldBoundaries(dom, BC::D, gr_bc::main); + FieldBoundaries(dom, BC::E, gr_bc::main); /** * aux::E <- alpha * em::D + beta x em0::B @@ -185,7 +185,7 @@ namespace ntt { /** * aux::E, aux::H <- boundary conditions */ - FieldBoundaries(dom, BC::B | BC::D, gr_bc::aux); + FieldBoundaries(dom, BC::B | BC::E, gr_bc::aux); // !ADD: GR -- particles? @@ -211,7 +211,7 @@ namespace ntt { * em0::D, em::D <- boundary conditions */ m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); - FieldBoundaries(dom, BC::D, gr_bc::main); + FieldBoundaries(dom, BC::E, gr_bc::main); /** * aux::H <- alpha * em0::B - beta x em0::D @@ -235,7 +235,7 @@ namespace ntt { * em0::D, em::D <- boundary conditions */ m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); - FieldBoundaries(dom, BC::D, gr_bc::main); + FieldBoundaries(dom, BC::E, gr_bc::main); /** * em::D <-> em0::D @@ -298,7 +298,7 @@ namespace ntt { /** * aux::E <- boundary conditions */ - FieldBoundaries(dom, BC::D, gr_bc::aux); + FieldBoundaries(dom, BC::E, gr_bc::aux); timers.stop("FieldBoundaries"); timers.start("FieldSolver"); @@ -399,7 +399,7 @@ namespace ntt { /** * aux::Е <- boundary conditions */ - FieldBoundaries(dom, BC::D, gr_bc::aux); + FieldBoundaries(dom, BC::E, gr_bc::aux); timers.stop("FieldBoundaries"); timers.start("FieldSolver"); @@ -449,7 +449,7 @@ namespace ntt { m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); timers.stop("Communications"); timers.start("FieldBoundaries"); - FieldBoundaries(dom, BC::D, gr_bc::main); + FieldBoundaries(dom, BC::E, gr_bc::main); timers.stop("FieldBoundaries"); timers.start("FieldSolver"); @@ -504,7 +504,7 @@ namespace ntt { m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); timers.stop("Communications"); timers.start("FieldBoundaries"); - FieldBoundaries(dom, BC::D, gr_bc::main); + FieldBoundaries(dom, BC::E, gr_bc::main); timers.stop("FieldBoundaries"); } /** @@ -833,7 +833,7 @@ namespace ntt { // First push, updates D0 with J. Kokkos::parallel_for("Ampere-1", range, - kernel::gr::Ampere_kernel(domain.fields.em0, + kernel::gr::Ampere_kernel(domain.fields.em0, // Din, Dout, aux domain.fields.em0, domain.fields.aux, domain.mesh.metric, From 418d035ae83cdea732a4e8206c0fe510bb2a96db Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Mon, 21 Oct 2024 19:05:10 -0400 Subject: [PATCH 058/234] minor notes --- src/engines/grpic.hpp | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 0c21c0728..b2ca0671e 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -730,9 +730,9 @@ namespace ntt { if (g == gr_getE::D0_B) { Kokkos::parallel_for("ComputeAuxE", range, - kernel::gr::ComputeAuxE_kernel(domain.fields.em0, - domain.fields.em, - domain.fields.aux, + kernel::gr::ComputeAuxE_kernel(domain.fields.em0, // D + domain.fields.em, // B + domain.fields.aux, // E domain.mesh.metric)); } else if (g == gr_getE::D_B0) { Kokkos::parallel_for("ComputeAuxE", @@ -751,9 +751,9 @@ namespace ntt { if (g == gr_getH::D_B0) { Kokkos::parallel_for("ComputeAuxH", range, - kernel::gr::ComputeAuxH_kernel(domain.fields.em, - domain.fields.em0, - domain.fields.aux, + kernel::gr::ComputeAuxH_kernel(domain.fields.em, // D + domain.fields.em0, // B + domain.fields.aux, // H domain.mesh.metric)); } else if (g == gr_getH::D0_B0) { Kokkos::parallel_for("ComputeAuxH", @@ -794,9 +794,9 @@ namespace ntt { Kokkos::parallel_for( "Faraday", domain.mesh.rangeActiveCells(), - kernel::gr::Faraday_kernel(domain.fields.em0, - domain.fields.em0, - domain.fields.aux, + kernel::gr::Faraday_kernel(domain.fields.em0, // Bin + domain.fields.em0, // Bout + domain.fields.aux, // E domain.mesh.metric, dT, domain.mesh.n_active(in::x2), @@ -825,16 +825,16 @@ namespace ntt { "algorithms.timestep.correction") * dt; auto range = CreateRangePolicy( - { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) + 1}, - { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2)}); + { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2)}, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1}); const auto ni2 = domain.mesh.n_active(in::x2); if (g == gr_ampere::aux) { // First push, updates D0 with J. Kokkos::parallel_for("Ampere-1", range, - kernel::gr::Ampere_kernel(domain.fields.em0, // Din, Dout, aux - domain.fields.em0, + kernel::gr::Ampere_kernel(domain.fields.em0, // Din + domain.fields.em0, // Dout domain.fields.aux, domain.mesh.metric, dT, From 117426ac52f7a7f55faa20ec05b2c5995192b58e Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 23 Oct 2024 17:27:09 -0400 Subject: [PATCH 059/234] implemented absorbing boundaries for gr --- src/engines/grpic.hpp | 25 ++++------ src/kernels/fields_bcs.hpp | 94 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 16 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index b2ca0671e..3f90220ec 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -562,25 +562,16 @@ namespace ntt { const auto dim = direction.get_dim(); real_t xg_min, xg_max, xg_edge; auto sign = direction.get_sign(); - if (sign > 0) { // + direction - xg_max = m_metadomain.mesh().extent(dim).second; - xg_min = xg_max - ds; - xg_edge = xg_max; - } else { // - direction - xg_min = m_metadomain.mesh().extent(dim).first; - xg_max = xg_min + ds; - xg_edge = xg_min; - } + xg_max = m_metadomain.mesh().extent(dim).second; + xg_min = xg_max - ds; + xg_edge = xg_max; + boundaries_t box; boundaries_t incl_ghosts; for (unsigned short d { 0 }; d < M::Dim; ++d) { if (d == static_cast(dim)) { box.push_back({ xg_min, xg_max }); - if (sign > 0) { - incl_ghosts.push_back({ false, true }); - } else { - incl_ghosts.push_back({ true, false }); - } + incl_ghosts.push_back({ false, true }); } else { box.push_back(Range::All); incl_ghosts.push_back({ true, true }); @@ -601,7 +592,8 @@ namespace ntt { Kokkos::parallel_for( "AbsorbFields", CreateRangePolicy(range_min, range_max), - kernel::AbsorbBoundaries_kernel(domain.fields.em, + kernel::AbsorbBoundariesGR_kernel(domain.fields.em, + m_pgen.init_flds, domain.mesh.metric, xg_edge, ds, @@ -609,7 +601,8 @@ namespace ntt { Kokkos::parallel_for( "AbsorbFields", CreateRangePolicy(range_min, range_max), - kernel::AbsorbBoundaries_kernel(domain.fields.em0, + kernel::AbsorbBoundariesGR_kernel(domain.fields.em0, + m_pgen.init_flds, domain.mesh.metric, xg_edge, ds, diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index c980d1e77..e69fbcc13 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -577,6 +577,100 @@ template } }; + template + struct AbsorbBoundariesGR_kernel { + static_assert(M::is_metric, "M must be a metric class"); + static_assert(i <= static_cast(M::Dim), + "Invalid component index"); + + ndfield_t Fld; + const I finit; + const M metric; + const real_t xg_edge; + const real_t dx_abs; + const BCTags tags; + + AbsorbBoundariesGR_kernel(ndfield_t Fld, + const I& finit, + const M& metric, + real_t xg_edge, + real_t dx_abs, + BCTags tags) + : Fld { Fld } + , finit { finit } + , metric { metric } + , xg_edge { xg_edge } + , dx_abs { dx_abs } + , tags { tags } {} + + Inline void operator()(index_t i1, index_t i2) const { + if constexpr (M::Dim == Dim::_2D) { + const auto i1_ = COORD(i1); + const auto i2_ = COORD(i2); + for (const auto comp : + { em::ex1, em::ex2, em::ex3, em::bx1, em::bx2, em::bx3 }) { + if ((comp == em::ex1) and not(tags & BC::Ex1)) { + continue; + } else if ((comp == em::ex2) and not(tags & BC::Ex2)) { + continue; + } else if ((comp == em::ex3) and not(tags & BC::Ex3)) { + continue; + } else if ((comp == em::bx1) and not(tags & BC::Bx1)) { + continue; + } else if ((comp == em::bx2) and not(tags & BC::Bx2)) { + continue; + } else if ((comp == em::bx3) and not(tags & BC::Bx3)) { + continue; + } + coord_t x_Cd { ZERO }; + if (comp == em::ex1 or comp == em::bx2 or comp == em::bx3) { + x_Cd[0] = i1_ + HALF; + x_Cd[1] = i2_; + } else if (comp == em::ex2 or comp == em::ex3 or comp == em::bx1) { + x_Cd[0] = i1_; + x_Cd[1] = i2_; + } + + const auto dx = math::abs( + metric.template convert(x_Cd[i - 1]) - xg_edge); + Fld(i1, i2, comp) *= math::tanh(dx / (INV_4 * dx_abs)); + + if (comp == em::bx1) { + const real_t x1_0 { metric.template convert<1, Crd::Cd, Crd::Ph>(i1_) }; + const real_t x2_H { metric.template convert<2, Crd::Cd, Crd::Ph>( + i2_ + HALF) }; + + vec_t b_PU { finit.bx1({ x1_0, x2_H }), + finit.bx2({ x1_0, x2_H }), + finit.bx3({ x1_0, x2_H }) }; + vec_t b_U { ZERO }; + metric.template transform({ i1_, i2_ + HALF }, + b_PU, + b_U); + Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * b_U[0]; + } else if (comp == em::bx2) { + const real_t x1_H { metric.template convert<1, Crd::Cd, Crd::Ph>( + i1_ + HALF) }; + const real_t x2_0 { metric.template convert<2, Crd::Cd, Crd::Ph>(i2_) }; + + vec_t b_PU { finit.bx1({ x1_H, x2_0 }), + finit.bx2({ x1_H, x2_0 }), + finit.bx3({ x1_H, x2_0 }) }; + vec_t b_U { ZERO }; + metric.template transform({ i1_ + HALF, i2_ }, + b_PU, + b_U); + Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * b_U[1]; + } + } + } else { + raise::KernelError( + HERE, + "AbsorbFields_kernel: 2D implementation called for D != 2"); + } + } + }; + } // namespace kernel #endif // KERNELS_FIELDS_BCS_HPP From 4abd1071ae7d33b3021860d1b050928bf8bda1c6 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Thu, 31 Oct 2024 15:49:40 -0400 Subject: [PATCH 060/234] working EM solver fully implemented --- setups/grpic/wald/pgen.hpp | 41 +++++++++++++++++++-- setups/grpic/wald/wald.toml | 72 +++++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 3 deletions(-) create mode 100644 setups/grpic/wald/wald.toml diff --git a/setups/grpic/wald/pgen.hpp b/setups/grpic/wald/pgen.hpp index a5f8f503e..51636314b 100644 --- a/setups/grpic/wald/pgen.hpp +++ b/setups/grpic/wald/pgen.hpp @@ -17,12 +17,30 @@ namespace user { struct InitFields { InitFields(M metric_) : metric { metric_ } {} - Inline auto VerticalPotential(const coord_t& x_Cd) const -> real_t { + Inline auto A_3(const coord_t& x_Cd) const -> real_t { + // return HALF * (metric.template h_<3, 3>(x_Cd) ); + // + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + // ); + coord_t x_Ph { ZERO }; metric.template convert(x_Cd, x_Ph); return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); } + Inline auto A_1(const coord_t& x_Cd) const -> real_t { + return HALF * (metric.template h_<1, 3>(x_Cd) + + TWO * metric.spin() * metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) + ); + } + + Inline auto A_0(const coord_t& x_Cd) const -> real_t { + real_t g_00 { -metric.alpha(x_Cd) * metric.alpha(x_Cd) + + metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) * metric.beta1(x_Cd) + }; + return HALF * (metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + + TWO * metric.spin() * g_00); + } + Inline auto bx1(const coord_t& x_Ph) const -> real_t { coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); @@ -33,7 +51,11 @@ namespace user { x0p[1] = xi[1] + HALF; real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; - return (VerticalPotential(x0p) - VerticalPotential(x0m)) * inv_sqrt_detH_ijP; + + if (cmp::AlmostZero(x_Ph[1])) + return ONE; + else + return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; } Inline auto bx2(const coord_t& x_Ph) const -> real_t { @@ -46,10 +68,23 @@ namespace user { x0p[1] = xi[1]; real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; - return -(VerticalPotential(x0p) - VerticalPotential(x0m)) * inv_sqrt_detH_ijP; + if (cmp::AlmostZero(x_Ph[1])) + return ZERO; + else + return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; } Inline auto bx3(const coord_t& x_Ph) const -> real_t { + // coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + // metric.template convert(x_Ph, xi); + + // x0m[0] = xi[0] + HALF - HALF; + // x0m[1] = xi[1]; + // x0p[0] = xi[0] + HALF + HALF; + // x0p[1] = xi[1]; + + // real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + // return -(A_1(x0p) - A_1(x0m)) * inv_sqrt_detH_ijP; return ZERO; } diff --git a/setups/grpic/wald/wald.toml b/setups/grpic/wald/wald.toml new file mode 100644 index 000000000..36c746ae5 --- /dev/null +++ b/setups/grpic/wald/wald.toml @@ -0,0 +1,72 @@ +[simulation] + name = "wald" + engine = "grpic" + runtime = 500.0 + +[grid] + resolution = [128,128] + extent = [[1.2, 8.0]] + + [grid.metric] + metric = "qkerr_schild" + qsph_r0 = 0.0 + qsph_h = 0.0 + ks_a = 0.95 + + [grid.boundaries] + fields = [["ABSORB"]] + particles = [["ABSORB"]] + + [grid.boundaries.absorb] + ds = 0.5 + +[scales] + larmor0 = 1.0 + skindepth0 = 1.0 + +[algorithms] + current_filters = 4 + + [algorithms.timestep] + CFL = 0.5 + + [algorithms.toggles] + deposit = false + fieldsolver = true + +[particles] + ppc0 = 8.0 + use_weights = true + sort_interval = 100 + +# [[particles.species]] +# label = "e-" +# mass = 1.0 +# charge = -1.0 +# maxnpart = 2e6 +# pusher = "Boris" +# +# [[particles.species]] +# label = "e+" +# mass = 1.0 +# charge = 1.0 +# maxnpart = 2e6 +# pusher = "Boris" + +[setup] + +[output] + format = "hdf5" + + [output.fields] + interval_time = 1.0 + quantities = ["D", "H", "B", "J", "A"] + + [output.particles] + enable = false + + [output.spectra] + enable = false + +[diagnostics] + interval = 1 From 59be02a6e9f4e50bb0bcf3cc5299dc3c73cb1440 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 5 Nov 2024 17:52:11 -0500 Subject: [PATCH 061/234] compilible pusher in gr --- src/engines/grpic.hpp | 41 ++++++++++++++++++-- src/kernels/particle_pusher_gr.hpp | 61 +++++++++++++++--------------- 2 files changed, 69 insertions(+), 33 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 3f90220ec..d48fb950e 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -344,7 +344,7 @@ namespace ntt { * Now: x_prtl at n + 1, u_prtl at n + 1/2 */ timers.start("ParticlePusher"); - // ParticlePush(dom); + ParticlePush(dom); timers.stop("ParticlePusher"); /** @@ -1004,10 +1004,16 @@ namespace ntt { // coeff = q / m (dt / 2) omegaB0 const auto coeff = q_ovr_m * HALF * dt * m_params.template get("scales.omegaB0"); + // clang-format off + + if (species.pusher() == PrtlPusher::PHOTON) { + auto range_policy = Kokkos::RangePolicy( + 0, + species.npart()); Kokkos::parallel_for( "ParticlePusher", - species.rangeActiveParticles(), + range_policy, kernel::gr::Pusher_kernel( domain.fields.em, domain.fields.em0, @@ -1022,9 +1028,38 @@ namespace ntt { domain.mesh.n_active(in::x1), domain.mesh.n_active(in::x2), domain.mesh.n_active(in::x3), - m_params.template get("algorithms.gr.pusher_eps"), m_params.template get("algorithms.gr.pusher_niter"), + m_params.template get("algorithms.gr.pusher_eps"), m_params.template get("algorithms.gr.pusher_niter"), domain.mesh.prtl_bc() )); + } else if (species.pusher() == PrtlPusher::BORIS) { + auto range_policy = Kokkos::RangePolicy( + 0, + species.npart()); + Kokkos::parallel_for( + "ParticlePusher", + range_policy, + kernel::gr::Pusher_kernel( + domain.fields.em, + domain.fields.em0, + species.i1, species.i2, species.i3, + species.i1_prev, species.i2_prev, species.i3_prev, + species.dx1, species.dx2, species.dx3, + species.dx1_prev, species.dx2_prev, species.dx3_prev, + species.ux1, species.ux2, species.ux3, + species.phi, species.tag, + domain.mesh.metric, + coeff, dt, + domain.mesh.n_active(in::x1), + domain.mesh.n_active(in::x2), + domain.mesh.n_active(in::x3), + m_params.template get("algorithms.gr.pusher_eps"), m_params.template get("algorithms.gr.pusher_niter"), + domain.mesh.prtl_bc() + )); + } else if (species.pusher() == PrtlPusher::NONE) { + // do nothing + } else { + raise::Error("not implemented", HERE); + } // clang-format on } } diff --git a/src/kernels/particle_pusher_gr.hpp b/src/kernels/particle_pusher_gr.hpp index 547463fa7..fd6246217 100644 --- a/src/kernels/particle_pusher_gr.hpp +++ b/src/kernels/particle_pusher_gr.hpp @@ -65,6 +65,7 @@ namespace kernel::gr { static_assert(M::is_metric, "M must be a metric class"); static constexpr auto D = M::Dim; + private: const randacc_ndfield_t DB; const randacc_ndfield_t DB0; array_t i1, i2, i3; @@ -86,34 +87,34 @@ namespace kernel::gr { bool is_absorb_i1min { false }, is_absorb_i1max { false }; public: - Pusher_kernel(const ndfield_t& DB, - const ndfield_t& DB0, - const array_t& i1, - const array_t& i2, - const array_t& i3, - const array_t& i1_prev, - const array_t& i2_prev, - const array_t& i3_prev, - const array_t& dx1, - const array_t& dx2, - const array_t& dx3, - const array_t& dx1_prev, - const array_t& dx2_prev, - const array_t& dx3_prev, - const array_t& ux1, - const array_t& ux2, - const array_t& ux3, - const array_t& phi, - const array_t& tag, - const M& metric, - const real_t& coeff, - const real_t& dt, - const int& ni1, - const int& ni2, - const int& ni3, - const real_t& epsilon, - const int& niter, - const boundaries_t& boundaries) + Pusher_kernel(const ndfield_t& DB, + const ndfield_t& DB0, + array_t& i1, + array_t& i2, + array_t& i3, + array_t& i1_prev, + array_t& i2_prev, + array_t& i3_prev, + array_t& dx1, + array_t& dx2, + array_t& dx3, + array_t& dx1_prev, + array_t& dx2_prev, + array_t& dx3_prev, + array_t& ux1, + array_t& ux2, + array_t& ux3, + array_t& phi, + array_t& tag, + const M& metric, + real_t coeff, + real_t dt, + int ni1, + int ni2, + int ni3, + const real_t& epsilon, + const int& niter, + const boundaries_t& boundaries) : DB { DB } , DB0 { DB0 } , i1 { i1 } @@ -351,8 +352,8 @@ namespace kernel::gr { vp_upd[1] = vp[1] + dt * - (-metric.alpha(xp) * u0 * DERIVATIVE_IN_TH(alpha, xp) + - vp_mid[1] * DERIVATIVE_IN_TH(beta1, xp) - + (-metric.alpha(xp) * u0 * DERIVATIVE_IN_TH(metric.alpha, xp) + + vp_mid[1] * DERIVATIVE_IN_TH(metric.beta1, xp) - (HALF / u0) * (DERIVATIVE_IN_TH((metric.template h<1, 1>), xp) * SQR(vp_mid[0]) + DERIVATIVE_IN_TH((metric.template h<2, 2>), xp) * SQR(vp_mid[1]) + From fb0788b5b5119cd69ca70a5fa8873250665254ef Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 6 Nov 2024 14:58:22 -0500 Subject: [PATCH 062/234] consistent type for pusher_niter --- src/engines/grpic.hpp | 4 ++-- src/kernels/particle_pusher_gr.hpp | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index d48fb950e..13d8899bf 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -1028,7 +1028,7 @@ namespace ntt { domain.mesh.n_active(in::x1), domain.mesh.n_active(in::x2), domain.mesh.n_active(in::x3), - m_params.template get("algorithms.gr.pusher_eps"), m_params.template get("algorithms.gr.pusher_niter"), + m_params.template get("algorithms.gr.pusher_eps"), m_params.template get("algorithms.gr.pusher_niter"), domain.mesh.prtl_bc() )); } else if (species.pusher() == PrtlPusher::BORIS) { @@ -1052,7 +1052,7 @@ namespace ntt { domain.mesh.n_active(in::x1), domain.mesh.n_active(in::x2), domain.mesh.n_active(in::x3), - m_params.template get("algorithms.gr.pusher_eps"), m_params.template get("algorithms.gr.pusher_niter"), + m_params.template get("algorithms.gr.pusher_eps"), m_params.template get("algorithms.gr.pusher_niter"), domain.mesh.prtl_bc() )); } else if (species.pusher() == PrtlPusher::NONE) { diff --git a/src/kernels/particle_pusher_gr.hpp b/src/kernels/particle_pusher_gr.hpp index fd6246217..fba40890f 100644 --- a/src/kernels/particle_pusher_gr.hpp +++ b/src/kernels/particle_pusher_gr.hpp @@ -77,11 +77,11 @@ namespace kernel::gr { array_t tag; const M metric; - const real_t coeff, dt; - const int ni1, ni2, ni3; - const real_t epsilon; - const int niter; - const int i1_absorb; + const real_t coeff, dt; + const int ni1, ni2, ni3; + const real_t epsilon; + const unsigned short niter; + const int i1_absorb; bool is_axis_i2min { false }, is_axis_i2max { false }; bool is_absorb_i1min { false }, is_absorb_i1max { false }; @@ -113,7 +113,7 @@ namespace kernel::gr { int ni2, int ni3, const real_t& epsilon, - const int& niter, + const unsigned short& niter, const boundaries_t& boundaries) : DB { DB } , DB0 { DB0 } From 619bacd377b7e63893b9d940f1afbab28f3958a1 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Thu, 7 Nov 2024 10:14:22 -0500 Subject: [PATCH 063/234] cleaned up an unused var --- src/engines/grpic.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 13d8899bf..40e2776e0 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -561,7 +561,6 @@ namespace ntt { "grid.boundaries.absorb.ds"); const auto dim = direction.get_dim(); real_t xg_min, xg_max, xg_edge; - auto sign = direction.get_sign(); xg_max = m_metadomain.mesh().extent(dim).second; xg_min = xg_max - ds; xg_edge = xg_max; From 483510184ad2aa69ca5c0608020831e61cb7269b Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Thu, 7 Nov 2024 16:01:16 -0500 Subject: [PATCH 064/234] fixed bug in output for dimensions of coordinates of particles in GR --- src/framework/domain/output.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/framework/domain/output.cpp b/src/framework/domain/output.cpp index be154ce16..c669eb8ae 100644 --- a/src/framework/domain/output.cpp +++ b/src/framework/domain/output.cpp @@ -79,7 +79,13 @@ namespace ntt { const auto species_to_write = params.template get>( "output.particles.species"); g_writer.defineFieldOutputs(S, all_fields_to_write); - g_writer.defineParticleOutputs(M::PrtlDim, species_to_write); + + Dimension dim = M::PrtlDim; + if constexpr (M::CoordType != Coord::Cart) { + dim = Dim::_3D; + } + g_writer.defineParticleOutputs(dim, species_to_write); + // spectra write all particle species std::vector spectra_species {}; for (const auto& sp : species_params()) { From 247177ee1a45a92f8c6fe02f0c73922e9af5c815 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 8 Nov 2024 16:08:26 -0500 Subject: [PATCH 065/234] fixed coordinate transformation for a global injector. When we inject a small amount of particles with given velocities, this is usually in physical coordinates rather than tetrads as in injection for a given energy distribution --- src/kernels/injectors.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kernels/injectors.hpp b/src/kernels/injectors.hpp index 9d3fd7d81..dd5e2d21d 100644 --- a/src/kernels/injectors.hpp +++ b/src/kernels/injectors.hpp @@ -335,7 +335,7 @@ namespace kernel { if constexpr (S == SimEngine::SRPIC) { global_metric.template transform_xyz(x_Cd_, u_Ph, u_Cd); } else if constexpr (S == SimEngine::GRPIC) { - global_metric.template transform(x_Cd, u_Ph, u_Cd); + global_metric.template transform(x_Cd, u_Ph, u_Cd); } else { raise::KernelError(HERE, "Unknown simulation engine"); } @@ -380,7 +380,7 @@ namespace kernel { if constexpr (S == SimEngine::SRPIC) { global_metric.template transform_xyz(x_Cd, u_Ph, u_Cd); } else if constexpr (S == SimEngine::GRPIC) { - global_metric.template transform(x_Cd, u_Ph, u_Cd); + global_metric.template transform(x_Cd, u_Ph, u_Cd); } else { raise::KernelError(HERE, "Unknown simulation engine"); } From a33d66442db07a14f36d890dd065e38415cde00d Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 8 Nov 2024 16:11:23 -0500 Subject: [PATCH 066/234] minor consistensy with sr --- src/kernels/particle_pusher_gr.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kernels/particle_pusher_gr.hpp b/src/kernels/particle_pusher_gr.hpp index fba40890f..2e9ddbde2 100644 --- a/src/kernels/particle_pusher_gr.hpp +++ b/src/kernels/particle_pusher_gr.hpp @@ -29,7 +29,7 @@ /* Local macros */ /* -------------------------------------------------------------------------- */ #define from_Xi_to_i(XI, I) \ - { I = static_cast((XI)); } + { I = static_cast((XI + 1)) - 1; } #define from_Xi_to_i_di(XI, I, DI) \ { \ From 702b0137794fb05def96aa7efccae34efb1863da Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 3 Dec 2024 10:18:51 -0500 Subject: [PATCH 067/234] kernels/particle_pusher_gr: 1. derivatives of the metric in geodesic pusher now use their functional form. Saved numerical derivatives for future testing 2. Added placeholders for storing previous coordinate 3. Fixed bug in geodesic pusher in updating utheta 4. Changes to boundary conditions at the axis --- src/kernels/particle_pusher_gr.hpp | 62 ++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/src/kernels/particle_pusher_gr.hpp b/src/kernels/particle_pusher_gr.hpp index 2e9ddbde2..306f19013 100644 --- a/src/kernels/particle_pusher_gr.hpp +++ b/src/kernels/particle_pusher_gr.hpp @@ -334,31 +334,53 @@ namespace kernel::gr { // find contravariant midpoint velocity metric.template transform(xp, vp_mid, vp_mid_cntrv); - // find Gamma / alpha at midpoint + // find Gamma / alpha at midpointы real_t u0 { computeGamma(T {}, vp_mid, vp_mid_cntrv) / metric.alpha(xp) }; // find updated velocity + // vp_upd[0] = + // vp[0] + + // dt * + // (-metric.alpha(xp) * u0 * DERIVATIVE_IN_R(metric.alpha, xp) + + // vp_mid[0] * DERIVATIVE_IN_R(metric.beta1, xp) - + // (HALF / u0) * + // (DERIVATIVE_IN_R((metric.template h<1, 1>), xp) * SQR(vp_mid[0]) + + // DERIVATIVE_IN_R((metric.template h<2, 2>), xp) * SQR(vp_mid[1]) + + // DERIVATIVE_IN_R((metric.template h<3, 3>), xp) * SQR(vp_mid[2]) + + // TWO * DERIVATIVE_IN_R((metric.template h<1, 3>), xp) * + // vp_mid[0] * vp_mid[2])); + // vp_upd[1] = + // vp[1] + + // dt * + // (-metric.alpha(xp) * u0 * DERIVATIVE_IN_TH(metric.alpha, xp) + + // vp_mid[0] * DERIVATIVE_IN_TH(metric.beta1, xp) - + // (HALF / u0) * + // (DERIVATIVE_IN_TH((metric.template h<1, 1>), xp) * SQR(vp_mid[0]) + + // DERIVATIVE_IN_TH((metric.template h<2, 2>), xp) * SQR(vp_mid[1]) + + // DERIVATIVE_IN_TH((metric.template h<3, 3>), xp) * SQR(vp_mid[2]) + + // TWO * DERIVATIVE_IN_TH((metric.template h<1, 3>), xp) * + // vp_mid[0] * vp_mid[2])); vp_upd[0] = vp[0] + dt * - (-metric.alpha(xp) * u0 * DERIVATIVE_IN_R(metric.alpha, xp) + - vp_mid[0] * DERIVATIVE_IN_R(metric.beta1, xp) - + (-metric.alpha(xp) * u0 * metric.dr_alpha(xp) + + vp_mid[0] * metric.dr_beta1(xp) - (HALF / u0) * - (DERIVATIVE_IN_R((metric.template h<1, 1>), xp) * SQR(vp_mid[0]) + - DERIVATIVE_IN_R((metric.template h<2, 2>), xp) * SQR(vp_mid[1]) + - DERIVATIVE_IN_R((metric.template h<3, 3>), xp) * SQR(vp_mid[2]) + - TWO * DERIVATIVE_IN_R((metric.template h<1, 3>), xp) * + (metric.dr_h11(xp) * SQR(vp_mid[0]) + + metric.dr_h22(xp) * SQR(vp_mid[1]) + + metric.dr_h33(xp) * SQR(vp_mid[2]) + + TWO * metric.dr_h13(xp) * vp_mid[0] * vp_mid[2])); vp_upd[1] = vp[1] + dt * - (-metric.alpha(xp) * u0 * DERIVATIVE_IN_TH(metric.alpha, xp) + - vp_mid[1] * DERIVATIVE_IN_TH(metric.beta1, xp) - + (-metric.alpha(xp) * u0 * metric.dt_alpha(xp) + + vp_mid[0] * metric.dt_beta1(xp) - (HALF / u0) * - (DERIVATIVE_IN_TH((metric.template h<1, 1>), xp) * SQR(vp_mid[0]) + - DERIVATIVE_IN_TH((metric.template h<2, 2>), xp) * SQR(vp_mid[1]) + - DERIVATIVE_IN_TH((metric.template h<3, 3>), xp) * SQR(vp_mid[2]) + - TWO * DERIVATIVE_IN_TH((metric.template h<1, 3>), xp) * + (metric.dt_h11(xp) * SQR(vp_mid[0]) + + metric.dt_h22(xp) * SQR(vp_mid[1]) + + metric.dt_h33(xp) * SQR(vp_mid[2]) + + TWO * metric.dt_h13(xp) * vp_mid[0] * vp_mid[2])); } } else if constexpr (D == Dim::_3D) { @@ -656,9 +678,13 @@ namespace kernel::gr { dx2_prev(p) = dx2(p); coord_t xp { ZERO }; + coord_t xp_prev { ZERO }; xp[0] = i_di_to_Xi(i1(p), dx1(p)); xp[1] = i_di_to_Xi(i2(p), dx2(p)); + + xp_prev[0] = i_di_to_Xi(i1_prev(p), dx1_prev(p)); + xp_prev[1] = i_di_to_Xi(i2_prev(p), dx2_prev(p)); vec_t Dp_cntrv { ZERO }, Bp_cntrv { ZERO }, Dp_hat { ZERO }, Bp_hat { ZERO }; @@ -692,7 +718,7 @@ namespace kernel::gr { { (xp[0] + xp_upd[0]) * HALF, (xp[1] + xp_upd[1]) * HALF }, vp_upd, phi(p)); - + // update coordinate int i1_, i2_; prtldx_t dx1_, dx2_; @@ -726,12 +752,16 @@ namespace kernel::gr { } } if constexpr (D == Dim::_2D || D == Dim::_3D) { - if (i2(p) < 1) { + if (i2(p) < 0) { if (is_axis_i2min) { + i2(p) = 0; + dx2(p) = ONE - dx2(p); ux2(p) = -ux2(p); } - } else if (i2(p) >= ni2 - 1) { + } else if (i2(p) >= ni2) { if (is_axis_i2min) { + i2(p) = ni2 - 1; + dx2(p) = ONE - dx2(p); ux2(p) = -ux2(p); } } From b58ede49f3cb7e70ffee675088a3d58252362b32 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 3 Dec 2024 10:22:49 -0500 Subject: [PATCH 068/234] metrics: all gr metrics now have functional form of derivatives of the metric componets --- src/metrics/kerr_schild.h | 163 +++++++++++++++++++++++++++++ src/metrics/kerr_schild_0.h | 102 ++++++++++++++++++ src/metrics/qkerr_schild.h | 201 ++++++++++++++++++++++++++++++++++++ 3 files changed, 466 insertions(+) diff --git a/src/metrics/kerr_schild.h b/src/metrics/kerr_schild.h index 5a60def53..9ec6f7c98 100644 --- a/src/metrics/kerr_schild.h +++ b/src/metrics/kerr_schild.h @@ -17,6 +17,7 @@ #include "arch/kokkos_aliases.h" #include "utils/numeric.h" +#include "utils/comparators.h" #include "metrics/metric_base.h" @@ -216,6 +217,28 @@ namespace metric { return ONE / math::sqrt(ONE + z(r, theta)); } + /** + * dr derivative of lapse function + * @param x coordinate array in code units + */ + Inline auto dr_alpha(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + const real_t dr_Sigma {TWO * r * dr}; + + return - (dr * Sigma(r, theta) - r * dr_Sigma) * CUBE(alpha(x)) / SQR(Sigma(r, theta)); + } + + /** + * dtheta derivative of lapse function + * @param x coordinate array in code units + */ + Inline auto dt_alpha(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + return CUBE(alpha(x)) * r * dt_Sigma(theta) / SQR(Sigma(r, theta)); + } + /** * radial component of shift vector * @param x coordinate array in code units @@ -225,6 +248,146 @@ namespace metric { return dr_inv * z_ / (ONE + z_); } + /** + * dr derivative of radial component of shift vector + * @param x coordinate array in code units + */ + Inline auto dr_beta1(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + const real_t dr_Sigma {TWO * r * dr}; + + return dr_inv * TWO * (dr * Sigma(r, theta) - r * dr_Sigma) / SQR(Sigma(r, theta) + TWO * r); + } + + /** + * dtheta derivative of radial component of shift vector + * @param x coordinate array in code units + */ + Inline auto dt_beta1(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + return - dr_inv * TWO * r * dt_Sigma(theta) / SQR(Sigma(r, theta) * (ONE + z(r, theta))); + } + + /** + * dr derivative of h^11 + * @param x coordinate array in code units + */ + Inline auto dr_h11(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + const real_t dr_Sigma {TWO * r * dr}; + const real_t dr_Delta {TWO * dr * (r - ONE)}; + const real_t dr_A {FOUR * r * dr * (SQR(r) + SQR(a)) - SQR(a) * SQR(math::sin(theta)) * dr_Delta}; + + return (Sigma(r, theta) * (Sigma(r, theta) + TWO * r) * dr_A + - TWO * A(r, theta) * (r * dr_Sigma + Sigma(r, theta) * (dr_Sigma + dr))) + / (SQR(Sigma(r, theta) * (Sigma(r, theta) + TWO * r))) * SQR(dr_inv); + } + + /** + * dr derivative of h^22 + * @param x coordinate array in code units + */ + Inline auto dr_h22(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + const real_t dr_Sigma {TWO * r * dr}; + + return - dr_Sigma / SQR(Sigma(r, theta)) * SQR(dtheta_inv); + } + + /** + * dr derivative of h^33 + * @param x coordinate array in code units + */ + Inline auto dr_h33(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + const real_t dr_Sigma {TWO * r * dr}; + + return - dr_Sigma / SQR(Sigma(r, theta)) / SQR(math::sin(theta)); + } + + /** + * dr derivative of h^13 + * @param x coordinate array in code units + */ + Inline auto dr_h13(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + const real_t dr_Sigma {TWO * r * dr}; + + return - a * dr_Sigma / SQR(Sigma(r, theta)) * dr_inv; + } + + /** + * dtheta derivative of Sigma + * @param x coordinate array in code units + */ + Inline auto dt_Sigma(const real_t& theta) const -> real_t { + const real_t dt_Sigma {- TWO * SQR(a) * math::sin(theta) * math::cos(theta) * dtheta}; + if (cmp::AlmostZero(dt_Sigma)) + return ZERO; + else + return dt_Sigma; + } + + /** + * dtheta derivative of A + * @param x coordinate array in code units + */ + Inline auto dt_A(const real_t& r, const real_t& theta) const -> real_t { + const real_t dt_A {- TWO * SQR(a) * math::sin(theta) * math::cos(theta) * Delta(r) * dtheta}; + if (cmp::AlmostZero(dt_A)) + return ZERO; + else + return dt_A; + } + + /** + * dtheta derivative of h^11 + * @param x coordinate array in code units + */ + Inline auto dt_h11(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + return (Sigma(r, theta) * (Sigma(r, theta) + TWO * r) * dt_A(r, theta) + - TWO * A(r, theta) * dt_Sigma(theta) * (r + Sigma(r, theta))) + / (SQR(Sigma(r, theta) * (Sigma(r, theta) + TWO * r))) * SQR(dr_inv); + } + + /** + * dtheta derivative of h^22 + * @param x coordinate array in code units + */ + Inline auto dt_h22(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + return - dt_Sigma(theta) / SQR(Sigma(r, theta)) * SQR(dtheta_inv); + } + + /** + * dtheta derivative of h^33 + * @param x coordinate array in code units + */ + Inline auto dt_h33(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + return - TWO * dtheta * math::cos(theta) * (Sigma(r, theta) - SQR(a) * SQR(math::sin(theta))) / CUBE(math::sin(theta)) / SQR(Sigma(r, theta)); + } + + /** + * dtheta derivative of h^13 + * @param x coordinate array in code units + */ + Inline auto dt_h13(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + return - a * dt_Sigma(theta) / SQR(Sigma(r, theta)) * dr_inv; + } + /** * sqrt(det(h_ij)) * @param x coordinate array in code units diff --git a/src/metrics/kerr_schild_0.h b/src/metrics/kerr_schild_0.h index 31080ed4c..baea9b4c1 100644 --- a/src/metrics/kerr_schild_0.h +++ b/src/metrics/kerr_schild_0.h @@ -171,6 +171,22 @@ namespace metric { return ONE; } + /** + * dr derivative of lapse function + * @param x coordinate array in code units + */ + Inline auto dr_alpha(const coord_t& x) const -> real_t { + return ZERO; + } + + /** + * dtheta derivative of lapse function + * @param x coordinate array in code units + */ + Inline auto dt_alpha(const coord_t& x) const -> real_t { + return ZERO; + } + /** * radial component of shift vector * @param x coordinate array in code units @@ -179,6 +195,92 @@ namespace metric { return ZERO; } + /** + * dr derivative of radial component of shift vector + * @param x coordinate array in code units + */ + Inline auto dr_beta1(const coord_t& x) const -> real_t { + return ZERO; + } + + /** + * dtheta derivative of radial component of shift vector + * @param x coordinate array in code units + */ + Inline auto dt_beta1(const coord_t& x) const -> real_t { + return ZERO; + } + + /** + * dr derivative of h^11 + * @param x coordinate array in code units + */ + Inline auto dr_h11(const coord_t& x) const -> real_t { + return ZERO; + } + + /** + * dr derivative of h^22 + * @param x coordinate array in code units + */ + Inline auto dr_h22(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + return - TWO / CUBE(r) * SQR(dtheta_inv) * dr; + } + + /** + * dr derivative of h^33 + * @param x coordinate array in code units + */ + Inline auto dr_h33(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + return - TWO / CUBE(r) / SQR(math::sin(theta)) * dr; + } + + /** + * dr derivative of h^13 + * @param x coordinate array in code units + */ + Inline auto dr_h13(const coord_t& x) const -> real_t { + return ZERO; + } + + /** + * dtheta derivative of h^11 + * @param x coordinate array in code units + */ + Inline auto dt_h11(const coord_t& x) const -> real_t { + return ZERO; + } + + /** + * dtheta derivative of h^22 + * @param x coordinate array in code units + */ + Inline auto dt_h22(const coord_t& x) const -> real_t { + return ZERO; + } + + /** + * dtheta derivative of h^33 + * @param x coordinate array in code units + */ + Inline auto dt_h33(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + return - TWO * math::cos(theta) / SQR(r) / CUBE(math::sin(theta)) * dtheta; + } + + /** + * dtheta derivative of h^13 + * @param x coordinate array in code units + */ + Inline auto dt_h13(const coord_t& x) const -> real_t { + return ZERO; + } + /** * sqrt(det(h_ij)) * @param x coordinate array in code units diff --git a/src/metrics/qkerr_schild.h b/src/metrics/qkerr_schild.h index 85507c6e1..e45376a24 100644 --- a/src/metrics/qkerr_schild.h +++ b/src/metrics/qkerr_schild.h @@ -234,6 +234,33 @@ namespace metric { return ONE / math::sqrt(ONE + z(r, theta)); } + /** + * dr derivative of lapse function + * @param x coordinate array in code units + */ + Inline auto dr_alpha(const coord_t& x) const -> real_t { + const real_t r { r0 + math::exp(x[0] * dchi + chi_min) }; + const real_t theta { eta2theta(x[1] * deta + eta_min) }; + const real_t dx_r {dchi * math::exp(x[0] * dchi + chi_min)}; + const real_t dr_Sigma {TWO * r * dx_r}; + + return - (dx_r * Sigma(r, theta) - r * dr_Sigma) * CUBE(alpha(x)) / SQR(Sigma(r, theta)); + } + + /** + * dtheta derivative of lapse function + * @param x coordinate array in code units + */ + Inline auto dt_alpha(const coord_t& x) const -> real_t { + const real_t r { r0 + math::exp(x[0] * dchi + chi_min) }; + const real_t eta {x[1] * deta + eta_min }; + const real_t theta { eta2theta(eta) }; + const real_t dx_dt {deta * (ONE + TWO * h0 * constant::INV_PI_SQR * (TWO * THREE * SQR(eta) - TWO * THREE * constant::PI * eta + constant::PI_SQR)) }; + const real_t dt_Sigma {- TWO * SQR(a) * math::sin(theta) * math::cos(theta) * dx_dt}; + + return r * dt_Sigma * CUBE(alpha(x)) / SQR(Sigma(r, theta)); + } + /** * radial component of shift vector * @param x coordinate array in code units @@ -246,6 +273,167 @@ namespace metric { return math::exp(-chi) * dchi_inv * z_ / (ONE + z_); } + /** + * dr derivative of radial component of shift vector + * @param x coordinate array in code units + */ + Inline auto dr_beta1(const coord_t& x) const -> real_t { + const real_t chi { x[0] * dchi + chi_min }; + const real_t r { r0 + math::exp(chi) }; + const real_t theta { eta2theta(x[1] * deta + eta_min) }; + const real_t z_ { z(r, theta) }; + const real_t dx_r {dchi * math::exp(x[0] * dchi + chi_min)}; + const real_t dr_Sigma {TWO * r * dx_r}; + + return math::exp(-chi) * dchi_inv * TWO * (dx_r * Sigma(r, theta) - r * dr_Sigma) / SQR(Sigma(r, theta) + TWO * r) + - dchi * math::exp(-chi) * dchi_inv * z_ / (ONE + z_); + } + + /** + * dr derivative of radial component of shift vector + * @param x coordinate array in code units + */ + Inline auto dt_beta1(const coord_t& x) const -> real_t { + const real_t chi { x[0] * dchi + chi_min }; + const real_t r { r0 + math::exp(chi) }; + const real_t eta {x[1] * deta + eta_min }; + const real_t theta { eta2theta(eta) }; + return - math::exp(-chi) * dchi_inv * TWO * r * dt_Sigma(eta) / SQR(Sigma(r, theta) * (ONE + z(r, theta))); + } + + /** + * dr derivative of h^11 + * @param x coordinate array in code units + */ + Inline auto dr_h11(const coord_t& x) const -> real_t { + const real_t r { r0 + math::exp(x[0] * dchi + chi_min) }; + const real_t theta { eta2theta(x[1] * deta + eta_min) }; + + const real_t dx_r {dchi * math::exp(x[0] * dchi + chi_min)}; + const real_t dr_Sigma {TWO * r * dx_r}; + const real_t dr_Delta {TWO * dx_r * (r - ONE)}; + const real_t dr_A {FOUR * r * dx_r * (SQR(r) + SQR(a)) - SQR(a) * SQR(math::sin(theta)) * dr_Delta}; + + return (math::exp(-TWO * (x[0] * dchi + chi_min)) / SQR(dchi) + * (Sigma(r, theta) * (Sigma(r, theta) + TWO * r) * dr_A + - TWO * A(r, theta) * (r * dr_Sigma + Sigma(r, theta) * (dr_Sigma + dx_r))) + / (SQR(Sigma(r, theta) * (Sigma(r, theta) + TWO * r))) ) + -TWO * dchi * math::exp(-TWO * (x[0] * dchi + chi_min)) / SQR(dchi) * A(r, theta) / (Sigma(r, theta) * (Sigma(r, theta) + TWO * r)); + } + + /** + * dr derivative of h^22 + * @param x coordinate array in code units + */ + Inline auto dr_h22(const coord_t& x) const -> real_t { + const real_t r { r0 + math::exp(x[0] * dchi + chi_min) }; + const real_t theta { eta2theta(x[1] * deta + eta_min) }; + const real_t dx_r {dchi * math::exp(x[0] * dchi + chi_min)}; + const real_t dr_Sigma {TWO * r * dx_r}; + + return - dr_Sigma / SQR(Sigma(r, theta)) / SQR(deta); + } + + /** + * dr derivative of h^33 + * @param x coordinate array in code units + */ + Inline auto dr_h33(const coord_t& x) const -> real_t { + const real_t r { r0 + math::exp(x[0] * dchi + chi_min) }; + const real_t theta { eta2theta(x[1] * deta + eta_min) }; + const real_t dx_r {dchi * math::exp(x[0] * dchi + chi_min)}; + const real_t dr_Sigma {TWO * r * dx_r}; + + return - dr_Sigma / SQR(Sigma(r, theta)) / SQR(math::sin(theta)); + } + + /** + * dr derivative of h^13 + * @param x coordinate array in code units + */ + Inline auto dr_h13(const coord_t& x) const -> real_t { + const real_t r { r0 + math::exp(x[0] * dchi + chi_min) }; + const real_t theta { eta2theta(x[1] * deta + eta_min) }; + const real_t dx_r {dchi * math::exp(x[0] * dchi + chi_min)}; + const real_t dr_Sigma {TWO * r * dx_r}; + + return - a * dr_Sigma / SQR(Sigma(r, theta)) * (math::exp(-(x[0] * dchi + chi_min)) * dchi_inv) + - dchi * (math::exp(-(x[0] * dchi + chi_min)) * dchi_inv) * a / Sigma(r, theta); + } + + /** + * dtheta derivative of Sigma + * @param x coordinate array in code units + */ + Inline auto dt_Sigma(const real_t& eta) const -> real_t { + const real_t theta { eta2theta(eta) }; + const real_t dt_Sigma {- TWO * SQR(a) * math::sin(theta) * math::cos(theta) * dx_dt(eta)}; + if (cmp::AlmostZero(dt_Sigma)) + return ZERO; + else + return dt_Sigma; + } + + /** + * dtheta derivative of A + * @param x coordinate array in code units + */ + Inline auto dt_A(const real_t& r, const real_t& eta) const -> real_t { + const real_t theta { eta2theta(eta) }; + const real_t dt_A {- TWO * SQR(a) * math::sin(theta) * math::cos(theta) * Delta(r) * dx_dt(eta)}; + if (cmp::AlmostZero(dt_A)) + return ZERO; + else + return dt_A; + } + + /** + * dtheta derivative of h^11 + * @param x coordinate array in code units + */ + Inline auto dt_h11(const coord_t& x) const -> real_t { + const real_t r { r0 + math::exp(x[0] * dchi + chi_min) }; + const real_t eta {x[1] * deta + eta_min }; + const real_t theta { eta2theta(eta) }; + return math::exp(-TWO * (x[0] * dchi + chi_min)) / SQR(dchi) + * (Sigma(r, theta) * (Sigma(r, theta) + TWO * r) * dt_A(r, eta) + - TWO * A(r, theta) * dt_Sigma(eta) * (r + Sigma(r, theta))) + / (SQR(Sigma(r, theta) * (Sigma(r, theta) + TWO * r))); + } + + /** + * dtheta derivative of h^22 + * @param x coordinate array in code units + */ + Inline auto dt_h22(const coord_t& x) const -> real_t { + const real_t r { r0 + math::exp(x[0] * dchi + chi_min) }; + const real_t eta {x[1] * deta + eta_min }; + const real_t theta { eta2theta(eta) }; + return - dt_Sigma(eta) / SQR(Sigma(r, theta)) / SQR(deta); + } + + /** + * dtheta derivative of h^33 + * @param x coordinate array in code units + */ + Inline auto dt_h33(const coord_t& x) const -> real_t { + const real_t r { r0 + math::exp(x[0] * dchi + chi_min) }; + const real_t eta {x[1] * deta + eta_min }; + const real_t theta { eta2theta(eta) }; + return - (dt_Sigma(eta) + TWO * math::cos(theta) / math::sin(theta) * Sigma(r, theta) * dx_dt(eta)) / SQR(Sigma(r, theta) * math::sin(theta)); + } + + /** + * dtheta derivative of h^13 + * @param x coordinate array in code units + */ + Inline auto dt_h13(const coord_t& x) const -> real_t { + const real_t r { r0 + math::exp(x[0] * dchi + chi_min) }; + const real_t eta {x[1] * deta + eta_min }; + const real_t theta { eta2theta(eta) }; + return - a * dt_Sigma(eta) / SQR(Sigma(r, theta)) * (math::exp(-(x[0] * dchi + chi_min)) * dchi_inv); + } + /** * sqrt(det(h_ij)) * @param x coordinate array in code units @@ -456,6 +644,19 @@ namespace metric { } } + /** + * @brief quasi-spherical eta to spherical theta + */ + Inline auto dx_dt(const real_t& eta) const -> real_t { + if (cmp::AlmostZero(h0)) { + return deta; + } else { + return deta * (ONE + + TWO * h0 * constant::INV_PI_SQR * + (TWO * THREE * SQR(eta) - TWO * THREE * constant::PI * eta + constant::PI_SQR)); + } + } + /** * @brief spherical theta to quasi-spherical eta */ From 2edb1972c037b1147be91b4e759d86b29cb69605 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 3 Dec 2024 10:24:38 -0500 Subject: [PATCH 069/234] setups/grpic/pusher: pgen and inputs for pusher tests --- setups/grpic/pusher/boris.toml | 97 +++++++++ setups/grpic/pusher/massive_gravity_3d.toml | 84 ++++++++ setups/grpic/pusher/massive_gravity_a0.toml | 93 +++++++++ .../grpic/pusher/massive_gravity_a0995.toml | 93 +++++++++ setups/grpic/pusher/pgen.hpp | 184 ++++++++++++++++++ 5 files changed, 551 insertions(+) create mode 100644 setups/grpic/pusher/boris.toml create mode 100644 setups/grpic/pusher/massive_gravity_3d.toml create mode 100644 setups/grpic/pusher/massive_gravity_a0.toml create mode 100644 setups/grpic/pusher/massive_gravity_a0995.toml create mode 100644 setups/grpic/pusher/pgen.hpp diff --git a/setups/grpic/pusher/boris.toml b/setups/grpic/pusher/boris.toml new file mode 100644 index 000000000..6483eb2c6 --- /dev/null +++ b/setups/grpic/pusher/boris.toml @@ -0,0 +1,97 @@ +[simulation] + name = "boris_a0" + engine = "grpic" + runtime = 100.0 + +[grid] + resolution = [128,128] + extent = [[0.9, 20.0]] + + [grid.metric] + metric = "qkerr_schild" + qsph_r0 = 0.0 + qsph_h = 0.0 + ks_a = 0.0 + + [grid.boundaries] + fields = [["ABSORB"]] + particles = [["ABSORB"]] + + [grid.boundaries.absorb] + ds = 0.5 + +[scales] + larmor0 = 5.0 + skindepth0 = 1e-3 + +[algorithms] + current_filters = 4 + + [algorithms.gr] + pusher_niter = 10 + pusher_eps = 1e-6 + + [algorithms.timestep] + CFL = 0.1 + + [algorithms.toggles] + deposit = false + fieldsolver = false + +[particles] + ppc0 = 1.0 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = -1.0 + maxnpart = 1e0 + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = -1.0 + maxnpart = 1e1 + +[setup] + x1s = [5.77] + y1s = [1.570796] + z1s = [0.000000] + ux1s = [0.4376657824933686] + uy1s = [0.000000] + uz1s = [4.0] +# x1s = [10.0] +# y1s = [1.570796] +# z1s = [0.000000] +# ux1s = [0.0] +# uy1s = [0.000000] +# uz1s = [10.00000] + + #x2s = [7.5] + #y2s = [0.15745454545454546] + #z2s = [0.000000] + #ux2s = [0.029637] + #uy2s = [0.000000] + #uz2s = [3.75] + +[output] + format = "hdf5" + + [output.fields] + enable = true + interval_time = 1.0 + quantities = ["D", "B"] + + [output.particles] + enable = true + stride = 1 + interval_time = 0.2 + species = [] + + [output.spectra] + enable = false + +[diagnostics] + interval = 10 + colored_stdout = true + blocking_timers = true diff --git a/setups/grpic/pusher/massive_gravity_3d.toml b/setups/grpic/pusher/massive_gravity_3d.toml new file mode 100644 index 000000000..5b87554b2 --- /dev/null +++ b/setups/grpic/pusher/massive_gravity_3d.toml @@ -0,0 +1,84 @@ +[simulation] + name = "massive_3d" + engine = "grpic" + runtime = 1000.0 + +[grid] + resolution = [128, 128] + extent = [[0.1, 15.0]] + + [grid.metric] + metric = "qkerr_schild" + qsph_r0 = 0.0 + qsph_h = 0.0 + ks_a = 0.995 + + [grid.boundaries] + fields = [["ABSORB"]] + particles = [["ABSORB"]] + + [grid.boundaries.absorb] + ds = 0.5 + +[scales] + larmor0 = 2e-3 + skindepth0 = 0.1 + +[algorithms] + current_filters = 4 + + [algorithms.gr] + pusher_niter = 20 + pusher_eps = 1e-3 + + [algorithms.timestep] + CFL = 0.8 + + [algorithms.toggles] + deposit = false + fieldsolver = false + +[particles] + ppc0 = 4.0 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = 0.0 + maxnpart = 1e1 + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = 0.0 + maxnpart = 1e1 + +[setup] + x1s = [10.00000] + y1s = [1.570796] + z1s = [0.000000] + ux1s = [0.2181435531189523] + uy1s = [2.89] + uz1s = [1.05769] + +[output] + format = "hdf5" + + [output.fields] + enable = true + interval_time = 5.0 + quantities = ["D", "B"] + + [output.particles] + enable = true + stride = 1 + interval_time = 0.5 + species = [] + + [output.spectra] + enable = false + +[diagnostics] + interval = 10 + colored_stdout = true + blocking_timers = true diff --git a/setups/grpic/pusher/massive_gravity_a0.toml b/setups/grpic/pusher/massive_gravity_a0.toml new file mode 100644 index 000000000..08b607d4b --- /dev/null +++ b/setups/grpic/pusher/massive_gravity_a0.toml @@ -0,0 +1,93 @@ +[simulation] + name = "massive_a0" + engine = "grpic" + runtime = 500.0 + +[grid] + resolution = [256,256] + extent = [[1.2, 15.0]] + + [grid.metric] + metric = "qkerr_schild" + qsph_r0 = 0.0 + qsph_h = 0.0 + ks_a = 0.0 + + [grid.boundaries] + fields = [["ABSORB"]] + particles = [["ABSORB"]] + + [grid.boundaries.absorb] + ds = 0.5 + +[scales] + larmor0 = 2e-3 + skindepth0 = 0.1 + +[algorithms] + current_filters = 4 + + [algorithms.gr] + pusher_niter = 10 + pusher_eps = 1e-3 + + [algorithms.timestep] + CFL = 0.8 + + [algorithms.toggles] + deposit = false + fieldsolver = false + +[particles] + ppc0 = 4.0 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = 0.0 + maxnpart = 1e1 + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = 0.0 + maxnpart = 1e1 + +[setup] + # (1,2,0) from Levin&Perez-Giz (2008) + x1s = [9.85180645] + y1s = [1.570796] + z1s = [0.5235987755982988] + ux1s = [0.24159816] + uy1s = [3.535534] + uz1s = [0.000000] + + # (4,1,1) from Levin&Perez-Giz (2008) + x2s = [68.600387] + y2s = [1.570796] + z2s = [0.000000] + ux2s = [0.029637] + uy2s = [0.000000] + uz2s = [3.900000] + +[output] + format = "hdf5" + + [output.fields] + enable = true + interval_time = 1.0 + quantities = ["D", "B"] + + [output.particles] + enable = true + stride = 1 + interval_time = 1.0 + species = [] + + [output.spectra] + enable = false + +[diagnostics] + interval = 10 + colored_stdout = true + blocking_timers = true diff --git a/setups/grpic/pusher/massive_gravity_a0995.toml b/setups/grpic/pusher/massive_gravity_a0995.toml new file mode 100644 index 000000000..b04538e10 --- /dev/null +++ b/setups/grpic/pusher/massive_gravity_a0995.toml @@ -0,0 +1,93 @@ +[simulation] + name = "massive_a0995" + engine = "grpic" + runtime = 2000.0 + +[grid] + resolution = [512, 512] + extent = [[0.9, 15.0]] + + [grid.metric] + metric = "kerr_schild" + qsph_r0 = 0.0 + qsph_h = 0.0 + ks_a = 0.0 + + [grid.boundaries] + fields = [["ABSORB"]] + particles = [["ABSORB"]] + + [grid.boundaries.absorb] + ds = 0.5 + +[scales] + larmor0 = 2e-3 + skindepth0 = 0.1 + +[algorithms] + current_filters = 4 + + [algorithms.gr] + pusher_niter = 10 + pusher_eps = 1e-10 + + [algorithms.timestep] + CFL = 6.0 + + [algorithms.toggles] + deposit = false + fieldsolver = false + +[particles] + ppc0 = 4.0 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = 0.0 + maxnpart = 1e1 + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = 0.0 + maxnpart = 1e1 + +[setup] + # (2,3,1) from Levin&Perez-Giz (2008) + x1s = [10.343515586064923] + y1s = [1.570796] + z1s = [0.000000] + ux1s = [0.19483849893499136] + uy1s = [0.000000] + uz1s = [2.000000] + + # (~1000, 3, ~671) from Levin&Perez-Giz (2008) + x2s = [10.64975354] + y2s = [1.570796] + z2s = [0.000000] + ux2s = [0.18914503] + uy2s = [0.000000] + uz2s = [2.000000] + +[output] + format = "hdf5" + + [output.fields] + enable = true + interval_time = 5.0 + quantities = ["D", "B"] + + [output.particles] + enable = true + stride = 1 + interval_time = 2.0 + species = [] + + [output.spectra] + enable = false + +[diagnostics] + interval = 10 + colored_stdout = true + blocking_timers = true diff --git a/setups/grpic/pusher/pgen.hpp b/setups/grpic/pusher/pgen.hpp new file mode 100644 index 000000000..d6a29a185 --- /dev/null +++ b/setups/grpic/pusher/pgen.hpp @@ -0,0 +1,184 @@ +#ifndef PROBLEM_GENERATOR_H +#define PROBLEM_GENERATOR_H + +#include "enums.h" +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "arch/traits.h" +#include "utils/comparators.h" +#include "utils/formatting.h" +#include "utils/log.h" +#include "utils/numeric.h" + +#include "archetypes/problem_generator.h" +#include "framework/domain/metadomain.h" +#include "framework/domain/domain.h" +#include "archetypes/energy_dist.h" +#include "archetypes/particle_injector.h" + +#include + +namespace user { + using namespace ntt; + + template + struct InitFields { + InitFields(M metric_) : metric { metric_ } {} + + Inline auto A_3(const coord_t& x_Cd) const -> real_t { + // return HALF * (metric.template h_<3, 3>(x_Cd) ); + // + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + // ); + coord_t x_Ph { ZERO }; + metric.template convert(x_Cd, x_Ph); + return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); + } + + Inline auto A_1(const coord_t& x_Cd) const -> real_t { + return HALF * (metric.template h_<1, 3>(x_Cd) + + TWO * metric.spin() * metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) + ); + } + + Inline auto A_0(const coord_t& x_Cd) const -> real_t { + real_t g_00 { -metric.alpha(x_Cd) * metric.alpha(x_Cd) + + metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) * metric.beta1(x_Cd) + }; + return HALF * (metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + + TWO * metric.spin() * g_00); + } + + Inline auto bx1(const coord_t& x_Ph) const -> real_t { + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0]; + x0m[1] = xi[1] - HALF; + x0p[0] = xi[0]; + x0p[1] = xi[1] + HALF; + + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + + if (cmp::AlmostZero(x_Ph[1])) + return ONE; + else + return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; + // return ZERO; + } + + Inline auto bx2(const coord_t& x_Ph) const -> real_t { + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0] + HALF - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF + HALF; + x0p[1] = xi[1]; + + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + if (cmp::AlmostZero(x_Ph[1])) + return ZERO; + else + return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; + // return ZERO; + } + + Inline auto bx3(const coord_t& x_Ph) const -> real_t { + // coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + // metric.template convert(x_Ph, xi); + + // x0m[0] = xi[0] + HALF - HALF; + // x0m[1] = xi[1]; + // x0p[0] = xi[0] + HALF + HALF; + // x0p[1] = xi[1]; + + // real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + // return -(A_1(x0p) - A_1(x0m)) * inv_sqrt_detH_ijP; + return ZERO; + } + + Inline auto dx1(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx2(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx3(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + private: + const M metric; + }; + + template + struct PGen : public arch::ProblemGenerator { + // compatibility traits for the problem generator + static constexpr auto engines { traits::compatible_with::value }; + static constexpr auto metrics { + traits::compatible_with::value + }; + static constexpr auto dimensions { traits::compatible_with::value }; + + // for easy access to variables in the child class + using arch::ProblemGenerator::D; + using arch::ProblemGenerator::C; + using arch::ProblemGenerator::params; + + InitFields init_flds; + const Metadomain& global_domain; + inline PGen(const SimulationParams& p, const Metadomain& m) + : arch::ProblemGenerator { p } + , global_domain { m } + , init_flds { m.mesh().metric } {} + + inline void InitPrtls(Domain& local_domain) { + const auto empty = std::vector {}; + const auto x1s = params.template get>("setup.x1s", empty); + const auto y1s = params.template get>("setup.y1s", empty); + const auto z1s = params.template get>("setup.z1s", empty); + const auto ux1s = params.template get>("setup.ux1s", + empty); + const auto uy1s = params.template get>("setup.uy1s", + empty); + const auto uz1s = params.template get>("setup.uz1s", + empty); + + const auto x2s = params.template get>("setup.x2s", empty); + const auto y2s = params.template get>("setup.y2s", empty); + const auto z2s = params.template get>("setup.z2s", empty); + const auto ux2s = params.template get>("setup.ux2s", + empty); + const auto uy2s = params.template get>("setup.uy2s", + empty); + const auto uz2s = params.template get>("setup.uz2s", + empty); + const std::map> data_1 { + { "x1", x1s}, + { "x2", y1s}, + { "phi", z1s}, + {"ux1", ux1s}, + {"ux2", uy1s}, + {"ux3", uz1s} + }; + const std::map> data_2 { + { "x1", x2s}, + { "x2", y2s}, + { "phi", z2s}, + {"ux1", ux2s}, + {"ux2", uy2s}, + {"ux3", uz2s} + }; + + arch::InjectGlobally(global_domain, local_domain, (arch::spidx_t)1, data_1); + arch::InjectGlobally(global_domain, local_domain, (arch::spidx_t)2, data_2); + } + // inline PGen() {} + }; + +} // namespace user + +#endif From 796cfcbb979240d094fa039a28a780d2be5ad99f Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 3 Dec 2024 12:28:14 -0500 Subject: [PATCH 070/234] engines/grpic: added timestep correction for gr pusher --- src/engines/grpic.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 40e2776e0..4c2278f1f 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -1001,9 +1001,9 @@ namespace ntt { ? species.charge() / species.mass() : ZERO; // coeff = q / m (dt / 2) omegaB0 - const auto coeff = q_ovr_m * HALF * dt * - m_params.template get("scales.omegaB0"); - + const auto coeff = q_ovr_m * HALF * dt * m_params.template get( + "algorithms.timestep.correction") * + m_params.template get("scales.omegaB0"); // clang-format off if (species.pusher() == PrtlPusher::PHOTON) { From 3d7e78156083c7566b1a80c1a94b154ac70c00fa Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 4 Dec 2024 14:58:18 -0500 Subject: [PATCH 071/234] erased stretching of initial field --- src/archetypes/field_setter.h | 52 ++++------------------------------- src/kernels/fields_bcs.hpp | 20 ++------------ 2 files changed, 8 insertions(+), 64 deletions(-) diff --git a/src/archetypes/field_setter.h b/src/archetypes/field_setter.h index 171e9f8a8..5c5c4dbe4 100644 --- a/src/archetypes/field_setter.h +++ b/src/archetypes/field_setter.h @@ -170,32 +170,13 @@ namespace arch { const real_t x2_H { metric.template convert<2, Crd::Cd, Crd::Ph>( i2_ + HALF) }; { // dx1 - vec_t d_PU { finit.dx1({ x1_H, x2_0 }), - finit.dx2({ x1_H, x2_0 }), - finit.dx3({ x1_H, x2_0 }) }; - vec_t d_U { ZERO }; - metric.template transform({ i1_ + HALF, i2_ }, - d_PU, - d_U); - EM(i1, i2, em::dx1) = d_U[0]; + EM(i1, i2, em::dx1) = finit.dx1({ x1_H, x2_0 }); } { // dx2 - vec_t d_PU { finit.dx1({ x1_0, x2_H }), - finit.dx2({ x1_0, x2_H }), - finit.dx3({ x1_0, x2_H }) }; - vec_t d_U { ZERO }; - metric.template transform({ i1_, i2_ + HALF }, - d_PU, - d_U); - EM(i1, i2, em::dx2) = d_U[1]; + EM(i1, i2, em::dx2) = finit.dx2({ x1_0, x2_H }); } { // dx3 - vec_t d_PU { finit.dx1({ x1_0, x2_0 }), - finit.dx2({ x1_0, x2_0 }), - finit.dx3({ x1_0, x2_0 }) }; - vec_t d_U { ZERO }; - metric.template transform({ i1_, i2_ }, d_PU, d_U); - EM(i1, i2, em::dx3) = d_U[2]; + EM(i1, i2, em::dx3) = finit.dx3({ x1_0, x2_0 }); } } if constexpr (defines_bx1 && defines_bx2 && defines_bx3) { @@ -206,34 +187,13 @@ namespace arch { const real_t x2_H { metric.template convert<2, Crd::Cd, Crd::Ph>( i2_ + HALF) }; { // bx1 - vec_t b_PU { finit.bx1({ x1_0, x2_H }), - finit.bx2({ x1_0, x2_H }), - finit.bx3({ x1_0, x2_H }) }; - vec_t b_U { ZERO }; - metric.template transform({ i1_, i2_ + HALF }, - b_PU, - b_U); - EM(i1, i2, em::bx1) = b_U[0]; + EM(i1, i2, em::bx1) = finit.bx1({ x1_0, x2_H }); } { // bx2 - vec_t b_PU { finit.bx1({ x1_H, x2_0 }), - finit.bx2({ x1_H, x2_0 }), - finit.bx3({ x1_H, x2_0 }) }; - vec_t b_U { ZERO }; - metric.template transform({ i1_ + HALF, i2_ }, - b_PU, - b_U); - EM(i1, i2, em::bx2) = b_U[1]; + EM(i1, i2, em::bx2) = finit.bx2({ x1_H, x2_0 }); } { // bx3 - vec_t b_PU { finit.bx1({ x1_H, x2_H }), - finit.bx2({ x1_H, x2_H }), - finit.bx3({ x1_H, x2_H }) }; - vec_t b_U { ZERO }; - metric.template transform({ i1_ + HALF, i2_ + HALF }, - b_PU, - b_U); - EM(i1, i2, em::bx3) = b_U[2]; + EM(i1, i2, em::bx3) = finit.bx3({ x1_H, x2_H }); } } } else { diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index e69fbcc13..e5b3de647 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -639,28 +639,12 @@ template const real_t x1_0 { metric.template convert<1, Crd::Cd, Crd::Ph>(i1_) }; const real_t x2_H { metric.template convert<2, Crd::Cd, Crd::Ph>( i2_ + HALF) }; - - vec_t b_PU { finit.bx1({ x1_0, x2_H }), - finit.bx2({ x1_0, x2_H }), - finit.bx3({ x1_0, x2_H }) }; - vec_t b_U { ZERO }; - metric.template transform({ i1_, i2_ + HALF }, - b_PU, - b_U); - Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * b_U[0]; + Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * finit.bx1({ x1_0, x2_H }); } else if (comp == em::bx2) { const real_t x1_H { metric.template convert<1, Crd::Cd, Crd::Ph>( i1_ + HALF) }; const real_t x2_0 { metric.template convert<2, Crd::Cd, Crd::Ph>(i2_) }; - - vec_t b_PU { finit.bx1({ x1_H, x2_0 }), - finit.bx2({ x1_H, x2_0 }), - finit.bx3({ x1_H, x2_0 }) }; - vec_t b_U { ZERO }; - metric.template transform({ i1_ + HALF, i2_ }, - b_PU, - b_U); - Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * b_U[1]; + Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * finit.bx2({ x1_H, x2_0 }); } } } else { From d0a44184ad84e043616c7fcd4a322842346fd429 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 4 Dec 2024 18:06:57 -0500 Subject: [PATCH 072/234] engines/grpic: updated routines for current (cur --> cur0) and added BC for currents --- src/engines/grpic.hpp | 74 +++++++++++++++++++++++++++++++++---------- 1 file changed, 57 insertions(+), 17 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 4c2278f1f..06088c1eb 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -354,7 +354,7 @@ namespace ntt { */ if (deposit_enabled) { timers.start("CurrentDeposit"); - Kokkos::deep_copy(dom.fields.cur, ZERO); + Kokkos::deep_copy(dom.fields.cur0, ZERO); CurrentsDeposit(dom); timers.stop("CurrentDeposit"); @@ -611,6 +611,50 @@ namespace ntt { } } + void CurrentsBoundaryConditions(domain_t& domain) { + /** + * absorbing boundaries + */ + const auto ds = m_params.template get( + "grid.boundaries.absorb.ds"); + const auto dim = in::x1; + real_t xg_min, xg_max, xg_edge; + xg_max = m_metadomain.mesh().extent(dim).second; + xg_min = xg_max - ds; + xg_edge = xg_max; + + boundaries_t box; + boundaries_t incl_ghosts; + for (unsigned short d { 0 }; d < M::Dim; ++d) { + if (d == static_cast(dim)) { + box.push_back({ xg_min, xg_max }); + incl_ghosts.push_back({ false, true }); + } else { + box.push_back(Range::All); + incl_ghosts.push_back({ true, true }); + } + } + if (not domain.mesh.Intersects(box)) { + return; + } + const auto intersect_range = domain.mesh.ExtentToRange(box, incl_ghosts); + tuple_t range_min { 0 }; + tuple_t range_max { 0 }; + + for (unsigned short d { 0 }; d < M::Dim; ++d) { + range_min[d] = intersect_range[d].first; + range_max[d] = intersect_range[d].second; + } + Kokkos::parallel_for( + "AbsorbCurrent", + CreateRangePolicy(range_min, range_max), + kernel::AbsorbCurrentGR_kernel(domain.fields.cur0, + domain.mesh.metric, + xg_edge, + ds, + tags)); + } + void OpenFieldsIn(dir::direction_t direction, domain_t& domain, BCTags tags, @@ -867,8 +911,8 @@ namespace ntt { const auto coeff = -dt * q0 * n0 / B0; // auto range = range_with_axis_BCs(domain); auto range = CreateRangePolicy( - { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) + 1}, - { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2)}); + { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2)}, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1}); const auto ni2 = domain.mesh.n_active(in::x2); if (g == gr_ampere::aux) { @@ -915,7 +959,7 @@ namespace ntt { void CurrentsDeposit(domain_t& domain) { auto scatter_cur = Kokkos::Experimental::create_scatter_view( - domain.fields.cur); + domain.fields.cur0); for (auto& species : domain.species) { logger::Checkpoint( fmt::format("Launching currents deposit kernel for %d [%s] : %lu %f", @@ -953,31 +997,27 @@ namespace ntt { (real_t)(species.charge()), dt)); } - Kokkos::Experimental::contribute(domain.fields.cur, scatter_cur); + Kokkos::Experimental::contribute(domain.fields.cur0, scatter_cur); } void CurrentsFilter(domain_t& domain) { logger::Checkpoint("Launching currents filtering kernels", HERE); - auto range = range_with_axis_BCs(domain); + auto range = CreateRangePolicy( + { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2)}, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1}); const auto nfilter = m_params.template get( "algorithms.current_filters"); tuple_t size; - if constexpr (M::Dim == Dim::_1D || M::Dim == Dim::_2D || M::Dim == Dim::_3D) { - size[0] = domain.mesh.n_active(in::x1); - } - if constexpr (M::Dim == Dim::_2D || M::Dim == Dim::_3D) { - size[1] = domain.mesh.n_active(in::x2); - } - if constexpr (M::Dim == Dim::_3D) { - size[2] = domain.mesh.n_active(in::x3); - } + size[0] = domain.mesh.n_active(in::x1); + size[1] = domain.mesh.n_active(in::x2); + // !TODO: this needs to be done more efficiently for (unsigned short i = 0; i < nfilter; ++i) { - Kokkos::deep_copy(domain.fields.buff, domain.fields.cur); + Kokkos::deep_copy(domain.fields.buff, domain.fields.cur0); Kokkos::parallel_for("CurrentsFilter", range, kernel::DigitalFilter_kernel( - domain.fields.cur, + domain.fields.cur0, domain.fields.buff, size, domain.mesh.flds_bc())); From 794beaab8a5afa9a635fc95d06413ac0cd50b2d3 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 4 Dec 2024 18:25:27 -0500 Subject: [PATCH 073/234] BC for currents --- src/engines/grpic.hpp | 3 +-- src/kernels/fields_bcs.hpp | 42 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 06088c1eb..6ce571bac 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -651,8 +651,7 @@ namespace ntt { kernel::AbsorbCurrentGR_kernel(domain.fields.cur0, domain.mesh.metric, xg_edge, - ds, - tags)); + ds)); } void OpenFieldsIn(dir::direction_t direction, diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index e5b3de647..34f4ac887 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -655,6 +655,48 @@ template } }; + template + struct AbsorbCurrentGR_kernel { + static_assert(M::is_metric, "M must be a metric class"); + static_assert(i <= static_cast(M::Dim), + "Invalid component index"); + + ndfield_t J; + const M metric; + const real_t xg_edge; + const real_t dx_abs; + const BCTags tags; + + AbsorbCurrentGR_kernel(ndfield_t J, + const M& metric, + real_t xg_edge, + real_t dx_abs, + BCTags tags) + : J { J } + , metric { metric } + , xg_edge { xg_edge } + , dx_abs { dx_abs } + , tags { tags } {} + + Inline void operator()(index_t i1, index_t i2) const { + if constexpr (M::Dim == Dim::_2D) { + const auto i1_ = COORD(i1); + const auto i2_ = COORD(i2); + coord_t x_Cd { ZERO }; + x_Cd[0] = i1_; + x_Cd[1] = i2_; + const auto dx = math::abs( + metric.template convert(x_Cd[i - 1]) - xg_edge); + J(i1, i2) *= math::tanh(dx / (INV_4 * dx_abs)); + + } else { + raise::KernelError( + HERE, + "AbsorbFields_kernel: 2D implementation called for D != 2"); + } + } + }; + } // namespace kernel #endif // KERNELS_FIELDS_BCS_HPP From 28f590ac3a53d2becd4f6335e9757b0cf5925a95 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 6 Dec 2024 11:22:07 -0500 Subject: [PATCH 074/234] engines/grpic: discard first step if fieldsolver = false --- src/engines/grpic.hpp | 292 ++++++++++++++++++++++-------------------- 1 file changed, 151 insertions(+), 141 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 6ce571bac..6b36ec35c 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -100,164 +100,174 @@ namespace ntt { "algorithms.toggles.deposit"); const auto sort_interval = m_params.template get( "particles.sort_interval"); - + if (step == 0) { - // communicate fields and apply BCs on the first timestep - /** - * Initially: em0::B -- - * em0::D -- - * em::B at -1/2 - * em::D at -1/2 - * - * cur0::J -- - * cur::J -- - * - * aux::E -- - * aux::H -- - * - * x_prtl at -1/2 - * u_prtl at -1/2 - */ + if (fieldsolver_enabled) { + // communicate fields and apply BCs on the first timestep + /** + * Initially: em0::B -- + * em0::D -- + * em::B at -1/2 + * em::D at -1/2 + * + * cur0::J -- + * cur::J -- + * + * aux::E -- + * aux::H -- + * + * x_prtl at -1/2 + * u_prtl at -1/2 + */ - /** - * em0::D, em::D, em0::B, em::B <- boundary conditions - */ - m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0 | Comm::D | Comm::D0); - FieldBoundaries(dom, BC::B | BC::E, gr_bc::main); + /** + * em0::D, em::D, em0::B, em::B <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0 | Comm::D | Comm::D0); + FieldBoundaries(dom, BC::B | BC::E, gr_bc::main); - /** - * em0::B <- em::B - * em0::D <- em::D - * - * Now: em0::B & em0::D at -1/2 - */ - CopyFields(dom); + /** + * em0::B <- em::B + * em0::D <- em::D + * + * Now: em0::B & em0::D at -1/2 + */ + CopyFields(dom); - /** - * aux::E <- alpha * em::D + beta x em0::B - * aux::H <- alpha * em::B0 - beta x em::D - * - * Now: aux::E & aux::H at -1/2 - */ - ComputeAuxE(dom, gr_getE::D_B0); - ComputeAuxH(dom, gr_getH::D_B0); + /** + * aux::E <- alpha * em::D + beta x em0::B + * aux::H <- alpha * em::B0 - beta x em::D + * + * Now: aux::E & aux::H at -1/2 + */ + ComputeAuxE(dom, gr_getE::D_B0); + ComputeAuxH(dom, gr_getH::D_B0); - /** - * aux::E, aux::H <- boundary conditions - */ - FieldBoundaries(dom, BC::B | BC::E, gr_bc::aux); + /** + * aux::E, aux::H <- boundary conditions + */ + FieldBoundaries(dom, BC::B | BC::E, gr_bc::aux); - /** - * em0::B <- (em0::B) <- -curl aux::E - * - * Now: em0::B at 0 - */ - Faraday(dom, gr_faraday::aux, HALF); + /** + * em0::B <- (em0::B) <- -curl aux::E + * + * Now: em0::B at 0 + */ + Faraday(dom, gr_faraday::aux, HALF); - /** - * em0::B, em::B <- boundary conditions - */ - m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); - FieldBoundaries(dom, BC::B, gr_bc::main); + /** + * em0::B, em::B <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); + FieldBoundaries(dom, BC::B, gr_bc::main); - /** - * em::D <- (em0::D) <- curl aux::H - * - * Now: em::D at 0 - */ - Ampere(dom, gr_ampere::init, HALF); + /** + * em::D <- (em0::D) <- curl aux::H + * + * Now: em::D at 0 + */ + Ampere(dom, gr_ampere::init, HALF); - /** - * em0::D, em::D <- boundary conditions - */ - m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); - FieldBoundaries(dom, BC::E, gr_bc::main); + /** + * em0::D, em::D <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + FieldBoundaries(dom, BC::E, gr_bc::main); - /** - * aux::E <- alpha * em::D + beta x em0::B - * aux::H <- alpha * em0::B - beta x em::D - * - * Now: aux::E & aux::H at 0 - */ - ComputeAuxE(dom, gr_getE::D_B0); - ComputeAuxH(dom, gr_getH::D_B0); + /** + * aux::E <- alpha * em::D + beta x em0::B + * aux::H <- alpha * em0::B - beta x em::D + * + * Now: aux::E & aux::H at 0 + */ + ComputeAuxE(dom, gr_getE::D_B0); + ComputeAuxH(dom, gr_getH::D_B0); - /** - * aux::E, aux::H <- boundary conditions - */ - FieldBoundaries(dom, BC::B | BC::E, gr_bc::aux); + /** + * aux::E, aux::H <- boundary conditions + */ + FieldBoundaries(dom, BC::B | BC::E, gr_bc::aux); - // !ADD: GR -- particles? + // !ADD: GR -- particles? - /** - * em0::B <- (em::B) <- -curl aux::E - * - * Now: em0::B at 1/2 - */ - Faraday(dom, gr_faraday::main, ONE); - /** - * em0::B, em::B <- boundary conditions - */ - m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); - FieldBoundaries(dom, BC::B, gr_bc::main); + /** + * em0::B <- (em::B) <- -curl aux::E + * + * Now: em0::B at 1/2 + */ + Faraday(dom, gr_faraday::main, ONE); + /** + * em0::B, em::B <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); + FieldBoundaries(dom, BC::B, gr_bc::main); - /** - * em0::D <- (em0::D) <- curl aux::H - * - * Now: em0::D at 1/2 - */ - Ampere(dom, gr_ampere::aux, ONE); - /** - * em0::D, em::D <- boundary conditions - */ - m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); - FieldBoundaries(dom, BC::E, gr_bc::main); + /** + * em0::D <- (em0::D) <- curl aux::H + * + * Now: em0::D at 1/2 + */ + Ampere(dom, gr_ampere::aux, ONE); + /** + * em0::D, em::D <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + FieldBoundaries(dom, BC::E, gr_bc::main); - /** - * aux::H <- alpha * em0::B - beta x em0::D - * - * Now: aux::H at 1/2 - */ - ComputeAuxH(dom, gr_getH::D0_B0); - /** - * aux::H <- boundary conditions - */ - FieldBoundaries(dom, BC::B, gr_bc::aux); + /** + * aux::H <- alpha * em0::B - beta x em0::D + * + * Now: aux::H at 1/2 + */ + ComputeAuxH(dom, gr_getH::D0_B0); + /** + * aux::H <- boundary conditions + */ + FieldBoundaries(dom, BC::B, gr_bc::aux); - /** - * em0::D <- (em::D) <- curl aux::H - * - * Now: em0::D at 1 - * em::D at 0 - */ - Ampere(dom, gr_ampere::main, ONE); - /** - * em0::D, em::D <- boundary conditions - */ - m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); - FieldBoundaries(dom, BC::E, gr_bc::main); + /** + * em0::D <- (em::D) <- curl aux::H + * + * Now: em0::D at 1 + * em::D at 0 + */ + Ampere(dom, gr_ampere::main, ONE); + /** + * em0::D, em::D <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + FieldBoundaries(dom, BC::E, gr_bc::main); - /** - * em::D <-> em0::D - * em::B <-> em0::B - * em::J <-> em0::J - */ - SwapFields(dom); - /** - * Finally: em0::B at -1/2 - * em0::D at 0 - * em::B at 1/2 - * em::D at 1 - * - * cur0::J -- - * cur::J -- - * - * aux::E -- - * aux::H -- - * - * x_prtl at 1 - * u_prtl at 1/2 - */ + /** + * em::D <-> em0::D + * em::B <-> em0::B + * em::J <-> em0::J + */ + SwapFields(dom); + /** + * Finally: em0::B at -1/2 + * em0::D at 0 + * em::B at 1/2 + * em::D at 1 + * + * cur0::J -- + * cur::J -- + * + * aux::E -- + * aux::H -- + * + * x_prtl at 1 + * u_prtl at 1/2 + */ + } + } else { + /** + * em0::B <- em::B + * em0::D <- em::D + * + * Now: em0::B & em0::D at -1/2 + */ + CopyFields(dom); } /** From 6ad04be0072bf42a28719bce32d16365d4a06b14 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 6 Dec 2024 13:58:38 -0500 Subject: [PATCH 075/234] GeodesicFullPush bug for utheta --- src/kernels/particle_pusher_gr.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kernels/particle_pusher_gr.hpp b/src/kernels/particle_pusher_gr.hpp index 306f19013..4b5432891 100644 --- a/src/kernels/particle_pusher_gr.hpp +++ b/src/kernels/particle_pusher_gr.hpp @@ -478,7 +478,7 @@ namespace kernel::gr { dt * (-metric.alpha(xp_mid) * u0 * DERIVATIVE_IN_TH(metric.alpha, xp_mid) + - vp_mid[1] * DERIVATIVE_IN_TH(metric.beta1, xp_mid) - + vp_mid[0] * DERIVATIVE_IN_TH(metric.beta1, xp_mid) - (HALF / u0) * (DERIVATIVE_IN_TH((metric.template h<1, 1>), xp_mid) * SQR(vp_mid[0]) + From 9f34be4dcc2f2cdb610df4401dbe192b3c562b2d Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 11 Dec 2024 16:36:19 -0500 Subject: [PATCH 076/234] minor gr pusher --- src/kernels/particle_pusher_gr.hpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/kernels/particle_pusher_gr.hpp b/src/kernels/particle_pusher_gr.hpp index 4b5432891..e220239bf 100644 --- a/src/kernels/particle_pusher_gr.hpp +++ b/src/kernels/particle_pusher_gr.hpp @@ -678,13 +678,9 @@ namespace kernel::gr { dx2_prev(p) = dx2(p); coord_t xp { ZERO }; - coord_t xp_prev { ZERO }; xp[0] = i_di_to_Xi(i1(p), dx1(p)); xp[1] = i_di_to_Xi(i2(p), dx2(p)); - - xp_prev[0] = i_di_to_Xi(i1_prev(p), dx1_prev(p)); - xp_prev[1] = i_di_to_Xi(i2_prev(p), dx2_prev(p)); vec_t Dp_cntrv { ZERO }, Bp_cntrv { ZERO }, Dp_hat { ZERO }, Bp_hat { ZERO }; From e5b0267f869d3028ded1513a9ebe0038ed363fa4 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 11 Dec 2024 18:20:10 -0500 Subject: [PATCH 077/234] pgen and inputs for pusher debugging updated --- setups/grpic/pusher/boris.toml | 52 ++++---- setups/grpic/pusher/massive_gravity_a0.toml | 4 +- .../grpic/pusher/massive_gravity_a0995.toml | 4 +- setups/grpic/pusher/pgen.hpp | 124 +++++++++++++----- 4 files changed, 122 insertions(+), 62 deletions(-) diff --git a/setups/grpic/pusher/boris.toml b/setups/grpic/pusher/boris.toml index 6483eb2c6..5d4d92a9d 100644 --- a/setups/grpic/pusher/boris.toml +++ b/setups/grpic/pusher/boris.toml @@ -1,14 +1,14 @@ [simulation] name = "boris_a0" engine = "grpic" - runtime = 100.0 + runtime = 500.0 [grid] resolution = [128,128] - extent = [[0.9, 20.0]] + extent = [[1.2, 20.0]] [grid.metric] - metric = "qkerr_schild" + metric = "kerr_schild" qsph_r0 = 0.0 qsph_h = 0.0 ks_a = 0.0 @@ -22,7 +22,7 @@ [scales] larmor0 = 5.0 - skindepth0 = 1e-3 + skindepth0 = 1e0 [algorithms] current_filters = 4 @@ -32,7 +32,7 @@ pusher_eps = 1e-6 [algorithms.timestep] - CFL = 0.1 + CFL = 1.0 [algorithms.toggles] deposit = false @@ -42,37 +42,33 @@ ppc0 = 1.0 [[particles.species]] - label = "e-" + label = "e+" mass = 1.0 - charge = -1.0 + charge = 1.0 maxnpart = 1e0 [[particles.species]] - label = "e+" + label = "e-" mass = 1.0 charge = -1.0 - maxnpart = 1e1 + maxnpart = 1e0 [setup] - x1s = [5.77] - y1s = [1.570796] - z1s = [0.000000] - ux1s = [0.4376657824933686] - uy1s = [0.000000] - uz1s = [4.0] -# x1s = [10.0] -# y1s = [1.570796] -# z1s = [0.000000] -# ux1s = [0.0] -# uy1s = [0.000000] -# uz1s = [10.00000] - - #x2s = [7.5] - #y2s = [0.15745454545454546] - #z2s = [0.000000] - #ux2s = [0.029637] - #uy2s = [0.000000] - #uz2s = [3.75] + # Fig. 6.4 from Kolos, Stuchlik and Tursunov (2015) + x1s = [5.77] + y1s = [1.5707963267948966] + z1s = [0.000000] + ux1s = [0.43766578249336874] + uy1s = [0.000000] + uz1s = [1.1707100000000000] + + # Fig. 5.3 from Kolos, Stuchlik and Tursunov (2015) + x2s = [6.79] + y2s = [1.5707963267948966] + z2s = [0.000000] + ux2s = [0.7398747390396659] + uy2s = [0.000000] + uz2s = [12.610410000000002] [output] format = "hdf5" diff --git a/setups/grpic/pusher/massive_gravity_a0.toml b/setups/grpic/pusher/massive_gravity_a0.toml index 08b607d4b..8179432da 100644 --- a/setups/grpic/pusher/massive_gravity_a0.toml +++ b/setups/grpic/pusher/massive_gravity_a0.toml @@ -56,7 +56,7 @@ [setup] # (1,2,0) from Levin&Perez-Giz (2008) x1s = [9.85180645] - y1s = [1.570796] + y1s = [1.5707963267948966] z1s = [0.5235987755982988] ux1s = [0.24159816] uy1s = [3.535534] @@ -64,7 +64,7 @@ # (4,1,1) from Levin&Perez-Giz (2008) x2s = [68.600387] - y2s = [1.570796] + y2s = [1.5707963267948966] z2s = [0.000000] ux2s = [0.029637] uy2s = [0.000000] diff --git a/setups/grpic/pusher/massive_gravity_a0995.toml b/setups/grpic/pusher/massive_gravity_a0995.toml index b04538e10..3b64d2066 100644 --- a/setups/grpic/pusher/massive_gravity_a0995.toml +++ b/setups/grpic/pusher/massive_gravity_a0995.toml @@ -56,7 +56,7 @@ [setup] # (2,3,1) from Levin&Perez-Giz (2008) x1s = [10.343515586064923] - y1s = [1.570796] + y1s = [1.5707963267948966] z1s = [0.000000] ux1s = [0.19483849893499136] uy1s = [0.000000] @@ -64,7 +64,7 @@ # (~1000, 3, ~671) from Levin&Perez-Giz (2008) x2s = [10.64975354] - y2s = [1.570796] + y2s = [1.5707963267948966] z2s = [0.000000] ux2s = [0.18914503] uy2s = [0.000000] diff --git a/setups/grpic/pusher/pgen.hpp b/setups/grpic/pusher/pgen.hpp index d6a29a185..8f60b1515 100644 --- a/setups/grpic/pusher/pgen.hpp +++ b/setups/grpic/pusher/pgen.hpp @@ -27,12 +27,12 @@ namespace user { InitFields(M metric_) : metric { metric_ } {} Inline auto A_3(const coord_t& x_Cd) const -> real_t { - // return HALF * (metric.template h_<3, 3>(x_Cd) ); - // + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) - // ); - coord_t x_Ph { ZERO }; - metric.template convert(x_Cd, x_Ph); - return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); + return HALF * (metric.template h_<3, 3>(x_Cd) + + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + ); + // coord_t x_Ph { ZERO }; + // metric.template convert(x_Cd, x_Ph); + // return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); } Inline auto A_1(const coord_t& x_Cd) const -> real_t { @@ -49,7 +49,7 @@ namespace user { + TWO * metric.spin() * g_00); } - Inline auto bx1(const coord_t& x_Ph) const -> real_t { + Inline auto bx1(const coord_t& x_Ph) const -> real_t { // at ( i , j + HALF ) coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); @@ -64,50 +64,114 @@ namespace user { return ONE; else return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; - // return ZERO; } - Inline auto bx2(const coord_t& x_Ph) const -> real_t { + Inline auto bx2(const coord_t& x_Ph) const -> real_t { // at ( i + HALF , j ) coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); - x0m[0] = xi[0] + HALF - HALF; + x0m[0] = xi[0] - HALF; x0m[1] = xi[1]; - x0p[0] = xi[0] + HALF + HALF; + x0p[0] = xi[0] + HALF; x0p[1] = xi[1]; - real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + real_t inv_sqrt_detH_iPj { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; if (cmp::AlmostZero(x_Ph[1])) return ZERO; else - return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; - // return ZERO; + return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_iPj; } - Inline auto bx3(const coord_t& x_Ph) const -> real_t { - // coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; - // metric.template convert(x_Ph, xi); + Inline auto bx3(const coord_t& x_Ph) const -> real_t { // at ( i + HALF , j + HALF ) + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); - // x0m[0] = xi[0] + HALF - HALF; - // x0m[1] = xi[1]; - // x0p[0] = xi[0] + HALF + HALF; - // x0p[1] = xi[1]; + x0m[0] = xi[0]; + x0m[1] = xi[1] - HALF; + x0p[0] = xi[0]; + x0p[1] = xi[1] + HALF; - // real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; - // return -(A_1(x0p) - A_1(x0m)) * inv_sqrt_detH_ijP; - return ZERO; + real_t inv_sqrt_detH_iPjP { ONE / metric.sqrt_det_h({ xi[0], xi[1]}) }; + return -(A_1(x0p) - A_1(x0m)) * inv_sqrt_detH_iPjP; } - Inline auto dx1(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto dx1(const coord_t& x_Ph) const -> real_t { // at ( i + HALF , j ) + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + real_t alpha_iPj { metric.alpha({ xi[0], xi[1] }) }; + real_t inv_sqrt_detH_ij { ONE / metric.sqrt_det_h({ xi[0] - HALF, xi[1] }) }; + real_t sqrt_detH_ij { metric.sqrt_det_h({ xi[0] - HALF, xi[1] }) }; + real_t beta_ij { metric.beta1({ xi[0] - HALF, xi[1]}) }; + real_t alpha_ij { metric.alpha({ xi[0] - HALF, xi[1]}) }; + + // D1 at ( i + HALF , j ) + x0m[0] = xi[0] - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF; + x0p[1] = xi[1]; + real_t E1d { (A_0(x0p) - A_0(x0m)) }; + real_t D1d { E1d / alpha_iPj }; + + // D3 at ( i , j ) + x0m[0] = xi[0] - HALF - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] - HALF + HALF; + x0p[1] = xi[1]; + real_t B2_aux { -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ij }; + real_t D3d { -sqrt_detH_ij * beta_ij * B2_aux / alpha_ij }; + + real_t D1u { metric.template h<1, 1>({ xi[0], xi[1] }) * D1d + metric.template h<1, 3>({ xi[0], xi[1] }) * D3d }; + + return D1u; } - Inline auto dx2(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto dx2(const coord_t& x_Ph) const -> real_t { // at ( i , j + HALF ) + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + x0m[0] = xi[0]; + x0m[1] = xi[1] - HALF; + x0p[0] = xi[0]; + x0p[1] = xi[1] + HALF; + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + real_t sqrt_detH_ijP { metric.sqrt_det_h({ xi[0], xi[1] }) }; + real_t alpha_ijP { metric.alpha({ xi[0], xi[1] }) }; + real_t beta_ijP { metric.beta1({ xi[0], xi[1] }) }; + + real_t E2d { (A_0(x0p) - A_0(x0m)) }; + real_t B3_aux { -(A_1(x0p) - A_1(x0m)) * inv_sqrt_detH_ijP }; + real_t D2d { E2d / alpha_ijP + sqrt_detH_ijP * beta_ijP * B3_aux / alpha_ijP }; + real_t D2u { metric.template h<2, 2>({ xi[0], xi[1] }) * D2d }; + + return D2u; } - Inline auto dx3(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto dx3(const coord_t& x_Ph) const -> real_t { // at ( i , j ) + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + real_t inv_sqrt_detH_ij { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + real_t sqrt_detH_ij { metric.sqrt_det_h({ xi[0], xi[1] }) }; + real_t beta_ij { metric.beta1({ xi[0], xi[1] }) }; + real_t alpha_ij { metric.alpha({ xi[0], xi[1] }) }; + real_t alpha_iPj { metric.alpha({ xi[0] + HALF, xi[1] }) }; + + // D3 at ( i , j ) + x0m[0] = xi[0] - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF; + x0p[1] = xi[1]; + real_t B2_aux { -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ij}; + real_t D3d { -sqrt_detH_ij * beta_ij * B2_aux / alpha_ij }; + + // D1 at ( i + HALF , j ) + x0m[0] = xi[0] + HALF - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF + HALF; + x0p[1] = xi[1]; + real_t E1d { (A_0(x0p) - A_0(x0m)) }; + real_t D1d { E1d / alpha_iPj }; + + return metric.template h<3, 3>({ xi[0], xi[1] }) * D3d + metric.template h<1, 3>({ xi[0], xi[1] }) * D1d; } private: From c9b70f528790f11a9a2827631502df78062b4f28 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Mon, 16 Dec 2024 15:38:24 -0500 Subject: [PATCH 078/234] boris tests with non-zero spin --- setups/grpic/pusher/boris_a09.toml | 85 ++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 setups/grpic/pusher/boris_a09.toml diff --git a/setups/grpic/pusher/boris_a09.toml b/setups/grpic/pusher/boris_a09.toml new file mode 100644 index 000000000..379e9c323 --- /dev/null +++ b/setups/grpic/pusher/boris_a09.toml @@ -0,0 +1,85 @@ +[simulation] + name = "boris_a09" + engine = "grpic" + runtime = 500.0 + +[grid] + resolution = [128,128] + extent = [[1.2, 20.0]] + + [grid.metric] + metric = "kerr_schild" + qsph_r0 = 0.0 + qsph_h = 0.0 + ks_a = 0.9 + + [grid.boundaries] + fields = [["ABSORB"]] + particles = [["ABSORB"]] + + [grid.boundaries.absorb] + ds = 0.5 + +[scales] + larmor0 = 0.5 + skindepth0 = 1e0 + +[algorithms] + current_filters = 4 + + [algorithms.gr] + pusher_niter = 10 + pusher_eps = 1e-6 + + [algorithms.timestep] + CFL = 1.0 + + [algorithms.toggles] + deposit = false + fieldsolver = false + +[particles] + ppc0 = 1.0 + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = 1.0 + maxnpart = 1e0 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = -1.0 + maxnpart = 1e0 + +[setup] + # Stuchlik and Kolos (2015) Fig.11 ~ RKA3 from Bacchini et al + x1s = [4.000000] + y1s = [1.3707963268] + z1s = [0.000000] + ux1s = [0.625666] + uy1s = [0.000000] + uz1s = [1.580567] + +[output] + format = "hdf5" + + [output.fields] + enable = true + interval_time = 1.0 + quantities = ["D", "B"] + + [output.particles] + enable = true + stride = 1 + interval_time = 0.2 + species = [] + + [output.spectra] + enable = false + +[diagnostics] + interval = 10 + colored_stdout = true + blocking_timers = true From 668db7b255f12b3ceb1420134986d8c846d9c669 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:22:31 -0500 Subject: [PATCH 079/234] updated setups --- setups/grpic/orbits/orbits.toml | 73 ++++++++++++++++ setups/grpic/orbits/pgen.hpp | 132 ++++++++++++++++++++++++++++ setups/grpic/pusher/deposit.toml | 91 +++++++++++++++++++ setups/grpic/vacuum/pgen.hpp | 146 +++++++++++++++++++++++++++++++ setups/grpic/vacuum/wald.toml | 93 ++++++++++++++++++++ setups/grpic/wald/pgen.hpp | 26 ++++-- setups/grpic/wald/wald.toml | 48 +++++----- 7 files changed, 583 insertions(+), 26 deletions(-) create mode 100644 setups/grpic/orbits/orbits.toml create mode 100644 setups/grpic/orbits/pgen.hpp create mode 100644 setups/grpic/pusher/deposit.toml create mode 100644 setups/grpic/vacuum/pgen.hpp create mode 100644 setups/grpic/vacuum/wald.toml diff --git a/setups/grpic/orbits/orbits.toml b/setups/grpic/orbits/orbits.toml new file mode 100644 index 000000000..ab8f2bef5 --- /dev/null +++ b/setups/grpic/orbits/orbits.toml @@ -0,0 +1,73 @@ +[simulation] + name = "wald" + engine = "grpic" + runtime = 110.0 + +[grid] + resolution = [128,128] + extent = [[1.2, 8.0]] + + [grid.metric] + metric = "kerr_schild" + qsph_r0 = 0.0 + qsph_h = 0.0 + ks_a = 0.95 + + [grid.boundaries] + fields = [["ABSORB"]] + particles = [["ABSORB"]] + + [grid.boundaries.absorb] + ds = 0.5 + +[scales] + larmor0 = 1.0 + skindepth0 = 1.0 + +[algorithms] + current_filters = 4 + + [algorithms.timestep] + CFL = 0.5 + + [algorithms.toggles] + deposit = false + fieldsolver = true + +[particles] + ppc0 = 8.0 + use_weights = true + sort_interval = 100 + +# [[particles.species]] +# label = "e-" +# mass = 1.0 +# charge = -1.0 +# maxnpart = 2e6 +# pusher = "Boris" +# +# [[particles.species]] +# label = "e+" +# mass = 1.0 +# charge = 1.0 +# maxnpart = 2e6 +# pusher = "Boris" + +[setup] + +[output] + format = "hdf5" + + [output.fields] + interval_time = 1.0 + quantities = ["D", "H", "B", "J"] + + [output.particles] + interval_time = 1.0 + quantities = ["X", "U"] + + [output.spectra] + enable = false + +[diagnostics] + interval = 1 diff --git a/setups/grpic/orbits/pgen.hpp b/setups/grpic/orbits/pgen.hpp new file mode 100644 index 000000000..54965b1ee --- /dev/null +++ b/setups/grpic/orbits/pgen.hpp @@ -0,0 +1,132 @@ +#ifndef PROBLEM_GENERATOR_H +#define PROBLEM_GENERATOR_H + +#include "enums.h" +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "arch/traits.h" + +#include "archetypes/problem_generator.h" +#include "framework/domain/metadomain.h" + +namespace user { + using namespace ntt; + + template + struct InitFields { + InitFields(M metric_) : metric { metric_ } {} + + Inline auto A_3(const coord_t& x_Cd) const -> real_t { + // return HALF * (metric.template h_<3, 3>(x_Cd) ); + // + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + // ); + + coord_t x_Ph { ZERO }; + metric.template convert(x_Cd, x_Ph); + return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); + } + + Inline auto A_1(const coord_t& x_Cd) const -> real_t { + return HALF * (metric.template h_<1, 3>(x_Cd) + + TWO * metric.spin() * metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) + ); + } + + Inline auto A_0(const coord_t& x_Cd) const -> real_t { + real_t g_00 { -metric.alpha(x_Cd) * metric.alpha(x_Cd) + + metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) * metric.beta1(x_Cd) + }; + return HALF * (metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + + TWO * metric.spin() * g_00); + } + + Inline auto bx1(const coord_t& x_Ph) const -> real_t { + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0]; + x0m[1] = xi[1] - HALF; + x0p[0] = xi[0]; + x0p[1] = xi[1] + HALF; + + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + + if (cmp::AlmostZero(x_Ph[1])) + return ONE; + else + return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; + } + + Inline auto bx2(const coord_t& x_Ph) const -> real_t { + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0] + HALF - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF + HALF; + x0p[1] = xi[1]; + + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + if (cmp::AlmostZero(x_Ph[1])) + return ZERO; + else + return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; + } + + Inline auto bx3(const coord_t& x_Ph) const -> real_t { + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0] + HALF - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF + HALF; + x0p[1] = xi[1]; + + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + return -(A_1(x0p) - A_1(x0m)) * inv_sqrt_detH_ijP; + // return ZERO; + } + + Inline auto dx1(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx2(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx3(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + private: + const M metric; + }; + + template + struct PGen : public arch::ProblemGenerator { + // compatibility traits for the problem generator + static constexpr auto engines { traits::compatible_with::value }; + static constexpr auto metrics { + traits::compatible_with::value + }; + static constexpr auto dimensions { traits::compatible_with::value }; + + // for easy access to variables in the child class + using arch::ProblemGenerator::D; + using arch::ProblemGenerator::C; + using arch::ProblemGenerator::params; + + InitFields init_flds; + + inline PGen(SimulationParams& p, const Metadomain& m) + : arch::ProblemGenerator(p) + , init_flds { m.mesh().metric } {} + + inline PGen() {} + }; + +} // namespace user + +#endif diff --git a/setups/grpic/pusher/deposit.toml b/setups/grpic/pusher/deposit.toml new file mode 100644 index 000000000..359d4eaa7 --- /dev/null +++ b/setups/grpic/pusher/deposit.toml @@ -0,0 +1,91 @@ +[simulation] + name = "deposit" + engine = "grpic" + runtime = 10.0 + +[grid] + resolution = [512,512] + extent = [[1.2, 20.0]] + + [grid.metric] + metric = "kerr_schild" + qsph_r0 = 0.0 + qsph_h = 0.0 + ks_a = 0.0 + + [grid.boundaries] + fields = [["ABSORB"]] + particles = [["ABSORB"]] + + [grid.boundaries.absorb] + ds = 2.0 + +[scales] + larmor0 = 2.0 + skindepth0 = 2.0 + +[algorithms] + current_filters = 4 + + [algorithms.gr] + pusher_niter = 10 + pusher_eps = 1e-6 + + [algorithms.timestep] + CFL = 0.5 + + [algorithms.toggles] + deposit = true + fieldsolver = true + +[particles] + ppc0 = 1.0 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = -1.0 + maxnpart = 1e0 + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = 1.0 + maxnpart = 1e0 + +[setup] + x1s = [17.00000] + y1s = [1.570796] + z1s = [0.000000] + ux1s = [8.000000] + uy1s = [0.000000] + uz1s = [0.000000] + + x2s = [17.00000] + y2s = [1.570796] + z2s = [0.000000] + ux2s = [-8.000000] + uy2s = [0.000000] + uz2s = [0.000000] + +[output] + format = "hdf5" + + [output.fields] + enable = true + interval_time = 0.5 + quantities = ["D", "B"] + + [output.particles] + enable = true + stride = 1 + interval_time = 0.5 + species = [] + + [output.spectra] + enable = false + +[diagnostics] + interval = 10 + colored_stdout = true + blocking_timers = true diff --git a/setups/grpic/vacuum/pgen.hpp b/setups/grpic/vacuum/pgen.hpp new file mode 100644 index 000000000..c02c198e8 --- /dev/null +++ b/setups/grpic/vacuum/pgen.hpp @@ -0,0 +1,146 @@ +#ifndef PROBLEM_GENERATOR_H +#define PROBLEM_GENERATOR_H + +#include "enums.h" +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "arch/traits.h" +#include "utils/comparators.h" +#include "utils/formatting.h" +#include "utils/log.h" +#include "utils/numeric.h" + +#include "archetypes/problem_generator.h" +#include "framework/domain/metadomain.h" +#include "framework/domain/domain.h" +#include "archetypes/energy_dist.h" +#include "archetypes/particle_injector.h" + +#include + +namespace user { + using namespace ntt; + + template + struct InitFields { + InitFields(M metric_) : metric { metric_ } {} + + Inline auto A_3(const coord_t& x_Cd) const -> real_t { + return HALF * (metric.template h_<3, 3>(x_Cd) ); + // + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + // ); + coord_t x_Ph { ZERO }; + metric.template convert(x_Cd, x_Ph); + return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); + } + + Inline auto A_1(const coord_t& x_Cd) const -> real_t { + return HALF * (metric.template h_<1, 3>(x_Cd) + + TWO * metric.spin() * metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) + ); + } + + Inline auto A_0(const coord_t& x_Cd) const -> real_t { + real_t g_00 { -metric.alpha(x_Cd) * metric.alpha(x_Cd) + + metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) * metric.beta1(x_Cd) + }; + return HALF * (metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + + TWO * metric.spin() * g_00); + } + + Inline auto bx1(const coord_t& x_Ph) const -> real_t { + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0]; + x0m[1] = xi[1] - HALF; + x0p[0] = xi[0]; + x0p[1] = xi[1] + HALF; + + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + + if (cmp::AlmostZero(x_Ph[1])) + return ONE; + else + return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; + } + + Inline auto bx2(const coord_t& x_Ph) const -> real_t { + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0] + HALF - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF + HALF; + x0p[1] = xi[1]; + + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + if (cmp::AlmostZero(x_Ph[1])) + return ZERO; + else + return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; + } + + Inline auto bx3(const coord_t& x_Ph) const -> real_t { + // coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + // metric.template convert(x_Ph, xi); + + // x0m[0] = xi[0] + HALF - HALF; + // x0m[1] = xi[1]; + // x0p[0] = xi[0] + HALF + HALF; + // x0p[1] = xi[1]; + + // real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + // return -(A_1(x0p) - A_1(x0m)) * inv_sqrt_detH_ijP; + return ZERO; + } + + Inline auto dx1(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx2(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx3(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + private: + const M metric; + }; + + template + struct PGen : public arch::ProblemGenerator { + // compatibility traits for the problem generator + static constexpr auto engines { traits::compatible_with::value }; + static constexpr auto metrics { + traits::compatible_with::value + }; + static constexpr auto dimensions { traits::compatible_with::value }; + + // for easy access to variables in the child class + using arch::ProblemGenerator::D; + using arch::ProblemGenerator::C; + using arch::ProblemGenerator::params; + + InitFields init_flds; + const Metadomain& global_domain; + inline PGen(const SimulationParams& p, const Metadomain& m) + : arch::ProblemGenerator { p } + , global_domain { m } + , init_flds { m.mesh().metric } {} + + // inline void InitPrtls(Domain& local_domain) { + + // arch::InjectGlobally(global_domain, local_domain, (arch::spidx_t)1, data_1); + // arch::InjectGlobally(global_domain, local_domain, (arch::spidx_t)2, data_2); + // } + // inline PGen() {} + }; + +} // namespace user + +#endif diff --git a/setups/grpic/vacuum/wald.toml b/setups/grpic/vacuum/wald.toml new file mode 100644 index 000000000..6864efb17 --- /dev/null +++ b/setups/grpic/vacuum/wald.toml @@ -0,0 +1,93 @@ +[simulation] + name = "wald" + engine = "grpic" + runtime = 5000.0 + +[grid] + resolution = [128,128] + extent = [[1.2, 100.0]] + + [grid.metric] + metric = "qkerr_schild" + qsph_r0 = 0.0 + qsph_h = 0.0 + ks_a = 0.0 + + [grid.boundaries] + fields = [["ABSORB"]] + particles = [["ABSORB"]] + + [grid.boundaries.absorb] + ds = 0.5 + +[scales] + larmor0 = 2e-3 + skindepth0 = 0.1 + +[algorithms] + current_filters = 4 + + [algorithms.gr] + pusher_niter = 10 + pusher_eps = 1e-2 + + [algorithms.timestep] + CFL = 10.0 + + [algorithms.toggles] + deposit = false + fieldsolver = false + +[particles] + ppc0 = 4.0 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = 0.0 + maxnpart = 1e1 + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = -1.0 + maxnpart = 1e1 + +[setup] + # (1,1,0) from Levin&Perez-Giz (2008) + x1s = [59.921203] + y1s = [1.570796] + z1s = [0.000000] + ux1s = [0.034020] + uy1s = [0.000000] + uz1s = [3.900000] + + # (4,1,1) from Levin&Perez-Giz (2008) + x2s = [68.600387] + y2s = [1.570796] + z2s = [0.000000] + ux2s = [0.029637] + uy2s = [0.000000] + uz2s = [3.900000] + +[output] + format = "hdf5" + + [output.fields] + enable = true + interval_time = 1.0 + quantities = ["D", "B"] + + [output.particles] + enable = true + stride = 1 + interval_time = 10.0 + species = [] + + [output.spectra] + enable = false + +[diagnostics] + interval = 10 + colored_stdout = true + blocking_timers = true diff --git a/setups/grpic/wald/pgen.hpp b/setups/grpic/wald/pgen.hpp index 51636314b..8dd1a6e4e 100644 --- a/setups/grpic/wald/pgen.hpp +++ b/setups/grpic/wald/pgen.hpp @@ -18,10 +18,9 @@ namespace user { InitFields(M metric_) : metric { metric_ } {} Inline auto A_3(const coord_t& x_Cd) const -> real_t { - // return HALF * (metric.template h_<3, 3>(x_Cd) ); - // + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + // return HALF * (metric.template h_<3, 3>(x_Cd) + // + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) // ); - coord_t x_Ph { ZERO }; metric.template convert(x_Cd, x_Ph); return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); @@ -122,9 +121,26 @@ namespace user { inline PGen(SimulationParams& p, const Metadomain& m) : arch::ProblemGenerator(p) + // , xi_min { p.template get>("setup.inj_rmin") } + // , xi_max { p.template get>("setup.inj_rmax") } , init_flds { m.mesh().metric } {} - - inline PGen() {} + + // inline void InitPrtls(Domain& local_domain) { + // const auto energy_dist = arch::ColdDist(local_domain.mesh.metric); + // const auto spatial_dist = PointDistribution(domain.mesh.metric, + // xi_min, + // xi_max); + // const auto injector = arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // { 1, 2 }); + // arch::InjectNonUniform>(params, + // local_domain, + // injector, + // 1.0); + // } + + // inline PGen() {} }; } // namespace user diff --git a/setups/grpic/wald/wald.toml b/setups/grpic/wald/wald.toml index 36c746ae5..44d1c8a24 100644 --- a/setups/grpic/wald/wald.toml +++ b/setups/grpic/wald/wald.toml @@ -1,17 +1,17 @@ [simulation] name = "wald" engine = "grpic" - runtime = 500.0 + runtime = 110.0 [grid] resolution = [128,128] - extent = [[1.2, 8.0]] + extent = [[2.0, 8.0]] [grid.metric] metric = "qkerr_schild" qsph_r0 = 0.0 qsph_h = 0.0 - ks_a = 0.95 + ks_a = 0.0 [grid.boundaries] fields = [["ABSORB"]] @@ -21,14 +21,14 @@ ds = 0.5 [scales] - larmor0 = 1.0 - skindepth0 = 1.0 + larmor0 = 1.0 #0.0025 + skindepth0 = 1.0 #0.05 [algorithms] current_filters = 4 [algorithms.timestep] - CFL = 0.5 + CFL = 0.3 [algorithms.toggles] deposit = false @@ -39,28 +39,32 @@ use_weights = true sort_interval = 100 -# [[particles.species]] -# label = "e-" -# mass = 1.0 -# charge = -1.0 -# maxnpart = 2e6 -# pusher = "Boris" -# -# [[particles.species]] -# label = "e+" -# mass = 1.0 -# charge = 1.0 -# maxnpart = 2e6 -# pusher = "Boris" + #[[particles.species]] + #label = "e-" + #mass = 1.0 + #charge = -1.0 + #maxnpart = 1e6 + #pusher = "Boris" + # + #[[particles.species]] + #label = "e+" + #mass = 1.0 + #charge = 1.0 + #maxnpart = 1e6 + #pusher = "Boris" [setup] + #multiplicity = 1.0 + #sigma_max = 1000.0 + #inj_rmin = 1.2 + #inj_rmax = 7.5 [output] format = "hdf5" [output.fields] interval_time = 1.0 - quantities = ["D", "H", "B", "J", "A"] + quantities = ["D", "H", "B", "J"] [output.particles] enable = false @@ -69,4 +73,6 @@ enable = false [diagnostics] - interval = 1 + interval = 2 + colored_stdout = true + blocking_timers = true From 98287db43bea318d0bb449486e24149e64d6620d Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 18 Dec 2024 10:59:29 -0500 Subject: [PATCH 080/234] engines/grpic: fixed the bug i introduced --- src/engines/grpic.hpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 6b36ec35c..01a61118e 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -259,15 +259,15 @@ namespace ntt { * x_prtl at 1 * u_prtl at 1/2 */ + } else { + /** + * em0::B <- em::B + * em0::D <- em::D + * + * Now: em0::B & em0::D at -1/2 + */ + CopyFields(dom); } - } else { - /** - * em0::B <- em::B - * em0::D <- em::D - * - * Now: em0::B & em0::D at -1/2 - */ - CopyFields(dom); } /** From fc48458292b1e5962cf1d5c45587ba2a3236de13 Mon Sep 17 00:00:00 2001 From: alisagk <55898700+alisagk@users.noreply.github.com> Date: Wed, 18 Dec 2024 11:10:37 -0500 Subject: [PATCH 081/234] Delete setups/grpic/orbits directory --- setups/grpic/orbits/orbits.toml | 73 ------------------ setups/grpic/orbits/pgen.hpp | 132 -------------------------------- 2 files changed, 205 deletions(-) delete mode 100644 setups/grpic/orbits/orbits.toml delete mode 100644 setups/grpic/orbits/pgen.hpp diff --git a/setups/grpic/orbits/orbits.toml b/setups/grpic/orbits/orbits.toml deleted file mode 100644 index ab8f2bef5..000000000 --- a/setups/grpic/orbits/orbits.toml +++ /dev/null @@ -1,73 +0,0 @@ -[simulation] - name = "wald" - engine = "grpic" - runtime = 110.0 - -[grid] - resolution = [128,128] - extent = [[1.2, 8.0]] - - [grid.metric] - metric = "kerr_schild" - qsph_r0 = 0.0 - qsph_h = 0.0 - ks_a = 0.95 - - [grid.boundaries] - fields = [["ABSORB"]] - particles = [["ABSORB"]] - - [grid.boundaries.absorb] - ds = 0.5 - -[scales] - larmor0 = 1.0 - skindepth0 = 1.0 - -[algorithms] - current_filters = 4 - - [algorithms.timestep] - CFL = 0.5 - - [algorithms.toggles] - deposit = false - fieldsolver = true - -[particles] - ppc0 = 8.0 - use_weights = true - sort_interval = 100 - -# [[particles.species]] -# label = "e-" -# mass = 1.0 -# charge = -1.0 -# maxnpart = 2e6 -# pusher = "Boris" -# -# [[particles.species]] -# label = "e+" -# mass = 1.0 -# charge = 1.0 -# maxnpart = 2e6 -# pusher = "Boris" - -[setup] - -[output] - format = "hdf5" - - [output.fields] - interval_time = 1.0 - quantities = ["D", "H", "B", "J"] - - [output.particles] - interval_time = 1.0 - quantities = ["X", "U"] - - [output.spectra] - enable = false - -[diagnostics] - interval = 1 diff --git a/setups/grpic/orbits/pgen.hpp b/setups/grpic/orbits/pgen.hpp deleted file mode 100644 index 54965b1ee..000000000 --- a/setups/grpic/orbits/pgen.hpp +++ /dev/null @@ -1,132 +0,0 @@ -#ifndef PROBLEM_GENERATOR_H -#define PROBLEM_GENERATOR_H - -#include "enums.h" -#include "global.h" - -#include "arch/kokkos_aliases.h" -#include "arch/traits.h" - -#include "archetypes/problem_generator.h" -#include "framework/domain/metadomain.h" - -namespace user { - using namespace ntt; - - template - struct InitFields { - InitFields(M metric_) : metric { metric_ } {} - - Inline auto A_3(const coord_t& x_Cd) const -> real_t { - // return HALF * (metric.template h_<3, 3>(x_Cd) ); - // + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) - // ); - - coord_t x_Ph { ZERO }; - metric.template convert(x_Cd, x_Ph); - return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); - } - - Inline auto A_1(const coord_t& x_Cd) const -> real_t { - return HALF * (metric.template h_<1, 3>(x_Cd) - + TWO * metric.spin() * metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) - ); - } - - Inline auto A_0(const coord_t& x_Cd) const -> real_t { - real_t g_00 { -metric.alpha(x_Cd) * metric.alpha(x_Cd) - + metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) * metric.beta1(x_Cd) - }; - return HALF * (metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) - + TWO * metric.spin() * g_00); - } - - Inline auto bx1(const coord_t& x_Ph) const -> real_t { - coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; - metric.template convert(x_Ph, xi); - - x0m[0] = xi[0]; - x0m[1] = xi[1] - HALF; - x0p[0] = xi[0]; - x0p[1] = xi[1] + HALF; - - real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; - - if (cmp::AlmostZero(x_Ph[1])) - return ONE; - else - return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; - } - - Inline auto bx2(const coord_t& x_Ph) const -> real_t { - coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; - metric.template convert(x_Ph, xi); - - x0m[0] = xi[0] + HALF - HALF; - x0m[1] = xi[1]; - x0p[0] = xi[0] + HALF + HALF; - x0p[1] = xi[1]; - - real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; - if (cmp::AlmostZero(x_Ph[1])) - return ZERO; - else - return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; - } - - Inline auto bx3(const coord_t& x_Ph) const -> real_t { - coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; - metric.template convert(x_Ph, xi); - - x0m[0] = xi[0] + HALF - HALF; - x0m[1] = xi[1]; - x0p[0] = xi[0] + HALF + HALF; - x0p[1] = xi[1]; - - real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; - return -(A_1(x0p) - A_1(x0m)) * inv_sqrt_detH_ijP; - // return ZERO; - } - - Inline auto dx1(const coord_t& x_Ph) const -> real_t { - return ZERO; - } - - Inline auto dx2(const coord_t& x_Ph) const -> real_t { - return ZERO; - } - - Inline auto dx3(const coord_t& x_Ph) const -> real_t { - return ZERO; - } - - private: - const M metric; - }; - - template - struct PGen : public arch::ProblemGenerator { - // compatibility traits for the problem generator - static constexpr auto engines { traits::compatible_with::value }; - static constexpr auto metrics { - traits::compatible_with::value - }; - static constexpr auto dimensions { traits::compatible_with::value }; - - // for easy access to variables in the child class - using arch::ProblemGenerator::D; - using arch::ProblemGenerator::C; - using arch::ProblemGenerator::params; - - InitFields init_flds; - - inline PGen(SimulationParams& p, const Metadomain& m) - : arch::ProblemGenerator(p) - , init_flds { m.mesh().metric } {} - - inline PGen() {} - }; - -} // namespace user - -#endif From 6c502a2b8f6135092b59578092a9c950ce4b7cfa Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 18 Dec 2024 11:08:29 -0500 Subject: [PATCH 082/234] setups/grpic: updated setup for vacuum --- setups/grpic/vacuum/pgen.hpp | 37 +--------------- setups/grpic/vacuum/vacuum.toml | 78 +++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 35 deletions(-) create mode 100644 setups/grpic/vacuum/vacuum.toml diff --git a/setups/grpic/vacuum/pgen.hpp b/setups/grpic/vacuum/pgen.hpp index c02c198e8..6d5e5fba5 100644 --- a/setups/grpic/vacuum/pgen.hpp +++ b/setups/grpic/vacuum/pgen.hpp @@ -27,28 +27,11 @@ namespace user { InitFields(M metric_) : metric { metric_ } {} Inline auto A_3(const coord_t& x_Cd) const -> real_t { - return HALF * (metric.template h_<3, 3>(x_Cd) ); - // + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) - // ); coord_t x_Ph { ZERO }; metric.template convert(x_Cd, x_Ph); return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); } - Inline auto A_1(const coord_t& x_Cd) const -> real_t { - return HALF * (metric.template h_<1, 3>(x_Cd) - + TWO * metric.spin() * metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) - ); - } - - Inline auto A_0(const coord_t& x_Cd) const -> real_t { - real_t g_00 { -metric.alpha(x_Cd) * metric.alpha(x_Cd) - + metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) * metric.beta1(x_Cd) - }; - return HALF * (metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) - + TWO * metric.spin() * g_00); - } - Inline auto bx1(const coord_t& x_Ph) const -> real_t { coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); @@ -70,9 +53,9 @@ namespace user { coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); - x0m[0] = xi[0] + HALF - HALF; + x0m[0] = xi[0] - HALF; x0m[1] = xi[1]; - x0p[0] = xi[0] + HALF + HALF; + x0p[0] = xi[0] + HALF; x0p[1] = xi[1]; real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; @@ -83,16 +66,6 @@ namespace user { } Inline auto bx3(const coord_t& x_Ph) const -> real_t { - // coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; - // metric.template convert(x_Ph, xi); - - // x0m[0] = xi[0] + HALF - HALF; - // x0m[1] = xi[1]; - // x0p[0] = xi[0] + HALF + HALF; - // x0p[1] = xi[1]; - - // real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; - // return -(A_1(x0p) - A_1(x0m)) * inv_sqrt_detH_ijP; return ZERO; } @@ -132,13 +105,7 @@ namespace user { : arch::ProblemGenerator { p } , global_domain { m } , init_flds { m.mesh().metric } {} - - // inline void InitPrtls(Domain& local_domain) { - // arch::InjectGlobally(global_domain, local_domain, (arch::spidx_t)1, data_1); - // arch::InjectGlobally(global_domain, local_domain, (arch::spidx_t)2, data_2); - // } - // inline PGen() {} }; } // namespace user diff --git a/setups/grpic/vacuum/vacuum.toml b/setups/grpic/vacuum/vacuum.toml new file mode 100644 index 000000000..3992b9f12 --- /dev/null +++ b/setups/grpic/vacuum/vacuum.toml @@ -0,0 +1,78 @@ +[simulation] + name = "wald" + engine = "grpic" + runtime = 500.0 + +[grid] + resolution = [128,128] + extent = [[2.0, 8.0]] + + [grid.metric] + metric = "qkerr_schild" + qsph_r0 = 0.0 + qsph_h = 0.0 + ks_a = 0.0 + + [grid.boundaries] + fields = [["ABSORB"]] + particles = [["ABSORB"]] + + [grid.boundaries.absorb] + ds = 0.5 + +[scales] + larmor0 = 0.0025 + skindepth0 = 0.05 + +[algorithms] + current_filters = 4 + + [algorithms.timestep] + CFL = 0.5 + + [algorithms.toggles] + deposit = false + fieldsolver = true + +[particles] + ppc0 = 8.0 + use_weights = true + sort_interval = 100 + + #[[particles.species]] + #label = "e-" + #mass = 1.0 + #charge = -1.0 + #maxnpart = 1e6 + #pusher = "Boris" + # + #[[particles.species]] + #label = "e+" + #mass = 1.0 + #charge = 1.0 + #maxnpart = 1e6 + #pusher = "Boris" + +[setup] + #multiplicity = 1.0 + #sigma_max = 1000.0 + inj_rmin = 1.2 + inj_rmax = 7.5 + +[output] + format = "hdf5" + + [output.fields] + interval_time = 1.0 + quantities = ["D", "H", "B", "J"] + + [output.particles] + enable = false + + [output.spectra] + enable = false + +[diagnostics] + interval = 2 + colored_stdout = true + blocking_timers = true From 1b81abe0653d29f34236208e6a3ac4fee8a6e892 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 18 Dec 2024 11:15:32 -0500 Subject: [PATCH 083/234] updated setups --- setups/grpic/vacuum/wald.toml | 93 ----------------------------------- setups/grpic/wald/pgen.hpp | 4 +- setups/grpic/wald/wald.toml | 12 ++--- 3 files changed, 8 insertions(+), 101 deletions(-) delete mode 100644 setups/grpic/vacuum/wald.toml diff --git a/setups/grpic/vacuum/wald.toml b/setups/grpic/vacuum/wald.toml deleted file mode 100644 index 6864efb17..000000000 --- a/setups/grpic/vacuum/wald.toml +++ /dev/null @@ -1,93 +0,0 @@ -[simulation] - name = "wald" - engine = "grpic" - runtime = 5000.0 - -[grid] - resolution = [128,128] - extent = [[1.2, 100.0]] - - [grid.metric] - metric = "qkerr_schild" - qsph_r0 = 0.0 - qsph_h = 0.0 - ks_a = 0.0 - - [grid.boundaries] - fields = [["ABSORB"]] - particles = [["ABSORB"]] - - [grid.boundaries.absorb] - ds = 0.5 - -[scales] - larmor0 = 2e-3 - skindepth0 = 0.1 - -[algorithms] - current_filters = 4 - - [algorithms.gr] - pusher_niter = 10 - pusher_eps = 1e-2 - - [algorithms.timestep] - CFL = 10.0 - - [algorithms.toggles] - deposit = false - fieldsolver = false - -[particles] - ppc0 = 4.0 - - [[particles.species]] - label = "e-" - mass = 1.0 - charge = 0.0 - maxnpart = 1e1 - - [[particles.species]] - label = "e+" - mass = 1.0 - charge = -1.0 - maxnpart = 1e1 - -[setup] - # (1,1,0) from Levin&Perez-Giz (2008) - x1s = [59.921203] - y1s = [1.570796] - z1s = [0.000000] - ux1s = [0.034020] - uy1s = [0.000000] - uz1s = [3.900000] - - # (4,1,1) from Levin&Perez-Giz (2008) - x2s = [68.600387] - y2s = [1.570796] - z2s = [0.000000] - ux2s = [0.029637] - uy2s = [0.000000] - uz2s = [3.900000] - -[output] - format = "hdf5" - - [output.fields] - enable = true - interval_time = 1.0 - quantities = ["D", "B"] - - [output.particles] - enable = true - stride = 1 - interval_time = 10.0 - species = [] - - [output.spectra] - enable = false - -[diagnostics] - interval = 10 - colored_stdout = true - blocking_timers = true diff --git a/setups/grpic/wald/pgen.hpp b/setups/grpic/wald/pgen.hpp index 8dd1a6e4e..b87d25ed2 100644 --- a/setups/grpic/wald/pgen.hpp +++ b/setups/grpic/wald/pgen.hpp @@ -61,9 +61,9 @@ namespace user { coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); - x0m[0] = xi[0] + HALF - HALF; + x0m[0] = xi[0] - HALF; x0m[1] = xi[1]; - x0p[0] = xi[0] + HALF + HALF; + x0p[0] = xi[0] + HALF; x0p[1] = xi[1]; real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; diff --git a/setups/grpic/wald/wald.toml b/setups/grpic/wald/wald.toml index 44d1c8a24..6e68d4324 100644 --- a/setups/grpic/wald/wald.toml +++ b/setups/grpic/wald/wald.toml @@ -1,7 +1,7 @@ [simulation] name = "wald" engine = "grpic" - runtime = 110.0 + runtime = 510.0 [grid] resolution = [128,128] @@ -21,14 +21,14 @@ ds = 0.5 [scales] - larmor0 = 1.0 #0.0025 - skindepth0 = 1.0 #0.05 + larmor0 = 0.0025 + skindepth0 = 0.05 [algorithms] current_filters = 4 [algorithms.timestep] - CFL = 0.3 + CFL = 0.5 [algorithms.toggles] deposit = false @@ -56,8 +56,8 @@ [setup] #multiplicity = 1.0 #sigma_max = 1000.0 - #inj_rmin = 1.2 - #inj_rmax = 7.5 + inj_rmin = 1.2 + inj_rmax = 7.5 [output] format = "hdf5" From 2e1cb0c69f570ee3c811be833f749465a19a06d1 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 18 Dec 2024 13:55:24 -0500 Subject: [PATCH 084/234] added separate setup for deposit debugging --- setups/grpic/deposit/deposit.toml | 91 ++++++++++++++++++++++++ setups/grpic/deposit/pgen.hpp | 113 ++++++++++++++++++++++++++++++ 2 files changed, 204 insertions(+) create mode 100644 setups/grpic/deposit/deposit.toml create mode 100644 setups/grpic/deposit/pgen.hpp diff --git a/setups/grpic/deposit/deposit.toml b/setups/grpic/deposit/deposit.toml new file mode 100644 index 000000000..359d4eaa7 --- /dev/null +++ b/setups/grpic/deposit/deposit.toml @@ -0,0 +1,91 @@ +[simulation] + name = "deposit" + engine = "grpic" + runtime = 10.0 + +[grid] + resolution = [512,512] + extent = [[1.2, 20.0]] + + [grid.metric] + metric = "kerr_schild" + qsph_r0 = 0.0 + qsph_h = 0.0 + ks_a = 0.0 + + [grid.boundaries] + fields = [["ABSORB"]] + particles = [["ABSORB"]] + + [grid.boundaries.absorb] + ds = 2.0 + +[scales] + larmor0 = 2.0 + skindepth0 = 2.0 + +[algorithms] + current_filters = 4 + + [algorithms.gr] + pusher_niter = 10 + pusher_eps = 1e-6 + + [algorithms.timestep] + CFL = 0.5 + + [algorithms.toggles] + deposit = true + fieldsolver = true + +[particles] + ppc0 = 1.0 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = -1.0 + maxnpart = 1e0 + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = 1.0 + maxnpart = 1e0 + +[setup] + x1s = [17.00000] + y1s = [1.570796] + z1s = [0.000000] + ux1s = [8.000000] + uy1s = [0.000000] + uz1s = [0.000000] + + x2s = [17.00000] + y2s = [1.570796] + z2s = [0.000000] + ux2s = [-8.000000] + uy2s = [0.000000] + uz2s = [0.000000] + +[output] + format = "hdf5" + + [output.fields] + enable = true + interval_time = 0.5 + quantities = ["D", "B"] + + [output.particles] + enable = true + stride = 1 + interval_time = 0.5 + species = [] + + [output.spectra] + enable = false + +[diagnostics] + interval = 10 + colored_stdout = true + blocking_timers = true diff --git a/setups/grpic/deposit/pgen.hpp b/setups/grpic/deposit/pgen.hpp new file mode 100644 index 000000000..6d5e5fba5 --- /dev/null +++ b/setups/grpic/deposit/pgen.hpp @@ -0,0 +1,113 @@ +#ifndef PROBLEM_GENERATOR_H +#define PROBLEM_GENERATOR_H + +#include "enums.h" +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "arch/traits.h" +#include "utils/comparators.h" +#include "utils/formatting.h" +#include "utils/log.h" +#include "utils/numeric.h" + +#include "archetypes/problem_generator.h" +#include "framework/domain/metadomain.h" +#include "framework/domain/domain.h" +#include "archetypes/energy_dist.h" +#include "archetypes/particle_injector.h" + +#include + +namespace user { + using namespace ntt; + + template + struct InitFields { + InitFields(M metric_) : metric { metric_ } {} + + Inline auto A_3(const coord_t& x_Cd) const -> real_t { + coord_t x_Ph { ZERO }; + metric.template convert(x_Cd, x_Ph); + return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); + } + + Inline auto bx1(const coord_t& x_Ph) const -> real_t { + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0]; + x0m[1] = xi[1] - HALF; + x0p[0] = xi[0]; + x0p[1] = xi[1] + HALF; + + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + + if (cmp::AlmostZero(x_Ph[1])) + return ONE; + else + return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; + } + + Inline auto bx2(const coord_t& x_Ph) const -> real_t { + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0] - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF; + x0p[1] = xi[1]; + + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + if (cmp::AlmostZero(x_Ph[1])) + return ZERO; + else + return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; + } + + Inline auto bx3(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx1(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx2(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx3(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + private: + const M metric; + }; + + template + struct PGen : public arch::ProblemGenerator { + // compatibility traits for the problem generator + static constexpr auto engines { traits::compatible_with::value }; + static constexpr auto metrics { + traits::compatible_with::value + }; + static constexpr auto dimensions { traits::compatible_with::value }; + + // for easy access to variables in the child class + using arch::ProblemGenerator::D; + using arch::ProblemGenerator::C; + using arch::ProblemGenerator::params; + + InitFields init_flds; + const Metadomain& global_domain; + inline PGen(const SimulationParams& p, const Metadomain& m) + : arch::ProblemGenerator { p } + , global_domain { m } + , init_flds { m.mesh().metric } {} + + }; + +} // namespace user + +#endif From 58a3e5daee471620a92fa53498d86e7c2e0ded49 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 18 Dec 2024 13:57:33 -0500 Subject: [PATCH 085/234] deleted deposit test from pusher tests --- setups/grpic/pusher/deposit.toml | 91 -------------------------------- 1 file changed, 91 deletions(-) delete mode 100644 setups/grpic/pusher/deposit.toml diff --git a/setups/grpic/pusher/deposit.toml b/setups/grpic/pusher/deposit.toml deleted file mode 100644 index 359d4eaa7..000000000 --- a/setups/grpic/pusher/deposit.toml +++ /dev/null @@ -1,91 +0,0 @@ -[simulation] - name = "deposit" - engine = "grpic" - runtime = 10.0 - -[grid] - resolution = [512,512] - extent = [[1.2, 20.0]] - - [grid.metric] - metric = "kerr_schild" - qsph_r0 = 0.0 - qsph_h = 0.0 - ks_a = 0.0 - - [grid.boundaries] - fields = [["ABSORB"]] - particles = [["ABSORB"]] - - [grid.boundaries.absorb] - ds = 2.0 - -[scales] - larmor0 = 2.0 - skindepth0 = 2.0 - -[algorithms] - current_filters = 4 - - [algorithms.gr] - pusher_niter = 10 - pusher_eps = 1e-6 - - [algorithms.timestep] - CFL = 0.5 - - [algorithms.toggles] - deposit = true - fieldsolver = true - -[particles] - ppc0 = 1.0 - - [[particles.species]] - label = "e-" - mass = 1.0 - charge = -1.0 - maxnpart = 1e0 - - [[particles.species]] - label = "e+" - mass = 1.0 - charge = 1.0 - maxnpart = 1e0 - -[setup] - x1s = [17.00000] - y1s = [1.570796] - z1s = [0.000000] - ux1s = [8.000000] - uy1s = [0.000000] - uz1s = [0.000000] - - x2s = [17.00000] - y2s = [1.570796] - z2s = [0.000000] - ux2s = [-8.000000] - uy2s = [0.000000] - uz2s = [0.000000] - -[output] - format = "hdf5" - - [output.fields] - enable = true - interval_time = 0.5 - quantities = ["D", "B"] - - [output.particles] - enable = true - stride = 1 - interval_time = 0.5 - species = [] - - [output.spectra] - enable = false - -[diagnostics] - interval = 10 - colored_stdout = true - blocking_timers = true From 44668406af783d1e05033a669feb39d654be67b1 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 18 Dec 2024 15:58:12 -0500 Subject: [PATCH 086/234] communication should be for j0 for GRPIC --- src/framework/domain/communications.cpp | 73 ++++++++++++++++++------- 1 file changed, 54 insertions(+), 19 deletions(-) diff --git a/src/framework/domain/communications.cpp b/src/framework/domain/communications.cpp index 60524eedd..65eb603c9 100644 --- a/src/framework/domain/communications.cpp +++ b/src/framework/domain/communications.cpp @@ -300,11 +300,10 @@ namespace ntt { comp_range_fld, false); } - } - if (comm_j) { + if (comm_j) { comm::CommunicateField(domain.index(), - domain.fields.cur, - domain.fields.cur, + domain.fields.cur0, + domain.fields.cur0, send_ind, recv_ind, send_rank, @@ -313,6 +312,21 @@ namespace ntt { recv_slice, comp_range_cur, false); + } + } else { + if (comm_j) { + comm::CommunicateField(domain.index(), + domain.fields.cur, + domain.fields.cur, + send_ind, + recv_ind, + send_rank, + recv_rank, + send_slice, + recv_slice, + comp_range_cur, + false); + } } } } @@ -432,17 +446,31 @@ namespace ntt { continue; } if (comm_j) { - comm::CommunicateField(domain.index(), - domain.fields.cur, - domain.fields.buff, - send_ind, - recv_ind, - send_rank, - recv_rank, - send_slice, - recv_slice, - comp_range_cur, - synchronize); + if constexpr (S == SimEngine::GRPIC) { + comm::CommunicateField(domain.index(), + domain.fields.cur0, + domain.fields.buff, + send_ind, + recv_ind, + send_rank, + recv_rank, + send_slice, + recv_slice, + comp_range_cur, + synchronize); + } else { + comm::CommunicateField(domain.index(), + domain.fields.cur, + domain.fields.buff, + send_ind, + recv_ind, + send_rank, + recv_rank, + send_slice, + recv_slice, + comp_range_cur, + synchronize); + } } if (comm_bckp) { comm::CommunicateField(domain.index(), @@ -472,10 +500,17 @@ namespace ntt { } } if (comm_j) { - AddBufferedFields(domain.fields.cur, - domain.fields.buff, - domain.mesh.rangeActiveCells(), - comp_range_cur); + if constexpr (S == SimEngine::GRPIC) { + AddBufferedFields(domain.fields.cur0, + domain.fields.buff, + domain.mesh.rangeActiveCells(), + comp_range_cur); + } else { + AddBufferedFields(domain.fields.cur, + domain.fields.buff, + domain.mesh.rangeActiveCells(), + comp_range_cur); + } } if (comm_bckp) { AddBufferedFields(domain.fields.bckp, From ef835907e620a3682671af118ac6a4ddbf8e9ced Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 18 Dec 2024 15:59:21 -0500 Subject: [PATCH 087/234] 1. implemented working boundary conditions for currents, 2. fixed normaliation for AmpereCurrents -- currents should not be normilized as they are used repeatedl3. minor style --- src/engines/grpic.hpp | 22 +++++++++++----------- src/kernels/fields_bcs.hpp | 11 +++++------ 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 01a61118e..7ce584680 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -373,9 +373,9 @@ namespace ntt { m_metadomain.CommunicateFields(dom, Comm::J); timers.stop("Communications"); - // timers.start("FieldBoundaries"); - // CurrentsBoundaryConditions(); - // timers.stop("FieldBoundaries"); + timers.start("FieldBoundaries"); + CurrentsBoundaryConditions(dom); + timers.stop("FieldBoundaries"); timers.start("CurrentFiltering"); CurrentsFilter(dom); @@ -659,9 +659,9 @@ namespace ntt { "AbsorbCurrent", CreateRangePolicy(range_min, range_max), kernel::AbsorbCurrentGR_kernel(domain.fields.cur0, - domain.mesh.metric, - xg_edge, - ds)); + domain.mesh.metric, + xg_edge, + ds)); } void OpenFieldsIn(dir::direction_t direction, @@ -917,7 +917,7 @@ namespace ntt { const auto q0 = m_params.template get("scales.q0"); const auto n0 = m_params.template get("scales.n0"); const auto B0 = m_params.template get("scales.B0"); - const auto coeff = -dt * q0 * n0 / B0; + const auto coeff = -dt * q0 / B0; // auto range = range_with_axis_BCs(domain); auto range = CreateRangePolicy( { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2)}, @@ -967,7 +967,7 @@ namespace ntt { } void CurrentsDeposit(domain_t& domain) { - auto scatter_cur = Kokkos::Experimental::create_scatter_view( + auto scatter_cur0 = Kokkos::Experimental::create_scatter_view( domain.fields.cur0); for (auto& species : domain.species) { logger::Checkpoint( @@ -983,7 +983,7 @@ namespace ntt { Kokkos::parallel_for("CurrentsDeposit", species.rangeActiveParticles(), kernel::DepositCurrents_kernel( - scatter_cur, + scatter_cur0, species.i1, species.i2, species.i3, @@ -1006,7 +1006,7 @@ namespace ntt { (real_t)(species.charge()), dt)); } - Kokkos::Experimental::contribute(domain.fields.cur0, scatter_cur); + Kokkos::Experimental::contribute(domain.fields.cur0, scatter_cur0); } void CurrentsFilter(domain_t& domain) { @@ -1030,7 +1030,7 @@ namespace ntt { domain.fields.buff, size, domain.mesh.flds_bc())); - m_metadomain.CommunicateFields(domain, Comm::J); + m_metadomain.CommunicateFields(domain, Comm::J); //J0 } } diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 34f4ac887..445e0be74 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -665,18 +665,15 @@ template const M metric; const real_t xg_edge; const real_t dx_abs; - const BCTags tags; AbsorbCurrentGR_kernel(ndfield_t J, const M& metric, real_t xg_edge, - real_t dx_abs, - BCTags tags) + real_t dx_abs) : J { J } , metric { metric } , xg_edge { xg_edge } - , dx_abs { dx_abs } - , tags { tags } {} + , dx_abs { dx_abs } {} Inline void operator()(index_t i1, index_t i2) const { if constexpr (M::Dim == Dim::_2D) { @@ -687,7 +684,9 @@ template x_Cd[1] = i2_; const auto dx = math::abs( metric.template convert(x_Cd[i - 1]) - xg_edge); - J(i1, i2) *= math::tanh(dx / (INV_4 * dx_abs)); + J(i1, i2, 0) *= math::tanh(dx / (INV_4 * dx_abs)); + J(i1, i2, 1) *= math::tanh(dx / (INV_4 * dx_abs)); + J(i1, i2, 2) *= math::tanh(dx / (INV_4 * dx_abs)); } else { raise::KernelError( From 03c0af14219de6d97e94172b757ad49cd64caf2d Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 18 Dec 2024 16:03:25 -0500 Subject: [PATCH 088/234] improved setup for deposit testing --- setups/grpic/deposit/deposit.py | 53 +++++++++++++++++++++ setups/grpic/deposit/deposit.toml | 12 ++--- setups/grpic/deposit/pgen.hpp | 79 ++++++++++++++++++------------- 3 files changed, 104 insertions(+), 40 deletions(-) create mode 100644 setups/grpic/deposit/deposit.py diff --git a/setups/grpic/deposit/deposit.py b/setups/grpic/deposit/deposit.py new file mode 100644 index 000000000..e3aa97dfc --- /dev/null +++ b/setups/grpic/deposit/deposit.py @@ -0,0 +1,53 @@ +import nt2.read as nt2r +import matplotlib as mpl +import matplotlib.pyplot as plt +import numpy as np + +data = nt2r.Data("deposit.h5") +def frame(ti): + t0 = data.t.isel(t=ti).values[()] + vmax = 1e-5 + fig, ax = plt.subplots(figsize=(6, 8), dpi=150) + data.Dr.isel(t=ti).polar.pcolor( + ax=ax, + norm=mpl.colors.SymLogNorm( + vmin=-vmax, vmax=vmax, linthresh=vmax / 1e3, linscale=1 + ), + cmap="RdBu_r", + ) + x1, x2 = data.particles[1].isel(t=ti).r * np.sin( + data.particles[1].isel(t=ti).th + ), data.particles[1].isel(t=ti).r * np.cos(data.particles[1].isel(t=ti).th) + ax.scatter(x1, x2, s=5, c="b", ec="w", lw=0.5) + x1, x2 = data.particles[2].isel(t=ti).r * np.sin( + data.particles[2].isel(t=ti).th + ), data.particles[2].isel(t=ti).r * np.cos(data.particles[2].isel(t=ti).th) + ax.scatter(x1, x2, s=5, c="r", ec="w", lw=0.5) + ax.add_artist( + mpl.patches.Arc( + (0, 0), + 2 * 10, + 2 * 10, + fill=False, + ec="k", + ls=":", + lw=0.5, + theta1=-90, + theta2=90, + ) + ) + axins = ax.inset_axes([0.4, 0.8, 0.6 - 0.025, 0.2 - 0.01]) + ( + data.Dr.sel(t=slice(data.t.min(), t0)).sel(r=10, method="nearest") + * np.sin(data.th) + ).sum(dim="th").plot(ax=axins) + axins.set( + ylim=(-3 * vmax, 3 * vmax), + xlim=(data.t.min(), data.t.max()), + title="", + ylabel=rf"$\int D_r d\Omega$ @ (r = 10)", + ) + axins.axvline(t0, color="b", lw=0.5, ls=":") + axins.axhline(0, color="r", lw=0.5, ls="--") + +frame(30) \ No newline at end of file diff --git a/setups/grpic/deposit/deposit.toml b/setups/grpic/deposit/deposit.toml index 359d4eaa7..c577f9c68 100644 --- a/setups/grpic/deposit/deposit.toml +++ b/setups/grpic/deposit/deposit.toml @@ -1,7 +1,7 @@ [simulation] name = "deposit" engine = "grpic" - runtime = 10.0 + runtime = 50.0 [grid] resolution = [512,512] @@ -21,8 +21,8 @@ ds = 2.0 [scales] - larmor0 = 2.0 - skindepth0 = 2.0 + larmor0 = 2e-3 + skindepth0 = 0.1 [algorithms] current_filters = 4 @@ -57,14 +57,14 @@ x1s = [17.00000] y1s = [1.570796] z1s = [0.000000] - ux1s = [8.000000] + ux1s = [5.000000] uy1s = [0.000000] uz1s = [0.000000] x2s = [17.00000] y2s = [1.570796] z2s = [0.000000] - ux2s = [-8.000000] + ux2s = [-5.000000] uy2s = [0.000000] uz2s = [0.000000] @@ -74,7 +74,7 @@ [output.fields] enable = true interval_time = 0.5 - quantities = ["D", "B"] + quantities = ["D"] [output.particles] enable = true diff --git a/setups/grpic/deposit/pgen.hpp b/setups/grpic/deposit/pgen.hpp index 6d5e5fba5..8ff89fa92 100644 --- a/setups/grpic/deposit/pgen.hpp +++ b/setups/grpic/deposit/pgen.hpp @@ -26,43 +26,12 @@ namespace user { struct InitFields { InitFields(M metric_) : metric { metric_ } {} - Inline auto A_3(const coord_t& x_Cd) const -> real_t { - coord_t x_Ph { ZERO }; - metric.template convert(x_Cd, x_Ph); - return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); - } - Inline auto bx1(const coord_t& x_Ph) const -> real_t { - coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; - metric.template convert(x_Ph, xi); - - x0m[0] = xi[0]; - x0m[1] = xi[1] - HALF; - x0p[0] = xi[0]; - x0p[1] = xi[1] + HALF; - - real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; - - if (cmp::AlmostZero(x_Ph[1])) - return ONE; - else - return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; + return ZERO; } Inline auto bx2(const coord_t& x_Ph) const -> real_t { - coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; - metric.template convert(x_Ph, xi); - - x0m[0] = xi[0] - HALF; - x0m[1] = xi[1]; - x0p[0] = xi[0] + HALF; - x0p[1] = xi[1]; - - real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; - if (cmp::AlmostZero(x_Ph[1])) - return ZERO; - else - return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; + return ZERO; } Inline auto bx3(const coord_t& x_Ph) const -> real_t { @@ -105,7 +74,49 @@ namespace user { : arch::ProblemGenerator { p } , global_domain { m } , init_flds { m.mesh().metric } {} - + + inline void InitPrtls(Domain& local_domain) { + const auto empty = std::vector {}; + const auto x1s = params.template get>("setup.x1s", empty); + const auto y1s = params.template get>("setup.y1s", empty); + const auto z1s = params.template get>("setup.z1s", empty); + const auto ux1s = params.template get>("setup.ux1s", + empty); + const auto uy1s = params.template get>("setup.uy1s", + empty); + const auto uz1s = params.template get>("setup.uz1s", + empty); + + const auto x2s = params.template get>("setup.x2s", empty); + const auto y2s = params.template get>("setup.y2s", empty); + const auto z2s = params.template get>("setup.z2s", empty); + const auto ux2s = params.template get>("setup.ux2s", + empty); + const auto uy2s = params.template get>("setup.uy2s", + empty); + const auto uz2s = params.template get>("setup.uz2s", + empty); + const std::map> data_1 { + { "x1", x1s}, + { "x2", y1s}, + { "phi", z1s}, + {"ux1", ux1s}, + {"ux2", uy1s}, + {"ux3", uz1s} + }; + const std::map> data_2 { + { "x1", x2s}, + { "x2", y2s}, + { "phi", z2s}, + {"ux1", ux2s}, + {"ux2", uy2s}, + {"ux3", uz2s} + }; + + arch::InjectGlobally(global_domain, local_domain, (arch::spidx_t)1, data_1); + arch::InjectGlobally(global_domain, local_domain, (arch::spidx_t)2, data_2); + } + // inline PGen() {} }; } // namespace user From 8faf54347cbed49c81b7064f730a243763020b9c Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Thu, 2 Jan 2025 15:33:52 -0500 Subject: [PATCH 089/234] minor fix for qks --- src/metrics/qkerr_schild.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/metrics/qkerr_schild.h b/src/metrics/qkerr_schild.h index e45376a24..e3d849952 100644 --- a/src/metrics/qkerr_schild.h +++ b/src/metrics/qkerr_schild.h @@ -255,7 +255,7 @@ namespace metric { const real_t r { r0 + math::exp(x[0] * dchi + chi_min) }; const real_t eta {x[1] * deta + eta_min }; const real_t theta { eta2theta(eta) }; - const real_t dx_dt {deta * (ONE + TWO * h0 * constant::INV_PI_SQR * (TWO * THREE * SQR(eta) - TWO * THREE * constant::PI * eta + constant::PI_SQR)) }; + const real_t dx_dt {deta * (ONE + TWO * h0 * static_cast(constant::INV_PI_SQR) * (TWO * THREE * SQR(eta) - TWO * THREE * static_cast(constant::PI) * eta + static_cast(constant::PI_SQR))) }; const real_t dt_Sigma {- TWO * SQR(a) * math::sin(theta) * math::cos(theta) * dx_dt}; return r * dt_Sigma * CUBE(alpha(x)) / SQR(Sigma(r, theta)); From b4a874e2bd15fdd647f2bb564628cd019567b74d Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Thu, 2 Jan 2025 15:52:50 -0500 Subject: [PATCH 090/234] minor for engines/grpic --- src/engines/grpic.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 7ce584680..d6a45c953 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -743,6 +743,7 @@ namespace ntt { (void)direction; (void)domain; (void)tags; + (void)g; raise::Error("Custom boundaries not implemented", HERE); // if constexpr ( // traits::has_member::value) { @@ -915,7 +916,7 @@ namespace ntt { void AmpereCurrents(domain_t& domain, const gr_ampere& g) { logger::Checkpoint("Launching Ampere kernel for adding currents", HERE); const auto q0 = m_params.template get("scales.q0"); - const auto n0 = m_params.template get("scales.n0"); + // const auto n0 = m_params.template get("scales.n0"); const auto B0 = m_params.template get("scales.B0"); const auto coeff = -dt * q0 / B0; // auto range = range_with_axis_BCs(domain); From d817b47603333d6498ba21b224b4b8434c9af18f Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 3 Jan 2025 15:01:25 -0500 Subject: [PATCH 091/234] implementing threshold-dependent injection is in progress --- setups/grpic/wald/pgen.hpp | 211 ++++++++++++++++++++++++++++-------- setups/grpic/wald/wald.toml | 48 ++++---- 2 files changed, 188 insertions(+), 71 deletions(-) diff --git a/setups/grpic/wald/pgen.hpp b/setups/grpic/wald/pgen.hpp index b87d25ed2..457c24107 100644 --- a/setups/grpic/wald/pgen.hpp +++ b/setups/grpic/wald/pgen.hpp @@ -7,8 +7,13 @@ #include "arch/kokkos_aliases.h" #include "arch/traits.h" +#include "archetypes/energy_dist.h" +#include "archetypes/spatial_dist.h" +#include "archetypes/particle_injector.h" #include "archetypes/problem_generator.h" #include "framework/domain/metadomain.h" +#include "kernels/particle_moments.hpp" +#include "utils/numeric.h" namespace user { using namespace ntt; @@ -18,26 +23,12 @@ namespace user { InitFields(M metric_) : metric { metric_ } {} Inline auto A_3(const coord_t& x_Cd) const -> real_t { - // return HALF * (metric.template h_<3, 3>(x_Cd) - // + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) - // ); - coord_t x_Ph { ZERO }; - metric.template convert(x_Cd, x_Ph); - return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); - } - - Inline auto A_1(const coord_t& x_Cd) const -> real_t { - return HALF * (metric.template h_<1, 3>(x_Cd) - + TWO * metric.spin() * metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) + return HALF * (metric.template h_<3, 3>(x_Cd) + + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) ); - } - - Inline auto A_0(const coord_t& x_Cd) const -> real_t { - real_t g_00 { -metric.alpha(x_Cd) * metric.alpha(x_Cd) - + metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) * metric.beta1(x_Cd) - }; - return HALF * (metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) - + TWO * metric.spin() * g_00); + // coord_t x_Ph { ZERO }; + // metric.template convert(x_Cd, x_Ph); + // return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); } Inline auto bx1(const coord_t& x_Ph) const -> real_t { @@ -74,16 +65,6 @@ namespace user { } Inline auto bx3(const coord_t& x_Ph) const -> real_t { - // coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; - // metric.template convert(x_Ph, xi); - - // x0m[0] = xi[0] + HALF - HALF; - // x0m[1] = xi[1]; - // x0p[0] = xi[0] + HALF + HALF; - // x0p[1] = xi[1]; - - // real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; - // return -(A_1(x0p) - A_1(x0m)) * inv_sqrt_detH_ijP; return ZERO; } @@ -103,6 +84,105 @@ namespace user { const M metric; }; + template + struct PointDistribution : public arch::SpatialDistribution { + PointDistribution(const std::vector& xi_min, + const std::vector& xi_max, + const real_t sigma_thr, + const real_t dens_thr, + long double time, + const SimulationParams& params, + Domain* domain_ptr + ) + : arch::SpatialDistribution { domain_ptr->mesh.metric } + , metric { domain_ptr->mesh.metric } + , EM { domain_ptr->fields.em } + , sigma_thr {sigma_thr} + , dens_thr {dens_thr} + , time {time} { + std::copy(xi_min.begin(), xi_min.end(), x_min); + std::copy(xi_max.begin(), xi_max.end(), x_max); + + std::vector specs {}; + for (auto& sp : domain_ptr->species) { + if (sp.mass() > 0) { + specs.push_back(sp.index()); + } + } + + auto scatter_buff = Kokkos::Experimental::create_scatter_view(domain_ptr->fields.buff); + printf("%f\n", domain_ptr->fields.buff(10, 10, 0)); + // some parameters + auto& mesh = domain_ptr->mesh; + const auto use_weights = params.template get("particles.use_weights"); + const auto ni2 = mesh.n_active(in::x2); + const auto inv_n0 = ONE / params.template get("scales.n0"); + + for (const auto& sp : specs) { + auto& prtl_spec = domain_ptr->species[sp - 1]; + // clang-format off + Kokkos::parallel_for( + "ComputeMoments", + prtl_spec.rangeActiveParticles(), + kernel::ParticleMoments_kernel({}, scatter_buff, 0u, + prtl_spec.i1, prtl_spec.i2, prtl_spec.i3, + prtl_spec.dx1, prtl_spec.dx2, prtl_spec.dx3, + prtl_spec.ux1, prtl_spec.ux2, prtl_spec.ux3, + prtl_spec.phi, prtl_spec.weight, prtl_spec.tag, + prtl_spec.mass(), prtl_spec.charge(), + use_weights, + metric, mesh.flds_bc(), + ni2, inv_n0, 0)); + // clang-format on + } + Kokkos::Experimental::contribute(domain_ptr->fields.buff, scatter_buff); + // auto density = domain_ptr->fields.buff; + auto density = Kokkos::create_mirror_view(domain_ptr->fields.buff); + Kokkos::deep_copy(density, domain_ptr->fields.buff); + } + + Inline auto sigma_crit(const coord_t& x_Ph) const -> real_t { + coord_t xi {ZERO}; + if constexpr (M::Dim == Dim::_2D) { + metric.template convert(x_Ph, xi); + const auto i1 = static_cast(xi[0]) + static_cast(N_GHOSTS); + const auto i2 = static_cast(xi[1]) + static_cast(N_GHOSTS); + // printf("%f %f\n", xi[0], xi[1]); + const vec_t B_cntrv { EM(i1, i2, em::bx1), EM(i1, i2, em::bx2), EM(i1, i2, em::bx3) }; + vec_t B_cov { ZERO }; + metric.template transform(xi, B_cntrv, B_cov); + const auto bsqr = DOT(B_cntrv[0], B_cntrv[1], B_cntrv[2], B_cov[0], B_cov[1], B_cov[2]); + const auto dens = density(i1, i2, 0); + // printf("%f\n", domain_ptr->fields.buff(10, 10, 0)); + // return (bsqr / dens > sigma_thr) || (dens < dens_thr); + return 2.0; + } + return ZERO; + } + + Inline auto operator()(const coord_t& x_Ph) const -> real_t override { + auto fill = true; + for (auto d = 0u; d < M::Dim; ++d) { + fill &= x_Ph[d] > x_min[d] and x_Ph[d] < x_max[d]; + } + if (time > ZERO) { + fill &= sigma_crit(x_Ph) > ONE; + } + return fill ? ONE : ZERO; + } + + private: + tuple_t x_min; + tuple_t x_max; + const real_t sigma_thr; + const real_t dens_thr; + long double time, + Domain* domain_ptr; + ndfield_t density; + ndfield_t EM; + const M metric; + }; + template struct PGen : public arch::ProblemGenerator { // compatibility traits for the problem generator @@ -117,30 +197,67 @@ namespace user { using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; + const std::vector xi_min; + const std::vector xi_max; + const real_t sigma0, sigma_max, multiplicity, nGJ; + InitFields init_flds; inline PGen(SimulationParams& p, const Metadomain& m) : arch::ProblemGenerator(p) - // , xi_min { p.template get>("setup.inj_rmin") } - // , xi_max { p.template get>("setup.inj_rmax") } + , xi_min { p.template get>("setup.xi_min") } + , xi_max { p.template get>("setup.xi_max") } + , sigma_max { p.template get("setup.sigma_max") } + , sigma0 { p.template get("scales.sigma0") } + , multiplicity { p.template get("setup.multiplicity") } + , nGJ { p.template get("scales.B0") * SQR(p.template get("scales.skindepth0")) } //m.mesh().metric.spin() * , init_flds { m.mesh().metric } {} - // inline void InitPrtls(Domain& local_domain) { - // const auto energy_dist = arch::ColdDist(local_domain.mesh.metric); - // const auto spatial_dist = PointDistribution(domain.mesh.metric, - // xi_min, - // xi_max); - // const auto injector = arch::NonUniformInjector( - // energy_dist, - // spatial_dist, - // { 1, 2 }); - // arch::InjectNonUniform>(params, - // local_domain, - // injector, - // 1.0); - // } - - // inline PGen() {} + inline void InitPrtls(Domain& local_domain) { + const auto energy_dist = arch::ColdDist(local_domain.mesh.metric); + const auto spatial_dist = PointDistribution(xi_min, + xi_max, + sigma_max / sigma0, + multiplicity * nGJ, + ZERO, + params, + &local_domain + ); + + const auto injector = arch::NonUniformInjector( + energy_dist, + spatial_dist, + { 1, 2 }); + arch::InjectNonUniform(params, + local_domain, + injector, + 1.0, + true); + } + + void CustomPostStep(std::size_t, long double time, Domain& local_domain) { + // const auto energy_dist = arch::ColdDist(local_domain.mesh.metric); + // const auto spatial_dist = PointDistribution(local_domain.mesh.metric, + // xi_min, + // xi_max); + + // const auto spatial_dist = ReplenishDist(local_domain.mesh.metric, + // const ndfield_t& density, + // unsigned short idx, + // const T& target_density, + // real_t target_max_density) + + // const auto injector = arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // { 1, 2 }); + // arch::InjectNonUniform(params, + // local_domain, + // injector, + // 1.0, + // true); + } + }; } // namespace user diff --git a/setups/grpic/wald/wald.toml b/setups/grpic/wald/wald.toml index 6e68d4324..2f12705de 100644 --- a/setups/grpic/wald/wald.toml +++ b/setups/grpic/wald/wald.toml @@ -1,17 +1,17 @@ [simulation] name = "wald" engine = "grpic" - runtime = 510.0 + runtime = 100.0 [grid] - resolution = [128,128] - extent = [[2.0, 8.0]] + resolution = [256,256] + extent = [[0.8, 8.0]] [grid.metric] metric = "qkerr_schild" qsph_r0 = 0.0 qsph_h = 0.0 - ks_a = 0.0 + ks_a = 0.95 [grid.boundaries] fields = [["ABSORB"]] @@ -31,7 +31,7 @@ CFL = 0.5 [algorithms.toggles] - deposit = false + deposit = true fieldsolver = true [particles] @@ -39,33 +39,33 @@ use_weights = true sort_interval = 100 - #[[particles.species]] - #label = "e-" - #mass = 1.0 - #charge = -1.0 - #maxnpart = 1e6 - #pusher = "Boris" - # - #[[particles.species]] - #label = "e+" - #mass = 1.0 - #charge = 1.0 - #maxnpart = 1e6 - #pusher = "Boris" + [[particles.species]] + label = "e-" + mass = 1.0 + charge = -1.0 + maxnpart = 1e6 + pusher = "Boris" + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = 1.0 + maxnpart = 1e6 + pusher = "Boris" [setup] - #multiplicity = 1.0 - #sigma_max = 1000.0 - inj_rmin = 1.2 - inj_rmax = 7.5 + multiplicity = 1.0 + sigma_max = 10.0 + xi_min = [1.2, 0.1] + xi_max = [7.5, 3.04159265] [output] format = "hdf5" [output.fields] interval_time = 1.0 - quantities = ["D", "H", "B", "J"] - + quantities = ["D", "B", "J", "N_1", "N_2", "Nppc_1", "Nppc_2"] + [output.particles] enable = false From 1328097476d0727f2a59861056825ca455d08ab4 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 3 Jan 2025 16:57:41 -0500 Subject: [PATCH 092/234] updated pgen --- setups/grpic/wald/pgen.hpp | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/setups/grpic/wald/pgen.hpp b/setups/grpic/wald/pgen.hpp index 457c24107..78f922700 100644 --- a/setups/grpic/wald/pgen.hpp +++ b/setups/grpic/wald/pgen.hpp @@ -90,7 +90,6 @@ namespace user { const std::vector& xi_max, const real_t sigma_thr, const real_t dens_thr, - long double time, const SimulationParams& params, Domain* domain_ptr ) @@ -98,8 +97,7 @@ namespace user { , metric { domain_ptr->mesh.metric } , EM { domain_ptr->fields.em } , sigma_thr {sigma_thr} - , dens_thr {dens_thr} - , time {time} { + , dens_thr {dens_thr} { std::copy(xi_min.begin(), xi_min.end(), x_min); std::copy(xi_max.begin(), xi_max.end(), x_max); @@ -136,7 +134,6 @@ namespace user { // clang-format on } Kokkos::Experimental::contribute(domain_ptr->fields.buff, scatter_buff); - // auto density = domain_ptr->fields.buff; auto density = Kokkos::create_mirror_view(domain_ptr->fields.buff); Kokkos::deep_copy(density, domain_ptr->fields.buff); } @@ -147,7 +144,6 @@ namespace user { metric.template convert(x_Ph, xi); const auto i1 = static_cast(xi[0]) + static_cast(N_GHOSTS); const auto i2 = static_cast(xi[1]) + static_cast(N_GHOSTS); - // printf("%f %f\n", xi[0], xi[1]); const vec_t B_cntrv { EM(i1, i2, em::bx1), EM(i1, i2, em::bx2), EM(i1, i2, em::bx3) }; vec_t B_cov { ZERO }; metric.template transform(xi, B_cntrv, B_cov); @@ -163,10 +159,7 @@ namespace user { Inline auto operator()(const coord_t& x_Ph) const -> real_t override { auto fill = true; for (auto d = 0u; d < M::Dim; ++d) { - fill &= x_Ph[d] > x_min[d] and x_Ph[d] < x_max[d]; - } - if (time > ZERO) { - fill &= sigma_crit(x_Ph) > ONE; + fill &= x_Ph[d] > x_min[d] and x_Ph[d] < x_max[d] and sigma_crit(x_Ph) > ONE; } return fill ? ONE : ZERO; } @@ -176,7 +169,6 @@ namespace user { tuple_t x_max; const real_t sigma_thr; const real_t dens_thr; - long double time, Domain* domain_ptr; ndfield_t density; ndfield_t EM; @@ -219,7 +211,6 @@ namespace user { xi_max, sigma_max / sigma0, multiplicity * nGJ, - ZERO, params, &local_domain ); From a400ced79c5d6838ed2189db5452f21b59df6567 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 3 Jan 2025 17:19:57 -0500 Subject: [PATCH 093/234] minor --- setups/grpic/wald/pgen.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/setups/grpic/wald/pgen.hpp b/setups/grpic/wald/pgen.hpp index 78f922700..41d9a2462 100644 --- a/setups/grpic/wald/pgen.hpp +++ b/setups/grpic/wald/pgen.hpp @@ -108,6 +108,7 @@ namespace user { } } + Kokkos::deep_copy(domain_ptr->fields.buff, ZERO); auto scatter_buff = Kokkos::Experimental::create_scatter_view(domain_ptr->fields.buff); printf("%f\n", domain_ptr->fields.buff(10, 10, 0)); // some parameters From 5b653e05e92e6d2b0175269bcf8b51bdf5092920 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 8 Jan 2025 19:14:18 -0500 Subject: [PATCH 094/234] implemented output of Aph for 2d GRPIC --- src/framework/domain/output.cpp | 44 +++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/framework/domain/output.cpp b/src/framework/domain/output.cpp index c669eb8ae..a5d4bc26d 100644 --- a/src/framework/domain/output.cpp +++ b/src/framework/domain/output.cpp @@ -173,6 +173,39 @@ namespace ntt { } } + template + void ComputeVectorPotential(ndfield_t& buffer, + ndfield_t& EM, + unsigned short buff_idx, + const Mesh mesh) { + if constexpr (M::Dim == Dim::_2D) { + const auto i2_min = mesh.i_min(in::x2); + // !TODO: this is quite slow + Kokkos::parallel_for( + "ComputeVectorPotential", + mesh.rangeActiveCells(), + Lambda(index_t i, index_t j) { + const real_t i_ { static_cast(static_cast(i) - (N_GHOSTS)) }; + const auto k_min = (i2_min - (N_GHOSTS)) + 1; + const auto k_max = (j - (N_GHOSTS)); + real_t A3 = ZERO; + for (auto k { k_min }; k <= k_max; ++k) { + real_t k_ = static_cast(k); + real_t sqrt_detH_ij1 { mesh.metric.sqrt_det_h({ i_, k_ - HALF }) }; + real_t sqrt_detH_ij2 { mesh.metric.sqrt_det_h({ i_, k_ + HALF }) }; + auto k1 { k + N_GHOSTS }; + A3 += HALF * (sqrt_detH_ij1 * EM(i, k1 - 1, em::bx1) + + sqrt_detH_ij2 * EM(i, k1, em::bx1)); + } + buffer(i, j, buff_idx) = A3; + }); + } else { + raise::KernelError( + HERE, + "ComputeVectorPotential: 2D implementation called for D != 2"); + } + } + template auto Metadomain::Write( const SimulationParams& params, @@ -313,6 +346,17 @@ namespace ntt { raise::Error("Custom output requested but no function provided", HERE); } + } else if (fld.is_vpotential()) { + if (S == SimEngine::GRPIC && M::Dim == Dim::_2D) { + const auto c = static_cast(addresses.back()); + ComputeVectorPotential(local_domain->fields.bckp, + local_domain->fields.em, + c, + local_domain->mesh); + } else { + raise::Error("Vector potential can only be computed for GRPIC in 2D", + HERE); + } } else { raise::Error("Wrong # of components requested for " "non-moment/non-custom output", From c8c81c7ce324c2e8251130a42463d002d94acab1 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 8 Jan 2025 19:17:43 -0500 Subject: [PATCH 095/234] updated pgen for wald solution with plasma --- setups/grpic/wald/pgen.hpp | 76 ++++++++++++++++++------------------- setups/grpic/wald/wald.toml | 17 +++++---- 2 files changed, 45 insertions(+), 48 deletions(-) diff --git a/setups/grpic/wald/pgen.hpp b/setups/grpic/wald/pgen.hpp index 41d9a2462..7e8eea216 100644 --- a/setups/grpic/wald/pgen.hpp +++ b/setups/grpic/wald/pgen.hpp @@ -26,9 +26,6 @@ namespace user { return HALF * (metric.template h_<3, 3>(x_Cd) + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) ); - // coord_t x_Ph { ZERO }; - // metric.template convert(x_Cd, x_Ph); - // return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); } Inline auto bx1(const coord_t& x_Ph) const -> real_t { @@ -96,6 +93,7 @@ namespace user { : arch::SpatialDistribution { domain_ptr->mesh.metric } , metric { domain_ptr->mesh.metric } , EM { domain_ptr->fields.em } + , density { domain_ptr->fields.buff } , sigma_thr {sigma_thr} , dens_thr {dens_thr} { std::copy(xi_min.begin(), xi_min.end(), x_min); @@ -108,9 +106,8 @@ namespace user { } } - Kokkos::deep_copy(domain_ptr->fields.buff, ZERO); - auto scatter_buff = Kokkos::Experimental::create_scatter_view(domain_ptr->fields.buff); - printf("%f\n", domain_ptr->fields.buff(10, 10, 0)); + Kokkos::deep_copy(density, ZERO); + auto scatter_buff = Kokkos::Experimental::create_scatter_view(density); // some parameters auto& mesh = domain_ptr->mesh; const auto use_weights = params.template get("particles.use_weights"); @@ -131,15 +128,13 @@ namespace user { prtl_spec.mass(), prtl_spec.charge(), use_weights, metric, mesh.flds_bc(), - ni2, inv_n0, 0)); + ni2, inv_n0, ZERO)); // clang-format on } - Kokkos::Experimental::contribute(domain_ptr->fields.buff, scatter_buff); - auto density = Kokkos::create_mirror_view(domain_ptr->fields.buff); - Kokkos::deep_copy(density, domain_ptr->fields.buff); + Kokkos::Experimental::contribute(density, scatter_buff); } - Inline auto sigma_crit(const coord_t& x_Ph) const -> real_t { + Inline auto sigma_crit(const coord_t& x_Ph) const -> bool { coord_t xi {ZERO}; if constexpr (M::Dim == Dim::_2D) { metric.template convert(x_Ph, xi); @@ -150,17 +145,15 @@ namespace user { metric.template transform(xi, B_cntrv, B_cov); const auto bsqr = DOT(B_cntrv[0], B_cntrv[1], B_cntrv[2], B_cov[0], B_cov[1], B_cov[2]); const auto dens = density(i1, i2, 0); - // printf("%f\n", domain_ptr->fields.buff(10, 10, 0)); - // return (bsqr / dens > sigma_thr) || (dens < dens_thr); - return 2.0; + return (bsqr > sigma_thr * dens) || (dens < dens_thr); } - return ZERO; + return false; } Inline auto operator()(const coord_t& x_Ph) const -> real_t override { auto fill = true; for (auto d = 0u; d < M::Dim; ++d) { - fill &= x_Ph[d] > x_min[d] and x_Ph[d] < x_max[d] and sigma_crit(x_Ph) > ONE; + fill &= x_Ph[d] > x_min[d] and x_Ph[d] < x_max[d] and sigma_crit(x_Ph); } return fill ? ONE : ZERO; } @@ -192,7 +185,7 @@ namespace user { const std::vector xi_min; const std::vector xi_max; - const real_t sigma0, sigma_max, multiplicity, nGJ; + const real_t sigma0, sigma_max, multiplicity, nGJ, temperature; InitFields init_flds; @@ -203,11 +196,14 @@ namespace user { , sigma_max { p.template get("setup.sigma_max") } , sigma0 { p.template get("scales.sigma0") } , multiplicity { p.template get("setup.multiplicity") } - , nGJ { p.template get("scales.B0") * SQR(p.template get("scales.skindepth0")) } //m.mesh().metric.spin() * + , nGJ { p.template get("scales.B0") * SQR(p.template get("scales.skindepth0")) } + , temperature { p.template get("setup.temperature") } , init_flds { m.mesh().metric } {} inline void InitPrtls(Domain& local_domain) { - const auto energy_dist = arch::ColdDist(local_domain.mesh.metric); + const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, + local_domain.random_pool, + temperature); const auto spatial_dist = PointDistribution(xi_min, xi_max, sigma_max / sigma0, @@ -216,7 +212,7 @@ namespace user { &local_domain ); - const auto injector = arch::NonUniformInjector( + const auto injector = arch::NonUniformInjector( energy_dist, spatial_dist, { 1, 2 }); @@ -228,26 +224,26 @@ namespace user { } void CustomPostStep(std::size_t, long double time, Domain& local_domain) { - // const auto energy_dist = arch::ColdDist(local_domain.mesh.metric); - // const auto spatial_dist = PointDistribution(local_domain.mesh.metric, - // xi_min, - // xi_max); - - // const auto spatial_dist = ReplenishDist(local_domain.mesh.metric, - // const ndfield_t& density, - // unsigned short idx, - // const T& target_density, - // real_t target_max_density) - - // const auto injector = arch::NonUniformInjector( - // energy_dist, - // spatial_dist, - // { 1, 2 }); - // arch::InjectNonUniform(params, - // local_domain, - // injector, - // 1.0, - // true); + const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, + local_domain.random_pool, + temperature); + const auto spatial_dist = PointDistribution(xi_min, + xi_max, + sigma_max / sigma0, + multiplicity * nGJ, + params, + &local_domain + ); + + const auto injector = arch::NonUniformInjector( + energy_dist, + spatial_dist, + { 1, 2 }); + arch::InjectNonUniform(params, + local_domain, + injector, + 1.0, + true); } }; diff --git a/setups/grpic/wald/wald.toml b/setups/grpic/wald/wald.toml index 2f12705de..3bf243afc 100644 --- a/setups/grpic/wald/wald.toml +++ b/setups/grpic/wald/wald.toml @@ -4,8 +4,8 @@ runtime = 100.0 [grid] - resolution = [256,256] - extent = [[0.8, 8.0]] + resolution = [512, 512] + extent = [[1.22, 6.0]] [grid.metric] metric = "qkerr_schild" @@ -43,28 +43,29 @@ label = "e-" mass = 1.0 charge = -1.0 - maxnpart = 1e6 + maxnpart = 2e8 pusher = "Boris" [[particles.species]] label = "e+" mass = 1.0 charge = 1.0 - maxnpart = 1e6 + maxnpart = 2e8 pusher = "Boris" [setup] multiplicity = 1.0 - sigma_max = 10.0 - xi_min = [1.2, 0.1] - xi_max = [7.5, 3.04159265] + sigma_max = 1000.0 + temperature = 0.01 + xi_min = [1.32, 0.1] + xi_max = [4.0, 3.04159265] [output] format = "hdf5" [output.fields] interval_time = 1.0 - quantities = ["D", "B", "J", "N_1", "N_2", "Nppc_1", "Nppc_2"] + quantities = ["D", "B", "J", "N_1", "N_2", "A"] [output.particles] enable = false From 2d9a0fa801f2afce0d88079df9775284c5238280 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Mon, 13 Jan 2025 14:26:55 -0500 Subject: [PATCH 096/234] gr minor --- src/kernels/fields_bcs.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 445e0be74..1bd45b8df 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -591,11 +591,11 @@ template const BCTags tags; AbsorbBoundariesGR_kernel(ndfield_t Fld, - const I& finit, - const M& metric, - real_t xg_edge, - real_t dx_abs, - BCTags tags) + const I& finit, + const M& metric, + real_t xg_edge, + real_t dx_abs, + BCTags tags) : Fld { Fld } , finit { finit } , metric { metric } From c836ceb3c5c3d04d7133aa3dc8f0c8f1fd7e35b6 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 14 Jan 2025 17:19:13 -0500 Subject: [PATCH 097/234] added an option for controlling initial B-field discretisation --- setups/grpic/wald/pgen.hpp | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/setups/grpic/wald/pgen.hpp b/setups/grpic/wald/pgen.hpp index 7e8eea216..b27dd370e 100644 --- a/setups/grpic/wald/pgen.hpp +++ b/setups/grpic/wald/pgen.hpp @@ -20,7 +20,7 @@ namespace user { template struct InitFields { - InitFields(M metric_) : metric { metric_ } {} + InitFields(M metric_, real_t m_eps) : metric { metric_ }, m_eps { m_eps } {} Inline auto A_3(const coord_t& x_Cd) const -> real_t { return HALF * (metric.template h_<3, 3>(x_Cd) @@ -28,37 +28,37 @@ namespace user { ); } - Inline auto bx1(const coord_t& x_Ph) const -> real_t { + Inline auto bx1(const coord_t& x_Ph) const -> real_t { // at ( i , j + HALF ) coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); x0m[0] = xi[0]; - x0m[1] = xi[1] - HALF; + x0m[1] = xi[1] - HALF * m_eps; x0p[0] = xi[0]; - x0p[1] = xi[1] + HALF; + x0p[1] = xi[1] + HALF * m_eps; real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; if (cmp::AlmostZero(x_Ph[1])) return ONE; else - return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; + return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP / m_eps; } - Inline auto bx2(const coord_t& x_Ph) const -> real_t { + Inline auto bx2(const coord_t& x_Ph) const -> real_t { // at ( i + HALF , j ) coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); - x0m[0] = xi[0] - HALF; + x0m[0] = xi[0] - HALF * m_eps; x0m[1] = xi[1]; - x0p[0] = xi[0] + HALF; + x0p[0] = xi[0] + HALF * m_eps; x0p[1] = xi[1]; - real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0] , xi[1] }) }; if (cmp::AlmostZero(x_Ph[1])) return ZERO; else - return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; + return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP / m_eps; } Inline auto bx3(const coord_t& x_Ph) const -> real_t { @@ -79,6 +79,7 @@ namespace user { private: const M metric; + const real_t m_eps; }; template @@ -185,7 +186,7 @@ namespace user { const std::vector xi_min; const std::vector xi_max; - const real_t sigma0, sigma_max, multiplicity, nGJ, temperature; + const real_t sigma0, sigma_max, multiplicity, nGJ, temperature, m_eps; InitFields init_flds; @@ -198,7 +199,8 @@ namespace user { , multiplicity { p.template get("setup.multiplicity") } , nGJ { p.template get("scales.B0") * SQR(p.template get("scales.skindepth0")) } , temperature { p.template get("setup.temperature") } - , init_flds { m.mesh().metric } {} + , m_eps { p.template get("setup.m_eps") } + , init_flds { m.mesh().metric, m_eps } {} inline void InitPrtls(Domain& local_domain) { const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, From 1227bf6950934296defd096f4d10a3ed237ddc95 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 15 Jan 2025 15:36:42 -0500 Subject: [PATCH 098/234] grpic vacuum wald test improved --- setups/grpic/vacuum/pgen.hpp | 116 ++++++++++++++++++++++++++++---- setups/grpic/vacuum/vacuum.toml | 38 +++-------- 2 files changed, 112 insertions(+), 42 deletions(-) diff --git a/setups/grpic/vacuum/pgen.hpp b/setups/grpic/vacuum/pgen.hpp index 6d5e5fba5..60a2bfe05 100644 --- a/setups/grpic/vacuum/pgen.hpp +++ b/setups/grpic/vacuum/pgen.hpp @@ -27,12 +27,26 @@ namespace user { InitFields(M metric_) : metric { metric_ } {} Inline auto A_3(const coord_t& x_Cd) const -> real_t { - coord_t x_Ph { ZERO }; - metric.template convert(x_Cd, x_Ph); - return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); + return HALF * (metric.template h_<3, 3>(x_Cd) + + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + ); } - Inline auto bx1(const coord_t& x_Ph) const -> real_t { + Inline auto A_1(const coord_t& x_Cd) const -> real_t { + return HALF * (metric.template h_<1, 3>(x_Cd) + + TWO * metric.spin() * metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) + ); + } + + Inline auto A_0(const coord_t& x_Cd) const -> real_t { + real_t g_00 { -metric.alpha(x_Cd) * metric.alpha(x_Cd) + + metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) * metric.beta1(x_Cd) + }; + return HALF * (metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + + TWO * metric.spin() * g_00); + } + + Inline auto bx1(const coord_t& x_Ph) const -> real_t { // at ( i , j + HALF ) coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); @@ -49,7 +63,7 @@ namespace user { return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; } - Inline auto bx2(const coord_t& x_Ph) const -> real_t { + Inline auto bx2(const coord_t& x_Ph) const -> real_t { // at ( i + HALF , j ) coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); @@ -65,20 +79,96 @@ namespace user { return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; } - Inline auto bx3(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto bx3(const coord_t& x_Ph) const -> real_t { // at ( i + HALF , j + HALF ) + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0]; + x0m[1] = xi[1] - HALF; + x0p[0] = xi[0]; + x0p[1] = xi[1] + HALF; + + real_t inv_sqrt_detH_iPjP { ONE / metric.sqrt_det_h({ xi[0], xi[1]}) }; + return -(A_1(x0p) - A_1(x0m)) * inv_sqrt_detH_iPjP; } - Inline auto dx1(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto dx1(const coord_t& x_Ph) const -> real_t { // at ( i + HALF , j ) + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + real_t alpha_iPj { metric.alpha({ xi[0], xi[1] }) }; + real_t inv_sqrt_detH_ij { ONE / metric.sqrt_det_h({ xi[0] - HALF, xi[1] }) }; + real_t sqrt_detH_ij { metric.sqrt_det_h({ xi[0] - HALF, xi[1] }) }; + real_t beta_ij { metric.beta1({ xi[0] - HALF, xi[1]}) }; + real_t alpha_ij { metric.alpha({ xi[0] - HALF, xi[1]}) }; + + // D1 at ( i + HALF , j ) + x0m[0] = xi[0] - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF; + x0p[1] = xi[1]; + real_t E1d { (A_0(x0p) - A_0(x0m)) }; + real_t D1d { E1d / alpha_iPj }; + + // D3 at ( i , j ) + x0m[0] = xi[0] - HALF - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] - HALF + HALF; + x0p[1] = xi[1]; + real_t D3d { (A_3(x0p) - A_3(x0m)) * beta_ij / alpha_ij }; + + real_t D1u { metric.template h<1, 1>({ xi[0], xi[1] }) * D1d + metric.template h<1, 3>({ xi[0], xi[1] }) * D3d }; + + return D1u; } - Inline auto dx2(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto dx2(const coord_t& x_Ph) const -> real_t { // at ( i , j + HALF ) + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + x0m[0] = xi[0]; + x0m[1] = xi[1] - HALF; + x0p[0] = xi[0]; + x0p[1] = xi[1] + HALF; + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + real_t sqrt_detH_ijP { metric.sqrt_det_h({ xi[0], xi[1] }) }; + real_t alpha_ijP { metric.alpha({ xi[0], xi[1] }) }; + real_t beta_ijP { metric.beta1({ xi[0], xi[1] }) }; + + real_t E2d { (A_0(x0p) - A_0(x0m)) }; + real_t D2d { E2d / alpha_ijP - (A_1(x0p) - A_1(x0m)) * beta_ijP / alpha_ijP }; + real_t D2u { metric.template h<2, 2>({ xi[0], xi[1] }) * D2d }; + + return D2u; } - Inline auto dx3(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto dx3(const coord_t& x_Ph) const -> real_t { // at ( i , j ) + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + real_t inv_sqrt_detH_ij { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + real_t sqrt_detH_ij { metric.sqrt_det_h({ xi[0], xi[1] }) }; + real_t beta_ij { metric.beta1({ xi[0], xi[1] }) }; + real_t alpha_ij { metric.alpha({ xi[0], xi[1] }) }; + real_t alpha_iPj { metric.alpha({ xi[0] + HALF, xi[1] }) }; + + // D3 at ( i , j ) + x0m[0] = xi[0] - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF; + x0p[1] = xi[1]; + real_t D3d { (A_3(x0p) - A_3(x0m)) * beta_ij / alpha_ij }; + + // D1 at ( i + HALF , j ) + x0m[0] = xi[0] + HALF - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF + HALF; + x0p[1] = xi[1]; + real_t E1d { (A_0(x0p) - A_0(x0m)) }; + real_t D1d { E1d / alpha_iPj }; + + if (cmp::AlmostZero(x_Ph[1])) + return metric.template h<1, 3>({ xi[0], xi[1] }) * D1d; + else + return metric.template h<3, 3>({ xi[0], xi[1] }) * D3d + metric.template h<1, 3>({ xi[0], xi[1] }) * D1d; } private: diff --git a/setups/grpic/vacuum/vacuum.toml b/setups/grpic/vacuum/vacuum.toml index 3992b9f12..621f5d127 100644 --- a/setups/grpic/vacuum/vacuum.toml +++ b/setups/grpic/vacuum/vacuum.toml @@ -1,31 +1,31 @@ [simulation] - name = "wald" + name = "wald_Dph_abs_a95" engine = "grpic" - runtime = 500.0 + runtime = 1.0 [grid] resolution = [128,128] - extent = [[2.0, 8.0]] + extent = [[1.0, 10.0]] [grid.metric] - metric = "qkerr_schild" + metric = "kerr_schild" qsph_r0 = 0.0 qsph_h = 0.0 - ks_a = 0.0 + ks_a = 0.95 [grid.boundaries] fields = [["ABSORB"]] particles = [["ABSORB"]] [grid.boundaries.absorb] - ds = 0.5 + ds = 1.0 [scales] larmor0 = 0.0025 skindepth0 = 0.05 [algorithms] - current_filters = 4 + current_filters = 0 [algorithms.timestep] CFL = 0.5 @@ -35,36 +35,16 @@ fieldsolver = true [particles] - ppc0 = 8.0 - use_weights = true - sort_interval = 100 - - #[[particles.species]] - #label = "e-" - #mass = 1.0 - #charge = -1.0 - #maxnpart = 1e6 - #pusher = "Boris" - # - #[[particles.species]] - #label = "e+" - #mass = 1.0 - #charge = 1.0 - #maxnpart = 1e6 - #pusher = "Boris" + ppc0 = 2.0 [setup] - #multiplicity = 1.0 - #sigma_max = 1000.0 - inj_rmin = 1.2 - inj_rmax = 7.5 [output] format = "hdf5" [output.fields] interval_time = 1.0 - quantities = ["D", "H", "B", "J"] + quantities = ["D", "H", "B", "A"] [output.particles] enable = false From 02c8d5e0c82990f6309c26dd6b1671b4b21ee36b Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 15 Jan 2025 15:37:20 -0500 Subject: [PATCH 099/234] kernels: all fields are absorbed in GR BC --- src/kernels/fields_bcs.hpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 1bd45b8df..c885d7d43 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -645,6 +645,26 @@ template i1_ + HALF) }; const real_t x2_0 { metric.template convert<2, Crd::Cd, Crd::Ph>(i2_) }; Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * finit.bx2({ x1_H, x2_0 }); + } else if (comp == em::bx3) { + const real_t x1_H { metric.template convert<1, Crd::Cd, Crd::Ph>( + i1_ + HALF) }; + const real_t x2_H { metric.template convert<2, Crd::Cd, Crd::Ph>( + i2_ + HALF) }; + Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * finit.bx3({ x1_H, x2_H }); + } else if (comp == em::ex1) { + const real_t x1_H { metric.template convert<1, Crd::Cd, Crd::Ph>( + i1_ + HALF) }; + const real_t x2_0 { metric.template convert<2, Crd::Cd, Crd::Ph>(i2_) }; + Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * finit.dx1({ x1_H, x2_0 }); + } else if (comp == em::ex2) { + const real_t x1_0 { metric.template convert<1, Crd::Cd, Crd::Ph>(i1_) }; + const real_t x2_H { metric.template convert<2, Crd::Cd, Crd::Ph>( + i2_ + HALF) }; + Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * finit.dx2({ x1_0, x2_H }); + } else if (comp == em::ex3) { + const real_t x1_0 { metric.template convert<1, Crd::Cd, Crd::Ph>(i1_) }; + const real_t x2_0 { metric.template convert<2, Crd::Cd, Crd::Ph>(i2_) }; + Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * finit.dx3({ x1_0, x2_0 }); } } } else { From df3be98ec134a27374c4b116504bb1fdde34b585 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Thu, 30 Jan 2025 10:30:17 -0500 Subject: [PATCH 100/234] updated deposit setup --- setups/grpic/deposit/deposit.toml | 60 +++++++++++++++++++------------ 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/setups/grpic/deposit/deposit.toml b/setups/grpic/deposit/deposit.toml index c577f9c68..66c848de9 100644 --- a/setups/grpic/deposit/deposit.toml +++ b/setups/grpic/deposit/deposit.toml @@ -1,17 +1,17 @@ [simulation] name = "deposit" engine = "grpic" - runtime = 50.0 + runtime = 500.0 [grid] - resolution = [512,512] - extent = [[1.2, 20.0]] + resolution = [256,256] + extent = [[0.8, 20.0]] [grid.metric] - metric = "kerr_schild" + metric = "qkerr_schild" qsph_r0 = 0.0 qsph_h = 0.0 - ks_a = 0.0 + ks_a = 0.95 [grid.boundaries] fields = [["ABSORB"]] @@ -29,7 +29,7 @@ [algorithms.gr] pusher_niter = 10 - pusher_eps = 1e-6 + pusher_eps = 1e-2 [algorithms.timestep] CFL = 0.5 @@ -45,41 +45,55 @@ label = "e-" mass = 1.0 charge = -1.0 - maxnpart = 1e0 + maxnpart = 5e0 [[particles.species]] label = "e+" mass = 1.0 charge = 1.0 - maxnpart = 1e0 + maxnpart = 5e0 [setup] - x1s = [17.00000] - y1s = [1.570796] - z1s = [0.000000] - ux1s = [5.000000] - uy1s = [0.000000] - uz1s = [0.000000] - - x2s = [17.00000] - y2s = [1.570796] - z2s = [0.000000] - ux2s = [-5.000000] - uy2s = [0.000000] - uz2s = [0.000000] + x1s = [5.000, 15.0, 17.0, 4.0, 12.0] + y1s = [1.5707963268, 0.7853981634, 2.3561944902, 0.1, 3.0415926536] + z1s = [0.0, 0.0, 0.0, 0.0, 0.0] + ux1s = [5.0, 5.0, 5.0, 5.0, 5.0] + uy1s = [4.0, 7.0, 4.0, -5.0, 9.0] + uz1s = [2.0, 1.0, 0.5, 3.0, 2.4] + + x2s = [5.000, 15.0, 17.0, 4.0, 12.0] + y2s = [1.5707963268, 0.7853981634, 2.3561944902, 0.1, 3.0415926536] + z2s = [0.0, 0.0, 0.0, 0.0, 0.0] + ux2s = [-5.0, -5.0, -5.0, -5.0, -5.0] + uy2s = [4.0, 7.0, 4.0, -5.0, 9.0] + uz2s = [2.0, 1.0, 0.5, 3.0, 2.4] + + #x1s = [17.00000] + #y1s = [1.570796] + #z1s = [0.000000] + #ux1s = [5.000000] + #uy1s = [0.000000] + #uz1s = [0.000000] + + #x2s = [17.00000] + #y2s = [1.570796] + #z2s = [0.000000] + #ux2s = [-5.000000] + #uy2s = [0.000000] + #uz2s = [0.000000] [output] format = "hdf5" [output.fields] enable = true - interval_time = 0.5 + interval_time = 1.0 quantities = ["D"] [output.particles] enable = true stride = 1 - interval_time = 0.5 + interval_time = 1.0 species = [] [output.spectra] From a9f25d28eb92c86883416d2057b18436892af38c Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Thu, 30 Jan 2025 10:32:00 -0500 Subject: [PATCH 101/234] engines/grpic: minor updates --- src/engines/grpic.hpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index d6a45c953..275743df0 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -916,18 +916,16 @@ namespace ntt { void AmpereCurrents(domain_t& domain, const gr_ampere& g) { logger::Checkpoint("Launching Ampere kernel for adding currents", HERE); const auto q0 = m_params.template get("scales.q0"); - // const auto n0 = m_params.template get("scales.n0"); const auto B0 = m_params.template get("scales.B0"); const auto coeff = -dt * q0 / B0; - // auto range = range_with_axis_BCs(domain); auto range = CreateRangePolicy( { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2)}, { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1}); const auto ni2 = domain.mesh.n_active(in::x2); if (g == gr_ampere::aux) { - // Updates D0 with J. - Kokkos::parallel_for("Ampere-1", + // Updates D0 with J: D0(n-1/2) -> (J(n)) -> D0(n+1/2) + Kokkos::parallel_for("AmpereCurrentsAux", range, kernel::gr::CurrentsAmpere_kernel(domain.fields.em0, domain.fields.cur, @@ -936,8 +934,8 @@ namespace ntt { ni2, domain.mesh.flds_bc())); } else if (g == gr_ampere::main) { - // Updates D0 with J0. - Kokkos::parallel_for("Ampere-2", + // Updates D0 with J0: D0(n) -> (J0(n+1/2)) -> D0(n+1) + Kokkos::parallel_for("mpereCurrentsMain", range, kernel::gr::CurrentsAmpere_kernel(domain.fields.em0, domain.fields.cur0, From 05b9b86aab9c86f1dc6b605a02f2d09f3ab3ea36 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova Date: Tue, 24 Sep 2024 12:20:06 -0400 Subject: [PATCH 102/234] compilable example for grpic --- setups/grpic/pgen_grpic_example.hpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/setups/grpic/pgen_grpic_example.hpp b/setups/grpic/pgen_grpic_example.hpp index f553ae849..3374a12d7 100644 --- a/setups/grpic/pgen_grpic_example.hpp +++ b/setups/grpic/pgen_grpic_example.hpp @@ -12,7 +12,7 @@ namespace user { using namespace ntt; template - struct PGen : public ProblemGenerator { + struct PGen : public arch::ProblemGenerator { // compatibility traits for the problem generator static constexpr auto engines { traits::compatible_with::value }; static constexpr auto metrics { @@ -21,13 +21,12 @@ namespace user { static constexpr auto dimensions { traits::compatible_with::value }; // for easy access to variables in the child class - using ProblemGenerator::D; - using ProblemGenerator::C; - using ProblemGenerator::params; - using ProblemGenerator::domain; + using arch::ProblemGenerator::D; + using arch::ProblemGenerator::C; + using arch::ProblemGenerator::params; inline PGen(SimulationParams& p, const Metadomain&) : - ProblemGenerator(p) {} + arch::ProblemGenerator(p) {} inline PGen() {} }; From 010d04115a1ecb5084ab6751087ddfc81f5f8bcf Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 24 Sep 2024 12:48:33 -0400 Subject: [PATCH 103/234] starting a test pgen --- setups/grpic/wald/pgen.hpp | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 setups/grpic/wald/pgen.hpp diff --git a/setups/grpic/wald/pgen.hpp b/setups/grpic/wald/pgen.hpp new file mode 100644 index 000000000..3374a12d7 --- /dev/null +++ b/setups/grpic/wald/pgen.hpp @@ -0,0 +1,36 @@ +#ifndef PROBLEM_GENERATOR_H +#define PROBLEM_GENERATOR_H + +#include "enums.h" +#include "global.h" + +#include "arch/traits.h" + +#include "archetypes/problem_generator.h" + +namespace user { + using namespace ntt; + + template + struct PGen : public arch::ProblemGenerator { + // compatibility traits for the problem generator + static constexpr auto engines { traits::compatible_with::value }; + static constexpr auto metrics { + traits::compatible_with::value + }; + static constexpr auto dimensions { traits::compatible_with::value }; + + // for easy access to variables in the child class + using arch::ProblemGenerator::D; + using arch::ProblemGenerator::C; + using arch::ProblemGenerator::params; + + inline PGen(SimulationParams& p, const Metadomain&) : + arch::ProblemGenerator(p) {} + + inline PGen() {} + }; + +} // namespace user + +#endif From 5efd5565d23d1ae5b45093d5b7d1b7959a24038e Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 24 Sep 2024 13:46:21 -0400 Subject: [PATCH 104/234] engines/grpic: added relevant libraries --- src/engines/grpic.hpp | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index d082d617f..f36b27758 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -13,15 +13,38 @@ #define ENGINES_GRPIC_GRPIC_H #include "enums.h" +#include "global.h" +#include "arch/kokkos_aliases.h" +#include "arch/traits.h" +#include "utils/log.h" +#include "utils/numeric.h" #include "utils/timer.h" #include "utils/toml.h" +#include "archetypes/particle_injector.h" #include "framework/domain/domain.h" #include "framework/parameters.h" #include "engines/engine.hpp" +#include "kernels/ampere_gr.hpp" +#include "kernels/aux_fields_gr.hpp" +#include "kernels/currents_deposit.hpp" +#include "kernels/digital_filter.hpp" +#include "kernels/faraday_gr.hpp" +#include "kernels/fields_bcs.hpp" +#include "kernels/particle_moments.hpp" +#include "kernels/particle_pusher_gr.hpp" + +#include "pgen.hpp" + +#include +#include + +#include +#include + namespace ntt { template @@ -31,6 +54,7 @@ namespace ntt { using Engine::m_params; using Engine::m_metadomain; + public: static constexpr auto S { SimEngine::SRPIC }; From 397a27fa1690cd9639f00ff4f3ef3ab51c50d629 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 24 Sep 2024 13:58:39 -0400 Subject: [PATCH 105/234] kernels/ampere_gr: fixed dublicate ErrorIF --- src/kernels/ampere_gr.hpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/kernels/ampere_gr.hpp b/src/kernels/ampere_gr.hpp index 5af0fa4ef..045eb1729 100644 --- a/src/kernels/ampere_gr.hpp +++ b/src/kernels/ampere_gr.hpp @@ -57,9 +57,6 @@ namespace kernel::gr { , coeff { coeff } { if constexpr ((D == Dim::_2D) || (D == Dim::_3D)) { raise::ErrorIf(boundaries.size() < 2, "boundaries defined incorrectly", HERE); - raise::ErrorIf(boundaries[1].size() < 2, - "boundaries defined incorrectly", - HERE); is_axis_i2min = (boundaries[1].first == FldsBC::AXIS); is_axis_i2max = (boundaries[1].second == FldsBC::AXIS); } From 86b5a61557458a5fefc907043de39fd1835469a5 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 24 Sep 2024 14:21:40 -0400 Subject: [PATCH 106/234] engines/grpic: minor syntax adustments to match engines/srpic --- src/engines/grpic.hpp | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index f36b27758..3a223cc79 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -4,7 +4,7 @@ * @implements * - ntt::GRPICEngine<> : ntt::Engine<> * @cpp: - * - srpic.cpp + * - grpic.cpp * @namespaces: * - ntt:: */ @@ -27,7 +27,6 @@ #include "framework/parameters.h" #include "engines/engine.hpp" - #include "kernels/ampere_gr.hpp" #include "kernels/aux_fields_gr.hpp" #include "kernels/currents_deposit.hpp" @@ -36,7 +35,6 @@ #include "kernels/fields_bcs.hpp" #include "kernels/particle_moments.hpp" #include "kernels/particle_pusher_gr.hpp" - #include "pgen.hpp" #include @@ -49,20 +47,32 @@ namespace ntt { template class GRPICEngine : public Engine { - using base_t = Engine; - - using Engine::m_params; - using Engine::m_metadomain; - + using base_t = Engine; + using pgen_t = user::PGen; + using domain_t = Domain; + // constexprs + using base_t::pgen_is_ok; + // contents + using base_t::m_metadomain; + using base_t::m_params; + using base_t::m_pgen; + // methods + using base_t::init; + // variables + using base_t::dt; + using base_t::max_steps; + using base_t::runtime; + using base_t::step; + using base_t::time; public: - static constexpr auto S { SimEngine::SRPIC }; + static constexpr auto S { SimEngine::GRPIC }; - GRPICEngine(const SimulationParams& params) : base_t { params } {} + GRPICEngine(SimulationParams& params) : base_t { params } {} ~GRPICEngine() = default; - void step_forward(timer::Timers&, Domain&) override {} + void step_forward(timer::Timers& timers, domain_t& dom) override {} }; } // namespace ntt From 48095eed5ddd41606bc1ab8b403bd2a2382a2190 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 24 Sep 2024 17:06:01 -0400 Subject: [PATCH 107/234] engines/grpic: added initialisation of fields and boundary conditions --- src/engines/grpic.hpp | 184 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 182 insertions(+), 2 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 3a223cc79..dbc421d96 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -72,9 +72,189 @@ namespace ntt { ~GRPICEngine() = default; - void step_forward(timer::Timers& timers, domain_t& dom) override {} - }; + void step_forward(timer::Timers& timers, domain_t& dom) override { + const auto fieldsolver_enabled = m_params.template get( + "algorithms.toggles.fieldsolver"); + const auto deposit_enabled = m_params.template get( + "algorithms.toggles.deposit"); + const auto sort_interval = m_params.template get( + "particles.sort_interval"); + + if (step == 0) { + // communicate fields and apply BCs on the first timestep + /** + * Initially: em0::B -- + * em0::D -- + * em::B at -1/2 + * em::D at -1/2 + * + * cur0::J -- + * cur::J -- + * + * aux::E -- + * aux::H -- + * + * x_prtl at -1/2 + * u_prtl at -1/2 + */ + + /** + * em0::D, em::D, em0::B, em::B <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0 | Comm::D | Comm::D0); + FieldBoundaries(dom, BC::B | BC::D); + + } + } + + void FieldBoundaries(domain_t& domain, BCTags tags) { + for (auto& direction : dir::Directions::orth) { + if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::ABSORB) { + AbsorbFieldsIn(direction, domain, tags); + } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::AXIS) { + if (domain.mesh.flds_bc_in(direction) == FldsBC::AXIS) { + AxisFieldsIn(direction, domain, tags); + } + } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::CUSTOM) { + if (domain.mesh.flds_bc_in(direction) == FldsBC::CUSTOM) { + CustomFieldsIn(direction, domain, tags); + } + } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { + raise::Error("HORIZON BCs only applicable for GR", HERE); + } + } // loop over directions + } + void AbsorbFieldsIn(dir::direction_t direction, + domain_t& domain, + BCTags tags) { + /** + * absorbing boundaries + */ + const auto ds = m_params.template get( + "grid.boundaries.absorb.ds"); + const auto dim = direction.get_dim(); + real_t xg_min, xg_max, xg_edge; + auto sign = direction.get_sign(); + if (sign > 0) { // + direction + xg_max = m_metadomain.mesh().extent(dim).second; + xg_min = xg_max - ds; + xg_edge = xg_max; + } else { // - direction + xg_min = m_metadomain.mesh().extent(dim).first; + xg_max = xg_min + ds; + xg_edge = xg_min; + } + boundaries_t box; + boundaries_t incl_ghosts; + for (unsigned short d { 0 }; d < M::Dim; ++d) { + if (d == static_cast(dim)) { + box.push_back({ xg_min, xg_max }); + if (sign > 0) { + incl_ghosts.push_back({ false, true }); + } else { + incl_ghosts.push_back({ true, false }); + } + } else { + box.push_back(Range::All); + incl_ghosts.push_back({ true, true }); + } + } + if (not domain.mesh.Intersects(box)) { + return; + } + const auto intersect_range = domain.mesh.ExtentToRange(box, incl_ghosts); + tuple_t range_min { 0 }; + tuple_t range_max { 0 }; + + for (unsigned short d { 0 }; d < M::Dim; ++d) { + range_min[d] = intersect_range[d].first; + range_max[d] = intersect_range[d].second; + } + if (dim == in::x1) { + Kokkos::parallel_for( + "AbsorbFields", + CreateRangePolicy(range_min, range_max), + kernel::AbsorbBoundaries_kernel(domain.fields.em, + domain.mesh.metric, + xg_edge, + ds, + tags)); + } else if (dim == in::x2) { + if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { + Kokkos::parallel_for( + "AbsorbFields", + CreateRangePolicy(range_min, range_max), + kernel::AbsorbBoundaries_kernel(domain.fields.em, + domain.mesh.metric, + xg_edge, + ds, + tags)); + } else { + raise::Error("Invalid dimension", HERE); + } + } else if (dim == in::x3) { + if constexpr (M::Dim == Dim::_3D) { + Kokkos::parallel_for( + "AbsorbFields", + CreateRangePolicy(range_min, range_max), + kernel::AbsorbBoundaries_kernel(domain.fields.em, + domain.mesh.metric, + xg_edge, + ds, + tags)); + } else { + raise::Error("Invalid dimension", HERE); + } + } + } + + void AxisFieldsIn(dir::direction_t direction, + domain_t& domain, + BCTags tags) { + /** + * axis boundaries + */ + raise::ErrorIf(M::CoordType == Coord::Cart, + "Invalid coordinate type for axis BCs", + HERE); + raise::ErrorIf(direction.get_dim() != in::x2, + "Invalid axis direction, should be x2", + HERE); + const auto i2_min = domain.mesh.i_min(in::x2); + const auto i2_max = domain.mesh.i_max(in::x2); + if (direction.get_sign() < 0) { + Kokkos::parallel_for( + "AxisBCFields", + domain.mesh.n_all(in::x1), + kernel::AxisBoundaries_kernel(domain.fields.em, i2_min, tags)); + } else { + Kokkos::parallel_for( + "AxisBCFields", + domain.mesh.n_all(in::x1), + kernel::AxisBoundaries_kernel(domain.fields.em, i2_max, tags)); + } + } + + void CustomFieldsIn(dir::direction_t direction, + domain_t& domain, + BCTags tags) { + (void)direction; + (void)domain; + (void)tags; + raise::Error("Custom boundaries not implemented", HERE); + // if constexpr ( + // traits::has_member::value) { + // const auto [box, custom_fields] = m_pgen.CustomFields(time); + // if (domain.mesh.Intersects(box)) { + // } + // + // } else { + // raise::Error("Custom boundaries not implemented", HERE); + // } + } + + }; } // namespace ntt #endif // ENGINES_GRPIC_GRPIC_H From 95efa492085d35c89be07dc566c322b47496d0da Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 25 Sep 2024 12:55:12 -0400 Subject: [PATCH 108/234] engines/grpic: added cope of em field into em0 --- src/engines/grpic.hpp | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index dbc421d96..61c1fcbec 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -104,6 +104,13 @@ namespace ntt { m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0 | Comm::D | Comm::D0); FieldBoundaries(dom, BC::B | BC::D); + /** + * em0::B <- em::B + * em0::D <- em::D + * + * Now: em0::B & em0::D at -1/2 + */ + CopyFields(dom); } } @@ -254,6 +261,22 @@ namespace ntt { // } } + /** + * @brief Swaps em and em0 fields, cur and cur0 currents. + */ + // void SwapFields() { + // auto& mblock = this->meshblock; + // std::swap(mblock.em, mblock.em0); + // std::swap(mblock.cur, mblock.cur0); + // } + + /** + * @brief Copies em fields into em0 + */ + void CopyFields(domain_t& domain) { + Kokkos::deep_copy(domain.fields.em0, domain.fields.em); + } + }; } // namespace ntt From efcb68f614616ccc4e124d8129620dea4ac672e4 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 25 Sep 2024 15:00:21 -0400 Subject: [PATCH 109/234] engines/grpic: added computation of aux fields --- src/engines/grpic.hpp | 82 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 81 insertions(+), 1 deletion(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 61c1fcbec..4fc72a2c2 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -45,6 +45,15 @@ namespace ntt { + enum class gr_getE { + D0_B, + D_B0 + }; + enum class gr_getH { + D_B0, + D0_B0 + }; + template class GRPICEngine : public Engine { using base_t = Engine; @@ -111,6 +120,15 @@ namespace ntt { * Now: em0::B & em0::D at -1/2 */ CopyFields(dom); + + /** + * aux::E <- alpha * em::D + beta x em0::B + * aux::H <- alpha * em::B0 - beta x em::D + * + * Now: aux::E & aux::H at -1/2 + */ + ComputeAuxE(dom, gr_getE::D0_B); + ComputeAuxH(dom, gr_getH::D_B0); } } @@ -273,10 +291,72 @@ namespace ntt { /** * @brief Copies em fields into em0 */ - void CopyFields(domain_t& domain) { + void CopyFields(domain_t& domain) { Kokkos::deep_copy(domain.fields.em0, domain.fields.em); } + void ComputeAuxE(domain_t& domain, const gr_getE& g) { + auto range = range_with_axis_BCs(domain); + if (g == gr_getE::D0_B) { + Kokkos::parallel_for("ComputeAuxE", + range, + kernel::gr::ComputeAuxE_kernel(domain.fields.em0, + domain.fields.em, + domain.fields.em, + domain.mesh.metric)); + } else if (g == gr_getE::D_B0) { + Kokkos::parallel_for("ComputeAuxE", + range, + kernel::gr::ComputeAuxE_kernel(domain.fields.em, + domain.fields.em0, + domain.fields.em, + domain.mesh.metric)); + } else { + raise::Error("Wrong option for `g`", HERE); + } + } + + void ComputeAuxH(domain_t& domain, const gr_getH& g) { + auto range = range_with_axis_BCs(domain); + if (g == gr_getH::D_B0) { + Kokkos::parallel_for("ComputeAuxH", + range, + kernel::gr::ComputeAuxH_kernel(domain.fields.em, + domain.fields.em0, + domain.fields.em, + domain.mesh.metric)); + } else if (g == gr_getH::D0_B0) { + Kokkos::parallel_for("ComputeAuxH", + range, + kernel::gr::ComputeAuxH_kernel(domain.fields.em0, + domain.fields.em0, + domain.fields.em, + domain.mesh.metric)); + } else { + raise::Error("Wrong option for `g`", HERE); + } + } + + auto range_with_axis_BCs(const domain_t& domain) -> range_t { + auto range = domain.mesh.rangeActiveCells(); + if constexpr (M::CoordType != Coord::Cart) { + /** + * @brief taking one extra cell in the x1 and x2 directions if AXIS BCs + */ + if constexpr (M::Dim == Dim::_2D) { + if (domain.mesh.flds_bc_in({ 0, +1 }) == FldsBC::AXIS) { + range = CreateRangePolicy( + { domain.mesh.i_min(in::x1) - 1, domain.mesh.i_min(in::x2) }, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1 }); + } + } else if constexpr (M::Dim == Dim::_3D) { + raise::Error("Invalid dimension", HERE); + } + } + return range; + } + + }; } // namespace ntt From 365d03b0309537f37fe045dd3ec5cd71a64700b2 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 25 Sep 2024 16:21:32 -0400 Subject: [PATCH 110/234] engines/grpis: fixed "aux" for passing to aux kernels, added Faraday function --- src/engines/grpic.hpp | 52 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 4fc72a2c2..f67e0aa24 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -53,6 +53,15 @@ namespace ntt { D_B0, D0_B0 }; + enum class gr_faraday { + aux, + main + }; + enum class gr_ampere { + init, + aux, + main + }; template class GRPICEngine : public Engine { @@ -129,9 +138,17 @@ namespace ntt { */ ComputeAuxE(dom, gr_getE::D0_B); ComputeAuxH(dom, gr_getH::D_B0); + + /** + * aux::E, aux::H <- boundary conditions + */ + // ?? aux field boundaries ?? + + Faraday(dom, gr_faraday::aux, HALF); } } +/* algorithm substeps --------------------------------------------------- */ void FieldBoundaries(domain_t& domain, BCTags tags) { for (auto& direction : dir::Directions::orth) { if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::ABSORB) { @@ -323,14 +340,14 @@ namespace ntt { range, kernel::gr::ComputeAuxH_kernel(domain.fields.em, domain.fields.em0, - domain.fields.em, + domain.fields.aux, domain.mesh.metric)); } else if (g == gr_getH::D0_B0) { Kokkos::parallel_for("ComputeAuxH", range, kernel::gr::ComputeAuxH_kernel(domain.fields.em0, domain.fields.em0, - domain.fields.em, + domain.fields.aux, domain.mesh.metric)); } else { raise::Error("Wrong option for `g`", HERE); @@ -356,6 +373,37 @@ namespace ntt { return range; } + void Faraday(domain_t& domain, const gr_faraday& g, real_t fraction = ONE) { + logger::Checkpoint("Launching Faraday kernel", HERE); + const auto dT = fraction * + m_params.template get( + "algorithms.timestep.correction") * + dt; + if (g == gr_faraday::aux) { + Kokkos::parallel_for("Faraday", + domain.mesh.rangeActiveCells(), + kernel::gr::Faraday_kernel(domain.fields.em0, + domain.fields.em0, + domain.fields.aux, + domain.mesh.metric, + dT, + domain.mesh.i_max(in::x2), + domain.mesh.flds_bc())); + } else if (g == gr_faraday::main) { + Kokkos::parallel_for("Faraday", + domain.mesh.rangeActiveCells(), + kernel::gr::Faraday_kernel(domain.fields.em0, + domain.fields.em0, + domain.fields.aux, + domain.mesh.metric, + dT, + domain.mesh.i_max(in::x2), + domain.mesh.flds_bc())); + + } else { + raise::Error("Wrong option for `g`", HERE); + } + } }; } // namespace ntt From 22262de0af08c6b8ad47135fe4b64c75a62b077a Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 25 Sep 2024 16:55:27 -0400 Subject: [PATCH 111/234] minor: added a comment on Faraday step --- src/engines/grpic.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index f67e0aa24..3578b4d79 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -144,6 +144,11 @@ namespace ntt { */ // ?? aux field boundaries ?? + /** + * em0::B <- (em0::B) <- -curl aux::E + * + * Now: em0::B at 0 + */ Faraday(dom, gr_faraday::aux, HALF); } } From 9912542a50e9f10afedbb132876521d4681842c2 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 25 Sep 2024 17:00:41 -0400 Subject: [PATCH 112/234] engines/grpic: bc for B&B0 --- src/engines/grpic.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 3578b4d79..0047d9a00 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -150,6 +150,12 @@ namespace ntt { * Now: em0::B at 0 */ Faraday(dom, gr_faraday::aux, HALF); + + /** + * em0::B, em::B <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); + FieldBoundaries(dom, BC::B); } } From 50dcef88d578e957e499eb2c281908da50f3e6c3 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 25 Sep 2024 17:21:12 -0400 Subject: [PATCH 113/234] engine/grpic: fixed passing ni2 to faraday kernel. prep for ampere kernel --- src/engines/grpic.hpp | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 0047d9a00..28d0f8c13 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -156,6 +156,13 @@ namespace ntt { */ m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); FieldBoundaries(dom, BC::B); + + /** + * em::D <- (em0::D) <- curl aux::H + * + * Now: em::D at 0 + */ + Ampere(dom, gr_ampere::init, HALF); } } @@ -398,7 +405,7 @@ namespace ntt { domain.fields.aux, domain.mesh.metric, dT, - domain.mesh.i_max(in::x2), + domain.mesh.n_active(in::x2), domain.mesh.flds_bc())); } else if (g == gr_faraday::main) { Kokkos::parallel_for("Faraday", @@ -408,13 +415,37 @@ namespace ntt { domain.fields.aux, domain.mesh.metric, dT, - domain.mesh.i_max(in::x2), + domain.mesh.n_active(in::x2), domain.mesh.flds_bc())); } else { raise::Error("Wrong option for `g`", HERE); } } + + void Ampere(domain_t& domain, const gr_ampere& g, real_t fraction = ONE) { + logger::Checkpoint("Launching Ampere kernel", HERE); + const auto dT = fraction * + m_params.template get( + "algorithms.timestep.correction") * + dt; + auto range = CreateRangePolicy( + { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) + 1}, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2)}); + auto range_pole = CreateRangePolicy( + { domain.mesh.i_min(in::x1)}, + { domain.mesh.i_max(in::x1)}); + const auto ni2 = domain.mesh.n_active(in::x2); + Kokkos::parallel_for("Ampere", + range, + kernel::gr::Ampere_kernel(domain.fields.em, + domain.fields.em, + domain.fields.em, + domain.mesh.metric, + dT, + ni2, + domain.mesh.flds_bc())); + } }; } // namespace ntt From 215f3c54759bca4b93f18f39eb62fc1be79fbc0d Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 25 Sep 2024 17:51:47 -0400 Subject: [PATCH 114/234] engines/grpic: added ampere and subsequent BC --- src/engines/grpic.hpp | 140 +++++++++++++++++++++++++----------------- 1 file changed, 85 insertions(+), 55 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 28d0f8c13..14d99003f 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -101,25 +101,26 @@ namespace ntt { if (step == 0) { // communicate fields and apply BCs on the first timestep /** - * Initially: em0::B -- - * em0::D -- - * em::B at -1/2 - * em::D at -1/2 - * - * cur0::J -- - * cur::J -- - * - * aux::E -- - * aux::H -- - * - * x_prtl at -1/2 - * u_prtl at -1/2 - */ + * Initially: em0::B -- + * em0::D -- + * em::B at -1/2 + * em::D at -1/2 + * + * cur0::J -- + * cur::J -- + * + * aux::E -- + * aux::H -- + * + * x_prtl at -1/2 + * u_prtl at -1/2 + */ /** - * em0::D, em::D, em0::B, em::B <- boundary conditions - */ - m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0 | Comm::D | Comm::D0); + * em0::D, em::D, em0::B, em::B <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, + Comm::B | Comm::B0 | Comm::D | Comm::D0); FieldBoundaries(dom, BC::B | BC::D); /** @@ -166,7 +167,7 @@ namespace ntt { } } -/* algorithm substeps --------------------------------------------------- */ + /* algorithm substeps --------------------------------------------------- */ void FieldBoundaries(domain_t& domain, BCTags tags) { for (auto& direction : dir::Directions::orth) { if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::ABSORB) { @@ -360,7 +361,7 @@ namespace ntt { domain.fields.em0, domain.fields.aux, domain.mesh.metric)); - } else if (g == gr_getH::D0_B0) { + } else if (g == gr_getH::D0_B0) { Kokkos::parallel_for("ComputeAuxH", range, kernel::gr::ComputeAuxH_kernel(domain.fields.em0, @@ -398,31 +399,33 @@ namespace ntt { "algorithms.timestep.correction") * dt; if (g == gr_faraday::aux) { - Kokkos::parallel_for("Faraday", - domain.mesh.rangeActiveCells(), - kernel::gr::Faraday_kernel(domain.fields.em0, - domain.fields.em0, - domain.fields.aux, - domain.mesh.metric, - dT, - domain.mesh.n_active(in::x2), - domain.mesh.flds_bc())); + Kokkos::parallel_for( + "Faraday", + domain.mesh.rangeActiveCells(), + kernel::gr::Faraday_kernel(domain.fields.em0, + domain.fields.em0, + domain.fields.aux, + domain.mesh.metric, + dT, + domain.mesh.n_active(in::x2), + domain.mesh.flds_bc())); } else if (g == gr_faraday::main) { - Kokkos::parallel_for("Faraday", - domain.mesh.rangeActiveCells(), - kernel::gr::Faraday_kernel(domain.fields.em0, - domain.fields.em0, - domain.fields.aux, - domain.mesh.metric, - dT, - domain.mesh.n_active(in::x2), - domain.mesh.flds_bc())); + Kokkos::parallel_for( + "Faraday", + domain.mesh.rangeActiveCells(), + kernel::gr::Faraday_kernel(domain.fields.em0, + domain.fields.em0, + domain.fields.aux, + domain.mesh.metric, + dT, + domain.mesh.n_active(in::x2), + domain.mesh.flds_bc())); } else { - raise::Error("Wrong option for `g`", HERE); - } + raise::Error("Wrong option for `g`", HERE); + } } - + void Ampere(domain_t& domain, const gr_ampere& g, real_t fraction = ONE) { logger::Checkpoint("Launching Ampere kernel", HERE); const auto dT = fraction * @@ -430,23 +433,50 @@ namespace ntt { "algorithms.timestep.correction") * dt; auto range = CreateRangePolicy( - { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) + 1}, - { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2)}); - auto range_pole = CreateRangePolicy( - { domain.mesh.i_min(in::x1)}, - { domain.mesh.i_max(in::x1)}); + { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) + 1 }, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) }); + auto range_pole = CreateRangePolicy({ domain.mesh.i_min(in::x1) }, + { domain.mesh.i_max(in::x1) }); const auto ni2 = domain.mesh.n_active(in::x2); - Kokkos::parallel_for("Ampere", - range, - kernel::gr::Ampere_kernel(domain.fields.em, - domain.fields.em, - domain.fields.em, - domain.mesh.metric, - dT, - ni2, - domain.mesh.flds_bc())); - } + if (g == gr_ampere::aux) { + // First push, updates D0 with J. + Kokkos::parallel_for("Ampere-1", + range, + kernel::gr::Ampere_kernel(domain.fields.em, // has to be zeros + domain.fields.em0, + domain.fields.aux, + domain.mesh.metric, + dT, + ni2, + domain.mesh.flds_bc())); + } else if (g == gr_ampere::main) { + // Second push, updates D with J0 but assigns it to D0. + Kokkos::parallel_for("Ampere-2", + range, + kernel::gr::Ampere_kernel(domain.fields.em, + domain.fields.em0, + domain.fields.aux, + domain.mesh.metric, + dT, + ni2, + domain.mesh.flds_bc())); + } else if (g == gr_ampere::init) { + // Second push, updates D with J0 and assigns it to D. + Kokkos::parallel_for("Ampere-3", + range, + kernel::gr::Ampere_kernel(domain.fields.em, //has to be zeros + domain.fields.em, + domain.fields.aux, + domain.mesh.metric, + dT, + ni2, + domain.mesh.flds_bc())); + } else { + raise::Error("Wrong option for `g`", HERE); + } + + } }; } // namespace ntt From afc162014101938e0050cb405ee95223e8d7b87b Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Thu, 26 Sep 2024 15:25:56 -0400 Subject: [PATCH 115/234] engine/grpic: added everything for initial step, except BC for aux --- src/engines/grpic.hpp | 102 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 97 insertions(+), 5 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 14d99003f..0e89ff400 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -164,6 +164,99 @@ namespace ntt { * Now: em::D at 0 */ Ampere(dom, gr_ampere::init, HALF); + + /** + * em0::D, em::D <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + FieldBoundaries(dom, BC::D); + + /** + * aux::E <- alpha * em::D + beta x em0::B + * aux::H <- alpha * em0::B - beta x em::D + * + * Now: aux::E & aux::H at 0 + */ + ComputeAuxE(dom, gr_getE::D_B0); + ComputeAuxH(dom, gr_getH::D_B0); + + /** + * aux::E, aux::H <- boundary conditions + */ + // ?? aux field boundaries ?? + + // !ADD: GR -- particles? + + /** + * em0::B <- (em::B) <- -curl aux::E + * + * Now: em0::B at 1/2 + */ + Faraday(dom, gr_faraday::main, ONE); + /** + * em0::B, em::B <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); + FieldBoundaries(dom, BC::B); + + /** + * em0::D <- (em0::D) <- curl aux::H + * + * Now: em0::D at 1/2 + */ + Ampere(dom, gr_ampere::aux, ONE); + /** + * em0::D, em::D <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + FieldBoundaries(dom, BC::D); + + /** + * aux::H <- alpha * em0::B - beta x em0::D + * + * Now: aux::H at 1/2 + */ + ComputeAuxH(dom, gr_getH::D0_B0); + /** + * aux::H <- boundary conditions + */ + // ?? aux field boundaries ?? + + /** + * em0::D <- (em::D) <- curl aux::H + * + * Now: em0::D at 1 + * em::D at 0 + */ + Ampere(dom, gr_ampere::main, ONE); + /** + * em0::D, em::D <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + FieldBoundaries(dom, BC::D); + + /** + * em::D <-> em0::D + * em::B <-> em0::B + * em::J <-> em0::J + */ + SwapFields(dom); + /** + * Finally: em0::B at -1/2 + * em0::D at 0 + * em::B at 1/2 + * em::D at 1 + * + * cur0::J -- + * cur::J -- + * + * aux::E -- + * aux::H -- + * + * x_prtl at 1 + * u_prtl at 1/2 + */ + } } @@ -318,11 +411,10 @@ namespace ntt { /** * @brief Swaps em and em0 fields, cur and cur0 currents. */ - // void SwapFields() { - // auto& mblock = this->meshblock; - // std::swap(mblock.em, mblock.em0); - // std::swap(mblock.cur, mblock.cur0); - // } + void SwapFields(domain_t& domain) { + std::swap(domain.fields.em, domain.fields.em0); + std::swap(domain.fields.cur, domain.fields.cur0); + } /** * @brief Copies em fields into em0 From f3496b9f66ded703bd80003e700a8fda7b3984f0 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 27 Sep 2024 13:26:17 -0400 Subject: [PATCH 116/234] engines/grpis: minor update to first aux step for consistency --- src/engines/grpic.hpp | 51 +++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 0e89ff400..a5f64a1e3 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -119,8 +119,7 @@ namespace ntt { /** * em0::D, em::D, em0::B, em::B <- boundary conditions */ - m_metadomain.CommunicateFields(dom, - Comm::B | Comm::B0 | Comm::D | Comm::D0); + m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0 | Comm::D | Comm::D0); FieldBoundaries(dom, BC::B | BC::D); /** @@ -137,7 +136,7 @@ namespace ntt { * * Now: aux::E & aux::H at -1/2 */ - ComputeAuxE(dom, gr_getE::D0_B); + ComputeAuxE(dom, gr_getE::D_B0); ComputeAuxH(dom, gr_getH::D_B0); /** @@ -258,6 +257,20 @@ namespace ntt { */ } + + if (fieldsolver_enabled) { + + } + + { + + + } + + if (fieldsolver_enabled) { + + } + } /* algorithm substeps --------------------------------------------------- */ @@ -467,19 +480,17 @@ namespace ntt { auto range_with_axis_BCs(const domain_t& domain) -> range_t { auto range = domain.mesh.rangeActiveCells(); - if constexpr (M::CoordType != Coord::Cart) { - /** - * @brief taking one extra cell in the x1 and x2 directions if AXIS BCs - */ - if constexpr (M::Dim == Dim::_2D) { - if (domain.mesh.flds_bc_in({ 0, +1 }) == FldsBC::AXIS) { - range = CreateRangePolicy( - { domain.mesh.i_min(in::x1) - 1, domain.mesh.i_min(in::x2) }, - { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1 }); - } - } else if constexpr (M::Dim == Dim::_3D) { - raise::Error("Invalid dimension", HERE); + /** + * @brief taking one extra cell in the x1 and x2 directions if AXIS BCs + */ + if constexpr (M::Dim == Dim::_2D) { + if (domain.mesh.flds_bc_in({ 0, +1 }) == FldsBC::AXIS) { + range = CreateRangePolicy( + { domain.mesh.i_min(in::x1) - 1, domain.mesh.i_min(in::x2) }, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1 }); } + } else if constexpr (M::Dim == Dim::_3D) { + raise::Error("Invalid dimension", HERE); } return range; } @@ -525,17 +536,15 @@ namespace ntt { "algorithms.timestep.correction") * dt; auto range = CreateRangePolicy( - { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) + 1 }, - { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) }); - auto range_pole = CreateRangePolicy({ domain.mesh.i_min(in::x1) }, - { domain.mesh.i_max(in::x1) }); + { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2)}, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1}); const auto ni2 = domain.mesh.n_active(in::x2); if (g == gr_ampere::aux) { // First push, updates D0 with J. Kokkos::parallel_for("Ampere-1", range, - kernel::gr::Ampere_kernel(domain.fields.em, // has to be zeros + kernel::gr::Ampere_kernel(domain.fields.em, domain.fields.em0, domain.fields.aux, domain.mesh.metric, @@ -557,7 +566,7 @@ namespace ntt { // Second push, updates D with J0 and assigns it to D. Kokkos::parallel_for("Ampere-3", range, - kernel::gr::Ampere_kernel(domain.fields.em, //has to be zeros + kernel::gr::Ampere_kernel(domain.fields.em, domain.fields.em, domain.fields.aux, domain.mesh.metric, From 2eaba26ac55c28eaa075e15c3138ed6e952f14d7 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 27 Sep 2024 13:28:35 -0400 Subject: [PATCH 117/234] engines/grpic: fixed ComputeAuxE function --- src/engines/grpic.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index a5f64a1e3..5a41c4724 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -443,14 +443,14 @@ namespace ntt { range, kernel::gr::ComputeAuxE_kernel(domain.fields.em0, domain.fields.em, - domain.fields.em, + domain.fields.aux, domain.mesh.metric)); } else if (g == gr_getE::D_B0) { Kokkos::parallel_for("ComputeAuxE", range, kernel::gr::ComputeAuxE_kernel(domain.fields.em, domain.fields.em0, - domain.fields.em, + domain.fields.aux, domain.mesh.metric)); } else { raise::Error("Wrong option for `g`", HERE); From 7adc70906f1ffadb0a616710514e7a2dd1b78824 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 27 Sep 2024 13:30:28 -0400 Subject: [PATCH 118/234] engines/grpic: fixed Faraday function --- src/engines/grpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 5a41c4724..ff1187353 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -516,7 +516,7 @@ namespace ntt { Kokkos::parallel_for( "Faraday", domain.mesh.rangeActiveCells(), - kernel::gr::Faraday_kernel(domain.fields.em0, + kernel::gr::Faraday_kernel(domain.fields.em, domain.fields.em0, domain.fields.aux, domain.mesh.metric, From 8c4451d021ef98a6b4b833a8fa5eeb1df8e61cc1 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 27 Sep 2024 13:32:17 -0400 Subject: [PATCH 119/234] engines/grpis: fixed Ampere function --- src/engines/grpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index ff1187353..51ff5fbfe 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -544,7 +544,7 @@ namespace ntt { // First push, updates D0 with J. Kokkos::parallel_for("Ampere-1", range, - kernel::gr::Ampere_kernel(domain.fields.em, + kernel::gr::Ampere_kernel(domain.fields.em0, domain.fields.em0, domain.fields.aux, domain.mesh.metric, From b4619ded7a40f630e3b626a11d47b43423fba339 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 27 Sep 2024 13:45:25 -0400 Subject: [PATCH 120/234] engines/grpic: added a class for field_bc to accomodate passing em, em0, aux to BC kernels --- src/engines/grpic.hpp | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 51ff5fbfe..02ad93959 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -62,6 +62,11 @@ namespace ntt { aux, main }; + enum class gr_bc { + main, + main0, + aux + }; template class GRPICEngine : public Engine { @@ -274,17 +279,17 @@ namespace ntt { } /* algorithm substeps --------------------------------------------------- */ - void FieldBoundaries(domain_t& domain, BCTags tags) { + void FieldBoundaries(domain_t& domain, BCTags tags, const gr_bc& g) { for (auto& direction : dir::Directions::orth) { if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::ABSORB) { - AbsorbFieldsIn(direction, domain, tags); + AbsorbFieldsIn(direction, domain, tags, g); } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::AXIS) { if (domain.mesh.flds_bc_in(direction) == FldsBC::AXIS) { - AxisFieldsIn(direction, domain, tags); + AxisFieldsIn(direction, domain, tags, g); } } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::CUSTOM) { if (domain.mesh.flds_bc_in(direction) == FldsBC::CUSTOM) { - CustomFieldsIn(direction, domain, tags); + CustomFieldsIn(direction, domain, tags, g); } } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { raise::Error("HORIZON BCs only applicable for GR", HERE); @@ -294,7 +299,8 @@ namespace ntt { void AbsorbFieldsIn(dir::direction_t direction, domain_t& domain, - BCTags tags) { + BCTags tags + const gr_getE& g) { /** * absorbing boundaries */ @@ -378,7 +384,8 @@ namespace ntt { void AxisFieldsIn(dir::direction_t direction, domain_t& domain, - BCTags tags) { + BCTags tags + const gr_getE& g) { /** * axis boundaries */ @@ -405,7 +412,8 @@ namespace ntt { void CustomFieldsIn(dir::direction_t direction, domain_t& domain, - BCTags tags) { + BCTags tags + const gr_getE& g) { (void)direction; (void)domain; (void)tags; From 01c9d0dcafe898f836ec275d6029a402dc893a68 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 27 Sep 2024 14:06:50 -0400 Subject: [PATCH 121/234] engines/grpic: cut out 3d from absorb BCs --- src/engines/grpic.hpp | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 02ad93959..ff43eba71 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -354,7 +354,7 @@ namespace ntt { ds, tags)); } else if (dim == in::x2) { - if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { + if constexpr (M::Dim == Dim::_2D) { Kokkos::parallel_for( "AbsorbFields", CreateRangePolicy(range_min, range_max), @@ -366,19 +366,8 @@ namespace ntt { } else { raise::Error("Invalid dimension", HERE); } - } else if (dim == in::x3) { - if constexpr (M::Dim == Dim::_3D) { - Kokkos::parallel_for( - "AbsorbFields", - CreateRangePolicy(range_min, range_max), - kernel::AbsorbBoundaries_kernel(domain.fields.em, - domain.mesh.metric, - xg_edge, - ds, - tags)); - } else { + } else { raise::Error("Invalid dimension", HERE); - } } } From d1fce86c9a678de4005b1e6deb23bbdf5e00a54a Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 27 Sep 2024 14:10:10 -0400 Subject: [PATCH 122/234] minot: fix syntax --- src/engines/grpic.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index ff43eba71..d53969b54 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -299,8 +299,8 @@ namespace ntt { void AbsorbFieldsIn(dir::direction_t direction, domain_t& domain, - BCTags tags - const gr_getE& g) { + BCTags tags, + const gr_bc& g) { /** * absorbing boundaries */ @@ -373,8 +373,8 @@ namespace ntt { void AxisFieldsIn(dir::direction_t direction, domain_t& domain, - BCTags tags - const gr_getE& g) { + BCTags tags, + const gr_bc& g) { /** * axis boundaries */ @@ -401,8 +401,8 @@ namespace ntt { void CustomFieldsIn(dir::direction_t direction, domain_t& domain, - BCTags tags - const gr_getE& g) { + BCTags tags, + const gr_bc& g) { (void)direction; (void)domain; (void)tags; From 8d410ad6f02477995b772f6f0f96e79b90f22fc0 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 27 Sep 2024 14:10:44 -0400 Subject: [PATCH 123/234] minor: cut out options from gr_bc class --- src/engines/grpic.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index d53969b54..82e06cc59 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -64,7 +64,6 @@ namespace ntt { }; enum class gr_bc { main, - main0, aux }; From 38931ab08d7b649aabf8bc6de410f8661f51825c Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 27 Sep 2024 14:12:48 -0400 Subject: [PATCH 124/234] minor: cut out gr_bc from most bc functions since only horizon bc needs aux fields; em and em0 are always set together --- src/engines/grpic.hpp | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 82e06cc59..0538d2936 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -281,14 +281,14 @@ namespace ntt { void FieldBoundaries(domain_t& domain, BCTags tags, const gr_bc& g) { for (auto& direction : dir::Directions::orth) { if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::ABSORB) { - AbsorbFieldsIn(direction, domain, tags, g); + AbsorbFieldsIn(direction, domain, tags); } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::AXIS) { if (domain.mesh.flds_bc_in(direction) == FldsBC::AXIS) { - AxisFieldsIn(direction, domain, tags, g); + AxisFieldsIn(direction, domain, tags); } } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::CUSTOM) { if (domain.mesh.flds_bc_in(direction) == FldsBC::CUSTOM) { - CustomFieldsIn(direction, domain, tags, g); + CustomFieldsIn(direction, domain, tags); } } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { raise::Error("HORIZON BCs only applicable for GR", HERE); @@ -298,8 +298,7 @@ namespace ntt { void AbsorbFieldsIn(dir::direction_t direction, domain_t& domain, - BCTags tags, - const gr_bc& g) { + BCTags tags) { /** * absorbing boundaries */ @@ -372,8 +371,7 @@ namespace ntt { void AxisFieldsIn(dir::direction_t direction, domain_t& domain, - BCTags tags, - const gr_bc& g) { + BCTags tags) { /** * axis boundaries */ From de10607153d62a3897a8c08ee81e550e0cdc97a5 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 27 Sep 2024 14:15:03 -0400 Subject: [PATCH 125/234] engines/grpic: added gr_bc into step=0 functions --- src/engines/grpic.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 0538d2936..4a48e0b33 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -124,7 +124,7 @@ namespace ntt { * em0::D, em::D, em0::B, em::B <- boundary conditions */ m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0 | Comm::D | Comm::D0); - FieldBoundaries(dom, BC::B | BC::D); + FieldBoundaries(dom, BC::B | BC::D, gr_bc::main); /** * em0::B <- em::B @@ -159,7 +159,7 @@ namespace ntt { * em0::B, em::B <- boundary conditions */ m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); - FieldBoundaries(dom, BC::B); + FieldBoundaries(dom, BC::B, gr_bc::main); /** * em::D <- (em0::D) <- curl aux::H @@ -172,7 +172,7 @@ namespace ntt { * em0::D, em::D <- boundary conditions */ m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); - FieldBoundaries(dom, BC::D); + FieldBoundaries(dom, BC::D, gr_bc::main); /** * aux::E <- alpha * em::D + beta x em0::B @@ -200,7 +200,7 @@ namespace ntt { * em0::B, em::B <- boundary conditions */ m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); - FieldBoundaries(dom, BC::B); + FieldBoundaries(dom, BC::B, gr_bc::main); /** * em0::D <- (em0::D) <- curl aux::H @@ -212,7 +212,7 @@ namespace ntt { * em0::D, em::D <- boundary conditions */ m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); - FieldBoundaries(dom, BC::D); + FieldBoundaries(dom, BC::D, gr_bc::main); /** * aux::H <- alpha * em0::B - beta x em0::D @@ -236,7 +236,7 @@ namespace ntt { * em0::D, em::D <- boundary conditions */ m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); - FieldBoundaries(dom, BC::D); + FieldBoundaries(dom, BC::D, gr_bc::main); /** * em::D <-> em0::D From c4649f405ce38957d438a1bc54f6aebaf72ce7f3 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 27 Sep 2024 14:22:49 -0400 Subject: [PATCH 126/234] engines/grpic: added BC for em0 --- src/engines/grpic.hpp | 56 +++++++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 4a48e0b33..68ba6edf7 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -279,21 +279,29 @@ namespace ntt { /* algorithm substeps --------------------------------------------------- */ void FieldBoundaries(domain_t& domain, BCTags tags, const gr_bc& g) { - for (auto& direction : dir::Directions::orth) { - if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::ABSORB) { - AbsorbFieldsIn(direction, domain, tags); - } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::AXIS) { - if (domain.mesh.flds_bc_in(direction) == FldsBC::AXIS) { - AxisFieldsIn(direction, domain, tags); + if (g == gr_bc::main) { + for (auto& direction : dir::Directions::orth) { + if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::ABSORB) { + AbsorbFieldsIn(direction, domain, tags); + } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::AXIS) { + if (domain.mesh.flds_bc_in(direction) == FldsBC::AXIS) { + AxisFieldsIn(direction, domain, tags); + } + } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::CUSTOM) { + if (domain.mesh.flds_bc_in(direction) == FldsBC::CUSTOM) { + CustomFieldsIn(direction, domain, tags, g); + } + } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { + raise::Error("HORIZON BCs only applicable for GR", HERE); } - } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::CUSTOM) { - if (domain.mesh.flds_bc_in(direction) == FldsBC::CUSTOM) { - CustomFieldsIn(direction, domain, tags); + } // loop over directions + } else if (g == gr_bc::aux) { + for (auto& direction : dir::Directions::orth) { + if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { + raise::Error("HORIZON BCs only applicable for GR", HERE); } - } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { - raise::Error("HORIZON BCs only applicable for GR", HERE); } - } // loop over directions + } } void AbsorbFieldsIn(dir::direction_t direction, @@ -351,6 +359,14 @@ namespace ntt { xg_edge, ds, tags)); + Kokkos::parallel_for( + "AbsorbFields", + CreateRangePolicy(range_min, range_max), + kernel::AbsorbBoundaries_kernel(domain.fields.em0, + domain.mesh.metric, + xg_edge, + ds, + tags)); } else if (dim == in::x2) { if constexpr (M::Dim == Dim::_2D) { Kokkos::parallel_for( @@ -361,6 +377,14 @@ namespace ntt { xg_edge, ds, tags)); + Kokkos::parallel_for( + "AbsorbFields", + CreateRangePolicy(range_min, range_max), + kernel::AbsorbBoundaries_kernel(domain.fields.em0, + domain.mesh.metric, + xg_edge, + ds, + tags)); } else { raise::Error("Invalid dimension", HERE); } @@ -388,11 +412,19 @@ namespace ntt { "AxisBCFields", domain.mesh.n_all(in::x1), kernel::AxisBoundaries_kernel(domain.fields.em, i2_min, tags)); + Kokkos::parallel_for( + "AxisBCFields", + domain.mesh.n_all(in::x1), + kernel::AxisBoundaries_kernel(domain.fields.em0, i2_min, tags)); } else { Kokkos::parallel_for( "AxisBCFields", domain.mesh.n_all(in::x1), kernel::AxisBoundaries_kernel(domain.fields.em, i2_max, tags)); + Kokkos::parallel_for( + "AxisBCFields", + domain.mesh.n_all(in::x1), + kernel::AxisBoundaries_kernel(domain.fields.em0, i2_max, tags)); } } From bb7afaf2e59e457a1f38058a45f71e11fb851e8e Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 27 Sep 2024 16:07:17 -0400 Subject: [PATCH 127/234] added a kernel for OpenBoundaries --- src/kernels/fields_bcs.hpp | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 363ff3ad2..68514222b 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -846,6 +846,40 @@ namespace kernel::bc { } }; + template + struct OpenBoundaries_kernel { + ndfield_t Fld; + const std::size_t i1_min; + const bool setE, setB; + + OpenBoundaries_kernel(ndfield_t Fld, BCTags tags) + : Fld { Fld } + , i1_min { i1_min } + , setE { tags & BC::Ex1 or tags & BC::Ex2 or tags & BC::Ex3 } + , setB { tags & BC::Bx1 or tags & BC::Bx2 or tags & BC::Bx3 } {} + + Inline void operator()(index_t i2) const { + if constexpr (M::Dim == Dim::_2D) { + if (setE) { + Fld(i1_min - 1, i2, em::ex1) = Fld(i1_min, i2, em::ex1); + Fld(i1_min, i2, em::ex2) = Fld(i1_min + 1, i2, em::ex2); + Fld(i1_min - 1, i2, em::ex2) = Fld(i1_min, i2, em::ex2); + Fld(i1_min, i2, em::ex3) = Fld(i1_min + 1, i2, em::ex3); + Fld(i1_min - 1, i2, em::ex3) = Fld(i1_min, i2, em::ex3); + } else if (setB) { + Fld(i1_min, i2, em::bx1) = Fld(i1_min + 1, i2, em::bx1); + Fld(i1_min - 1, i2, em::bx1) = Fld(i1_min, i2, em::bx1); + Fld(i1_min - 1, i2, em::bx2) = Fld(i1_min, i2, em::bx2); + Fld(i1_min - 1, i2, em::bx3) = Fld(i1_min, i2, em::bx3); + } + } else { + raise::KernelError( + HERE, + "AbsorbFields_kernel: 2D implementation called for D != 2"); + } + } + }; + } // namespace kernel::bc #endif // KERNELS_FIELDS_BCS_HPP From 2a73574b5dd4220a9772f45f4826514881a75dc8 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Mon, 30 Sep 2024 15:48:22 -0400 Subject: [PATCH 128/234] engines/grpic: implemented function to call for OpenBoundaries_kernel kernels/fields_bcs: fixed parameter passing in OpenBoundaries_kernel --- src/engines/grpic.hpp | 35 +++++++++++++++++++++++++++++++++-- src/kernels/fields_bcs.hpp | 2 +- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 68ba6edf7..5f4a5e7a5 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -292,13 +292,13 @@ namespace ntt { CustomFieldsIn(direction, domain, tags, g); } } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { - raise::Error("HORIZON BCs only applicable for GR", HERE); + OpenFieldsIn(direction, domain, tags); } } // loop over directions } else if (g == gr_bc::aux) { for (auto& direction : dir::Directions::orth) { if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { - raise::Error("HORIZON BCs only applicable for GR", HERE); + OpenFieldsIn(direction, domain, tags); } } } @@ -393,6 +393,37 @@ namespace ntt { } } + void OpenFieldsIn(dir::direction_t direction, + domain_t& domain, + BCTags tags, + const gr_bc& g) { + /** + * open boundaries + */ + raise::ErrorIf(M::CoordType == Coord::Cart, + "Invalid coordinate type for axis BCs", + HERE); + raise::ErrorIf(direction.get_dim() != in::x1, + "Invalid axis direction, should be x2", + HERE); + const auto i1_min = domain.mesh.i_min(in::x1); + if (g == gr_bc::main) { + Kokkos::parallel_for( + "OpenBCFields", + domain.mesh.n_all(in::x1), + kernel::OpenBoundaries_kernel(domain.fields.em, i1_min, tags)); + Kokkos::parallel_for( + "OpenBCFields", + domain.mesh.n_all(in::x1), + kernel::AxisBoundaries_kernel(domain.fields.em0, i1_min, tags)); + } else if (g == gr_bc::aux) { + Kokkos::parallel_for( + "OpenBCFields", + domain.mesh.n_all(in::x1), + kernel::OpenBoundaries_kernel(domain.fields.aux, i1_min, tags)); + } + } + void AxisFieldsIn(dir::direction_t direction, domain_t& domain, BCTags tags) { diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 68514222b..7c574f058 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -852,7 +852,7 @@ namespace kernel::bc { const std::size_t i1_min; const bool setE, setB; - OpenBoundaries_kernel(ndfield_t Fld, BCTags tags) + OpenBoundaries_kernel(ndfield_t Fld, std::size_t i1_min, BCTags tags) : Fld { Fld } , i1_min { i1_min } , setE { tags & BC::Ex1 or tags & BC::Ex2 or tags & BC::Ex3 } From 37a34eacdcc0397bf5182271c50f1d32bd6afbec Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Mon, 30 Sep 2024 15:54:30 -0400 Subject: [PATCH 129/234] enginesgrpic: added aux field boundaries calls to initial step --- src/engines/grpic.hpp | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 5f4a5e7a5..2ceca46ee 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -146,7 +146,7 @@ namespace ntt { /** * aux::E, aux::H <- boundary conditions */ - // ?? aux field boundaries ?? + FieldBoundaries(dom, BC::B | BC::D, gr_bc::aux); /** * em0::B <- (em0::B) <- -curl aux::E @@ -186,7 +186,7 @@ namespace ntt { /** * aux::E, aux::H <- boundary conditions */ - // ?? aux field boundaries ?? + FieldBoundaries(dom, BC::B | BC::D, gr_bc::aux); // !ADD: GR -- particles? @@ -223,7 +223,7 @@ namespace ntt { /** * aux::H <- boundary conditions */ - // ?? aux field boundaries ?? + FieldBoundaries(dom, BC::B, gr_bc::aux); /** * em0::D <- (em::D) <- curl aux::H @@ -259,7 +259,6 @@ namespace ntt { * x_prtl at 1 * u_prtl at 1/2 */ - } if (fieldsolver_enabled) { @@ -292,13 +291,13 @@ namespace ntt { CustomFieldsIn(direction, domain, tags, g); } } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { - OpenFieldsIn(direction, domain, tags); + OpenFieldsIn(direction, domain, tags, g); } } // loop over directions } else if (g == gr_bc::aux) { for (auto& direction : dir::Directions::orth) { if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { - OpenFieldsIn(direction, domain, tags); + OpenFieldsIn(direction, domain, tags, g); } } } @@ -410,16 +409,16 @@ namespace ntt { if (g == gr_bc::main) { Kokkos::parallel_for( "OpenBCFields", - domain.mesh.n_all(in::x1), + domain.mesh.n_all(in::x2), kernel::OpenBoundaries_kernel(domain.fields.em, i1_min, tags)); Kokkos::parallel_for( "OpenBCFields", - domain.mesh.n_all(in::x1), - kernel::AxisBoundaries_kernel(domain.fields.em0, i1_min, tags)); + domain.mesh.n_all(in::x2), + kernel::OpenBoundaries_kernel(domain.fields.em0, i1_min, tags)); } else if (g == gr_bc::aux) { Kokkos::parallel_for( "OpenBCFields", - domain.mesh.n_all(in::x1), + domain.mesh.n_all(in::x2), kernel::OpenBoundaries_kernel(domain.fields.aux, i1_min, tags)); } } From 454b590ab45c1b2f3c9e40af354bdada3f2a23fc Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Mon, 30 Sep 2024 16:16:18 -0400 Subject: [PATCH 130/234] added TimeAverageDB() and kernel for it in kernels/aux --- src/engines/grpic.hpp | 34 ++++++++++++++++++++++++++++++- src/kernels/aux_fields_gr.hpp | 38 +++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 2ceca46ee..8097e1fb1 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -261,8 +261,31 @@ namespace ntt { */ } - if (fieldsolver_enabled) { + /** + * Initially: em0::B at n-3/2 + * em0::D at n-1 + * em::B at n-1/2 + * em::D at n + * + * cur0::J -- + * cur::J at n-1/2 + * + * aux::E -- + * aux::H -- + * + * x_prtl at n + * u_prtl at n-1/2 + */ + if (fieldsolver_enabled) { + /** + * em0::D <- (em0::D + em::D) / 2 + * em0::B <- (em0::B + em::B) / 2 + * + * Now: em0::D at n-1/2 + * em0::B at n-1 + */ + TimeAverageDB(dom); } { @@ -634,6 +657,15 @@ namespace ntt { } } + + void TimeAverageDB(domain_t& domain) { + auto range = range_with_axis_BCs(domain); + Kokkos::parallel_for("TimeAverageDB", + range, + kernel::gr::TimeAverageDB_kernel(domain.fields.em, + domain.fields.em0, + domain.mesh.metric)); + } }; } // namespace ntt diff --git a/src/kernels/aux_fields_gr.hpp b/src/kernels/aux_fields_gr.hpp index 5744c3092..4f87ba1f2 100644 --- a/src/kernels/aux_fields_gr.hpp +++ b/src/kernels/aux_fields_gr.hpp @@ -230,6 +230,44 @@ namespace kernel::gr { } } }; + + /** + * @brief Kernel for computing time average of B and D + * @tparam M Metric + */ + template + class TimeAverageDB_kernel { + static_assert(M::is_metric, "M must be a metric class"); + static constexpr auto D = M::Dim; + + const ndfield_t BDf; + ndfield_t BDf0; + const M metric; + + public: + TimeAverageDB_kernel(const ndfield_t& BDf, + const ndfield_t& BDf0, + const M& metric) + : BDf { BDf } + , BDf0 { BDf0 } + , metric { metric } {} + + Inline void operator()(index_t i1, index_t i2) const { + if constexpr (D == Dim::_2D) { + BDf0(i1, i2, em::bx1) = HALF * (BDf0(i1, i2, em::bx1) + BDf(i1, i2, em::bx1)); + BDf0(i1, i2, em::bx2) = HALF * (BDf0(i1, i2, em::bx2) + BDf(i1, i2, em::bx2)); + BDf0(i1, i2, em::bx3) = HALF * (BDf0(i1, i2, em::bx3) + BDf(i1, i2, em::bx3)); + BDf0(i1, i2, em::ex1) = HALF * (BDf0(i1, i2, em::bx1) + BDf(i1, i2, em::ex1)); + BDf0(i1, i2, em::ex2) = HALF * (BDf0(i1, i2, em::bx2) + BDf(i1, i2, em::ex2)); + BDf0(i1, i2, em::ex3) = HALF * (BDf0(i1, i2, em::bx3) + BDf(i1, i2, em::ex3)); + } else { + raise::KernelError( + HERE, + "ComputeAuxH_kernel: 2D implementation called for D != 2"); + } + } + + }; } // namespace kernel::gr #endif // KERNELS_AUX_FIELDS_GR_HPP From fd5d1b4f6710858646d23e0dd58d97cc18014065 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Mon, 30 Sep 2024 16:26:17 -0400 Subject: [PATCH 131/234] engines/grpic: first half of the main step for field solver --- src/engines/grpic.hpp | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 8097e1fb1..db984049e 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -286,15 +286,53 @@ namespace ntt { * em0::B at n-1 */ TimeAverageDB(dom); + /** + * aux::E <- alpha * em0::D + beta x em::B + * + * Now: aux::E at n-1/2 + */ + ComputeAuxE(dom, gr_getE::D0_B); + /** + * aux::E <- boundary conditions + */ + FieldBoundaries(dom, BC::D, gr_bc::aux); + /** + * em0::B <- (em0::B) <- -curl aux::E + * + * Now: em0::B at n + */ + Faraday(dom, gr_faraday::aux, ONE); + + + /** + * em0::B, em::B <- boundary conditions + */ + FieldBoundaries(dom, BC::B, gr_bc::main); + m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); + + + /** + * aux::H <- alpha * em0::B - beta x em::D + * + * Now: aux::H at n + */ + ComputeAuxH(dom, gr_getH::D_B0); + /** + * aux::H <- boundary conditions + */ + FieldBoundaries(dom, BC::B, gr_bc::aux); } { - + /** + * particle pusher goes here + * + */ } if (fieldsolver_enabled) { - + } } From 5aba20a14f61c665423e9fa2eb9e68f84f74b59a Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Mon, 30 Sep 2024 16:36:45 -0400 Subject: [PATCH 132/234] added TimeAverageJ() and its kernel in kernels/aux --- src/engines/grpic.hpp | 11 ++++++++++- src/kernels/aux_fields_gr.hpp | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index db984049e..4043968ff 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -332,7 +332,7 @@ namespace ntt { } if (fieldsolver_enabled) { - + TimeAverageJ(dom); } } @@ -704,6 +704,15 @@ namespace ntt { domain.fields.em0, domain.mesh.metric)); } + + void TimeAverageJ(domain_t& domain) { + auto range = range_with_axis_BCs(domain); + Kokkos::parallel_for("TimeAverageJ", + range, + kernel::gr::TimeAverageJ_kernel(domain.fields.cur, + domain.fields.cur0, + domain.mesh.metric)); + } }; } // namespace ntt diff --git a/src/kernels/aux_fields_gr.hpp b/src/kernels/aux_fields_gr.hpp index 4f87ba1f2..b56a819f9 100644 --- a/src/kernels/aux_fields_gr.hpp +++ b/src/kernels/aux_fields_gr.hpp @@ -266,7 +266,40 @@ namespace kernel::gr { "ComputeAuxH_kernel: 2D implementation called for D != 2"); } } + }; + /** + * @brief Kernel for computing time average of J + * @tparam M Metric + */ + template + class TimeAverageJ_kernel { + static_assert(M::is_metric, "M must be a metric class"); + static constexpr auto D = M::Dim; + + const ndfield_t Jf; + ndfield_t Jf0; + const M metric; + + public: + TimeAverageJ_kernel(const ndfield_t& Jf, + const ndfield_t& Jf0, + const M& metric) + : Jf { Jf } + , Jf0 { Jf0 } + , metric { metric } {} + + Inline void operator()(index_t i1, index_t i2) const { + if constexpr (D == Dim::_2D) { + Jf0(i1, i2, cur::jx1) = HALF * (Jf0(i1, i2, cur::jx1) + Jf(i1, i2, cur::jx1)); + Jf0(i1, i2, cur::jx2) = HALF * (Jf0(i1, i2, cur::jx2) + Jf(i1, i2, cur::jx2)); + Jf0(i1, i2, cur::jx3) = HALF * (Jf0(i1, i2, cur::jx3) + Jf(i1, i2, cur::jx3)); + } else { + raise::KernelError( + HERE, + "ComputeAuxH_kernel: 2D implementation called for D != 2"); + } + } }; } // namespace kernel::gr From b5207e366085203951057a8c22fc322a3170f7a9 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Mon, 30 Sep 2024 16:50:56 -0400 Subject: [PATCH 133/234] engines/grpic: added all steps relevant to field solver --- src/engines/grpic.hpp | 109 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 108 insertions(+), 1 deletion(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 4043968ff..1e4f873f9 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -332,9 +332,116 @@ namespace ntt { } if (fieldsolver_enabled) { + /** + * cur::J <- (cur0::J + cur::J) / 2 + * + * Now: cur::J at n + */ TimeAverageJ(dom); - } + + /** + * aux::Е <- alpha * em::D + beta x em0::B + * + * Now: aux::Е at n + */ + ComputeAuxE(dom, gr_getE::D_B0); + /** + * aux::Е <- boundary conditions + */ + FieldBoundaries(dom, BC::D, gr_bc::aux); + /** + * em0::B <- (em::B) <- -curl aux::E + * + * Now: em0::B at n+1/2 + * em::B at n-1/2 + */ + Faraday(dom, gr_faraday::main, ONE); + + /** + * em0::B, em::B <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); + FieldBoundaries(dom, BC::B, gr_bc::main); + + + /** + * em0::D <- (em0::D) <- curl aux::H + * + * Now: em0::D at n+1/2 + */ + Ampere(dom, gr_ampere::aux, ONE); + + + if (deposit_enabled) { + /** + * em0::D <- (em0::D) <- cur::J + * + * Now: em0::D at n+1/2 + */ + // AmpereCurrents(gr_ampere::aux); + } + + /** + * em0::D, em::D <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + FieldBoundaries(dom, BC::D, gr_bc::main); + + /** + * aux::H <- alpha * em0::B - beta x em0::D + * + * Now: aux::H at n+1/2 + */ + ComputeAuxH(dom, gr_getH::D0_B0); + /** + * aux::H <- boundary conditions + */ + FieldBoundaries(dom, BC::B, gr_bc::aux); + /** + * em0::D <- (em::D) <- curl aux::H + * + * Now: em0::D at n+1 + * em::D at n + */ + Ampere(dom, gr_ampere::main, ONE); + if (deposit_enabled) { + /** + * em0::D <- (em0::D) <- cur0::J + * + * Now: em0::D at n+1 + */ + // AmpereCurrents(gr_ampere::main); + } + /** + * em::D <-> em0::D + * em::B <-> em0::B + * cur::J <-> cur0::J + */ + SwapFields(dom); + + + /** + * em0::D, em::D <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + FieldBoundaries(dom, BC::D, gr_bc::main); + } + /** + * Finally: em0::B at n-1/2 + * em0::D at n + * em::B at n+1/2 + * em::D at n+1 + * + * cur0::J (at n) + * cur::J at n+1/2 + * + * aux::E (at n+1/2) + * aux::H (at n) + * + * x_prtl at n+1 + * u_prtl at n+1/2 + */ } /* algorithm substeps --------------------------------------------------- */ From 4d1d0b0b4b9bb882daa04559a720827dc42f8135 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Mon, 30 Sep 2024 17:09:21 -0400 Subject: [PATCH 134/234] engines/grpic: added function for Ampere currents --- src/engines/grpic.hpp | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 1e4f873f9..c5a946b00 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -800,6 +800,40 @@ namespace ntt { } else { raise::Error("Wrong option for `g`", HERE); } + } + + void AmpereCurrents(domain_t& domain, const gr_ampere& g) { + logger::Checkpoint("Launching Ampere kernel for adding currents", HERE); + const auto q0 = m_params.template get("scales.q0"); + const auto n0 = m_params.template get("scales.n0"); + const auto B0 = m_params.template get("scales.B0"); + const auto coeff = -dt * q0 * n0 / B0; + auto range = range_with_axis_BCs(domain); + const auto ni2 = domain.mesh.n_active(in::x2); + + if (g == gr_ampere::aux) { + // Updates D0 with J. + Kokkos::parallel_for("Ampere-1", + range, + kernel::gr::CurrentsAmpere_kernel(domain.fields.em0, + domain.fields.cur, + domain.mesh.metric, + coeff, + ni2, + domain.mesh.flds_bc())); + } else if (g == gr_ampere::main) { + // Updates D0 with J0. + Kokkos::parallel_for("Ampere-2", + range, + kernel::gr::CurrentsAmpere_kernel(domain.fields.em0, + domain.fields.cur0, + domain.mesh.metric, + coeff, + ni2, + domain.mesh.flds_bc())); + } else { + raise::Error("Wrong option for `g`", HERE); + } } From 04201cee737f4f77dee85f9954a39deb3b40ecba Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Mon, 30 Sep 2024 17:14:21 -0400 Subject: [PATCH 135/234] see below -- added call from the time step --- src/engines/grpic.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index c5a946b00..30f9e35ad 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -378,7 +378,7 @@ namespace ntt { * * Now: em0::D at n+1/2 */ - // AmpereCurrents(gr_ampere::aux); + AmpereCurrents(dom, gr_ampere::aux); } /** @@ -411,7 +411,7 @@ namespace ntt { * * Now: em0::D at n+1 */ - // AmpereCurrents(gr_ampere::main); + AmpereCurrents(dom, gr_ampere::main); } /** * em::D <-> em0::D From a67362130655f140f6235e60b0ffcec9cb609e4c Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 1 Oct 2024 12:16:48 -0400 Subject: [PATCH 136/234] engines/grpic: started ParticlePush --- src/engines/grpic.hpp | 154 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 152 insertions(+), 2 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 30f9e35ad..d8b38208e 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -325,10 +325,44 @@ namespace ntt { { /** - * particle pusher goes here - * + * x_prtl, u_prtl <- em::D, em0::B + * + * Now: x_prtl at n + 1, u_prtl at n + 1/2 */ + timers.start("ParticlePusher"); + ParticlePush(dom); + timers.stop("ParticlePusher"); + + /** + * cur0::J <- current deposition + * + * Now: cur0::J at n+1/2 + */ + if (deposit_enabled) { + timers.start("CurrentDeposit"); + Kokkos::deep_copy(dom.fields.cur, ZERO); + CurrentsDeposit(dom); + timers.stop("CurrentDeposit"); + + timers.start("Communications"); + m_metadomain.SynchronizeFields(dom, Comm::J); + m_metadomain.CommunicateFields(dom, Comm::J); + timers.stop("Communications"); + + // timers.start("FieldBoundaries"); + // CurrentsBoundaryConditions(); + // timers.stop("FieldBoundaries"); + + timers.start("CurrentFiltering"); + CurrentsFilter(dom); + timers.stop("CurrentFiltering"); + } + timers.start("Communications"); + if ((sort_interval > 0) and (step % sort_interval == 0)) { + m_metadomain.CommunicateParticles(dom, &timers); + } + timers.stop("Communications"); } if (fieldsolver_enabled) { @@ -854,6 +888,122 @@ namespace ntt { domain.fields.cur0, domain.mesh.metric)); } + + void CurrentsDeposit(domain_t& domain) { + auto scatter_cur = Kokkos::Experimental::create_scatter_view( + domain.fields.cur); + for (auto& species : domain.species) { + logger::Checkpoint( + fmt::format("Launching currents deposit kernel for %d [%s] : %lu %f", + species.index(), + species.label().c_str(), + species.npart(), + (double)species.charge()), + HERE); + if (species.npart() == 0 || cmp::AlmostZero(species.charge())) { + continue; + } + Kokkos::parallel_for("CurrentsDeposit", + species.rangeActiveParticles(), + kernel::DepositCurrents_kernel( + scatter_cur, + species.i1, + species.i2, + species.i3, + species.i1_prev, + species.i2_prev, + species.i3_prev, + species.dx1, + species.dx2, + species.dx3, + species.dx1_prev, + species.dx2_prev, + species.dx3_prev, + species.ux1, + species.ux2, + species.ux3, + species.phi, + species.weight, + species.tag, + domain.mesh.metric, + (real_t)(species.charge()), + dt)); + } + Kokkos::Experimental::contribute(domain.fields.cur, scatter_cur); + } + + void CurrentsFilter(domain_t& domain) { + logger::Checkpoint("Launching currents filtering kernels", HERE); + auto range = range_with_axis_BCs(domain); + const auto nfilter = m_params.template get( + "algorithms.current_filters"); + tuple_t size; + if constexpr (M::Dim == Dim::_1D || M::Dim == Dim::_2D || M::Dim == Dim::_3D) { + size[0] = domain.mesh.n_active(in::x1); + } + if constexpr (M::Dim == Dim::_2D || M::Dim == Dim::_3D) { + size[1] = domain.mesh.n_active(in::x2); + } + if constexpr (M::Dim == Dim::_3D) { + size[2] = domain.mesh.n_active(in::x3); + } + // !TODO: this needs to be done more efficiently + for (unsigned short i = 0; i < nfilter; ++i) { + Kokkos::deep_copy(domain.fields.buff, domain.fields.cur); + Kokkos::parallel_for("CurrentsFilter", + range, + kernel::DigitalFilter_kernel( + domain.fields.cur, + domain.fields.buff, + size, + domain.mesh.flds_bc())); + m_metadomain.CommunicateFields(domain, Comm::J); + } + } + + void ParticlePush(domain_t& domain) { + for (auto& species : domain.species) { + species.set_unsorted(); + logger::Checkpoint( + fmt::format("Launching particle pusher kernel for %d [%s] : %lu", + species.index(), + species.label().c_str(), + species.npart()), + HERE); + if (species.npart() == 0) { + continue; + } + const auto q_ovr_m = species.mass() > ZERO + ? species.charge() / species.mass() + : ZERO; + // coeff = q / m (dt / 2) omegaB0 + const auto coeff = q_ovr_m * HALF * dt * + m_params.template get("scales.omegaB0"); + // clang-format off + Kokkos::parallel_for( + "ParticlePusher", + species.rangeActiveParticles(), + kernel::gr::Pusher_kernel( + domain.fields.em, + domain.fields.em0, + species.i1, species.i2, species.i3, + species.i1_prev, species.i2_prev, species.i3_prev, + species.dx1, species.dx2, species.dx3, + species.dx1_prev, species.dx2_prev, species.dx3_prev, + species.ux1, species.ux2, species.ux3, + species.phi, species.tag, + domain.mesh.metric, + coeff, dt, + domain.mesh.n_active(in::x1), + domain.mesh.n_active(in::x2), + domain.mesh.n_active(in::x3), + m_params.template get("algorithms.gr.pusher_eps"), m_params.template get("algorithms.gr.pusher_niter"), + domain.mesh.prtl_bc() + )); + // clang-format on + } + } + }; } // namespace ntt From 76f4e81830570b6ceccc5a4bfdc430349e9a0fdd Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 1 Oct 2024 12:21:25 -0400 Subject: [PATCH 137/234] see below --- src/engines/grpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index d8b38208e..9b71977cd 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -330,7 +330,7 @@ namespace ntt { * Now: x_prtl at n + 1, u_prtl at n + 1/2 */ timers.start("ParticlePusher"); - ParticlePush(dom); + // ParticlePush(dom); timers.stop("ParticlePusher"); /** From 1d53dc9b13b52f13ffef9e25cd1d42f71115417d Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 1 Oct 2024 12:32:47 -0400 Subject: [PATCH 138/234] engines/grpic: added timers --- src/engines/grpic.hpp | 65 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 9b71977cd..c6e7b657f 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -278,6 +278,7 @@ namespace ntt { */ if (fieldsolver_enabled) { + timers.start("FieldSolver"); /** * em0::D <- (em0::D + em::D) / 2 * em0::B <- (em0::B + em::B) / 2 @@ -292,35 +293,49 @@ namespace ntt { * Now: aux::E at n-1/2 */ ComputeAuxE(dom, gr_getE::D0_B); + timers.stop("FieldSolver"); + + timers.start("FieldBoundaries"); /** * aux::E <- boundary conditions */ FieldBoundaries(dom, BC::D, gr_bc::aux); + timers.stop("FieldBoundaries"); + + timers.start("FieldSolver"); /** * em0::B <- (em0::B) <- -curl aux::E * * Now: em0::B at n */ Faraday(dom, gr_faraday::aux, ONE); - + timers.stop("FieldSolver"); /** * em0::B, em::B <- boundary conditions */ + timers.start("FieldBoundaries"); FieldBoundaries(dom, BC::B, gr_bc::main); + timers.stop("FieldBoundaries"); + timers.start("Communications"); m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); + timers.stop("Communications"); - + timers.start("FieldSolver"); /** * aux::H <- alpha * em0::B - beta x em::D * * Now: aux::H at n */ ComputeAuxH(dom, gr_getH::D_B0); + timers.stop("FieldSolver"); + + timers.start("FieldBoundaries"); /** * aux::H <- boundary conditions */ FieldBoundaries(dom, BC::B, gr_bc::aux); + timers.stop("FieldBoundaries"); } { @@ -366,23 +381,29 @@ namespace ntt { } if (fieldsolver_enabled) { + timers.start("FieldSolver"); /** * cur::J <- (cur0::J + cur::J) / 2 * * Now: cur::J at n */ TimeAverageJ(dom); - /** * aux::Е <- alpha * em::D + beta x em0::B * * Now: aux::Е at n */ ComputeAuxE(dom, gr_getE::D_B0); + timers.stop("FieldSolver"); + + timers.start("FieldBoundaries"); /** * aux::Е <- boundary conditions */ FieldBoundaries(dom, BC::D, gr_bc::aux); + timers.stop("FieldBoundaries"); + + timers.start("FieldSolver"); /** * em0::B <- (em::B) <- -curl aux::E * @@ -390,48 +411,65 @@ namespace ntt { * em::B at n-1/2 */ Faraday(dom, gr_faraday::main, ONE); - + timers.stop("FieldSolver"); + /** * em0::B, em::B <- boundary conditions */ + timers.start("Communications"); m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); + timers.stop("Communications"); + timers.start("FieldBoundaries"); FieldBoundaries(dom, BC::B, gr_bc::main); + timers.stop("FieldBoundaries"); - + timers.start("FieldSolver"); /** * em0::D <- (em0::D) <- curl aux::H * * Now: em0::D at n+1/2 */ Ampere(dom, gr_ampere::aux, ONE); - + timers.stop("FieldSolver"); if (deposit_enabled) { + timers.start("FieldSolver"); /** * em0::D <- (em0::D) <- cur::J * * Now: em0::D at n+1/2 */ AmpereCurrents(dom, gr_ampere::aux); + timers.stop("FieldSolver"); } - + /** * em0::D, em::D <- boundary conditions */ + timers.start("Communications"); m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + timers.stop("Communications"); + timers.start("FieldBoundaries"); FieldBoundaries(dom, BC::D, gr_bc::main); - + timers.stop("FieldBoundaries"); + timers.start("FieldSolver"); /** * aux::H <- alpha * em0::B - beta x em0::D * * Now: aux::H at n+1/2 */ ComputeAuxH(dom, gr_getH::D0_B0); + timers.stop("FieldSolver"); + + timers.start("FieldBoundaries"); /** * aux::H <- boundary conditions */ FieldBoundaries(dom, BC::B, gr_bc::aux); + timers.stop("FieldBoundaries"); + + timers.start("FieldSolver"); /** * em0::D <- (em::D) <- curl aux::H * @@ -439,27 +477,36 @@ namespace ntt { * em::D at n */ Ampere(dom, gr_ampere::main, ONE); + timers.stop("FieldSolver"); + if (deposit_enabled) { + timers.start("FieldSolver"); /** * em0::D <- (em0::D) <- cur0::J * * Now: em0::D at n+1 */ AmpereCurrents(dom, gr_ampere::main); + timers.stop("FieldSolver"); } + timers.start("FieldSolver"); /** * em::D <-> em0::D * em::B <-> em0::B * cur::J <-> cur0::J */ SwapFields(dom); - + timers.stop("FieldSolver"); /** * em0::D, em::D <- boundary conditions */ + timers.start("Communications"); m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + timers.stop("Communications"); + timers.start("FieldBoundaries"); FieldBoundaries(dom, BC::D, gr_bc::main); + timers.stop("FieldBoundaries"); } /** * Finally: em0::B at n-1/2 From 77b162f697edb8fc9bfc0f176a027a1622be9824 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 1 Oct 2024 14:00:57 -0400 Subject: [PATCH 139/234] parameters: changed to requiring n=2 for GRPIC --- src/framework/parameters.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/framework/parameters.cpp b/src/framework/parameters.cpp index 4a9b3056a..f86b5042f 100644 --- a/src/framework/parameters.cpp +++ b/src/framework/parameters.cpp @@ -686,10 +686,10 @@ namespace ntt { prtl_bc_enum.push_back({ PrtlBC::PERIODIC, PrtlBC::PERIODIC }); } } else { - raise::ErrorIf(flds_bc[0].size() != 1, + raise::ErrorIf(flds_bc[0].size() != 2, "invalid `grid.boundaries.fields`", HERE); - raise::ErrorIf(prtl_bc[0].size() != 1, + raise::ErrorIf(prtl_bc[0].size() != 2, "invalid `grid.boundaries.particles`", HERE); flds_bc_enum.push_back( From 717672742db6d44645c9074d7c27149e815c7a32 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 1 Oct 2024 15:27:17 -0400 Subject: [PATCH 140/234] minor: fixed description for qkerr metric for h_tilde --- src/metrics/qkerr_schild.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/metrics/qkerr_schild.h b/src/metrics/qkerr_schild.h index d531b8b3b..85507c6e1 100644 --- a/src/metrics/qkerr_schild.h +++ b/src/metrics/qkerr_schild.h @@ -265,7 +265,7 @@ namespace metric { } /** - * sqrt(det(h_ij)) + * sqrt(det(h_ij)) divided by sin(theta). * @param x coordinate array in code units */ Inline auto sqrt_det_h_tilde(const coord_t& x) const -> real_t { From df2a47dc6b8e9c8a6194f7152f9a3e32751dc65f Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 1 Oct 2024 17:37:59 -0400 Subject: [PATCH 141/234] parameters: restored !=1 --- src/framework/parameters.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/framework/parameters.cpp b/src/framework/parameters.cpp index f86b5042f..4a9b3056a 100644 --- a/src/framework/parameters.cpp +++ b/src/framework/parameters.cpp @@ -686,10 +686,10 @@ namespace ntt { prtl_bc_enum.push_back({ PrtlBC::PERIODIC, PrtlBC::PERIODIC }); } } else { - raise::ErrorIf(flds_bc[0].size() != 2, + raise::ErrorIf(flds_bc[0].size() != 1, "invalid `grid.boundaries.fields`", HERE); - raise::ErrorIf(prtl_bc[0].size() != 2, + raise::ErrorIf(prtl_bc[0].size() != 1, "invalid `grid.boundaries.particles`", HERE); flds_bc_enum.push_back( From baafca9853357fdf6069ce98a7288ebb58e32194 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 2 Oct 2024 16:42:43 -0400 Subject: [PATCH 142/234] archetypes/field_setter: fixed wrong conversion of theta and call for d() [instead of b] in 3D grpic --- src/archetypes/field_setter.h | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/archetypes/field_setter.h b/src/archetypes/field_setter.h index 281c28df6..171e9f8a8 100644 --- a/src/archetypes/field_setter.h +++ b/src/archetypes/field_setter.h @@ -202,13 +202,13 @@ namespace arch { const real_t x1_0 { metric.template convert<1, Crd::Cd, Crd::Ph>(i1_) }; const real_t x1_H { metric.template convert<1, Crd::Cd, Crd::Ph>( i1_ + HALF) }; - const real_t x2_0 { metric.template convert<1, Crd::Cd, Crd::Ph>(i2_) }; - const real_t x2_H { metric.template convert<1, Crd::Cd, Crd::Ph>( + const real_t x2_0 { metric.template convert<2, Crd::Cd, Crd::Ph>(i2_) }; + const real_t x2_H { metric.template convert<2, Crd::Cd, Crd::Ph>( i2_ + HALF) }; { // bx1 - vec_t b_PU { finit.dx1({ x1_0, x2_H }), - finit.dx2({ x1_0, x2_H }), - finit.dx3({ x1_0, x2_H }) }; + vec_t b_PU { finit.bx1({ x1_0, x2_H }), + finit.bx2({ x1_0, x2_H }), + finit.bx3({ x1_0, x2_H }) }; vec_t b_U { ZERO }; metric.template transform({ i1_, i2_ + HALF }, b_PU, @@ -216,9 +216,9 @@ namespace arch { EM(i1, i2, em::bx1) = b_U[0]; } { // bx2 - vec_t b_PU { finit.dx1({ x1_H, x2_0 }), - finit.dx2({ x1_H, x2_0 }), - finit.dx3({ x1_H, x2_0 }) }; + vec_t b_PU { finit.bx1({ x1_H, x2_0 }), + finit.bx2({ x1_H, x2_0 }), + finit.bx3({ x1_H, x2_0 }) }; vec_t b_U { ZERO }; metric.template transform({ i1_ + HALF, i2_ }, b_PU, @@ -226,9 +226,9 @@ namespace arch { EM(i1, i2, em::bx2) = b_U[1]; } { // bx3 - vec_t b_PU { finit.dx1({ x1_H, x2_H }), - finit.dx2({ x1_H, x2_H }), - finit.dx3({ x1_H, x2_H }) }; + vec_t b_PU { finit.bx1({ x1_H, x2_H }), + finit.bx2({ x1_H, x2_H }), + finit.bx3({ x1_H, x2_H }) }; vec_t b_U { ZERO }; metric.template transform({ i1_ + HALF, i2_ + HALF }, b_PU, From c660f8a7cd4b1667cfe8251d547311a6ab91a4b0 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:14:11 -0400 Subject: [PATCH 143/234] kernels/fields_bc: fixed if statement for open BC --- src/kernels/fields_bcs.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 7c574f058..50fc27dd5 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -866,7 +866,8 @@ namespace kernel::bc { Fld(i1_min - 1, i2, em::ex2) = Fld(i1_min, i2, em::ex2); Fld(i1_min, i2, em::ex3) = Fld(i1_min + 1, i2, em::ex3); Fld(i1_min - 1, i2, em::ex3) = Fld(i1_min, i2, em::ex3); - } else if (setB) { + } + if (setB) { Fld(i1_min, i2, em::bx1) = Fld(i1_min + 1, i2, em::bx1); Fld(i1_min - 1, i2, em::bx1) = Fld(i1_min, i2, em::bx1); Fld(i1_min - 1, i2, em::bx2) = Fld(i1_min, i2, em::bx2); From 896c7f7bfccfe181985ccc3b3540db4b1f473639 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:46:55 -0400 Subject: [PATCH 144/234] kernels/fields_bc: added a separate kernel for aux fields --- src/kernels/fields_bcs.hpp | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 50fc27dd5..59cbbd76a 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -881,6 +881,38 @@ namespace kernel::bc { } }; + template + struct OpenBoundariesAux_kernel { + ndfield_t Fld; + const std::size_t i1_min; + const bool setE, setB; + + OpenBoundariesAux_kernel(ndfield_t Fld, std::size_t i1_min, BCTags tags) + : Fld { Fld } + , i1_min { i1_min } + , setE { tags & BC::Ex1 or tags & BC::Ex2 or tags & BC::Ex3 } + , setB { tags & BC::Bx1 or tags & BC::Bx2 or tags & BC::Bx3 } {} + + Inline void operator()(index_t i2) const { + if constexpr (M::Dim == Dim::_2D) { + if (setE) { + Fld(i1_min - 1, i2, em::ex1) = Fld(i1_min, i2, em::ex1); + Fld(i1_min - 1, i2, em::ex2) = Fld(i1_min, i2, em::ex2); + Fld(i1_min - 1, i2, em::ex3) = Fld(i1_min, i2, em::ex3); + } + if (setB) { + Fld(i1_min - 1, i2, em::bx1) = Fld(i1_min, i2, em::bx1); + Fld(i1_min - 1, i2, em::bx2) = Fld(i1_min, i2, em::bx2); + Fld(i1_min - 1, i2, em::bx3) = Fld(i1_min, i2, em::bx3); + } + } else { + raise::KernelError( + HERE, + "AbsorbFields_kernel: 2D implementation called for D != 2"); + } + } + }; + } // namespace kernel::bc #endif // KERNELS_FIELDS_BCS_HPP From cfa70037303f64f58a9be6b02f5e2fa3e0ca93ea Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:47:57 -0400 Subject: [PATCH 145/234] see below --- src/engines/grpic.hpp | 30 ++++++------------------------ 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index c6e7b657f..97aea35f7 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -615,27 +615,6 @@ namespace ntt { xg_edge, ds, tags)); - } else if (dim == in::x2) { - if constexpr (M::Dim == Dim::_2D) { - Kokkos::parallel_for( - "AbsorbFields", - CreateRangePolicy(range_min, range_max), - kernel::AbsorbBoundaries_kernel(domain.fields.em, - domain.mesh.metric, - xg_edge, - ds, - tags)); - Kokkos::parallel_for( - "AbsorbFields", - CreateRangePolicy(range_min, range_max), - kernel::AbsorbBoundaries_kernel(domain.fields.em0, - domain.mesh.metric, - xg_edge, - ds, - tags)); - } else { - raise::Error("Invalid dimension", HERE); - } } else { raise::Error("Invalid dimension", HERE); } @@ -655,10 +634,13 @@ namespace ntt { "Invalid axis direction, should be x2", HERE); const auto i1_min = domain.mesh.i_min(in::x1); + auto range = CreateRangePolicy( + {domain.mesh.i_min(in::x2)}, + {domain.mesh.i_max(in::x2) + 1}); if (g == gr_bc::main) { Kokkos::parallel_for( "OpenBCFields", - domain.mesh.n_all(in::x2), + range, kernel::OpenBoundaries_kernel(domain.fields.em, i1_min, tags)); Kokkos::parallel_for( "OpenBCFields", @@ -667,8 +649,8 @@ namespace ntt { } else if (g == gr_bc::aux) { Kokkos::parallel_for( "OpenBCFields", - domain.mesh.n_all(in::x2), - kernel::OpenBoundaries_kernel(domain.fields.aux, i1_min, tags)); + range, + kernel::OpenBoundariesAux_kernel(domain.fields.aux, i1_min, tags)); } } From e2e91d20f1aa0608f7a89512193d49384d7018d5 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:51:13 -0400 Subject: [PATCH 146/234] engines/grpic: also changed range for axis BC in this commit and commit below --- src/engines/grpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 97aea35f7..cce672747 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -644,7 +644,7 @@ namespace ntt { kernel::OpenBoundaries_kernel(domain.fields.em, i1_min, tags)); Kokkos::parallel_for( "OpenBCFields", - domain.mesh.n_all(in::x2), + range, kernel::OpenBoundaries_kernel(domain.fields.em0, i1_min, tags)); } else if (g == gr_bc::aux) { Kokkos::parallel_for( From f277b2ad6a9598c2234e221453729efc4253406b Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 2 Oct 2024 18:04:58 -0400 Subject: [PATCH 147/234] engines/grpic: changed range for axis BC --- src/engines/grpic.hpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index cce672747..cb1ec1e85 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -668,23 +668,26 @@ namespace ntt { HERE); const auto i2_min = domain.mesh.i_min(in::x2); const auto i2_max = domain.mesh.i_max(in::x2); + auto range = CreateRangePolicy( + {domain.mesh.i_min(in::x1) - 1}, + {domain.mesh.i_max(in::x1)}); if (direction.get_sign() < 0) { Kokkos::parallel_for( "AxisBCFields", - domain.mesh.n_all(in::x1), + range, kernel::AxisBoundaries_kernel(domain.fields.em, i2_min, tags)); Kokkos::parallel_for( "AxisBCFields", - domain.mesh.n_all(in::x1), + range, kernel::AxisBoundaries_kernel(domain.fields.em0, i2_min, tags)); } else { Kokkos::parallel_for( "AxisBCFields", - domain.mesh.n_all(in::x1), + range, kernel::AxisBoundaries_kernel(domain.fields.em, i2_max, tags)); Kokkos::parallel_for( "AxisBCFields", - domain.mesh.n_all(in::x1), + range, kernel::AxisBoundaries_kernel(domain.fields.em0, i2_max, tags)); } } From fce3b6ea44b1c2bef79e97808282138547c7bc69 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Thu, 3 Oct 2024 18:30:39 -0400 Subject: [PATCH 148/234] updated pgen for wald --- setups/grpic/wald/pgen.hpp | 47 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/setups/grpic/wald/pgen.hpp b/setups/grpic/wald/pgen.hpp index 3374a12d7..efb9c78e7 100644 --- a/setups/grpic/wald/pgen.hpp +++ b/setups/grpic/wald/pgen.hpp @@ -4,13 +4,51 @@ #include "enums.h" #include "global.h" +#include "arch/kokkos_aliases.h" #include "arch/traits.h" #include "archetypes/problem_generator.h" +#include "framework/domain/metadomain.h" namespace user { using namespace ntt; + template + struct InitFields { + InitFields() {} + + Inline auto VerticalPotential(const coord_t& x_Ph) const -> real_t { + return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); + } + + Inline auto bx1(const coord_t& x_Ph) const -> real_t { + return x_Ph[0] * math::cos(x_Ph[1]); + } + + Inline auto bx2(const coord_t& x_Ph) const -> real_t { + return -x_Ph[0] * math::sin(x_Ph[1]); + } + + Inline auto bx3(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx1(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx2(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx3(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + private: + // const real_t Bsurf, Rstar; + }; + template struct PGen : public arch::ProblemGenerator { // compatibility traits for the problem generator @@ -24,13 +62,18 @@ namespace user { using arch::ProblemGenerator::D; using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; + + InitFields init_flds; - inline PGen(SimulationParams& p, const Metadomain&) : - arch::ProblemGenerator(p) {} + inline PGen(SimulationParams& p, const Metadomain& m) : + arch::ProblemGenerator(p) + // , init_flds { } + {} inline PGen() {} }; + } // namespace user #endif From b1e4146fde3686e8132f86e28bbea39aebe99c93 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Thu, 3 Oct 2024 19:12:38 -0400 Subject: [PATCH 149/234] engines/grpic: adjusted ranges for all kernels --- src/engines/grpic.hpp | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index cb1ec1e85..78aac20af 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -675,20 +675,20 @@ namespace ntt { Kokkos::parallel_for( "AxisBCFields", range, - kernel::AxisBoundaries_kernel(domain.fields.em, i2_min, tags)); + kernel::AxisBoundariesGR_kernel(domain.fields.em, i2_min, tags)); Kokkos::parallel_for( "AxisBCFields", range, - kernel::AxisBoundaries_kernel(domain.fields.em0, i2_min, tags)); + kernel::AxisBoundariesGR_kernel(domain.fields.em0, i2_min, tags)); } else { Kokkos::parallel_for( "AxisBCFields", range, - kernel::AxisBoundaries_kernel(domain.fields.em, i2_max, tags)); + kernel::AxisBoundariesGR_kernel(domain.fields.em, i2_max, tags)); Kokkos::parallel_for( "AxisBCFields", range, - kernel::AxisBoundaries_kernel(domain.fields.em0, i2_max, tags)); + kernel::AxisBoundariesGR_kernel(domain.fields.em0, i2_max, tags)); } } @@ -826,8 +826,8 @@ namespace ntt { "algorithms.timestep.correction") * dt; auto range = CreateRangePolicy( - { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2)}, - { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1}); + { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) + 1}, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2)}); const auto ni2 = domain.mesh.n_active(in::x2); if (g == gr_ampere::aux) { @@ -874,7 +874,10 @@ namespace ntt { const auto n0 = m_params.template get("scales.n0"); const auto B0 = m_params.template get("scales.B0"); const auto coeff = -dt * q0 * n0 / B0; - auto range = range_with_axis_BCs(domain); + // auto range = range_with_axis_BCs(domain); + auto range = CreateRangePolicy( + { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) + 1}, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2)}); const auto ni2 = domain.mesh.n_active(in::x2); if (g == gr_ampere::aux) { @@ -904,18 +907,16 @@ namespace ntt { } void TimeAverageDB(domain_t& domain) { - auto range = range_with_axis_BCs(domain); Kokkos::parallel_for("TimeAverageDB", - range, + domain.mesh.rangeActiveCells(), kernel::gr::TimeAverageDB_kernel(domain.fields.em, domain.fields.em0, domain.mesh.metric)); } void TimeAverageJ(domain_t& domain) { - auto range = range_with_axis_BCs(domain); Kokkos::parallel_for("TimeAverageJ", - range, + domain.mesh.rangeActiveCells(), kernel::gr::TimeAverageJ_kernel(domain.fields.cur, domain.fields.cur0, domain.mesh.metric)); From c2b00e05c5d272e9ba236d2679dc935a1034e24a Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Thu, 3 Oct 2024 19:13:28 -0400 Subject: [PATCH 150/234] minor changes to kernels/fields_bc gr-specific --- src/kernels/fields_bcs.hpp | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 59cbbd76a..9df53e3e7 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -540,6 +540,29 @@ namespace kernel::bc { * * @brief Applies enforced boundary conditions (fixed value) */ + template + struct AxisBoundariesGR_kernel { + ndfield_t Fld; + const std::size_t i_edge; + const bool setE, setB; + + AxisBoundariesGR_kernel(ndfield_t Fld, std::size_t i_edge, BCTags tags) + : Fld { Fld } + , i_edge { i_edge } + , setE { tags & BC::Ex1 or tags & BC::Ex2 or tags & BC::Ex3 } + , setB { tags & BC::Bx1 or tags & BC::Bx2 or tags & BC::Bx3 } {} + + Inline void operator()(index_t i1) const { + if constexpr (D == Dim::_2D) { + if (setB) { + Fld(i1, i_edge, em::bx2) = ZERO; + } + } else { + raise::KernelError(HERE, "AxisBoundaries_kernel: D != 2"); + } + } + }; + template struct EnforcedBoundaries_kernel { static constexpr Dimension D = M::Dim; From 693d67cf8adb616308fd5a5a728313e28873362d Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 4 Oct 2024 14:50:26 -0400 Subject: [PATCH 151/234] wald: vertical potential setup --- setups/grpic/wald/pgen.hpp | 44 +++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/setups/grpic/wald/pgen.hpp b/setups/grpic/wald/pgen.hpp index efb9c78e7..a5f8f503e 100644 --- a/setups/grpic/wald/pgen.hpp +++ b/setups/grpic/wald/pgen.hpp @@ -13,20 +13,40 @@ namespace user { using namespace ntt; - template + template struct InitFields { - InitFields() {} + InitFields(M metric_) : metric { metric_ } {} - Inline auto VerticalPotential(const coord_t& x_Ph) const -> real_t { + Inline auto VerticalPotential(const coord_t& x_Cd) const -> real_t { + coord_t x_Ph { ZERO }; + metric.template convert(x_Cd, x_Ph); return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); } Inline auto bx1(const coord_t& x_Ph) const -> real_t { - return x_Ph[0] * math::cos(x_Ph[1]); + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0]; + x0m[1] = xi[1] - HALF; + x0p[0] = xi[0]; + x0p[1] = xi[1] + HALF; + + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + return (VerticalPotential(x0p) - VerticalPotential(x0m)) * inv_sqrt_detH_ijP; } Inline auto bx2(const coord_t& x_Ph) const -> real_t { - return -x_Ph[0] * math::sin(x_Ph[1]); + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0] + HALF - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF + HALF; + x0p[1] = xi[1]; + + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + return -(VerticalPotential(x0p) - VerticalPotential(x0m)) * inv_sqrt_detH_ijP; } Inline auto bx3(const coord_t& x_Ph) const -> real_t { @@ -46,7 +66,7 @@ namespace user { } private: - // const real_t Bsurf, Rstar; + const M metric; }; template @@ -62,18 +82,16 @@ namespace user { using arch::ProblemGenerator::D; using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; - - InitFields init_flds; - inline PGen(SimulationParams& p, const Metadomain& m) : - arch::ProblemGenerator(p) - // , init_flds { } - {} + InitFields init_flds; + + inline PGen(SimulationParams& p, const Metadomain& m) + : arch::ProblemGenerator(p) + , init_flds { m.mesh().metric } {} inline PGen() {} }; - } // namespace user #endif From 7aea0ad1250f51cdbea1c83e1c4896c7bf553611 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 4 Oct 2024 18:25:02 -0400 Subject: [PATCH 152/234] kernel/ampere_gr: bug for theta=0 --- src/kernels/ampere_gr.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/kernels/ampere_gr.hpp b/src/kernels/ampere_gr.hpp index 045eb1729..46b950796 100644 --- a/src/kernels/ampere_gr.hpp +++ b/src/kernels/ampere_gr.hpp @@ -74,6 +74,8 @@ namespace kernel::gr { if ((i2 == i2min) && is_axis_i2min) { // theta = 0 const real_t inv_polar_area_pH { ONE / metric.polar_area(i1_ + HALF) }; + const real_t inv_sqrt_detH_0pH { ONE / + metric.sqrt_det_h({ i1_, HALF }) }; Dout(i1, i2, em::dx1) = Din(i1, i2, em::dx1) + inv_polar_area_pH * coeff * H(i1, i2, em::hx3); Dout(i1, i2, em::dx2) = Din(i1, i2, em::dx2) + From 0d011a1716177a9171340873d62dd493e7f9aa46 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 4 Oct 2024 18:47:18 -0400 Subject: [PATCH 153/234] kernels/faraday_gr: bug at i2min --- src/kernels/faraday_gr.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/kernels/faraday_gr.hpp b/src/kernels/faraday_gr.hpp index 19eede5f2..5f6e5590b 100644 --- a/src/kernels/faraday_gr.hpp +++ b/src/kernels/faraday_gr.hpp @@ -73,7 +73,9 @@ namespace kernel::gr { Bout(i1, i2, em::bx1) = Bin(i1, i2, em::bx1) + coeff * inv_sqrt_detH_0pH * (E(i1, i2, em::ex3) - E(i1, i2 + 1, em::ex3)); - if ((i2 != i2min) || !is_axis_i2min) { + if ((i2 == i2min) && is_axis_i2min) { + Bout(i1, i2, em::bx2) = ZERO; + } else { const real_t inv_sqrt_detH_pH0 { ONE / metric.sqrt_det_h( { i1_ + HALF, i2_ }) }; Bout(i1, i2, em::bx2) = Bin(i1, i2, em::bx2) + From 159543217c244c6107c7bfd84476a4ff1ed60de9 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 4 Oct 2024 19:03:04 -0400 Subject: [PATCH 154/234] kernels/aux: bugs in time averaging --- src/kernels/aux_fields_gr.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/kernels/aux_fields_gr.hpp b/src/kernels/aux_fields_gr.hpp index b56a819f9..c739cffbd 100644 --- a/src/kernels/aux_fields_gr.hpp +++ b/src/kernels/aux_fields_gr.hpp @@ -257,9 +257,9 @@ namespace kernel::gr { BDf0(i1, i2, em::bx1) = HALF * (BDf0(i1, i2, em::bx1) + BDf(i1, i2, em::bx1)); BDf0(i1, i2, em::bx2) = HALF * (BDf0(i1, i2, em::bx2) + BDf(i1, i2, em::bx2)); BDf0(i1, i2, em::bx3) = HALF * (BDf0(i1, i2, em::bx3) + BDf(i1, i2, em::bx3)); - BDf0(i1, i2, em::ex1) = HALF * (BDf0(i1, i2, em::bx1) + BDf(i1, i2, em::ex1)); - BDf0(i1, i2, em::ex2) = HALF * (BDf0(i1, i2, em::bx2) + BDf(i1, i2, em::ex2)); - BDf0(i1, i2, em::ex3) = HALF * (BDf0(i1, i2, em::bx3) + BDf(i1, i2, em::ex3)); + BDf0(i1, i2, em::ex1) = HALF * (BDf0(i1, i2, em::ex1) + BDf(i1, i2, em::ex1)); + BDf0(i1, i2, em::ex2) = HALF * (BDf0(i1, i2, em::ex2) + BDf(i1, i2, em::ex2)); + BDf0(i1, i2, em::ex3) = HALF * (BDf0(i1, i2, em::ex3) + BDf(i1, i2, em::ex3)); } else { raise::KernelError( HERE, @@ -291,9 +291,9 @@ namespace kernel::gr { Inline void operator()(index_t i1, index_t i2) const { if constexpr (D == Dim::_2D) { - Jf0(i1, i2, cur::jx1) = HALF * (Jf0(i1, i2, cur::jx1) + Jf(i1, i2, cur::jx1)); - Jf0(i1, i2, cur::jx2) = HALF * (Jf0(i1, i2, cur::jx2) + Jf(i1, i2, cur::jx2)); - Jf0(i1, i2, cur::jx3) = HALF * (Jf0(i1, i2, cur::jx3) + Jf(i1, i2, cur::jx3)); + Jf(i1, i2, cur::jx1) = HALF * (Jf0(i1, i2, cur::jx1) + Jf(i1, i2, cur::jx1)); + Jf(i1, i2, cur::jx2) = HALF * (Jf0(i1, i2, cur::jx2) + Jf(i1, i2, cur::jx2)); + Jf(i1, i2, cur::jx3) = HALF * (Jf0(i1, i2, cur::jx3) + Jf(i1, i2, cur::jx3)); } else { raise::KernelError( HERE, From c15b6cfb4dfe4cfd6be9ea7084a883fd88a8c06e Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 4 Oct 2024 20:13:52 -0400 Subject: [PATCH 155/234] metric: kerr-schild a=0 minor fixes --- src/metrics/kerr_schild_0.h | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/metrics/kerr_schild_0.h b/src/metrics/kerr_schild_0.h index 70689f4f0..31080ed4c 100644 --- a/src/metrics/kerr_schild_0.h +++ b/src/metrics/kerr_schild_0.h @@ -36,6 +36,7 @@ namespace metric { private: const real_t dr, dtheta, dphi; const real_t dr_inv, dtheta_inv, dphi_inv; + const real_t a, rg_, rh_; public: static constexpr const char* Label { "kerr_schild_0" }; @@ -57,6 +58,9 @@ namespace metric { boundaries_t ext, const std::map& = {}) : MetricBase { res, ext } + , a { ZERO } + , rg_ { ONE } + , rh_ { TWO } , dr { (x1_max - x1_min) / nx1 } , dtheta { (x2_max - x2_min) / nx2 } , dphi { (x3_max - x3_min) / nx3 } @@ -70,17 +74,17 @@ namespace metric { [[nodiscard]] Inline auto spin() const -> const real_t& { - return ZERO; + return a; } [[nodiscard]] Inline auto rhorizon() const -> const real_t& { - return ZERO; + return rh_; } [[nodiscard]] Inline auto rg() const -> const real_t& { - return ZERO; + return rg_; } /** From 5a81f695280b9458942b08efddfa097bc3cd1440 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 18 Oct 2024 17:15:06 -0400 Subject: [PATCH 156/234] minor kernels/aux_gr --- src/kernels/aux_fields_gr.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kernels/aux_fields_gr.hpp b/src/kernels/aux_fields_gr.hpp index c739cffbd..0983b6b7e 100644 --- a/src/kernels/aux_fields_gr.hpp +++ b/src/kernels/aux_fields_gr.hpp @@ -277,8 +277,8 @@ namespace kernel::gr { static_assert(M::is_metric, "M must be a metric class"); static constexpr auto D = M::Dim; - const ndfield_t Jf; - ndfield_t Jf0; + ndfield_t Jf; + const ndfield_t Jf0; const M metric; public: From fb952096c3f7b04b5684dffc8edc6a06a84a48aa Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 18 Oct 2024 17:17:40 -0400 Subject: [PATCH 157/234] engines/grpic: parameters for field boundaries --- src/engines/grpic.hpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 78aac20af..f4cc46b1b 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -124,7 +124,7 @@ namespace ntt { * em0::D, em::D, em0::B, em::B <- boundary conditions */ m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0 | Comm::D | Comm::D0); - FieldBoundaries(dom, BC::B | BC::D, gr_bc::main); + FieldBoundaries(dom, BC::B | BC::E, gr_bc::main); /** * em0::B <- em::B @@ -146,7 +146,7 @@ namespace ntt { /** * aux::E, aux::H <- boundary conditions */ - FieldBoundaries(dom, BC::B | BC::D, gr_bc::aux); + FieldBoundaries(dom, BC::B | BC::E, gr_bc::aux); /** * em0::B <- (em0::B) <- -curl aux::E @@ -172,7 +172,7 @@ namespace ntt { * em0::D, em::D <- boundary conditions */ m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); - FieldBoundaries(dom, BC::D, gr_bc::main); + FieldBoundaries(dom, BC::E, gr_bc::main); /** * aux::E <- alpha * em::D + beta x em0::B @@ -186,7 +186,7 @@ namespace ntt { /** * aux::E, aux::H <- boundary conditions */ - FieldBoundaries(dom, BC::B | BC::D, gr_bc::aux); + FieldBoundaries(dom, BC::B | BC::E, gr_bc::aux); // !ADD: GR -- particles? @@ -212,7 +212,7 @@ namespace ntt { * em0::D, em::D <- boundary conditions */ m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); - FieldBoundaries(dom, BC::D, gr_bc::main); + FieldBoundaries(dom, BC::E, gr_bc::main); /** * aux::H <- alpha * em0::B - beta x em0::D @@ -236,7 +236,7 @@ namespace ntt { * em0::D, em::D <- boundary conditions */ m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); - FieldBoundaries(dom, BC::D, gr_bc::main); + FieldBoundaries(dom, BC::E, gr_bc::main); /** * em::D <-> em0::D @@ -299,7 +299,7 @@ namespace ntt { /** * aux::E <- boundary conditions */ - FieldBoundaries(dom, BC::D, gr_bc::aux); + FieldBoundaries(dom, BC::E, gr_bc::aux); timers.stop("FieldBoundaries"); timers.start("FieldSolver"); @@ -400,7 +400,7 @@ namespace ntt { /** * aux::Е <- boundary conditions */ - FieldBoundaries(dom, BC::D, gr_bc::aux); + FieldBoundaries(dom, BC::E, gr_bc::aux); timers.stop("FieldBoundaries"); timers.start("FieldSolver"); @@ -450,7 +450,7 @@ namespace ntt { m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); timers.stop("Communications"); timers.start("FieldBoundaries"); - FieldBoundaries(dom, BC::D, gr_bc::main); + FieldBoundaries(dom, BC::E, gr_bc::main); timers.stop("FieldBoundaries"); timers.start("FieldSolver"); @@ -505,7 +505,7 @@ namespace ntt { m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); timers.stop("Communications"); timers.start("FieldBoundaries"); - FieldBoundaries(dom, BC::D, gr_bc::main); + FieldBoundaries(dom, BC::E, gr_bc::main); timers.stop("FieldBoundaries"); } /** @@ -834,7 +834,7 @@ namespace ntt { // First push, updates D0 with J. Kokkos::parallel_for("Ampere-1", range, - kernel::gr::Ampere_kernel(domain.fields.em0, + kernel::gr::Ampere_kernel(domain.fields.em0, // Din, Dout, aux domain.fields.em0, domain.fields.aux, domain.mesh.metric, From ee2bd29754613c057e184932232ad3b6020e767c Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Mon, 21 Oct 2024 19:05:10 -0400 Subject: [PATCH 158/234] minor notes --- src/engines/grpic.hpp | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index f4cc46b1b..c5d99b302 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -731,9 +731,9 @@ namespace ntt { if (g == gr_getE::D0_B) { Kokkos::parallel_for("ComputeAuxE", range, - kernel::gr::ComputeAuxE_kernel(domain.fields.em0, - domain.fields.em, - domain.fields.aux, + kernel::gr::ComputeAuxE_kernel(domain.fields.em0, // D + domain.fields.em, // B + domain.fields.aux, // E domain.mesh.metric)); } else if (g == gr_getE::D_B0) { Kokkos::parallel_for("ComputeAuxE", @@ -752,9 +752,9 @@ namespace ntt { if (g == gr_getH::D_B0) { Kokkos::parallel_for("ComputeAuxH", range, - kernel::gr::ComputeAuxH_kernel(domain.fields.em, - domain.fields.em0, - domain.fields.aux, + kernel::gr::ComputeAuxH_kernel(domain.fields.em, // D + domain.fields.em0, // B + domain.fields.aux, // H domain.mesh.metric)); } else if (g == gr_getH::D0_B0) { Kokkos::parallel_for("ComputeAuxH", @@ -795,9 +795,9 @@ namespace ntt { Kokkos::parallel_for( "Faraday", domain.mesh.rangeActiveCells(), - kernel::gr::Faraday_kernel(domain.fields.em0, - domain.fields.em0, - domain.fields.aux, + kernel::gr::Faraday_kernel(domain.fields.em0, // Bin + domain.fields.em0, // Bout + domain.fields.aux, // E domain.mesh.metric, dT, domain.mesh.n_active(in::x2), @@ -826,16 +826,16 @@ namespace ntt { "algorithms.timestep.correction") * dt; auto range = CreateRangePolicy( - { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) + 1}, - { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2)}); + { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2)}, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1}); const auto ni2 = domain.mesh.n_active(in::x2); if (g == gr_ampere::aux) { // First push, updates D0 with J. Kokkos::parallel_for("Ampere-1", range, - kernel::gr::Ampere_kernel(domain.fields.em0, // Din, Dout, aux - domain.fields.em0, + kernel::gr::Ampere_kernel(domain.fields.em0, // Din + domain.fields.em0, // Dout domain.fields.aux, domain.mesh.metric, dT, From dd7277c7c63bf8f8d408cc9de8538ba1409cd23a Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 23 Oct 2024 17:27:09 -0400 Subject: [PATCH 159/234] implemented absorbing boundaries for gr --- src/engines/grpic.hpp | 25 ++++------- src/kernels/fields_bcs.hpp | 92 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 16 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index c5d99b302..375916c41 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -563,25 +563,16 @@ namespace ntt { const auto dim = direction.get_dim(); real_t xg_min, xg_max, xg_edge; auto sign = direction.get_sign(); - if (sign > 0) { // + direction - xg_max = m_metadomain.mesh().extent(dim).second; - xg_min = xg_max - ds; - xg_edge = xg_max; - } else { // - direction - xg_min = m_metadomain.mesh().extent(dim).first; - xg_max = xg_min + ds; - xg_edge = xg_min; - } + xg_max = m_metadomain.mesh().extent(dim).second; + xg_min = xg_max - ds; + xg_edge = xg_max; + boundaries_t box; boundaries_t incl_ghosts; for (unsigned short d { 0 }; d < M::Dim; ++d) { if (d == static_cast(dim)) { box.push_back({ xg_min, xg_max }); - if (sign > 0) { - incl_ghosts.push_back({ false, true }); - } else { - incl_ghosts.push_back({ true, false }); - } + incl_ghosts.push_back({ false, true }); } else { box.push_back(Range::All); incl_ghosts.push_back({ true, true }); @@ -602,7 +593,8 @@ namespace ntt { Kokkos::parallel_for( "AbsorbFields", CreateRangePolicy(range_min, range_max), - kernel::AbsorbBoundaries_kernel(domain.fields.em, + kernel::AbsorbBoundariesGR_kernel(domain.fields.em, + m_pgen.init_flds, domain.mesh.metric, xg_edge, ds, @@ -610,7 +602,8 @@ namespace ntt { Kokkos::parallel_for( "AbsorbFields", CreateRangePolicy(range_min, range_max), - kernel::AbsorbBoundaries_kernel(domain.fields.em0, + kernel::AbsorbBoundariesGR_kernel(domain.fields.em0, + m_pgen.init_flds, domain.mesh.metric, xg_edge, ds, diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 9df53e3e7..9d9db5488 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -936,6 +936,98 @@ namespace kernel::bc { } }; + template + struct AbsorbBoundariesGR_kernel { + static_assert(M::is_metric, "M must be a metric class"); + static_assert(i <= static_cast(M::Dim), + "Invalid component index"); + + ndfield_t Fld; + const I finit; + const M metric; + const real_t xg_edge; + const real_t dx_abs; + const BCTags tags; + + AbsorbBoundariesGR_kernel(ndfield_t Fld, + const I& finit, + const M& metric, + real_t xg_edge, + real_t dx_abs, + BCTags tags) + : Fld { Fld } + , finit { finit } + , metric { metric } + , xg_edge { xg_edge } + , dx_abs { dx_abs } + , tags { tags } {} + + Inline void operator()(index_t i1, index_t i2) const { + if constexpr (M::Dim == Dim::_2D) { + const auto i1_ = COORD(i1); + const auto i2_ = COORD(i2); + for (const auto comp : + { em::ex1, em::ex2, em::ex3, em::bx1, em::bx2, em::bx3 }) { + if ((comp == em::ex1) and not(tags & BC::Ex1)) { + continue; + } else if ((comp == em::ex2) and not(tags & BC::Ex2)) { + continue; + } else if ((comp == em::ex3) and not(tags & BC::Ex3)) { + continue; + } else if ((comp == em::bx1) and not(tags & BC::Bx1)) { + continue; + } else if ((comp == em::bx2) and not(tags & BC::Bx2)) { + continue; + } else if ((comp == em::bx3) and not(tags & BC::Bx3)) { + continue; + } + coord_t x_Cd { ZERO }; + if (comp == em::ex1 or comp == em::bx2 or comp == em::bx3) { + x_Cd[0] = i1_ + HALF; + x_Cd[1] = i2_; + } else if (comp == em::ex2 or comp == em::ex3 or comp == em::bx1) { + x_Cd[0] = i1_; + x_Cd[1] = i2_; + } + + const auto dx = math::abs( + metric.template convert(x_Cd[i - 1]) - xg_edge); + Fld(i1, i2, comp) *= math::tanh(dx / (INV_4 * dx_abs)); + + if (comp == em::bx1) { + const real_t x1_0 { metric.template convert<1, Crd::Cd, Crd::Ph>(i1_) }; + const real_t x2_H { metric.template convert<2, Crd::Cd, Crd::Ph>( + i2_ + HALF) }; + + vec_t b_PU { finit.bx1({ x1_0, x2_H }), + finit.bx2({ x1_0, x2_H }), + finit.bx3({ x1_0, x2_H }) }; + vec_t b_U { ZERO }; + metric.template transform({ i1_, i2_ + HALF }, b_PU, b_U); + Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * + b_U[0]; + } else if (comp == em::bx2) { + const real_t x1_H { metric.template convert<1, Crd::Cd, Crd::Ph>( + i1_ + HALF) }; + const real_t x2_0 { metric.template convert<2, Crd::Cd, Crd::Ph>(i2_) }; + + vec_t b_PU { finit.bx1({ x1_H, x2_0 }), + finit.bx2({ x1_H, x2_0 }), + finit.bx3({ x1_H, x2_0 }) }; + vec_t b_U { ZERO }; + metric.template transform({ i1_ + HALF, i2_ }, b_PU, b_U); + Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * + b_U[1]; + } + } + } else { + raise::KernelError( + HERE, + "AbsorbFields_kernel: 2D implementation called for D != 2"); + } + } + }; + } // namespace kernel::bc #endif // KERNELS_FIELDS_BCS_HPP From 7aad0cc7a85dec44fe72d4db8906bcfd32105b8f Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Thu, 31 Oct 2024 15:49:40 -0400 Subject: [PATCH 160/234] working EM solver fully implemented --- setups/grpic/wald/pgen.hpp | 41 +++++++++++++++++++-- setups/grpic/wald/wald.toml | 72 +++++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 3 deletions(-) create mode 100644 setups/grpic/wald/wald.toml diff --git a/setups/grpic/wald/pgen.hpp b/setups/grpic/wald/pgen.hpp index a5f8f503e..51636314b 100644 --- a/setups/grpic/wald/pgen.hpp +++ b/setups/grpic/wald/pgen.hpp @@ -17,12 +17,30 @@ namespace user { struct InitFields { InitFields(M metric_) : metric { metric_ } {} - Inline auto VerticalPotential(const coord_t& x_Cd) const -> real_t { + Inline auto A_3(const coord_t& x_Cd) const -> real_t { + // return HALF * (metric.template h_<3, 3>(x_Cd) ); + // + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + // ); + coord_t x_Ph { ZERO }; metric.template convert(x_Cd, x_Ph); return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); } + Inline auto A_1(const coord_t& x_Cd) const -> real_t { + return HALF * (metric.template h_<1, 3>(x_Cd) + + TWO * metric.spin() * metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) + ); + } + + Inline auto A_0(const coord_t& x_Cd) const -> real_t { + real_t g_00 { -metric.alpha(x_Cd) * metric.alpha(x_Cd) + + metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) * metric.beta1(x_Cd) + }; + return HALF * (metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + + TWO * metric.spin() * g_00); + } + Inline auto bx1(const coord_t& x_Ph) const -> real_t { coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); @@ -33,7 +51,11 @@ namespace user { x0p[1] = xi[1] + HALF; real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; - return (VerticalPotential(x0p) - VerticalPotential(x0m)) * inv_sqrt_detH_ijP; + + if (cmp::AlmostZero(x_Ph[1])) + return ONE; + else + return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; } Inline auto bx2(const coord_t& x_Ph) const -> real_t { @@ -46,10 +68,23 @@ namespace user { x0p[1] = xi[1]; real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; - return -(VerticalPotential(x0p) - VerticalPotential(x0m)) * inv_sqrt_detH_ijP; + if (cmp::AlmostZero(x_Ph[1])) + return ZERO; + else + return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; } Inline auto bx3(const coord_t& x_Ph) const -> real_t { + // coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + // metric.template convert(x_Ph, xi); + + // x0m[0] = xi[0] + HALF - HALF; + // x0m[1] = xi[1]; + // x0p[0] = xi[0] + HALF + HALF; + // x0p[1] = xi[1]; + + // real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + // return -(A_1(x0p) - A_1(x0m)) * inv_sqrt_detH_ijP; return ZERO; } diff --git a/setups/grpic/wald/wald.toml b/setups/grpic/wald/wald.toml new file mode 100644 index 000000000..36c746ae5 --- /dev/null +++ b/setups/grpic/wald/wald.toml @@ -0,0 +1,72 @@ +[simulation] + name = "wald" + engine = "grpic" + runtime = 500.0 + +[grid] + resolution = [128,128] + extent = [[1.2, 8.0]] + + [grid.metric] + metric = "qkerr_schild" + qsph_r0 = 0.0 + qsph_h = 0.0 + ks_a = 0.95 + + [grid.boundaries] + fields = [["ABSORB"]] + particles = [["ABSORB"]] + + [grid.boundaries.absorb] + ds = 0.5 + +[scales] + larmor0 = 1.0 + skindepth0 = 1.0 + +[algorithms] + current_filters = 4 + + [algorithms.timestep] + CFL = 0.5 + + [algorithms.toggles] + deposit = false + fieldsolver = true + +[particles] + ppc0 = 8.0 + use_weights = true + sort_interval = 100 + +# [[particles.species]] +# label = "e-" +# mass = 1.0 +# charge = -1.0 +# maxnpart = 2e6 +# pusher = "Boris" +# +# [[particles.species]] +# label = "e+" +# mass = 1.0 +# charge = 1.0 +# maxnpart = 2e6 +# pusher = "Boris" + +[setup] + +[output] + format = "hdf5" + + [output.fields] + interval_time = 1.0 + quantities = ["D", "H", "B", "J", "A"] + + [output.particles] + enable = false + + [output.spectra] + enable = false + +[diagnostics] + interval = 1 From e7615254fe962e55ba12f64e670854e3b41f341f Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 5 Nov 2024 17:52:11 -0500 Subject: [PATCH 161/234] compilible pusher in gr --- src/engines/grpic.hpp | 41 ++++++++++++++++++-- src/kernels/particle_pusher_gr.hpp | 61 +++++++++++++++--------------- 2 files changed, 69 insertions(+), 33 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 375916c41..333742eaf 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -345,7 +345,7 @@ namespace ntt { * Now: x_prtl at n + 1, u_prtl at n + 1/2 */ timers.start("ParticlePusher"); - // ParticlePush(dom); + ParticlePush(dom); timers.stop("ParticlePusher"); /** @@ -1005,10 +1005,16 @@ namespace ntt { // coeff = q / m (dt / 2) omegaB0 const auto coeff = q_ovr_m * HALF * dt * m_params.template get("scales.omegaB0"); + // clang-format off + + if (species.pusher() == PrtlPusher::PHOTON) { + auto range_policy = Kokkos::RangePolicy( + 0, + species.npart()); Kokkos::parallel_for( "ParticlePusher", - species.rangeActiveParticles(), + range_policy, kernel::gr::Pusher_kernel( domain.fields.em, domain.fields.em0, @@ -1023,9 +1029,38 @@ namespace ntt { domain.mesh.n_active(in::x1), domain.mesh.n_active(in::x2), domain.mesh.n_active(in::x3), - m_params.template get("algorithms.gr.pusher_eps"), m_params.template get("algorithms.gr.pusher_niter"), + m_params.template get("algorithms.gr.pusher_eps"), m_params.template get("algorithms.gr.pusher_niter"), domain.mesh.prtl_bc() )); + } else if (species.pusher() == PrtlPusher::BORIS) { + auto range_policy = Kokkos::RangePolicy( + 0, + species.npart()); + Kokkos::parallel_for( + "ParticlePusher", + range_policy, + kernel::gr::Pusher_kernel( + domain.fields.em, + domain.fields.em0, + species.i1, species.i2, species.i3, + species.i1_prev, species.i2_prev, species.i3_prev, + species.dx1, species.dx2, species.dx3, + species.dx1_prev, species.dx2_prev, species.dx3_prev, + species.ux1, species.ux2, species.ux3, + species.phi, species.tag, + domain.mesh.metric, + coeff, dt, + domain.mesh.n_active(in::x1), + domain.mesh.n_active(in::x2), + domain.mesh.n_active(in::x3), + m_params.template get("algorithms.gr.pusher_eps"), m_params.template get("algorithms.gr.pusher_niter"), + domain.mesh.prtl_bc() + )); + } else if (species.pusher() == PrtlPusher::NONE) { + // do nothing + } else { + raise::Error("not implemented", HERE); + } // clang-format on } } diff --git a/src/kernels/particle_pusher_gr.hpp b/src/kernels/particle_pusher_gr.hpp index 547463fa7..fd6246217 100644 --- a/src/kernels/particle_pusher_gr.hpp +++ b/src/kernels/particle_pusher_gr.hpp @@ -65,6 +65,7 @@ namespace kernel::gr { static_assert(M::is_metric, "M must be a metric class"); static constexpr auto D = M::Dim; + private: const randacc_ndfield_t DB; const randacc_ndfield_t DB0; array_t i1, i2, i3; @@ -86,34 +87,34 @@ namespace kernel::gr { bool is_absorb_i1min { false }, is_absorb_i1max { false }; public: - Pusher_kernel(const ndfield_t& DB, - const ndfield_t& DB0, - const array_t& i1, - const array_t& i2, - const array_t& i3, - const array_t& i1_prev, - const array_t& i2_prev, - const array_t& i3_prev, - const array_t& dx1, - const array_t& dx2, - const array_t& dx3, - const array_t& dx1_prev, - const array_t& dx2_prev, - const array_t& dx3_prev, - const array_t& ux1, - const array_t& ux2, - const array_t& ux3, - const array_t& phi, - const array_t& tag, - const M& metric, - const real_t& coeff, - const real_t& dt, - const int& ni1, - const int& ni2, - const int& ni3, - const real_t& epsilon, - const int& niter, - const boundaries_t& boundaries) + Pusher_kernel(const ndfield_t& DB, + const ndfield_t& DB0, + array_t& i1, + array_t& i2, + array_t& i3, + array_t& i1_prev, + array_t& i2_prev, + array_t& i3_prev, + array_t& dx1, + array_t& dx2, + array_t& dx3, + array_t& dx1_prev, + array_t& dx2_prev, + array_t& dx3_prev, + array_t& ux1, + array_t& ux2, + array_t& ux3, + array_t& phi, + array_t& tag, + const M& metric, + real_t coeff, + real_t dt, + int ni1, + int ni2, + int ni3, + const real_t& epsilon, + const int& niter, + const boundaries_t& boundaries) : DB { DB } , DB0 { DB0 } , i1 { i1 } @@ -351,8 +352,8 @@ namespace kernel::gr { vp_upd[1] = vp[1] + dt * - (-metric.alpha(xp) * u0 * DERIVATIVE_IN_TH(alpha, xp) + - vp_mid[1] * DERIVATIVE_IN_TH(beta1, xp) - + (-metric.alpha(xp) * u0 * DERIVATIVE_IN_TH(metric.alpha, xp) + + vp_mid[1] * DERIVATIVE_IN_TH(metric.beta1, xp) - (HALF / u0) * (DERIVATIVE_IN_TH((metric.template h<1, 1>), xp) * SQR(vp_mid[0]) + DERIVATIVE_IN_TH((metric.template h<2, 2>), xp) * SQR(vp_mid[1]) + From 3fc4497633b958eca5fd81e6205218e971428fa0 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 6 Nov 2024 14:58:22 -0500 Subject: [PATCH 162/234] consistent type for pusher_niter --- src/engines/grpic.hpp | 4 ++-- src/kernels/particle_pusher_gr.hpp | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 333742eaf..67ef1ef46 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -1029,7 +1029,7 @@ namespace ntt { domain.mesh.n_active(in::x1), domain.mesh.n_active(in::x2), domain.mesh.n_active(in::x3), - m_params.template get("algorithms.gr.pusher_eps"), m_params.template get("algorithms.gr.pusher_niter"), + m_params.template get("algorithms.gr.pusher_eps"), m_params.template get("algorithms.gr.pusher_niter"), domain.mesh.prtl_bc() )); } else if (species.pusher() == PrtlPusher::BORIS) { @@ -1053,7 +1053,7 @@ namespace ntt { domain.mesh.n_active(in::x1), domain.mesh.n_active(in::x2), domain.mesh.n_active(in::x3), - m_params.template get("algorithms.gr.pusher_eps"), m_params.template get("algorithms.gr.pusher_niter"), + m_params.template get("algorithms.gr.pusher_eps"), m_params.template get("algorithms.gr.pusher_niter"), domain.mesh.prtl_bc() )); } else if (species.pusher() == PrtlPusher::NONE) { diff --git a/src/kernels/particle_pusher_gr.hpp b/src/kernels/particle_pusher_gr.hpp index fd6246217..fba40890f 100644 --- a/src/kernels/particle_pusher_gr.hpp +++ b/src/kernels/particle_pusher_gr.hpp @@ -77,11 +77,11 @@ namespace kernel::gr { array_t tag; const M metric; - const real_t coeff, dt; - const int ni1, ni2, ni3; - const real_t epsilon; - const int niter; - const int i1_absorb; + const real_t coeff, dt; + const int ni1, ni2, ni3; + const real_t epsilon; + const unsigned short niter; + const int i1_absorb; bool is_axis_i2min { false }, is_axis_i2max { false }; bool is_absorb_i1min { false }, is_absorb_i1max { false }; @@ -113,7 +113,7 @@ namespace kernel::gr { int ni2, int ni3, const real_t& epsilon, - const int& niter, + const unsigned short& niter, const boundaries_t& boundaries) : DB { DB } , DB0 { DB0 } From c7d78cda1b9ac2ce16ec4bf33c0c7607b3554cdd Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Thu, 7 Nov 2024 10:14:22 -0500 Subject: [PATCH 163/234] cleaned up an unused var --- src/engines/grpic.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 67ef1ef46..745503793 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -562,7 +562,6 @@ namespace ntt { "grid.boundaries.absorb.ds"); const auto dim = direction.get_dim(); real_t xg_min, xg_max, xg_edge; - auto sign = direction.get_sign(); xg_max = m_metadomain.mesh().extent(dim).second; xg_min = xg_max - ds; xg_edge = xg_max; From 8cc33c7cd26a54eabf96400a4a008f73a3b5be9a Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Thu, 7 Nov 2024 16:01:16 -0500 Subject: [PATCH 164/234] fixed bug in output for dimensions of coordinates of particles in GR --- src/framework/domain/output.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/framework/domain/output.cpp b/src/framework/domain/output.cpp index 6961d2826..ae138bac1 100644 --- a/src/framework/domain/output.cpp +++ b/src/framework/domain/output.cpp @@ -87,7 +87,13 @@ namespace ntt { const auto species_to_write = params.template get>( "output.particles.species"); g_writer.defineFieldOutputs(S, all_fields_to_write); - g_writer.defineParticleOutputs(M::PrtlDim, species_to_write); + + Dimension dim = M::PrtlDim; + if constexpr (M::CoordType != Coord::Cart) { + dim = Dim::_3D; + } + g_writer.defineParticleOutputs(dim, species_to_write); + // spectra write all particle species std::vector spectra_species {}; for (const auto& sp : species_params()) { From 470f733932b5b3952c7dbcecfb26791c07eeaffa Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 8 Nov 2024 16:08:26 -0500 Subject: [PATCH 165/234] fixed coordinate transformation for a global injector. When we inject a small amount of particles with given velocities, this is usually in physical coordinates rather than tetrads as in injection for a given energy distribution --- src/kernels/injectors.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kernels/injectors.hpp b/src/kernels/injectors.hpp index 9d3fd7d81..dd5e2d21d 100644 --- a/src/kernels/injectors.hpp +++ b/src/kernels/injectors.hpp @@ -335,7 +335,7 @@ namespace kernel { if constexpr (S == SimEngine::SRPIC) { global_metric.template transform_xyz(x_Cd_, u_Ph, u_Cd); } else if constexpr (S == SimEngine::GRPIC) { - global_metric.template transform(x_Cd, u_Ph, u_Cd); + global_metric.template transform(x_Cd, u_Ph, u_Cd); } else { raise::KernelError(HERE, "Unknown simulation engine"); } @@ -380,7 +380,7 @@ namespace kernel { if constexpr (S == SimEngine::SRPIC) { global_metric.template transform_xyz(x_Cd, u_Ph, u_Cd); } else if constexpr (S == SimEngine::GRPIC) { - global_metric.template transform(x_Cd, u_Ph, u_Cd); + global_metric.template transform(x_Cd, u_Ph, u_Cd); } else { raise::KernelError(HERE, "Unknown simulation engine"); } From 37c4b0d622845683e6c04e283a421f83cb5c2167 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 8 Nov 2024 16:11:23 -0500 Subject: [PATCH 166/234] minor consistensy with sr --- src/kernels/particle_pusher_gr.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kernels/particle_pusher_gr.hpp b/src/kernels/particle_pusher_gr.hpp index fba40890f..2e9ddbde2 100644 --- a/src/kernels/particle_pusher_gr.hpp +++ b/src/kernels/particle_pusher_gr.hpp @@ -29,7 +29,7 @@ /* Local macros */ /* -------------------------------------------------------------------------- */ #define from_Xi_to_i(XI, I) \ - { I = static_cast((XI)); } + { I = static_cast((XI + 1)) - 1; } #define from_Xi_to_i_di(XI, I, DI) \ { \ From 2b2e6a9f6ace65b4fa430b432a71fb026fc42896 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 3 Dec 2024 10:18:51 -0500 Subject: [PATCH 167/234] kernels/particle_pusher_gr: 1. derivatives of the metric in geodesic pusher now use their functional form. Saved numerical derivatives for future testing 2. Added placeholders for storing previous coordinate 3. Fixed bug in geodesic pusher in updating utheta 4. Changes to boundary conditions at the axis --- src/kernels/particle_pusher_gr.hpp | 62 ++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/src/kernels/particle_pusher_gr.hpp b/src/kernels/particle_pusher_gr.hpp index 2e9ddbde2..306f19013 100644 --- a/src/kernels/particle_pusher_gr.hpp +++ b/src/kernels/particle_pusher_gr.hpp @@ -334,31 +334,53 @@ namespace kernel::gr { // find contravariant midpoint velocity metric.template transform(xp, vp_mid, vp_mid_cntrv); - // find Gamma / alpha at midpoint + // find Gamma / alpha at midpointы real_t u0 { computeGamma(T {}, vp_mid, vp_mid_cntrv) / metric.alpha(xp) }; // find updated velocity + // vp_upd[0] = + // vp[0] + + // dt * + // (-metric.alpha(xp) * u0 * DERIVATIVE_IN_R(metric.alpha, xp) + + // vp_mid[0] * DERIVATIVE_IN_R(metric.beta1, xp) - + // (HALF / u0) * + // (DERIVATIVE_IN_R((metric.template h<1, 1>), xp) * SQR(vp_mid[0]) + + // DERIVATIVE_IN_R((metric.template h<2, 2>), xp) * SQR(vp_mid[1]) + + // DERIVATIVE_IN_R((metric.template h<3, 3>), xp) * SQR(vp_mid[2]) + + // TWO * DERIVATIVE_IN_R((metric.template h<1, 3>), xp) * + // vp_mid[0] * vp_mid[2])); + // vp_upd[1] = + // vp[1] + + // dt * + // (-metric.alpha(xp) * u0 * DERIVATIVE_IN_TH(metric.alpha, xp) + + // vp_mid[0] * DERIVATIVE_IN_TH(metric.beta1, xp) - + // (HALF / u0) * + // (DERIVATIVE_IN_TH((metric.template h<1, 1>), xp) * SQR(vp_mid[0]) + + // DERIVATIVE_IN_TH((metric.template h<2, 2>), xp) * SQR(vp_mid[1]) + + // DERIVATIVE_IN_TH((metric.template h<3, 3>), xp) * SQR(vp_mid[2]) + + // TWO * DERIVATIVE_IN_TH((metric.template h<1, 3>), xp) * + // vp_mid[0] * vp_mid[2])); vp_upd[0] = vp[0] + dt * - (-metric.alpha(xp) * u0 * DERIVATIVE_IN_R(metric.alpha, xp) + - vp_mid[0] * DERIVATIVE_IN_R(metric.beta1, xp) - + (-metric.alpha(xp) * u0 * metric.dr_alpha(xp) + + vp_mid[0] * metric.dr_beta1(xp) - (HALF / u0) * - (DERIVATIVE_IN_R((metric.template h<1, 1>), xp) * SQR(vp_mid[0]) + - DERIVATIVE_IN_R((metric.template h<2, 2>), xp) * SQR(vp_mid[1]) + - DERIVATIVE_IN_R((metric.template h<3, 3>), xp) * SQR(vp_mid[2]) + - TWO * DERIVATIVE_IN_R((metric.template h<1, 3>), xp) * + (metric.dr_h11(xp) * SQR(vp_mid[0]) + + metric.dr_h22(xp) * SQR(vp_mid[1]) + + metric.dr_h33(xp) * SQR(vp_mid[2]) + + TWO * metric.dr_h13(xp) * vp_mid[0] * vp_mid[2])); vp_upd[1] = vp[1] + dt * - (-metric.alpha(xp) * u0 * DERIVATIVE_IN_TH(metric.alpha, xp) + - vp_mid[1] * DERIVATIVE_IN_TH(metric.beta1, xp) - + (-metric.alpha(xp) * u0 * metric.dt_alpha(xp) + + vp_mid[0] * metric.dt_beta1(xp) - (HALF / u0) * - (DERIVATIVE_IN_TH((metric.template h<1, 1>), xp) * SQR(vp_mid[0]) + - DERIVATIVE_IN_TH((metric.template h<2, 2>), xp) * SQR(vp_mid[1]) + - DERIVATIVE_IN_TH((metric.template h<3, 3>), xp) * SQR(vp_mid[2]) + - TWO * DERIVATIVE_IN_TH((metric.template h<1, 3>), xp) * + (metric.dt_h11(xp) * SQR(vp_mid[0]) + + metric.dt_h22(xp) * SQR(vp_mid[1]) + + metric.dt_h33(xp) * SQR(vp_mid[2]) + + TWO * metric.dt_h13(xp) * vp_mid[0] * vp_mid[2])); } } else if constexpr (D == Dim::_3D) { @@ -656,9 +678,13 @@ namespace kernel::gr { dx2_prev(p) = dx2(p); coord_t xp { ZERO }; + coord_t xp_prev { ZERO }; xp[0] = i_di_to_Xi(i1(p), dx1(p)); xp[1] = i_di_to_Xi(i2(p), dx2(p)); + + xp_prev[0] = i_di_to_Xi(i1_prev(p), dx1_prev(p)); + xp_prev[1] = i_di_to_Xi(i2_prev(p), dx2_prev(p)); vec_t Dp_cntrv { ZERO }, Bp_cntrv { ZERO }, Dp_hat { ZERO }, Bp_hat { ZERO }; @@ -692,7 +718,7 @@ namespace kernel::gr { { (xp[0] + xp_upd[0]) * HALF, (xp[1] + xp_upd[1]) * HALF }, vp_upd, phi(p)); - + // update coordinate int i1_, i2_; prtldx_t dx1_, dx2_; @@ -726,12 +752,16 @@ namespace kernel::gr { } } if constexpr (D == Dim::_2D || D == Dim::_3D) { - if (i2(p) < 1) { + if (i2(p) < 0) { if (is_axis_i2min) { + i2(p) = 0; + dx2(p) = ONE - dx2(p); ux2(p) = -ux2(p); } - } else if (i2(p) >= ni2 - 1) { + } else if (i2(p) >= ni2) { if (is_axis_i2min) { + i2(p) = ni2 - 1; + dx2(p) = ONE - dx2(p); ux2(p) = -ux2(p); } } From e69021ab474577c837dd74dff375af33ed48494b Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 3 Dec 2024 10:22:49 -0500 Subject: [PATCH 168/234] metrics: all gr metrics now have functional form of derivatives of the metric componets --- src/metrics/kerr_schild.h | 163 +++++++++++++++++++++++++++++ src/metrics/kerr_schild_0.h | 102 ++++++++++++++++++ src/metrics/qkerr_schild.h | 201 ++++++++++++++++++++++++++++++++++++ 3 files changed, 466 insertions(+) diff --git a/src/metrics/kerr_schild.h b/src/metrics/kerr_schild.h index 5a60def53..9ec6f7c98 100644 --- a/src/metrics/kerr_schild.h +++ b/src/metrics/kerr_schild.h @@ -17,6 +17,7 @@ #include "arch/kokkos_aliases.h" #include "utils/numeric.h" +#include "utils/comparators.h" #include "metrics/metric_base.h" @@ -216,6 +217,28 @@ namespace metric { return ONE / math::sqrt(ONE + z(r, theta)); } + /** + * dr derivative of lapse function + * @param x coordinate array in code units + */ + Inline auto dr_alpha(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + const real_t dr_Sigma {TWO * r * dr}; + + return - (dr * Sigma(r, theta) - r * dr_Sigma) * CUBE(alpha(x)) / SQR(Sigma(r, theta)); + } + + /** + * dtheta derivative of lapse function + * @param x coordinate array in code units + */ + Inline auto dt_alpha(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + return CUBE(alpha(x)) * r * dt_Sigma(theta) / SQR(Sigma(r, theta)); + } + /** * radial component of shift vector * @param x coordinate array in code units @@ -225,6 +248,146 @@ namespace metric { return dr_inv * z_ / (ONE + z_); } + /** + * dr derivative of radial component of shift vector + * @param x coordinate array in code units + */ + Inline auto dr_beta1(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + const real_t dr_Sigma {TWO * r * dr}; + + return dr_inv * TWO * (dr * Sigma(r, theta) - r * dr_Sigma) / SQR(Sigma(r, theta) + TWO * r); + } + + /** + * dtheta derivative of radial component of shift vector + * @param x coordinate array in code units + */ + Inline auto dt_beta1(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + return - dr_inv * TWO * r * dt_Sigma(theta) / SQR(Sigma(r, theta) * (ONE + z(r, theta))); + } + + /** + * dr derivative of h^11 + * @param x coordinate array in code units + */ + Inline auto dr_h11(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + const real_t dr_Sigma {TWO * r * dr}; + const real_t dr_Delta {TWO * dr * (r - ONE)}; + const real_t dr_A {FOUR * r * dr * (SQR(r) + SQR(a)) - SQR(a) * SQR(math::sin(theta)) * dr_Delta}; + + return (Sigma(r, theta) * (Sigma(r, theta) + TWO * r) * dr_A + - TWO * A(r, theta) * (r * dr_Sigma + Sigma(r, theta) * (dr_Sigma + dr))) + / (SQR(Sigma(r, theta) * (Sigma(r, theta) + TWO * r))) * SQR(dr_inv); + } + + /** + * dr derivative of h^22 + * @param x coordinate array in code units + */ + Inline auto dr_h22(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + const real_t dr_Sigma {TWO * r * dr}; + + return - dr_Sigma / SQR(Sigma(r, theta)) * SQR(dtheta_inv); + } + + /** + * dr derivative of h^33 + * @param x coordinate array in code units + */ + Inline auto dr_h33(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + const real_t dr_Sigma {TWO * r * dr}; + + return - dr_Sigma / SQR(Sigma(r, theta)) / SQR(math::sin(theta)); + } + + /** + * dr derivative of h^13 + * @param x coordinate array in code units + */ + Inline auto dr_h13(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + const real_t dr_Sigma {TWO * r * dr}; + + return - a * dr_Sigma / SQR(Sigma(r, theta)) * dr_inv; + } + + /** + * dtheta derivative of Sigma + * @param x coordinate array in code units + */ + Inline auto dt_Sigma(const real_t& theta) const -> real_t { + const real_t dt_Sigma {- TWO * SQR(a) * math::sin(theta) * math::cos(theta) * dtheta}; + if (cmp::AlmostZero(dt_Sigma)) + return ZERO; + else + return dt_Sigma; + } + + /** + * dtheta derivative of A + * @param x coordinate array in code units + */ + Inline auto dt_A(const real_t& r, const real_t& theta) const -> real_t { + const real_t dt_A {- TWO * SQR(a) * math::sin(theta) * math::cos(theta) * Delta(r) * dtheta}; + if (cmp::AlmostZero(dt_A)) + return ZERO; + else + return dt_A; + } + + /** + * dtheta derivative of h^11 + * @param x coordinate array in code units + */ + Inline auto dt_h11(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + return (Sigma(r, theta) * (Sigma(r, theta) + TWO * r) * dt_A(r, theta) + - TWO * A(r, theta) * dt_Sigma(theta) * (r + Sigma(r, theta))) + / (SQR(Sigma(r, theta) * (Sigma(r, theta) + TWO * r))) * SQR(dr_inv); + } + + /** + * dtheta derivative of h^22 + * @param x coordinate array in code units + */ + Inline auto dt_h22(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + return - dt_Sigma(theta) / SQR(Sigma(r, theta)) * SQR(dtheta_inv); + } + + /** + * dtheta derivative of h^33 + * @param x coordinate array in code units + */ + Inline auto dt_h33(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + return - TWO * dtheta * math::cos(theta) * (Sigma(r, theta) - SQR(a) * SQR(math::sin(theta))) / CUBE(math::sin(theta)) / SQR(Sigma(r, theta)); + } + + /** + * dtheta derivative of h^13 + * @param x coordinate array in code units + */ + Inline auto dt_h13(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + return - a * dt_Sigma(theta) / SQR(Sigma(r, theta)) * dr_inv; + } + /** * sqrt(det(h_ij)) * @param x coordinate array in code units diff --git a/src/metrics/kerr_schild_0.h b/src/metrics/kerr_schild_0.h index 31080ed4c..baea9b4c1 100644 --- a/src/metrics/kerr_schild_0.h +++ b/src/metrics/kerr_schild_0.h @@ -171,6 +171,22 @@ namespace metric { return ONE; } + /** + * dr derivative of lapse function + * @param x coordinate array in code units + */ + Inline auto dr_alpha(const coord_t& x) const -> real_t { + return ZERO; + } + + /** + * dtheta derivative of lapse function + * @param x coordinate array in code units + */ + Inline auto dt_alpha(const coord_t& x) const -> real_t { + return ZERO; + } + /** * radial component of shift vector * @param x coordinate array in code units @@ -179,6 +195,92 @@ namespace metric { return ZERO; } + /** + * dr derivative of radial component of shift vector + * @param x coordinate array in code units + */ + Inline auto dr_beta1(const coord_t& x) const -> real_t { + return ZERO; + } + + /** + * dtheta derivative of radial component of shift vector + * @param x coordinate array in code units + */ + Inline auto dt_beta1(const coord_t& x) const -> real_t { + return ZERO; + } + + /** + * dr derivative of h^11 + * @param x coordinate array in code units + */ + Inline auto dr_h11(const coord_t& x) const -> real_t { + return ZERO; + } + + /** + * dr derivative of h^22 + * @param x coordinate array in code units + */ + Inline auto dr_h22(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + return - TWO / CUBE(r) * SQR(dtheta_inv) * dr; + } + + /** + * dr derivative of h^33 + * @param x coordinate array in code units + */ + Inline auto dr_h33(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + return - TWO / CUBE(r) / SQR(math::sin(theta)) * dr; + } + + /** + * dr derivative of h^13 + * @param x coordinate array in code units + */ + Inline auto dr_h13(const coord_t& x) const -> real_t { + return ZERO; + } + + /** + * dtheta derivative of h^11 + * @param x coordinate array in code units + */ + Inline auto dt_h11(const coord_t& x) const -> real_t { + return ZERO; + } + + /** + * dtheta derivative of h^22 + * @param x coordinate array in code units + */ + Inline auto dt_h22(const coord_t& x) const -> real_t { + return ZERO; + } + + /** + * dtheta derivative of h^33 + * @param x coordinate array in code units + */ + Inline auto dt_h33(const coord_t& x) const -> real_t { + const real_t r {x[0] * dr + x1_min}; + const real_t theta {x[1] * dtheta + x2_min}; + return - TWO * math::cos(theta) / SQR(r) / CUBE(math::sin(theta)) * dtheta; + } + + /** + * dtheta derivative of h^13 + * @param x coordinate array in code units + */ + Inline auto dt_h13(const coord_t& x) const -> real_t { + return ZERO; + } + /** * sqrt(det(h_ij)) * @param x coordinate array in code units diff --git a/src/metrics/qkerr_schild.h b/src/metrics/qkerr_schild.h index 85507c6e1..e45376a24 100644 --- a/src/metrics/qkerr_schild.h +++ b/src/metrics/qkerr_schild.h @@ -234,6 +234,33 @@ namespace metric { return ONE / math::sqrt(ONE + z(r, theta)); } + /** + * dr derivative of lapse function + * @param x coordinate array in code units + */ + Inline auto dr_alpha(const coord_t& x) const -> real_t { + const real_t r { r0 + math::exp(x[0] * dchi + chi_min) }; + const real_t theta { eta2theta(x[1] * deta + eta_min) }; + const real_t dx_r {dchi * math::exp(x[0] * dchi + chi_min)}; + const real_t dr_Sigma {TWO * r * dx_r}; + + return - (dx_r * Sigma(r, theta) - r * dr_Sigma) * CUBE(alpha(x)) / SQR(Sigma(r, theta)); + } + + /** + * dtheta derivative of lapse function + * @param x coordinate array in code units + */ + Inline auto dt_alpha(const coord_t& x) const -> real_t { + const real_t r { r0 + math::exp(x[0] * dchi + chi_min) }; + const real_t eta {x[1] * deta + eta_min }; + const real_t theta { eta2theta(eta) }; + const real_t dx_dt {deta * (ONE + TWO * h0 * constant::INV_PI_SQR * (TWO * THREE * SQR(eta) - TWO * THREE * constant::PI * eta + constant::PI_SQR)) }; + const real_t dt_Sigma {- TWO * SQR(a) * math::sin(theta) * math::cos(theta) * dx_dt}; + + return r * dt_Sigma * CUBE(alpha(x)) / SQR(Sigma(r, theta)); + } + /** * radial component of shift vector * @param x coordinate array in code units @@ -246,6 +273,167 @@ namespace metric { return math::exp(-chi) * dchi_inv * z_ / (ONE + z_); } + /** + * dr derivative of radial component of shift vector + * @param x coordinate array in code units + */ + Inline auto dr_beta1(const coord_t& x) const -> real_t { + const real_t chi { x[0] * dchi + chi_min }; + const real_t r { r0 + math::exp(chi) }; + const real_t theta { eta2theta(x[1] * deta + eta_min) }; + const real_t z_ { z(r, theta) }; + const real_t dx_r {dchi * math::exp(x[0] * dchi + chi_min)}; + const real_t dr_Sigma {TWO * r * dx_r}; + + return math::exp(-chi) * dchi_inv * TWO * (dx_r * Sigma(r, theta) - r * dr_Sigma) / SQR(Sigma(r, theta) + TWO * r) + - dchi * math::exp(-chi) * dchi_inv * z_ / (ONE + z_); + } + + /** + * dr derivative of radial component of shift vector + * @param x coordinate array in code units + */ + Inline auto dt_beta1(const coord_t& x) const -> real_t { + const real_t chi { x[0] * dchi + chi_min }; + const real_t r { r0 + math::exp(chi) }; + const real_t eta {x[1] * deta + eta_min }; + const real_t theta { eta2theta(eta) }; + return - math::exp(-chi) * dchi_inv * TWO * r * dt_Sigma(eta) / SQR(Sigma(r, theta) * (ONE + z(r, theta))); + } + + /** + * dr derivative of h^11 + * @param x coordinate array in code units + */ + Inline auto dr_h11(const coord_t& x) const -> real_t { + const real_t r { r0 + math::exp(x[0] * dchi + chi_min) }; + const real_t theta { eta2theta(x[1] * deta + eta_min) }; + + const real_t dx_r {dchi * math::exp(x[0] * dchi + chi_min)}; + const real_t dr_Sigma {TWO * r * dx_r}; + const real_t dr_Delta {TWO * dx_r * (r - ONE)}; + const real_t dr_A {FOUR * r * dx_r * (SQR(r) + SQR(a)) - SQR(a) * SQR(math::sin(theta)) * dr_Delta}; + + return (math::exp(-TWO * (x[0] * dchi + chi_min)) / SQR(dchi) + * (Sigma(r, theta) * (Sigma(r, theta) + TWO * r) * dr_A + - TWO * A(r, theta) * (r * dr_Sigma + Sigma(r, theta) * (dr_Sigma + dx_r))) + / (SQR(Sigma(r, theta) * (Sigma(r, theta) + TWO * r))) ) + -TWO * dchi * math::exp(-TWO * (x[0] * dchi + chi_min)) / SQR(dchi) * A(r, theta) / (Sigma(r, theta) * (Sigma(r, theta) + TWO * r)); + } + + /** + * dr derivative of h^22 + * @param x coordinate array in code units + */ + Inline auto dr_h22(const coord_t& x) const -> real_t { + const real_t r { r0 + math::exp(x[0] * dchi + chi_min) }; + const real_t theta { eta2theta(x[1] * deta + eta_min) }; + const real_t dx_r {dchi * math::exp(x[0] * dchi + chi_min)}; + const real_t dr_Sigma {TWO * r * dx_r}; + + return - dr_Sigma / SQR(Sigma(r, theta)) / SQR(deta); + } + + /** + * dr derivative of h^33 + * @param x coordinate array in code units + */ + Inline auto dr_h33(const coord_t& x) const -> real_t { + const real_t r { r0 + math::exp(x[0] * dchi + chi_min) }; + const real_t theta { eta2theta(x[1] * deta + eta_min) }; + const real_t dx_r {dchi * math::exp(x[0] * dchi + chi_min)}; + const real_t dr_Sigma {TWO * r * dx_r}; + + return - dr_Sigma / SQR(Sigma(r, theta)) / SQR(math::sin(theta)); + } + + /** + * dr derivative of h^13 + * @param x coordinate array in code units + */ + Inline auto dr_h13(const coord_t& x) const -> real_t { + const real_t r { r0 + math::exp(x[0] * dchi + chi_min) }; + const real_t theta { eta2theta(x[1] * deta + eta_min) }; + const real_t dx_r {dchi * math::exp(x[0] * dchi + chi_min)}; + const real_t dr_Sigma {TWO * r * dx_r}; + + return - a * dr_Sigma / SQR(Sigma(r, theta)) * (math::exp(-(x[0] * dchi + chi_min)) * dchi_inv) + - dchi * (math::exp(-(x[0] * dchi + chi_min)) * dchi_inv) * a / Sigma(r, theta); + } + + /** + * dtheta derivative of Sigma + * @param x coordinate array in code units + */ + Inline auto dt_Sigma(const real_t& eta) const -> real_t { + const real_t theta { eta2theta(eta) }; + const real_t dt_Sigma {- TWO * SQR(a) * math::sin(theta) * math::cos(theta) * dx_dt(eta)}; + if (cmp::AlmostZero(dt_Sigma)) + return ZERO; + else + return dt_Sigma; + } + + /** + * dtheta derivative of A + * @param x coordinate array in code units + */ + Inline auto dt_A(const real_t& r, const real_t& eta) const -> real_t { + const real_t theta { eta2theta(eta) }; + const real_t dt_A {- TWO * SQR(a) * math::sin(theta) * math::cos(theta) * Delta(r) * dx_dt(eta)}; + if (cmp::AlmostZero(dt_A)) + return ZERO; + else + return dt_A; + } + + /** + * dtheta derivative of h^11 + * @param x coordinate array in code units + */ + Inline auto dt_h11(const coord_t& x) const -> real_t { + const real_t r { r0 + math::exp(x[0] * dchi + chi_min) }; + const real_t eta {x[1] * deta + eta_min }; + const real_t theta { eta2theta(eta) }; + return math::exp(-TWO * (x[0] * dchi + chi_min)) / SQR(dchi) + * (Sigma(r, theta) * (Sigma(r, theta) + TWO * r) * dt_A(r, eta) + - TWO * A(r, theta) * dt_Sigma(eta) * (r + Sigma(r, theta))) + / (SQR(Sigma(r, theta) * (Sigma(r, theta) + TWO * r))); + } + + /** + * dtheta derivative of h^22 + * @param x coordinate array in code units + */ + Inline auto dt_h22(const coord_t& x) const -> real_t { + const real_t r { r0 + math::exp(x[0] * dchi + chi_min) }; + const real_t eta {x[1] * deta + eta_min }; + const real_t theta { eta2theta(eta) }; + return - dt_Sigma(eta) / SQR(Sigma(r, theta)) / SQR(deta); + } + + /** + * dtheta derivative of h^33 + * @param x coordinate array in code units + */ + Inline auto dt_h33(const coord_t& x) const -> real_t { + const real_t r { r0 + math::exp(x[0] * dchi + chi_min) }; + const real_t eta {x[1] * deta + eta_min }; + const real_t theta { eta2theta(eta) }; + return - (dt_Sigma(eta) + TWO * math::cos(theta) / math::sin(theta) * Sigma(r, theta) * dx_dt(eta)) / SQR(Sigma(r, theta) * math::sin(theta)); + } + + /** + * dtheta derivative of h^13 + * @param x coordinate array in code units + */ + Inline auto dt_h13(const coord_t& x) const -> real_t { + const real_t r { r0 + math::exp(x[0] * dchi + chi_min) }; + const real_t eta {x[1] * deta + eta_min }; + const real_t theta { eta2theta(eta) }; + return - a * dt_Sigma(eta) / SQR(Sigma(r, theta)) * (math::exp(-(x[0] * dchi + chi_min)) * dchi_inv); + } + /** * sqrt(det(h_ij)) * @param x coordinate array in code units @@ -456,6 +644,19 @@ namespace metric { } } + /** + * @brief quasi-spherical eta to spherical theta + */ + Inline auto dx_dt(const real_t& eta) const -> real_t { + if (cmp::AlmostZero(h0)) { + return deta; + } else { + return deta * (ONE + + TWO * h0 * constant::INV_PI_SQR * + (TWO * THREE * SQR(eta) - TWO * THREE * constant::PI * eta + constant::PI_SQR)); + } + } + /** * @brief spherical theta to quasi-spherical eta */ From 05b30f827f2371ae15361ee619872134ca742e1f Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 3 Dec 2024 10:24:38 -0500 Subject: [PATCH 169/234] setups/grpic/pusher: pgen and inputs for pusher tests --- setups/grpic/pusher/boris.toml | 97 +++++++++ setups/grpic/pusher/massive_gravity_3d.toml | 84 ++++++++ setups/grpic/pusher/massive_gravity_a0.toml | 93 +++++++++ .../grpic/pusher/massive_gravity_a0995.toml | 93 +++++++++ setups/grpic/pusher/pgen.hpp | 184 ++++++++++++++++++ 5 files changed, 551 insertions(+) create mode 100644 setups/grpic/pusher/boris.toml create mode 100644 setups/grpic/pusher/massive_gravity_3d.toml create mode 100644 setups/grpic/pusher/massive_gravity_a0.toml create mode 100644 setups/grpic/pusher/massive_gravity_a0995.toml create mode 100644 setups/grpic/pusher/pgen.hpp diff --git a/setups/grpic/pusher/boris.toml b/setups/grpic/pusher/boris.toml new file mode 100644 index 000000000..6483eb2c6 --- /dev/null +++ b/setups/grpic/pusher/boris.toml @@ -0,0 +1,97 @@ +[simulation] + name = "boris_a0" + engine = "grpic" + runtime = 100.0 + +[grid] + resolution = [128,128] + extent = [[0.9, 20.0]] + + [grid.metric] + metric = "qkerr_schild" + qsph_r0 = 0.0 + qsph_h = 0.0 + ks_a = 0.0 + + [grid.boundaries] + fields = [["ABSORB"]] + particles = [["ABSORB"]] + + [grid.boundaries.absorb] + ds = 0.5 + +[scales] + larmor0 = 5.0 + skindepth0 = 1e-3 + +[algorithms] + current_filters = 4 + + [algorithms.gr] + pusher_niter = 10 + pusher_eps = 1e-6 + + [algorithms.timestep] + CFL = 0.1 + + [algorithms.toggles] + deposit = false + fieldsolver = false + +[particles] + ppc0 = 1.0 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = -1.0 + maxnpart = 1e0 + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = -1.0 + maxnpart = 1e1 + +[setup] + x1s = [5.77] + y1s = [1.570796] + z1s = [0.000000] + ux1s = [0.4376657824933686] + uy1s = [0.000000] + uz1s = [4.0] +# x1s = [10.0] +# y1s = [1.570796] +# z1s = [0.000000] +# ux1s = [0.0] +# uy1s = [0.000000] +# uz1s = [10.00000] + + #x2s = [7.5] + #y2s = [0.15745454545454546] + #z2s = [0.000000] + #ux2s = [0.029637] + #uy2s = [0.000000] + #uz2s = [3.75] + +[output] + format = "hdf5" + + [output.fields] + enable = true + interval_time = 1.0 + quantities = ["D", "B"] + + [output.particles] + enable = true + stride = 1 + interval_time = 0.2 + species = [] + + [output.spectra] + enable = false + +[diagnostics] + interval = 10 + colored_stdout = true + blocking_timers = true diff --git a/setups/grpic/pusher/massive_gravity_3d.toml b/setups/grpic/pusher/massive_gravity_3d.toml new file mode 100644 index 000000000..5b87554b2 --- /dev/null +++ b/setups/grpic/pusher/massive_gravity_3d.toml @@ -0,0 +1,84 @@ +[simulation] + name = "massive_3d" + engine = "grpic" + runtime = 1000.0 + +[grid] + resolution = [128, 128] + extent = [[0.1, 15.0]] + + [grid.metric] + metric = "qkerr_schild" + qsph_r0 = 0.0 + qsph_h = 0.0 + ks_a = 0.995 + + [grid.boundaries] + fields = [["ABSORB"]] + particles = [["ABSORB"]] + + [grid.boundaries.absorb] + ds = 0.5 + +[scales] + larmor0 = 2e-3 + skindepth0 = 0.1 + +[algorithms] + current_filters = 4 + + [algorithms.gr] + pusher_niter = 20 + pusher_eps = 1e-3 + + [algorithms.timestep] + CFL = 0.8 + + [algorithms.toggles] + deposit = false + fieldsolver = false + +[particles] + ppc0 = 4.0 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = 0.0 + maxnpart = 1e1 + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = 0.0 + maxnpart = 1e1 + +[setup] + x1s = [10.00000] + y1s = [1.570796] + z1s = [0.000000] + ux1s = [0.2181435531189523] + uy1s = [2.89] + uz1s = [1.05769] + +[output] + format = "hdf5" + + [output.fields] + enable = true + interval_time = 5.0 + quantities = ["D", "B"] + + [output.particles] + enable = true + stride = 1 + interval_time = 0.5 + species = [] + + [output.spectra] + enable = false + +[diagnostics] + interval = 10 + colored_stdout = true + blocking_timers = true diff --git a/setups/grpic/pusher/massive_gravity_a0.toml b/setups/grpic/pusher/massive_gravity_a0.toml new file mode 100644 index 000000000..08b607d4b --- /dev/null +++ b/setups/grpic/pusher/massive_gravity_a0.toml @@ -0,0 +1,93 @@ +[simulation] + name = "massive_a0" + engine = "grpic" + runtime = 500.0 + +[grid] + resolution = [256,256] + extent = [[1.2, 15.0]] + + [grid.metric] + metric = "qkerr_schild" + qsph_r0 = 0.0 + qsph_h = 0.0 + ks_a = 0.0 + + [grid.boundaries] + fields = [["ABSORB"]] + particles = [["ABSORB"]] + + [grid.boundaries.absorb] + ds = 0.5 + +[scales] + larmor0 = 2e-3 + skindepth0 = 0.1 + +[algorithms] + current_filters = 4 + + [algorithms.gr] + pusher_niter = 10 + pusher_eps = 1e-3 + + [algorithms.timestep] + CFL = 0.8 + + [algorithms.toggles] + deposit = false + fieldsolver = false + +[particles] + ppc0 = 4.0 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = 0.0 + maxnpart = 1e1 + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = 0.0 + maxnpart = 1e1 + +[setup] + # (1,2,0) from Levin&Perez-Giz (2008) + x1s = [9.85180645] + y1s = [1.570796] + z1s = [0.5235987755982988] + ux1s = [0.24159816] + uy1s = [3.535534] + uz1s = [0.000000] + + # (4,1,1) from Levin&Perez-Giz (2008) + x2s = [68.600387] + y2s = [1.570796] + z2s = [0.000000] + ux2s = [0.029637] + uy2s = [0.000000] + uz2s = [3.900000] + +[output] + format = "hdf5" + + [output.fields] + enable = true + interval_time = 1.0 + quantities = ["D", "B"] + + [output.particles] + enable = true + stride = 1 + interval_time = 1.0 + species = [] + + [output.spectra] + enable = false + +[diagnostics] + interval = 10 + colored_stdout = true + blocking_timers = true diff --git a/setups/grpic/pusher/massive_gravity_a0995.toml b/setups/grpic/pusher/massive_gravity_a0995.toml new file mode 100644 index 000000000..b04538e10 --- /dev/null +++ b/setups/grpic/pusher/massive_gravity_a0995.toml @@ -0,0 +1,93 @@ +[simulation] + name = "massive_a0995" + engine = "grpic" + runtime = 2000.0 + +[grid] + resolution = [512, 512] + extent = [[0.9, 15.0]] + + [grid.metric] + metric = "kerr_schild" + qsph_r0 = 0.0 + qsph_h = 0.0 + ks_a = 0.0 + + [grid.boundaries] + fields = [["ABSORB"]] + particles = [["ABSORB"]] + + [grid.boundaries.absorb] + ds = 0.5 + +[scales] + larmor0 = 2e-3 + skindepth0 = 0.1 + +[algorithms] + current_filters = 4 + + [algorithms.gr] + pusher_niter = 10 + pusher_eps = 1e-10 + + [algorithms.timestep] + CFL = 6.0 + + [algorithms.toggles] + deposit = false + fieldsolver = false + +[particles] + ppc0 = 4.0 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = 0.0 + maxnpart = 1e1 + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = 0.0 + maxnpart = 1e1 + +[setup] + # (2,3,1) from Levin&Perez-Giz (2008) + x1s = [10.343515586064923] + y1s = [1.570796] + z1s = [0.000000] + ux1s = [0.19483849893499136] + uy1s = [0.000000] + uz1s = [2.000000] + + # (~1000, 3, ~671) from Levin&Perez-Giz (2008) + x2s = [10.64975354] + y2s = [1.570796] + z2s = [0.000000] + ux2s = [0.18914503] + uy2s = [0.000000] + uz2s = [2.000000] + +[output] + format = "hdf5" + + [output.fields] + enable = true + interval_time = 5.0 + quantities = ["D", "B"] + + [output.particles] + enable = true + stride = 1 + interval_time = 2.0 + species = [] + + [output.spectra] + enable = false + +[diagnostics] + interval = 10 + colored_stdout = true + blocking_timers = true diff --git a/setups/grpic/pusher/pgen.hpp b/setups/grpic/pusher/pgen.hpp new file mode 100644 index 000000000..d6a29a185 --- /dev/null +++ b/setups/grpic/pusher/pgen.hpp @@ -0,0 +1,184 @@ +#ifndef PROBLEM_GENERATOR_H +#define PROBLEM_GENERATOR_H + +#include "enums.h" +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "arch/traits.h" +#include "utils/comparators.h" +#include "utils/formatting.h" +#include "utils/log.h" +#include "utils/numeric.h" + +#include "archetypes/problem_generator.h" +#include "framework/domain/metadomain.h" +#include "framework/domain/domain.h" +#include "archetypes/energy_dist.h" +#include "archetypes/particle_injector.h" + +#include + +namespace user { + using namespace ntt; + + template + struct InitFields { + InitFields(M metric_) : metric { metric_ } {} + + Inline auto A_3(const coord_t& x_Cd) const -> real_t { + // return HALF * (metric.template h_<3, 3>(x_Cd) ); + // + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + // ); + coord_t x_Ph { ZERO }; + metric.template convert(x_Cd, x_Ph); + return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); + } + + Inline auto A_1(const coord_t& x_Cd) const -> real_t { + return HALF * (metric.template h_<1, 3>(x_Cd) + + TWO * metric.spin() * metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) + ); + } + + Inline auto A_0(const coord_t& x_Cd) const -> real_t { + real_t g_00 { -metric.alpha(x_Cd) * metric.alpha(x_Cd) + + metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) * metric.beta1(x_Cd) + }; + return HALF * (metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + + TWO * metric.spin() * g_00); + } + + Inline auto bx1(const coord_t& x_Ph) const -> real_t { + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0]; + x0m[1] = xi[1] - HALF; + x0p[0] = xi[0]; + x0p[1] = xi[1] + HALF; + + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + + if (cmp::AlmostZero(x_Ph[1])) + return ONE; + else + return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; + // return ZERO; + } + + Inline auto bx2(const coord_t& x_Ph) const -> real_t { + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0] + HALF - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF + HALF; + x0p[1] = xi[1]; + + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + if (cmp::AlmostZero(x_Ph[1])) + return ZERO; + else + return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; + // return ZERO; + } + + Inline auto bx3(const coord_t& x_Ph) const -> real_t { + // coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + // metric.template convert(x_Ph, xi); + + // x0m[0] = xi[0] + HALF - HALF; + // x0m[1] = xi[1]; + // x0p[0] = xi[0] + HALF + HALF; + // x0p[1] = xi[1]; + + // real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + // return -(A_1(x0p) - A_1(x0m)) * inv_sqrt_detH_ijP; + return ZERO; + } + + Inline auto dx1(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx2(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx3(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + private: + const M metric; + }; + + template + struct PGen : public arch::ProblemGenerator { + // compatibility traits for the problem generator + static constexpr auto engines { traits::compatible_with::value }; + static constexpr auto metrics { + traits::compatible_with::value + }; + static constexpr auto dimensions { traits::compatible_with::value }; + + // for easy access to variables in the child class + using arch::ProblemGenerator::D; + using arch::ProblemGenerator::C; + using arch::ProblemGenerator::params; + + InitFields init_flds; + const Metadomain& global_domain; + inline PGen(const SimulationParams& p, const Metadomain& m) + : arch::ProblemGenerator { p } + , global_domain { m } + , init_flds { m.mesh().metric } {} + + inline void InitPrtls(Domain& local_domain) { + const auto empty = std::vector {}; + const auto x1s = params.template get>("setup.x1s", empty); + const auto y1s = params.template get>("setup.y1s", empty); + const auto z1s = params.template get>("setup.z1s", empty); + const auto ux1s = params.template get>("setup.ux1s", + empty); + const auto uy1s = params.template get>("setup.uy1s", + empty); + const auto uz1s = params.template get>("setup.uz1s", + empty); + + const auto x2s = params.template get>("setup.x2s", empty); + const auto y2s = params.template get>("setup.y2s", empty); + const auto z2s = params.template get>("setup.z2s", empty); + const auto ux2s = params.template get>("setup.ux2s", + empty); + const auto uy2s = params.template get>("setup.uy2s", + empty); + const auto uz2s = params.template get>("setup.uz2s", + empty); + const std::map> data_1 { + { "x1", x1s}, + { "x2", y1s}, + { "phi", z1s}, + {"ux1", ux1s}, + {"ux2", uy1s}, + {"ux3", uz1s} + }; + const std::map> data_2 { + { "x1", x2s}, + { "x2", y2s}, + { "phi", z2s}, + {"ux1", ux2s}, + {"ux2", uy2s}, + {"ux3", uz2s} + }; + + arch::InjectGlobally(global_domain, local_domain, (arch::spidx_t)1, data_1); + arch::InjectGlobally(global_domain, local_domain, (arch::spidx_t)2, data_2); + } + // inline PGen() {} + }; + +} // namespace user + +#endif From 048dd573aae7662b6d77f3e821dbf7d9e7f347d7 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 3 Dec 2024 12:28:14 -0500 Subject: [PATCH 170/234] engines/grpic: added timestep correction for gr pusher --- src/engines/grpic.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 745503793..785e6df97 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -1002,9 +1002,9 @@ namespace ntt { ? species.charge() / species.mass() : ZERO; // coeff = q / m (dt / 2) omegaB0 - const auto coeff = q_ovr_m * HALF * dt * - m_params.template get("scales.omegaB0"); - + const auto coeff = q_ovr_m * HALF * dt * m_params.template get( + "algorithms.timestep.correction") * + m_params.template get("scales.omegaB0"); // clang-format off if (species.pusher() == PrtlPusher::PHOTON) { From a86182c766af5b7e19189ec7b1c9aba63793958a Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 4 Dec 2024 14:58:18 -0500 Subject: [PATCH 171/234] erased stretching of initial field --- src/archetypes/field_setter.h | 52 ++++------------------------------- src/kernels/fields_bcs.hpp | 16 ++--------- 2 files changed, 8 insertions(+), 60 deletions(-) diff --git a/src/archetypes/field_setter.h b/src/archetypes/field_setter.h index 171e9f8a8..5c5c4dbe4 100644 --- a/src/archetypes/field_setter.h +++ b/src/archetypes/field_setter.h @@ -170,32 +170,13 @@ namespace arch { const real_t x2_H { metric.template convert<2, Crd::Cd, Crd::Ph>( i2_ + HALF) }; { // dx1 - vec_t d_PU { finit.dx1({ x1_H, x2_0 }), - finit.dx2({ x1_H, x2_0 }), - finit.dx3({ x1_H, x2_0 }) }; - vec_t d_U { ZERO }; - metric.template transform({ i1_ + HALF, i2_ }, - d_PU, - d_U); - EM(i1, i2, em::dx1) = d_U[0]; + EM(i1, i2, em::dx1) = finit.dx1({ x1_H, x2_0 }); } { // dx2 - vec_t d_PU { finit.dx1({ x1_0, x2_H }), - finit.dx2({ x1_0, x2_H }), - finit.dx3({ x1_0, x2_H }) }; - vec_t d_U { ZERO }; - metric.template transform({ i1_, i2_ + HALF }, - d_PU, - d_U); - EM(i1, i2, em::dx2) = d_U[1]; + EM(i1, i2, em::dx2) = finit.dx2({ x1_0, x2_H }); } { // dx3 - vec_t d_PU { finit.dx1({ x1_0, x2_0 }), - finit.dx2({ x1_0, x2_0 }), - finit.dx3({ x1_0, x2_0 }) }; - vec_t d_U { ZERO }; - metric.template transform({ i1_, i2_ }, d_PU, d_U); - EM(i1, i2, em::dx3) = d_U[2]; + EM(i1, i2, em::dx3) = finit.dx3({ x1_0, x2_0 }); } } if constexpr (defines_bx1 && defines_bx2 && defines_bx3) { @@ -206,34 +187,13 @@ namespace arch { const real_t x2_H { metric.template convert<2, Crd::Cd, Crd::Ph>( i2_ + HALF) }; { // bx1 - vec_t b_PU { finit.bx1({ x1_0, x2_H }), - finit.bx2({ x1_0, x2_H }), - finit.bx3({ x1_0, x2_H }) }; - vec_t b_U { ZERO }; - metric.template transform({ i1_, i2_ + HALF }, - b_PU, - b_U); - EM(i1, i2, em::bx1) = b_U[0]; + EM(i1, i2, em::bx1) = finit.bx1({ x1_0, x2_H }); } { // bx2 - vec_t b_PU { finit.bx1({ x1_H, x2_0 }), - finit.bx2({ x1_H, x2_0 }), - finit.bx3({ x1_H, x2_0 }) }; - vec_t b_U { ZERO }; - metric.template transform({ i1_ + HALF, i2_ }, - b_PU, - b_U); - EM(i1, i2, em::bx2) = b_U[1]; + EM(i1, i2, em::bx2) = finit.bx2({ x1_H, x2_0 }); } { // bx3 - vec_t b_PU { finit.bx1({ x1_H, x2_H }), - finit.bx2({ x1_H, x2_H }), - finit.bx3({ x1_H, x2_H }) }; - vec_t b_U { ZERO }; - metric.template transform({ i1_ + HALF, i2_ + HALF }, - b_PU, - b_U); - EM(i1, i2, em::bx3) = b_U[2]; + EM(i1, i2, em::bx3) = finit.bx3({ x1_H, x2_H }); } } } else { diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 9d9db5488..73b169a71 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -998,26 +998,14 @@ namespace kernel::bc { const real_t x1_0 { metric.template convert<1, Crd::Cd, Crd::Ph>(i1_) }; const real_t x2_H { metric.template convert<2, Crd::Cd, Crd::Ph>( i2_ + HALF) }; - - vec_t b_PU { finit.bx1({ x1_0, x2_H }), - finit.bx2({ x1_0, x2_H }), - finit.bx3({ x1_0, x2_H }) }; - vec_t b_U { ZERO }; - metric.template transform({ i1_, i2_ + HALF }, b_PU, b_U); Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * - b_U[0]; + finit.bx1({ x1_0, x2_H }); } else if (comp == em::bx2) { const real_t x1_H { metric.template convert<1, Crd::Cd, Crd::Ph>( i1_ + HALF) }; const real_t x2_0 { metric.template convert<2, Crd::Cd, Crd::Ph>(i2_) }; - - vec_t b_PU { finit.bx1({ x1_H, x2_0 }), - finit.bx2({ x1_H, x2_0 }), - finit.bx3({ x1_H, x2_0 }) }; - vec_t b_U { ZERO }; - metric.template transform({ i1_ + HALF, i2_ }, b_PU, b_U); Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * - b_U[1]; + finit.bx2({ x1_H, x2_0 }); } } } else { From c1ed57712790dbfa5a08deb94b250f550d433787 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 4 Dec 2024 18:06:57 -0500 Subject: [PATCH 172/234] engines/grpic: updated routines for current (cur --> cur0) and added BC for currents --- src/engines/grpic.hpp | 74 +++++++++++++++++++++++++++++++++---------- 1 file changed, 57 insertions(+), 17 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 785e6df97..1bab9bc39 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -355,7 +355,7 @@ namespace ntt { */ if (deposit_enabled) { timers.start("CurrentDeposit"); - Kokkos::deep_copy(dom.fields.cur, ZERO); + Kokkos::deep_copy(dom.fields.cur0, ZERO); CurrentsDeposit(dom); timers.stop("CurrentDeposit"); @@ -612,6 +612,50 @@ namespace ntt { } } + void CurrentsBoundaryConditions(domain_t& domain) { + /** + * absorbing boundaries + */ + const auto ds = m_params.template get( + "grid.boundaries.absorb.ds"); + const auto dim = in::x1; + real_t xg_min, xg_max, xg_edge; + xg_max = m_metadomain.mesh().extent(dim).second; + xg_min = xg_max - ds; + xg_edge = xg_max; + + boundaries_t box; + boundaries_t incl_ghosts; + for (unsigned short d { 0 }; d < M::Dim; ++d) { + if (d == static_cast(dim)) { + box.push_back({ xg_min, xg_max }); + incl_ghosts.push_back({ false, true }); + } else { + box.push_back(Range::All); + incl_ghosts.push_back({ true, true }); + } + } + if (not domain.mesh.Intersects(box)) { + return; + } + const auto intersect_range = domain.mesh.ExtentToRange(box, incl_ghosts); + tuple_t range_min { 0 }; + tuple_t range_max { 0 }; + + for (unsigned short d { 0 }; d < M::Dim; ++d) { + range_min[d] = intersect_range[d].first; + range_max[d] = intersect_range[d].second; + } + Kokkos::parallel_for( + "AbsorbCurrent", + CreateRangePolicy(range_min, range_max), + kernel::AbsorbCurrentGR_kernel(domain.fields.cur0, + domain.mesh.metric, + xg_edge, + ds, + tags)); + } + void OpenFieldsIn(dir::direction_t direction, domain_t& domain, BCTags tags, @@ -868,8 +912,8 @@ namespace ntt { const auto coeff = -dt * q0 * n0 / B0; // auto range = range_with_axis_BCs(domain); auto range = CreateRangePolicy( - { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) + 1}, - { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2)}); + { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2)}, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1}); const auto ni2 = domain.mesh.n_active(in::x2); if (g == gr_ampere::aux) { @@ -916,7 +960,7 @@ namespace ntt { void CurrentsDeposit(domain_t& domain) { auto scatter_cur = Kokkos::Experimental::create_scatter_view( - domain.fields.cur); + domain.fields.cur0); for (auto& species : domain.species) { logger::Checkpoint( fmt::format("Launching currents deposit kernel for %d [%s] : %lu %f", @@ -954,31 +998,27 @@ namespace ntt { (real_t)(species.charge()), dt)); } - Kokkos::Experimental::contribute(domain.fields.cur, scatter_cur); + Kokkos::Experimental::contribute(domain.fields.cur0, scatter_cur); } void CurrentsFilter(domain_t& domain) { logger::Checkpoint("Launching currents filtering kernels", HERE); - auto range = range_with_axis_BCs(domain); + auto range = CreateRangePolicy( + { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2)}, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1}); const auto nfilter = m_params.template get( "algorithms.current_filters"); tuple_t size; - if constexpr (M::Dim == Dim::_1D || M::Dim == Dim::_2D || M::Dim == Dim::_3D) { - size[0] = domain.mesh.n_active(in::x1); - } - if constexpr (M::Dim == Dim::_2D || M::Dim == Dim::_3D) { - size[1] = domain.mesh.n_active(in::x2); - } - if constexpr (M::Dim == Dim::_3D) { - size[2] = domain.mesh.n_active(in::x3); - } + size[0] = domain.mesh.n_active(in::x1); + size[1] = domain.mesh.n_active(in::x2); + // !TODO: this needs to be done more efficiently for (unsigned short i = 0; i < nfilter; ++i) { - Kokkos::deep_copy(domain.fields.buff, domain.fields.cur); + Kokkos::deep_copy(domain.fields.buff, domain.fields.cur0); Kokkos::parallel_for("CurrentsFilter", range, kernel::DigitalFilter_kernel( - domain.fields.cur, + domain.fields.cur0, domain.fields.buff, size, domain.mesh.flds_bc())); From 062a8b8ca3d1f165ab4da16f6a22ab05e0e820d6 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 4 Dec 2024 18:25:27 -0500 Subject: [PATCH 173/234] BC for currents --- src/engines/grpic.hpp | 3 +-- src/kernels/fields_bcs.hpp | 43 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 1bab9bc39..271017917 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -652,8 +652,7 @@ namespace ntt { kernel::AbsorbCurrentGR_kernel(domain.fields.cur0, domain.mesh.metric, xg_edge, - ds, - tags)); + ds)); } void OpenFieldsIn(dir::direction_t direction, diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 73b169a71..3a949671a 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -1016,6 +1016,49 @@ namespace kernel::bc { } }; + template + struct AbsorbCurrentGR_kernel { + static_assert(M::is_metric, "M must be a metric class"); + static_assert(i <= static_cast(M::Dim), + "Invalid component index"); + + ndfield_t J; + const M metric; + const real_t xg_edge; + const real_t dx_abs; + const BCTags tags; + + AbsorbCurrentGR_kernel(ndfield_t J, + const M& metric, + real_t xg_edge, + real_t dx_abs, + BCTags tags) + : J { J } + , metric { metric } + , xg_edge { xg_edge } + , dx_abs { dx_abs } + , tags { tags } {} + + Inline void operator()(index_t i1, index_t i2) const { + if constexpr (M::Dim == Dim::_2D) { + const auto i1_ = COORD(i1); + const auto i2_ = COORD(i2); + coord_t x_Cd { ZERO }; + x_Cd[0] = i1_; + x_Cd[1] = i2_; + const auto dx = math::abs( + metric.template convert(x_Cd[i - 1]) - xg_edge); + J(i1, i2) *= math::tanh(dx / (INV_4 * dx_abs)); + + } else { + raise::KernelError( + HERE, + "AbsorbFields_kernel: 2D implementation called for D != 2"); + } + } + }; + } // namespace kernel::bc +>>>>>>> 794beaab (BC for currents) #endif // KERNELS_FIELDS_BCS_HPP From ca04d9686e965b1c452b6d759ca51a55ec82e700 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 6 Dec 2024 11:22:07 -0500 Subject: [PATCH 174/234] engines/grpic: discard first step if fieldsolver = false --- src/engines/grpic.hpp | 292 ++++++++++++++++++++++-------------------- 1 file changed, 151 insertions(+), 141 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 271017917..74ae4049e 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -101,164 +101,174 @@ namespace ntt { "algorithms.toggles.deposit"); const auto sort_interval = m_params.template get( "particles.sort_interval"); - + if (step == 0) { - // communicate fields and apply BCs on the first timestep - /** - * Initially: em0::B -- - * em0::D -- - * em::B at -1/2 - * em::D at -1/2 - * - * cur0::J -- - * cur::J -- - * - * aux::E -- - * aux::H -- - * - * x_prtl at -1/2 - * u_prtl at -1/2 - */ + if (fieldsolver_enabled) { + // communicate fields and apply BCs on the first timestep + /** + * Initially: em0::B -- + * em0::D -- + * em::B at -1/2 + * em::D at -1/2 + * + * cur0::J -- + * cur::J -- + * + * aux::E -- + * aux::H -- + * + * x_prtl at -1/2 + * u_prtl at -1/2 + */ - /** - * em0::D, em::D, em0::B, em::B <- boundary conditions - */ - m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0 | Comm::D | Comm::D0); - FieldBoundaries(dom, BC::B | BC::E, gr_bc::main); + /** + * em0::D, em::D, em0::B, em::B <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0 | Comm::D | Comm::D0); + FieldBoundaries(dom, BC::B | BC::E, gr_bc::main); - /** - * em0::B <- em::B - * em0::D <- em::D - * - * Now: em0::B & em0::D at -1/2 - */ - CopyFields(dom); + /** + * em0::B <- em::B + * em0::D <- em::D + * + * Now: em0::B & em0::D at -1/2 + */ + CopyFields(dom); - /** - * aux::E <- alpha * em::D + beta x em0::B - * aux::H <- alpha * em::B0 - beta x em::D - * - * Now: aux::E & aux::H at -1/2 - */ - ComputeAuxE(dom, gr_getE::D_B0); - ComputeAuxH(dom, gr_getH::D_B0); + /** + * aux::E <- alpha * em::D + beta x em0::B + * aux::H <- alpha * em::B0 - beta x em::D + * + * Now: aux::E & aux::H at -1/2 + */ + ComputeAuxE(dom, gr_getE::D_B0); + ComputeAuxH(dom, gr_getH::D_B0); - /** - * aux::E, aux::H <- boundary conditions - */ - FieldBoundaries(dom, BC::B | BC::E, gr_bc::aux); + /** + * aux::E, aux::H <- boundary conditions + */ + FieldBoundaries(dom, BC::B | BC::E, gr_bc::aux); - /** - * em0::B <- (em0::B) <- -curl aux::E - * - * Now: em0::B at 0 - */ - Faraday(dom, gr_faraday::aux, HALF); + /** + * em0::B <- (em0::B) <- -curl aux::E + * + * Now: em0::B at 0 + */ + Faraday(dom, gr_faraday::aux, HALF); - /** - * em0::B, em::B <- boundary conditions - */ - m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); - FieldBoundaries(dom, BC::B, gr_bc::main); + /** + * em0::B, em::B <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); + FieldBoundaries(dom, BC::B, gr_bc::main); - /** - * em::D <- (em0::D) <- curl aux::H - * - * Now: em::D at 0 - */ - Ampere(dom, gr_ampere::init, HALF); + /** + * em::D <- (em0::D) <- curl aux::H + * + * Now: em::D at 0 + */ + Ampere(dom, gr_ampere::init, HALF); - /** - * em0::D, em::D <- boundary conditions - */ - m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); - FieldBoundaries(dom, BC::E, gr_bc::main); + /** + * em0::D, em::D <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + FieldBoundaries(dom, BC::E, gr_bc::main); - /** - * aux::E <- alpha * em::D + beta x em0::B - * aux::H <- alpha * em0::B - beta x em::D - * - * Now: aux::E & aux::H at 0 - */ - ComputeAuxE(dom, gr_getE::D_B0); - ComputeAuxH(dom, gr_getH::D_B0); + /** + * aux::E <- alpha * em::D + beta x em0::B + * aux::H <- alpha * em0::B - beta x em::D + * + * Now: aux::E & aux::H at 0 + */ + ComputeAuxE(dom, gr_getE::D_B0); + ComputeAuxH(dom, gr_getH::D_B0); - /** - * aux::E, aux::H <- boundary conditions - */ - FieldBoundaries(dom, BC::B | BC::E, gr_bc::aux); + /** + * aux::E, aux::H <- boundary conditions + */ + FieldBoundaries(dom, BC::B | BC::E, gr_bc::aux); - // !ADD: GR -- particles? + // !ADD: GR -- particles? - /** - * em0::B <- (em::B) <- -curl aux::E - * - * Now: em0::B at 1/2 - */ - Faraday(dom, gr_faraday::main, ONE); - /** - * em0::B, em::B <- boundary conditions - */ - m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); - FieldBoundaries(dom, BC::B, gr_bc::main); + /** + * em0::B <- (em::B) <- -curl aux::E + * + * Now: em0::B at 1/2 + */ + Faraday(dom, gr_faraday::main, ONE); + /** + * em0::B, em::B <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); + FieldBoundaries(dom, BC::B, gr_bc::main); - /** - * em0::D <- (em0::D) <- curl aux::H - * - * Now: em0::D at 1/2 - */ - Ampere(dom, gr_ampere::aux, ONE); - /** - * em0::D, em::D <- boundary conditions - */ - m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); - FieldBoundaries(dom, BC::E, gr_bc::main); + /** + * em0::D <- (em0::D) <- curl aux::H + * + * Now: em0::D at 1/2 + */ + Ampere(dom, gr_ampere::aux, ONE); + /** + * em0::D, em::D <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + FieldBoundaries(dom, BC::E, gr_bc::main); - /** - * aux::H <- alpha * em0::B - beta x em0::D - * - * Now: aux::H at 1/2 - */ - ComputeAuxH(dom, gr_getH::D0_B0); - /** - * aux::H <- boundary conditions - */ - FieldBoundaries(dom, BC::B, gr_bc::aux); + /** + * aux::H <- alpha * em0::B - beta x em0::D + * + * Now: aux::H at 1/2 + */ + ComputeAuxH(dom, gr_getH::D0_B0); + /** + * aux::H <- boundary conditions + */ + FieldBoundaries(dom, BC::B, gr_bc::aux); - /** - * em0::D <- (em::D) <- curl aux::H - * - * Now: em0::D at 1 - * em::D at 0 - */ - Ampere(dom, gr_ampere::main, ONE); - /** - * em0::D, em::D <- boundary conditions - */ - m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); - FieldBoundaries(dom, BC::E, gr_bc::main); + /** + * em0::D <- (em::D) <- curl aux::H + * + * Now: em0::D at 1 + * em::D at 0 + */ + Ampere(dom, gr_ampere::main, ONE); + /** + * em0::D, em::D <- boundary conditions + */ + m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + FieldBoundaries(dom, BC::E, gr_bc::main); - /** - * em::D <-> em0::D - * em::B <-> em0::B - * em::J <-> em0::J - */ - SwapFields(dom); - /** - * Finally: em0::B at -1/2 - * em0::D at 0 - * em::B at 1/2 - * em::D at 1 - * - * cur0::J -- - * cur::J -- - * - * aux::E -- - * aux::H -- - * - * x_prtl at 1 - * u_prtl at 1/2 - */ + /** + * em::D <-> em0::D + * em::B <-> em0::B + * em::J <-> em0::J + */ + SwapFields(dom); + /** + * Finally: em0::B at -1/2 + * em0::D at 0 + * em::B at 1/2 + * em::D at 1 + * + * cur0::J -- + * cur::J -- + * + * aux::E -- + * aux::H -- + * + * x_prtl at 1 + * u_prtl at 1/2 + */ + } + } else { + /** + * em0::B <- em::B + * em0::D <- em::D + * + * Now: em0::B & em0::D at -1/2 + */ + CopyFields(dom); } /** From 30f87f49c905559ab4b059ae0909f5e44f68c97f Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 6 Dec 2024 13:58:38 -0500 Subject: [PATCH 175/234] GeodesicFullPush bug for utheta --- src/kernels/particle_pusher_gr.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kernels/particle_pusher_gr.hpp b/src/kernels/particle_pusher_gr.hpp index 306f19013..4b5432891 100644 --- a/src/kernels/particle_pusher_gr.hpp +++ b/src/kernels/particle_pusher_gr.hpp @@ -478,7 +478,7 @@ namespace kernel::gr { dt * (-metric.alpha(xp_mid) * u0 * DERIVATIVE_IN_TH(metric.alpha, xp_mid) + - vp_mid[1] * DERIVATIVE_IN_TH(metric.beta1, xp_mid) - + vp_mid[0] * DERIVATIVE_IN_TH(metric.beta1, xp_mid) - (HALF / u0) * (DERIVATIVE_IN_TH((metric.template h<1, 1>), xp_mid) * SQR(vp_mid[0]) + From 1fd9cc2dbbb7d8297704c00ff5b4150d2ae95f9b Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 11 Dec 2024 16:36:19 -0500 Subject: [PATCH 176/234] minor gr pusher --- src/kernels/particle_pusher_gr.hpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/kernels/particle_pusher_gr.hpp b/src/kernels/particle_pusher_gr.hpp index 4b5432891..e220239bf 100644 --- a/src/kernels/particle_pusher_gr.hpp +++ b/src/kernels/particle_pusher_gr.hpp @@ -678,13 +678,9 @@ namespace kernel::gr { dx2_prev(p) = dx2(p); coord_t xp { ZERO }; - coord_t xp_prev { ZERO }; xp[0] = i_di_to_Xi(i1(p), dx1(p)); xp[1] = i_di_to_Xi(i2(p), dx2(p)); - - xp_prev[0] = i_di_to_Xi(i1_prev(p), dx1_prev(p)); - xp_prev[1] = i_di_to_Xi(i2_prev(p), dx2_prev(p)); vec_t Dp_cntrv { ZERO }, Bp_cntrv { ZERO }, Dp_hat { ZERO }, Bp_hat { ZERO }; From cd11618a646d6aac17e1ce73defa64284d562780 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 11 Dec 2024 18:20:10 -0500 Subject: [PATCH 177/234] pgen and inputs for pusher debugging updated --- setups/grpic/pusher/boris.toml | 52 ++++---- setups/grpic/pusher/massive_gravity_a0.toml | 4 +- .../grpic/pusher/massive_gravity_a0995.toml | 4 +- setups/grpic/pusher/pgen.hpp | 124 +++++++++++++----- 4 files changed, 122 insertions(+), 62 deletions(-) diff --git a/setups/grpic/pusher/boris.toml b/setups/grpic/pusher/boris.toml index 6483eb2c6..5d4d92a9d 100644 --- a/setups/grpic/pusher/boris.toml +++ b/setups/grpic/pusher/boris.toml @@ -1,14 +1,14 @@ [simulation] name = "boris_a0" engine = "grpic" - runtime = 100.0 + runtime = 500.0 [grid] resolution = [128,128] - extent = [[0.9, 20.0]] + extent = [[1.2, 20.0]] [grid.metric] - metric = "qkerr_schild" + metric = "kerr_schild" qsph_r0 = 0.0 qsph_h = 0.0 ks_a = 0.0 @@ -22,7 +22,7 @@ [scales] larmor0 = 5.0 - skindepth0 = 1e-3 + skindepth0 = 1e0 [algorithms] current_filters = 4 @@ -32,7 +32,7 @@ pusher_eps = 1e-6 [algorithms.timestep] - CFL = 0.1 + CFL = 1.0 [algorithms.toggles] deposit = false @@ -42,37 +42,33 @@ ppc0 = 1.0 [[particles.species]] - label = "e-" + label = "e+" mass = 1.0 - charge = -1.0 + charge = 1.0 maxnpart = 1e0 [[particles.species]] - label = "e+" + label = "e-" mass = 1.0 charge = -1.0 - maxnpart = 1e1 + maxnpart = 1e0 [setup] - x1s = [5.77] - y1s = [1.570796] - z1s = [0.000000] - ux1s = [0.4376657824933686] - uy1s = [0.000000] - uz1s = [4.0] -# x1s = [10.0] -# y1s = [1.570796] -# z1s = [0.000000] -# ux1s = [0.0] -# uy1s = [0.000000] -# uz1s = [10.00000] - - #x2s = [7.5] - #y2s = [0.15745454545454546] - #z2s = [0.000000] - #ux2s = [0.029637] - #uy2s = [0.000000] - #uz2s = [3.75] + # Fig. 6.4 from Kolos, Stuchlik and Tursunov (2015) + x1s = [5.77] + y1s = [1.5707963267948966] + z1s = [0.000000] + ux1s = [0.43766578249336874] + uy1s = [0.000000] + uz1s = [1.1707100000000000] + + # Fig. 5.3 from Kolos, Stuchlik and Tursunov (2015) + x2s = [6.79] + y2s = [1.5707963267948966] + z2s = [0.000000] + ux2s = [0.7398747390396659] + uy2s = [0.000000] + uz2s = [12.610410000000002] [output] format = "hdf5" diff --git a/setups/grpic/pusher/massive_gravity_a0.toml b/setups/grpic/pusher/massive_gravity_a0.toml index 08b607d4b..8179432da 100644 --- a/setups/grpic/pusher/massive_gravity_a0.toml +++ b/setups/grpic/pusher/massive_gravity_a0.toml @@ -56,7 +56,7 @@ [setup] # (1,2,0) from Levin&Perez-Giz (2008) x1s = [9.85180645] - y1s = [1.570796] + y1s = [1.5707963267948966] z1s = [0.5235987755982988] ux1s = [0.24159816] uy1s = [3.535534] @@ -64,7 +64,7 @@ # (4,1,1) from Levin&Perez-Giz (2008) x2s = [68.600387] - y2s = [1.570796] + y2s = [1.5707963267948966] z2s = [0.000000] ux2s = [0.029637] uy2s = [0.000000] diff --git a/setups/grpic/pusher/massive_gravity_a0995.toml b/setups/grpic/pusher/massive_gravity_a0995.toml index b04538e10..3b64d2066 100644 --- a/setups/grpic/pusher/massive_gravity_a0995.toml +++ b/setups/grpic/pusher/massive_gravity_a0995.toml @@ -56,7 +56,7 @@ [setup] # (2,3,1) from Levin&Perez-Giz (2008) x1s = [10.343515586064923] - y1s = [1.570796] + y1s = [1.5707963267948966] z1s = [0.000000] ux1s = [0.19483849893499136] uy1s = [0.000000] @@ -64,7 +64,7 @@ # (~1000, 3, ~671) from Levin&Perez-Giz (2008) x2s = [10.64975354] - y2s = [1.570796] + y2s = [1.5707963267948966] z2s = [0.000000] ux2s = [0.18914503] uy2s = [0.000000] diff --git a/setups/grpic/pusher/pgen.hpp b/setups/grpic/pusher/pgen.hpp index d6a29a185..8f60b1515 100644 --- a/setups/grpic/pusher/pgen.hpp +++ b/setups/grpic/pusher/pgen.hpp @@ -27,12 +27,12 @@ namespace user { InitFields(M metric_) : metric { metric_ } {} Inline auto A_3(const coord_t& x_Cd) const -> real_t { - // return HALF * (metric.template h_<3, 3>(x_Cd) ); - // + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) - // ); - coord_t x_Ph { ZERO }; - metric.template convert(x_Cd, x_Ph); - return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); + return HALF * (metric.template h_<3, 3>(x_Cd) + + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + ); + // coord_t x_Ph { ZERO }; + // metric.template convert(x_Cd, x_Ph); + // return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); } Inline auto A_1(const coord_t& x_Cd) const -> real_t { @@ -49,7 +49,7 @@ namespace user { + TWO * metric.spin() * g_00); } - Inline auto bx1(const coord_t& x_Ph) const -> real_t { + Inline auto bx1(const coord_t& x_Ph) const -> real_t { // at ( i , j + HALF ) coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); @@ -64,50 +64,114 @@ namespace user { return ONE; else return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; - // return ZERO; } - Inline auto bx2(const coord_t& x_Ph) const -> real_t { + Inline auto bx2(const coord_t& x_Ph) const -> real_t { // at ( i + HALF , j ) coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); - x0m[0] = xi[0] + HALF - HALF; + x0m[0] = xi[0] - HALF; x0m[1] = xi[1]; - x0p[0] = xi[0] + HALF + HALF; + x0p[0] = xi[0] + HALF; x0p[1] = xi[1]; - real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + real_t inv_sqrt_detH_iPj { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; if (cmp::AlmostZero(x_Ph[1])) return ZERO; else - return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; - // return ZERO; + return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_iPj; } - Inline auto bx3(const coord_t& x_Ph) const -> real_t { - // coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; - // metric.template convert(x_Ph, xi); + Inline auto bx3(const coord_t& x_Ph) const -> real_t { // at ( i + HALF , j + HALF ) + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); - // x0m[0] = xi[0] + HALF - HALF; - // x0m[1] = xi[1]; - // x0p[0] = xi[0] + HALF + HALF; - // x0p[1] = xi[1]; + x0m[0] = xi[0]; + x0m[1] = xi[1] - HALF; + x0p[0] = xi[0]; + x0p[1] = xi[1] + HALF; - // real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; - // return -(A_1(x0p) - A_1(x0m)) * inv_sqrt_detH_ijP; - return ZERO; + real_t inv_sqrt_detH_iPjP { ONE / metric.sqrt_det_h({ xi[0], xi[1]}) }; + return -(A_1(x0p) - A_1(x0m)) * inv_sqrt_detH_iPjP; } - Inline auto dx1(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto dx1(const coord_t& x_Ph) const -> real_t { // at ( i + HALF , j ) + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + real_t alpha_iPj { metric.alpha({ xi[0], xi[1] }) }; + real_t inv_sqrt_detH_ij { ONE / metric.sqrt_det_h({ xi[0] - HALF, xi[1] }) }; + real_t sqrt_detH_ij { metric.sqrt_det_h({ xi[0] - HALF, xi[1] }) }; + real_t beta_ij { metric.beta1({ xi[0] - HALF, xi[1]}) }; + real_t alpha_ij { metric.alpha({ xi[0] - HALF, xi[1]}) }; + + // D1 at ( i + HALF , j ) + x0m[0] = xi[0] - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF; + x0p[1] = xi[1]; + real_t E1d { (A_0(x0p) - A_0(x0m)) }; + real_t D1d { E1d / alpha_iPj }; + + // D3 at ( i , j ) + x0m[0] = xi[0] - HALF - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] - HALF + HALF; + x0p[1] = xi[1]; + real_t B2_aux { -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ij }; + real_t D3d { -sqrt_detH_ij * beta_ij * B2_aux / alpha_ij }; + + real_t D1u { metric.template h<1, 1>({ xi[0], xi[1] }) * D1d + metric.template h<1, 3>({ xi[0], xi[1] }) * D3d }; + + return D1u; } - Inline auto dx2(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto dx2(const coord_t& x_Ph) const -> real_t { // at ( i , j + HALF ) + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + x0m[0] = xi[0]; + x0m[1] = xi[1] - HALF; + x0p[0] = xi[0]; + x0p[1] = xi[1] + HALF; + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + real_t sqrt_detH_ijP { metric.sqrt_det_h({ xi[0], xi[1] }) }; + real_t alpha_ijP { metric.alpha({ xi[0], xi[1] }) }; + real_t beta_ijP { metric.beta1({ xi[0], xi[1] }) }; + + real_t E2d { (A_0(x0p) - A_0(x0m)) }; + real_t B3_aux { -(A_1(x0p) - A_1(x0m)) * inv_sqrt_detH_ijP }; + real_t D2d { E2d / alpha_ijP + sqrt_detH_ijP * beta_ijP * B3_aux / alpha_ijP }; + real_t D2u { metric.template h<2, 2>({ xi[0], xi[1] }) * D2d }; + + return D2u; } - Inline auto dx3(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto dx3(const coord_t& x_Ph) const -> real_t { // at ( i , j ) + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + real_t inv_sqrt_detH_ij { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + real_t sqrt_detH_ij { metric.sqrt_det_h({ xi[0], xi[1] }) }; + real_t beta_ij { metric.beta1({ xi[0], xi[1] }) }; + real_t alpha_ij { metric.alpha({ xi[0], xi[1] }) }; + real_t alpha_iPj { metric.alpha({ xi[0] + HALF, xi[1] }) }; + + // D3 at ( i , j ) + x0m[0] = xi[0] - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF; + x0p[1] = xi[1]; + real_t B2_aux { -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ij}; + real_t D3d { -sqrt_detH_ij * beta_ij * B2_aux / alpha_ij }; + + // D1 at ( i + HALF , j ) + x0m[0] = xi[0] + HALF - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF + HALF; + x0p[1] = xi[1]; + real_t E1d { (A_0(x0p) - A_0(x0m)) }; + real_t D1d { E1d / alpha_iPj }; + + return metric.template h<3, 3>({ xi[0], xi[1] }) * D3d + metric.template h<1, 3>({ xi[0], xi[1] }) * D1d; } private: From 9e3c47f0ae155c168af9c8301a5ac6dbb0bdd0e1 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Mon, 16 Dec 2024 15:38:24 -0500 Subject: [PATCH 178/234] boris tests with non-zero spin --- setups/grpic/pusher/boris_a09.toml | 85 ++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 setups/grpic/pusher/boris_a09.toml diff --git a/setups/grpic/pusher/boris_a09.toml b/setups/grpic/pusher/boris_a09.toml new file mode 100644 index 000000000..379e9c323 --- /dev/null +++ b/setups/grpic/pusher/boris_a09.toml @@ -0,0 +1,85 @@ +[simulation] + name = "boris_a09" + engine = "grpic" + runtime = 500.0 + +[grid] + resolution = [128,128] + extent = [[1.2, 20.0]] + + [grid.metric] + metric = "kerr_schild" + qsph_r0 = 0.0 + qsph_h = 0.0 + ks_a = 0.9 + + [grid.boundaries] + fields = [["ABSORB"]] + particles = [["ABSORB"]] + + [grid.boundaries.absorb] + ds = 0.5 + +[scales] + larmor0 = 0.5 + skindepth0 = 1e0 + +[algorithms] + current_filters = 4 + + [algorithms.gr] + pusher_niter = 10 + pusher_eps = 1e-6 + + [algorithms.timestep] + CFL = 1.0 + + [algorithms.toggles] + deposit = false + fieldsolver = false + +[particles] + ppc0 = 1.0 + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = 1.0 + maxnpart = 1e0 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = -1.0 + maxnpart = 1e0 + +[setup] + # Stuchlik and Kolos (2015) Fig.11 ~ RKA3 from Bacchini et al + x1s = [4.000000] + y1s = [1.3707963268] + z1s = [0.000000] + ux1s = [0.625666] + uy1s = [0.000000] + uz1s = [1.580567] + +[output] + format = "hdf5" + + [output.fields] + enable = true + interval_time = 1.0 + quantities = ["D", "B"] + + [output.particles] + enable = true + stride = 1 + interval_time = 0.2 + species = [] + + [output.spectra] + enable = false + +[diagnostics] + interval = 10 + colored_stdout = true + blocking_timers = true From d0349bf87b83aec3e8b2f204afcbc11758f64ff6 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:22:31 -0500 Subject: [PATCH 179/234] updated setups --- setups/grpic/orbits/orbits.toml | 73 ++++++++++++++++ setups/grpic/orbits/pgen.hpp | 132 ++++++++++++++++++++++++++++ setups/grpic/pusher/deposit.toml | 91 +++++++++++++++++++ setups/grpic/vacuum/pgen.hpp | 146 +++++++++++++++++++++++++++++++ setups/grpic/vacuum/wald.toml | 93 ++++++++++++++++++++ setups/grpic/wald/pgen.hpp | 26 ++++-- setups/grpic/wald/wald.toml | 48 +++++----- 7 files changed, 583 insertions(+), 26 deletions(-) create mode 100644 setups/grpic/orbits/orbits.toml create mode 100644 setups/grpic/orbits/pgen.hpp create mode 100644 setups/grpic/pusher/deposit.toml create mode 100644 setups/grpic/vacuum/pgen.hpp create mode 100644 setups/grpic/vacuum/wald.toml diff --git a/setups/grpic/orbits/orbits.toml b/setups/grpic/orbits/orbits.toml new file mode 100644 index 000000000..ab8f2bef5 --- /dev/null +++ b/setups/grpic/orbits/orbits.toml @@ -0,0 +1,73 @@ +[simulation] + name = "wald" + engine = "grpic" + runtime = 110.0 + +[grid] + resolution = [128,128] + extent = [[1.2, 8.0]] + + [grid.metric] + metric = "kerr_schild" + qsph_r0 = 0.0 + qsph_h = 0.0 + ks_a = 0.95 + + [grid.boundaries] + fields = [["ABSORB"]] + particles = [["ABSORB"]] + + [grid.boundaries.absorb] + ds = 0.5 + +[scales] + larmor0 = 1.0 + skindepth0 = 1.0 + +[algorithms] + current_filters = 4 + + [algorithms.timestep] + CFL = 0.5 + + [algorithms.toggles] + deposit = false + fieldsolver = true + +[particles] + ppc0 = 8.0 + use_weights = true + sort_interval = 100 + +# [[particles.species]] +# label = "e-" +# mass = 1.0 +# charge = -1.0 +# maxnpart = 2e6 +# pusher = "Boris" +# +# [[particles.species]] +# label = "e+" +# mass = 1.0 +# charge = 1.0 +# maxnpart = 2e6 +# pusher = "Boris" + +[setup] + +[output] + format = "hdf5" + + [output.fields] + interval_time = 1.0 + quantities = ["D", "H", "B", "J"] + + [output.particles] + interval_time = 1.0 + quantities = ["X", "U"] + + [output.spectra] + enable = false + +[diagnostics] + interval = 1 diff --git a/setups/grpic/orbits/pgen.hpp b/setups/grpic/orbits/pgen.hpp new file mode 100644 index 000000000..54965b1ee --- /dev/null +++ b/setups/grpic/orbits/pgen.hpp @@ -0,0 +1,132 @@ +#ifndef PROBLEM_GENERATOR_H +#define PROBLEM_GENERATOR_H + +#include "enums.h" +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "arch/traits.h" + +#include "archetypes/problem_generator.h" +#include "framework/domain/metadomain.h" + +namespace user { + using namespace ntt; + + template + struct InitFields { + InitFields(M metric_) : metric { metric_ } {} + + Inline auto A_3(const coord_t& x_Cd) const -> real_t { + // return HALF * (metric.template h_<3, 3>(x_Cd) ); + // + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + // ); + + coord_t x_Ph { ZERO }; + metric.template convert(x_Cd, x_Ph); + return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); + } + + Inline auto A_1(const coord_t& x_Cd) const -> real_t { + return HALF * (metric.template h_<1, 3>(x_Cd) + + TWO * metric.spin() * metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) + ); + } + + Inline auto A_0(const coord_t& x_Cd) const -> real_t { + real_t g_00 { -metric.alpha(x_Cd) * metric.alpha(x_Cd) + + metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) * metric.beta1(x_Cd) + }; + return HALF * (metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + + TWO * metric.spin() * g_00); + } + + Inline auto bx1(const coord_t& x_Ph) const -> real_t { + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0]; + x0m[1] = xi[1] - HALF; + x0p[0] = xi[0]; + x0p[1] = xi[1] + HALF; + + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + + if (cmp::AlmostZero(x_Ph[1])) + return ONE; + else + return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; + } + + Inline auto bx2(const coord_t& x_Ph) const -> real_t { + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0] + HALF - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF + HALF; + x0p[1] = xi[1]; + + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + if (cmp::AlmostZero(x_Ph[1])) + return ZERO; + else + return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; + } + + Inline auto bx3(const coord_t& x_Ph) const -> real_t { + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0] + HALF - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF + HALF; + x0p[1] = xi[1]; + + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + return -(A_1(x0p) - A_1(x0m)) * inv_sqrt_detH_ijP; + // return ZERO; + } + + Inline auto dx1(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx2(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx3(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + private: + const M metric; + }; + + template + struct PGen : public arch::ProblemGenerator { + // compatibility traits for the problem generator + static constexpr auto engines { traits::compatible_with::value }; + static constexpr auto metrics { + traits::compatible_with::value + }; + static constexpr auto dimensions { traits::compatible_with::value }; + + // for easy access to variables in the child class + using arch::ProblemGenerator::D; + using arch::ProblemGenerator::C; + using arch::ProblemGenerator::params; + + InitFields init_flds; + + inline PGen(SimulationParams& p, const Metadomain& m) + : arch::ProblemGenerator(p) + , init_flds { m.mesh().metric } {} + + inline PGen() {} + }; + +} // namespace user + +#endif diff --git a/setups/grpic/pusher/deposit.toml b/setups/grpic/pusher/deposit.toml new file mode 100644 index 000000000..359d4eaa7 --- /dev/null +++ b/setups/grpic/pusher/deposit.toml @@ -0,0 +1,91 @@ +[simulation] + name = "deposit" + engine = "grpic" + runtime = 10.0 + +[grid] + resolution = [512,512] + extent = [[1.2, 20.0]] + + [grid.metric] + metric = "kerr_schild" + qsph_r0 = 0.0 + qsph_h = 0.0 + ks_a = 0.0 + + [grid.boundaries] + fields = [["ABSORB"]] + particles = [["ABSORB"]] + + [grid.boundaries.absorb] + ds = 2.0 + +[scales] + larmor0 = 2.0 + skindepth0 = 2.0 + +[algorithms] + current_filters = 4 + + [algorithms.gr] + pusher_niter = 10 + pusher_eps = 1e-6 + + [algorithms.timestep] + CFL = 0.5 + + [algorithms.toggles] + deposit = true + fieldsolver = true + +[particles] + ppc0 = 1.0 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = -1.0 + maxnpart = 1e0 + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = 1.0 + maxnpart = 1e0 + +[setup] + x1s = [17.00000] + y1s = [1.570796] + z1s = [0.000000] + ux1s = [8.000000] + uy1s = [0.000000] + uz1s = [0.000000] + + x2s = [17.00000] + y2s = [1.570796] + z2s = [0.000000] + ux2s = [-8.000000] + uy2s = [0.000000] + uz2s = [0.000000] + +[output] + format = "hdf5" + + [output.fields] + enable = true + interval_time = 0.5 + quantities = ["D", "B"] + + [output.particles] + enable = true + stride = 1 + interval_time = 0.5 + species = [] + + [output.spectra] + enable = false + +[diagnostics] + interval = 10 + colored_stdout = true + blocking_timers = true diff --git a/setups/grpic/vacuum/pgen.hpp b/setups/grpic/vacuum/pgen.hpp new file mode 100644 index 000000000..c02c198e8 --- /dev/null +++ b/setups/grpic/vacuum/pgen.hpp @@ -0,0 +1,146 @@ +#ifndef PROBLEM_GENERATOR_H +#define PROBLEM_GENERATOR_H + +#include "enums.h" +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "arch/traits.h" +#include "utils/comparators.h" +#include "utils/formatting.h" +#include "utils/log.h" +#include "utils/numeric.h" + +#include "archetypes/problem_generator.h" +#include "framework/domain/metadomain.h" +#include "framework/domain/domain.h" +#include "archetypes/energy_dist.h" +#include "archetypes/particle_injector.h" + +#include + +namespace user { + using namespace ntt; + + template + struct InitFields { + InitFields(M metric_) : metric { metric_ } {} + + Inline auto A_3(const coord_t& x_Cd) const -> real_t { + return HALF * (metric.template h_<3, 3>(x_Cd) ); + // + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + // ); + coord_t x_Ph { ZERO }; + metric.template convert(x_Cd, x_Ph); + return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); + } + + Inline auto A_1(const coord_t& x_Cd) const -> real_t { + return HALF * (metric.template h_<1, 3>(x_Cd) + + TWO * metric.spin() * metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) + ); + } + + Inline auto A_0(const coord_t& x_Cd) const -> real_t { + real_t g_00 { -metric.alpha(x_Cd) * metric.alpha(x_Cd) + + metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) * metric.beta1(x_Cd) + }; + return HALF * (metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + + TWO * metric.spin() * g_00); + } + + Inline auto bx1(const coord_t& x_Ph) const -> real_t { + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0]; + x0m[1] = xi[1] - HALF; + x0p[0] = xi[0]; + x0p[1] = xi[1] + HALF; + + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + + if (cmp::AlmostZero(x_Ph[1])) + return ONE; + else + return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; + } + + Inline auto bx2(const coord_t& x_Ph) const -> real_t { + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0] + HALF - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF + HALF; + x0p[1] = xi[1]; + + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + if (cmp::AlmostZero(x_Ph[1])) + return ZERO; + else + return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; + } + + Inline auto bx3(const coord_t& x_Ph) const -> real_t { + // coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + // metric.template convert(x_Ph, xi); + + // x0m[0] = xi[0] + HALF - HALF; + // x0m[1] = xi[1]; + // x0p[0] = xi[0] + HALF + HALF; + // x0p[1] = xi[1]; + + // real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + // return -(A_1(x0p) - A_1(x0m)) * inv_sqrt_detH_ijP; + return ZERO; + } + + Inline auto dx1(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx2(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx3(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + private: + const M metric; + }; + + template + struct PGen : public arch::ProblemGenerator { + // compatibility traits for the problem generator + static constexpr auto engines { traits::compatible_with::value }; + static constexpr auto metrics { + traits::compatible_with::value + }; + static constexpr auto dimensions { traits::compatible_with::value }; + + // for easy access to variables in the child class + using arch::ProblemGenerator::D; + using arch::ProblemGenerator::C; + using arch::ProblemGenerator::params; + + InitFields init_flds; + const Metadomain& global_domain; + inline PGen(const SimulationParams& p, const Metadomain& m) + : arch::ProblemGenerator { p } + , global_domain { m } + , init_flds { m.mesh().metric } {} + + // inline void InitPrtls(Domain& local_domain) { + + // arch::InjectGlobally(global_domain, local_domain, (arch::spidx_t)1, data_1); + // arch::InjectGlobally(global_domain, local_domain, (arch::spidx_t)2, data_2); + // } + // inline PGen() {} + }; + +} // namespace user + +#endif diff --git a/setups/grpic/vacuum/wald.toml b/setups/grpic/vacuum/wald.toml new file mode 100644 index 000000000..6864efb17 --- /dev/null +++ b/setups/grpic/vacuum/wald.toml @@ -0,0 +1,93 @@ +[simulation] + name = "wald" + engine = "grpic" + runtime = 5000.0 + +[grid] + resolution = [128,128] + extent = [[1.2, 100.0]] + + [grid.metric] + metric = "qkerr_schild" + qsph_r0 = 0.0 + qsph_h = 0.0 + ks_a = 0.0 + + [grid.boundaries] + fields = [["ABSORB"]] + particles = [["ABSORB"]] + + [grid.boundaries.absorb] + ds = 0.5 + +[scales] + larmor0 = 2e-3 + skindepth0 = 0.1 + +[algorithms] + current_filters = 4 + + [algorithms.gr] + pusher_niter = 10 + pusher_eps = 1e-2 + + [algorithms.timestep] + CFL = 10.0 + + [algorithms.toggles] + deposit = false + fieldsolver = false + +[particles] + ppc0 = 4.0 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = 0.0 + maxnpart = 1e1 + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = -1.0 + maxnpart = 1e1 + +[setup] + # (1,1,0) from Levin&Perez-Giz (2008) + x1s = [59.921203] + y1s = [1.570796] + z1s = [0.000000] + ux1s = [0.034020] + uy1s = [0.000000] + uz1s = [3.900000] + + # (4,1,1) from Levin&Perez-Giz (2008) + x2s = [68.600387] + y2s = [1.570796] + z2s = [0.000000] + ux2s = [0.029637] + uy2s = [0.000000] + uz2s = [3.900000] + +[output] + format = "hdf5" + + [output.fields] + enable = true + interval_time = 1.0 + quantities = ["D", "B"] + + [output.particles] + enable = true + stride = 1 + interval_time = 10.0 + species = [] + + [output.spectra] + enable = false + +[diagnostics] + interval = 10 + colored_stdout = true + blocking_timers = true diff --git a/setups/grpic/wald/pgen.hpp b/setups/grpic/wald/pgen.hpp index 51636314b..8dd1a6e4e 100644 --- a/setups/grpic/wald/pgen.hpp +++ b/setups/grpic/wald/pgen.hpp @@ -18,10 +18,9 @@ namespace user { InitFields(M metric_) : metric { metric_ } {} Inline auto A_3(const coord_t& x_Cd) const -> real_t { - // return HALF * (metric.template h_<3, 3>(x_Cd) ); - // + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + // return HALF * (metric.template h_<3, 3>(x_Cd) + // + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) // ); - coord_t x_Ph { ZERO }; metric.template convert(x_Cd, x_Ph); return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); @@ -122,9 +121,26 @@ namespace user { inline PGen(SimulationParams& p, const Metadomain& m) : arch::ProblemGenerator(p) + // , xi_min { p.template get>("setup.inj_rmin") } + // , xi_max { p.template get>("setup.inj_rmax") } , init_flds { m.mesh().metric } {} - - inline PGen() {} + + // inline void InitPrtls(Domain& local_domain) { + // const auto energy_dist = arch::ColdDist(local_domain.mesh.metric); + // const auto spatial_dist = PointDistribution(domain.mesh.metric, + // xi_min, + // xi_max); + // const auto injector = arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // { 1, 2 }); + // arch::InjectNonUniform>(params, + // local_domain, + // injector, + // 1.0); + // } + + // inline PGen() {} }; } // namespace user diff --git a/setups/grpic/wald/wald.toml b/setups/grpic/wald/wald.toml index 36c746ae5..44d1c8a24 100644 --- a/setups/grpic/wald/wald.toml +++ b/setups/grpic/wald/wald.toml @@ -1,17 +1,17 @@ [simulation] name = "wald" engine = "grpic" - runtime = 500.0 + runtime = 110.0 [grid] resolution = [128,128] - extent = [[1.2, 8.0]] + extent = [[2.0, 8.0]] [grid.metric] metric = "qkerr_schild" qsph_r0 = 0.0 qsph_h = 0.0 - ks_a = 0.95 + ks_a = 0.0 [grid.boundaries] fields = [["ABSORB"]] @@ -21,14 +21,14 @@ ds = 0.5 [scales] - larmor0 = 1.0 - skindepth0 = 1.0 + larmor0 = 1.0 #0.0025 + skindepth0 = 1.0 #0.05 [algorithms] current_filters = 4 [algorithms.timestep] - CFL = 0.5 + CFL = 0.3 [algorithms.toggles] deposit = false @@ -39,28 +39,32 @@ use_weights = true sort_interval = 100 -# [[particles.species]] -# label = "e-" -# mass = 1.0 -# charge = -1.0 -# maxnpart = 2e6 -# pusher = "Boris" -# -# [[particles.species]] -# label = "e+" -# mass = 1.0 -# charge = 1.0 -# maxnpart = 2e6 -# pusher = "Boris" + #[[particles.species]] + #label = "e-" + #mass = 1.0 + #charge = -1.0 + #maxnpart = 1e6 + #pusher = "Boris" + # + #[[particles.species]] + #label = "e+" + #mass = 1.0 + #charge = 1.0 + #maxnpart = 1e6 + #pusher = "Boris" [setup] + #multiplicity = 1.0 + #sigma_max = 1000.0 + #inj_rmin = 1.2 + #inj_rmax = 7.5 [output] format = "hdf5" [output.fields] interval_time = 1.0 - quantities = ["D", "H", "B", "J", "A"] + quantities = ["D", "H", "B", "J"] [output.particles] enable = false @@ -69,4 +73,6 @@ enable = false [diagnostics] - interval = 1 + interval = 2 + colored_stdout = true + blocking_timers = true From 1e1c50b3d9c2fdc5243e99752639d1c7c2a7d0bf Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 18 Dec 2024 10:59:29 -0500 Subject: [PATCH 180/234] engines/grpic: fixed the bug i introduced --- src/engines/grpic.hpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 74ae4049e..a4bffadf2 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -260,15 +260,15 @@ namespace ntt { * x_prtl at 1 * u_prtl at 1/2 */ + } else { + /** + * em0::B <- em::B + * em0::D <- em::D + * + * Now: em0::B & em0::D at -1/2 + */ + CopyFields(dom); } - } else { - /** - * em0::B <- em::B - * em0::D <- em::D - * - * Now: em0::B & em0::D at -1/2 - */ - CopyFields(dom); } /** From 1aefb656081e5fc76e9f77153a4ffa55e698c80a Mon Sep 17 00:00:00 2001 From: alisagk <55898700+alisagk@users.noreply.github.com> Date: Wed, 18 Dec 2024 11:10:37 -0500 Subject: [PATCH 181/234] Delete setups/grpic/orbits directory --- setups/grpic/orbits/orbits.toml | 73 ------------------ setups/grpic/orbits/pgen.hpp | 132 -------------------------------- 2 files changed, 205 deletions(-) delete mode 100644 setups/grpic/orbits/orbits.toml delete mode 100644 setups/grpic/orbits/pgen.hpp diff --git a/setups/grpic/orbits/orbits.toml b/setups/grpic/orbits/orbits.toml deleted file mode 100644 index ab8f2bef5..000000000 --- a/setups/grpic/orbits/orbits.toml +++ /dev/null @@ -1,73 +0,0 @@ -[simulation] - name = "wald" - engine = "grpic" - runtime = 110.0 - -[grid] - resolution = [128,128] - extent = [[1.2, 8.0]] - - [grid.metric] - metric = "kerr_schild" - qsph_r0 = 0.0 - qsph_h = 0.0 - ks_a = 0.95 - - [grid.boundaries] - fields = [["ABSORB"]] - particles = [["ABSORB"]] - - [grid.boundaries.absorb] - ds = 0.5 - -[scales] - larmor0 = 1.0 - skindepth0 = 1.0 - -[algorithms] - current_filters = 4 - - [algorithms.timestep] - CFL = 0.5 - - [algorithms.toggles] - deposit = false - fieldsolver = true - -[particles] - ppc0 = 8.0 - use_weights = true - sort_interval = 100 - -# [[particles.species]] -# label = "e-" -# mass = 1.0 -# charge = -1.0 -# maxnpart = 2e6 -# pusher = "Boris" -# -# [[particles.species]] -# label = "e+" -# mass = 1.0 -# charge = 1.0 -# maxnpart = 2e6 -# pusher = "Boris" - -[setup] - -[output] - format = "hdf5" - - [output.fields] - interval_time = 1.0 - quantities = ["D", "H", "B", "J"] - - [output.particles] - interval_time = 1.0 - quantities = ["X", "U"] - - [output.spectra] - enable = false - -[diagnostics] - interval = 1 diff --git a/setups/grpic/orbits/pgen.hpp b/setups/grpic/orbits/pgen.hpp deleted file mode 100644 index 54965b1ee..000000000 --- a/setups/grpic/orbits/pgen.hpp +++ /dev/null @@ -1,132 +0,0 @@ -#ifndef PROBLEM_GENERATOR_H -#define PROBLEM_GENERATOR_H - -#include "enums.h" -#include "global.h" - -#include "arch/kokkos_aliases.h" -#include "arch/traits.h" - -#include "archetypes/problem_generator.h" -#include "framework/domain/metadomain.h" - -namespace user { - using namespace ntt; - - template - struct InitFields { - InitFields(M metric_) : metric { metric_ } {} - - Inline auto A_3(const coord_t& x_Cd) const -> real_t { - // return HALF * (metric.template h_<3, 3>(x_Cd) ); - // + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) - // ); - - coord_t x_Ph { ZERO }; - metric.template convert(x_Cd, x_Ph); - return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); - } - - Inline auto A_1(const coord_t& x_Cd) const -> real_t { - return HALF * (metric.template h_<1, 3>(x_Cd) - + TWO * metric.spin() * metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) - ); - } - - Inline auto A_0(const coord_t& x_Cd) const -> real_t { - real_t g_00 { -metric.alpha(x_Cd) * metric.alpha(x_Cd) - + metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) * metric.beta1(x_Cd) - }; - return HALF * (metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) - + TWO * metric.spin() * g_00); - } - - Inline auto bx1(const coord_t& x_Ph) const -> real_t { - coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; - metric.template convert(x_Ph, xi); - - x0m[0] = xi[0]; - x0m[1] = xi[1] - HALF; - x0p[0] = xi[0]; - x0p[1] = xi[1] + HALF; - - real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; - - if (cmp::AlmostZero(x_Ph[1])) - return ONE; - else - return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; - } - - Inline auto bx2(const coord_t& x_Ph) const -> real_t { - coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; - metric.template convert(x_Ph, xi); - - x0m[0] = xi[0] + HALF - HALF; - x0m[1] = xi[1]; - x0p[0] = xi[0] + HALF + HALF; - x0p[1] = xi[1]; - - real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; - if (cmp::AlmostZero(x_Ph[1])) - return ZERO; - else - return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; - } - - Inline auto bx3(const coord_t& x_Ph) const -> real_t { - coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; - metric.template convert(x_Ph, xi); - - x0m[0] = xi[0] + HALF - HALF; - x0m[1] = xi[1]; - x0p[0] = xi[0] + HALF + HALF; - x0p[1] = xi[1]; - - real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; - return -(A_1(x0p) - A_1(x0m)) * inv_sqrt_detH_ijP; - // return ZERO; - } - - Inline auto dx1(const coord_t& x_Ph) const -> real_t { - return ZERO; - } - - Inline auto dx2(const coord_t& x_Ph) const -> real_t { - return ZERO; - } - - Inline auto dx3(const coord_t& x_Ph) const -> real_t { - return ZERO; - } - - private: - const M metric; - }; - - template - struct PGen : public arch::ProblemGenerator { - // compatibility traits for the problem generator - static constexpr auto engines { traits::compatible_with::value }; - static constexpr auto metrics { - traits::compatible_with::value - }; - static constexpr auto dimensions { traits::compatible_with::value }; - - // for easy access to variables in the child class - using arch::ProblemGenerator::D; - using arch::ProblemGenerator::C; - using arch::ProblemGenerator::params; - - InitFields init_flds; - - inline PGen(SimulationParams& p, const Metadomain& m) - : arch::ProblemGenerator(p) - , init_flds { m.mesh().metric } {} - - inline PGen() {} - }; - -} // namespace user - -#endif From 755aaa82c8d033e9c3222adbf1fb7e2595563ced Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 18 Dec 2024 11:08:29 -0500 Subject: [PATCH 182/234] setups/grpic: updated setup for vacuum --- setups/grpic/vacuum/pgen.hpp | 37 +--------------- setups/grpic/vacuum/vacuum.toml | 78 +++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 35 deletions(-) create mode 100644 setups/grpic/vacuum/vacuum.toml diff --git a/setups/grpic/vacuum/pgen.hpp b/setups/grpic/vacuum/pgen.hpp index c02c198e8..6d5e5fba5 100644 --- a/setups/grpic/vacuum/pgen.hpp +++ b/setups/grpic/vacuum/pgen.hpp @@ -27,28 +27,11 @@ namespace user { InitFields(M metric_) : metric { metric_ } {} Inline auto A_3(const coord_t& x_Cd) const -> real_t { - return HALF * (metric.template h_<3, 3>(x_Cd) ); - // + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) - // ); coord_t x_Ph { ZERO }; metric.template convert(x_Cd, x_Ph); return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); } - Inline auto A_1(const coord_t& x_Cd) const -> real_t { - return HALF * (metric.template h_<1, 3>(x_Cd) - + TWO * metric.spin() * metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) - ); - } - - Inline auto A_0(const coord_t& x_Cd) const -> real_t { - real_t g_00 { -metric.alpha(x_Cd) * metric.alpha(x_Cd) - + metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) * metric.beta1(x_Cd) - }; - return HALF * (metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) - + TWO * metric.spin() * g_00); - } - Inline auto bx1(const coord_t& x_Ph) const -> real_t { coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); @@ -70,9 +53,9 @@ namespace user { coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); - x0m[0] = xi[0] + HALF - HALF; + x0m[0] = xi[0] - HALF; x0m[1] = xi[1]; - x0p[0] = xi[0] + HALF + HALF; + x0p[0] = xi[0] + HALF; x0p[1] = xi[1]; real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; @@ -83,16 +66,6 @@ namespace user { } Inline auto bx3(const coord_t& x_Ph) const -> real_t { - // coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; - // metric.template convert(x_Ph, xi); - - // x0m[0] = xi[0] + HALF - HALF; - // x0m[1] = xi[1]; - // x0p[0] = xi[0] + HALF + HALF; - // x0p[1] = xi[1]; - - // real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; - // return -(A_1(x0p) - A_1(x0m)) * inv_sqrt_detH_ijP; return ZERO; } @@ -132,13 +105,7 @@ namespace user { : arch::ProblemGenerator { p } , global_domain { m } , init_flds { m.mesh().metric } {} - - // inline void InitPrtls(Domain& local_domain) { - // arch::InjectGlobally(global_domain, local_domain, (arch::spidx_t)1, data_1); - // arch::InjectGlobally(global_domain, local_domain, (arch::spidx_t)2, data_2); - // } - // inline PGen() {} }; } // namespace user diff --git a/setups/grpic/vacuum/vacuum.toml b/setups/grpic/vacuum/vacuum.toml new file mode 100644 index 000000000..3992b9f12 --- /dev/null +++ b/setups/grpic/vacuum/vacuum.toml @@ -0,0 +1,78 @@ +[simulation] + name = "wald" + engine = "grpic" + runtime = 500.0 + +[grid] + resolution = [128,128] + extent = [[2.0, 8.0]] + + [grid.metric] + metric = "qkerr_schild" + qsph_r0 = 0.0 + qsph_h = 0.0 + ks_a = 0.0 + + [grid.boundaries] + fields = [["ABSORB"]] + particles = [["ABSORB"]] + + [grid.boundaries.absorb] + ds = 0.5 + +[scales] + larmor0 = 0.0025 + skindepth0 = 0.05 + +[algorithms] + current_filters = 4 + + [algorithms.timestep] + CFL = 0.5 + + [algorithms.toggles] + deposit = false + fieldsolver = true + +[particles] + ppc0 = 8.0 + use_weights = true + sort_interval = 100 + + #[[particles.species]] + #label = "e-" + #mass = 1.0 + #charge = -1.0 + #maxnpart = 1e6 + #pusher = "Boris" + # + #[[particles.species]] + #label = "e+" + #mass = 1.0 + #charge = 1.0 + #maxnpart = 1e6 + #pusher = "Boris" + +[setup] + #multiplicity = 1.0 + #sigma_max = 1000.0 + inj_rmin = 1.2 + inj_rmax = 7.5 + +[output] + format = "hdf5" + + [output.fields] + interval_time = 1.0 + quantities = ["D", "H", "B", "J"] + + [output.particles] + enable = false + + [output.spectra] + enable = false + +[diagnostics] + interval = 2 + colored_stdout = true + blocking_timers = true From ba72d3ba16bae7c5165dfa66999d60a27d50d7d3 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 18 Dec 2024 11:15:32 -0500 Subject: [PATCH 183/234] updated setups --- setups/grpic/vacuum/wald.toml | 93 ----------------------------------- setups/grpic/wald/pgen.hpp | 4 +- setups/grpic/wald/wald.toml | 12 ++--- 3 files changed, 8 insertions(+), 101 deletions(-) delete mode 100644 setups/grpic/vacuum/wald.toml diff --git a/setups/grpic/vacuum/wald.toml b/setups/grpic/vacuum/wald.toml deleted file mode 100644 index 6864efb17..000000000 --- a/setups/grpic/vacuum/wald.toml +++ /dev/null @@ -1,93 +0,0 @@ -[simulation] - name = "wald" - engine = "grpic" - runtime = 5000.0 - -[grid] - resolution = [128,128] - extent = [[1.2, 100.0]] - - [grid.metric] - metric = "qkerr_schild" - qsph_r0 = 0.0 - qsph_h = 0.0 - ks_a = 0.0 - - [grid.boundaries] - fields = [["ABSORB"]] - particles = [["ABSORB"]] - - [grid.boundaries.absorb] - ds = 0.5 - -[scales] - larmor0 = 2e-3 - skindepth0 = 0.1 - -[algorithms] - current_filters = 4 - - [algorithms.gr] - pusher_niter = 10 - pusher_eps = 1e-2 - - [algorithms.timestep] - CFL = 10.0 - - [algorithms.toggles] - deposit = false - fieldsolver = false - -[particles] - ppc0 = 4.0 - - [[particles.species]] - label = "e-" - mass = 1.0 - charge = 0.0 - maxnpart = 1e1 - - [[particles.species]] - label = "e+" - mass = 1.0 - charge = -1.0 - maxnpart = 1e1 - -[setup] - # (1,1,0) from Levin&Perez-Giz (2008) - x1s = [59.921203] - y1s = [1.570796] - z1s = [0.000000] - ux1s = [0.034020] - uy1s = [0.000000] - uz1s = [3.900000] - - # (4,1,1) from Levin&Perez-Giz (2008) - x2s = [68.600387] - y2s = [1.570796] - z2s = [0.000000] - ux2s = [0.029637] - uy2s = [0.000000] - uz2s = [3.900000] - -[output] - format = "hdf5" - - [output.fields] - enable = true - interval_time = 1.0 - quantities = ["D", "B"] - - [output.particles] - enable = true - stride = 1 - interval_time = 10.0 - species = [] - - [output.spectra] - enable = false - -[diagnostics] - interval = 10 - colored_stdout = true - blocking_timers = true diff --git a/setups/grpic/wald/pgen.hpp b/setups/grpic/wald/pgen.hpp index 8dd1a6e4e..b87d25ed2 100644 --- a/setups/grpic/wald/pgen.hpp +++ b/setups/grpic/wald/pgen.hpp @@ -61,9 +61,9 @@ namespace user { coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); - x0m[0] = xi[0] + HALF - HALF; + x0m[0] = xi[0] - HALF; x0m[1] = xi[1]; - x0p[0] = xi[0] + HALF + HALF; + x0p[0] = xi[0] + HALF; x0p[1] = xi[1]; real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; diff --git a/setups/grpic/wald/wald.toml b/setups/grpic/wald/wald.toml index 44d1c8a24..6e68d4324 100644 --- a/setups/grpic/wald/wald.toml +++ b/setups/grpic/wald/wald.toml @@ -1,7 +1,7 @@ [simulation] name = "wald" engine = "grpic" - runtime = 110.0 + runtime = 510.0 [grid] resolution = [128,128] @@ -21,14 +21,14 @@ ds = 0.5 [scales] - larmor0 = 1.0 #0.0025 - skindepth0 = 1.0 #0.05 + larmor0 = 0.0025 + skindepth0 = 0.05 [algorithms] current_filters = 4 [algorithms.timestep] - CFL = 0.3 + CFL = 0.5 [algorithms.toggles] deposit = false @@ -56,8 +56,8 @@ [setup] #multiplicity = 1.0 #sigma_max = 1000.0 - #inj_rmin = 1.2 - #inj_rmax = 7.5 + inj_rmin = 1.2 + inj_rmax = 7.5 [output] format = "hdf5" From 70edd434022261a954cdf7f44e83f0ba707975e2 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 18 Dec 2024 13:55:24 -0500 Subject: [PATCH 184/234] added separate setup for deposit debugging --- setups/grpic/deposit/deposit.toml | 91 ++++++++++++++++++++++++ setups/grpic/deposit/pgen.hpp | 113 ++++++++++++++++++++++++++++++ 2 files changed, 204 insertions(+) create mode 100644 setups/grpic/deposit/deposit.toml create mode 100644 setups/grpic/deposit/pgen.hpp diff --git a/setups/grpic/deposit/deposit.toml b/setups/grpic/deposit/deposit.toml new file mode 100644 index 000000000..359d4eaa7 --- /dev/null +++ b/setups/grpic/deposit/deposit.toml @@ -0,0 +1,91 @@ +[simulation] + name = "deposit" + engine = "grpic" + runtime = 10.0 + +[grid] + resolution = [512,512] + extent = [[1.2, 20.0]] + + [grid.metric] + metric = "kerr_schild" + qsph_r0 = 0.0 + qsph_h = 0.0 + ks_a = 0.0 + + [grid.boundaries] + fields = [["ABSORB"]] + particles = [["ABSORB"]] + + [grid.boundaries.absorb] + ds = 2.0 + +[scales] + larmor0 = 2.0 + skindepth0 = 2.0 + +[algorithms] + current_filters = 4 + + [algorithms.gr] + pusher_niter = 10 + pusher_eps = 1e-6 + + [algorithms.timestep] + CFL = 0.5 + + [algorithms.toggles] + deposit = true + fieldsolver = true + +[particles] + ppc0 = 1.0 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = -1.0 + maxnpart = 1e0 + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = 1.0 + maxnpart = 1e0 + +[setup] + x1s = [17.00000] + y1s = [1.570796] + z1s = [0.000000] + ux1s = [8.000000] + uy1s = [0.000000] + uz1s = [0.000000] + + x2s = [17.00000] + y2s = [1.570796] + z2s = [0.000000] + ux2s = [-8.000000] + uy2s = [0.000000] + uz2s = [0.000000] + +[output] + format = "hdf5" + + [output.fields] + enable = true + interval_time = 0.5 + quantities = ["D", "B"] + + [output.particles] + enable = true + stride = 1 + interval_time = 0.5 + species = [] + + [output.spectra] + enable = false + +[diagnostics] + interval = 10 + colored_stdout = true + blocking_timers = true diff --git a/setups/grpic/deposit/pgen.hpp b/setups/grpic/deposit/pgen.hpp new file mode 100644 index 000000000..6d5e5fba5 --- /dev/null +++ b/setups/grpic/deposit/pgen.hpp @@ -0,0 +1,113 @@ +#ifndef PROBLEM_GENERATOR_H +#define PROBLEM_GENERATOR_H + +#include "enums.h" +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "arch/traits.h" +#include "utils/comparators.h" +#include "utils/formatting.h" +#include "utils/log.h" +#include "utils/numeric.h" + +#include "archetypes/problem_generator.h" +#include "framework/domain/metadomain.h" +#include "framework/domain/domain.h" +#include "archetypes/energy_dist.h" +#include "archetypes/particle_injector.h" + +#include + +namespace user { + using namespace ntt; + + template + struct InitFields { + InitFields(M metric_) : metric { metric_ } {} + + Inline auto A_3(const coord_t& x_Cd) const -> real_t { + coord_t x_Ph { ZERO }; + metric.template convert(x_Cd, x_Ph); + return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); + } + + Inline auto bx1(const coord_t& x_Ph) const -> real_t { + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0]; + x0m[1] = xi[1] - HALF; + x0p[0] = xi[0]; + x0p[1] = xi[1] + HALF; + + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + + if (cmp::AlmostZero(x_Ph[1])) + return ONE; + else + return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; + } + + Inline auto bx2(const coord_t& x_Ph) const -> real_t { + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0] - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF; + x0p[1] = xi[1]; + + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + if (cmp::AlmostZero(x_Ph[1])) + return ZERO; + else + return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; + } + + Inline auto bx3(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx1(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx2(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx3(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + private: + const M metric; + }; + + template + struct PGen : public arch::ProblemGenerator { + // compatibility traits for the problem generator + static constexpr auto engines { traits::compatible_with::value }; + static constexpr auto metrics { + traits::compatible_with::value + }; + static constexpr auto dimensions { traits::compatible_with::value }; + + // for easy access to variables in the child class + using arch::ProblemGenerator::D; + using arch::ProblemGenerator::C; + using arch::ProblemGenerator::params; + + InitFields init_flds; + const Metadomain& global_domain; + inline PGen(const SimulationParams& p, const Metadomain& m) + : arch::ProblemGenerator { p } + , global_domain { m } + , init_flds { m.mesh().metric } {} + + }; + +} // namespace user + +#endif From 50818fc1fb0e5805528a3c424eb3441e58326de5 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 18 Dec 2024 13:57:33 -0500 Subject: [PATCH 185/234] deleted deposit test from pusher tests --- setups/grpic/pusher/deposit.toml | 91 -------------------------------- 1 file changed, 91 deletions(-) delete mode 100644 setups/grpic/pusher/deposit.toml diff --git a/setups/grpic/pusher/deposit.toml b/setups/grpic/pusher/deposit.toml deleted file mode 100644 index 359d4eaa7..000000000 --- a/setups/grpic/pusher/deposit.toml +++ /dev/null @@ -1,91 +0,0 @@ -[simulation] - name = "deposit" - engine = "grpic" - runtime = 10.0 - -[grid] - resolution = [512,512] - extent = [[1.2, 20.0]] - - [grid.metric] - metric = "kerr_schild" - qsph_r0 = 0.0 - qsph_h = 0.0 - ks_a = 0.0 - - [grid.boundaries] - fields = [["ABSORB"]] - particles = [["ABSORB"]] - - [grid.boundaries.absorb] - ds = 2.0 - -[scales] - larmor0 = 2.0 - skindepth0 = 2.0 - -[algorithms] - current_filters = 4 - - [algorithms.gr] - pusher_niter = 10 - pusher_eps = 1e-6 - - [algorithms.timestep] - CFL = 0.5 - - [algorithms.toggles] - deposit = true - fieldsolver = true - -[particles] - ppc0 = 1.0 - - [[particles.species]] - label = "e-" - mass = 1.0 - charge = -1.0 - maxnpart = 1e0 - - [[particles.species]] - label = "e+" - mass = 1.0 - charge = 1.0 - maxnpart = 1e0 - -[setup] - x1s = [17.00000] - y1s = [1.570796] - z1s = [0.000000] - ux1s = [8.000000] - uy1s = [0.000000] - uz1s = [0.000000] - - x2s = [17.00000] - y2s = [1.570796] - z2s = [0.000000] - ux2s = [-8.000000] - uy2s = [0.000000] - uz2s = [0.000000] - -[output] - format = "hdf5" - - [output.fields] - enable = true - interval_time = 0.5 - quantities = ["D", "B"] - - [output.particles] - enable = true - stride = 1 - interval_time = 0.5 - species = [] - - [output.spectra] - enable = false - -[diagnostics] - interval = 10 - colored_stdout = true - blocking_timers = true From 8e04145ed37722b1d77c67f815a82ad6a48235e8 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 18 Dec 2024 15:58:12 -0500 Subject: [PATCH 186/234] communication should be for j0 for GRPIC --- src/framework/domain/communications.cpp | 73 ++++++++++++++++++------- 1 file changed, 54 insertions(+), 19 deletions(-) diff --git a/src/framework/domain/communications.cpp b/src/framework/domain/communications.cpp index 7dc5d285a..728d8fba2 100644 --- a/src/framework/domain/communications.cpp +++ b/src/framework/domain/communications.cpp @@ -303,11 +303,10 @@ namespace ntt { comp_range_fld, false); } - } - if (comm_j) { + if (comm_j) { comm::CommunicateField(domain.index(), - domain.fields.cur, - domain.fields.cur, + domain.fields.cur0, + domain.fields.cur0, send_ind, recv_ind, send_rank, @@ -316,6 +315,21 @@ namespace ntt { recv_slice, comp_range_cur, false); + } + } else { + if (comm_j) { + comm::CommunicateField(domain.index(), + domain.fields.cur, + domain.fields.cur, + send_ind, + recv_ind, + send_rank, + recv_rank, + send_slice, + recv_slice, + comp_range_cur, + false); + } } } } @@ -435,17 +449,31 @@ namespace ntt { continue; } if (comm_j) { - comm::CommunicateField(domain.index(), - domain.fields.cur, - domain.fields.buff, - send_ind, - recv_ind, - send_rank, - recv_rank, - send_slice, - recv_slice, - comp_range_cur, - synchronize); + if constexpr (S == SimEngine::GRPIC) { + comm::CommunicateField(domain.index(), + domain.fields.cur0, + domain.fields.buff, + send_ind, + recv_ind, + send_rank, + recv_rank, + send_slice, + recv_slice, + comp_range_cur, + synchronize); + } else { + comm::CommunicateField(domain.index(), + domain.fields.cur, + domain.fields.buff, + send_ind, + recv_ind, + send_rank, + recv_rank, + send_slice, + recv_slice, + comp_range_cur, + synchronize); + } } if (comm_bckp) { comm::CommunicateField(domain.index(), @@ -475,10 +503,17 @@ namespace ntt { } } if (comm_j) { - AddBufferedFields(domain.fields.cur, - domain.fields.buff, - domain.mesh.rangeActiveCells(), - comp_range_cur); + if constexpr (S == SimEngine::GRPIC) { + AddBufferedFields(domain.fields.cur0, + domain.fields.buff, + domain.mesh.rangeActiveCells(), + comp_range_cur); + } else { + AddBufferedFields(domain.fields.cur, + domain.fields.buff, + domain.mesh.rangeActiveCells(), + comp_range_cur); + } } if (comm_bckp) { AddBufferedFields(domain.fields.bckp, From 277bccf6aaf29f088c4f9ea83c4bd41a72e1a190 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 18 Dec 2024 15:59:21 -0500 Subject: [PATCH 187/234] 1. implemented working boundary conditions for currents, 2. fixed normaliation for AmpereCurrents -- currents should not be normilized as they are used repeatedl3. minor style --- src/engines/grpic.hpp | 22 +++++++++++----------- src/kernels/fields_bcs.hpp | 12 +++++------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index a4bffadf2..10ea9258a 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -374,9 +374,9 @@ namespace ntt { m_metadomain.CommunicateFields(dom, Comm::J); timers.stop("Communications"); - // timers.start("FieldBoundaries"); - // CurrentsBoundaryConditions(); - // timers.stop("FieldBoundaries"); + timers.start("FieldBoundaries"); + CurrentsBoundaryConditions(dom); + timers.stop("FieldBoundaries"); timers.start("CurrentFiltering"); CurrentsFilter(dom); @@ -660,9 +660,9 @@ namespace ntt { "AbsorbCurrent", CreateRangePolicy(range_min, range_max), kernel::AbsorbCurrentGR_kernel(domain.fields.cur0, - domain.mesh.metric, - xg_edge, - ds)); + domain.mesh.metric, + xg_edge, + ds)); } void OpenFieldsIn(dir::direction_t direction, @@ -918,7 +918,7 @@ namespace ntt { const auto q0 = m_params.template get("scales.q0"); const auto n0 = m_params.template get("scales.n0"); const auto B0 = m_params.template get("scales.B0"); - const auto coeff = -dt * q0 * n0 / B0; + const auto coeff = -dt * q0 / B0; // auto range = range_with_axis_BCs(domain); auto range = CreateRangePolicy( { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2)}, @@ -968,7 +968,7 @@ namespace ntt { } void CurrentsDeposit(domain_t& domain) { - auto scatter_cur = Kokkos::Experimental::create_scatter_view( + auto scatter_cur0 = Kokkos::Experimental::create_scatter_view( domain.fields.cur0); for (auto& species : domain.species) { logger::Checkpoint( @@ -984,7 +984,7 @@ namespace ntt { Kokkos::parallel_for("CurrentsDeposit", species.rangeActiveParticles(), kernel::DepositCurrents_kernel( - scatter_cur, + scatter_cur0, species.i1, species.i2, species.i3, @@ -1007,7 +1007,7 @@ namespace ntt { (real_t)(species.charge()), dt)); } - Kokkos::Experimental::contribute(domain.fields.cur0, scatter_cur); + Kokkos::Experimental::contribute(domain.fields.cur0, scatter_cur0); } void CurrentsFilter(domain_t& domain) { @@ -1031,7 +1031,7 @@ namespace ntt { domain.fields.buff, size, domain.mesh.flds_bc())); - m_metadomain.CommunicateFields(domain, Comm::J); + m_metadomain.CommunicateFields(domain, Comm::J); //J0 } } diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 3a949671a..ee6604577 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -1026,18 +1026,15 @@ namespace kernel::bc { const M metric; const real_t xg_edge; const real_t dx_abs; - const BCTags tags; AbsorbCurrentGR_kernel(ndfield_t J, const M& metric, real_t xg_edge, - real_t dx_abs, - BCTags tags) + real_t dx_abs) : J { J } , metric { metric } , xg_edge { xg_edge } - , dx_abs { dx_abs } - , tags { tags } {} + , dx_abs { dx_abs } {} Inline void operator()(index_t i1, index_t i2) const { if constexpr (M::Dim == Dim::_2D) { @@ -1048,8 +1045,9 @@ namespace kernel::bc { x_Cd[1] = i2_; const auto dx = math::abs( metric.template convert(x_Cd[i - 1]) - xg_edge); - J(i1, i2) *= math::tanh(dx / (INV_4 * dx_abs)); - + J(i1, i2, cur::jx1) *= math::tanh(dx / (INV_4 * dx_abs)); + J(i1, i2, cur::jx2) *= math::tanh(dx / (INV_4 * dx_abs)); + J(i1, i2, cur::jx3) *= math::tanh(dx / (INV_4 * dx_abs)); } else { raise::KernelError( HERE, From cc07d4501aad8200fc3cd23286b5d1e7202b739d Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 18 Dec 2024 16:03:25 -0500 Subject: [PATCH 188/234] improved setup for deposit testing --- setups/grpic/deposit/deposit.py | 53 +++++++++++++++++++++ setups/grpic/deposit/deposit.toml | 12 ++--- setups/grpic/deposit/pgen.hpp | 79 ++++++++++++++++++------------- 3 files changed, 104 insertions(+), 40 deletions(-) create mode 100644 setups/grpic/deposit/deposit.py diff --git a/setups/grpic/deposit/deposit.py b/setups/grpic/deposit/deposit.py new file mode 100644 index 000000000..e3aa97dfc --- /dev/null +++ b/setups/grpic/deposit/deposit.py @@ -0,0 +1,53 @@ +import nt2.read as nt2r +import matplotlib as mpl +import matplotlib.pyplot as plt +import numpy as np + +data = nt2r.Data("deposit.h5") +def frame(ti): + t0 = data.t.isel(t=ti).values[()] + vmax = 1e-5 + fig, ax = plt.subplots(figsize=(6, 8), dpi=150) + data.Dr.isel(t=ti).polar.pcolor( + ax=ax, + norm=mpl.colors.SymLogNorm( + vmin=-vmax, vmax=vmax, linthresh=vmax / 1e3, linscale=1 + ), + cmap="RdBu_r", + ) + x1, x2 = data.particles[1].isel(t=ti).r * np.sin( + data.particles[1].isel(t=ti).th + ), data.particles[1].isel(t=ti).r * np.cos(data.particles[1].isel(t=ti).th) + ax.scatter(x1, x2, s=5, c="b", ec="w", lw=0.5) + x1, x2 = data.particles[2].isel(t=ti).r * np.sin( + data.particles[2].isel(t=ti).th + ), data.particles[2].isel(t=ti).r * np.cos(data.particles[2].isel(t=ti).th) + ax.scatter(x1, x2, s=5, c="r", ec="w", lw=0.5) + ax.add_artist( + mpl.patches.Arc( + (0, 0), + 2 * 10, + 2 * 10, + fill=False, + ec="k", + ls=":", + lw=0.5, + theta1=-90, + theta2=90, + ) + ) + axins = ax.inset_axes([0.4, 0.8, 0.6 - 0.025, 0.2 - 0.01]) + ( + data.Dr.sel(t=slice(data.t.min(), t0)).sel(r=10, method="nearest") + * np.sin(data.th) + ).sum(dim="th").plot(ax=axins) + axins.set( + ylim=(-3 * vmax, 3 * vmax), + xlim=(data.t.min(), data.t.max()), + title="", + ylabel=rf"$\int D_r d\Omega$ @ (r = 10)", + ) + axins.axvline(t0, color="b", lw=0.5, ls=":") + axins.axhline(0, color="r", lw=0.5, ls="--") + +frame(30) \ No newline at end of file diff --git a/setups/grpic/deposit/deposit.toml b/setups/grpic/deposit/deposit.toml index 359d4eaa7..c577f9c68 100644 --- a/setups/grpic/deposit/deposit.toml +++ b/setups/grpic/deposit/deposit.toml @@ -1,7 +1,7 @@ [simulation] name = "deposit" engine = "grpic" - runtime = 10.0 + runtime = 50.0 [grid] resolution = [512,512] @@ -21,8 +21,8 @@ ds = 2.0 [scales] - larmor0 = 2.0 - skindepth0 = 2.0 + larmor0 = 2e-3 + skindepth0 = 0.1 [algorithms] current_filters = 4 @@ -57,14 +57,14 @@ x1s = [17.00000] y1s = [1.570796] z1s = [0.000000] - ux1s = [8.000000] + ux1s = [5.000000] uy1s = [0.000000] uz1s = [0.000000] x2s = [17.00000] y2s = [1.570796] z2s = [0.000000] - ux2s = [-8.000000] + ux2s = [-5.000000] uy2s = [0.000000] uz2s = [0.000000] @@ -74,7 +74,7 @@ [output.fields] enable = true interval_time = 0.5 - quantities = ["D", "B"] + quantities = ["D"] [output.particles] enable = true diff --git a/setups/grpic/deposit/pgen.hpp b/setups/grpic/deposit/pgen.hpp index 6d5e5fba5..8ff89fa92 100644 --- a/setups/grpic/deposit/pgen.hpp +++ b/setups/grpic/deposit/pgen.hpp @@ -26,43 +26,12 @@ namespace user { struct InitFields { InitFields(M metric_) : metric { metric_ } {} - Inline auto A_3(const coord_t& x_Cd) const -> real_t { - coord_t x_Ph { ZERO }; - metric.template convert(x_Cd, x_Ph); - return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); - } - Inline auto bx1(const coord_t& x_Ph) const -> real_t { - coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; - metric.template convert(x_Ph, xi); - - x0m[0] = xi[0]; - x0m[1] = xi[1] - HALF; - x0p[0] = xi[0]; - x0p[1] = xi[1] + HALF; - - real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; - - if (cmp::AlmostZero(x_Ph[1])) - return ONE; - else - return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; + return ZERO; } Inline auto bx2(const coord_t& x_Ph) const -> real_t { - coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; - metric.template convert(x_Ph, xi); - - x0m[0] = xi[0] - HALF; - x0m[1] = xi[1]; - x0p[0] = xi[0] + HALF; - x0p[1] = xi[1]; - - real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; - if (cmp::AlmostZero(x_Ph[1])) - return ZERO; - else - return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; + return ZERO; } Inline auto bx3(const coord_t& x_Ph) const -> real_t { @@ -105,7 +74,49 @@ namespace user { : arch::ProblemGenerator { p } , global_domain { m } , init_flds { m.mesh().metric } {} - + + inline void InitPrtls(Domain& local_domain) { + const auto empty = std::vector {}; + const auto x1s = params.template get>("setup.x1s", empty); + const auto y1s = params.template get>("setup.y1s", empty); + const auto z1s = params.template get>("setup.z1s", empty); + const auto ux1s = params.template get>("setup.ux1s", + empty); + const auto uy1s = params.template get>("setup.uy1s", + empty); + const auto uz1s = params.template get>("setup.uz1s", + empty); + + const auto x2s = params.template get>("setup.x2s", empty); + const auto y2s = params.template get>("setup.y2s", empty); + const auto z2s = params.template get>("setup.z2s", empty); + const auto ux2s = params.template get>("setup.ux2s", + empty); + const auto uy2s = params.template get>("setup.uy2s", + empty); + const auto uz2s = params.template get>("setup.uz2s", + empty); + const std::map> data_1 { + { "x1", x1s}, + { "x2", y1s}, + { "phi", z1s}, + {"ux1", ux1s}, + {"ux2", uy1s}, + {"ux3", uz1s} + }; + const std::map> data_2 { + { "x1", x2s}, + { "x2", y2s}, + { "phi", z2s}, + {"ux1", ux2s}, + {"ux2", uy2s}, + {"ux3", uz2s} + }; + + arch::InjectGlobally(global_domain, local_domain, (arch::spidx_t)1, data_1); + arch::InjectGlobally(global_domain, local_domain, (arch::spidx_t)2, data_2); + } + // inline PGen() {} }; } // namespace user From d2df46782211630b9ab56f345d8a150692a28f6d Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Thu, 2 Jan 2025 15:33:52 -0500 Subject: [PATCH 189/234] minor fix for qks --- src/metrics/qkerr_schild.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/metrics/qkerr_schild.h b/src/metrics/qkerr_schild.h index e45376a24..e3d849952 100644 --- a/src/metrics/qkerr_schild.h +++ b/src/metrics/qkerr_schild.h @@ -255,7 +255,7 @@ namespace metric { const real_t r { r0 + math::exp(x[0] * dchi + chi_min) }; const real_t eta {x[1] * deta + eta_min }; const real_t theta { eta2theta(eta) }; - const real_t dx_dt {deta * (ONE + TWO * h0 * constant::INV_PI_SQR * (TWO * THREE * SQR(eta) - TWO * THREE * constant::PI * eta + constant::PI_SQR)) }; + const real_t dx_dt {deta * (ONE + TWO * h0 * static_cast(constant::INV_PI_SQR) * (TWO * THREE * SQR(eta) - TWO * THREE * static_cast(constant::PI) * eta + static_cast(constant::PI_SQR))) }; const real_t dt_Sigma {- TWO * SQR(a) * math::sin(theta) * math::cos(theta) * dx_dt}; return r * dt_Sigma * CUBE(alpha(x)) / SQR(Sigma(r, theta)); From 68aeddc8e8431256b66335db3eb837647206d8fc Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Thu, 2 Jan 2025 15:52:50 -0500 Subject: [PATCH 190/234] minor for engines/grpic --- src/engines/grpic.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 10ea9258a..18ba4db0b 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -744,6 +744,7 @@ namespace ntt { (void)direction; (void)domain; (void)tags; + (void)g; raise::Error("Custom boundaries not implemented", HERE); // if constexpr ( // traits::has_member::value) { @@ -916,7 +917,7 @@ namespace ntt { void AmpereCurrents(domain_t& domain, const gr_ampere& g) { logger::Checkpoint("Launching Ampere kernel for adding currents", HERE); const auto q0 = m_params.template get("scales.q0"); - const auto n0 = m_params.template get("scales.n0"); + // const auto n0 = m_params.template get("scales.n0"); const auto B0 = m_params.template get("scales.B0"); const auto coeff = -dt * q0 / B0; // auto range = range_with_axis_BCs(domain); From 4e873ffde63da1965dfb4a7bb3c21f326b92a9ce Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 3 Jan 2025 15:01:25 -0500 Subject: [PATCH 191/234] implementing threshold-dependent injection is in progress --- setups/grpic/wald/pgen.hpp | 211 ++++++++++++++++++++++++++++-------- setups/grpic/wald/wald.toml | 48 ++++---- 2 files changed, 188 insertions(+), 71 deletions(-) diff --git a/setups/grpic/wald/pgen.hpp b/setups/grpic/wald/pgen.hpp index b87d25ed2..457c24107 100644 --- a/setups/grpic/wald/pgen.hpp +++ b/setups/grpic/wald/pgen.hpp @@ -7,8 +7,13 @@ #include "arch/kokkos_aliases.h" #include "arch/traits.h" +#include "archetypes/energy_dist.h" +#include "archetypes/spatial_dist.h" +#include "archetypes/particle_injector.h" #include "archetypes/problem_generator.h" #include "framework/domain/metadomain.h" +#include "kernels/particle_moments.hpp" +#include "utils/numeric.h" namespace user { using namespace ntt; @@ -18,26 +23,12 @@ namespace user { InitFields(M metric_) : metric { metric_ } {} Inline auto A_3(const coord_t& x_Cd) const -> real_t { - // return HALF * (metric.template h_<3, 3>(x_Cd) - // + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) - // ); - coord_t x_Ph { ZERO }; - metric.template convert(x_Cd, x_Ph); - return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); - } - - Inline auto A_1(const coord_t& x_Cd) const -> real_t { - return HALF * (metric.template h_<1, 3>(x_Cd) - + TWO * metric.spin() * metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) + return HALF * (metric.template h_<3, 3>(x_Cd) + + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) ); - } - - Inline auto A_0(const coord_t& x_Cd) const -> real_t { - real_t g_00 { -metric.alpha(x_Cd) * metric.alpha(x_Cd) - + metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) * metric.beta1(x_Cd) - }; - return HALF * (metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) - + TWO * metric.spin() * g_00); + // coord_t x_Ph { ZERO }; + // metric.template convert(x_Cd, x_Ph); + // return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); } Inline auto bx1(const coord_t& x_Ph) const -> real_t { @@ -74,16 +65,6 @@ namespace user { } Inline auto bx3(const coord_t& x_Ph) const -> real_t { - // coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; - // metric.template convert(x_Ph, xi); - - // x0m[0] = xi[0] + HALF - HALF; - // x0m[1] = xi[1]; - // x0p[0] = xi[0] + HALF + HALF; - // x0p[1] = xi[1]; - - // real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; - // return -(A_1(x0p) - A_1(x0m)) * inv_sqrt_detH_ijP; return ZERO; } @@ -103,6 +84,105 @@ namespace user { const M metric; }; + template + struct PointDistribution : public arch::SpatialDistribution { + PointDistribution(const std::vector& xi_min, + const std::vector& xi_max, + const real_t sigma_thr, + const real_t dens_thr, + long double time, + const SimulationParams& params, + Domain* domain_ptr + ) + : arch::SpatialDistribution { domain_ptr->mesh.metric } + , metric { domain_ptr->mesh.metric } + , EM { domain_ptr->fields.em } + , sigma_thr {sigma_thr} + , dens_thr {dens_thr} + , time {time} { + std::copy(xi_min.begin(), xi_min.end(), x_min); + std::copy(xi_max.begin(), xi_max.end(), x_max); + + std::vector specs {}; + for (auto& sp : domain_ptr->species) { + if (sp.mass() > 0) { + specs.push_back(sp.index()); + } + } + + auto scatter_buff = Kokkos::Experimental::create_scatter_view(domain_ptr->fields.buff); + printf("%f\n", domain_ptr->fields.buff(10, 10, 0)); + // some parameters + auto& mesh = domain_ptr->mesh; + const auto use_weights = params.template get("particles.use_weights"); + const auto ni2 = mesh.n_active(in::x2); + const auto inv_n0 = ONE / params.template get("scales.n0"); + + for (const auto& sp : specs) { + auto& prtl_spec = domain_ptr->species[sp - 1]; + // clang-format off + Kokkos::parallel_for( + "ComputeMoments", + prtl_spec.rangeActiveParticles(), + kernel::ParticleMoments_kernel({}, scatter_buff, 0u, + prtl_spec.i1, prtl_spec.i2, prtl_spec.i3, + prtl_spec.dx1, prtl_spec.dx2, prtl_spec.dx3, + prtl_spec.ux1, prtl_spec.ux2, prtl_spec.ux3, + prtl_spec.phi, prtl_spec.weight, prtl_spec.tag, + prtl_spec.mass(), prtl_spec.charge(), + use_weights, + metric, mesh.flds_bc(), + ni2, inv_n0, 0)); + // clang-format on + } + Kokkos::Experimental::contribute(domain_ptr->fields.buff, scatter_buff); + // auto density = domain_ptr->fields.buff; + auto density = Kokkos::create_mirror_view(domain_ptr->fields.buff); + Kokkos::deep_copy(density, domain_ptr->fields.buff); + } + + Inline auto sigma_crit(const coord_t& x_Ph) const -> real_t { + coord_t xi {ZERO}; + if constexpr (M::Dim == Dim::_2D) { + metric.template convert(x_Ph, xi); + const auto i1 = static_cast(xi[0]) + static_cast(N_GHOSTS); + const auto i2 = static_cast(xi[1]) + static_cast(N_GHOSTS); + // printf("%f %f\n", xi[0], xi[1]); + const vec_t B_cntrv { EM(i1, i2, em::bx1), EM(i1, i2, em::bx2), EM(i1, i2, em::bx3) }; + vec_t B_cov { ZERO }; + metric.template transform(xi, B_cntrv, B_cov); + const auto bsqr = DOT(B_cntrv[0], B_cntrv[1], B_cntrv[2], B_cov[0], B_cov[1], B_cov[2]); + const auto dens = density(i1, i2, 0); + // printf("%f\n", domain_ptr->fields.buff(10, 10, 0)); + // return (bsqr / dens > sigma_thr) || (dens < dens_thr); + return 2.0; + } + return ZERO; + } + + Inline auto operator()(const coord_t& x_Ph) const -> real_t override { + auto fill = true; + for (auto d = 0u; d < M::Dim; ++d) { + fill &= x_Ph[d] > x_min[d] and x_Ph[d] < x_max[d]; + } + if (time > ZERO) { + fill &= sigma_crit(x_Ph) > ONE; + } + return fill ? ONE : ZERO; + } + + private: + tuple_t x_min; + tuple_t x_max; + const real_t sigma_thr; + const real_t dens_thr; + long double time, + Domain* domain_ptr; + ndfield_t density; + ndfield_t EM; + const M metric; + }; + template struct PGen : public arch::ProblemGenerator { // compatibility traits for the problem generator @@ -117,30 +197,67 @@ namespace user { using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; + const std::vector xi_min; + const std::vector xi_max; + const real_t sigma0, sigma_max, multiplicity, nGJ; + InitFields init_flds; inline PGen(SimulationParams& p, const Metadomain& m) : arch::ProblemGenerator(p) - // , xi_min { p.template get>("setup.inj_rmin") } - // , xi_max { p.template get>("setup.inj_rmax") } + , xi_min { p.template get>("setup.xi_min") } + , xi_max { p.template get>("setup.xi_max") } + , sigma_max { p.template get("setup.sigma_max") } + , sigma0 { p.template get("scales.sigma0") } + , multiplicity { p.template get("setup.multiplicity") } + , nGJ { p.template get("scales.B0") * SQR(p.template get("scales.skindepth0")) } //m.mesh().metric.spin() * , init_flds { m.mesh().metric } {} - // inline void InitPrtls(Domain& local_domain) { - // const auto energy_dist = arch::ColdDist(local_domain.mesh.metric); - // const auto spatial_dist = PointDistribution(domain.mesh.metric, - // xi_min, - // xi_max); - // const auto injector = arch::NonUniformInjector( - // energy_dist, - // spatial_dist, - // { 1, 2 }); - // arch::InjectNonUniform>(params, - // local_domain, - // injector, - // 1.0); - // } - - // inline PGen() {} + inline void InitPrtls(Domain& local_domain) { + const auto energy_dist = arch::ColdDist(local_domain.mesh.metric); + const auto spatial_dist = PointDistribution(xi_min, + xi_max, + sigma_max / sigma0, + multiplicity * nGJ, + ZERO, + params, + &local_domain + ); + + const auto injector = arch::NonUniformInjector( + energy_dist, + spatial_dist, + { 1, 2 }); + arch::InjectNonUniform(params, + local_domain, + injector, + 1.0, + true); + } + + void CustomPostStep(std::size_t, long double time, Domain& local_domain) { + // const auto energy_dist = arch::ColdDist(local_domain.mesh.metric); + // const auto spatial_dist = PointDistribution(local_domain.mesh.metric, + // xi_min, + // xi_max); + + // const auto spatial_dist = ReplenishDist(local_domain.mesh.metric, + // const ndfield_t& density, + // unsigned short idx, + // const T& target_density, + // real_t target_max_density) + + // const auto injector = arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // { 1, 2 }); + // arch::InjectNonUniform(params, + // local_domain, + // injector, + // 1.0, + // true); + } + }; } // namespace user diff --git a/setups/grpic/wald/wald.toml b/setups/grpic/wald/wald.toml index 6e68d4324..2f12705de 100644 --- a/setups/grpic/wald/wald.toml +++ b/setups/grpic/wald/wald.toml @@ -1,17 +1,17 @@ [simulation] name = "wald" engine = "grpic" - runtime = 510.0 + runtime = 100.0 [grid] - resolution = [128,128] - extent = [[2.0, 8.0]] + resolution = [256,256] + extent = [[0.8, 8.0]] [grid.metric] metric = "qkerr_schild" qsph_r0 = 0.0 qsph_h = 0.0 - ks_a = 0.0 + ks_a = 0.95 [grid.boundaries] fields = [["ABSORB"]] @@ -31,7 +31,7 @@ CFL = 0.5 [algorithms.toggles] - deposit = false + deposit = true fieldsolver = true [particles] @@ -39,33 +39,33 @@ use_weights = true sort_interval = 100 - #[[particles.species]] - #label = "e-" - #mass = 1.0 - #charge = -1.0 - #maxnpart = 1e6 - #pusher = "Boris" - # - #[[particles.species]] - #label = "e+" - #mass = 1.0 - #charge = 1.0 - #maxnpart = 1e6 - #pusher = "Boris" + [[particles.species]] + label = "e-" + mass = 1.0 + charge = -1.0 + maxnpart = 1e6 + pusher = "Boris" + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = 1.0 + maxnpart = 1e6 + pusher = "Boris" [setup] - #multiplicity = 1.0 - #sigma_max = 1000.0 - inj_rmin = 1.2 - inj_rmax = 7.5 + multiplicity = 1.0 + sigma_max = 10.0 + xi_min = [1.2, 0.1] + xi_max = [7.5, 3.04159265] [output] format = "hdf5" [output.fields] interval_time = 1.0 - quantities = ["D", "H", "B", "J"] - + quantities = ["D", "B", "J", "N_1", "N_2", "Nppc_1", "Nppc_2"] + [output.particles] enable = false From 32fcfaf9dfa4ce45f4af7202de814881f0a17a39 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 3 Jan 2025 16:57:41 -0500 Subject: [PATCH 192/234] updated pgen --- setups/grpic/wald/pgen.hpp | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/setups/grpic/wald/pgen.hpp b/setups/grpic/wald/pgen.hpp index 457c24107..78f922700 100644 --- a/setups/grpic/wald/pgen.hpp +++ b/setups/grpic/wald/pgen.hpp @@ -90,7 +90,6 @@ namespace user { const std::vector& xi_max, const real_t sigma_thr, const real_t dens_thr, - long double time, const SimulationParams& params, Domain* domain_ptr ) @@ -98,8 +97,7 @@ namespace user { , metric { domain_ptr->mesh.metric } , EM { domain_ptr->fields.em } , sigma_thr {sigma_thr} - , dens_thr {dens_thr} - , time {time} { + , dens_thr {dens_thr} { std::copy(xi_min.begin(), xi_min.end(), x_min); std::copy(xi_max.begin(), xi_max.end(), x_max); @@ -136,7 +134,6 @@ namespace user { // clang-format on } Kokkos::Experimental::contribute(domain_ptr->fields.buff, scatter_buff); - // auto density = domain_ptr->fields.buff; auto density = Kokkos::create_mirror_view(domain_ptr->fields.buff); Kokkos::deep_copy(density, domain_ptr->fields.buff); } @@ -147,7 +144,6 @@ namespace user { metric.template convert(x_Ph, xi); const auto i1 = static_cast(xi[0]) + static_cast(N_GHOSTS); const auto i2 = static_cast(xi[1]) + static_cast(N_GHOSTS); - // printf("%f %f\n", xi[0], xi[1]); const vec_t B_cntrv { EM(i1, i2, em::bx1), EM(i1, i2, em::bx2), EM(i1, i2, em::bx3) }; vec_t B_cov { ZERO }; metric.template transform(xi, B_cntrv, B_cov); @@ -163,10 +159,7 @@ namespace user { Inline auto operator()(const coord_t& x_Ph) const -> real_t override { auto fill = true; for (auto d = 0u; d < M::Dim; ++d) { - fill &= x_Ph[d] > x_min[d] and x_Ph[d] < x_max[d]; - } - if (time > ZERO) { - fill &= sigma_crit(x_Ph) > ONE; + fill &= x_Ph[d] > x_min[d] and x_Ph[d] < x_max[d] and sigma_crit(x_Ph) > ONE; } return fill ? ONE : ZERO; } @@ -176,7 +169,6 @@ namespace user { tuple_t x_max; const real_t sigma_thr; const real_t dens_thr; - long double time, Domain* domain_ptr; ndfield_t density; ndfield_t EM; @@ -219,7 +211,6 @@ namespace user { xi_max, sigma_max / sigma0, multiplicity * nGJ, - ZERO, params, &local_domain ); From 5ba83978500f092cd2ad9c433dfc8ddbab900a7d Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 3 Jan 2025 17:19:57 -0500 Subject: [PATCH 193/234] minor --- setups/grpic/wald/pgen.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/setups/grpic/wald/pgen.hpp b/setups/grpic/wald/pgen.hpp index 78f922700..41d9a2462 100644 --- a/setups/grpic/wald/pgen.hpp +++ b/setups/grpic/wald/pgen.hpp @@ -108,6 +108,7 @@ namespace user { } } + Kokkos::deep_copy(domain_ptr->fields.buff, ZERO); auto scatter_buff = Kokkos::Experimental::create_scatter_view(domain_ptr->fields.buff); printf("%f\n", domain_ptr->fields.buff(10, 10, 0)); // some parameters From fdf2e738cd0dc769628602125c17f8baa75ec816 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 8 Jan 2025 19:14:18 -0500 Subject: [PATCH 194/234] implemented output of Aph for 2d GRPIC --- src/framework/domain/output.cpp | 44 +++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/framework/domain/output.cpp b/src/framework/domain/output.cpp index ae138bac1..c13e354b3 100644 --- a/src/framework/domain/output.cpp +++ b/src/framework/domain/output.cpp @@ -185,6 +185,39 @@ namespace ntt { } } + template + void ComputeVectorPotential(ndfield_t& buffer, + ndfield_t& EM, + unsigned short buff_idx, + const Mesh mesh) { + if constexpr (M::Dim == Dim::_2D) { + const auto i2_min = mesh.i_min(in::x2); + // !TODO: this is quite slow + Kokkos::parallel_for( + "ComputeVectorPotential", + mesh.rangeActiveCells(), + Lambda(index_t i, index_t j) { + const real_t i_ { static_cast(static_cast(i) - (N_GHOSTS)) }; + const auto k_min = (i2_min - (N_GHOSTS)) + 1; + const auto k_max = (j - (N_GHOSTS)); + real_t A3 = ZERO; + for (auto k { k_min }; k <= k_max; ++k) { + real_t k_ = static_cast(k); + real_t sqrt_detH_ij1 { mesh.metric.sqrt_det_h({ i_, k_ - HALF }) }; + real_t sqrt_detH_ij2 { mesh.metric.sqrt_det_h({ i_, k_ + HALF }) }; + auto k1 { k + N_GHOSTS }; + A3 += HALF * (sqrt_detH_ij1 * EM(i, k1 - 1, em::bx1) + + sqrt_detH_ij2 * EM(i, k1, em::bx1)); + } + buffer(i, j, buff_idx) = A3; + }); + } else { + raise::KernelError( + HERE, + "ComputeVectorPotential: 2D implementation called for D != 2"); + } + } + template auto Metadomain::Write( const SimulationParams& params, @@ -347,6 +380,17 @@ namespace ntt { raise::Error("Custom output requested but no function provided", HERE); } + } else if (fld.is_vpotential()) { + if (S == SimEngine::GRPIC && M::Dim == Dim::_2D) { + const auto c = static_cast(addresses.back()); + ComputeVectorPotential(local_domain->fields.bckp, + local_domain->fields.em, + c, + local_domain->mesh); + } else { + raise::Error("Vector potential can only be computed for GRPIC in 2D", + HERE); + } } else { raise::Error("Wrong # of components requested for " "non-moment/non-custom output", From 1d11b58bfdb2bc0dec3376f9a6453f62ea626ef6 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 8 Jan 2025 19:17:43 -0500 Subject: [PATCH 195/234] updated pgen for wald solution with plasma --- setups/grpic/wald/pgen.hpp | 76 ++++++++++++++++++------------------- setups/grpic/wald/wald.toml | 17 +++++---- 2 files changed, 45 insertions(+), 48 deletions(-) diff --git a/setups/grpic/wald/pgen.hpp b/setups/grpic/wald/pgen.hpp index 41d9a2462..7e8eea216 100644 --- a/setups/grpic/wald/pgen.hpp +++ b/setups/grpic/wald/pgen.hpp @@ -26,9 +26,6 @@ namespace user { return HALF * (metric.template h_<3, 3>(x_Cd) + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) ); - // coord_t x_Ph { ZERO }; - // metric.template convert(x_Cd, x_Ph); - // return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); } Inline auto bx1(const coord_t& x_Ph) const -> real_t { @@ -96,6 +93,7 @@ namespace user { : arch::SpatialDistribution { domain_ptr->mesh.metric } , metric { domain_ptr->mesh.metric } , EM { domain_ptr->fields.em } + , density { domain_ptr->fields.buff } , sigma_thr {sigma_thr} , dens_thr {dens_thr} { std::copy(xi_min.begin(), xi_min.end(), x_min); @@ -108,9 +106,8 @@ namespace user { } } - Kokkos::deep_copy(domain_ptr->fields.buff, ZERO); - auto scatter_buff = Kokkos::Experimental::create_scatter_view(domain_ptr->fields.buff); - printf("%f\n", domain_ptr->fields.buff(10, 10, 0)); + Kokkos::deep_copy(density, ZERO); + auto scatter_buff = Kokkos::Experimental::create_scatter_view(density); // some parameters auto& mesh = domain_ptr->mesh; const auto use_weights = params.template get("particles.use_weights"); @@ -131,15 +128,13 @@ namespace user { prtl_spec.mass(), prtl_spec.charge(), use_weights, metric, mesh.flds_bc(), - ni2, inv_n0, 0)); + ni2, inv_n0, ZERO)); // clang-format on } - Kokkos::Experimental::contribute(domain_ptr->fields.buff, scatter_buff); - auto density = Kokkos::create_mirror_view(domain_ptr->fields.buff); - Kokkos::deep_copy(density, domain_ptr->fields.buff); + Kokkos::Experimental::contribute(density, scatter_buff); } - Inline auto sigma_crit(const coord_t& x_Ph) const -> real_t { + Inline auto sigma_crit(const coord_t& x_Ph) const -> bool { coord_t xi {ZERO}; if constexpr (M::Dim == Dim::_2D) { metric.template convert(x_Ph, xi); @@ -150,17 +145,15 @@ namespace user { metric.template transform(xi, B_cntrv, B_cov); const auto bsqr = DOT(B_cntrv[0], B_cntrv[1], B_cntrv[2], B_cov[0], B_cov[1], B_cov[2]); const auto dens = density(i1, i2, 0); - // printf("%f\n", domain_ptr->fields.buff(10, 10, 0)); - // return (bsqr / dens > sigma_thr) || (dens < dens_thr); - return 2.0; + return (bsqr > sigma_thr * dens) || (dens < dens_thr); } - return ZERO; + return false; } Inline auto operator()(const coord_t& x_Ph) const -> real_t override { auto fill = true; for (auto d = 0u; d < M::Dim; ++d) { - fill &= x_Ph[d] > x_min[d] and x_Ph[d] < x_max[d] and sigma_crit(x_Ph) > ONE; + fill &= x_Ph[d] > x_min[d] and x_Ph[d] < x_max[d] and sigma_crit(x_Ph); } return fill ? ONE : ZERO; } @@ -192,7 +185,7 @@ namespace user { const std::vector xi_min; const std::vector xi_max; - const real_t sigma0, sigma_max, multiplicity, nGJ; + const real_t sigma0, sigma_max, multiplicity, nGJ, temperature; InitFields init_flds; @@ -203,11 +196,14 @@ namespace user { , sigma_max { p.template get("setup.sigma_max") } , sigma0 { p.template get("scales.sigma0") } , multiplicity { p.template get("setup.multiplicity") } - , nGJ { p.template get("scales.B0") * SQR(p.template get("scales.skindepth0")) } //m.mesh().metric.spin() * + , nGJ { p.template get("scales.B0") * SQR(p.template get("scales.skindepth0")) } + , temperature { p.template get("setup.temperature") } , init_flds { m.mesh().metric } {} inline void InitPrtls(Domain& local_domain) { - const auto energy_dist = arch::ColdDist(local_domain.mesh.metric); + const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, + local_domain.random_pool, + temperature); const auto spatial_dist = PointDistribution(xi_min, xi_max, sigma_max / sigma0, @@ -216,7 +212,7 @@ namespace user { &local_domain ); - const auto injector = arch::NonUniformInjector( + const auto injector = arch::NonUniformInjector( energy_dist, spatial_dist, { 1, 2 }); @@ -228,26 +224,26 @@ namespace user { } void CustomPostStep(std::size_t, long double time, Domain& local_domain) { - // const auto energy_dist = arch::ColdDist(local_domain.mesh.metric); - // const auto spatial_dist = PointDistribution(local_domain.mesh.metric, - // xi_min, - // xi_max); - - // const auto spatial_dist = ReplenishDist(local_domain.mesh.metric, - // const ndfield_t& density, - // unsigned short idx, - // const T& target_density, - // real_t target_max_density) - - // const auto injector = arch::NonUniformInjector( - // energy_dist, - // spatial_dist, - // { 1, 2 }); - // arch::InjectNonUniform(params, - // local_domain, - // injector, - // 1.0, - // true); + const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, + local_domain.random_pool, + temperature); + const auto spatial_dist = PointDistribution(xi_min, + xi_max, + sigma_max / sigma0, + multiplicity * nGJ, + params, + &local_domain + ); + + const auto injector = arch::NonUniformInjector( + energy_dist, + spatial_dist, + { 1, 2 }); + arch::InjectNonUniform(params, + local_domain, + injector, + 1.0, + true); } }; diff --git a/setups/grpic/wald/wald.toml b/setups/grpic/wald/wald.toml index 2f12705de..3bf243afc 100644 --- a/setups/grpic/wald/wald.toml +++ b/setups/grpic/wald/wald.toml @@ -4,8 +4,8 @@ runtime = 100.0 [grid] - resolution = [256,256] - extent = [[0.8, 8.0]] + resolution = [512, 512] + extent = [[1.22, 6.0]] [grid.metric] metric = "qkerr_schild" @@ -43,28 +43,29 @@ label = "e-" mass = 1.0 charge = -1.0 - maxnpart = 1e6 + maxnpart = 2e8 pusher = "Boris" [[particles.species]] label = "e+" mass = 1.0 charge = 1.0 - maxnpart = 1e6 + maxnpart = 2e8 pusher = "Boris" [setup] multiplicity = 1.0 - sigma_max = 10.0 - xi_min = [1.2, 0.1] - xi_max = [7.5, 3.04159265] + sigma_max = 1000.0 + temperature = 0.01 + xi_min = [1.32, 0.1] + xi_max = [4.0, 3.04159265] [output] format = "hdf5" [output.fields] interval_time = 1.0 - quantities = ["D", "B", "J", "N_1", "N_2", "Nppc_1", "Nppc_2"] + quantities = ["D", "B", "J", "N_1", "N_2", "A"] [output.particles] enable = false From 5a72128e7bd703afcf675ebf92ba11f0627a078c Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 14 Jan 2025 17:19:13 -0500 Subject: [PATCH 196/234] added an option for controlling initial B-field discretisation --- setups/grpic/wald/pgen.hpp | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/setups/grpic/wald/pgen.hpp b/setups/grpic/wald/pgen.hpp index 7e8eea216..b27dd370e 100644 --- a/setups/grpic/wald/pgen.hpp +++ b/setups/grpic/wald/pgen.hpp @@ -20,7 +20,7 @@ namespace user { template struct InitFields { - InitFields(M metric_) : metric { metric_ } {} + InitFields(M metric_, real_t m_eps) : metric { metric_ }, m_eps { m_eps } {} Inline auto A_3(const coord_t& x_Cd) const -> real_t { return HALF * (metric.template h_<3, 3>(x_Cd) @@ -28,37 +28,37 @@ namespace user { ); } - Inline auto bx1(const coord_t& x_Ph) const -> real_t { + Inline auto bx1(const coord_t& x_Ph) const -> real_t { // at ( i , j + HALF ) coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); x0m[0] = xi[0]; - x0m[1] = xi[1] - HALF; + x0m[1] = xi[1] - HALF * m_eps; x0p[0] = xi[0]; - x0p[1] = xi[1] + HALF; + x0p[1] = xi[1] + HALF * m_eps; real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; if (cmp::AlmostZero(x_Ph[1])) return ONE; else - return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; + return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP / m_eps; } - Inline auto bx2(const coord_t& x_Ph) const -> real_t { + Inline auto bx2(const coord_t& x_Ph) const -> real_t { // at ( i + HALF , j ) coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); - x0m[0] = xi[0] - HALF; + x0m[0] = xi[0] - HALF * m_eps; x0m[1] = xi[1]; - x0p[0] = xi[0] + HALF; + x0p[0] = xi[0] + HALF * m_eps; x0p[1] = xi[1]; - real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0] , xi[1] }) }; if (cmp::AlmostZero(x_Ph[1])) return ZERO; else - return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; + return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP / m_eps; } Inline auto bx3(const coord_t& x_Ph) const -> real_t { @@ -79,6 +79,7 @@ namespace user { private: const M metric; + const real_t m_eps; }; template @@ -185,7 +186,7 @@ namespace user { const std::vector xi_min; const std::vector xi_max; - const real_t sigma0, sigma_max, multiplicity, nGJ, temperature; + const real_t sigma0, sigma_max, multiplicity, nGJ, temperature, m_eps; InitFields init_flds; @@ -198,7 +199,8 @@ namespace user { , multiplicity { p.template get("setup.multiplicity") } , nGJ { p.template get("scales.B0") * SQR(p.template get("scales.skindepth0")) } , temperature { p.template get("setup.temperature") } - , init_flds { m.mesh().metric } {} + , m_eps { p.template get("setup.m_eps") } + , init_flds { m.mesh().metric, m_eps } {} inline void InitPrtls(Domain& local_domain) { const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, From 9bc2f6d0d380f263cfd6c447814323aca97d8994 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 15 Jan 2025 15:36:42 -0500 Subject: [PATCH 197/234] grpic vacuum wald test improved --- setups/grpic/vacuum/pgen.hpp | 116 ++++++++++++++++++++++++++++---- setups/grpic/vacuum/vacuum.toml | 38 +++-------- 2 files changed, 112 insertions(+), 42 deletions(-) diff --git a/setups/grpic/vacuum/pgen.hpp b/setups/grpic/vacuum/pgen.hpp index 6d5e5fba5..60a2bfe05 100644 --- a/setups/grpic/vacuum/pgen.hpp +++ b/setups/grpic/vacuum/pgen.hpp @@ -27,12 +27,26 @@ namespace user { InitFields(M metric_) : metric { metric_ } {} Inline auto A_3(const coord_t& x_Cd) const -> real_t { - coord_t x_Ph { ZERO }; - metric.template convert(x_Cd, x_Ph); - return HALF * SQR(x_Ph[0]) * SQR(math::sin(x_Ph[1])); + return HALF * (metric.template h_<3, 3>(x_Cd) + + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + ); } - Inline auto bx1(const coord_t& x_Ph) const -> real_t { + Inline auto A_1(const coord_t& x_Cd) const -> real_t { + return HALF * (metric.template h_<1, 3>(x_Cd) + + TWO * metric.spin() * metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) + ); + } + + Inline auto A_0(const coord_t& x_Cd) const -> real_t { + real_t g_00 { -metric.alpha(x_Cd) * metric.alpha(x_Cd) + + metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) * metric.beta1(x_Cd) + }; + return HALF * (metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + + TWO * metric.spin() * g_00); + } + + Inline auto bx1(const coord_t& x_Ph) const -> real_t { // at ( i , j + HALF ) coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); @@ -49,7 +63,7 @@ namespace user { return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; } - Inline auto bx2(const coord_t& x_Ph) const -> real_t { + Inline auto bx2(const coord_t& x_Ph) const -> real_t { // at ( i + HALF , j ) coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); @@ -65,20 +79,96 @@ namespace user { return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; } - Inline auto bx3(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto bx3(const coord_t& x_Ph) const -> real_t { // at ( i + HALF , j + HALF ) + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0]; + x0m[1] = xi[1] - HALF; + x0p[0] = xi[0]; + x0p[1] = xi[1] + HALF; + + real_t inv_sqrt_detH_iPjP { ONE / metric.sqrt_det_h({ xi[0], xi[1]}) }; + return -(A_1(x0p) - A_1(x0m)) * inv_sqrt_detH_iPjP; } - Inline auto dx1(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto dx1(const coord_t& x_Ph) const -> real_t { // at ( i + HALF , j ) + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + real_t alpha_iPj { metric.alpha({ xi[0], xi[1] }) }; + real_t inv_sqrt_detH_ij { ONE / metric.sqrt_det_h({ xi[0] - HALF, xi[1] }) }; + real_t sqrt_detH_ij { metric.sqrt_det_h({ xi[0] - HALF, xi[1] }) }; + real_t beta_ij { metric.beta1({ xi[0] - HALF, xi[1]}) }; + real_t alpha_ij { metric.alpha({ xi[0] - HALF, xi[1]}) }; + + // D1 at ( i + HALF , j ) + x0m[0] = xi[0] - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF; + x0p[1] = xi[1]; + real_t E1d { (A_0(x0p) - A_0(x0m)) }; + real_t D1d { E1d / alpha_iPj }; + + // D3 at ( i , j ) + x0m[0] = xi[0] - HALF - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] - HALF + HALF; + x0p[1] = xi[1]; + real_t D3d { (A_3(x0p) - A_3(x0m)) * beta_ij / alpha_ij }; + + real_t D1u { metric.template h<1, 1>({ xi[0], xi[1] }) * D1d + metric.template h<1, 3>({ xi[0], xi[1] }) * D3d }; + + return D1u; } - Inline auto dx2(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto dx2(const coord_t& x_Ph) const -> real_t { // at ( i , j + HALF ) + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + x0m[0] = xi[0]; + x0m[1] = xi[1] - HALF; + x0p[0] = xi[0]; + x0p[1] = xi[1] + HALF; + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + real_t sqrt_detH_ijP { metric.sqrt_det_h({ xi[0], xi[1] }) }; + real_t alpha_ijP { metric.alpha({ xi[0], xi[1] }) }; + real_t beta_ijP { metric.beta1({ xi[0], xi[1] }) }; + + real_t E2d { (A_0(x0p) - A_0(x0m)) }; + real_t D2d { E2d / alpha_ijP - (A_1(x0p) - A_1(x0m)) * beta_ijP / alpha_ijP }; + real_t D2u { metric.template h<2, 2>({ xi[0], xi[1] }) * D2d }; + + return D2u; } - Inline auto dx3(const coord_t& x_Ph) const -> real_t { - return ZERO; + Inline auto dx3(const coord_t& x_Ph) const -> real_t { // at ( i , j ) + coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + real_t inv_sqrt_detH_ij { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + real_t sqrt_detH_ij { metric.sqrt_det_h({ xi[0], xi[1] }) }; + real_t beta_ij { metric.beta1({ xi[0], xi[1] }) }; + real_t alpha_ij { metric.alpha({ xi[0], xi[1] }) }; + real_t alpha_iPj { metric.alpha({ xi[0] + HALF, xi[1] }) }; + + // D3 at ( i , j ) + x0m[0] = xi[0] - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF; + x0p[1] = xi[1]; + real_t D3d { (A_3(x0p) - A_3(x0m)) * beta_ij / alpha_ij }; + + // D1 at ( i + HALF , j ) + x0m[0] = xi[0] + HALF - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF + HALF; + x0p[1] = xi[1]; + real_t E1d { (A_0(x0p) - A_0(x0m)) }; + real_t D1d { E1d / alpha_iPj }; + + if (cmp::AlmostZero(x_Ph[1])) + return metric.template h<1, 3>({ xi[0], xi[1] }) * D1d; + else + return metric.template h<3, 3>({ xi[0], xi[1] }) * D3d + metric.template h<1, 3>({ xi[0], xi[1] }) * D1d; } private: diff --git a/setups/grpic/vacuum/vacuum.toml b/setups/grpic/vacuum/vacuum.toml index 3992b9f12..621f5d127 100644 --- a/setups/grpic/vacuum/vacuum.toml +++ b/setups/grpic/vacuum/vacuum.toml @@ -1,31 +1,31 @@ [simulation] - name = "wald" + name = "wald_Dph_abs_a95" engine = "grpic" - runtime = 500.0 + runtime = 1.0 [grid] resolution = [128,128] - extent = [[2.0, 8.0]] + extent = [[1.0, 10.0]] [grid.metric] - metric = "qkerr_schild" + metric = "kerr_schild" qsph_r0 = 0.0 qsph_h = 0.0 - ks_a = 0.0 + ks_a = 0.95 [grid.boundaries] fields = [["ABSORB"]] particles = [["ABSORB"]] [grid.boundaries.absorb] - ds = 0.5 + ds = 1.0 [scales] larmor0 = 0.0025 skindepth0 = 0.05 [algorithms] - current_filters = 4 + current_filters = 0 [algorithms.timestep] CFL = 0.5 @@ -35,36 +35,16 @@ fieldsolver = true [particles] - ppc0 = 8.0 - use_weights = true - sort_interval = 100 - - #[[particles.species]] - #label = "e-" - #mass = 1.0 - #charge = -1.0 - #maxnpart = 1e6 - #pusher = "Boris" - # - #[[particles.species]] - #label = "e+" - #mass = 1.0 - #charge = 1.0 - #maxnpart = 1e6 - #pusher = "Boris" + ppc0 = 2.0 [setup] - #multiplicity = 1.0 - #sigma_max = 1000.0 - inj_rmin = 1.2 - inj_rmax = 7.5 [output] format = "hdf5" [output.fields] interval_time = 1.0 - quantities = ["D", "H", "B", "J"] + quantities = ["D", "H", "B", "A"] [output.particles] enable = false From 65054962e6613c154f7e458dfa3f3d7a239f7a79 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Wed, 15 Jan 2025 15:37:20 -0500 Subject: [PATCH 198/234] kernels: all fields are absorbed in GR BC --- src/kernels/fields_bcs.hpp | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index ee6604577..c528a79f9 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -1006,6 +1006,30 @@ namespace kernel::bc { const real_t x2_0 { metric.template convert<2, Crd::Cd, Crd::Ph>(i2_) }; Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * finit.bx2({ x1_H, x2_0 }); + } else if (comp == em::bx3) { + const real_t x1_H { metric.template convert<1, Crd::Cd, Crd::Ph>( + i1_ + HALF) }; + const real_t x2_H { metric.template convert<2, Crd::Cd, Crd::Ph>( + i2_ + HALF) }; + Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * + finit.bx3({ x1_H, x2_H }); + } else if (comp == em::ex1) { + const real_t x1_H { metric.template convert<1, Crd::Cd, Crd::Ph>( + i1_ + HALF) }; + const real_t x2_0 { metric.template convert<2, Crd::Cd, Crd::Ph>(i2_) }; + Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * + finit.dx1({ x1_H, x2_0 }); + } else if (comp == em::ex2) { + const real_t x1_0 { metric.template convert<1, Crd::Cd, Crd::Ph>(i1_) }; + const real_t x2_H { metric.template convert<2, Crd::Cd, Crd::Ph>( + i2_ + HALF) }; + Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * + finit.dx2({ x1_0, x2_H }); + } else if (comp == em::ex3) { + const real_t x1_0 { metric.template convert<1, Crd::Cd, Crd::Ph>(i1_) }; + const real_t x2_0 { metric.template convert<2, Crd::Cd, Crd::Ph>(i2_) }; + Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * + finit.dx3({ x1_0, x2_0 }); } } } else { @@ -1057,6 +1081,5 @@ namespace kernel::bc { }; } // namespace kernel::bc ->>>>>>> 794beaab (BC for currents) #endif // KERNELS_FIELDS_BCS_HPP From 6dcb21a23a9c1aaae6be18436b80e2ef4667a45c Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Thu, 30 Jan 2025 17:02:48 -0500 Subject: [PATCH 199/234] gr pusher: particles are deleted under the horizon when they reach N_GHOSTS --- src/kernels/particle_pusher_gr.hpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/kernels/particle_pusher_gr.hpp b/src/kernels/particle_pusher_gr.hpp index e220239bf..71cfae3b4 100644 --- a/src/kernels/particle_pusher_gr.hpp +++ b/src/kernels/particle_pusher_gr.hpp @@ -142,9 +142,7 @@ namespace kernel::gr { , ni3 { ni3 } , epsilon { epsilon } , niter { niter } - , i1_absorb { static_cast(metric.template convert<1, Crd::Ph, Crd::Cd>( - metric.rhorizon())) - - 5 } { + , i1_absorb {N_GHOSTS} { raise::ErrorIf(boundaries.size() < 2, "boundaries defined incorrectly", HERE); is_absorb_i1min = (boundaries[0].first == PrtlBC::ABSORB) || @@ -706,6 +704,9 @@ namespace kernel::gr { EMHalfPush(xp, vp, Dp_hat, Bp_hat, vp_upd); /* x^i(n) -> x^i(n + 1) */ coord_t xp_upd { ZERO }; + if (i1(p) < 15 && is_absorb_i1min) { + vp_upd[0] = -10.0; + } GeodesicCoordinatePush(Massive_t {}, xp, vp_upd, xp_upd); // update phi From 4a1883317d45fb30a8a89504abe08729b685f2f5 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Thu, 30 Jan 2025 18:31:04 -0500 Subject: [PATCH 200/234] engines/grpic: adjustments after merging --- src/engines/grpic.hpp | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 9050d2ff1..6a1575a5a 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -99,8 +99,8 @@ namespace ntt { "algorithms.toggles.fieldsolver"); const auto deposit_enabled = m_params.template get( "algorithms.toggles.deposit"); - const auto sort_interval = m_params.template get( - "particles.sort_interval"); + const auto clear_interval = m_params.template get( + "particles.clear_interval"); if (step == 0) { if (fieldsolver_enabled) { @@ -384,9 +384,10 @@ namespace ntt { } timers.start("Communications"); - if ((sort_interval > 0) and (step % sort_interval == 0)) { - m_metadomain.CommunicateParticles(dom, &timers); - } + // if ((sort_interval > 0) and (step % sort_interval == 0)) { + // m_metadomain.CommunicateParticles(dom, &timers); + // } + m_metadomain.CommunicateParticles(dom); timers.stop("Communications"); } @@ -518,6 +519,13 @@ namespace ntt { FieldBoundaries(dom, BC::E, gr_bc::main); timers.stop("FieldBoundaries"); } + + if (clear_interval > 0 and step % clear_interval == 0 and step > 0) { + timers.start("PrtlClear"); + m_metadomain.RemoveDeadParticles(dom); + timers.stop("PrtlClear"); + } + /** * Finally: em0::B at n-1/2 * em0::D at n @@ -539,7 +547,7 @@ namespace ntt { void FieldBoundaries(domain_t& domain, BCTags tags, const gr_bc& g) { if (g == gr_bc::main) { for (auto& direction : dir::Directions::orth) { - if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::ABSORB) { + if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::MATCH) { AbsorbFieldsIn(direction, domain, tags); } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::AXIS) { if (domain.mesh.flds_bc_in(direction) == FldsBC::AXIS) { @@ -602,7 +610,7 @@ namespace ntt { Kokkos::parallel_for( "AbsorbFields", CreateRangePolicy(range_min, range_max), - kernel::AbsorbBoundariesGR_kernel(domain.fields.em, + kernel::bc::AbsorbBoundariesGR_kernel(domain.fields.em, m_pgen.init_flds, domain.mesh.metric, xg_edge, @@ -611,7 +619,7 @@ namespace ntt { Kokkos::parallel_for( "AbsorbFields", CreateRangePolicy(range_min, range_max), - kernel::AbsorbBoundariesGR_kernel(domain.fields.em0, + kernel::bc::AbsorbBoundariesGR_kernel(domain.fields.em0, m_pgen.init_flds, domain.mesh.metric, xg_edge, @@ -659,7 +667,7 @@ namespace ntt { Kokkos::parallel_for( "AbsorbCurrent", CreateRangePolicy(range_min, range_max), - kernel::AbsorbCurrentGR_kernel(domain.fields.cur0, + kernel::bc::AbsorbCurrentGR_kernel(domain.fields.cur0, domain.mesh.metric, xg_edge, ds)); @@ -686,16 +694,16 @@ namespace ntt { Kokkos::parallel_for( "OpenBCFields", range, - kernel::OpenBoundaries_kernel(domain.fields.em, i1_min, tags)); + kernel::bc::OpenBoundaries_kernel(domain.fields.em, i1_min, tags)); Kokkos::parallel_for( "OpenBCFields", range, - kernel::OpenBoundaries_kernel(domain.fields.em0, i1_min, tags)); + kernel::bc::OpenBoundaries_kernel(domain.fields.em0, i1_min, tags)); } else if (g == gr_bc::aux) { Kokkos::parallel_for( "OpenBCFields", range, - kernel::OpenBoundariesAux_kernel(domain.fields.aux, i1_min, tags)); + kernel::bc::OpenBoundariesAux_kernel(domain.fields.aux, i1_min, tags)); } } @@ -720,20 +728,20 @@ namespace ntt { Kokkos::parallel_for( "AxisBCFields", range, - kernel::AxisBoundariesGR_kernel(domain.fields.em, i2_min, tags)); + kernel::bc::AxisBoundariesGR_kernel(domain.fields.em, i2_min, tags)); Kokkos::parallel_for( "AxisBCFields", range, - kernel::AxisBoundariesGR_kernel(domain.fields.em0, i2_min, tags)); + kernel::bc::AxisBoundariesGR_kernel(domain.fields.em0, i2_min, tags)); } else { Kokkos::parallel_for( "AxisBCFields", range, - kernel::AxisBoundariesGR_kernel(domain.fields.em, i2_max, tags)); + kernel::bc::AxisBoundariesGR_kernel(domain.fields.em, i2_max, tags)); Kokkos::parallel_for( "AxisBCFields", range, - kernel::AxisBoundariesGR_kernel(domain.fields.em0, i2_max, tags)); + kernel::bc::AxisBoundariesGR_kernel(domain.fields.em0, i2_max, tags)); } } From a243e75c64a87bfa470c51d19e43a6b3cf6a3862 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 31 Jan 2025 12:31:26 -0500 Subject: [PATCH 201/234] gr absorb BC for fields: minor, tanh is computed only once now --- src/kernels/fields_bcs.hpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 45336830e..fcd0e46de 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -992,43 +992,44 @@ namespace kernel::bc { const auto dx = math::abs( metric.template convert(x_Cd[i - 1]) - xg_edge); - Fld(i1, i2, comp) *= math::tanh(dx / (INV_4 * dx_abs)); + const auto tanh = math::tanh(dx / (INV_4 * dx_abs)); + Fld(i1, i2, comp) *= tanh; if (comp == em::bx1) { const real_t x1_0 { metric.template convert<1, Crd::Cd, Crd::Ph>(i1_) }; const real_t x2_H { metric.template convert<2, Crd::Cd, Crd::Ph>( i2_ + HALF) }; - Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * + Fld(i1, i2, comp) += (ONE - tanh) * finit.bx1({ x1_0, x2_H }); } else if (comp == em::bx2) { const real_t x1_H { metric.template convert<1, Crd::Cd, Crd::Ph>( i1_ + HALF) }; const real_t x2_0 { metric.template convert<2, Crd::Cd, Crd::Ph>(i2_) }; - Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * + Fld(i1, i2, comp) += (ONE - tanh) * finit.bx2({ x1_H, x2_0 }); } else if (comp == em::bx3) { const real_t x1_H { metric.template convert<1, Crd::Cd, Crd::Ph>( i1_ + HALF) }; const real_t x2_H { metric.template convert<2, Crd::Cd, Crd::Ph>( i2_ + HALF) }; - Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * + Fld(i1, i2, comp) += (ONE - tanh) * finit.bx3({ x1_H, x2_H }); } else if (comp == em::ex1) { const real_t x1_H { metric.template convert<1, Crd::Cd, Crd::Ph>( i1_ + HALF) }; const real_t x2_0 { metric.template convert<2, Crd::Cd, Crd::Ph>(i2_) }; - Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * + Fld(i1, i2, comp) += (ONE - tanh) * finit.dx1({ x1_H, x2_0 }); } else if (comp == em::ex2) { const real_t x1_0 { metric.template convert<1, Crd::Cd, Crd::Ph>(i1_) }; const real_t x2_H { metric.template convert<2, Crd::Cd, Crd::Ph>( i2_ + HALF) }; - Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * + Fld(i1, i2, comp) += (ONE - tanh) * finit.dx2({ x1_0, x2_H }); } else if (comp == em::ex3) { const real_t x1_0 { metric.template convert<1, Crd::Cd, Crd::Ph>(i1_) }; const real_t x2_0 { metric.template convert<2, Crd::Cd, Crd::Ph>(i2_) }; - Fld(i1, i2, comp) += (ONE - math::tanh(dx / (INV_4 * dx_abs))) * + Fld(i1, i2, comp) += (ONE - tanh) * finit.dx3({ x1_0, x2_0 }); } } From 6c0ae144be546154d57320c8f22e84d045a7ddb4 Mon Sep 17 00:00:00 2001 From: haykh Date: Fri, 31 Jan 2025 13:26:59 -0500 Subject: [PATCH 202/234] dxmin compare between meshblocks --- src/framework/domain/metadomain.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/framework/domain/metadomain.cpp b/src/framework/domain/metadomain.cpp index ed4373df2..a1daaa07d 100644 --- a/src/framework/domain/metadomain.cpp +++ b/src/framework/domain/metadomain.cpp @@ -392,9 +392,10 @@ namespace ntt { mpi::get_type(), MPI_COMM_WORLD); for (const auto& dx : dx_mins) { - raise::ErrorIf(!cmp::AlmostEqual(dx, dx_min), - "dx_min is not the same across all MPI ranks", - HERE); + raise::ErrorIf( + !cmp::AlmostEqual(dx, dx_min, std::numeric_limits::epsilon()), + "dx_min is not the same across all MPI ranks", + HERE); } #endif } From fb0d7066253d3f81c21bcfe971e18ed44ebca5f9 Mon Sep 17 00:00:00 2001 From: haykh Date: Fri, 31 Jan 2025 13:28:18 -0500 Subject: [PATCH 203/234] dxmin compare between meshblocks (smaller precision) --- src/framework/domain/metadomain.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/framework/domain/metadomain.cpp b/src/framework/domain/metadomain.cpp index a1daaa07d..4f0d0d805 100644 --- a/src/framework/domain/metadomain.cpp +++ b/src/framework/domain/metadomain.cpp @@ -392,10 +392,12 @@ namespace ntt { mpi::get_type(), MPI_COMM_WORLD); for (const auto& dx : dx_mins) { - raise::ErrorIf( - !cmp::AlmostEqual(dx, dx_min, std::numeric_limits::epsilon()), - "dx_min is not the same across all MPI ranks", - HERE); + raise::ErrorIf(!cmp::AlmostEqual(dx, + dx_min, + std::numeric_limits::epsilon() * + static_cast(10.0)), + "dx_min is not the same across all MPI ranks", + HERE); } #endif } From 6cc29ac78cf261d0dfd3ac8e86136ebf024a5873 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 31 Jan 2025 15:00:05 -0500 Subject: [PATCH 204/234] bug in gr pusher -- BC at one of the axes --- src/kernels/particle_pusher_gr.hpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/kernels/particle_pusher_gr.hpp b/src/kernels/particle_pusher_gr.hpp index 71cfae3b4..c80fe188e 100644 --- a/src/kernels/particle_pusher_gr.hpp +++ b/src/kernels/particle_pusher_gr.hpp @@ -704,9 +704,6 @@ namespace kernel::gr { EMHalfPush(xp, vp, Dp_hat, Bp_hat, vp_upd); /* x^i(n) -> x^i(n + 1) */ coord_t xp_upd { ZERO }; - if (i1(p) < 15 && is_absorb_i1min) { - vp_upd[0] = -10.0; - } GeodesicCoordinatePush(Massive_t {}, xp, vp_upd, xp_upd); // update phi @@ -756,7 +753,7 @@ namespace kernel::gr { ux2(p) = -ux2(p); } } else if (i2(p) >= ni2) { - if (is_axis_i2min) { + if (is_axis_i2max) { i2(p) = ni2 - 1; dx2(p) = ONE - dx2(p); ux2(p) = -ux2(p); From f861da44441feeac63ceff71b3a8f4f73d90f89e Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 31 Jan 2025 15:03:04 -0500 Subject: [PATCH 205/234] vacuum wald setup updated after merging --- setups/grpic/vacuum/vacuum.toml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/setups/grpic/vacuum/vacuum.toml b/setups/grpic/vacuum/vacuum.toml index 621f5d127..e6aecef58 100644 --- a/setups/grpic/vacuum/vacuum.toml +++ b/setups/grpic/vacuum/vacuum.toml @@ -1,10 +1,10 @@ [simulation] - name = "wald_Dph_abs_a95" + name = "vacuum" engine = "grpic" - runtime = 1.0 + runtime = 100.0 [grid] - resolution = [128,128] + resolution = [128, 128] extent = [[1.0, 10.0]] [grid.metric] @@ -14,7 +14,7 @@ ks_a = 0.95 [grid.boundaries] - fields = [["ABSORB"]] + fields = [["MATCH"]] particles = [["ABSORB"]] [grid.boundaries.absorb] @@ -41,6 +41,7 @@ [output] format = "hdf5" + separate_files = false [output.fields] interval_time = 1.0 From 2e4df9a57c51e22c53bd45bab1328aec474835b8 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 31 Jan 2025 15:06:13 -0500 Subject: [PATCH 206/234] updated input for wald after merging --- setups/grpic/wald/wald.toml | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/setups/grpic/wald/wald.toml b/setups/grpic/wald/wald.toml index 3bf243afc..c515ae024 100644 --- a/setups/grpic/wald/wald.toml +++ b/setups/grpic/wald/wald.toml @@ -1,43 +1,48 @@ [simulation] name = "wald" engine = "grpic" - runtime = 100.0 + runtime = 500.0 [grid] - resolution = [512, 512] - extent = [[1.22, 6.0]] + resolution = [256, 256] + extent = [[1.0, 6.0]] [grid.metric] metric = "qkerr_schild" - qsph_r0 = 0.0 + qsph_r0 = 0.0 qsph_h = 0.0 ks_a = 0.95 [grid.boundaries] - fields = [["ABSORB"]] + fields = [["MATCH"]] particles = [["ABSORB"]] [grid.boundaries.absorb] - ds = 0.5 + ds = 1.0 [scales] - larmor0 = 0.0025 - skindepth0 = 0.05 + larmor0 = 0.025 + skindepth0 = 0.5 [algorithms] current_filters = 4 + [algorithms.gr] + pusher_niter = 10 + pusher_eps = 1e-2 + [algorithms.timestep] CFL = 0.5 + correction = 1.0 [algorithms.toggles] deposit = true fieldsolver = true [particles] - ppc0 = 8.0 + ppc0 = 4.0 use_weights = true - sort_interval = 100 + clear_interval = 100 [[particles.species]] label = "e-" @@ -57,15 +62,17 @@ multiplicity = 1.0 sigma_max = 1000.0 temperature = 0.01 - xi_min = [1.32, 0.1] - xi_max = [4.0, 3.04159265] + xi_min = [1.5, 0.0] + xi_max = [4.0, 3.14159265] + m_eps = 1.0 [output] format = "hdf5" + separate_files = false [output.fields] interval_time = 1.0 - quantities = ["D", "B", "J", "N_1", "N_2", "A"] + quantities = ["D", "B", "N_1", "N_2", "A"] [output.particles] enable = false From 146056912a7be95f86afc1d3b10f7983ab8c6cc4 Mon Sep 17 00:00:00 2001 From: haykh Date: Fri, 31 Jan 2025 15:08:15 -0500 Subject: [PATCH 207/234] BCs for GR same as SR --- setups/grpic/wald/pgen.hpp | 126 ++--- src/engines/grpic.hpp | 301 ++++++------ src/framework/domain/metadomain.cpp | 9 +- src/global/global.h | 4 + src/kernels/fields_bcs.hpp | 696 +++++++++++++--------------- 5 files changed, 562 insertions(+), 574 deletions(-) diff --git a/setups/grpic/wald/pgen.hpp b/setups/grpic/wald/pgen.hpp index b27dd370e..7f8146bb9 100644 --- a/setups/grpic/wald/pgen.hpp +++ b/setups/grpic/wald/pgen.hpp @@ -6,14 +6,15 @@ #include "arch/kokkos_aliases.h" #include "arch/traits.h" +#include "utils/numeric.h" #include "archetypes/energy_dist.h" -#include "archetypes/spatial_dist.h" #include "archetypes/particle_injector.h" #include "archetypes/problem_generator.h" +#include "archetypes/spatial_dist.h" #include "framework/domain/metadomain.h" + #include "kernels/particle_moments.hpp" -#include "utils/numeric.h" namespace user { using namespace ntt; @@ -23,30 +24,31 @@ namespace user { InitFields(M metric_, real_t m_eps) : metric { metric_ }, m_eps { m_eps } {} Inline auto A_3(const coord_t& x_Cd) const -> real_t { - return HALF * (metric.template h_<3, 3>(x_Cd) - + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) - ); + return HALF * (metric.template h_<3, 3>(x_Cd) + + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * + metric.beta1(x_Cd)); } Inline auto bx1(const coord_t& x_Ph) const -> real_t { // at ( i , j + HALF ) - coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + coord_t xi { ZERO }, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); x0m[0] = xi[0]; x0m[1] = xi[1] - HALF * m_eps; x0p[0] = xi[0]; x0p[1] = xi[1] + HALF * m_eps; - + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; - if (cmp::AlmostZero(x_Ph[1])) + if (cmp::AlmostZero(x_Ph[1])) { return ONE; - else + } else { return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP / m_eps; + } } Inline auto bx2(const coord_t& x_Ph) const -> real_t { // at ( i + HALF , j ) - coord_t xi {ZERO}, x0m { ZERO }, x0p { ZERO }; + coord_t xi { ZERO }, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); x0m[0] = xi[0] - HALF * m_eps; @@ -54,11 +56,12 @@ namespace user { x0p[0] = xi[0] + HALF * m_eps; x0p[1] = xi[1]; - real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0] , xi[1] }) }; - if (cmp::AlmostZero(x_Ph[1])) + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + if (cmp::AlmostZero(x_Ph[1])) { return ZERO; - else + } else { return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP / m_eps; + } } Inline auto bx3(const coord_t& x_Ph) const -> real_t { @@ -78,7 +81,7 @@ namespace user { } private: - const M metric; + const M metric; const real_t m_eps; }; @@ -89,14 +92,13 @@ namespace user { const real_t sigma_thr, const real_t dens_thr, const SimulationParams& params, - Domain* domain_ptr - ) - : arch::SpatialDistribution { domain_ptr->mesh.metric } + Domain* domain_ptr) + : arch::SpatialDistribution { domain_ptr->mesh.metric } , metric { domain_ptr->mesh.metric } - , EM { domain_ptr->fields.em } + , EM { domain_ptr->fields.em } , density { domain_ptr->fields.buff } - , sigma_thr {sigma_thr} - , dens_thr {dens_thr} { + , sigma_thr { sigma_thr } + , dens_thr { dens_thr } { std::copy(xi_min.begin(), xi_min.end(), x_min); std::copy(xi_max.begin(), xi_max.end(), x_max); @@ -108,12 +110,13 @@ namespace user { } Kokkos::deep_copy(density, ZERO); - auto scatter_buff = Kokkos::Experimental::create_scatter_view(density); + auto scatter_buff = Kokkos::Experimental::create_scatter_view(density); // some parameters - auto& mesh = domain_ptr->mesh; - const auto use_weights = params.template get("particles.use_weights"); - const auto ni2 = mesh.n_active(in::x2); - const auto inv_n0 = ONE / params.template get("scales.n0"); + auto& mesh = domain_ptr->mesh; + const auto use_weights = params.template get( + "particles.use_weights"); + const auto ni2 = mesh.n_active(in::x2); + const auto inv_n0 = ONE / params.template get("scales.n0"); for (const auto& sp : specs) { auto& prtl_spec = domain_ptr->species[sp - 1]; @@ -136,15 +139,18 @@ namespace user { } Inline auto sigma_crit(const coord_t& x_Ph) const -> bool { - coord_t xi {ZERO}; + coord_t xi { ZERO }; if constexpr (M::Dim == Dim::_2D) { metric.template convert(x_Ph, xi); const auto i1 = static_cast(xi[0]) + static_cast(N_GHOSTS); const auto i2 = static_cast(xi[1]) + static_cast(N_GHOSTS); - const vec_t B_cntrv { EM(i1, i2, em::bx1), EM(i1, i2, em::bx2), EM(i1, i2, em::bx3) }; - vec_t B_cov { ZERO }; + const vec_t B_cntrv { EM(i1, i2, em::bx1), + EM(i1, i2, em::bx2), + EM(i1, i2, em::bx3) }; + vec_t B_cov { ZERO }; metric.template transform(xi, B_cntrv, B_cov); - const auto bsqr = DOT(B_cntrv[0], B_cntrv[1], B_cntrv[2], B_cov[0], B_cov[1], B_cov[2]); + const auto bsqr = + DOT(B_cntrv[0], B_cntrv[1], B_cntrv[2], B_cov[0], B_cov[1], B_cov[2]); const auto dens = density(i1, i2, 0); return (bsqr > sigma_thr * dens) || (dens < dens_thr); } @@ -162,12 +168,12 @@ namespace user { private: tuple_t x_min; tuple_t x_max; - const real_t sigma_thr; - const real_t dens_thr; - Domain* domain_ptr; - ndfield_t density; - ndfield_t EM; - const M metric; + const real_t sigma_thr; + const real_t dens_thr; + Domain* domain_ptr; + ndfield_t density; + ndfield_t EM; + const M metric; }; template @@ -197,13 +203,14 @@ namespace user { , sigma_max { p.template get("setup.sigma_max") } , sigma0 { p.template get("scales.sigma0") } , multiplicity { p.template get("setup.multiplicity") } - , nGJ { p.template get("scales.B0") * SQR(p.template get("scales.skindepth0")) } + , nGJ { p.template get("scales.B0") * + SQR(p.template get("scales.skindepth0")) } , temperature { p.template get("setup.temperature") } , m_eps { p.template get("setup.m_eps") } , init_flds { m.mesh().metric, m_eps } {} - + inline void InitPrtls(Domain& local_domain) { - const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, + const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, local_domain.random_pool, temperature); const auto spatial_dist = PointDistribution(xi_min, @@ -211,22 +218,22 @@ namespace user { sigma_max / sigma0, multiplicity * nGJ, params, - &local_domain - ); + &local_domain); - const auto injector = arch::NonUniformInjector( - energy_dist, - spatial_dist, - { 1, 2 }); + const auto injector = + arch::NonUniformInjector( + energy_dist, + spatial_dist, + { 1, 2 }); arch::InjectNonUniform(params, - local_domain, - injector, - 1.0, - true); + local_domain, + injector, + 1.0, + true); } void CustomPostStep(std::size_t, long double time, Domain& local_domain) { - const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, + const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, local_domain.random_pool, temperature); const auto spatial_dist = PointDistribution(xi_min, @@ -234,20 +241,19 @@ namespace user { sigma_max / sigma0, multiplicity * nGJ, params, - &local_domain - ); + &local_domain); - const auto injector = arch::NonUniformInjector( - energy_dist, - spatial_dist, - { 1, 2 }); + const auto injector = + arch::NonUniformInjector( + energy_dist, + spatial_dist, + { 1, 2 }); arch::InjectNonUniform(params, - local_domain, - injector, - 1.0, - true); + local_domain, + injector, + 1.0, + true); } - }; } // namespace user diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 6a1575a5a..d6a40bbc4 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -101,7 +101,7 @@ namespace ntt { "algorithms.toggles.deposit"); const auto clear_interval = m_params.template get( "particles.clear_interval"); - + if (step == 0) { if (fieldsolver_enabled) { // communicate fields and apply BCs on the first timestep @@ -124,8 +124,9 @@ namespace ntt { /** * em0::D, em::D, em0::B, em::B <- boundary conditions */ - m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0 | Comm::D | Comm::D0); - FieldBoundaries(dom, BC::B | BC::E, gr_bc::main); + m_metadomain.CommunicateFields(dom, + Comm::B | Comm::B0 | Comm::D | Comm::D0); + FieldBoundaries(dom, BC::B | BC::D, gr_bc::main); /** * em0::B <- em::B @@ -147,7 +148,7 @@ namespace ntt { /** * aux::E, aux::H <- boundary conditions */ - FieldBoundaries(dom, BC::B | BC::E, gr_bc::aux); + FieldBoundaries(dom, BC::H | BC::E, gr_bc::aux); /** * em0::B <- (em0::B) <- -curl aux::E @@ -173,7 +174,7 @@ namespace ntt { * em0::D, em::D <- boundary conditions */ m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); - FieldBoundaries(dom, BC::E, gr_bc::main); + FieldBoundaries(dom, BC::D, gr_bc::main); /** * aux::E <- alpha * em::D + beta x em0::B @@ -187,7 +188,7 @@ namespace ntt { /** * aux::E, aux::H <- boundary conditions */ - FieldBoundaries(dom, BC::B | BC::E, gr_bc::aux); + FieldBoundaries(dom, BC::H | BC::E, gr_bc::aux); // !ADD: GR -- particles? @@ -213,7 +214,7 @@ namespace ntt { * em0::D, em::D <- boundary conditions */ m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); - FieldBoundaries(dom, BC::E, gr_bc::main); + FieldBoundaries(dom, BC::D, gr_bc::main); /** * aux::H <- alpha * em0::B - beta x em0::D @@ -224,7 +225,7 @@ namespace ntt { /** * aux::H <- boundary conditions */ - FieldBoundaries(dom, BC::B, gr_bc::aux); + FieldBoundaries(dom, BC::H, gr_bc::aux); /** * em0::D <- (em::D) <- curl aux::H @@ -237,7 +238,7 @@ namespace ntt { * em0::D, em::D <- boundary conditions */ m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); - FieldBoundaries(dom, BC::E, gr_bc::main); + FieldBoundaries(dom, BC::D, gr_bc::main); /** * em::D <-> em0::D @@ -261,31 +262,31 @@ namespace ntt { * u_prtl at 1/2 */ } else { - /** - * em0::B <- em::B - * em0::D <- em::D - * - * Now: em0::B & em0::D at -1/2 - */ - CopyFields(dom); + /** + * em0::B <- em::B + * em0::D <- em::D + * + * Now: em0::B & em0::D at -1/2 + */ + CopyFields(dom); } } - /** - * Initially: em0::B at n-3/2 - * em0::D at n-1 - * em::B at n-1/2 - * em::D at n - * - * cur0::J -- - * cur::J at n-1/2 - * - * aux::E -- - * aux::H -- - * - * x_prtl at n - * u_prtl at n-1/2 - */ + /** + * Initially: em0::B at n-3/2 + * em0::D at n-1 + * em::B at n-1/2 + * em::D at n + * + * cur0::J -- + * cur::J at n-1/2 + * + * aux::E -- + * aux::H -- + * + * x_prtl at n + * u_prtl at n-1/2 + */ if (fieldsolver_enabled) { timers.start("FieldSolver"); @@ -304,14 +305,14 @@ namespace ntt { */ ComputeAuxE(dom, gr_getE::D0_B); timers.stop("FieldSolver"); - + timers.start("FieldBoundaries"); /** * aux::E <- boundary conditions */ FieldBoundaries(dom, BC::E, gr_bc::aux); timers.stop("FieldBoundaries"); - + timers.start("FieldSolver"); /** * em0::B <- (em0::B) <- -curl aux::E @@ -344,7 +345,7 @@ namespace ntt { /** * aux::H <- boundary conditions */ - FieldBoundaries(dom, BC::B, gr_bc::aux); + FieldBoundaries(dom, BC::H, gr_bc::aux); timers.stop("FieldBoundaries"); } @@ -384,9 +385,6 @@ namespace ntt { } timers.start("Communications"); - // if ((sort_interval > 0) and (step % sort_interval == 0)) { - // m_metadomain.CommunicateParticles(dom, &timers); - // } m_metadomain.CommunicateParticles(dom); timers.stop("Communications"); } @@ -423,7 +421,7 @@ namespace ntt { */ Faraday(dom, gr_faraday::main, ONE); timers.stop("FieldSolver"); - + /** * em0::B, em::B <- boundary conditions */ @@ -453,7 +451,7 @@ namespace ntt { AmpereCurrents(dom, gr_ampere::aux); timers.stop("FieldSolver"); } - + /** * em0::D, em::D <- boundary conditions */ @@ -461,7 +459,7 @@ namespace ntt { m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); timers.stop("Communications"); timers.start("FieldBoundaries"); - FieldBoundaries(dom, BC::E, gr_bc::main); + FieldBoundaries(dom, BC::D, gr_bc::main); timers.stop("FieldBoundaries"); timers.start("FieldSolver"); @@ -516,7 +514,7 @@ namespace ntt { m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); timers.stop("Communications"); timers.start("FieldBoundaries"); - FieldBoundaries(dom, BC::E, gr_bc::main); + FieldBoundaries(dom, BC::D, gr_bc::main); timers.stop("FieldBoundaries"); } @@ -548,7 +546,7 @@ namespace ntt { if (g == gr_bc::main) { for (auto& direction : dir::Directions::orth) { if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::MATCH) { - AbsorbFieldsIn(direction, domain, tags); + MatchFieldsIn(direction, domain, tags); } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::AXIS) { if (domain.mesh.flds_bc_in(direction) == FldsBC::AXIS) { AxisFieldsIn(direction, domain, tags); @@ -558,26 +556,25 @@ namespace ntt { CustomFieldsIn(direction, domain, tags, g); } } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { - OpenFieldsIn(direction, domain, tags, g); + HorizonFieldsIn(direction, domain, tags, g); } } // loop over directions } else if (g == gr_bc::aux) { for (auto& direction : dir::Directions::orth) { if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { - OpenFieldsIn(direction, domain, tags, g); + HorizonFieldsIn(direction, domain, tags, g); } } } } - void AbsorbFieldsIn(dir::direction_t direction, - domain_t& domain, - BCTags tags) { + void MatchFieldsIn(dir::direction_t direction, + domain_t& domain, + BCTags tags) { /** - * absorbing boundaries + * match boundaries */ - const auto ds = m_params.template get( - "grid.boundaries.absorb.ds"); + const auto ds = m_params.template get("grid.boundaries.match.ds"); const auto dim = direction.get_dim(); real_t xg_min, xg_max, xg_edge; xg_max = m_metadomain.mesh().extent(dim).second; @@ -608,34 +605,35 @@ namespace ntt { } if (dim == in::x1) { Kokkos::parallel_for( - "AbsorbFields", + "MatchBoundaries", CreateRangePolicy(range_min, range_max), - kernel::bc::AbsorbBoundariesGR_kernel(domain.fields.em, - m_pgen.init_flds, - domain.mesh.metric, - xg_edge, - ds, - tags)); + kernel::bc::MatchBoundaries_kernel( + domain.fields.em, + m_pgen.init_flds, + domain.mesh.metric, + xg_edge, + ds, + tags)); Kokkos::parallel_for( - "AbsorbFields", + "MatchBoundaries", CreateRangePolicy(range_min, range_max), - kernel::bc::AbsorbBoundariesGR_kernel(domain.fields.em0, - m_pgen.init_flds, - domain.mesh.metric, - xg_edge, - ds, - tags)); + kernel::bc::MatchBoundaries_kernel( + domain.fields.em0, + m_pgen.init_flds, + domain.mesh.metric, + xg_edge, + ds, + tags)); } else { - raise::Error("Invalid dimension", HERE); + raise::Error("Invalid dimension", HERE); } } void CurrentsBoundaryConditions(domain_t& domain) { /** - * absorbing boundaries + * match boundaries */ - const auto ds = m_params.template get( - "grid.boundaries.absorb.ds"); + const auto ds = m_params.template get("grid.boundaries.match.ds"); const auto dim = in::x1; real_t xg_min, xg_max, xg_edge; xg_max = m_metadomain.mesh().extent(dim).second; @@ -665,18 +663,18 @@ namespace ntt { range_max[d] = intersect_range[d].second; } Kokkos::parallel_for( - "AbsorbCurrent", - CreateRangePolicy(range_min, range_max), - kernel::bc::AbsorbCurrentGR_kernel(domain.fields.cur0, - domain.mesh.metric, - xg_edge, - ds)); + "AbsorbCurrentsGR", + CreateRangePolicy(range_min, range_max), + kernel::bc::AbsorbCurrentsGR_kernel(domain.fields.cur0, + domain.mesh.metric, + xg_edge, + ds)); } - void OpenFieldsIn(dir::direction_t direction, - domain_t& domain, - BCTags tags, - const gr_bc& g) { + void HorizonFieldsIn(dir::direction_t direction, + domain_t& domain, + BCTags tags, + const gr_bc& g) { /** * open boundaries */ @@ -687,23 +685,28 @@ namespace ntt { "Invalid axis direction, should be x2", HERE); const auto i1_min = domain.mesh.i_min(in::x1); - auto range = CreateRangePolicy( - {domain.mesh.i_min(in::x2)}, - {domain.mesh.i_max(in::x2) + 1}); + auto range = CreateRangePolicy({ domain.mesh.i_min(in::x2) }, + { domain.mesh.i_max(in::x2) + 1 }); if (g == gr_bc::main) { Kokkos::parallel_for( "OpenBCFields", range, - kernel::bc::OpenBoundaries_kernel(domain.fields.em, i1_min, tags)); + kernel::bc::HorizonBoundaries_kernel(domain.fields.em, + i1_min, + tags)); Kokkos::parallel_for( "OpenBCFields", - range, - kernel::bc::OpenBoundaries_kernel(domain.fields.em0, i1_min, tags)); + range, + kernel::bc::HorizonBoundaries_kernel(domain.fields.em0, + i1_min, + tags)); } else if (g == gr_bc::aux) { Kokkos::parallel_for( "OpenBCFields", range, - kernel::bc::OpenBoundariesAux_kernel(domain.fields.aux, i1_min, tags)); + kernel::bc::HorizonBoundaries_kernel(domain.fields.aux, + i1_min, + tags)); } } @@ -721,27 +724,34 @@ namespace ntt { HERE); const auto i2_min = domain.mesh.i_min(in::x2); const auto i2_max = domain.mesh.i_max(in::x2); - auto range = CreateRangePolicy( - {domain.mesh.i_min(in::x1) - 1}, - {domain.mesh.i_max(in::x1)}); + auto range = CreateRangePolicy({ domain.mesh.i_min(in::x1) - 1 }, + { domain.mesh.i_max(in::x1) }); if (direction.get_sign() < 0) { Kokkos::parallel_for( "AxisBCFields", range, - kernel::bc::AxisBoundariesGR_kernel(domain.fields.em, i2_min, tags)); + kernel::bc::AxisBoundariesGR_kernel(domain.fields.em, + i2_min, + tags)); Kokkos::parallel_for( "AxisBCFields", range, - kernel::bc::AxisBoundariesGR_kernel(domain.fields.em0, i2_min, tags)); + kernel::bc::AxisBoundariesGR_kernel(domain.fields.em0, + i2_min, + tags)); } else { Kokkos::parallel_for( "AxisBCFields", range, - kernel::bc::AxisBoundariesGR_kernel(domain.fields.em, i2_max, tags)); + kernel::bc::AxisBoundariesGR_kernel(domain.fields.em, + i2_max, + tags)); Kokkos::parallel_for( "AxisBCFields", range, - kernel::bc::AxisBoundariesGR_kernel(domain.fields.em0, i2_max, tags)); + kernel::bc::AxisBoundariesGR_kernel(domain.fields.em0, + i2_max, + tags)); } } @@ -783,12 +793,13 @@ namespace ntt { void ComputeAuxE(domain_t& domain, const gr_getE& g) { auto range = range_with_axis_BCs(domain); if (g == gr_getE::D0_B) { - Kokkos::parallel_for("ComputeAuxE", - range, - kernel::gr::ComputeAuxE_kernel(domain.fields.em0, // D - domain.fields.em, // B - domain.fields.aux, // E - domain.mesh.metric)); + Kokkos::parallel_for( + "ComputeAuxE", + range, + kernel::gr::ComputeAuxE_kernel(domain.fields.em0, // D + domain.fields.em, // B + domain.fields.aux, // E + domain.mesh.metric)); } else if (g == gr_getE::D_B0) { Kokkos::parallel_for("ComputeAuxE", range, @@ -804,12 +815,13 @@ namespace ntt { void ComputeAuxH(domain_t& domain, const gr_getH& g) { auto range = range_with_axis_BCs(domain); if (g == gr_getH::D_B0) { - Kokkos::parallel_for("ComputeAuxH", - range, - kernel::gr::ComputeAuxH_kernel(domain.fields.em, // D - domain.fields.em0, // B - domain.fields.aux, // H - domain.mesh.metric)); + Kokkos::parallel_for( + "ComputeAuxH", + range, + kernel::gr::ComputeAuxH_kernel(domain.fields.em, // D + domain.fields.em0, // B + domain.fields.aux, // H + domain.mesh.metric)); } else if (g == gr_getH::D0_B0) { Kokkos::parallel_for("ComputeAuxH", range, @@ -880,8 +892,8 @@ namespace ntt { "algorithms.timestep.correction") * dt; auto range = CreateRangePolicy( - { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2)}, - { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1}); + { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) }, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1 }); const auto ni2 = domain.mesh.n_active(in::x2); if (g == gr_ampere::aux) { @@ -927,51 +939,52 @@ namespace ntt { const auto q0 = m_params.template get("scales.q0"); const auto B0 = m_params.template get("scales.B0"); const auto coeff = -dt * q0 / B0; - auto range = CreateRangePolicy( - { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2)}, - { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1}); - const auto ni2 = domain.mesh.n_active(in::x2); + auto range = CreateRangePolicy( + { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) }, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1 }); + const auto ni2 = domain.mesh.n_active(in::x2); if (g == gr_ampere::aux) { // Updates D0 with J: D0(n-1/2) -> (J(n)) -> D0(n+1/2) - Kokkos::parallel_for("AmpereCurrentsAux", - range, - kernel::gr::CurrentsAmpere_kernel(domain.fields.em0, - domain.fields.cur, - domain.mesh.metric, - coeff, - ni2, - domain.mesh.flds_bc())); + Kokkos::parallel_for( + "AmpereCurrentsAux", + range, + kernel::gr::CurrentsAmpere_kernel(domain.fields.em0, + domain.fields.cur, + domain.mesh.metric, + coeff, + ni2, + domain.mesh.flds_bc())); } else if (g == gr_ampere::main) { // Updates D0 with J0: D0(n) -> (J0(n+1/2)) -> D0(n+1) - Kokkos::parallel_for("mpereCurrentsMain", - range, - kernel::gr::CurrentsAmpere_kernel(domain.fields.em0, - domain.fields.cur0, - domain.mesh.metric, - coeff, - ni2, - domain.mesh.flds_bc())); + Kokkos::parallel_for( + "AmpereCurrentsMain", + range, + kernel::gr::CurrentsAmpere_kernel(domain.fields.em0, + domain.fields.cur0, + domain.mesh.metric, + coeff, + ni2, + domain.mesh.flds_bc())); } else { raise::Error("Wrong option for `g`", HERE); } - } void TimeAverageDB(domain_t& domain) { - Kokkos::parallel_for("TimeAverageDB", - domain.mesh.rangeActiveCells(), - kernel::gr::TimeAverageDB_kernel(domain.fields.em, - domain.fields.em0, - domain.mesh.metric)); + Kokkos::parallel_for("TimeAverageDB", + domain.mesh.rangeActiveCells(), + kernel::gr::TimeAverageDB_kernel(domain.fields.em, + domain.fields.em0, + domain.mesh.metric)); } void TimeAverageJ(domain_t& domain) { - Kokkos::parallel_for("TimeAverageJ", - domain.mesh.rangeActiveCells(), - kernel::gr::TimeAverageJ_kernel(domain.fields.cur, - domain.fields.cur0, - domain.mesh.metric)); + Kokkos::parallel_for("TimeAverageJ", + domain.mesh.rangeActiveCells(), + kernel::gr::TimeAverageJ_kernel(domain.fields.cur, + domain.fields.cur0, + domain.mesh.metric)); } void CurrentsDeposit(domain_t& domain) { @@ -1020,8 +1033,8 @@ namespace ntt { void CurrentsFilter(domain_t& domain) { logger::Checkpoint("Launching currents filtering kernels", HERE); auto range = CreateRangePolicy( - { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2)}, - { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1}); + { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) }, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1 }); const auto nfilter = m_params.template get( "algorithms.current_filters"); tuple_t size; @@ -1038,7 +1051,7 @@ namespace ntt { domain.fields.buff, size, domain.mesh.flds_bc())); - m_metadomain.CommunicateFields(domain, Comm::J); //J0 + m_metadomain.CommunicateFields(domain, Comm::J); // J0 } } @@ -1058,9 +1071,10 @@ namespace ntt { ? species.charge() / species.mass() : ZERO; // coeff = q / m (dt / 2) omegaB0 - const auto coeff = q_ovr_m * HALF * dt * m_params.template get( - "algorithms.timestep.correction") * - m_params.template get("scales.omegaB0"); + const auto coeff = q_ovr_m * HALF * dt * + m_params.template get( + "algorithms.timestep.correction") * + m_params.template get("scales.omegaB0"); // clang-format off if (species.pusher() == PrtlPusher::PHOTON) { @@ -1119,7 +1133,6 @@ namespace ntt { // clang-format on } } - }; } // namespace ntt diff --git a/src/framework/domain/metadomain.cpp b/src/framework/domain/metadomain.cpp index 4f0d0d805..101bf2144 100644 --- a/src/framework/domain/metadomain.cpp +++ b/src/framework/domain/metadomain.cpp @@ -369,6 +369,8 @@ namespace ntt { template void Metadomain::metricCompatibilityCheck() const { + const auto epsilon = std::numeric_limits::epsilon() * + static_cast(100.0); const auto dx_min = g_mesh.metric.dxMin(); auto dx_min_from_domains = std::numeric_limits::infinity(); for (unsigned int idx { 0 }; idx < g_ndomains; ++idx) { @@ -377,7 +379,7 @@ namespace ntt { dx_min_from_domains = std::min(dx_min_from_domains, current_dx_min); } raise::ErrorIf( - not cmp::AlmostEqual(dx_min, dx_min_from_domains), + not cmp::AlmostEqual(dx_min / dx_min_from_domains, ONE, epsilon), "dx_min is not the same across all domains: " + std::to_string(dx_min) + " " + std::to_string(dx_min_from_domains), HERE); @@ -392,10 +394,7 @@ namespace ntt { mpi::get_type(), MPI_COMM_WORLD); for (const auto& dx : dx_mins) { - raise::ErrorIf(!cmp::AlmostEqual(dx, - dx_min, - std::numeric_limits::epsilon() * - static_cast(10.0)), + raise::ErrorIf(not cmp::AlmostEqual(dx / dx_min, ONE, epsilon), "dx_min is not the same across all MPI ranks", HERE); } diff --git a/src/global/global.h b/src/global/global.h index 77fa8c51c..3f63eafbe 100644 --- a/src/global/global.h +++ b/src/global/global.h @@ -285,9 +285,13 @@ namespace BC { Dx1 = 1 << 0, Dx2 = 1 << 1, Dx3 = 1 << 2, + Hx1 = 1 << 3, + Hx2 = 1 << 4, + Hx3 = 1 << 5, B = Bx1 | Bx2 | Bx3, E = Ex1 | Ex2 | Ex3, D = Dx1 | Dx2 | Dx3, + H = Hx1 | Hx2 | Hx3, }; } // namespace BC diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index fcd0e46de..6c3bd2760 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -4,7 +4,10 @@ * @implements * - kernel::bc::MatchBoundaries_kernel<> * - kernel::bc::AxisBoundaries_kernel<> + * - kernel::bc::AxisBoundariesGR_kernel<> + * - kernel::bc::AbsorbCurrentsGR_kernel<> * - kernel::bc::EnforcedBoundaries_kernel<> + * - kernel::bc::HorizonBoundaries_kernel<> * @namespaces: * - kernel::bc:: */ @@ -89,80 +92,64 @@ namespace kernel::bc { metric.template convert({ i1_ }, x_Ph_0); metric.template convert({ i1_ + HALF }, x_Ph_H); - // SRPIC - auto ex1_U { ZERO }, ex2_U { ZERO }, ex3_U { ZERO }, bx1_U { ZERO }, - bx2_U { ZERO }, bx3_U { ZERO }; - if (tags & BC::E) { - if constexpr (defines_ex1) { - ex1_U = metric.template transform<1, Idx::T, Idx::U>( - { i1_ + HALF }, - fset.ex1(x_Ph_H)); - } - if constexpr (defines_ex2) { - ex2_U = metric.template transform<2, Idx::T, Idx::U>( - { i1_ }, - fset.ex2(x_Ph_0)); - } - if constexpr (defines_ex3) { - ex3_U = metric.template transform<3, Idx::T, Idx::U>( - { i1_ }, - fset.ex3(x_Ph_0)); - } - } - if (tags & BC::B) { - if constexpr (defines_bx1) { - bx1_U = metric.template transform<1, Idx::T, Idx::U>( - { i1_ }, - fset.bx1(x_Ph_0)); - } - if constexpr (defines_bx2) { - bx2_U = metric.template transform<2, Idx::T, Idx::U>( - { i1_ + HALF }, - fset.bx2(x_Ph_H)); - } - if constexpr (defines_bx3) { - bx3_U = metric.template transform<3, Idx::T, Idx::U>( - { i1_ + HALF }, - fset.bx3(x_Ph_H)); - } - } - if constexpr (defines_ex1 or defines_bx2 or defines_bx3) { - const auto dx = math::abs( - metric.template convert(i1_ + HALF) - xg_edge); - const auto s = shape(dx); + const auto s = shape(math::abs( + metric.template convert(i1_ + HALF) - xg_edge)); if constexpr (defines_ex1) { if (tags & BC::E) { - Fld(i1, em::ex1) = s * Fld(i1, em::ex1) + (ONE - s) * ex1_U; + Fld(i1, em::ex1) = s * Fld(i1, em::ex1) + + (ONE - s) * + metric.template transform<1, Idx::T, Idx::U>( + { i1_ + HALF }, + fset.ex1(x_Ph_H)); } } if constexpr (defines_bx2 or defines_bx3) { if (tags & BC::B) { if constexpr (defines_bx2) { - Fld(i1, em::bx2) = s * Fld(i1, em::bx2) + (ONE - s) * bx2_U; + Fld(i1, em::bx2) = s * Fld(i1, em::bx2) + + (ONE - s) * + metric.template transform<2, Idx::T, Idx::U>( + { i1_ + HALF }, + fset.bx2(x_Ph_H)); } if constexpr (defines_bx3) { - Fld(i1, em::bx3) = s * Fld(i1, em::bx3) + (ONE - s) * bx3_U; + Fld(i1, em::bx3) = s * Fld(i1, em::bx3) + + (ONE - s) * + metric.template transform<3, Idx::T, Idx::U>( + { i1_ + HALF }, + fset.bx3(x_Ph_H)); } } } } if constexpr (defines_bx1 or defines_ex2 or defines_ex3) { - const auto dx = math::abs( - metric.template convert(i1_) - xg_edge); - const auto s = shape(dx); + const auto s = shape(math::abs( + metric.template convert(i1_) - xg_edge)); if constexpr (defines_bx1) { if (tags & BC::B) { - Fld(i1, em::bx1) = s * Fld(i1, em::bx1) + (ONE - s) * bx1_U; + Fld(i1, em::bx1) = s * Fld(i1, em::bx1) + + (ONE - s) * + metric.template transform<1, Idx::T, Idx::U>( + { i1_ }, + fset.bx1(x_Ph_0)); } } if constexpr (defines_ex2 or defines_ex3) { if (tags & BC::E) { if constexpr (defines_ex2) { - Fld(i1, em::ex2) = s * Fld(i1, em::ex2) + (ONE - s) * ex2_U; + Fld(i1, em::ex2) = s * Fld(i1, em::ex2) + + (ONE - s) * + metric.template transform<2, Idx::T, Idx::U>( + { i1_ }, + fset.ex2(x_Ph_0)); } if constexpr (defines_ex3) { - Fld(i1, em::ex3) = s * Fld(i1, em::ex3) + (ONE - s) * ex3_U; + Fld(i1, em::ex3) = s * Fld(i1, em::ex3) + + (ONE - s) * + metric.template transform<3, Idx::T, Idx::U>( + { i1_ }, + fset.ex3(x_Ph_0)); } } } @@ -183,123 +170,156 @@ namespace kernel::bc { const auto i1_ = COORD(i1); const auto i2_ = COORD(i2); - if constexpr (S == SimEngine::SRPIC) { - // SRPIC - if constexpr (defines_ex1 or defines_bx2) { - coord_t x_Ph_H0 { ZERO }; - metric.template convert({ i1_ + HALF, i2_ }, x_Ph_H0); - // i1 + 1/2, i2 - real_t xi_Cd; - if constexpr (o == in::x1) { - xi_Cd = i1_ + HALF; - } else { - xi_Cd = i2_; + // SRPIC + if constexpr (defines_ex1 or defines_dx1 or defines_bx2) { + // i1 + 1/2, i2 + real_t xi_Cd; + if constexpr (o == in::x1) { + xi_Cd = i1_ + HALF; + } else { + xi_Cd = i2_; + } + + const auto s = shape(math::abs( + metric.template convert(xi_Cd) - xg_edge)); + + coord_t x_Ph_H0 { ZERO }; + metric.template convert({ i1_ + HALF, i2_ }, x_Ph_H0); + + if constexpr (defines_ex1 or defines_dx1) { + if ((tags & BC::E) or (tags & BC::D)) { + if constexpr (defines_ex1 and S == SimEngine::SRPIC) { + Fld(i1, i2, em::ex1) = s * Fld(i1, i2, em::ex1) + + (ONE - s) * + metric.template transform<1, Idx::T, Idx::U>( + { i1_ + HALF, i2_ }, + fset.ex1(x_Ph_H0)); + } else if constexpr (defines_dx1 and S == SimEngine::GRPIC) { + Fld(i1, i2, em::dx1) = s * Fld(i1, i2, em::dx1) + + (ONE - s) * fset.dx1(x_Ph_H0); + } } + } - const auto dx = math::abs( - metric.template convert(xi_Cd) - xg_edge); - const auto s = shape(dx); + if constexpr (defines_bx2) { + if (tags & BC::B) { + if constexpr (S == SimEngine::SRPIC) { + Fld(i1, i2, em::bx2) = s * Fld(i1, i2, em::bx2) + + (ONE - s) * + metric.template transform<2, Idx::T, Idx::U>( + { i1_ + HALF, i2_ }, + fset.bx2(x_Ph_H0)); + } else if constexpr (S == SimEngine::GRPIC) { + Fld(i1, i2, em::bx2) = s * Fld(i1, i2, em::bx2) + + (ONE - s) * fset.bx2(x_Ph_H0); + } + } + } + } - if constexpr (defines_ex1) { - if (tags & BC::E) { - const auto ex1_U = metric.template transform<1, Idx::T, Idx::U>( - { i1_ + HALF, i2_ }, - fset.ex1(x_Ph_H0)); - Fld(i1, i2, em::ex1) = s * Fld(i1, i2, em::ex1) + (ONE - s) * ex1_U; + if constexpr (defines_ex2 or defines_dx2 or defines_bx1) { + // i1, i2 + 1/2 + real_t xi_Cd; + if constexpr (o == in::x1) { + xi_Cd = i1_; + } else { + xi_Cd = i2_ + HALF; + } + + const auto s = shape(math::abs( + metric.template convert(xi_Cd) - xg_edge)); + + coord_t x_Ph_0H { ZERO }; + metric.template convert({ i1_, i2_ + HALF }, x_Ph_0H); + + if constexpr (defines_ex2 or defines_dx2) { + if ((tags & BC::E) or (tags & BC::D)) { + if constexpr (defines_ex2 and S == SimEngine::SRPIC) { + Fld(i1, i2, em::ex2) = s * Fld(i1, i2, em::ex2) + + (ONE - s) * + metric.template transform<2, Idx::T, Idx::U>( + { i1_, i2_ + HALF }, + fset.ex2(x_Ph_0H)); + } else if constexpr (defines_dx2 and S == SimEngine::GRPIC) { + Fld(i1, i2, em::dx2) = s * Fld(i1, i2, em::dx2) + + (ONE - s) * fset.dx2(x_Ph_0H); } } - if constexpr (defines_bx2) { - if (tags & BC::B) { - const auto bx2_U = metric.template transform<2, Idx::T, Idx::U>( - { i1_ + HALF, i2_ }, - fset.bx2(x_Ph_H0)); - Fld(i1, i2, em::bx2) = s * Fld(i1, i2, em::bx2) + (ONE - s) * bx2_U; + } + + if constexpr (defines_bx1) { + if (tags & BC::B) { + if constexpr (S == SimEngine::SRPIC) { + Fld(i1, i2, em::bx1) = s * Fld(i1, i2, em::bx1) + + (ONE - s) * + metric.template transform<1, Idx::T, Idx::U>( + { i1_, i2_ + HALF }, + fset.bx1(x_Ph_0H)); + } else if constexpr (S == SimEngine::GRPIC) { + Fld(i1, i2, em::bx1) = s * Fld(i1, i2, em::bx1) + + (ONE - s) * fset.bx1(x_Ph_0H); } } } + } - if constexpr (defines_ex2 or defines_bx1) { - coord_t x_Ph_0H { ZERO }; - metric.template convert({ i1_, i2_ + HALF }, x_Ph_0H); - // i1, i2 + 1/2 + if constexpr (defines_ex3 or defines_dx3) { + if (tags & BC::E) { + // i1, i2 real_t xi_Cd; if constexpr (o == in::x1) { xi_Cd = i1_; } else { - xi_Cd = i2_ + HALF; + xi_Cd = i2_; } - const auto dx = math::abs( - metric.template convert(xi_Cd) - xg_edge); - const auto s = shape(dx); - if constexpr (defines_ex2) { - if (tags & BC::E) { - auto ex2_U { ZERO }; - ex2_U = metric.template transform<2, Idx::T, Idx::U>( - { i1_, i2_ + HALF }, - fset.ex2(x_Ph_0H)); - Fld(i1, i2, em::ex2) = s * Fld(i1, i2, em::ex2) + (ONE - s) * ex2_U; - } - } - if constexpr (defines_bx1) { - if (tags & BC::B) { - auto bx1_U { ZERO }; - bx1_U = metric.template transform<1, Idx::T, Idx::U>( - { i1_, i2_ + HALF }, - fset.bx1(x_Ph_0H)); - Fld(i1, i2, em::bx1) = s * Fld(i1, i2, em::bx1) + (ONE - s) * bx1_U; - } + const auto s = shape(math::abs( + metric.template convert(xi_Cd) - xg_edge)); + + coord_t x_Ph_00 { ZERO }; + metric.template convert({ i1_, i2_ }, x_Ph_00); + + if constexpr (defines_ex3 and S == SimEngine::SRPIC) { + Fld(i1, i2, em::ex3) = s * Fld(i1, i2, em::ex3) + + (ONE - s) * + metric.template transform<3, Idx::T, Idx::U>( + { i1_, i2_ }, + fset.ex3(x_Ph_00)); + } else if constexpr (defines_dx3 and S == SimEngine::GRPIC) { + Fld(i1, i2, em::dx3) = s * Fld(i1, i2, em::dx3) + + (ONE - s) * fset.dx3(x_Ph_00); } } + } - if constexpr (defines_ex3) { - if (tags & BC::E) { - auto ex3_U { ZERO }; - coord_t x_Ph_00 { ZERO }; - metric.template convert({ i1_, i2_ }, x_Ph_00); - ex3_U = metric.template transform<3, Idx::T, Idx::U>( - { i1_, i2_ }, - fset.ex3(x_Ph_00)); - // i1, i2 - real_t xi_Cd; - if constexpr (o == in::x1) { - xi_Cd = i1_; - } else { - xi_Cd = i2_; - } - const auto dx = math::abs( - metric.template convert(xi_Cd) - xg_edge); - const auto s = shape(dx); - Fld(i1, i2, em::ex3) = s * Fld(i1, i2, em::ex3) + (ONE - s) * ex3_U; + if constexpr (defines_bx3) { + if (tags & BC::B) { + // i1 + 1/2, i2 + 1/2 + real_t xi_Cd; + if constexpr (o == in::x1) { + xi_Cd = i1_ + HALF; + } else { + xi_Cd = i2_ + HALF; } - } - if constexpr (defines_bx3) { - if (tags & BC::B) { - auto bx3_U { ZERO }; - coord_t x_Ph_HH { ZERO }; - metric.template convert({ i1_ + HALF, i2_ + HALF }, - x_Ph_HH); - bx3_U = metric.template transform<3, Idx::T, Idx::U>( - { i1_ + HALF, i2_ + HALF }, - fset.bx3(x_Ph_HH)); - // i1 + 1/2, i2 + 1/2 - real_t xi_Cd; - if constexpr (o == in::x1) { - xi_Cd = i1_ + HALF; - } else { - xi_Cd = i2_ + HALF; - } - const auto dx = math::abs( - metric.template convert(xi_Cd) - xg_edge); - const auto s = shape(dx); - // bx3 - Fld(i1, i2, em::bx3) = s * Fld(i1, i2, em::bx3) + (ONE - s) * bx3_U; + const auto s = shape(math::abs( + metric.template convert(xi_Cd) - xg_edge)); + + coord_t x_Ph_HH { ZERO }; + metric.template convert({ i1_ + HALF, i2_ + HALF }, + x_Ph_HH); + + if constexpr (S == SimEngine::SRPIC) { + Fld(i1, i2, em::bx3) = s * Fld(i1, i2, em::bx3) + + (ONE - s) * + metric.template transform<3, Idx::T, Idx::U>( + { i1_ + HALF, i2_ + HALF }, + fset.bx3(x_Ph_HH)); + } else if constexpr (S == SimEngine::GRPIC) { + Fld(i1, i2, em::bx3) = s * Fld(i1, i2, em::bx3) + + (ONE - s) * fset.bx3(x_Ph_HH); } } - } else { - // GRPIC - raise::KernelError(HERE, "GRPIC not implemented"); } } else { raise::KernelError( @@ -328,18 +348,18 @@ namespace kernel::bc { } else { xi_Cd = i3_; } - const auto dx = math::abs( - metric.template convert(xi_Cd) - xg_edge); - const auto s = shape(dx); - auto ex1_U { ZERO }; + const auto s = shape(math::abs( + metric.template convert(xi_Cd) - xg_edge)); + coord_t x_Ph_H00 { ZERO }; metric.template convert({ i1_ + HALF, i2_, i3_ }, x_Ph_H00); - ex1_U = metric.template transform<1, Idx::T, Idx::U>( - { i1_ + HALF, i2_, i3_ }, - fset.ex1(x_Ph_H00)); - Fld(i1, i2, i3, em::ex1) = s * Fld(i1, i2, i3, em::ex1) + - (ONE - s) * ex1_U; + + Fld(i1, i2, i3, em::ex1) = + s * Fld(i1, i2, i3, em::ex1) + + (ONE - s) * metric.template transform<1, Idx::T, Idx::U>( + { i1_ + HALF, i2_, i3_ }, + fset.ex1(x_Ph_H00)); } if constexpr (defines_ex2) { @@ -352,18 +372,18 @@ namespace kernel::bc { } else { xi_Cd = i3_; } - const auto dx = math::abs( - metric.template convert(xi_Cd) - xg_edge); - const auto s = shape(dx); - auto ex2_U { ZERO }; + const auto s = shape(math::abs( + metric.template convert(xi_Cd) - xg_edge)); + coord_t x_Ph_0H0 { ZERO }; metric.template convert({ i1_, i2_ + HALF, i3_ }, x_Ph_0H0); - ex2_U = metric.template transform<2, Idx::T, Idx::U>( - { i1_, i2_ + HALF, i3_ }, - fset.ex2(x_Ph_0H0)); - Fld(i1, i2, i3, em::ex2) = s * Fld(i1, i2, i3, em::ex2) + - (ONE - s) * ex2_U; + + Fld(i1, i2, i3, em::ex2) = + s * Fld(i1, i2, i3, em::ex2) + + (ONE - s) * metric.template transform<2, Idx::T, Idx::U>( + { i1_, i2_ + HALF, i3_ }, + fset.ex2(x_Ph_0H0)); } if constexpr (defines_ex3) { @@ -376,18 +396,17 @@ namespace kernel::bc { } else { xi_Cd = i3_ + HALF; } - const auto dx = math::abs( - metric.template convert(xi_Cd) - xg_edge); - const auto s = shape(dx); - auto ex3_U { ZERO }; + const auto s = shape(math::abs( + metric.template convert(xi_Cd) - xg_edge)); + coord_t x_Ph_00H { ZERO }; metric.template convert({ i1_, i2_, i3_ + HALF }, x_Ph_00H); - ex3_U = metric.template transform<3, Idx::T, Idx::U>( - { i1_, i2_, i3_ + HALF }, - fset.ex3(x_Ph_00H)); - Fld(i1, i2, i3, em::ex3) = s * Fld(i1, i2, i3, em::ex3) + - (ONE - s) * ex3_U; + Fld(i1, i2, i3, em::ex3) = + s * Fld(i1, i2, i3, em::ex3) + + (ONE - s) * metric.template transform<3, Idx::T, Idx::U>( + { i1_, i2_, i3_ + HALF }, + fset.ex3(x_Ph_00H)); } } } @@ -404,22 +423,19 @@ namespace kernel::bc { } else { xi_Cd = i3_ + HALF; } - const auto dx = math::abs( - metric.template convert(xi_Cd) - xg_edge); - const auto s = shape(dx); - auto bx1_U { ZERO }; - if constexpr (defines_bx1) { - coord_t x_Ph_0HH { ZERO }; - metric.template convert( - { i1_, i2_ + HALF, i3_ + HALF }, - x_Ph_0HH); - bx1_U = metric.template transform<1, Idx::T, Idx::U>( - { i1_, i2_ + HALF, i3_ + HALF }, - fset.bx1(x_Ph_0HH)); - } - // bx1 - Fld(i1, i2, i3, em::bx1) = s * Fld(i1, i2, i3, em::bx1) + - (ONE - s) * bx1_U; + const auto s = shape(math::abs( + metric.template convert(xi_Cd) - xg_edge)); + + coord_t x_Ph_0HH { ZERO }; + metric.template convert( + { i1_, i2_ + HALF, i3_ + HALF }, + x_Ph_0HH); + + Fld(i1, i2, i3, em::bx1) = + s * Fld(i1, i2, i3, em::bx1) + + (ONE - s) * metric.template transform<1, Idx::T, Idx::U>( + { i1_, i2_ + HALF, i3_ + HALF }, + fset.bx1(x_Ph_0HH)); } if constexpr (defines_bx2) { @@ -432,19 +448,19 @@ namespace kernel::bc { } else { xi_Cd = i3_ + HALF; } - const auto dx = math::abs( - metric.template convert(xi_Cd) - xg_edge); - const auto s = shape(dx); - auto bx2_U { ZERO }; + const auto s = shape(math::abs( + metric.template convert(xi_Cd) - xg_edge)); + coord_t x_Ph_H0H { ZERO }; metric.template convert( { i1_ + HALF, i2_, i3_ + HALF }, x_Ph_H0H); - bx2_U = metric.template transform<2, Idx::T, Idx::U>( - { i1_ + HALF, i2_, i3_ + HALF }, - fset.bx2(x_Ph_H0H)); - Fld(i1, i2, i3, em::bx2) = s * Fld(i1, i2, i3, em::bx2) + - (ONE - s) * bx2_U; + + Fld(i1, i2, i3, em::bx2) = + s * Fld(i1, i2, i3, em::bx2) + + (ONE - s) * metric.template transform<2, Idx::T, Idx::U>( + { i1_ + HALF, i2_, i3_ + HALF }, + fset.bx2(x_Ph_H0H)); } if constexpr (defines_bx3) { @@ -457,19 +473,20 @@ namespace kernel::bc { } else { xi_Cd = i3_; } - const auto dx = math::abs( - metric.template convert(xi_Cd) - xg_edge); - const auto s = shape(dx); - auto bx3_U { ZERO }; + + const auto s = shape(math::abs( + metric.template convert(xi_Cd) - xg_edge)); + coord_t x_Ph_HH0 { ZERO }; metric.template convert( { i1_ + HALF, i2_ + HALF, i3_ }, x_Ph_HH0); - bx3_U = metric.template transform<3, Idx::T, Idx::U>( - { i1_ + HALF, i2_ + HALF, i3_ }, - fset.bx3(x_Ph_HH0)); - Fld(i1, i2, i3, em::bx3) = s * Fld(i1, i2, i3, em::bx3) + - (ONE - s) * bx3_U; + + Fld(i1, i2, i3, em::bx3) = + s * Fld(i1, i2, i3, em::bx3) + + (ONE - s) * metric.template transform<3, Idx::T, Idx::U>( + { i1_ + HALF, i2_ + HALF, i3_ }, + fset.bx3(x_Ph_HH0)); } } } @@ -558,7 +575,7 @@ namespace kernel::bc { Fld(i1, i_edge, em::bx2) = ZERO; } } else { - raise::KernelError(HERE, "AxisBoundaries_kernel: D != 2"); + raise::KernelError(HERE, "AxisBoundariesGR_kernel: D != 2"); } } }; @@ -869,180 +886,129 @@ namespace kernel::bc { } }; - template - struct OpenBoundaries_kernel { + template + struct HorizonBoundaries_kernel { ndfield_t Fld; const std::size_t i1_min; const bool setE, setB; - OpenBoundaries_kernel(ndfield_t Fld, std::size_t i1_min, BCTags tags) + HorizonBoundaries_kernel(ndfield_t Fld, std::size_t i1_min, BCTags tags) : Fld { Fld } , i1_min { i1_min } - , setE { tags & BC::Ex1 or tags & BC::Ex2 or tags & BC::Ex3 } - , setB { tags & BC::Bx1 or tags & BC::Bx2 or tags & BC::Bx3 } {} + , setE { (tags & BC::Ex1 or tags & BC::Ex2 or tags & BC::Ex3) or + (tags & BC::Dx1 or tags & BC::Dx2 or tags & BC::Dx3) } + , setB { (tags & BC::Bx1 or tags & BC::Bx2 or tags & BC::Bx3) or + (tags & BC::Hx1 or tags & BC::Hx2 or tags & BC::Hx3) } {} Inline void operator()(index_t i2) const { if constexpr (M::Dim == Dim::_2D) { - if (setE) { - Fld(i1_min - 1, i2, em::ex1) = Fld(i1_min, i2, em::ex1); - Fld(i1_min, i2, em::ex2) = Fld(i1_min + 1, i2, em::ex2); - Fld(i1_min - 1, i2, em::ex2) = Fld(i1_min, i2, em::ex2); - Fld(i1_min, i2, em::ex3) = Fld(i1_min + 1, i2, em::ex3); - Fld(i1_min - 1, i2, em::ex3) = Fld(i1_min, i2, em::ex3); - } - if (setB) { - Fld(i1_min, i2, em::bx1) = Fld(i1_min + 1, i2, em::bx1); - Fld(i1_min - 1, i2, em::bx1) = Fld(i1_min, i2, em::bx1); - Fld(i1_min - 1, i2, em::bx2) = Fld(i1_min, i2, em::bx2); - Fld(i1_min - 1, i2, em::bx3) = Fld(i1_min, i2, em::bx3); - } - } else { - raise::KernelError( - HERE, - "AbsorbFields_kernel: 2D implementation called for D != 2"); - } - } - }; - - template - struct OpenBoundariesAux_kernel { - ndfield_t Fld; - const std::size_t i1_min; - const bool setE, setB; - - OpenBoundariesAux_kernel(ndfield_t Fld, std::size_t i1_min, BCTags tags) - : Fld { Fld } - , i1_min { i1_min } - , setE { tags & BC::Ex1 or tags & BC::Ex2 or tags & BC::Ex3 } - , setB { tags & BC::Bx1 or tags & BC::Bx2 or tags & BC::Bx3 } {} - - Inline void operator()(index_t i2) const { - if constexpr (M::Dim == Dim::_2D) { - if (setE) { - Fld(i1_min - 1, i2, em::ex1) = Fld(i1_min, i2, em::ex1); - Fld(i1_min - 1, i2, em::ex2) = Fld(i1_min, i2, em::ex2); - Fld(i1_min - 1, i2, em::ex3) = Fld(i1_min, i2, em::ex3); - } - if (setB) { - Fld(i1_min - 1, i2, em::bx1) = Fld(i1_min, i2, em::bx1); - Fld(i1_min - 1, i2, em::bx2) = Fld(i1_min, i2, em::bx2); - Fld(i1_min - 1, i2, em::bx3) = Fld(i1_min, i2, em::bx3); - } - } else { - raise::KernelError( - HERE, - "AbsorbFields_kernel: 2D implementation called for D != 2"); - } - } - }; - - template - struct AbsorbBoundariesGR_kernel { - static_assert(M::is_metric, "M must be a metric class"); - static_assert(i <= static_cast(M::Dim), - "Invalid component index"); - - ndfield_t Fld; - const I finit; - const M metric; - const real_t xg_edge; - const real_t dx_abs; - const BCTags tags; - - AbsorbBoundariesGR_kernel(ndfield_t Fld, - const I& finit, - const M& metric, - real_t xg_edge, - real_t dx_abs, - BCTags tags) - : Fld { Fld } - , finit { finit } - , metric { metric } - , xg_edge { xg_edge } - , dx_abs { dx_abs } - , tags { tags } {} - - Inline void operator()(index_t i1, index_t i2) const { - if constexpr (M::Dim == Dim::_2D) { - const auto i1_ = COORD(i1); - const auto i2_ = COORD(i2); - for (const auto comp : - { em::ex1, em::ex2, em::ex3, em::bx1, em::bx2, em::bx3 }) { - if ((comp == em::ex1) and not(tags & BC::Ex1)) { - continue; - } else if ((comp == em::ex2) and not(tags & BC::Ex2)) { - continue; - } else if ((comp == em::ex3) and not(tags & BC::Ex3)) { - continue; - } else if ((comp == em::bx1) and not(tags & BC::Bx1)) { - continue; - } else if ((comp == em::bx2) and not(tags & BC::Bx2)) { - continue; - } else if ((comp == em::bx3) and not(tags & BC::Bx3)) { - continue; + if constexpr (not IsAux) { + if (setE) { + Fld(i1_min - 1, i2, em::dx1) = Fld(i1_min, i2, em::dx1); + Fld(i1_min, i2, em::dx2) = Fld(i1_min + 1, i2, em::dx2); + Fld(i1_min - 1, i2, em::dx2) = Fld(i1_min, i2, em::dx2); + Fld(i1_min, i2, em::dx3) = Fld(i1_min + 1, i2, em::dx3); + Fld(i1_min - 1, i2, em::dx3) = Fld(i1_min, i2, em::dx3); } - coord_t x_Cd { ZERO }; - if (comp == em::ex1 or comp == em::bx2 or comp == em::bx3) { - x_Cd[0] = i1_ + HALF; - x_Cd[1] = i2_; - } else if (comp == em::ex2 or comp == em::ex3 or comp == em::bx1) { - x_Cd[0] = i1_; - x_Cd[1] = i2_; + if (setB) { + Fld(i1_min, i2, em::bx1) = Fld(i1_min + 1, i2, em::bx1); + Fld(i1_min - 1, i2, em::bx1) = Fld(i1_min, i2, em::bx1); + Fld(i1_min - 1, i2, em::bx2) = Fld(i1_min, i2, em::bx2); + Fld(i1_min - 1, i2, em::bx3) = Fld(i1_min, i2, em::bx3); } - - const auto dx = math::abs( - metric.template convert(x_Cd[i - 1]) - xg_edge); - const auto tanh = math::tanh(dx / (INV_4 * dx_abs)); - Fld(i1, i2, comp) *= tanh; - - if (comp == em::bx1) { - const real_t x1_0 { metric.template convert<1, Crd::Cd, Crd::Ph>(i1_) }; - const real_t x2_H { metric.template convert<2, Crd::Cd, Crd::Ph>( - i2_ + HALF) }; - Fld(i1, i2, comp) += (ONE - tanh) * - finit.bx1({ x1_0, x2_H }); - } else if (comp == em::bx2) { - const real_t x1_H { metric.template convert<1, Crd::Cd, Crd::Ph>( - i1_ + HALF) }; - const real_t x2_0 { metric.template convert<2, Crd::Cd, Crd::Ph>(i2_) }; - Fld(i1, i2, comp) += (ONE - tanh) * - finit.bx2({ x1_H, x2_0 }); - } else if (comp == em::bx3) { - const real_t x1_H { metric.template convert<1, Crd::Cd, Crd::Ph>( - i1_ + HALF) }; - const real_t x2_H { metric.template convert<2, Crd::Cd, Crd::Ph>( - i2_ + HALF) }; - Fld(i1, i2, comp) += (ONE - tanh) * - finit.bx3({ x1_H, x2_H }); - } else if (comp == em::ex1) { - const real_t x1_H { metric.template convert<1, Crd::Cd, Crd::Ph>( - i1_ + HALF) }; - const real_t x2_0 { metric.template convert<2, Crd::Cd, Crd::Ph>(i2_) }; - Fld(i1, i2, comp) += (ONE - tanh) * - finit.dx1({ x1_H, x2_0 }); - } else if (comp == em::ex2) { - const real_t x1_0 { metric.template convert<1, Crd::Cd, Crd::Ph>(i1_) }; - const real_t x2_H { metric.template convert<2, Crd::Cd, Crd::Ph>( - i2_ + HALF) }; - Fld(i1, i2, comp) += (ONE - tanh) * - finit.dx2({ x1_0, x2_H }); - } else if (comp == em::ex3) { - const real_t x1_0 { metric.template convert<1, Crd::Cd, Crd::Ph>(i1_) }; - const real_t x2_0 { metric.template convert<2, Crd::Cd, Crd::Ph>(i2_) }; - Fld(i1, i2, comp) += (ONE - tanh) * - finit.dx3({ x1_0, x2_0 }); + } else { + if (setE) { + Fld(i1_min - 1, i2, em::ex1) = Fld(i1_min, i2, em::ex1); + Fld(i1_min - 1, i2, em::ex2) = Fld(i1_min, i2, em::ex2); + Fld(i1_min - 1, i2, em::ex3) = Fld(i1_min, i2, em::ex3); + } + if (setB) { + Fld(i1_min - 1, i2, em::hx1) = Fld(i1_min, i2, em::hx1); + Fld(i1_min - 1, i2, em::hx2) = Fld(i1_min, i2, em::hx2); + Fld(i1_min - 1, i2, em::hx3) = Fld(i1_min, i2, em::hx3); } } } else { raise::KernelError( HERE, - "AbsorbFields_kernel: 2D implementation called for D != 2"); + "HorizonBoundaries_kernel: 2D implementation called for D != 2"); } } }; + // template + // struct OpenBoundariesAux_kernel { + // ndfield_t Fld; + // const std::size_t i1_min; + // const bool setE, setB; + // + // OpenBoundariesAux_kernel(ndfield_t Fld, std::size_t i1_min, BCTags tags) + // : Fld { Fld } + // , i1_min { i1_min } + // , setE { tags & BC::Ex1 or tags & BC::Ex2 or tags & BC::Ex3 } + // , setB { tags & BC::Bx1 or tags & BC::Bx2 or tags & BC::Bx3 } {} + // + // Inline void operator()(index_t i2) const { + // if constexpr (M::Dim == Dim::_2D) { + // if (setE) { + // Fld(i1_min - 1, i2, em::ex1) = Fld(i1_min, i2, em::ex1); + // Fld(i1_min - 1, i2, em::ex2) = Fld(i1_min, i2, em::ex2); + // Fld(i1_min - 1, i2, em::ex3) = Fld(i1_min, i2, em::ex3); + // } + // if (setB) { + // Fld(i1_min - 1, i2, em::bx1) = Fld(i1_min, i2, em::bx1); + // Fld(i1_min - 1, i2, em::bx2) = Fld(i1_min, i2, em::bx2); + // Fld(i1_min - 1, i2, em::bx3) = Fld(i1_min, i2, em::bx3); + // } + // } else { + // raise::KernelError( + // HERE, + // "AbsorbFields_kernel: 2D implementation called for D != 2"); + // } + // } + // }; + // + // template + // struct AbsorbBoundariesGR_kernel { + // static_assert(M::is_metric, "M must be a metric class"); + // static_assert(i <= static_cast(M::Dim), + // "Invalid component index"); + // + // ndfield_t Fld; + // const I finit; + // const M metric; + // const real_t xg_edge; + // const real_t dx_abs; + // const BCTags tags; + // + // AbsorbBoundariesGR_kernel(ndfield_t Fld, + // const I& finit, + // const M& metric, + // real_t xg_edge, + // real_t dx_abs, + // BCTags tags) + // : Fld { Fld } + // , finit { finit } + // , metric { metric } + // , xg_edge { xg_edge } + // , dx_abs { dx_abs } + // , tags { tags } {} + // + // Inline void operator()(index_t i1, index_t i2) const { + // if constexpr (M::Dim == Dim::_2D) { + // } + // } + // + // else { + // raise::KernelError( + // HERE, + // "AbsorbFields_kernel: 2D implementation called for D != 2"); + // } + // } + // }; + template - struct AbsorbCurrentGR_kernel { + struct AbsorbCurrentsGR_kernel { static_assert(M::is_metric, "M must be a metric class"); static_assert(i <= static_cast(M::Dim), "Invalid component index"); @@ -1052,10 +1018,10 @@ namespace kernel::bc { const real_t xg_edge; const real_t dx_abs; - AbsorbCurrentGR_kernel(ndfield_t J, - const M& metric, - real_t xg_edge, - real_t dx_abs) + AbsorbCurrentsGR_kernel(ndfield_t J, + const M& metric, + real_t xg_edge, + real_t dx_abs) : J { J } , metric { metric } , xg_edge { xg_edge } @@ -1077,7 +1043,7 @@ namespace kernel::bc { } else { raise::KernelError( HERE, - "AbsorbFields_kernel: 2D implementation called for D != 2"); + "AbsorbCurrentsGR_kernel: 2D implementation called for D != 2"); } } }; From 043a1bea263b4c7b64e20136fa0cd1d124ee5647 Mon Sep 17 00:00:00 2001 From: haykh Date: Fri, 31 Jan 2025 18:21:37 -0500 Subject: [PATCH 208/234] temporary hack for aux BCs (need to be done carefully) --- src/engines/grpic.hpp | 36 ++++++++------ src/framework/domain/communications.cpp | 62 +++++++++++++++---------- src/global/utils/plog.h | 2 +- 3 files changed, 61 insertions(+), 39 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index d6a40bbc4..2de876891 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -391,12 +391,14 @@ namespace ntt { if (fieldsolver_enabled) { timers.start("FieldSolver"); - /** - * cur::J <- (cur0::J + cur::J) / 2 - * - * Now: cur::J at n - */ - TimeAverageJ(dom); + if (deposit_enabled) { + /** + * cur::J <- (cur0::J + cur::J) / 2 + * + * Now: cur::J at n + */ + TimeAverageJ(dom); + } /** * aux::Е <- alpha * em::D + beta x em0::B * @@ -556,13 +558,17 @@ namespace ntt { CustomFieldsIn(direction, domain, tags, g); } } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { - HorizonFieldsIn(direction, domain, tags, g); + if (domain.mesh.flds_bc_in(direction) == FldsBC::HORIZON) { + HorizonFieldsIn(direction, domain, tags, g); + } } } // loop over directions } else if (g == gr_bc::aux) { for (auto& direction : dir::Directions::orth) { if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { - HorizonFieldsIn(direction, domain, tags, g); + if (domain.mesh.flds_bc_in(direction) == FldsBC::HORIZON) { + HorizonFieldsIn(direction, domain, tags, g); + } } } } @@ -679,10 +685,10 @@ namespace ntt { * open boundaries */ raise::ErrorIf(M::CoordType == Coord::Cart, - "Invalid coordinate type for axis BCs", + "Invalid coordinate type for horizon BCs", HERE); raise::ErrorIf(direction.get_dim() != in::x1, - "Invalid axis direction, should be x2", + "Invalid horizon direction, should be x1", HERE); const auto i1_min = domain.mesh.i_min(in::x1); auto range = CreateRangePolicy({ domain.mesh.i_min(in::x2) }, @@ -1075,12 +1081,16 @@ namespace ntt { m_params.template get( "algorithms.timestep.correction") * m_params.template get("scales.omegaB0"); + const auto eps = m_params.template get( + "algorithms.gr.pusher_eps"); + const auto niter = m_params.template get( + "algorithms.gr.pusher_niter"); // clang-format off - if (species.pusher() == PrtlPusher::PHOTON) { auto range_policy = Kokkos::RangePolicy( 0, species.npart()); + Kokkos::parallel_for( "ParticlePusher", range_policy, @@ -1098,7 +1108,7 @@ namespace ntt { domain.mesh.n_active(in::x1), domain.mesh.n_active(in::x2), domain.mesh.n_active(in::x3), - m_params.template get("algorithms.gr.pusher_eps"), m_params.template get("algorithms.gr.pusher_niter"), + eps, niter, domain.mesh.prtl_bc() )); } else if (species.pusher() == PrtlPusher::BORIS) { @@ -1122,7 +1132,7 @@ namespace ntt { domain.mesh.n_active(in::x1), domain.mesh.n_active(in::x2), domain.mesh.n_active(in::x3), - m_params.template get("algorithms.gr.pusher_eps"), m_params.template get("algorithms.gr.pusher_niter"), + eps, niter, domain.mesh.prtl_bc() )); } else if (species.pusher() == PrtlPusher::NONE) { diff --git a/src/framework/domain/communications.cpp b/src/framework/domain/communications.cpp index 728d8fba2..e8ef8e219 100644 --- a/src/framework/domain/communications.cpp +++ b/src/framework/domain/communications.cpp @@ -36,10 +36,10 @@ namespace ntt { using comm_params_t = std::pair>; template - auto GetSendRecvRanks(Metadomain* metadomain, - Domain& domain, - dir::direction_t direction) - -> std::pair { + auto GetSendRecvRanks( + Metadomain* metadomain, + Domain& domain, + dir::direction_t direction) -> std::pair { Domain* send_to_nghbr_ptr = nullptr; Domain* recv_from_nghbr_ptr = nullptr; // set pointers to the correct send/recv domains @@ -119,11 +119,11 @@ namespace ntt { } template - auto GetSendRecvParams(Metadomain* metadomain, - Domain& domain, - dir::direction_t direction, - bool synchronize) - -> std::pair { + auto GetSendRecvParams( + Metadomain* metadomain, + Domain& domain, + dir::direction_t direction, + bool synchronize) -> std::pair { const auto [send_indrank, recv_indrank] = GetSendRecvRanks(metadomain, domain, direction); const auto [send_ind, send_rank] = send_indrank; @@ -206,11 +206,11 @@ namespace ntt { template void Metadomain::CommunicateFields(Domain& domain, CommTags tags) { - const auto comm_fields = (tags & Comm::E) || (tags & Comm::B) || - (tags & Comm::J) || (tags & Comm::D) || - (tags & Comm::D0) || (tags & Comm::B0); - const bool comm_em = (tags & Comm::E) || (tags & Comm::B) || (tags & Comm::D); - const bool comm_em0 = (tags & Comm::B0) || (tags & Comm::D0); + const auto comm_fields = (tags & Comm::E) or (tags & Comm::B) or + (tags & Comm::J) or (tags & Comm::D) or + (tags & Comm::D0) or (tags & Comm::B0); + const bool comm_em = (tags & Comm::E) or (tags & Comm::B) || (tags & Comm::D); + const bool comm_em0 = (tags & Comm::B0) or (tags & Comm::D0); const bool comm_j = (tags & Comm::J); raise::ErrorIf(not comm_fields, "CommunicateFields called with no task", HERE); @@ -302,19 +302,31 @@ namespace ntt { recv_slice, comp_range_fld, false); + // @HACK_GR_1.2.0 -- this has to be done carefully + comm::CommunicateField(domain.index(), + domain.fields.aux, + domain.fields.aux, + send_ind, + recv_ind, + send_rank, + recv_rank, + send_slice, + recv_slice, + comp_range_fld, + false); } if (comm_j) { - comm::CommunicateField(domain.index(), - domain.fields.cur0, - domain.fields.cur0, - send_ind, - recv_ind, - send_rank, - recv_rank, - send_slice, - recv_slice, - comp_range_cur, - false); + comm::CommunicateField(domain.index(), + domain.fields.cur0, + domain.fields.cur0, + send_ind, + recv_ind, + send_rank, + recv_rank, + send_slice, + recv_slice, + comp_range_cur, + false); } } else { if (comm_j) { diff --git a/src/global/utils/plog.h b/src/global/utils/plog.h index 03dc19319..ce6475c36 100644 --- a/src/global/utils/plog.h +++ b/src/global/utils/plog.h @@ -93,4 +93,4 @@ namespace logger { } // namespace logger -#endif // GLOBAL_UTILS_PLOG_H \ No newline at end of file +#endif // GLOBAL_UTILS_PLOG_H From 888c7dea5aa259b6f8799f914e9cc57b0477a9e0 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 18 Feb 2025 11:21:27 -0500 Subject: [PATCH 209/234] changed BC flag to MATCH everywhere --- setups/grpic/deposit/deposit.toml | 2 +- setups/grpic/pusher/massive_gravity_a0995.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setups/grpic/deposit/deposit.toml b/setups/grpic/deposit/deposit.toml index 66c848de9..438caef88 100644 --- a/setups/grpic/deposit/deposit.toml +++ b/setups/grpic/deposit/deposit.toml @@ -14,7 +14,7 @@ ks_a = 0.95 [grid.boundaries] - fields = [["ABSORB"]] + fields = [["MATCH"]] particles = [["ABSORB"]] [grid.boundaries.absorb] diff --git a/setups/grpic/pusher/massive_gravity_a0995.toml b/setups/grpic/pusher/massive_gravity_a0995.toml index 3b64d2066..55b3f96ad 100644 --- a/setups/grpic/pusher/massive_gravity_a0995.toml +++ b/setups/grpic/pusher/massive_gravity_a0995.toml @@ -14,7 +14,7 @@ ks_a = 0.0 [grid.boundaries] - fields = [["ABSORB"]] + fields = [["MATCH"]] particles = [["ABSORB"]] [grid.boundaries.absorb] From 61d735cc477bd20b8e41bb4b38bfb02bdf7187c9 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 18 Feb 2025 11:22:51 -0500 Subject: [PATCH 210/234] MATCH BCs --- setups/grpic/pusher/boris.toml | 2 +- setups/grpic/pusher/boris_a09.toml | 2 +- setups/grpic/pusher/massive_gravity_3d.toml | 2 +- setups/grpic/pusher/massive_gravity_a0.toml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/setups/grpic/pusher/boris.toml b/setups/grpic/pusher/boris.toml index 5d4d92a9d..698ee6cb3 100644 --- a/setups/grpic/pusher/boris.toml +++ b/setups/grpic/pusher/boris.toml @@ -14,7 +14,7 @@ ks_a = 0.0 [grid.boundaries] - fields = [["ABSORB"]] + fields = [["MATCH"]] particles = [["ABSORB"]] [grid.boundaries.absorb] diff --git a/setups/grpic/pusher/boris_a09.toml b/setups/grpic/pusher/boris_a09.toml index 379e9c323..09cad8e76 100644 --- a/setups/grpic/pusher/boris_a09.toml +++ b/setups/grpic/pusher/boris_a09.toml @@ -14,7 +14,7 @@ ks_a = 0.9 [grid.boundaries] - fields = [["ABSORB"]] + fields = [["MATCH"]] particles = [["ABSORB"]] [grid.boundaries.absorb] diff --git a/setups/grpic/pusher/massive_gravity_3d.toml b/setups/grpic/pusher/massive_gravity_3d.toml index 5b87554b2..142db7743 100644 --- a/setups/grpic/pusher/massive_gravity_3d.toml +++ b/setups/grpic/pusher/massive_gravity_3d.toml @@ -14,7 +14,7 @@ ks_a = 0.995 [grid.boundaries] - fields = [["ABSORB"]] + fields = [["MATCH"]] particles = [["ABSORB"]] [grid.boundaries.absorb] diff --git a/setups/grpic/pusher/massive_gravity_a0.toml b/setups/grpic/pusher/massive_gravity_a0.toml index 8179432da..bdb8c802b 100644 --- a/setups/grpic/pusher/massive_gravity_a0.toml +++ b/setups/grpic/pusher/massive_gravity_a0.toml @@ -14,7 +14,7 @@ ks_a = 0.0 [grid.boundaries] - fields = [["ABSORB"]] + fields = [["MATCH"]] particles = [["ABSORB"]] [grid.boundaries.absorb] From 4a4f858c8c3a57a099881e61188a4f086815f2ec Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Thu, 6 Mar 2025 14:38:44 -0500 Subject: [PATCH 211/234] kernels/faraday_gr: bug -- set ghost cells between mesh boundaries --- src/kernels/faraday_gr.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/kernels/faraday_gr.hpp b/src/kernels/faraday_gr.hpp index 5f6e5590b..241fa35dc 100644 --- a/src/kernels/faraday_gr.hpp +++ b/src/kernels/faraday_gr.hpp @@ -73,8 +73,10 @@ namespace kernel::gr { Bout(i1, i2, em::bx1) = Bin(i1, i2, em::bx1) + coeff * inv_sqrt_detH_0pH * (E(i1, i2, em::ex3) - E(i1, i2 + 1, em::ex3)); - if ((i2 == i2min) && is_axis_i2min) { - Bout(i1, i2, em::bx2) = ZERO; + if (i2 == i2min) { + if (is_axis_i2min) { + Bout(i1, i2, em::bx2) = ZERO; + } } else { const real_t inv_sqrt_detH_pH0 { ONE / metric.sqrt_det_h( { i1_ + HALF, i2_ }) }; From 01aa65ab0e29c39cd37adffe48f7c7b15d943dc9 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Mon, 10 Mar 2025 15:11:24 -0400 Subject: [PATCH 212/234] restored faraday_gr at i2=i2min because it was correct --- src/kernels/faraday_gr.hpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/kernels/faraday_gr.hpp b/src/kernels/faraday_gr.hpp index 241fa35dc..5f6e5590b 100644 --- a/src/kernels/faraday_gr.hpp +++ b/src/kernels/faraday_gr.hpp @@ -73,10 +73,8 @@ namespace kernel::gr { Bout(i1, i2, em::bx1) = Bin(i1, i2, em::bx1) + coeff * inv_sqrt_detH_0pH * (E(i1, i2, em::ex3) - E(i1, i2 + 1, em::ex3)); - if (i2 == i2min) { - if (is_axis_i2min) { - Bout(i1, i2, em::bx2) = ZERO; - } + if ((i2 == i2min) && is_axis_i2min) { + Bout(i1, i2, em::bx2) = ZERO; } else { const real_t inv_sqrt_detH_pH0 { ONE / metric.sqrt_det_h( { i1_ + HALF, i2_ }) }; From 2cee034e501817ef0e1d97f1ae4921497f035bdf Mon Sep 17 00:00:00 2001 From: haykh Date: Wed, 12 Mar 2025 14:56:57 -0400 Subject: [PATCH 213/234] inj fixed --- dev/nix/kokkos.nix | 2 +- setups/grpic/wald/pgen.hpp | 187 ++++++++++++++++++++++------------- src/archetypes/energy_dist.h | 27 +---- src/kernels/injectors.hpp | 24 ++--- 4 files changed, 134 insertions(+), 106 deletions(-) diff --git a/dev/nix/kokkos.nix b/dev/nix/kokkos.nix index cfe583c7a..48e5d8c71 100644 --- a/dev/nix/kokkos.nix +++ b/dev/nix/kokkos.nix @@ -41,7 +41,7 @@ pkgs.stdenv.mkDerivation { version = "${version}"; src = pkgs.fetchgit { url = "https://github.com/kokkos/kokkos/"; - rev = "v${version}"; + rev = "${version}"; sha256 = "sha256-cI2p+6J+8BRV5fXTDxxHTfh6P5PeeLUiF73o5zVysHQ="; }; diff --git a/setups/grpic/wald/pgen.hpp b/setups/grpic/wald/pgen.hpp index 7f8146bb9..236d361a6 100644 --- a/setups/grpic/wald/pgen.hpp +++ b/setups/grpic/wald/pgen.hpp @@ -30,38 +30,40 @@ namespace user { } Inline auto bx1(const coord_t& x_Ph) const -> real_t { // at ( i , j + HALF ) - coord_t xi { ZERO }, x0m { ZERO }, x0p { ZERO }; - metric.template convert(x_Ph, xi); + // coord_t xi { ZERO }, x0m { ZERO }, x0p { ZERO }; + // metric.template convert(x_Ph, xi); - x0m[0] = xi[0]; - x0m[1] = xi[1] - HALF * m_eps; - x0p[0] = xi[0]; - x0p[1] = xi[1] + HALF * m_eps; + // x0m[0] = xi[0]; + // x0m[1] = xi[1] - HALF * m_eps; + // x0p[0] = xi[0]; + // x0p[1] = xi[1] + HALF * m_eps; - real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + // real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; - if (cmp::AlmostZero(x_Ph[1])) { - return ONE; - } else { - return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP / m_eps; - } + // if (cmp::AlmostZero(x_Ph[1])) { + // return ONE; + // } else { + // return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP / m_eps; + // } + return ZERO; } Inline auto bx2(const coord_t& x_Ph) const -> real_t { // at ( i + HALF , j ) - coord_t xi { ZERO }, x0m { ZERO }, x0p { ZERO }; - metric.template convert(x_Ph, xi); - - x0m[0] = xi[0] - HALF * m_eps; - x0m[1] = xi[1]; - x0p[0] = xi[0] + HALF * m_eps; - x0p[1] = xi[1]; - - real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; - if (cmp::AlmostZero(x_Ph[1])) { - return ZERO; - } else { - return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP / m_eps; - } + // coord_t xi { ZERO }, x0m { ZERO }, x0p { ZERO }; + // metric.template convert(x_Ph, xi); + + // x0m[0] = xi[0] - HALF * m_eps; + // x0m[1] = xi[1]; + // x0p[0] = xi[0] + HALF * m_eps; + // x0p[1] = xi[1]; + + // real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + // if (cmp::AlmostZero(x_Ph[1])) { + // return ZERO; + // } else { + // return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP / m_eps; + // } + return ZERO; } Inline auto bx3(const coord_t& x_Ph) const -> real_t { @@ -147,12 +149,18 @@ namespace user { const vec_t B_cntrv { EM(i1, i2, em::bx1), EM(i1, i2, em::bx2), EM(i1, i2, em::bx3) }; + const vec_t D_cntrv { EM(i1, i2, em::dx1), + EM(i1, i2, em::dx2), + EM(i1, i2, em::dx3) }; vec_t B_cov { ZERO }; metric.template transform(xi, B_cntrv, B_cov); const auto bsqr = DOT(B_cntrv[0], B_cntrv[1], B_cntrv[2], B_cov[0], B_cov[1], B_cov[2]); + const auto DdotB = + DOT(D_cntrv[0], D_cntrv[1], D_cntrv[2], B_cov[0], B_cov[1], B_cov[2]); const auto dens = density(i1, i2, 0); - return (bsqr > sigma_thr * dens) || (dens < dens_thr); + // return (bsqr > sigma_thr * dens) || (dens < dens_thr); + return (DdotB / bsqr > 0.001) || (bsqr > sigma_thr * dens); } return false; } @@ -194,7 +202,8 @@ namespace user { const std::vector xi_max; const real_t sigma0, sigma_max, multiplicity, nGJ, temperature, m_eps; - InitFields init_flds; + InitFields init_flds; + const Metadomain* metadomain; inline PGen(SimulationParams& p, const Metadomain& m) : arch::ProblemGenerator(p) @@ -207,52 +216,90 @@ namespace user { SQR(p.template get("scales.skindepth0")) } , temperature { p.template get("setup.temperature") } , m_eps { p.template get("setup.m_eps") } - , init_flds { m.mesh().metric, m_eps } {} - - inline void InitPrtls(Domain& local_domain) { - const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, - local_domain.random_pool, + , init_flds { m.mesh().metric, m_eps } + , metadomain { &m } {} + + inline void InitPrtls(Domain& domain) { + // arch::InjectGlobally(*metadomain, + // domain, + // 1, + // { + // { "x1", { 3.2 } }, + // { "x2", { 1.2 } }, + // { "phi", { 0.0 } }, + // { "ux1", { 0.0 } }, + // { "ux2", { -1.0 } }, + // { "ux3", { 0.5 } } + // }); + // arch::InjectGlobally(*metadomain, + // domain, + // 2, + // { + // { "x1", { 2.05 } }, + // { "x2", { 2.3 } }, + // { "phi", { 0.0 } }, + // { "ux1", { 0.5 } }, + // { "ux2", { -0.5 } }, + // { "ux3", { -0.5 } } + // }); + const auto energy_dist = arch::Maxwellian(domain.mesh.metric, + domain.random_pool, temperature); - const auto spatial_dist = PointDistribution(xi_min, - xi_max, - sigma_max / sigma0, - multiplicity * nGJ, - params, - &local_domain); - - const auto injector = - arch::NonUniformInjector( - energy_dist, - spatial_dist, - { 1, 2 }); - arch::InjectNonUniform(params, - local_domain, - injector, - 1.0, - true); + // const auto npart0 = domain.species[0].npart(); + // auto& ux3_0 = domain.species[0].ux3; + // auto& ux3_1 = domain.species[1].ux3; + // Kokkos::parallel_for( + // "zero", + // npart0, + // Lambda(index_t p) { + // ux3_0(p) = ZERO; + // ux3_1(p) = ZERO; + // }); + // const auto spatial_dist = PointDistribution(xi_min, + // xi_max, + // sigma_max / sigma0, + // multiplicity * nGJ, + // params, + // &local_domain); + + // const auto injector = + // arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // { 1, 2 }); + // arch::InjectNonUniform(params, + // local_domain, + // injector, + // 1.0, + // true); + // const auto energy_dist = arch::Cold(local_domain.mesh.metric); + const auto injector = arch::UniformInjector( + energy_dist, + { 1, 2 }); + arch::InjectUniform(params, domain, injector, 1.0, true); } void CustomPostStep(std::size_t, long double time, Domain& local_domain) { - const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, - local_domain.random_pool, - temperature); - const auto spatial_dist = PointDistribution(xi_min, - xi_max, - sigma_max / sigma0, - multiplicity * nGJ, - params, - &local_domain); - - const auto injector = - arch::NonUniformInjector( - energy_dist, - spatial_dist, - { 1, 2 }); - arch::InjectNonUniform(params, - local_domain, - injector, - 1.0, - true); + // const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, + // local_domain.random_pool, + // temperature); + // const auto spatial_dist = PointDistribution(xi_min, + // xi_max, + // sigma_max / sigma0, + // multiplicity * nGJ, + // params, + // &local_domain); + + // const auto injector = + // arch::NonUniformInjector( + // energy_dist, + // spatial_dist, + // { 1, 2 }); + // arch::InjectNonUniform(params, + // local_domain, + // injector, + // 1.0, + // true); } }; diff --git a/src/archetypes/energy_dist.h b/src/archetypes/energy_dist.h index e9bc9051a..1cd9d63a4 100644 --- a/src/archetypes/energy_dist.h +++ b/src/archetypes/energy_dist.h @@ -83,9 +83,7 @@ namespace arch { , g_max { g_max } , pl_ind { pl_ind } {} - Inline void operator()(const coord_t& x_Code, - vec_t& v, - unsigned short sp = 0) const override { + Inline void operator()(vec_t& v, unsigned short sp = 0) const override { auto rand_gen = pool.get_state(); auto rand_X1 = Random(rand_gen); auto rand_gam = ONE; @@ -108,15 +106,6 @@ namespace arch { v[1] = v[2] * math::cos(constant::TWO_PI * rand_X3); v[2] = v[2] * math::sin(constant::TWO_PI * rand_X3); - if constexpr (S == SimEngine::GRPIC) { - // convert from the tetrad basis to covariant - vec_t v_Hat; - v_Hat[0] = v[0]; - v_Hat[1] = v[1]; - v_Hat[2] = v[2]; - metric.template transform(x_Code, v_Hat, v); - } - pool.free_state(rand_gen); } @@ -223,9 +212,9 @@ namespace arch { (v[boost_dir] + boost_beta * gamma); } - Inline void operator()(const coord_t& x_Code, - vec_t& v, - unsigned short s = 0) const override { + Inline void operator()(const coord_t&, + vec_t& v, + unsigned short s = 0) const override { if (cmp::AlmostZero(temperature)) { v[0] = ZERO; v[1] = ZERO; @@ -233,14 +222,6 @@ namespace arch { } else { JS(v, temperature); } - if constexpr (S == SimEngine::GRPIC) { - // convert from the tetrad basis to covariant - vec_t v_Hat; - v_Hat[0] = v[0]; - v_Hat[1] = v[1]; - v_Hat[2] = v[2]; - metric.template transform(x_Code, v_Hat, v); - } if constexpr (M::CoordType == Coord::Cart) { // boost only when using cartesian coordinates if (not cmp::AlmostZero(boost_velocity)) { diff --git a/src/kernels/injectors.hpp b/src/kernels/injectors.hpp index dd5e2d21d..8cab09019 100644 --- a/src/kernels/injectors.hpp +++ b/src/kernels/injectors.hpp @@ -118,26 +118,26 @@ namespace kernel { metric.template convert(x_Cd, x_Ph); if constexpr (M::CoordType == Coord::Cart) { vec_t v_Ph { ZERO }; - energy_dist(x_Ph, v_Ph, spidx1); - metric.template transform_xyz(x_Ph, v_Ph, v1); - energy_dist(x_Ph, v_Ph, spidx2); - metric.template transform_xyz(x_Ph, v_Ph, v2); + energy_dist(x_Cd, v_Ph, spidx1); + metric.template transform_xyz(x_Cd, v_Ph, v1); + energy_dist(x_Cd, v_Ph, spidx2); + metric.template transform_xyz(x_Cd, v_Ph, v2); } else if constexpr (S == SimEngine::SRPIC) { - coord_t x_Ph_ { ZERO }; - x_Ph_[0] = x_Ph[0]; - x_Ph_[1] = x_Ph[1]; - x_Ph_[2] = ZERO; // phi = 0 + coord_t x_Cd_ { ZERO }; + x_Cd_[0] = x_Cd[0]; + x_Cd_[1] = x_Cd[1]; + x_Cd_[2] = ZERO; // phi = 0 vec_t v_Ph { ZERO }; energy_dist(x_Ph, v_Ph, spidx1); - metric.template transform_xyz(x_Ph_, v_Ph, v1); + metric.template transform_xyz(x_Cd_, v_Ph, v1); energy_dist(x_Ph, v_Ph, spidx2); - metric.template transform_xyz(x_Ph_, v_Ph, v2); + metric.template transform_xyz(x_Cd_, v_Ph, v2); } else if constexpr (S == SimEngine::GRPIC) { vec_t v_Ph { ZERO }; energy_dist(x_Ph, v_Ph, spidx1); - metric.template transform(x_Ph, v_Ph, v1); + metric.template transform(x_Cd, v_Ph, v1); energy_dist(x_Ph, v_Ph, spidx2); - metric.template transform(x_Ph, v_Ph, v2); + metric.template transform(x_Cd, v_Ph, v2); } else { raise::KernelError(HERE, "Unknown simulation engine"); } From 26ea9a3806d96f565d170b41af12dbbd72cf8289 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 8 Apr 2025 18:47:54 -0400 Subject: [PATCH 214/234] bug in polar_area qks --- src/metrics/qkerr_schild.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/metrics/qkerr_schild.h b/src/metrics/qkerr_schild.h index e3d849952..5e6a8c977 100644 --- a/src/metrics/qkerr_schild.h +++ b/src/metrics/qkerr_schild.h @@ -480,7 +480,7 @@ namespace metric { math::sqrt( ONE + TWO * (r0 + math::exp(x1 * dchi + chi_min)) / (SQR(r0 + math::exp(x1 * dchi + chi_min)) + SQR(a))) * - (ONE - math::cos(eta2theta(HALF * deta + eta_min))); + (ONE - math::cos(eta2theta(HALF * deta))); } /** From f4c9a6cd59769d03712c42feb7bba2e9ca44e193 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Tue, 8 Apr 2025 18:48:55 -0400 Subject: [PATCH 215/234] axis boundary conditions now match SRPIC --- src/kernels/fields_bcs.hpp | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 6c3bd2760..30009b7b9 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -565,14 +565,36 @@ namespace kernel::bc { AxisBoundariesGR_kernel(ndfield_t Fld, std::size_t i_edge, BCTags tags) : Fld { Fld } - , i_edge { i_edge } + // , i_edge { i_edge } + , i_edge { P ? (i_edge + 1) : i_edge } , setE { tags & BC::Ex1 or tags & BC::Ex2 or tags & BC::Ex3 } , setB { tags & BC::Bx1 or tags & BC::Bx2 or tags & BC::Bx3 } {} Inline void operator()(index_t i1) const { if constexpr (D == Dim::_2D) { - if (setB) { - Fld(i1, i_edge, em::bx2) = ZERO; + // if (setB) { + // Fld(i1, i_edge, em::bx2) = ZERO; + // } + if constexpr (not P) { + if (setE) { + Fld(i1, i_edge - 1, em::ex2) = -Fld(i1, i_edge, em::ex2); + Fld(i1, i_edge, em::ex3) = ZERO; + } + if (setB) { + Fld(i1, i_edge - 1, em::bx1) = Fld(i1, i_edge, em::bx1); + Fld(i1, i_edge, em::bx2) = ZERO; + Fld(i1, i_edge - 1, em::bx3) = Fld(i1, i_edge, em::bx3); + } + } else { + if (setE) { + Fld(i1, i_edge + 1, em::ex2) = -Fld(i1, i_edge, em::ex2); + Fld(i1, i_edge + 1, em::ex3) = ZERO; + } + if (setB) { + Fld(i1, i_edge + 1, em::bx1) = Fld(i1, i_edge, em::bx1); + Fld(i1, i_edge + 1, em::bx2) = ZERO; + Fld(i1, i_edge + 1, em::bx3) = Fld(i1, i_edge, em::bx3); + } } } else { raise::KernelError(HERE, "AxisBoundariesGR_kernel: D != 2"); From b43010738a70012a2cc94d79c8d6db64fad7823c Mon Sep 17 00:00:00 2001 From: haykh Date: Thu, 10 Apr 2025 14:16:36 -0400 Subject: [PATCH 216/234] comm aux --- src/engines/grpic.hpp | 16 ++++++ src/framework/domain/communications.cpp | 73 +++++++++++++++++++------ 2 files changed, 72 insertions(+), 17 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 2de876891..42fcfe74c 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -313,6 +313,10 @@ namespace ntt { FieldBoundaries(dom, BC::E, gr_bc::aux); timers.stop("FieldBoundaries"); + timers.start("Communications"); + m_metadomain.CommunicateFields(dom, Comm::E); + timers.stop("Communications"); + timers.start("FieldSolver"); /** * em0::B <- (em0::B) <- -curl aux::E @@ -347,6 +351,10 @@ namespace ntt { */ FieldBoundaries(dom, BC::H, gr_bc::aux); timers.stop("FieldBoundaries"); + + timers.start("Communications"); + m_metadomain.CommunicateFields(dom, Comm::H); + timers.stop("Communications"); } { @@ -414,6 +422,10 @@ namespace ntt { FieldBoundaries(dom, BC::E, gr_bc::aux); timers.stop("FieldBoundaries"); + timers.start("Communications"); + m_metadomain.CommunicateFields(dom, Comm::E); + timers.stop("Communications"); + timers.start("FieldSolver"); /** * em0::B <- (em::B) <- -curl aux::E @@ -480,6 +492,10 @@ namespace ntt { FieldBoundaries(dom, BC::B, gr_bc::aux); timers.stop("FieldBoundaries"); + timers.start("Communications"); + m_metadomain.CommunicateFields(dom, Comm::H); + timers.stop("Communications"); + timers.start("FieldSolver"); /** * em0::D <- (em::D) <- curl aux::H diff --git a/src/framework/domain/communications.cpp b/src/framework/domain/communications.cpp index e8ef8e219..11225071d 100644 --- a/src/framework/domain/communications.cpp +++ b/src/framework/domain/communications.cpp @@ -206,13 +206,22 @@ namespace ntt { template void Metadomain::CommunicateFields(Domain& domain, CommTags tags) { - const auto comm_fields = (tags & Comm::E) or (tags & Comm::B) or - (tags & Comm::J) or (tags & Comm::D) or - (tags & Comm::D0) or (tags & Comm::B0); - const bool comm_em = (tags & Comm::E) or (tags & Comm::B) || (tags & Comm::D); - const bool comm_em0 = (tags & Comm::B0) or (tags & Comm::D0); + // const auto comm_fields = (tags & Comm::E) or (tags & Comm::B) or + // (tags & Comm::J) or (tags & Comm::D) or + // (tags & Comm::D0) or (tags & Comm::B0) or + // (tags & Comm::H); + const auto comm_em = ((S == SimEngine::SRPIC) and + ((tags & Comm::E) or (tags & Comm::B))) or + ((S == SimEngine::GRPIC) and + ((tags & Comm::D) or (tags & Comm::B))); + const bool comm_em0 = (S == SimEngine::GRPIC) and + ((tags & Comm::B0) or (tags & Comm::D0)); const bool comm_j = (tags & Comm::J); - raise::ErrorIf(not comm_fields, "CommunicateFields called with no task", HERE); + const bool comm_aux = (S == SimEngine::GRPIC) and + ((tags & Comm::E) or (tags & Comm::H)); + raise::ErrorIf(not(comm_em or comm_em0 or comm_j or comm_aux), + "CommunicateFields called with no task", + HERE); std::string comms = ""; if (tags & Comm::E) { @@ -227,6 +236,9 @@ namespace ntt { if (tags & Comm::D) { comms += "D "; } + if (tags & Comm::H) { + comms += "H "; + } if (tags & Comm::D0) { comms += "D0 "; } @@ -243,16 +255,17 @@ namespace ntt { auto comp_range_fld = range_tuple_t {}; auto comp_range_cur = range_tuple_t {}; if constexpr (S == SimEngine::GRPIC) { - if (((tags & Comm::D) && (tags & Comm::B)) || - ((tags & Comm::D0) && (tags & Comm::B0))) { + if (((tags & Comm::D) and (tags & Comm::B)) or + ((tags & Comm::D0) and (tags & Comm::B0)) or + ((tags & Comm::E) and (tags & Comm::H))) { comp_range_fld = range_tuple_t(em::dx1, em::bx3 + 1); - } else if ((tags & Comm::D) || (tags & Comm::D0)) { + } else if ((tags & Comm::D) or (tags & Comm::D0) or (tags & Comm::E)) { comp_range_fld = range_tuple_t(em::dx1, em::dx3 + 1); - } else if ((tags & Comm::B) || (tags & Comm::B0)) { + } else if ((tags & Comm::B) or (tags & Comm::B0) or (tags & Comm::H)) { comp_range_fld = range_tuple_t(em::bx1, em::bx3 + 1); } } else if constexpr (S == SimEngine::SRPIC) { - if ((tags & Comm::E) && (tags & Comm::B)) { + if ((tags & Comm::E) and (tags & Comm::B)) { comp_range_fld = range_tuple_t(em::ex1, em::bx3 + 1); } else if (tags & Comm::E) { comp_range_fld = range_tuple_t(em::ex1, em::ex3 + 1); @@ -290,10 +303,10 @@ namespace ntt { false); } if constexpr (S == SimEngine::GRPIC) { - if (comm_em0) { + if (comm_aux) { comm::CommunicateField(domain.index(), - domain.fields.em0, - domain.fields.em0, + domain.fields.aux, + domain.fields.aux, send_ind, recv_ind, send_rank, @@ -302,10 +315,11 @@ namespace ntt { recv_slice, comp_range_fld, false); - // @HACK_GR_1.2.0 -- this has to be done carefully + } + if (comm_em0) { comm::CommunicateField(domain.index(), - domain.fields.aux, - domain.fields.aux, + domain.fields.em0, + domain.fields.em0, send_ind, recv_ind, send_rank, @@ -314,6 +328,18 @@ namespace ntt { recv_slice, comp_range_fld, false); + // @HACK_GR_1.2.0 -- this has to be done carefully + // comm::CommunicateField(domain.index(), + // domain.fields.aux, + // domain.fields.aux, + // send_ind, + // recv_ind, + // send_rank, + // recv_rank, + // send_slice, + // recv_slice, + // comp_range_fld, + // false); } if (comm_j) { comm::CommunicateField(domain.index(), @@ -329,6 +355,19 @@ namespace ntt { false); } } else { + if (comm_em) { + comm::CommunicateField(domain.index(), + domain.fields.em, + domain.fields.em, + send_ind, + recv_ind, + send_rank, + recv_rank, + send_slice, + recv_slice, + comp_range_fld, + false); + } if (comm_j) { comm::CommunicateField(domain.index(), domain.fields.cur, From fbd6c26150bbb659764d08fcfc587d187f3cc542 Mon Sep 17 00:00:00 2001 From: haykh Date: Thu, 10 Apr 2025 14:41:00 -0400 Subject: [PATCH 217/234] proper ordering of comm vs flds --- src/engines/grpic.hpp | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 42fcfe74c..a5046242f 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -148,6 +148,7 @@ namespace ntt { /** * aux::E, aux::H <- boundary conditions */ + m_metadomain.CommunicateFields(dom, Comm::H | Comm::E); FieldBoundaries(dom, BC::H | BC::E, gr_bc::aux); /** @@ -188,6 +189,7 @@ namespace ntt { /** * aux::E, aux::H <- boundary conditions */ + m_metadomain.CommunicateFields(dom, Comm::H | Comm::E); FieldBoundaries(dom, BC::H | BC::E, gr_bc::aux); // !ADD: GR -- particles? @@ -225,6 +227,7 @@ namespace ntt { /** * aux::H <- boundary conditions */ + m_metadomain.CommunicateFields(dom, Comm::H); FieldBoundaries(dom, BC::H, gr_bc::aux); /** @@ -306,6 +309,9 @@ namespace ntt { ComputeAuxE(dom, gr_getE::D0_B); timers.stop("FieldSolver"); + timers.start("Communications"); + m_metadomain.CommunicateFields(dom, Comm::E); + timers.stop("Communications"); timers.start("FieldBoundaries"); /** * aux::E <- boundary conditions @@ -313,10 +319,6 @@ namespace ntt { FieldBoundaries(dom, BC::E, gr_bc::aux); timers.stop("FieldBoundaries"); - timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::E); - timers.stop("Communications"); - timers.start("FieldSolver"); /** * em0::B <- (em0::B) <- -curl aux::E @@ -326,15 +328,15 @@ namespace ntt { Faraday(dom, gr_faraday::aux, ONE); timers.stop("FieldSolver"); + timers.start("Communications"); + m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); + timers.stop("Communications"); /** * em0::B, em::B <- boundary conditions */ timers.start("FieldBoundaries"); FieldBoundaries(dom, BC::B, gr_bc::main); timers.stop("FieldBoundaries"); - timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); - timers.stop("Communications"); timers.start("FieldSolver"); /** @@ -345,16 +347,15 @@ namespace ntt { ComputeAuxH(dom, gr_getH::D_B0); timers.stop("FieldSolver"); + timers.start("Communications"); + m_metadomain.CommunicateFields(dom, Comm::H); + timers.stop("Communications"); timers.start("FieldBoundaries"); /** * aux::H <- boundary conditions */ FieldBoundaries(dom, BC::H, gr_bc::aux); timers.stop("FieldBoundaries"); - - timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::H); - timers.stop("Communications"); } { @@ -415,6 +416,9 @@ namespace ntt { ComputeAuxE(dom, gr_getE::D_B0); timers.stop("FieldSolver"); + timers.start("Communications"); + m_metadomain.CommunicateFields(dom, Comm::E); + timers.stop("Communications"); timers.start("FieldBoundaries"); /** * aux::Е <- boundary conditions @@ -422,10 +426,6 @@ namespace ntt { FieldBoundaries(dom, BC::E, gr_bc::aux); timers.stop("FieldBoundaries"); - timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::E); - timers.stop("Communications"); - timers.start("FieldSolver"); /** * em0::B <- (em::B) <- -curl aux::E @@ -485,6 +485,9 @@ namespace ntt { ComputeAuxH(dom, gr_getH::D0_B0); timers.stop("FieldSolver"); + timers.start("Communications"); + m_metadomain.CommunicateFields(dom, Comm::H); + timers.stop("Communications"); timers.start("FieldBoundaries"); /** * aux::H <- boundary conditions @@ -492,10 +495,6 @@ namespace ntt { FieldBoundaries(dom, BC::B, gr_bc::aux); timers.stop("FieldBoundaries"); - timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::H); - timers.stop("Communications"); - timers.start("FieldSolver"); /** * em0::D <- (em::D) <- curl aux::H From 4fc9809b413b70ab665a86c99ada6fd56e318ed1 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 11 Apr 2025 12:35:06 -0400 Subject: [PATCH 218/234] updated wald setup --- setups/grpic/wald/pgen.hpp | 193 ++++++++++++++++-------------------- setups/grpic/wald/wald.toml | 1 - 2 files changed, 83 insertions(+), 111 deletions(-) diff --git a/setups/grpic/wald/pgen.hpp b/setups/grpic/wald/pgen.hpp index 236d361a6..7d101b376 100644 --- a/setups/grpic/wald/pgen.hpp +++ b/setups/grpic/wald/pgen.hpp @@ -29,41 +29,53 @@ namespace user { metric.beta1(x_Cd)); } + Inline auto A_1(const coord_t& x_Cd) const -> real_t { + return HALF * (metric.template h_<1, 3>(x_Cd) + + TWO * metric.spin() * metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) + ); + } + + Inline auto A_0(const coord_t& x_Cd) const -> real_t { + real_t g_00 { -metric.alpha(x_Cd) * metric.alpha(x_Cd) + + metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) * metric.beta1(x_Cd) + }; + return HALF * (metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + + TWO * metric.spin() * g_00); + } + Inline auto bx1(const coord_t& x_Ph) const -> real_t { // at ( i , j + HALF ) - // coord_t xi { ZERO }, x0m { ZERO }, x0p { ZERO }; - // metric.template convert(x_Ph, xi); + coord_t xi { ZERO }, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); - // x0m[0] = xi[0]; - // x0m[1] = xi[1] - HALF * m_eps; - // x0p[0] = xi[0]; - // x0p[1] = xi[1] + HALF * m_eps; + x0m[0] = xi[0]; + x0m[1] = xi[1] - HALF * m_eps; + x0p[0] = xi[0]; + x0p[1] = xi[1] + HALF * m_eps; - // real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; - // if (cmp::AlmostZero(x_Ph[1])) { - // return ONE; - // } else { - // return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP / m_eps; - // } - return ZERO; + if (cmp::AlmostZero(x_Ph[1])) { + return ONE; + } else { + return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP / m_eps; + } } Inline auto bx2(const coord_t& x_Ph) const -> real_t { // at ( i + HALF , j ) - // coord_t xi { ZERO }, x0m { ZERO }, x0p { ZERO }; - // metric.template convert(x_Ph, xi); - - // x0m[0] = xi[0] - HALF * m_eps; - // x0m[1] = xi[1]; - // x0p[0] = xi[0] + HALF * m_eps; - // x0p[1] = xi[1]; - - // real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; - // if (cmp::AlmostZero(x_Ph[1])) { - // return ZERO; - // } else { - // return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP / m_eps; - // } - return ZERO; + coord_t xi { ZERO }, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0] - HALF * m_eps; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF * m_eps; + x0p[1] = xi[1]; + + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + if (cmp::AlmostZero(x_Ph[1])) { + return ZERO; + } else { + return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP / m_eps; + } } Inline auto bx3(const coord_t& x_Ph) const -> real_t { @@ -100,6 +112,7 @@ namespace user { , EM { domain_ptr->fields.em } , density { domain_ptr->fields.buff } , sigma_thr { sigma_thr } + , inv_n0 { ONE / params.template get("scales.n0") } , dens_thr { dens_thr } { std::copy(xi_min.begin(), xi_min.end(), x_min); std::copy(xi_max.begin(), xi_max.end(), x_max); @@ -118,7 +131,6 @@ namespace user { const auto use_weights = params.template get( "particles.use_weights"); const auto ni2 = mesh.n_active(in::x2); - const auto inv_n0 = ONE / params.template get("scales.n0"); for (const auto& sp : specs) { auto& prtl_spec = domain_ptr->species[sp - 1]; @@ -156,11 +168,8 @@ namespace user { metric.template transform(xi, B_cntrv, B_cov); const auto bsqr = DOT(B_cntrv[0], B_cntrv[1], B_cntrv[2], B_cov[0], B_cov[1], B_cov[2]); - const auto DdotB = - DOT(D_cntrv[0], D_cntrv[1], D_cntrv[2], B_cov[0], B_cov[1], B_cov[2]); const auto dens = density(i1, i2, 0); - // return (bsqr > sigma_thr * dens) || (dens < dens_thr); - return (DdotB / bsqr > 0.001) || (bsqr > sigma_thr * dens); + return (bsqr > sigma_thr * dens) || (dens < dens_thr); } return false; } @@ -178,6 +187,7 @@ namespace user { tuple_t x_max; const real_t sigma_thr; const real_t dens_thr; + const real_t inv_n0; Domain* domain_ptr; ndfield_t density; ndfield_t EM; @@ -219,87 +229,50 @@ namespace user { , init_flds { m.mesh().metric, m_eps } , metadomain { &m } {} - inline void InitPrtls(Domain& domain) { - // arch::InjectGlobally(*metadomain, - // domain, - // 1, - // { - // { "x1", { 3.2 } }, - // { "x2", { 1.2 } }, - // { "phi", { 0.0 } }, - // { "ux1", { 0.0 } }, - // { "ux2", { -1.0 } }, - // { "ux3", { 0.5 } } - // }); - // arch::InjectGlobally(*metadomain, - // domain, - // 2, - // { - // { "x1", { 2.05 } }, - // { "x2", { 2.3 } }, - // { "phi", { 0.0 } }, - // { "ux1", { 0.5 } }, - // { "ux2", { -0.5 } }, - // { "ux3", { -0.5 } } - // }); - const auto energy_dist = arch::Maxwellian(domain.mesh.metric, - domain.random_pool, + inline void InitPrtls(Domain& local_domain) { + const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, + local_domain.random_pool, temperature); - // const auto npart0 = domain.species[0].npart(); - // auto& ux3_0 = domain.species[0].ux3; - // auto& ux3_1 = domain.species[1].ux3; - // Kokkos::parallel_for( - // "zero", - // npart0, - // Lambda(index_t p) { - // ux3_0(p) = ZERO; - // ux3_1(p) = ZERO; - // }); - // const auto spatial_dist = PointDistribution(xi_min, - // xi_max, - // sigma_max / sigma0, - // multiplicity * nGJ, - // params, - // &local_domain); - - // const auto injector = - // arch::NonUniformInjector( - // energy_dist, - // spatial_dist, - // { 1, 2 }); - // arch::InjectNonUniform(params, - // local_domain, - // injector, - // 1.0, - // true); - // const auto energy_dist = arch::Cold(local_domain.mesh.metric); - const auto injector = arch::UniformInjector( - energy_dist, - { 1, 2 }); - arch::InjectUniform(params, domain, injector, 1.0, true); + const auto spatial_dist = PointDistribution(xi_min, + xi_max, + sigma_max / sigma0, + multiplicity * nGJ, + params, + &local_domain); + + const auto injector = + arch::NonUniformInjector( + energy_dist, + spatial_dist, + { 1, 2 }); + arch::InjectNonUniform(params, + local_domain, + injector, + 1.0, + true); } void CustomPostStep(std::size_t, long double time, Domain& local_domain) { - // const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, - // local_domain.random_pool, - // temperature); - // const auto spatial_dist = PointDistribution(xi_min, - // xi_max, - // sigma_max / sigma0, - // multiplicity * nGJ, - // params, - // &local_domain); - - // const auto injector = - // arch::NonUniformInjector( - // energy_dist, - // spatial_dist, - // { 1, 2 }); - // arch::InjectNonUniform(params, - // local_domain, - // injector, - // 1.0, - // true); + const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, + local_domain.random_pool, + temperature); + const auto spatial_dist = PointDistribution(xi_min, + xi_max, + sigma_max / sigma0, + multiplicity * nGJ, + params, + &local_domain); + + const auto injector = + arch::NonUniformInjector( + energy_dist, + spatial_dist, + { 1, 2 }); + arch::InjectNonUniform(params, + local_domain, + injector, + 1.0, + true); } }; diff --git a/setups/grpic/wald/wald.toml b/setups/grpic/wald/wald.toml index c515ae024..61d1a4697 100644 --- a/setups/grpic/wald/wald.toml +++ b/setups/grpic/wald/wald.toml @@ -68,7 +68,6 @@ [output] format = "hdf5" - separate_files = false [output.fields] interval_time = 1.0 From dd4f7562ce5f243b451d2f1b445e82cd819b9b36 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 25 Apr 2025 12:32:59 -0400 Subject: [PATCH 219/234] minor change in input --- setups/grpic/pusher/massive_gravity_a0.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setups/grpic/pusher/massive_gravity_a0.toml b/setups/grpic/pusher/massive_gravity_a0.toml index bdb8c802b..02c6062dc 100644 --- a/setups/grpic/pusher/massive_gravity_a0.toml +++ b/setups/grpic/pusher/massive_gravity_a0.toml @@ -72,6 +72,7 @@ [output] format = "hdf5" + separate_files = false [output.fields] enable = true @@ -82,7 +83,7 @@ enable = true stride = 1 interval_time = 1.0 - species = [] + species = [1, 2] [output.spectra] enable = false From 368b5a8bc1fd2b1f6bb2ba77e4c79425c7d8abac Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 25 Apr 2025 12:34:15 -0400 Subject: [PATCH 220/234] particles are now deleted when they cross ghost zone --- src/kernels/particle_pusher_gr.hpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/kernels/particle_pusher_gr.hpp b/src/kernels/particle_pusher_gr.hpp index c80fe188e..2318d5436 100644 --- a/src/kernels/particle_pusher_gr.hpp +++ b/src/kernels/particle_pusher_gr.hpp @@ -81,7 +81,6 @@ namespace kernel::gr { const int ni1, ni2, ni3; const real_t epsilon; const unsigned short niter; - const int i1_absorb; bool is_axis_i2min { false }, is_axis_i2max { false }; bool is_absorb_i1min { false }, is_absorb_i1max { false }; @@ -141,8 +140,7 @@ namespace kernel::gr { , ni2 { ni2 } , ni3 { ni3 } , epsilon { epsilon } - , niter { niter } - , i1_absorb {N_GHOSTS} { + , niter { niter } { raise::ErrorIf(boundaries.size() < 2, "boundaries defined incorrectly", HERE); is_absorb_i1min = (boundaries[0].first == PrtlBC::ABSORB) || @@ -739,7 +737,7 @@ namespace kernel::gr { template Inline void Pusher_kernel::boundaryConditions(index_t& p) const { if constexpr (D == Dim::_1D || D == Dim::_2D || D == Dim::_3D) { - if (i1(p) < i1_absorb && is_absorb_i1min) { + if (i1(p) < 0 && is_absorb_i1min) { tag(p) = ParticleTag::dead; } else if (i1(p) >= ni1 && is_absorb_i1max) { tag(p) = ParticleTag::dead; From 7e73df901fe9334215529d4db6eaa15c5da511f4 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Fri, 25 Apr 2025 14:04:57 -0400 Subject: [PATCH 221/234] updated horizon boundaries for fields --- src/engines/grpic.hpp | 9 +------- src/kernels/fields_bcs.hpp | 42 ++++++++++++++++++++++++++++++-------- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index a5046242f..eb35be4b2 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -721,14 +721,7 @@ namespace ntt { kernel::bc::HorizonBoundaries_kernel(domain.fields.em0, i1_min, tags)); - } else if (g == gr_bc::aux) { - Kokkos::parallel_for( - "OpenBCFields", - range, - kernel::bc::HorizonBoundaries_kernel(domain.fields.aux, - i1_min, - tags)); - } + } } void AxisFieldsIn(dir::direction_t direction, diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 30009b7b9..b3fb1761d 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -926,17 +926,41 @@ namespace kernel::bc { if constexpr (M::Dim == Dim::_2D) { if constexpr (not IsAux) { if (setE) { - Fld(i1_min - 1, i2, em::dx1) = Fld(i1_min, i2, em::dx1); - Fld(i1_min, i2, em::dx2) = Fld(i1_min + 1, i2, em::dx2); - Fld(i1_min - 1, i2, em::dx2) = Fld(i1_min, i2, em::dx2); - Fld(i1_min, i2, em::dx3) = Fld(i1_min + 1, i2, em::dx3); - Fld(i1_min - 1, i2, em::dx3) = Fld(i1_min, i2, em::dx3); + // Fld(i1_min - 1, i2, em::dx1) = Fld(i1_min, i2, em::dx1); + // Fld(i1_min, i2, em::dx2) = Fld(i1_min + 1, i2, em::dx2); + // Fld(i1_min - 1, i2, em::dx2) = Fld(i1_min, i2, em::dx2); + // Fld(i1_min, i2, em::dx3) = Fld(i1_min + 1, i2, em::dx3); + // Fld(i1_min - 1, i2, em::dx3) = Fld(i1_min, i2, em::dx3); + + Fld(i1_min , i2, em::dx1) = Fld(i1_min + 1, i2, em::dx1); + Fld(i1_min - 1, i2, em::dx1) = Fld(i1_min + 1, i2, em::dx1); + Fld(i1_min - 2, i2, em::dx1) = Fld(i1_min + 1, i2, em::dx1); + + Fld(i1_min , i2, em::dx2) = Fld(i1_min + 1, i2, em::dx2); + Fld(i1_min - 1, i2, em::dx2) = Fld(i1_min + 1, i2, em::dx2); + Fld(i1_min - 2, i2, em::dx2) = Fld(i1_min + 1, i2, em::dx2); + + Fld(i1_min , i2, em::dx3) = Fld(i1_min + 1, i2, em::dx3); + Fld(i1_min - 1, i2, em::dx3) = Fld(i1_min + 1, i2, em::dx3); + Fld(i1_min - 2, i2, em::dx3) = Fld(i1_min + 1, i2, em::dx3); } if (setB) { - Fld(i1_min, i2, em::bx1) = Fld(i1_min + 1, i2, em::bx1); - Fld(i1_min - 1, i2, em::bx1) = Fld(i1_min, i2, em::bx1); - Fld(i1_min - 1, i2, em::bx2) = Fld(i1_min, i2, em::bx2); - Fld(i1_min - 1, i2, em::bx3) = Fld(i1_min, i2, em::bx3); + // Fld(i1_min, i2, em::bx1) = Fld(i1_min + 1, i2, em::bx1); + // Fld(i1_min - 1, i2, em::bx1) = Fld(i1_min, i2, em::bx1); + // Fld(i1_min - 1, i2, em::bx2) = Fld(i1_min, i2, em::bx2); + // Fld(i1_min - 1, i2, em::bx3) = Fld(i1_min, i2, em::bx3); + + Fld(i1_min , i2, em::bx1) = Fld(i1_min + 1, i2, em::bx1); + Fld(i1_min - 1, i2, em::bx1) = Fld(i1_min + 1, i2, em::bx1); + Fld(i1_min - 2, i2, em::bx1) = Fld(i1_min + 1, i2, em::bx1); + + Fld(i1_min , i2, em::bx2) = Fld(i1_min + 1, i2, em::bx2); + Fld(i1_min - 1, i2, em::bx2) = Fld(i1_min + 1, i2, em::bx2); + Fld(i1_min - 2, i2, em::bx2) = Fld(i1_min + 1, i2, em::bx2); + + Fld(i1_min , i2, em::bx3) = Fld(i1_min + 1, i2, em::bx3); + Fld(i1_min - 1, i2, em::bx3) = Fld(i1_min + 1, i2, em::bx3); + Fld(i1_min - 2, i2, em::bx3) = Fld(i1_min + 1, i2, em::bx3); } } else { if (setE) { From 1d2bb1b8902c306b81a962faf17cc87677fb40f8 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Mon, 19 May 2025 20:26:29 -0400 Subject: [PATCH 222/234] adjusted horizon BCs to work with nfilters>0; depleted aux horizon BCs --- src/kernels/fields_bcs.hpp | 66 +++++++++----------------------------- 1 file changed, 16 insertions(+), 50 deletions(-) diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index b3fb1761d..3f63b650a 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -908,70 +908,36 @@ namespace kernel::bc { } }; - template + template struct HorizonBoundaries_kernel { ndfield_t Fld; const std::size_t i1_min; const bool setE, setB; + const std::size_t nfilter; - HorizonBoundaries_kernel(ndfield_t Fld, std::size_t i1_min, BCTags tags) + HorizonBoundaries_kernel(ndfield_t Fld, std::size_t i1_min, BCTags tags, std::size_t nfilter) : Fld { Fld } , i1_min { i1_min } , setE { (tags & BC::Ex1 or tags & BC::Ex2 or tags & BC::Ex3) or (tags & BC::Dx1 or tags & BC::Dx2 or tags & BC::Dx3) } , setB { (tags & BC::Bx1 or tags & BC::Bx2 or tags & BC::Bx3) or - (tags & BC::Hx1 or tags & BC::Hx2 or tags & BC::Hx3) } {} + (tags & BC::Hx1 or tags & BC::Hx2 or tags & BC::Hx3) } + , nfilter { nfilter } {} Inline void operator()(index_t i2) const { if constexpr (M::Dim == Dim::_2D) { - if constexpr (not IsAux) { - if (setE) { - // Fld(i1_min - 1, i2, em::dx1) = Fld(i1_min, i2, em::dx1); - // Fld(i1_min, i2, em::dx2) = Fld(i1_min + 1, i2, em::dx2); - // Fld(i1_min - 1, i2, em::dx2) = Fld(i1_min, i2, em::dx2); - // Fld(i1_min, i2, em::dx3) = Fld(i1_min + 1, i2, em::dx3); - // Fld(i1_min - 1, i2, em::dx3) = Fld(i1_min, i2, em::dx3); - - Fld(i1_min , i2, em::dx1) = Fld(i1_min + 1, i2, em::dx1); - Fld(i1_min - 1, i2, em::dx1) = Fld(i1_min + 1, i2, em::dx1); - Fld(i1_min - 2, i2, em::dx1) = Fld(i1_min + 1, i2, em::dx1); - - Fld(i1_min , i2, em::dx2) = Fld(i1_min + 1, i2, em::dx2); - Fld(i1_min - 1, i2, em::dx2) = Fld(i1_min + 1, i2, em::dx2); - Fld(i1_min - 2, i2, em::dx2) = Fld(i1_min + 1, i2, em::dx2); - - Fld(i1_min , i2, em::dx3) = Fld(i1_min + 1, i2, em::dx3); - Fld(i1_min - 1, i2, em::dx3) = Fld(i1_min + 1, i2, em::dx3); - Fld(i1_min - 2, i2, em::dx3) = Fld(i1_min + 1, i2, em::dx3); - } - if (setB) { - // Fld(i1_min, i2, em::bx1) = Fld(i1_min + 1, i2, em::bx1); - // Fld(i1_min - 1, i2, em::bx1) = Fld(i1_min, i2, em::bx1); - // Fld(i1_min - 1, i2, em::bx2) = Fld(i1_min, i2, em::bx2); - // Fld(i1_min - 1, i2, em::bx3) = Fld(i1_min, i2, em::bx3); - - Fld(i1_min , i2, em::bx1) = Fld(i1_min + 1, i2, em::bx1); - Fld(i1_min - 1, i2, em::bx1) = Fld(i1_min + 1, i2, em::bx1); - Fld(i1_min - 2, i2, em::bx1) = Fld(i1_min + 1, i2, em::bx1); - - Fld(i1_min , i2, em::bx2) = Fld(i1_min + 1, i2, em::bx2); - Fld(i1_min - 1, i2, em::bx2) = Fld(i1_min + 1, i2, em::bx2); - Fld(i1_min - 2, i2, em::bx2) = Fld(i1_min + 1, i2, em::bx2); - - Fld(i1_min , i2, em::bx3) = Fld(i1_min + 1, i2, em::bx3); - Fld(i1_min - 1, i2, em::bx3) = Fld(i1_min + 1, i2, em::bx3); - Fld(i1_min - 2, i2, em::bx3) = Fld(i1_min + 1, i2, em::bx3); - } - } else { - if (setE) { - Fld(i1_min - 1, i2, em::ex1) = Fld(i1_min, i2, em::ex1); - Fld(i1_min - 1, i2, em::ex2) = Fld(i1_min, i2, em::ex2); - Fld(i1_min - 1, i2, em::ex3) = Fld(i1_min, i2, em::ex3); + if (setE) { + for (unsigned short i = 0; i <= 2 + nfilter; ++i) { + Fld(i1_min - N_GHOSTS + i, i2, em::dx1) = Fld(i1_min + 1 + nfilter, i2, em::dx1); + Fld(i1_min - N_GHOSTS + i, i2, em::dx2) = Fld(i1_min + 1 + nfilter, i2, em::dx2); + Fld(i1_min - N_GHOSTS + i, i2, em::dx3) = Fld(i1_min + 1 + nfilter, i2, em::dx3); } - if (setB) { - Fld(i1_min - 1, i2, em::hx1) = Fld(i1_min, i2, em::hx1); - Fld(i1_min - 1, i2, em::hx2) = Fld(i1_min, i2, em::hx2); - Fld(i1_min - 1, i2, em::hx3) = Fld(i1_min, i2, em::hx3); + } + if (setB) { + for (unsigned short i = 0; i <= 2 + nfilter; ++i) { + Fld(i1_min - N_GHOSTS + i, i2, em::bx1) = Fld(i1_min + 1 + nfilter, i2, em::bx1); + Fld(i1_min - N_GHOSTS + i, i2, em::bx2) = Fld(i1_min + 1 + nfilter, i2, em::bx2); + Fld(i1_min - N_GHOSTS + i, i2, em::bx3) = Fld(i1_min + 1 + nfilter, i2, em::bx3); } } } else { From 7ac1b68bb3abb7b6371db1f21a431664f631f6f7 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Mon, 19 May 2025 20:53:31 -0400 Subject: [PATCH 223/234] adjusted engine accordingly (previous below) --- src/engines/grpic.hpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index eb35be4b2..04197d12b 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -708,19 +708,23 @@ namespace ntt { const auto i1_min = domain.mesh.i_min(in::x1); auto range = CreateRangePolicy({ domain.mesh.i_min(in::x2) }, { domain.mesh.i_max(in::x2) + 1 }); + const auto nfilter = m_params.template get( + "algorithms.current_filters"); if (g == gr_bc::main) { Kokkos::parallel_for( "OpenBCFields", range, - kernel::bc::HorizonBoundaries_kernel(domain.fields.em, + kernel::bc::HorizonBoundaries_kernel(domain.fields.em, i1_min, - tags)); + tags, + nfilter)); Kokkos::parallel_for( "OpenBCFields", range, - kernel::bc::HorizonBoundaries_kernel(domain.fields.em0, + kernel::bc::HorizonBoundaries_kernel(domain.fields.em0, i1_min, - tags)); + tags, + nfilter)); } } From 81cdc01ebfef90780807a5bb41cb82474a7b91f2 Mon Sep 17 00:00:00 2001 From: haykh Date: Thu, 12 Jun 2025 17:33:08 -0700 Subject: [PATCH 224/234] merge 1.2.0rc + pgens separation --- pgens/accretion/accretion.toml | 85 ++++++++++ pgens/accretion/pgen.hpp | 281 +++++++++++++++++++++++++++++++++ pgens/wald/pgen.hpp | 278 ++++++++++++-------------------- pgens/wald/wald.toml | 54 ++----- src/engines/grpic.hpp | 18 +-- 5 files changed, 491 insertions(+), 225 deletions(-) create mode 100644 pgens/accretion/accretion.toml create mode 100644 pgens/accretion/pgen.hpp diff --git a/pgens/accretion/accretion.toml b/pgens/accretion/accretion.toml new file mode 100644 index 000000000..1ec641430 --- /dev/null +++ b/pgens/accretion/accretion.toml @@ -0,0 +1,85 @@ +[simulation] + name = "wald" + engine = "grpic" + runtime = 500.0 + +[grid] + resolution = [256, 256] + extent = [[1.0, 6.0]] + + [grid.metric] + metric = "qkerr_schild" + qsph_r0 = 0.0 + qsph_h = 0.0 + ks_a = 0.95 + + [grid.boundaries] + fields = [["MATCH"]] + particles = [["ABSORB"]] + + [grid.boundaries.absorb] + ds = 1.0 + +[scales] + larmor0 = 0.025 + skindepth0 = 0.5 + +[algorithms] + current_filters = 4 + + [algorithms.gr] + pusher_niter = 10 + pusher_eps = 1e-2 + + [algorithms.timestep] + CFL = 0.5 + correction = 1.0 + + [algorithms.toggles] + deposit = true + fieldsolver = true + +[particles] + ppc0 = 4.0 + use_weights = true + clear_interval = 100 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = -1.0 + maxnpart = 2e8 + pusher = "Boris" + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = 1.0 + maxnpart = 2e8 + pusher = "Boris" + +[setup] + multiplicity = 1.0 + sigma_max = 1000.0 + temperature = 0.01 + xi_min = [1.5, 0.0] + xi_max = [4.0, 3.14159265] + m_eps = 1.0 + +[output] + format = "hdf5" + + [output.fields] + interval_time = 1.0 + quantities = ["D", "B", "N_1", "N_2", "A"] + + [output.particles] + enable = false + + [output.spectra] + enable = false + +[diagnostics] + interval = 2 + colored_stdout = true + blocking_timers = true diff --git a/pgens/accretion/pgen.hpp b/pgens/accretion/pgen.hpp new file mode 100644 index 000000000..864140dfe --- /dev/null +++ b/pgens/accretion/pgen.hpp @@ -0,0 +1,281 @@ +#ifndef PROBLEM_GENERATOR_H +#define PROBLEM_GENERATOR_H + +#include "enums.h" +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "arch/traits.h" +#include "utils/numeric.h" + +#include "archetypes/energy_dist.h" +#include "archetypes/particle_injector.h" +#include "archetypes/problem_generator.h" +#include "archetypes/spatial_dist.h" +#include "framework/domain/metadomain.h" + +#include "kernels/particle_moments.hpp" + +namespace user { + using namespace ntt; + + template + struct InitFields { + InitFields(M metric_, real_t m_eps) : metric { metric_ }, m_eps { m_eps } {} + + Inline auto A_3(const coord_t& x_Cd) const -> real_t { + return HALF * (metric.template h_<3, 3>(x_Cd) + + TWO * metric.spin() * metric.template h_<1, 3>(x_Cd) * + metric.beta1(x_Cd)); + } + + Inline auto A_1(const coord_t& x_Cd) const -> real_t { + return HALF * (metric.template h_<1, 3>(x_Cd) + + TWO * metric.spin() * metric.template h_<1, 1>(x_Cd) * + metric.beta1(x_Cd)); + } + + Inline auto A_0(const coord_t& x_Cd) const -> real_t { + real_t g_00 { -metric.alpha(x_Cd) * metric.alpha(x_Cd) + + metric.template h_<1, 1>(x_Cd) * metric.beta1(x_Cd) * + metric.beta1(x_Cd) }; + return HALF * (metric.template h_<1, 3>(x_Cd) * metric.beta1(x_Cd) + + TWO * metric.spin() * g_00); + } + + Inline auto bx1(const coord_t& x_Ph) const -> real_t { // at ( i , j + HALF ) + coord_t xi { ZERO }, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0]; + x0m[1] = xi[1] - HALF * m_eps; + x0p[0] = xi[0]; + x0p[1] = xi[1] + HALF * m_eps; + + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + + if (cmp::AlmostZero(x_Ph[1])) { + return ONE; + } else { + return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP / m_eps; + } + } + + Inline auto bx2(const coord_t& x_Ph) const -> real_t { // at ( i + HALF , j ) + coord_t xi { ZERO }, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + + x0m[0] = xi[0] - HALF * m_eps; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF * m_eps; + x0p[1] = xi[1]; + + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + if (cmp::AlmostZero(x_Ph[1])) { + return ZERO; + } else { + return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP / m_eps; + } + } + + Inline auto bx3(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx1(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx2(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + Inline auto dx3(const coord_t& x_Ph) const -> real_t { + return ZERO; + } + + private: + const M metric; + const real_t m_eps; + }; + + template + struct PointDistribution : public arch::SpatialDistribution { + PointDistribution(const std::vector& xi_min, + const std::vector& xi_max, + const real_t sigma_thr, + const real_t dens_thr, + const SimulationParams& params, + Domain* domain_ptr) + : arch::SpatialDistribution { domain_ptr->mesh.metric } + , metric { domain_ptr->mesh.metric } + , EM { domain_ptr->fields.em } + , density { domain_ptr->fields.buff } + , sigma_thr { sigma_thr } + , inv_n0 { ONE / params.template get("scales.n0") } + , dens_thr { dens_thr } { + std::copy(xi_min.begin(), xi_min.end(), x_min); + std::copy(xi_max.begin(), xi_max.end(), x_max); + + std::vector specs {}; + for (auto& sp : domain_ptr->species) { + if (sp.mass() > 0) { + specs.push_back(sp.index()); + } + } + + Kokkos::deep_copy(density, ZERO); + auto scatter_buff = Kokkos::Experimental::create_scatter_view(density); + // some parameters + auto& mesh = domain_ptr->mesh; + const auto use_weights = params.template get( + "particles.use_weights"); + const auto ni2 = mesh.n_active(in::x2); + + for (const auto& sp : specs) { + auto& prtl_spec = domain_ptr->species[sp - 1]; + // clang-format off + Kokkos::parallel_for( + "ComputeMoments", + prtl_spec.rangeActiveParticles(), + kernel::ParticleMoments_kernel({}, scatter_buff, 0u, + prtl_spec.i1, prtl_spec.i2, prtl_spec.i3, + prtl_spec.dx1, prtl_spec.dx2, prtl_spec.dx3, + prtl_spec.ux1, prtl_spec.ux2, prtl_spec.ux3, + prtl_spec.phi, prtl_spec.weight, prtl_spec.tag, + prtl_spec.mass(), prtl_spec.charge(), + use_weights, + metric, mesh.flds_bc(), + ni2, inv_n0, ZERO)); + // clang-format on + } + Kokkos::Experimental::contribute(density, scatter_buff); + } + + Inline auto sigma_crit(const coord_t& x_Ph) const -> bool { + coord_t xi { ZERO }; + if constexpr (M::Dim == Dim::_2D) { + metric.template convert(x_Ph, xi); + const auto i1 = static_cast(xi[0]) + static_cast(N_GHOSTS); + const auto i2 = static_cast(xi[1]) + static_cast(N_GHOSTS); + const vec_t B_cntrv { EM(i1, i2, em::bx1), + EM(i1, i2, em::bx2), + EM(i1, i2, em::bx3) }; + const vec_t D_cntrv { EM(i1, i2, em::dx1), + EM(i1, i2, em::dx2), + EM(i1, i2, em::dx3) }; + vec_t B_cov { ZERO }; + metric.template transform(xi, B_cntrv, B_cov); + const auto bsqr = + DOT(B_cntrv[0], B_cntrv[1], B_cntrv[2], B_cov[0], B_cov[1], B_cov[2]); + const auto dens = density(i1, i2, 0); + return (bsqr > sigma_thr * dens) || (dens < dens_thr); + } + return false; + } + + Inline auto operator()(const coord_t& x_Ph) const -> real_t override { + auto fill = true; + for (auto d = 0u; d < M::Dim; ++d) { + fill &= x_Ph[d] > x_min[d] and x_Ph[d] < x_max[d] and sigma_crit(x_Ph); + } + return fill ? ONE : ZERO; + } + + private: + tuple_t x_min; + tuple_t x_max; + const real_t sigma_thr; + const real_t dens_thr; + const real_t inv_n0; + Domain* domain_ptr; + ndfield_t density; + ndfield_t EM; + const M metric; + }; + + template + struct PGen : public arch::ProblemGenerator { + // compatibility traits for the problem generator + static constexpr auto engines { traits::compatible_with::value }; + static constexpr auto metrics { + traits::compatible_with::value + }; + static constexpr auto dimensions { traits::compatible_with::value }; + + // for easy access to variables in the child class + using arch::ProblemGenerator::D; + using arch::ProblemGenerator::C; + using arch::ProblemGenerator::params; + + const std::vector xi_min; + const std::vector xi_max; + const real_t sigma0, sigma_max, multiplicity, nGJ, temperature, m_eps; + + InitFields init_flds; + const Metadomain* metadomain; + + inline PGen(SimulationParams& p, const Metadomain& m) + : arch::ProblemGenerator(p) + , xi_min { p.template get>("setup.xi_min") } + , xi_max { p.template get>("setup.xi_max") } + , sigma_max { p.template get("setup.sigma_max") } + , sigma0 { p.template get("scales.sigma0") } + , multiplicity { p.template get("setup.multiplicity") } + , nGJ { p.template get("scales.B0") * + SQR(p.template get("scales.skindepth0")) } + , temperature { p.template get("setup.temperature") } + , m_eps { p.template get("setup.m_eps") } + , init_flds { m.mesh().metric, m_eps } + , metadomain { &m } {} + + inline void InitPrtls(Domain& local_domain) { + const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, + local_domain.random_pool, + temperature); + const auto spatial_dist = PointDistribution(xi_min, + xi_max, + sigma_max / sigma0, + multiplicity * nGJ, + params, + &local_domain); + + const auto injector = + arch::NonUniformInjector( + energy_dist, + spatial_dist, + { 1, 2 }); + arch::InjectNonUniform(params, + local_domain, + injector, + 1.0, + true); + } + + void CustomPostStep(std::size_t, long double time, Domain& local_domain) { + const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, + local_domain.random_pool, + temperature); + const auto spatial_dist = PointDistribution(xi_min, + xi_max, + sigma_max / sigma0, + multiplicity * nGJ, + params, + &local_domain); + + const auto injector = + arch::NonUniformInjector( + energy_dist, + spatial_dist, + { 1, 2 }); + arch::InjectNonUniform(params, + local_domain, + injector, + 1.0, + true); + } + }; + +} // namespace user + +#endif diff --git a/pgens/wald/pgen.hpp b/pgens/wald/pgen.hpp index 04a80de57..b2c2aeb0c 100644 --- a/pgens/wald/pgen.hpp +++ b/pgens/wald/pgen.hpp @@ -6,22 +6,25 @@ #include "arch/kokkos_aliases.h" #include "arch/traits.h" +#include "utils/comparators.h" +#include "utils/formatting.h" +#include "utils/log.h" #include "utils/numeric.h" #include "archetypes/energy_dist.h" #include "archetypes/particle_injector.h" #include "archetypes/problem_generator.h" -#include "archetypes/spatial_dist.h" +#include "framework/domain/domain.h" #include "framework/domain/metadomain.h" -#include "kernels/particle_moments.hpp" +#include namespace user { using namespace ntt; template struct InitFields { - InitFields(M metric_, real_t m_eps) : metric { metric_ }, m_eps { m_eps } {} + InitFields(M metric_) : metric { metric_ } {} Inline auto A_3(const coord_t& x_Cd) const -> real_t { return HALF * (metric.template h_<3, 3>(x_Cd) + @@ -43,157 +46,139 @@ namespace user { TWO * metric.spin() * g_00); } - Inline auto bx1(const coord_t& x_Ph) const - -> real_t { // at ( i , j + HALF ) + Inline auto bx1(const coord_t& x_Ph) const -> real_t { // at ( i , j + HALF ) coord_t xi { ZERO }, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); x0m[0] = xi[0]; - x0m[1] = xi[1] - HALF * m_eps; + x0m[1] = xi[1] - HALF; x0p[0] = xi[0]; - x0p[1] = xi[1] + HALF * m_eps; + x0p[1] = xi[1] + HALF; real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; if (cmp::AlmostZero(x_Ph[1])) { return ONE; } else { - return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP / m_eps; + return (A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; } } - Inline auto bx2(const coord_t& x_Ph) const - -> real_t { // at ( i + HALF , j ) + Inline auto bx2(const coord_t& x_Ph) const -> real_t { // at ( i + HALF , j ) coord_t xi { ZERO }, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); - x0m[0] = xi[0] - HALF * m_eps; + x0m[0] = xi[0] - HALF; x0m[1] = xi[1]; - x0p[0] = xi[0] + HALF * m_eps; + x0p[0] = xi[0] + HALF; x0p[1] = xi[1]; real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; if (cmp::AlmostZero(x_Ph[1])) { return ZERO; } else { - return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP / m_eps; + return -(A_3(x0p) - A_3(x0m)) * inv_sqrt_detH_ijP; } } - Inline auto bx3(const coord_t& x_Ph) const -> real_t { - return ZERO; - } + Inline auto bx3( + const coord_t& x_Ph) const -> real_t { // at ( i + HALF , j + HALF ) + coord_t xi { ZERO }, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); - Inline auto dx1(const coord_t& x_Ph) const -> real_t { - return ZERO; - } + x0m[0] = xi[0]; + x0m[1] = xi[1] - HALF; + x0p[0] = xi[0]; + x0p[1] = xi[1] + HALF; - Inline auto dx2(const coord_t& x_Ph) const -> real_t { - return ZERO; + real_t inv_sqrt_detH_iPjP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + return -(A_1(x0p) - A_1(x0m)) * inv_sqrt_detH_iPjP; } - Inline auto dx3(const coord_t& x_Ph) const -> real_t { - return ZERO; - } + Inline auto dx1(const coord_t& x_Ph) const -> real_t { // at ( i + HALF , j ) + coord_t xi { ZERO }, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); - private: - const M metric; - const real_t m_eps; - }; + real_t alpha_iPj { metric.alpha({ xi[0], xi[1] }) }; + real_t inv_sqrt_detH_ij { ONE / metric.sqrt_det_h({ xi[0] - HALF, xi[1] }) }; + real_t sqrt_detH_ij { metric.sqrt_det_h({ xi[0] - HALF, xi[1] }) }; + real_t beta_ij { metric.beta1({ xi[0] - HALF, xi[1] }) }; + real_t alpha_ij { metric.alpha({ xi[0] - HALF, xi[1] }) }; - template - struct PointDistribution : public arch::SpatialDistribution { - PointDistribution(const std::vector& xi_min, - const std::vector& xi_max, - const real_t sigma_thr, - const real_t dens_thr, - const SimulationParams& params, - Domain* domain_ptr) - : arch::SpatialDistribution { domain_ptr->mesh.metric } - , metric { domain_ptr->mesh.metric } - , EM { domain_ptr->fields.em } - , density { domain_ptr->fields.buff } - , sigma_thr { sigma_thr } - , inv_n0 { ONE / params.template get("scales.n0") } - , dens_thr { dens_thr } { - std::copy(xi_min.begin(), xi_min.end(), x_min); - std::copy(xi_max.begin(), xi_max.end(), x_max); - - std::vector specs {}; - for (auto& sp : domain_ptr->species) { - if (sp.mass() > 0) { - specs.push_back(sp.index()); - } - } + // D1 at ( i + HALF , j ) + x0m[0] = xi[0] - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF; + x0p[1] = xi[1]; + real_t E1d { (A_0(x0p) - A_0(x0m)) }; + real_t D1d { E1d / alpha_iPj }; - Kokkos::deep_copy(density, ZERO); - auto scatter_buff = Kokkos::Experimental::create_scatter_view(density); - // some parameters - auto& mesh = domain_ptr->mesh; - const auto use_weights = params.template get( - "particles.use_weights"); - const auto ni2 = mesh.n_active(in::x2); - - for (const auto& sp : specs) { - auto& prtl_spec = domain_ptr->species[sp - 1]; - // clang-format off - Kokkos::parallel_for( - "ComputeMoments", - prtl_spec.rangeActiveParticles(), - kernel::ParticleMoments_kernel({}, scatter_buff, 0u, - prtl_spec.i1, prtl_spec.i2, prtl_spec.i3, - prtl_spec.dx1, prtl_spec.dx2, prtl_spec.dx3, - prtl_spec.ux1, prtl_spec.ux2, prtl_spec.ux3, - prtl_spec.phi, prtl_spec.weight, prtl_spec.tag, - prtl_spec.mass(), prtl_spec.charge(), - use_weights, - metric, mesh.flds_bc(), - ni2, inv_n0, ZERO)); - // clang-format on - } - Kokkos::Experimental::contribute(density, scatter_buff); + // D3 at ( i , j ) + x0m[0] = xi[0] - HALF - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] - HALF + HALF; + x0p[1] = xi[1]; + real_t D3d { (A_3(x0p) - A_3(x0m)) * beta_ij / alpha_ij }; + + real_t D1u { metric.template h<1, 1>({ xi[0], xi[1] }) * D1d + + metric.template h<1, 3>({ xi[0], xi[1] }) * D3d }; + + return D1u; } - Inline auto sigma_crit(const coord_t& x_Ph) const -> bool { - coord_t xi { ZERO }; - if constexpr (M::Dim == Dim::_2D) { - metric.template convert(x_Ph, xi); - const auto i1 = static_cast(xi[0]) + static_cast(N_GHOSTS); - const auto i2 = static_cast(xi[1]) + static_cast(N_GHOSTS); - const vec_t B_cntrv { EM(i1, i2, em::bx1), - EM(i1, i2, em::bx2), - EM(i1, i2, em::bx3) }; - const vec_t D_cntrv { EM(i1, i2, em::dx1), - EM(i1, i2, em::dx2), - EM(i1, i2, em::dx3) }; - vec_t B_cov { ZERO }; - metric.template transform(xi, B_cntrv, B_cov); - const auto bsqr = - DOT(B_cntrv[0], B_cntrv[1], B_cntrv[2], B_cov[0], B_cov[1], B_cov[2]); - const auto dens = density(i1, i2, 0); - return (bsqr > sigma_thr * dens) || (dens < dens_thr); - } - return false; + Inline auto dx2(const coord_t& x_Ph) const -> real_t { // at ( i , j + HALF ) + coord_t xi { ZERO }, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + x0m[0] = xi[0]; + x0m[1] = xi[1] - HALF; + x0p[0] = xi[0]; + x0p[1] = xi[1] + HALF; + real_t inv_sqrt_detH_ijP { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + real_t sqrt_detH_ijP { metric.sqrt_det_h({ xi[0], xi[1] }) }; + real_t alpha_ijP { metric.alpha({ xi[0], xi[1] }) }; + real_t beta_ijP { metric.beta1({ xi[0], xi[1] }) }; + + real_t E2d { (A_0(x0p) - A_0(x0m)) }; + real_t D2d { E2d / alpha_ijP - (A_1(x0p) - A_1(x0m)) * beta_ijP / alpha_ijP }; + real_t D2u { metric.template h<2, 2>({ xi[0], xi[1] }) * D2d }; + + return D2u; } - Inline auto operator()(const coord_t& x_Ph) const -> real_t override { - auto fill = true; - for (auto d = 0u; d < M::Dim; ++d) { - fill &= x_Ph[d] > x_min[d] and x_Ph[d] < x_max[d] and sigma_crit(x_Ph); + Inline auto dx3(const coord_t& x_Ph) const -> real_t { // at ( i , j ) + coord_t xi { ZERO }, x0m { ZERO }, x0p { ZERO }; + metric.template convert(x_Ph, xi); + real_t inv_sqrt_detH_ij { ONE / metric.sqrt_det_h({ xi[0], xi[1] }) }; + real_t sqrt_detH_ij { metric.sqrt_det_h({ xi[0], xi[1] }) }; + real_t beta_ij { metric.beta1({ xi[0], xi[1] }) }; + real_t alpha_ij { metric.alpha({ xi[0], xi[1] }) }; + real_t alpha_iPj { metric.alpha({ xi[0] + HALF, xi[1] }) }; + + // D3 at ( i , j ) + x0m[0] = xi[0] - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF; + x0p[1] = xi[1]; + real_t D3d { (A_3(x0p) - A_3(x0m)) * beta_ij / alpha_ij }; + + // D1 at ( i + HALF , j ) + x0m[0] = xi[0] + HALF - HALF; + x0m[1] = xi[1]; + x0p[0] = xi[0] + HALF + HALF; + x0p[1] = xi[1]; + real_t E1d { (A_0(x0p) - A_0(x0m)) }; + real_t D1d { E1d / alpha_iPj }; + + if (cmp::AlmostZero(x_Ph[1])) { + return metric.template h<1, 3>({ xi[0], xi[1] }) * D1d; + } else { + return metric.template h<3, 3>({ xi[0], xi[1] }) * D3d + + metric.template h<1, 3>({ xi[0], xi[1] }) * D1d; } - return fill ? ONE : ZERO; } private: - tuple_t x_min; - tuple_t x_max; - const real_t sigma_thr; - const real_t dens_thr; - const real_t inv_n0; - Domain* domain_ptr; - ndfield_t density; - ndfield_t EM; - const M metric; + const M metric; }; template @@ -210,72 +195,13 @@ namespace user { using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; - const std::vector xi_min; - const std::vector xi_max; - const real_t sigma0, sigma_max, multiplicity, nGJ, temperature, m_eps; - InitFields init_flds; - const Metadomain* metadomain; - - inline PGen(SimulationParams& p, const Metadomain& m) - : arch::ProblemGenerator(p) - , xi_min { p.template get>("setup.xi_min") } - , xi_max { p.template get>("setup.xi_max") } - , sigma_max { p.template get("setup.sigma_max") } - , sigma0 { p.template get("scales.sigma0") } - , multiplicity { p.template get("setup.multiplicity") } - , nGJ { p.template get("scales.B0") * - SQR(p.template get("scales.skindepth0")) } - , temperature { p.template get("setup.temperature") } - , m_eps { p.template get("setup.m_eps") } - , init_flds { m.mesh().metric, m_eps } - , metadomain { &m } {} - - inline void InitPrtls(Domain& local_domain) { - const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, - local_domain.random_pool, - temperature); - const auto spatial_dist = PointDistribution(xi_min, - xi_max, - sigma_max / sigma0, - multiplicity * nGJ, - params, - &local_domain); - - const auto injector = - arch::NonUniformInjector( - energy_dist, - spatial_dist, - { 1, 2 }); - arch::InjectNonUniform(params, - local_domain, - injector, - 1.0, - true); - } + const Metadomain& global_domain; - void CustomPostStep(std::size_t, long double time, Domain& local_domain) { - const auto energy_dist = arch::Maxwellian(local_domain.mesh.metric, - local_domain.random_pool, - temperature); - const auto spatial_dist = PointDistribution(xi_min, - xi_max, - sigma_max / sigma0, - multiplicity * nGJ, - params, - &local_domain); - - const auto injector = - arch::NonUniformInjector( - energy_dist, - spatial_dist, - { 1, 2 }); - arch::InjectNonUniform(params, - local_domain, - injector, - 1.0, - true); - } + inline PGen(const SimulationParams& p, const Metadomain& m) + : arch::ProblemGenerator { p } + , global_domain { m } + , init_flds { m.mesh().metric } {} }; } // namespace user diff --git a/pgens/wald/wald.toml b/pgens/wald/wald.toml index 1ec641430..1104f7a08 100644 --- a/pgens/wald/wald.toml +++ b/pgens/wald/wald.toml @@ -1,14 +1,14 @@ [simulation] - name = "wald" + name = "vacuum" engine = "grpic" - runtime = 500.0 + runtime = 100.0 [grid] - resolution = [256, 256] - extent = [[1.0, 6.0]] + resolution = [128, 128] + extent = [[1.0, 10.0]] [grid.metric] - metric = "qkerr_schild" + metric = "kerr_schild" qsph_r0 = 0.0 qsph_h = 0.0 ks_a = 0.95 @@ -21,57 +21,31 @@ ds = 1.0 [scales] - larmor0 = 0.025 - skindepth0 = 0.5 + larmor0 = 0.0025 + skindepth0 = 0.05 [algorithms] - current_filters = 4 - - [algorithms.gr] - pusher_niter = 10 - pusher_eps = 1e-2 + current_filters = 0 [algorithms.timestep] - CFL = 0.5 - correction = 1.0 + CFL = 0.5 [algorithms.toggles] - deposit = true + deposit = false fieldsolver = true [particles] - ppc0 = 4.0 - use_weights = true - clear_interval = 100 - - [[particles.species]] - label = "e-" - mass = 1.0 - charge = -1.0 - maxnpart = 2e8 - pusher = "Boris" - - [[particles.species]] - label = "e+" - mass = 1.0 - charge = 1.0 - maxnpart = 2e8 - pusher = "Boris" + ppc0 = 2.0 [setup] - multiplicity = 1.0 - sigma_max = 1000.0 - temperature = 0.01 - xi_min = [1.5, 0.0] - xi_max = [4.0, 3.14159265] - m_eps = 1.0 [output] - format = "hdf5" + format = "hdf5" + separate_files = false [output.fields] interval_time = 1.0 - quantities = ["D", "B", "N_1", "N_2", "A"] + quantities = ["D", "H", "B", "A"] [output.particles] enable = false diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 04197d12b..6abbfd337 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -715,17 +715,17 @@ namespace ntt { "OpenBCFields", range, kernel::bc::HorizonBoundaries_kernel(domain.fields.em, - i1_min, - tags, - nfilter)); + i1_min, + tags, + nfilter)); Kokkos::parallel_for( "OpenBCFields", range, kernel::bc::HorizonBoundaries_kernel(domain.fields.em0, - i1_min, - tags, - nfilter)); - } + i1_min, + tags, + nfilter)); + } } void AxisFieldsIn(dir::direction_t direction, @@ -1099,7 +1099,7 @@ namespace ntt { "algorithms.gr.pusher_niter"); // clang-format off if (species.pusher() == PrtlPusher::PHOTON) { - auto range_policy = Kokkos::RangePolicy( + auto range_policy = Kokkos::RangePolicy( 0, species.npart()); @@ -1124,7 +1124,7 @@ namespace ntt { domain.mesh.prtl_bc() )); } else if (species.pusher() == PrtlPusher::BORIS) { - auto range_policy = Kokkos::RangePolicy( + auto range_policy = Kokkos::RangePolicy( 0, species.npart()); Kokkos::parallel_for( From f667d72ed2e5cf7ff33234f4263764fe590af073 Mon Sep 17 00:00:00 2001 From: haykh Date: Thu, 12 Jun 2025 20:38:26 -0700 Subject: [PATCH 225/234] ds param fix --- src/engines/grpic.hpp | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 6abbfd337..6a563eb7c 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -595,13 +595,24 @@ namespace ntt { /** * match boundaries */ - const auto ds = m_params.template get("grid.boundaries.match.ds"); + // const auto ds = m_params.template get("grid.boundaries.match.ds"); + const auto ds_array = m_params.template get>( + "grid.boundaries.match.ds"); const auto dim = direction.get_dim(); real_t xg_min, xg_max, xg_edge; - xg_max = m_metadomain.mesh().extent(dim).second; - xg_min = xg_max - ds; - xg_edge = xg_max; - + auto sign = direction.get_sign(); + real_t ds; + if (sign > 0) { // + direction + ds = ds_array[(short)dim].second; + xg_max = m_metadomain.mesh().extent(dim).second; + xg_min = xg_max - ds; + xg_edge = xg_max; + } else { // - direction + ds = ds_array[(short)dim].first; + xg_min = m_metadomain.mesh().extent(dim).first; + xg_max = xg_min + ds; + xg_edge = xg_min; + } boundaries_t box; boundaries_t incl_ghosts; for (unsigned short d { 0 }; d < M::Dim; ++d) { @@ -654,9 +665,13 @@ namespace ntt { /** * match boundaries */ - const auto ds = m_params.template get("grid.boundaries.match.ds"); + // @TODO: also in MatchFieldsIn (need a better way) + const auto ds_array = m_params.template get>( + "grid.boundaries.match.ds"); const auto dim = in::x1; real_t xg_min, xg_max, xg_edge; + real_t ds; + ds = ds_array[(short)dim].second; xg_max = m_metadomain.mesh().extent(dim).second; xg_min = xg_max - ds; xg_edge = xg_max; From dcfea6206b05a4cd979900546ffbc8899762e5cc Mon Sep 17 00:00:00 2001 From: haykh Date: Sat, 14 Jun 2025 10:54:01 -0700 Subject: [PATCH 226/234] minor refactor + BCs aligned with sr --- src/engines/grpic.hpp | 241 ++++++++++++++++---------------- src/global/global.h | 4 + src/kernels/fields_bcs.hpp | 272 +++++++++++++++++++------------------ 3 files changed, 268 insertions(+), 249 deletions(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 6a563eb7c..345c91d5d 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -16,13 +16,11 @@ #include "global.h" #include "arch/kokkos_aliases.h" -#include "arch/traits.h" #include "utils/log.h" #include "utils/numeric.h" #include "utils/timer.h" #include "utils/toml.h" -#include "archetypes/particle_injector.h" #include "framework/domain/domain.h" #include "framework/parameters.h" @@ -33,7 +31,6 @@ #include "kernels/digital_filter.hpp" #include "kernels/faraday_gr.hpp" #include "kernels/fields_bcs.hpp" -#include "kernels/particle_moments.hpp" #include "kernels/particle_pusher_gr.hpp" #include "pgen.hpp" @@ -64,7 +61,8 @@ namespace ntt { }; enum class gr_bc { main, - aux + aux, + curr }; template @@ -385,7 +383,7 @@ namespace ntt { timers.stop("Communications"); timers.start("FieldBoundaries"); - CurrentsBoundaryConditions(dom); + FieldBoundaries(dom, BC::J, gr_bc::curr); timers.stop("FieldBoundaries"); timers.start("CurrentFiltering"); @@ -563,27 +561,25 @@ namespace ntt { if (g == gr_bc::main) { for (auto& direction : dir::Directions::orth) { if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::MATCH) { - MatchFieldsIn(direction, domain, tags); - } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::AXIS) { - if (domain.mesh.flds_bc_in(direction) == FldsBC::AXIS) { - AxisFieldsIn(direction, domain, tags); - } + MatchFieldsIn(direction, domain, tags, g); + } else if (domain.mesh.flds_bc_in(direction) == FldsBC::AXIS) { + AxisFieldsIn(direction, domain, tags); } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::CUSTOM) { - if (domain.mesh.flds_bc_in(direction) == FldsBC::CUSTOM) { - CustomFieldsIn(direction, domain, tags, g); - } - } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { - if (domain.mesh.flds_bc_in(direction) == FldsBC::HORIZON) { - HorizonFieldsIn(direction, domain, tags, g); - } + CustomFieldsIn(direction, domain, tags, g); + } else if (domain.mesh.flds_bc_in(direction) == FldsBC::HORIZON) { + HorizonFieldsIn(direction, domain, tags, g); } } // loop over directions } else if (g == gr_bc::aux) { for (auto& direction : dir::Directions::orth) { - if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { - if (domain.mesh.flds_bc_in(direction) == FldsBC::HORIZON) { - HorizonFieldsIn(direction, domain, tags, g); - } + if (domain.mesh.flds_bc_in(direction) == FldsBC::HORIZON) { + HorizonFieldsIn(direction, domain, tags, g); + } + } + } else if (g == gr_bc::curr) { + for (auto& direction : dir::Directions::orth) { + if (domain.mesh.prtl_bc_in(direction) == PrtlBC::ABSORB) { + MatchFieldsIn(direction, domain, tags, g); } } } @@ -591,17 +587,22 @@ namespace ntt { void MatchFieldsIn(dir::direction_t direction, domain_t& domain, - BCTags tags) { + BCTags tags, + const gr_bc& g) { /** * match boundaries */ - // const auto ds = m_params.template get("grid.boundaries.match.ds"); const auto ds_array = m_params.template get>( "grid.boundaries.match.ds"); const auto dim = direction.get_dim(); real_t xg_min, xg_max, xg_edge; auto sign = direction.get_sign(); - real_t ds; + + raise::ErrorIf(((dim != in::x1) or (sign > 0)) and (g == gr_bc::curr), + "Absorption of currents only possible in +x1 (+r)", + HERE); + + real_t ds; if (sign > 0) { // + direction ds = ds_array[(short)dim].second; xg_max = m_metadomain.mesh().extent(dim).second; @@ -636,76 +637,88 @@ namespace ntt { range_max[d] = intersect_range[d].second; } if (dim == in::x1) { - Kokkos::parallel_for( - "MatchBoundaries", - CreateRangePolicy(range_min, range_max), - kernel::bc::MatchBoundaries_kernel( - domain.fields.em, - m_pgen.init_flds, - domain.mesh.metric, - xg_edge, - ds, - tags)); - Kokkos::parallel_for( - "MatchBoundaries", - CreateRangePolicy(range_min, range_max), - kernel::bc::MatchBoundaries_kernel( - domain.fields.em0, - m_pgen.init_flds, - domain.mesh.metric, - xg_edge, - ds, - tags)); + if (g != gr_bc::curr) { + Kokkos::parallel_for( + "MatchBoundaries", + CreateRangePolicy(range_min, range_max), + kernel::bc::MatchBoundaries_kernel( + domain.fields.em, + m_pgen.init_flds, + domain.mesh.metric, + xg_edge, + ds, + tags)); + Kokkos::parallel_for( + "MatchBoundaries", + CreateRangePolicy(range_min, range_max), + kernel::bc::MatchBoundaries_kernel( + domain.fields.em0, + m_pgen.init_flds, + domain.mesh.metric, + xg_edge, + ds, + tags)); + } else { + Kokkos::parallel_for( + "AbsorbCurrents", + CreateRangePolicy(range_min, range_max), + kernel::bc::gr::AbsorbCurrents_kernel(domain.fields.cur0, + domain.mesh.metric, + xg_edge, + ds)); + } } else { raise::Error("Invalid dimension", HERE); } } - void CurrentsBoundaryConditions(domain_t& domain) { - /** - * match boundaries - */ - // @TODO: also in MatchFieldsIn (need a better way) - const auto ds_array = m_params.template get>( - "grid.boundaries.match.ds"); - const auto dim = in::x1; - real_t xg_min, xg_max, xg_edge; - real_t ds; - ds = ds_array[(short)dim].second; - xg_max = m_metadomain.mesh().extent(dim).second; - xg_min = xg_max - ds; - xg_edge = xg_max; - - boundaries_t box; - boundaries_t incl_ghosts; - for (unsigned short d { 0 }; d < M::Dim; ++d) { - if (d == static_cast(dim)) { - box.push_back({ xg_min, xg_max }); - incl_ghosts.push_back({ false, true }); - } else { - box.push_back(Range::All); - incl_ghosts.push_back({ true, true }); - } - } - if (not domain.mesh.Intersects(box)) { - return; - } - const auto intersect_range = domain.mesh.ExtentToRange(box, incl_ghosts); - tuple_t range_min { 0 }; - tuple_t range_max { 0 }; - - for (unsigned short d { 0 }; d < M::Dim; ++d) { - range_min[d] = intersect_range[d].first; - range_max[d] = intersect_range[d].second; - } - Kokkos::parallel_for( - "AbsorbCurrentsGR", - CreateRangePolicy(range_min, range_max), - kernel::bc::AbsorbCurrentsGR_kernel(domain.fields.cur0, - domain.mesh.metric, - xg_edge, - ds)); - } + // void AbsorbCurrentsIn(dir::direction_t direction, + // domain_t& domain, + // BCTags tags) { + // /** + // * absorbing currents at the boundaries + // */ + // // @TODO: also in MatchFieldsIn (need a better way) + // const auto ds_array = m_params.template get>( + // "grid.boundaries.match.ds"); + // const auto dim = in::x1; + // real_t xg_min, xg_max, xg_edge; + // real_t ds; + // ds = ds_array[(short)dim].second; + // xg_max = m_metadomain.mesh().extent(dim).second; + // xg_min = xg_max - ds; + // xg_edge = xg_max; + // + // boundaries_t box; + // boundaries_t incl_ghosts; + // for (unsigned short d { 0 }; d < M::Dim; ++d) { + // if (d == static_cast(dim)) { + // box.push_back({ xg_min, xg_max }); + // incl_ghosts.push_back({ false, true }); + // } else { + // box.push_back(Range::All); + // incl_ghosts.push_back({ true, true }); + // } + // } + // if (not domain.mesh.Intersects(box)) { + // return; + // } + // const auto intersect_range = domain.mesh.ExtentToRange(box, incl_ghosts); + // tuple_t range_min { 0 }; + // tuple_t range_max { 0 }; + // + // for (unsigned short d { 0 }; d < M::Dim; ++d) { + // range_min[d] = intersect_range[d].first; + // range_max[d] = intersect_range[d].second; + // } + // Kokkos::parallel_for( + // "AbsorbCurrentsGR", + // CreateRangePolicy(range_min, range_max), + // kernel::bc::AbsorbCurrentsGR_kernel(domain.fields.cur0, + // domain.mesh.metric, + // xg_edge, + // ds)); + // } void HorizonFieldsIn(dir::direction_t direction, domain_t& domain, @@ -729,17 +742,17 @@ namespace ntt { Kokkos::parallel_for( "OpenBCFields", range, - kernel::bc::HorizonBoundaries_kernel(domain.fields.em, - i1_min, - tags, - nfilter)); + kernel::bc::gr::HorizonBoundaries_kernel(domain.fields.em, + i1_min, + tags, + nfilter)); Kokkos::parallel_for( "OpenBCFields", range, - kernel::bc::HorizonBoundaries_kernel(domain.fields.em0, - i1_min, - tags, - nfilter)); + kernel::bc::gr::HorizonBoundaries_kernel(domain.fields.em0, + i1_min, + tags, + nfilter)); } } @@ -757,34 +770,32 @@ namespace ntt { HERE); const auto i2_min = domain.mesh.i_min(in::x2); const auto i2_max = domain.mesh.i_max(in::x2); - auto range = CreateRangePolicy({ domain.mesh.i_min(in::x1) - 1 }, - { domain.mesh.i_max(in::x1) }); if (direction.get_sign() < 0) { Kokkos::parallel_for( "AxisBCFields", - range, - kernel::bc::AxisBoundariesGR_kernel(domain.fields.em, - i2_min, - tags)); + domain.mesh.n_all(in::x1), + kernel::bc::AxisBoundaries_kernel(domain.fields.em, + i2_min, + tags)); Kokkos::parallel_for( "AxisBCFields", - range, - kernel::bc::AxisBoundariesGR_kernel(domain.fields.em0, - i2_min, - tags)); + domain.mesh.n_all(in::x1), + kernel::bc::AxisBoundaries_kernel(domain.fields.em0, + i2_min, + tags)); } else { Kokkos::parallel_for( "AxisBCFields", - range, - kernel::bc::AxisBoundariesGR_kernel(domain.fields.em, - i2_max, - tags)); + domain.mesh.n_all(in::x1), + kernel::bc::AxisBoundaries_kernel(domain.fields.em, + i2_max, + tags)); Kokkos::parallel_for( "AxisBCFields", - range, - kernel::bc::AxisBoundariesGR_kernel(domain.fields.em0, - i2_max, - tags)); + domain.mesh.n_all(in::x1), + kernel::bc::AxisBoundaries_kernel(domain.fields.em0, + i2_max, + tags)); } } diff --git a/src/global/global.h b/src/global/global.h index 16cb4fc8a..adffcf6e9 100644 --- a/src/global/global.h +++ b/src/global/global.h @@ -291,10 +291,14 @@ namespace BC { Hx1 = 1 << 3, Hx2 = 1 << 4, Hx3 = 1 << 5, + Jx1 = 1 << 6, + Jx2 = 1 << 7, + Jx3 = 1 << 8, B = Bx1 | Bx2 | Bx3, E = Ex1 | Ex2 | Ex3, D = Dx1 | Dx2 | Dx3, H = Hx1 | Hx2 | Hx3, + J = Jx1 | Jx2 | Jx3, }; } // namespace BC diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 3a35e4d8b..40dd3d13d 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -16,6 +16,7 @@ #ifndef KERNELS_FIELDS_BCS_HPP #define KERNELS_FIELDS_BCS_HPP +#include "enums.h" #include "global.h" #include "arch/kokkos_aliases.h" @@ -844,57 +845,57 @@ namespace kernel::bc { } }; - /* - * @tparam I: Field Setter class - * @tparam M: Metric - * @tparam P: Positive/Negative direction - * @tparam O: Orientation - * - * @brief Applies enforced boundary conditions (fixed value) - */ - template - struct AxisBoundariesGR_kernel { - ndfield_t Fld; - const std::size_t i_edge; - const bool setE, setB; - - AxisBoundariesGR_kernel(ndfield_t Fld, std::size_t i_edge, BCTags tags) - : Fld { Fld } // , i_edge { i_edge } - , i_edge { P ? (i_edge + 1) : i_edge } - , setE { tags & BC::Ex1 or tags & BC::Ex2 or tags & BC::Ex3 } - , setB { tags & BC::Bx1 or tags & BC::Bx2 or tags & BC::Bx3 } {} - - Inline void operator()(index_t i1) const { - if constexpr (D == Dim::_2D) { - // if (setB) { - // Fld(i1, i_edge, em::bx2) = ZERO; - // } - if constexpr (not P) { - if (setE) { - Fld(i1, i_edge - 1, em::ex2) = -Fld(i1, i_edge, em::ex2); - Fld(i1, i_edge, em::ex3) = ZERO; - } - if (setB) { - Fld(i1, i_edge - 1, em::bx1) = Fld(i1, i_edge, em::bx1); - Fld(i1, i_edge, em::bx2) = ZERO; - Fld(i1, i_edge - 1, em::bx3) = Fld(i1, i_edge, em::bx3); - } - } else { - if (setE) { - Fld(i1, i_edge + 1, em::ex2) = -Fld(i1, i_edge, em::ex2); - Fld(i1, i_edge + 1, em::ex3) = ZERO; - } - if (setB) { - Fld(i1, i_edge + 1, em::bx1) = Fld(i1, i_edge, em::bx1); - Fld(i1, i_edge + 1, em::bx2) = ZERO; - Fld(i1, i_edge + 1, em::bx3) = Fld(i1, i_edge, em::bx3); - } - } - } else { - raise::KernelError(HERE, "AxisBoundariesGR_kernel: D != 2"); - } - } - }; + // /* + // * @tparam I: Field Setter class + // * @tparam M: Metric + // * @tparam P: Positive/Negative direction + // * @tparam O: Orientation + // * + // * @brief Applies enforced boundary conditions (fixed value) + // */ + // template + // struct AxisBoundariesGR_kernel { + // ndfield_t Fld; + // const std::size_t i_edge; + // const bool setE, setB; + // + // AxisBoundariesGR_kernel(ndfield_t Fld, std::size_t i_edge, BCTags tags) + // : Fld { Fld } // , i_edge { i_edge } + // , i_edge { P ? (i_edge + 1) : i_edge } + // , setE { tags & BC::Ex1 or tags & BC::Ex2 or tags & BC::Ex3 } + // , setB { tags & BC::Bx1 or tags & BC::Bx2 or tags & BC::Bx3 } {} + // + // Inline void operator()(index_t i1) const { + // if constexpr (D == Dim::_2D) { + // // if (setB) { + // // Fld(i1, i_edge, em::bx2) = ZERO; + // // } + // if constexpr (not P) { + // if (setE) { + // Fld(i1, i_edge - 1, em::ex2) = -Fld(i1, i_edge, em::ex2); + // Fld(i1, i_edge, em::ex3) = ZERO; + // } + // if (setB) { + // Fld(i1, i_edge - 1, em::bx1) = Fld(i1, i_edge, em::bx1); + // Fld(i1, i_edge, em::bx2) = ZERO; + // Fld(i1, i_edge - 1, em::bx3) = Fld(i1, i_edge, em::bx3); + // } + // } else { + // if (setE) { + // Fld(i1, i_edge + 1, em::ex2) = -Fld(i1, i_edge, em::ex2); + // Fld(i1, i_edge + 1, em::ex3) = ZERO; + // } + // if (setB) { + // Fld(i1, i_edge + 1, em::bx1) = Fld(i1, i_edge, em::bx1); + // Fld(i1, i_edge + 1, em::bx2) = ZERO; + // Fld(i1, i_edge + 1, em::bx3) = Fld(i1, i_edge, em::bx3); + // } + // } + // } else { + // raise::KernelError(HERE, "AxisBoundariesGR_kernel: D != 2"); + // } + // } + // }; template struct EnforcedBoundaries_kernel { @@ -1215,101 +1216,104 @@ namespace kernel::bc { } }; - template - struct HorizonBoundaries_kernel { - ndfield_t Fld; - const std::size_t i1_min; - const bool setE, setB; - const std::size_t nfilter; - - HorizonBoundaries_kernel(ndfield_t Fld, - std::size_t i1_min, - BCTags tags, - std::size_t nfilter) - : Fld { Fld } - , i1_min { i1_min } - , setE { (tags & BC::Ex1 or tags & BC::Ex2 or tags & BC::Ex3) or - (tags & BC::Dx1 or tags & BC::Dx2 or tags & BC::Dx3) } - , setB { (tags & BC::Bx1 or tags & BC::Bx2 or tags & BC::Bx3) or - (tags & BC::Hx1 or tags & BC::Hx2 or tags & BC::Hx3) } - , nfilter { nfilter } {} - - Inline void operator()(index_t i2) const { - if constexpr (M::Dim == Dim::_2D) { - if (setE) { - for (unsigned short i = 0; i <= 2 + nfilter; ++i) { - Fld(i1_min - N_GHOSTS + i, i2, em::dx1) = Fld(i1_min + 1 + nfilter, - i2, - em::dx1); - Fld(i1_min - N_GHOSTS + i, i2, em::dx2) = Fld(i1_min + 1 + nfilter, - i2, - em::dx2); - Fld(i1_min - N_GHOSTS + i, i2, em::dx3) = Fld(i1_min + 1 + nfilter, - i2, - em::dx3); + namespace gr { + + template + struct HorizonBoundaries_kernel { + ndfield_t Fld; + const std::size_t i1_min; + const bool setE, setB; + const std::size_t nfilter; + + HorizonBoundaries_kernel(ndfield_t Fld, + std::size_t i1_min, + BCTags tags, + std::size_t nfilter) + : Fld { Fld } + , i1_min { i1_min } + , setE { (tags & BC::Ex1 or tags & BC::Ex2 or tags & BC::Ex3) or + (tags & BC::Dx1 or tags & BC::Dx2 or tags & BC::Dx3) } + , setB { (tags & BC::Bx1 or tags & BC::Bx2 or tags & BC::Bx3) or + (tags & BC::Hx1 or tags & BC::Hx2 or tags & BC::Hx3) } + , nfilter { nfilter } {} + + Inline void operator()(index_t i2) const { + if constexpr (M::Dim == Dim::_2D) { + if (setE) { + for (unsigned short i = 0; i <= 2 + nfilter; ++i) { + Fld(i1_min - N_GHOSTS + i, + i2, + em::dx1) = Fld(i1_min + 1 + nfilter, i2, em::dx1); + Fld(i1_min - N_GHOSTS + i, + i2, + em::dx2) = Fld(i1_min + 1 + nfilter, i2, em::dx2); + Fld(i1_min - N_GHOSTS + i, + i2, + em::dx3) = Fld(i1_min + 1 + nfilter, i2, em::dx3); + } } - } - if (setB) { - for (unsigned short i = 0; i <= 2 + nfilter; ++i) { - Fld(i1_min - N_GHOSTS + i, i2, em::bx1) = Fld(i1_min + 1 + nfilter, - i2, - em::bx1); - Fld(i1_min - N_GHOSTS + i, i2, em::bx2) = Fld(i1_min + 1 + nfilter, - i2, - em::bx2); - Fld(i1_min - N_GHOSTS + i, i2, em::bx3) = Fld(i1_min + 1 + nfilter, - i2, - em::bx3); + if (setB) { + for (unsigned short i = 0; i <= 2 + nfilter; ++i) { + Fld(i1_min - N_GHOSTS + i, + i2, + em::bx1) = Fld(i1_min + 1 + nfilter, i2, em::bx1); + Fld(i1_min - N_GHOSTS + i, + i2, + em::bx2) = Fld(i1_min + 1 + nfilter, i2, em::bx2); + Fld(i1_min - N_GHOSTS + i, + i2, + em::bx3) = Fld(i1_min + 1 + nfilter, i2, em::bx3); + } } + } else { + raise::KernelError( + HERE, + "HorizonBoundaries_kernel: 2D implementation called for D != 2"); } - } else { - raise::KernelError( - HERE, - "HorizonBoundaries_kernel: 2D implementation called for D != 2"); } - } - }; + }; - template - struct AbsorbCurrentsGR_kernel { - static_assert(M::is_metric, "M must be a metric class"); - static_assert(i <= static_cast(M::Dim), - "Invalid component index"); + template + struct AbsorbCurrents_kernel { + static_assert(M::is_metric, "M must be a metric class"); + static_assert(i <= static_cast(M::Dim), + "Invalid component index"); - ndfield_t J; - const M metric; - const real_t xg_edge; - const real_t dx_abs; + ndfield_t J; + const M metric; + const real_t xg_edge; + const real_t dx_abs; - AbsorbCurrentsGR_kernel(ndfield_t J, + AbsorbCurrents_kernel(ndfield_t J, const M& metric, real_t xg_edge, real_t dx_abs) - : J { J } - , metric { metric } - , xg_edge { xg_edge } - , dx_abs { dx_abs } {} - - Inline void operator()(index_t i1, index_t i2) const { - if constexpr (M::Dim == Dim::_2D) { - const auto i1_ = COORD(i1); - const auto i2_ = COORD(i2); - coord_t x_Cd { ZERO }; - x_Cd[0] = i1_; - x_Cd[1] = i2_; - const auto dx = math::abs( - metric.template convert(x_Cd[i - 1]) - xg_edge); - J(i1, i2, 0) *= math::tanh(dx / (INV_4 * dx_abs)); - J(i1, i2, 1) *= math::tanh(dx / (INV_4 * dx_abs)); - J(i1, i2, 2) *= math::tanh(dx / (INV_4 * dx_abs)); + : J { J } + , metric { metric } + , xg_edge { xg_edge } + , dx_abs { dx_abs } {} + + Inline void operator()(index_t i1, index_t i2) const { + if constexpr (M::Dim == Dim::_2D) { + const auto i1_ = COORD(i1); + const auto i2_ = COORD(i2); + coord_t x_Cd { ZERO }; + x_Cd[0] = i1_; + x_Cd[1] = i2_; + const auto dx = math::abs( + metric.template convert(x_Cd[i - 1]) - xg_edge); + J(i1, i2, 0) *= math::tanh(dx / (INV_4 * dx_abs)); + J(i1, i2, 1) *= math::tanh(dx / (INV_4 * dx_abs)); + J(i1, i2, 2) *= math::tanh(dx / (INV_4 * dx_abs)); - } else { - raise::KernelError( - HERE, - "AbsorbCurrentsGR_kernel: 2D implementation called for D != 2"); + } else { + raise::KernelError( + HERE, + "gr::AbsorbCurrents_kernel: 2D implementation called for D != 2"); + } } - } - }; + }; + } // namespace gr } // namespace kernel::bc From 7470a4124acf5c4ef2d57d55d89de12f0e2000f1 Mon Sep 17 00:00:00 2001 From: Alisa Galishnikova <55898700+alisagk@users.noreply.github.com> Date: Sat, 14 Jun 2025 16:07:35 -0400 Subject: [PATCH 227/234] minor bug in AbsorbCur: sign < 0 --- src/engines/grpic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 345c91d5d..78966a25e 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -598,7 +598,7 @@ namespace ntt { real_t xg_min, xg_max, xg_edge; auto sign = direction.get_sign(); - raise::ErrorIf(((dim != in::x1) or (sign > 0)) and (g == gr_bc::curr), + raise::ErrorIf(((dim != in::x1) or (sign < 0)) and (g == gr_bc::curr), "Absorption of currents only possible in +x1 (+r)", HERE); From ffe110cc455bc960abc97279f8d3f222c75f574e Mon Sep 17 00:00:00 2001 From: hayk Date: Sun, 15 Jun 2025 16:13:19 -0400 Subject: [PATCH 228/234] (RUNTEST) From d750384b0d1aed8b8028d7673eb528f2e253700d Mon Sep 17 00:00:00 2001 From: haykh Date: Sun, 15 Jun 2025 13:24:09 -0700 Subject: [PATCH 229/234] minor bug in test for plaw dist GR --- src/archetypes/tests/powerlaw.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/archetypes/tests/powerlaw.cpp b/src/archetypes/tests/powerlaw.cpp index 58df1f4cf..3cb76763f 100644 --- a/src/archetypes/tests/powerlaw.cpp +++ b/src/archetypes/tests/powerlaw.cpp @@ -45,10 +45,12 @@ struct Caller { raise::KernelError(HERE, "Gamma out of bounds"); } } else { - vec_t vup { ZERO }; - metric.template transform(xp, vp, vup); + vec_t vd { ZERO }; + vec_t vu { ZERO }; + metric.template transform(xp, vp, vd); + metric.template transform(xp, vp, vu); const auto gamma = math::sqrt( - ONE + vup[0] * vp[0] + vup[1] * vp[1] + vup[2] * vp[2]); + ONE + vu[0] * vd[0] + vu[1] * vd[1] + vu[2] * vd[2]); if (gamma < 10 or gamma > 1000) { raise::KernelError(HERE, "Gamma out of bounds"); } From 9a4d2b62e5b43c144b7bf697c95ac3a6da4fe7b6 Mon Sep 17 00:00:00 2001 From: haykh Date: Sun, 15 Jun 2025 17:23:07 -0400 Subject: [PATCH 230/234] minor nix-shell fix for newest HIP/ROCm --- dev/nix/kokkos.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev/nix/kokkos.nix b/dev/nix/kokkos.nix index 8dfe1f380..e12b9d57d 100644 --- a/dev/nix/kokkos.nix +++ b/dev/nix/kokkos.nix @@ -29,13 +29,15 @@ let getArch = _: if gpu != "NONE" && arch == "NATIVE" then - throw "Please specify an architecture when the GPU support is enabled. Available architectures: https://kokkos.org/kokkos-core-wiki/keywords.html#architectures" + throw "Please specify an architecture when the GPU support is enabled. Available architectures: https://kokkos.org/kokkos-core-wiki/get-started/configuration-guide.html#gpu-architectures" else arch; cmakeExtraFlags = { "HIP" = [ "-D Kokkos_ENABLE_HIP=ON" "-D Kokkos_ARCH_${getArch { }}=ON" + # remove leading AMD_ + "-D AMDGPU_TARGETS=${builtins.replaceStrings [ "amd_" ] [ "" ] (pkgs.lib.toLower (getArch { }))}" "-D CMAKE_C_COMPILER=hipcc" "-D CMAKE_CXX_COMPILER=hipcc" ]; From 1149632ea71afb952dcd9cec32769e96525a6720 Mon Sep 17 00:00:00 2001 From: haykh Date: Sun, 15 Jun 2025 17:23:23 -0400 Subject: [PATCH 231/234] minor bugfix in test for stats --- src/kernels/tests/reduced_stats.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/kernels/tests/reduced_stats.cpp b/src/kernels/tests/reduced_stats.cpp index 499c24aa9..ee036395a 100644 --- a/src/kernels/tests/reduced_stats.cpp +++ b/src/kernels/tests/reduced_stats.cpp @@ -4,13 +4,11 @@ #include "global.h" #include "arch/kokkos_aliases.h" -#include "utils/comparators.h" #include "utils/error.h" #include "metrics/minkowski.h" #include -#include #include using namespace ntt; @@ -18,15 +16,17 @@ using namespace metric; template class Fill_kernel { - ndfield_t& arr; - real_t v; - unsigned short c; + ndfield_t arr; + real_t v; + unsigned short c; public: Fill_kernel(ndfield_t& arr_, real_t v_, unsigned short c_) : arr { arr_ } , v { v_ } - , c { c_ } {} + , c { c_ } { + raise::ErrorIf(c_ >= N, "c > N", HERE); + } Inline void operator()(index_t i1) const { arr(i1, c) = v; From b581ba340a97c8936f0221fd516424b993cfb141 Mon Sep 17 00:00:00 2001 From: haykh Date: Sun, 15 Jun 2025 17:23:44 -0400 Subject: [PATCH 232/234] (RUNTEST) From 9d3a6e7820167a0f30496dc8cd7d484e5e9ff7ca Mon Sep 17 00:00:00 2001 From: haykh Date: Sun, 15 Jun 2025 17:34:52 -0400 Subject: [PATCH 233/234] minor bug in parallel checkpoint test --- src/checkpoint/tests/checkpoint-mpi.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/checkpoint/tests/checkpoint-mpi.cpp b/src/checkpoint/tests/checkpoint-mpi.cpp index bc6d6038a..2372d81bc 100644 --- a/src/checkpoint/tests/checkpoint-mpi.cpp +++ b/src/checkpoint/tests/checkpoint-mpi.cpp @@ -20,7 +20,7 @@ using namespace checkpoint; void cleanup() { namespace fs = std::filesystem; - fs::path temp_path { "checkpoints" }; + fs::path temp_path { "chck" }; fs::remove_all(temp_path); } @@ -126,11 +126,12 @@ auto main(int argc, char* argv[]) -> int { } adios2::ADIOS adios; + const path_t checkpoint_path { "chck" }; { // write checkpoint Writer writer; - writer.init(&adios, 0, 0.0, 1); + writer.init(&adios, checkpoint_path, 0, 0.0, 1); writer.defineFieldVariables(SimEngine::GRPIC, { g_nx1_gh, g_nx2_gh }, @@ -190,7 +191,7 @@ auto main(int argc, char* argv[]) -> int { array_t plds1_read { "plds_1", npart1, 3 }; adios2::IO io = adios.DeclareIO("checkpointRead"); - adios2::Engine reader = io.Open("checkpoints/step-00000000.bp", + adios2::Engine reader = io.Open(checkpoint_path / "step-00000000.bp", adios2::Mode::Read); reader.BeginStep(); From d50161fb407963fcc421569f91bb7d81b1abf2cb Mon Sep 17 00:00:00 2001 From: haykh Date: Sun, 15 Jun 2025 17:53:49 -0400 Subject: [PATCH 234/234] fixed deposit test for dprec (RUNTEST) --- src/kernels/tests/deposit.cpp | 70 +++++++++++++---------------------- 1 file changed, 25 insertions(+), 45 deletions(-) diff --git a/src/kernels/tests/deposit.cpp b/src/kernels/tests/deposit.cpp index 1570937b8..c9acafaed 100644 --- a/src/kernels/tests/deposit.cpp +++ b/src/kernels/tests/deposit.cpp @@ -27,13 +27,16 @@ void errorIf(bool condition, const std::string& message) { } } -inline static constexpr auto epsilon = std::numeric_limits::epsilon(); +const real_t eps = std::is_same_v ? (real_t)(1e-6) + : (real_t)(1e-3); -Inline auto equal(real_t a, real_t b, const char* msg = "", real_t acc = ONE) - -> bool { - const auto eps = epsilon * acc; - if (not cmp::AlmostEqual(a, b, eps)) { +Inline auto equal(real_t a, real_t b, const char* msg, real_t eps) -> bool { + if ((a - b) >= eps * math::max(math::fabs(a), math::fabs(b))) { printf("%.12e != %.12e %s\n", a, b, msg); + printf("%.12e >= %.12e %s\n", + a - b, + eps * math::max(math::fabs(a), math::fabs(b)), + msg); return false; } return true; @@ -49,8 +52,8 @@ void put_value(array_t arr, T value, int i) { template void testDeposit(const std::vector& res, const boundaries_t& ext, - const std::map& params = {}, - const real_t acc = ONE) { + const std::map& params, + const real_t eps) { static_assert(M::Dim == 2); errorIf(res.size() != M::Dim, "res.size() != M::Dim"); using namespace ntt; @@ -158,13 +161,13 @@ void testDeposit(const std::vector& res, if (not cmp::AlmostZero(SumDivJ)) { throw std::logic_error("DepositCurrents_kernel::SumDivJ != 0"); } - errorIf(not equal(J_h(i0 + N_GHOSTS, j0 + N_GHOSTS, cur::jx1), Jx1, "", acc), + errorIf(not equal(J_h(i0 + N_GHOSTS, j0 + N_GHOSTS, cur::jx1), Jx1, "", eps), "DepositCurrents_kernel::Jx1 is incorrect"); - errorIf(not equal(J_h(i0 + N_GHOSTS, j0 + 1 + N_GHOSTS, cur::jx1), Jx2, "", acc), + errorIf(not equal(J_h(i0 + N_GHOSTS, j0 + 1 + N_GHOSTS, cur::jx1), Jx2, "", eps), "DepositCurrents_kernel::Jx2 is incorrect"); - errorIf(not equal(J_h(i0 + N_GHOSTS, j0 + N_GHOSTS, cur::jx2), Jy1, "", acc), + errorIf(not equal(J_h(i0 + N_GHOSTS, j0 + N_GHOSTS, cur::jx2), Jy1, "", eps), "DepositCurrents_kernel::Jy1 is incorrect"); - errorIf(not equal(J_h(i0 + 1 + N_GHOSTS, j0 + N_GHOSTS, cur::jx2), Jy2, "", acc), + errorIf(not equal(J_h(i0 + 1 + N_GHOSTS, j0 + N_GHOSTS, cur::jx2), Jy2, "", eps), "DepositCurrents_kernel::Jy2 is incorrect"); } @@ -183,41 +186,18 @@ auto main(int argc, char* argv[]) -> int { { 0.0, 55.0 }, { 0.0, 55.0 } }; + const std::map params { + { "r0", 0.0 }, + { "h", 0.25 }, + { "a", 0.9 } + }; - testDeposit, SimEngine::SRPIC>(res, xy_extent, {}, 500); - - testDeposit, SimEngine::SRPIC>(res, r_extent, {}, 500); - - testDeposit, SimEngine::SRPIC>(res, - r_extent, - { - { "r0", 0.0 }, - { "h", 0.25 } - }, - 500); - - testDeposit, SimEngine::GRPIC>(res, - r_extent, - { - { "a", 0.9 } - }, - 500); - - testDeposit, SimEngine::GRPIC>(res, - r_extent, - { - { "r0", 0.0 }, - { "h", 0.25 }, - { "a", 0.9 } - }, - 500); - - testDeposit, SimEngine::GRPIC>(res, - r_extent, - { - { "a", 0.9 } - }, - 500); + testDeposit, SimEngine::SRPIC>(res, xy_extent, {}, eps); + testDeposit, SimEngine::SRPIC>(res, r_extent, {}, eps); + testDeposit, SimEngine::SRPIC>(res, r_extent, params, eps); + testDeposit, SimEngine::GRPIC>(res, r_extent, params, eps); + testDeposit, SimEngine::GRPIC>(res, r_extent, params, eps); + testDeposit, SimEngine::GRPIC>(res, r_extent, params, eps); } catch (std::exception& e) { std::cerr << e.what() << std::endl;