From 72244f8b857cc19cad88ed03fa83b6d355f615af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Victor=20R=C3=A9ville?= Date: Tue, 3 Dec 2024 10:57:15 +0100 Subject: [PATCH 01/82] Added new branch with collisionless TC from develop --- src/fluid/braginskii/bragThermalDiffusion.cpp | 2 +- src/fluid/braginskii/bragThermalDiffusion.hpp | 80 ++++++++++++++++--- src/fluid/fluid_defs.hpp | 4 + 3 files changed, 76 insertions(+), 10 deletions(-) diff --git a/src/fluid/braginskii/bragThermalDiffusion.cpp b/src/fluid/braginskii/bragThermalDiffusion.cpp index 82d5dac94..2d340adb0 100644 --- a/src/fluid/braginskii/bragThermalDiffusion.cpp +++ b/src/fluid/braginskii/bragThermalDiffusion.cpp @@ -46,7 +46,7 @@ void BragThermalDiffusion::ShowConfig() { } } -void BragThermalDiffusion::EnrollBragThermalDiffusivity(BragDiffusivityFunc myFunc) { +void BragThermalDiffusion::EnrollBragThermalDiffusivity(FourArrayDiffusivityFunc myFunc) { if(this->status.status != UserDefFunction) { IDEFIX_WARNING("Braginskii thermal diffusivity enrollment requires Hydro/BragThermalDiffusion " "to be set to userdef in .ini file"); diff --git a/src/fluid/braginskii/bragThermalDiffusion.hpp b/src/fluid/braginskii/bragThermalDiffusion.hpp index 1fe774a41..217039994 100644 --- a/src/fluid/braginskii/bragThermalDiffusion.hpp +++ b/src/fluid/braginskii/bragThermalDiffusion.hpp @@ -35,11 +35,13 @@ class BragThermalDiffusion { void AddBragDiffusiveFluxLim(int, const real, const IdefixArray4D &); // Enroll user-defined thermal conductivity - void EnrollBragThermalDiffusivity(BragDiffusivityFunc); + void EnrollBragThermalDiffusivity(FourArrayDiffusivityFunc); IdefixArray3D heatSrc; // Source terms of the thermal operator IdefixArray3D knorArr; IdefixArray3D kparArr; + IdefixArray3D alpha; // Transition from Brag to collisionless tc + IdefixArray3D clessQ; // Collisionless tc flux coefficient // pre-computed geometrical factors in non-cartesian geometry IdefixArray1D one_dmu; @@ -50,7 +52,7 @@ class BragThermalDiffusion { // status of the module ParabolicModuleStatus &status; - BragDiffusivityFunc diffusivityFunc; + FourArrayDiffusivityFunc diffusivityFunc; bool haveSlopeLimiter{false}; @@ -110,6 +112,12 @@ BragThermalDiffusion::BragThermalDiffusion(Input &input, Grid &grid, Fluid this->knorArr = IdefixArray3D("BragThermalDiffusionKnorArray",data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); + this->alpha = IdefixArray3D("ClessThermalDiffusionAlphaArray",data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); + this->clessQ = IdefixArray3D("ClessThermalDiffusionClessQ",data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); } else { IDEFIX_ERROR("Unknown braginskii thermal diffusion definition in idefix.ini. " "Can only be constant or userdef."); @@ -256,7 +264,7 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, if(haveThermalDiffusion == UserDefFunction && dir == IDIR) { if(diffusivityFunc) { idfx::pushRegion("UserDef::BragThermalDiffusivityFunction"); - diffusivityFunc(*this->data, t, kparArr, knorArr); + diffusivityFunc(*this->data, t, kparArr, knorArr, alpha, clessQ); idfx::popRegion(); } else { IDEFIX_ERROR("No user-defined thermal diffusion function has been enrolled"); @@ -274,6 +282,8 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, [[maybe_unused]] real dTi, dTj, dTk, dTn; dTi = dTj = dTk = dTn = ZERO_F; + [[maybe_unused]] real dvp, dvm, dpp, dpm, Pn, Vn; /* For the collisionless / saturated flux */ + real locdmax = 0; /////////////////////////////////////////// // IDIR sweep // @@ -292,9 +302,9 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, kpar = kparConstant; } - EXPAND( Bi = BX_I; , - Bj = BY_I; , - Bk = BZ_I; ) + D_EXPAND( Bi = BX_I; , + Bj = BY_I; , + Bk = BZ_I; ) Bn = BX_I; #if GEOMETRY == CARTESIAN @@ -370,7 +380,9 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, } dTi = D_DX_I_T(Vc)/dx1(i); + #if DIMENSIONS >= 2 + if (haveSlopeLimiter) { dTj = 1./x1(i-1)*SL::PLMLim(SL_DY_T(Vc,k,j,i-1)/dx2(j), SL_DY_T(Vc,k,j+1,i-1)/dx2(j+1)); @@ -399,6 +411,22 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, locdmax = FMAX(kpar,knor)/(0.5*(Vc(RHO,k,j,i) + Vc(RHO,k,j,i - 1)))*gamma_m1; dTn = dTi; + + /* For collisionless / saturated heat flux */ + + dvp = Vc(VX1,k,j,i+1) - Vc(VX1,k,j,i); + dvm = Vc(VX1,k,j,i) - Vc(VX1,k,j,i-1); + + dpp = Vc(PRS,k,j,i+1) - Vc(PRS,k,j,i); + dpm = Vc(PRS,k,j,i) - Vc(PRS,k,j,i-1); + + if(haveSlopeLimiter) { + Vn = Vc(VX1,k,j,i)+0.5*SL::PLMLim(dvp, dvm); + Pn = Vc(PRS,k,j,i)+0.5*SL::PLMLim(dpp, dpm); + } else { + Vn = 0.5*(Vc(VX1,k, j, i) + Vc(VX1,k,j,i+1)); + Pn = 0.5*(Vc(PRS,k, j, i) + Vc(PRS,k,j,i+1)); + } } else if(dir == JDIR) { ////////////// // JDIR @@ -533,6 +561,22 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, locdmax = FMAX(kpar,knor)/(0.5*(Vc(RHO,k,j,i) + Vc(RHO,k,j - 1,i)))*gamma_m1; dTn = dTj; + + /* For the collisionless / saturated flux */ + + dvp = Vc(VX2,k,j+1,i) - Vc(VX2,k,j,i); + dvm = Vc(VX2,k,j,i) - Vc(VX2,k,j-1,i); + + dpp = Vc(PRS,k,j+1,i) - Vc(PRS,k,j,i); + dpm = Vc(PRS,k,j,i) - Vc(PRS,k,j-1,i); + + if(haveSlopeLimiter) { + Vn = Vc(VX2,k,j,i)+0.5*SL::PLMLim(dvp, dvm); + Pn = Vc(PRS,k,j,i)+0.5*SL::PLMLim(dpp, dpm); + } else { + Vn = 0.5*(Vc(VX2,k,j, i) + Vc(VX2,k,j+1,i)); + Pn = 0.5*(Vc(PRS,k,j, i) + Vc(PRS,k,j+1,i)); + } } else if(dir == KDIR) { ////////////// // KDIR @@ -625,11 +669,28 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, locdmax = FMAX(kpar,knor)/(0.5*(Vc(RHO,k,j,i) + Vc(RHO,k-1,j,i)))*gamma_m1; dTn = dTk; + + /* For the collisionless / saturated flux */ + dvp = Vc(VX3,k+1,j,i) - Vc(VX3,k,j,i); + dvm = Vc(VX3,k,j,i) - Vc(VX3,k-1,j,i); + + dpp = Vc(PRS,k+1,j,i) - Vc(PRS,k,j,i); + dpm = Vc(PRS,k,j,i) - Vc(PRS,k-1,j,i); + + if(haveSlopeLimiter) { + Vn = Vc(VX3,k,j,i)+0.5*SL::PLMLim(dvp, dvm); + Pn = Vc(PRS,k,j,i)+0.5*SL::PLMLim(dpp, dpm); + } else { + Vn = 0.5*(Vc(VX3,k,j,i) + Vc(VX3,k+1,j,i)); + Pn = 0.5*(Vc(PRS,k,j,i) + Vc(PRS,k+1,j,i)); + } } // From here, gradients and normal have been computed, so we just need to get the fluxes - bgradT = EXPAND( Bi*dTi , + Bj*dTj, +Bk*dTk); - Bmag = EXPAND( Bi*Bi , + Bj*Bj, + Bk*Bk); + bgradT = D_EXPAND( Bi*dTi , + Bj*dTj, +Bk*dTk); + Bmag = D_EXPAND( Bi*Bi , + Bj*Bj, + Bk*Bk); + //printf("%f , %f\n", Bmag, EXPAND(Bi*Bi, + Bj*Bj, + Bk*Bk)); + //printf("%f , %f \n", 0.5*(Vc(VX1,k,j,i) + Vc(VX1,k,j,i+1)), Vn); Bmag = sqrt(Bmag); Bmag = FMAX(1e-6*SMALL_NUMBER,Bmag); @@ -637,10 +698,11 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, bn = Bn/Bmag; /* -- unit vector component -- */ q = kpar*bgradT*bn + knor*(dTn - bn*bgradT); + q = alpha(k,j,i)*q + (1-alpha(k,j,i))*clessQ(k,j,i)*Pn*Vn; Flux(ENG, k, j, i) -= q; - dMax(k,j,i) = FMAX(dMax(k,j,i),locdmax); + dMax(k,j,i) = FMAX(dMax(k,j,i),locdmax)*alpha(k,j,i); }); idfx::popRegion(); } diff --git a/src/fluid/fluid_defs.hpp b/src/fluid/fluid_defs.hpp index 2269fc803..90e18b876 100644 --- a/src/fluid/fluid_defs.hpp +++ b/src/fluid/fluid_defs.hpp @@ -52,6 +52,10 @@ using DiffusivityFunc = void (*) (DataBlock &, const real t, IdefixArray3D using BragDiffusivityFunc = void (*) (DataBlock &, const real t, IdefixArray3D &, IdefixArray3D &); +using FourArrayDiffusivityFunc = void (*) (DataBlock &, const real t, + IdefixArray3D &, IdefixArray3D &, + IdefixArray3D &, IdefixArray3D &); + // Deprecated signatures using SrcTermFuncOld = void (*) (DataBlock &, const real t, const real dt); From cb3bea853eff493db3d5b03f415c0aa98048c90b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Victor=20R=C3=A9ville?= Date: Thu, 12 Dec 2024 16:29:24 +0100 Subject: [PATCH 02/82] Fixed undeclared CL arrays (GPU compiling) --- reference | 2 +- src/fluid/braginskii/bragThermalDiffusion.hpp | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/reference b/reference index b675bceaa..609a60164 160000 --- a/reference +++ b/reference @@ -1 +1 @@ -Subproject commit b675bceaa6aabc01dded346e2d631857f698dc76 +Subproject commit 609a60164461fb2d4fbeffa89ce148acf1191525 diff --git a/src/fluid/braginskii/bragThermalDiffusion.hpp b/src/fluid/braginskii/bragThermalDiffusion.hpp index 217039994..187209ee2 100644 --- a/src/fluid/braginskii/bragThermalDiffusion.hpp +++ b/src/fluid/braginskii/bragThermalDiffusion.hpp @@ -260,6 +260,8 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, real kparConstant = this->kpar; IdefixArray3D knorArr = this->knorArr; IdefixArray3D kparArr = this->kparArr; + IdefixArray3D alpha = this->alpha; + IdefixArray3D clessQ = this->clessQ; if(haveThermalDiffusion == UserDefFunction && dir == IDIR) { if(diffusivityFunc) { From 6ec6d0db33c0f1c6ac2637fa400c832ec8a83640 Mon Sep 17 00:00:00 2001 From: Hal Bal Date: Sat, 14 Dec 2024 10:18:43 +0100 Subject: [PATCH 03/82] collisionless tc as an option of bragTDiffusion --- src/fluid/braginskii/bragThermalDiffusion.cpp | 22 ++- src/fluid/braginskii/bragThermalDiffusion.hpp | 155 +++++++++++------- src/fluid/fluid_defs.hpp | 10 +- 3 files changed, 116 insertions(+), 71 deletions(-) diff --git a/src/fluid/braginskii/bragThermalDiffusion.cpp b/src/fluid/braginskii/bragThermalDiffusion.cpp index 2d340adb0..43477c9d6 100644 --- a/src/fluid/braginskii/bragThermalDiffusion.cpp +++ b/src/fluid/braginskii/bragThermalDiffusion.cpp @@ -27,11 +27,17 @@ void BragThermalDiffusion::ShowConfig() { } else if (status.status==UserDefFunction) { idfx::cout << "Braginskii Thermal Diffusion: ENABLED with user-defined diffusivity function." << std::endl; - if(!diffusivityFunc) { + if(!bragDiffusivityFunc) { IDEFIX_ERROR("No braginskii thermal diffusion function has been enrolled"); } + } else if (status.status==CollisionLess) { + idfx::cout << "CollisionLess / Braginskii Thermal Diffusion: ENABLED with user-defined " + "diffusivity function."<< std::endl; + if(!clessDiffusivityFunc) { + IDEFIX_ERROR("No collisionless / Braginskii thermal diffusion function has been enrolled"); + } } else { - IDEFIX_ERROR("Unknown braginskii thermal diffusion mode"); + IDEFIX_ERROR("Unknown Braginskii thermal diffusion mode"); } if(status.isExplicit) { idfx::cout << "Braginskii Thermal Diffusion: uses an explicit time integration." << std::endl; @@ -46,12 +52,20 @@ void BragThermalDiffusion::ShowConfig() { } } -void BragThermalDiffusion::EnrollBragThermalDiffusivity(FourArrayDiffusivityFunc myFunc) { +void BragThermalDiffusion::EnrollBragThermalDiffusivity(TwoArrayDiffusivityFunc myFunc) { if(this->status.status != UserDefFunction) { IDEFIX_WARNING("Braginskii thermal diffusivity enrollment requires Hydro/BragThermalDiffusion " "to be set to userdef in .ini file"); } - this->diffusivityFunc = myFunc; + this->bragDiffusivityFunc = myFunc; +} + +void BragThermalDiffusion::EnrollClessThermalDiffusivity(FourArrayDiffusivityFunc myFunc) { + if(this->status.status != UserDefFunction) { + IDEFIX_WARNING("Collisionless/Braginskii thermal diffusivity enrollment requires " + "Hydro/BragThermalDiffusion to be set to collisionless in .ini file"); + } + this->clessDiffusivityFunc = myFunc; } void BragThermalDiffusion::AddBragDiffusiveFlux(int dir, const real t, diff --git a/src/fluid/braginskii/bragThermalDiffusion.hpp b/src/fluid/braginskii/bragThermalDiffusion.hpp index 187209ee2..8cf892e5a 100644 --- a/src/fluid/braginskii/bragThermalDiffusion.hpp +++ b/src/fluid/braginskii/bragThermalDiffusion.hpp @@ -35,13 +35,14 @@ class BragThermalDiffusion { void AddBragDiffusiveFluxLim(int, const real, const IdefixArray4D &); // Enroll user-defined thermal conductivity - void EnrollBragThermalDiffusivity(FourArrayDiffusivityFunc); + void EnrollBragThermalDiffusivity(TwoArrayDiffusivityFunc); + void EnrollClessThermalDiffusivity(FourArrayDiffusivityFunc); IdefixArray3D heatSrc; // Source terms of the thermal operator IdefixArray3D knorArr; IdefixArray3D kparArr; - IdefixArray3D alpha; // Transition from Brag to collisionless tc - IdefixArray3D clessQ; // Collisionless tc flux coefficient + IdefixArray3D clessAlpha; // Transition from Brag to collisionless tc + IdefixArray3D clessBeta; // Collisionless tc flux coefficient // pre-computed geometrical factors in non-cartesian geometry IdefixArray1D one_dmu; @@ -52,7 +53,10 @@ class BragThermalDiffusion { // status of the module ParabolicModuleStatus &status; - FourArrayDiffusivityFunc diffusivityFunc; + TwoArrayDiffusivityFunc bragDiffusivityFunc; + FourArrayDiffusivityFunc clessDiffusivityFunc; + + bool includeCollisionlessTD{false}; bool haveSlopeLimiter{false}; @@ -107,17 +111,26 @@ BragThermalDiffusion::BragThermalDiffusion(Input &input, Grid &grid, Fluid } else if(input.Get("Hydro","bragTDiffusion",2).compare("userdef") == 0) { this->status.status = UserDefFunction; this->kparArr = IdefixArray3D("BragThermalDiffusionKparArray",data->np_tot[KDIR], - data->np_tot[JDIR], - data->np_tot[IDIR]); + data->np_tot[JDIR], + data->np_tot[IDIR]); + this->knorArr = IdefixArray3D("BragThermalDiffusionKnorArray",data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); + } else if(input.Get("Hydro","bragTDiffusion",2).compare("collisionless") == 0) { + this->includeCollisionlessTD = true; + this->status.status = CollisionLess; + this->kparArr = IdefixArray3D("BragThermalDiffusionKparArray",data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); this->knorArr = IdefixArray3D("BragThermalDiffusionKnorArray",data->np_tot[KDIR], - data->np_tot[JDIR], - data->np_tot[IDIR]); - this->alpha = IdefixArray3D("ClessThermalDiffusionAlphaArray",data->np_tot[KDIR], - data->np_tot[JDIR], - data->np_tot[IDIR]); - this->clessQ = IdefixArray3D("ClessThermalDiffusionClessQ",data->np_tot[KDIR], - data->np_tot[JDIR], - data->np_tot[IDIR]); + data->np_tot[JDIR], + data->np_tot[IDIR]); + this->clessAlpha = IdefixArray3D("ClessThermalDiffusionAlphaArray",data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); + this->clessBeta = IdefixArray3D("ClessThermalDiffusionBetaArray",data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); } else { IDEFIX_ERROR("Unknown braginskii thermal diffusion definition in idefix.ini. " "Can only be constant or userdef."); @@ -218,6 +231,7 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, HydroModuleStatus haveThermalDiffusion = this->status.status; bool haveSlopeLimiter = this->haveSlopeLimiter; + bool includeCollisionlessTD = this->includeCollisionlessTD; using SL = SlopeLimiter; int ibeg, iend, jbeg, jend, kbeg, kend; @@ -260,17 +274,26 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, real kparConstant = this->kpar; IdefixArray3D knorArr = this->knorArr; IdefixArray3D kparArr = this->kparArr; - IdefixArray3D alpha = this->alpha; - IdefixArray3D clessQ = this->clessQ; + IdefixArray3D clessAlpha = this->clessAlpha; + IdefixArray3D clessBeta = this->clessBeta; if(haveThermalDiffusion == UserDefFunction && dir == IDIR) { - if(diffusivityFunc) { + if(bragDiffusivityFunc) { idfx::pushRegion("UserDef::BragThermalDiffusivityFunction"); - diffusivityFunc(*this->data, t, kparArr, knorArr, alpha, clessQ); + bragDiffusivityFunc(*this->data, t, kparArr, knorArr); idfx::popRegion(); - } else { + } + else { IDEFIX_ERROR("No user-defined thermal diffusion function has been enrolled"); } + } else if(haveThermalDiffusion == CollisionLess && dir == IDIR) { + if (clessDiffusivityFunc) { + idfx::pushRegion("UserDef::ClessThermalDiffusivityFunction"); + clessDiffusivityFunc(*this->data, t, kparArr, knorArr, clessAlpha, clessBeta); + idfx::popRegion(); + } else { + IDEFIX_ERROR("No user-defined collisionless thermal diffusion function has been enrolled"); + } } idefix_for("BragDiffusiveFlux",kbeg, kend, jbeg, jend, ibeg, iend, @@ -292,7 +315,7 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, /////////////////////////////////////////// if(dir == IDIR) { - if(haveThermalDiffusion == UserDefFunction) { + if(haveThermalDiffusion == UserDefFunction || haveThermalDiffusion == CollisionLess) { knor = HALF_F*(knorArr(k,j,i-1)+knorArr(k,j,i)); if(haveSlopeLimiter) { kpar = 2.*(kparArr(k,j,i-1)*kparArr(k,j,i))/(kparArr(k,j,i-1)+kparArr(k,j,i)); @@ -415,26 +438,27 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, dTn = dTi; /* For collisionless / saturated heat flux */ - - dvp = Vc(VX1,k,j,i+1) - Vc(VX1,k,j,i); - dvm = Vc(VX1,k,j,i) - Vc(VX1,k,j,i-1); - - dpp = Vc(PRS,k,j,i+1) - Vc(PRS,k,j,i); - dpm = Vc(PRS,k,j,i) - Vc(PRS,k,j,i-1); - - if(haveSlopeLimiter) { - Vn = Vc(VX1,k,j,i)+0.5*SL::PLMLim(dvp, dvm); - Pn = Vc(PRS,k,j,i)+0.5*SL::PLMLim(dpp, dpm); - } else { - Vn = 0.5*(Vc(VX1,k, j, i) + Vc(VX1,k,j,i+1)); - Pn = 0.5*(Vc(PRS,k, j, i) + Vc(PRS,k,j,i+1)); + if (includeCollisionlessTD) { + dvp = Vc(VX1,k,j,i+1) - Vc(VX1,k,j,i); + dvm = Vc(VX1,k,j,i) - Vc(VX1,k,j,i-1); + + dpp = Vc(PRS,k,j,i+1) - Vc(PRS,k,j,i); + dpm = Vc(PRS,k,j,i) - Vc(PRS,k,j,i-1); + + if(haveSlopeLimiter) { + Vn = Vc(VX1,k,j,i)+0.5*SL::PLMLim(dvp, dvm); + Pn = Vc(PRS,k,j,i)+0.5*SL::PLMLim(dpp, dpm); + } else { + Vn = 0.5*(Vc(VX1,k, j, i) + Vc(VX1,k,j,i+1)); + Pn = 0.5*(Vc(PRS,k, j, i) + Vc(PRS,k,j,i+1)); + } } } else if(dir == JDIR) { ////////////// // JDIR ///////////// - if(haveThermalDiffusion == UserDefFunction) { + if(haveThermalDiffusion == UserDefFunction || haveThermalDiffusion == CollisionLess) { knor = HALF_F*(knorArr(k,j-1,i)+knorArr(k,j,i)); if(haveSlopeLimiter) { kpar = 2.*(kparArr(k,j-1,i)*kparArr(k,j,i))/(kparArr(k,j-1,i)+kparArr(k,j,i)); @@ -565,25 +589,26 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, dTn = dTj; /* For the collisionless / saturated flux */ - - dvp = Vc(VX2,k,j+1,i) - Vc(VX2,k,j,i); - dvm = Vc(VX2,k,j,i) - Vc(VX2,k,j-1,i); - - dpp = Vc(PRS,k,j+1,i) - Vc(PRS,k,j,i); - dpm = Vc(PRS,k,j,i) - Vc(PRS,k,j-1,i); - - if(haveSlopeLimiter) { - Vn = Vc(VX2,k,j,i)+0.5*SL::PLMLim(dvp, dvm); - Pn = Vc(PRS,k,j,i)+0.5*SL::PLMLim(dpp, dpm); - } else { - Vn = 0.5*(Vc(VX2,k,j, i) + Vc(VX2,k,j+1,i)); - Pn = 0.5*(Vc(PRS,k,j, i) + Vc(PRS,k,j+1,i)); - } + if(includeCollisionlessTD) { + dvp = Vc(VX2,k,j+1,i) - Vc(VX2,k,j,i); + dvm = Vc(VX2,k,j,i) - Vc(VX2,k,j-1,i); + + dpp = Vc(PRS,k,j+1,i) - Vc(PRS,k,j,i); + dpm = Vc(PRS,k,j,i) - Vc(PRS,k,j-1,i); + + if(haveSlopeLimiter) { + Vn = Vc(VX2,k,j,i)+0.5*SL::PLMLim(dvp, dvm); + Pn = Vc(PRS,k,j,i)+0.5*SL::PLMLim(dpp, dpm); + } else { + Vn = 0.5*(Vc(VX2,k,j, i) + Vc(VX2,k,j+1,i)); + Pn = 0.5*(Vc(PRS,k,j, i) + Vc(PRS,k,j+1,i)); + } + } } else if(dir == KDIR) { ////////////// // KDIR ///////////// - if(haveThermalDiffusion == UserDefFunction) { + if(haveThermalDiffusion == UserDefFunction || haveThermalDiffusion == CollisionLess) { knor = HALF_F*(knorArr(k-1,j,i)+knorArr(k,j,i)); if(haveSlopeLimiter) { kpar = 2.*(kparArr(k-1,j,i)*kparArr(k,j,i))/(kparArr(k-1,j,i)+kparArr(k,j,i)); @@ -673,18 +698,20 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, dTn = dTk; /* For the collisionless / saturated flux */ - dvp = Vc(VX3,k+1,j,i) - Vc(VX3,k,j,i); - dvm = Vc(VX3,k,j,i) - Vc(VX3,k-1,j,i); + if(includeCollisionlessTD) { + dvp = Vc(VX3,k+1,j,i) - Vc(VX3,k,j,i); + dvm = Vc(VX3,k,j,i) - Vc(VX3,k-1,j,i); - dpp = Vc(PRS,k+1,j,i) - Vc(PRS,k,j,i); - dpm = Vc(PRS,k,j,i) - Vc(PRS,k-1,j,i); + dpp = Vc(PRS,k+1,j,i) - Vc(PRS,k,j,i); + dpm = Vc(PRS,k,j,i) - Vc(PRS,k-1,j,i); - if(haveSlopeLimiter) { - Vn = Vc(VX3,k,j,i)+0.5*SL::PLMLim(dvp, dvm); - Pn = Vc(PRS,k,j,i)+0.5*SL::PLMLim(dpp, dpm); - } else { - Vn = 0.5*(Vc(VX3,k,j,i) + Vc(VX3,k+1,j,i)); - Pn = 0.5*(Vc(PRS,k,j,i) + Vc(PRS,k+1,j,i)); + if(haveSlopeLimiter) { + Vn = Vc(VX3,k,j,i)+0.5*SL::PLMLim(dvp, dvm); + Pn = Vc(PRS,k,j,i)+0.5*SL::PLMLim(dpp, dpm); + } else { + Vn = 0.5*(Vc(VX3,k,j,i) + Vc(VX3,k+1,j,i)); + Pn = 0.5*(Vc(PRS,k,j,i) + Vc(PRS,k+1,j,i)); + } } } // From here, gradients and normal have been computed, so we just need to get the fluxes @@ -700,11 +727,15 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, bn = Bn/Bmag; /* -- unit vector component -- */ q = kpar*bgradT*bn + knor*(dTn - bn*bgradT); - q = alpha(k,j,i)*q + (1-alpha(k,j,i))*clessQ(k,j,i)*Pn*Vn; - + if(includeCollisionlessTD) { + q = clessAlpha(k,j,i)*q + (1-clessAlpha(k,j,i))*clessBeta(k,j,i)*Pn*Vn; + } Flux(ENG, k, j, i) -= q; + dMax(k,j,i) = FMAX(dMax(k,j,i),locdmax); - dMax(k,j,i) = FMAX(dMax(k,j,i),locdmax)*alpha(k,j,i); + if(includeCollisionlessTD) { + dMax(k,j,i) *= clessAlpha(k,j,i); + } }); idfx::popRegion(); } diff --git a/src/fluid/fluid_defs.hpp b/src/fluid/fluid_defs.hpp index 90e18b876..62a2099c4 100644 --- a/src/fluid/fluid_defs.hpp +++ b/src/fluid/fluid_defs.hpp @@ -25,7 +25,7 @@ class Fluid; // Parabolic terms can have different status -enum HydroModuleStatus {Disabled, Constant, UserDefFunction }; +enum HydroModuleStatus {Disabled, Constant, UserDefFunction, CollisionLess}; // Structure to describe the status of parabolic modules struct ParabolicModuleStatus { @@ -49,12 +49,12 @@ using SrcTermFunc = void (*) (Fluid *, const real t, const real dt); using EmfBoundaryFunc = void (*) (DataBlock &, const real t); using DiffusivityFunc = void (*) (DataBlock &, const real t, IdefixArray3D &); -using BragDiffusivityFunc = void (*) (DataBlock &, const real t, - IdefixArray3D &, IdefixArray3D &); +using TwoArrayDiffusivityFunc = void (*) (DataBlock &, const real t, + IdefixArray3D &, IdefixArray3D &); using FourArrayDiffusivityFunc = void (*) (DataBlock &, const real t, - IdefixArray3D &, IdefixArray3D &, - IdefixArray3D &, IdefixArray3D &); + IdefixArray3D &, IdefixArray3D &, + IdefixArray3D &, IdefixArray3D &); // Deprecated signatures using SrcTermFuncOld = void (*) (DataBlock &, const real t, const real dt); From 9ac669865703c68324b6ef35999cc786deaaf8c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Victor=20R=C3=A9ville?= Date: Mon, 16 Dec 2024 10:32:16 +0100 Subject: [PATCH 04/82] idefix error includes collisionless status --- src/fluid/braginskii/bragThermalDiffusion.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/fluid/braginskii/bragThermalDiffusion.hpp b/src/fluid/braginskii/bragThermalDiffusion.hpp index 8cf892e5a..ca1cf115d 100644 --- a/src/fluid/braginskii/bragThermalDiffusion.hpp +++ b/src/fluid/braginskii/bragThermalDiffusion.hpp @@ -133,7 +133,7 @@ BragThermalDiffusion::BragThermalDiffusion(Input &input, Grid &grid, Fluid data->np_tot[IDIR]); } else { IDEFIX_ERROR("Unknown braginskii thermal diffusion definition in idefix.ini. " - "Can only be constant or userdef."); + "Can only be constant, userdef or collisionless."); } } else { IDEFIX_ERROR("I cannot create a BragThermalDiffusion object without bragTDiffusion defined" @@ -728,13 +728,13 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, bn = Bn/Bmag; /* -- unit vector component -- */ q = kpar*bgradT*bn + knor*(dTn - bn*bgradT); if(includeCollisionlessTD) { - q = clessAlpha(k,j,i)*q + (1-clessAlpha(k,j,i))*clessBeta(k,j,i)*Pn*Vn; + q = clessAlpha(k,j,i)*q + (1-clessAlpha(k,j,i))*clessBeta(k,j,i)*Pn*Vn; } Flux(ENG, k, j, i) -= q; dMax(k,j,i) = FMAX(dMax(k,j,i),locdmax); if(includeCollisionlessTD) { - dMax(k,j,i) *= clessAlpha(k,j,i); + dMax(k,j,i) *= clessAlpha(k,j,i); } }); idfx::popRegion(); From 94a71d229c565fe21be263c4f06d65b4555477ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Victor=20R=C3=A9ville?= Date: Fri, 20 Dec 2024 15:58:55 +0100 Subject: [PATCH 05/82] Added a saturation option for collisionless formulation --- src/fluid/braginskii/bragThermalDiffusion.cpp | 15 +- src/fluid/braginskii/bragThermalDiffusion.hpp | 164 +++++++++++------- src/fluid/fluid_defs.hpp | 2 +- 3 files changed, 113 insertions(+), 68 deletions(-) diff --git a/src/fluid/braginskii/bragThermalDiffusion.cpp b/src/fluid/braginskii/bragThermalDiffusion.cpp index 43477c9d6..ee1d5b996 100644 --- a/src/fluid/braginskii/bragThermalDiffusion.cpp +++ b/src/fluid/braginskii/bragThermalDiffusion.cpp @@ -27,15 +27,9 @@ void BragThermalDiffusion::ShowConfig() { } else if (status.status==UserDefFunction) { idfx::cout << "Braginskii Thermal Diffusion: ENABLED with user-defined diffusivity function." << std::endl; - if(!bragDiffusivityFunc) { + if(!bragDiffusivityFunc && !clessDiffusivityFunc) { IDEFIX_ERROR("No braginskii thermal diffusion function has been enrolled"); } - } else if (status.status==CollisionLess) { - idfx::cout << "CollisionLess / Braginskii Thermal Diffusion: ENABLED with user-defined " - "diffusivity function."<< std::endl; - if(!clessDiffusivityFunc) { - IDEFIX_ERROR("No collisionless / Braginskii thermal diffusion function has been enrolled"); - } } else { IDEFIX_ERROR("Unknown Braginskii thermal diffusion mode"); } @@ -50,12 +44,15 @@ void BragThermalDiffusion::ShowConfig() { if(haveSlopeLimiter) { idfx::cout << "Braginskii Thermal Diffusion: uses a slope limiter." << std::endl; } + if(includeCollisionlessTD) { + idfx::cout << "Saturation with collisionless flux is enabled." << std::endl; + } } void BragThermalDiffusion::EnrollBragThermalDiffusivity(TwoArrayDiffusivityFunc myFunc) { if(this->status.status != UserDefFunction) { IDEFIX_WARNING("Braginskii thermal diffusivity enrollment requires Hydro/BragThermalDiffusion " - "to be set to userdef in .ini file"); + "to be set to userdef in .ini file"); } this->bragDiffusivityFunc = myFunc; } @@ -63,7 +60,7 @@ void BragThermalDiffusion::EnrollBragThermalDiffusivity(TwoArrayDiffusivityFunc void BragThermalDiffusion::EnrollClessThermalDiffusivity(FourArrayDiffusivityFunc myFunc) { if(this->status.status != UserDefFunction) { IDEFIX_WARNING("Collisionless/Braginskii thermal diffusivity enrollment requires " - "Hydro/BragThermalDiffusion to be set to collisionless in .ini file"); + "Hydro/BragThermalDiffusion to be set to userdef in .ini file"); } this->clessDiffusivityFunc = myFunc; } diff --git a/src/fluid/braginskii/bragThermalDiffusion.hpp b/src/fluid/braginskii/bragThermalDiffusion.hpp index ca1cf115d..db01497b7 100644 --- a/src/fluid/braginskii/bragThermalDiffusion.hpp +++ b/src/fluid/braginskii/bragThermalDiffusion.hpp @@ -41,8 +41,8 @@ class BragThermalDiffusion { IdefixArray3D heatSrc; // Source terms of the thermal operator IdefixArray3D knorArr; IdefixArray3D kparArr; - IdefixArray3D clessAlpha; // Transition from Brag to collisionless tc - IdefixArray3D clessBeta; // Collisionless tc flux coefficient + IdefixArray3D clessAlphaArr; // Transition from Brag to collisionless tc + IdefixArray3D clessBetaArr; // Collisionless tc flux coefficient (before pv) // pre-computed geometrical factors in non-cartesian geometry IdefixArray1D one_dmu; @@ -67,6 +67,7 @@ class BragThermalDiffusion { // constant diffusion coefficient (when needed) real knor, kpar; + real clessAlpha, clessBeta; // equation of state (required to get the heat capacity) EquationOfState *eos; @@ -104,36 +105,55 @@ BragThermalDiffusion::BragThermalDiffusion(Input &input, Grid &grid, Fluid IDEFIX_ERROR("Unknown braginskii thermal diffusion limiter in idefix.ini. " "Can only be vanleer, mc or nolimiter."); } - if(input.Get("Hydro","bragTDiffusion",2).compare("constant") == 0) { - this->kpar = input.Get("Hydro","bragTDiffusion",3); - this->knor = input.GetOrSet("Hydro","bragTDiffusion",4,0.); - this->status.status = Constant; - } else if(input.Get("Hydro","bragTDiffusion",2).compare("userdef") == 0) { - this->status.status = UserDefFunction; - this->kparArr = IdefixArray3D("BragThermalDiffusionKparArray",data->np_tot[KDIR], - data->np_tot[JDIR], - data->np_tot[IDIR]); - this->knorArr = IdefixArray3D("BragThermalDiffusionKnorArray",data->np_tot[KDIR], - data->np_tot[JDIR], - data->np_tot[IDIR]); - } else if(input.Get("Hydro","bragTDiffusion",2).compare("collisionless") == 0) { - this->includeCollisionlessTD = true; - this->status.status = CollisionLess; - this->kparArr = IdefixArray3D("BragThermalDiffusionKparArray",data->np_tot[KDIR], - data->np_tot[JDIR], - data->np_tot[IDIR]); - this->knorArr = IdefixArray3D("BragThermalDiffusionKnorArray",data->np_tot[KDIR], - data->np_tot[JDIR], - data->np_tot[IDIR]); - this->clessAlpha = IdefixArray3D("ClessThermalDiffusionAlphaArray",data->np_tot[KDIR], - data->np_tot[JDIR], - data->np_tot[IDIR]); - this->clessBeta = IdefixArray3D("ClessThermalDiffusionBetaArray",data->np_tot[KDIR], - data->np_tot[JDIR], - data->np_tot[IDIR]); + if(input.Get("Hydro","bragTDiffusion",2).compare("nosat") == 0) { + if(input.Get("Hydro","bragTDiffusion",3).compare("constant") == 0) { + this->kpar = input.Get("Hydro","bragTDiffusion",4); + this->knor = input.GetOrSet("Hydro","bragTDiffusion",5,0.); + this->status.status = Constant; + } else if(input.Get("Hydro","bragTDiffusion",3).compare("userdef") == 0) { + this->status.status = UserDefFunction; + this->kparArr = IdefixArray3D("BragThermalDiffusionKparArray",data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); + this->knorArr = IdefixArray3D("BragThermalDiffusionKnorArray",data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); + } else { + IDEFIX_ERROR("Unknown braginskii thermal diffusion definition in idefix.ini. " + "Can only be constant or userdef."); + } + } else if(input.Get("Hydro","bragTDiffusion",2).compare("wcollisionless") == 0.0) { + if(input.Get("Hydro","bragTDiffusion",3).compare("constant") == 0) { + this->includeCollisionlessTD = true; + this->kpar = input.Get("Hydro","bragTDiffusion",4); + this->knor = input.GetOrSet("Hydro","bragTDiffusion",5,0.); + this->clessAlpha = input.GetOrSet("Hydro","bragTDiffusion",6,1.); + this->clessBeta = input.GetOrSet("Hydro","bragTDiffusion",7,0.); + this->status.status = Constant; + } else if(input.Get("Hydro","bragTDiffusion",3).compare("userdef") == 0) { + this->includeCollisionlessTD = true; + this->status.status = UserDefFunction; + this->kparArr = IdefixArray3D("BragThermalDiffusionKparArray",data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); + this->knorArr = IdefixArray3D("BragThermalDiffusionKnorArray",data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); + this->clessAlphaArr = IdefixArray3D("ClessThermalDiffusionAlphaArray", + data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); + this->clessBetaArr = IdefixArray3D("ClessThermalDiffusionBetaArray", + data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); + } else { + IDEFIX_ERROR("Unknown braginskii/collsiionless thermal diffusion definition in idefix.ini. " + "Can only be constant or userdef."); + } } else { - IDEFIX_ERROR("Unknown braginskii thermal diffusion definition in idefix.ini. " - "Can only be constant, userdef or collisionless."); + IDEFIX_ERROR("Unknown braginskii thermal diffusion saturation in idefix.ini. " + "Can only be nosat or wcollisionless."); } } else { IDEFIX_ERROR("I cannot create a BragThermalDiffusion object without bragTDiffusion defined" @@ -156,7 +176,7 @@ BragThermalDiffusion::BragThermalDiffusion(Input &input, Grid &grid, Fluid // and therefore not available as such in the DataBlock. // It is rather defined as PRS/RHO. // Special spatial derivative macros are therefore needed and defined here -// directly at the right cell interface according to the direciton of the flux. +// directly at the right cell interface according to the direction of the flux. #define D_DX_I_T(q) (q(PRS,k,j,i)/q(RHO,k,j,i) - q(PRS,k,j,i - 1)/q(RHO,k,j,i - 1)) #define D_DY_J_T(q) (q(PRS,k,j,i)/q(RHO,k,j,i) - q(PRS,k,j - 1,i)/q(RHO,k,j - 1,i)) #define D_DZ_K_T(q) (q(PRS,k,j,i)/q(RHO,k,j,i) - q(PRS,k - 1,j,i)/q(RHO,k - 1,j,i)) @@ -272,27 +292,31 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, #endif real knorConstant = this->knor; real kparConstant = this->kpar; + real clessAlphaConst = this->clessAlpha; + real clessBetaConst = this->clessBeta; IdefixArray3D knorArr = this->knorArr; IdefixArray3D kparArr = this->kparArr; - IdefixArray3D clessAlpha = this->clessAlpha; - IdefixArray3D clessBeta = this->clessBeta; + IdefixArray3D clessAlphaArr = this->clessAlphaArr; + IdefixArray3D clessBetaArr = this->clessBetaArr; - if(haveThermalDiffusion == UserDefFunction && dir == IDIR) { + if(includeCollisionlessTD == false && haveThermalDiffusion == UserDefFunction && dir == IDIR) { if(bragDiffusivityFunc) { idfx::pushRegion("UserDef::BragThermalDiffusivityFunction"); bragDiffusivityFunc(*this->data, t, kparArr, knorArr); idfx::popRegion(); } else { - IDEFIX_ERROR("No user-defined thermal diffusion function has been enrolled"); + IDEFIX_ERROR("No user-defined Braginskii thermal diffusion function has been enrolled"); } - } else if(haveThermalDiffusion == CollisionLess && dir == IDIR) { + } else if(includeCollisionlessTD == true && haveThermalDiffusion == UserDefFunction + && dir == IDIR) { if (clessDiffusivityFunc) { idfx::pushRegion("UserDef::ClessThermalDiffusivityFunction"); - clessDiffusivityFunc(*this->data, t, kparArr, knorArr, clessAlpha, clessBeta); + clessDiffusivityFunc(*this->data, t, kparArr, knorArr, clessAlphaArr, clessBetaArr); idfx::popRegion(); } else { - IDEFIX_ERROR("No user-defined collisionless thermal diffusion function has been enrolled"); + IDEFIX_ERROR("No user-defined Braginskii/collisionless " + "thermal diffusion function has been enrolled"); } } @@ -306,8 +330,8 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, [[maybe_unused]] real dTi, dTj, dTk, dTn; dTi = dTj = dTk = dTn = ZERO_F; - - [[maybe_unused]] real dvp, dvm, dpp, dpm, Pn, Vn; /* For the collisionless / saturated flux */ + /* For the collisionless / saturated flux */ + [[maybe_unused]] real clessAlpha, clessBeta, dvp, dvm, dpp, dpm, Pn, Vn; real locdmax = 0; /////////////////////////////////////////// @@ -315,16 +339,24 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, /////////////////////////////////////////// if(dir == IDIR) { - if(haveThermalDiffusion == UserDefFunction || haveThermalDiffusion == CollisionLess) { + if(haveThermalDiffusion == UserDefFunction) { knor = HALF_F*(knorArr(k,j,i-1)+knorArr(k,j,i)); if(haveSlopeLimiter) { kpar = 2.*(kparArr(k,j,i-1)*kparArr(k,j,i))/(kparArr(k,j,i-1)+kparArr(k,j,i)); } else { kpar = HALF_F*(kparArr(k,j,i-1)+kparArr(k,j,i)); } + if(includeCollisionlessTD) { + clessAlpha=HALF_F*(clessAlphaArr(k,j,i-1)+clessAlphaArr(k,j,i)); + clessBeta=HALF_F*(clessBetaArr(k,j,i-1)+clessAlphaArr(k,j,i)); + } } else { knor = knorConstant; kpar = kparConstant; + if(includeCollisionlessTD) { + clessAlpha=clessAlphaConst; + clessBeta=clessBetaConst; + } } D_EXPAND( Bi = BX_I; , @@ -446,11 +478,11 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, dpm = Vc(PRS,k,j,i) - Vc(PRS,k,j,i-1); if(haveSlopeLimiter) { - Vn = Vc(VX1,k,j,i)+0.5*SL::PLMLim(dvp, dvm); - Pn = Vc(PRS,k,j,i)+0.5*SL::PLMLim(dpp, dpm); + Vn = Vc(VX1,k,j,i)-HALF_F*SL::PLMLim(dvp, dvm); + Pn = Vc(PRS,k,j,i)-HALF_F*SL::PLMLim(dpp, dpm); } else { - Vn = 0.5*(Vc(VX1,k, j, i) + Vc(VX1,k,j,i+1)); - Pn = 0.5*(Vc(PRS,k, j, i) + Vc(PRS,k,j,i+1)); + Vn = HALF_F*(Vc(VX1,k,j,i) + Vc(VX1,k,j,i-1)); + Pn = HALF_F*(Vc(PRS,k,j,i) + Vc(PRS,k,j,i-1)); } } } else if(dir == JDIR) { @@ -458,16 +490,24 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, // JDIR ///////////// - if(haveThermalDiffusion == UserDefFunction || haveThermalDiffusion == CollisionLess) { + if(haveThermalDiffusion == UserDefFunction) { knor = HALF_F*(knorArr(k,j-1,i)+knorArr(k,j,i)); if(haveSlopeLimiter) { kpar = 2.*(kparArr(k,j-1,i)*kparArr(k,j,i))/(kparArr(k,j-1,i)+kparArr(k,j,i)); } else { kpar = HALF_F*(kparArr(k,j-1,i)+kparArr(k,j,i)); } + if(includeCollisionlessTD) { + clessAlpha=HALF_F*(clessAlphaArr(k,j-1,i)+clessAlphaArr(k,j,i)); + clessBeta=HALF_F*(clessBetaArr(k,j-1,i)+clessAlphaArr(k,j,i)); + } } else { knor = knorConstant; kpar = kparConstant; + if(includeCollisionlessTD) { + clessAlpha=clessAlphaConst; + clessBeta=clessBetaConst; + } } EXPAND( Bi = BX_J; , @@ -597,27 +637,35 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, dpm = Vc(PRS,k,j,i) - Vc(PRS,k,j-1,i); if(haveSlopeLimiter) { - Vn = Vc(VX2,k,j,i)+0.5*SL::PLMLim(dvp, dvm); - Pn = Vc(PRS,k,j,i)+0.5*SL::PLMLim(dpp, dpm); + Vn = Vc(VX2,k,j,i)-HALF_F*SL::PLMLim(dvp, dvm); + Pn = Vc(PRS,k,j,i)-HALF_F*SL::PLMLim(dpp, dpm); } else { - Vn = 0.5*(Vc(VX2,k,j, i) + Vc(VX2,k,j+1,i)); - Pn = 0.5*(Vc(PRS,k,j, i) + Vc(PRS,k,j+1,i)); + Vn = 0.5*(Vc(VX2,k,j-1,i) + Vc(VX2,k,j,i)); + Pn = 0.5*(Vc(PRS,k,j-1,i) + Vc(PRS,k,j,i)); } } } else if(dir == KDIR) { ////////////// // KDIR ///////////// - if(haveThermalDiffusion == UserDefFunction || haveThermalDiffusion == CollisionLess) { + if(haveThermalDiffusion == UserDefFunction) { knor = HALF_F*(knorArr(k-1,j,i)+knorArr(k,j,i)); if(haveSlopeLimiter) { kpar = 2.*(kparArr(k-1,j,i)*kparArr(k,j,i))/(kparArr(k-1,j,i)+kparArr(k,j,i)); } else { kpar = HALF_F*(kparArr(k-1,j,i)+kparArr(k,j,i)); } + if(includeCollisionlessTD) { + clessAlpha=HALF_F*(clessAlphaArr(k-1,j,i)+clessAlphaArr(k,j,i)); + clessBeta=HALF_F*(clessBetaArr(k-1,j,i)+clessBetaArr(k,j,i)); + } } else { knor = knorConstant; kpar = kparConstant; + if(includeCollisionlessTD) { + clessAlpha=clessAlphaConst; + clessBeta=clessBetaConst; + } } Bi = BX_K; @@ -706,11 +754,11 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, dpm = Vc(PRS,k,j,i) - Vc(PRS,k-1,j,i); if(haveSlopeLimiter) { - Vn = Vc(VX3,k,j,i)+0.5*SL::PLMLim(dvp, dvm); - Pn = Vc(PRS,k,j,i)+0.5*SL::PLMLim(dpp, dpm); + Vn = Vc(VX3,k,j,i)-HALF_F*SL::PLMLim(dvp, dvm); + Pn = Vc(PRS,k,j,i)-HALF_F*SL::PLMLim(dpp, dpm); } else { - Vn = 0.5*(Vc(VX3,k,j,i) + Vc(VX3,k+1,j,i)); - Pn = 0.5*(Vc(PRS,k,j,i) + Vc(PRS,k+1,j,i)); + Vn = HALF_F*(Vc(VX3,k-1,j,i) + Vc(VX3,k,j,i)); + Pn = HALF_F*(Vc(PRS,k-1,j,i) + Vc(PRS,k,j,i)); } } } @@ -728,13 +776,13 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, bn = Bn/Bmag; /* -- unit vector component -- */ q = kpar*bgradT*bn + knor*(dTn - bn*bgradT); if(includeCollisionlessTD) { - q = clessAlpha(k,j,i)*q + (1-clessAlpha(k,j,i))*clessBeta(k,j,i)*Pn*Vn; + q = clessAlpha*q + (1-clessAlpha)*clessBeta*Pn*Vn; } Flux(ENG, k, j, i) -= q; dMax(k,j,i) = FMAX(dMax(k,j,i),locdmax); if(includeCollisionlessTD) { - dMax(k,j,i) *= clessAlpha(k,j,i); + dMax(k,j,i) *= clessAlpha; } }); idfx::popRegion(); diff --git a/src/fluid/fluid_defs.hpp b/src/fluid/fluid_defs.hpp index 62a2099c4..1d040071c 100644 --- a/src/fluid/fluid_defs.hpp +++ b/src/fluid/fluid_defs.hpp @@ -25,7 +25,7 @@ class Fluid; // Parabolic terms can have different status -enum HydroModuleStatus {Disabled, Constant, UserDefFunction, CollisionLess}; +enum HydroModuleStatus {Disabled, Constant, UserDefFunction}; // Structure to describe the status of parabolic modules struct ParabolicModuleStatus { From e2f7a9a16bf97e2f76245757bbf91cdba188a120 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Sun, 12 Jan 2025 18:07:36 +0100 Subject: [PATCH 06/82] add a column Density example --- test/utils/columnDensity/column.cpp | 215 +++++++++++++++++++++++ test/utils/columnDensity/column.hpp | 48 +++++ test/utils/columnDensity/definitions.hpp | 4 + test/utils/columnDensity/idefix.ini | 24 +++ test/utils/columnDensity/setup.cpp | 154 ++++++++++++++++ 5 files changed, 445 insertions(+) create mode 100644 test/utils/columnDensity/column.cpp create mode 100644 test/utils/columnDensity/column.hpp create mode 100644 test/utils/columnDensity/definitions.hpp create mode 100644 test/utils/columnDensity/idefix.ini create mode 100644 test/utils/columnDensity/setup.cpp diff --git a/test/utils/columnDensity/column.cpp b/test/utils/columnDensity/column.cpp new file mode 100644 index 000000000..9ad4c3573 --- /dev/null +++ b/test/utils/columnDensity/column.cpp @@ -0,0 +1,215 @@ +#include "column.hpp" +#include "loop.hpp" + +Column::Column(int dir, int sign, int variable, DataBlock *data) : direction(dir), sign(sign), variable(variable) { + idfx::pushRegion("Column::Column"); + this->np_tot = data->np_tot; + this->np_int = data->np_int; + this->beg = data->beg; + + if(dir>= DIMENSIONS || dir < IDIR) IDEFIX_ERROR("Unknown direction for Column constructor"); + + // Allocate the array on which we do the average + this->ColumnArray = IdefixArray3D("ColumnArray",np_tot[KDIR], np_tot[JDIR], np_tot[IDIR]); + + // Elementary volumes and area + this->Volume = data->dV; + this->Area = data->A[dir]; + + // allocate helper array + if(dir == IDIR) { + localSum = IdefixArray2D("localSum",np_tot[KDIR], np_tot[JDIR]); + } + if(dir == JDIR) { + localSum = IdefixArray2D("localSum",np_tot[KDIR], np_tot[IDIR]); + } + if(dir == KDIR) { + localSum = IdefixArray2D("localSum",np_tot[JDIR], np_tot[IDIR]); + } + #ifdef WITH_MPI + // Create sub-MPI communicator dedicated to scan + int remainDims[3] = {false, false, false}; + remainDims[dir] = true; + MPI_Cart_sub(data->mygrid->CartComm, remainDims, &this->ColumnComm); + MPI_Comm_rank(this->ColumnComm, &this->MPIrank); + MPI_Comm_size(this->ColumnComm, &this->MPIsize); + + // create MPI class for boundary Xchanges + int ntarget = 0; + std::vector mapVars; + mapVars.push_back(ntarget); + + this->mpi.Init(data->mygrid, mapVars, data->nghost.data(), data->np_int.data()); + this->nproc = data->mygrid->nproc; + #endif + idfx::popRegion(); +} + +void Column::ComputeColumn(IdefixArray4D in) { + idfx::pushRegion("Column::ComputeColumn"); + const int nk = np_int[KDIR]; + const int nj = np_int[JDIR]; + const int ni = np_int[IDIR]; + const int kb = beg[KDIR]; + const int jb = beg[JDIR]; + const int ib = beg[IDIR]; + const int ke = kb+nk; + const int je = jb+nj; + const int ie = ib+ni; + + const int direction = this->direction; + auto column = this->ColumnArray; + auto dV = this->Volume; + auto A = this->Area; + auto localSum = this->localSum; + + const int var = this->variable; + if(direction==IDIR) { + // Inspired from loop.hpp + Kokkos::parallel_for("ColumnX1", team_policy (nk*nj, Kokkos::AUTO), + KOKKOS_LAMBDA (member_type team_member) { + int k = team_member.league_rank() / nj; + int j = team_member.league_rank() - k*nj + jb; + k += kb; + Kokkos::parallel_scan(Kokkos::TeamThreadRange<>(team_member,ib,ie), + [=] (int i, real &partial_sum, bool is_final) { + partial_sum += in(var,k,j,i)*dV(k,j,i) / (0.5*(A(k,j,i)+A(k,j,i+1))); + if(is_final) column(k,j,i) = partial_sum; + //partial_sum += in(var,k,j,i)*dV(k,j,i) / (0.5*(A(k,j,i)+A(k,j,i+1))); + }); + }); + } + if(direction==JDIR) { + // Inspired from loop.hpp + Kokkos::parallel_for("ColumnX2", team_policy (nk*ni, Kokkos::AUTO), + KOKKOS_LAMBDA (member_type team_member) { + int k = team_member.league_rank() / ni; + int i = team_member.league_rank() - k*ni + ib; + k += kb; + Kokkos::parallel_scan(Kokkos::TeamThreadRange<>(team_member,jb,je), + [=] (int j, real &partial_sum, bool is_final) { + partial_sum += in(var,k,j,i)*dV(k,j,i) / (0.5*(A(k,j,i)+A(k,j+1,i))); + if(is_final) column(k,j,i) = partial_sum; + }); + }); + } + if(direction==KDIR) { + // Inspired from loop.hpp + Kokkos::parallel_for("ColumnX2", team_policy (nj*ni, Kokkos::AUTO), + KOKKOS_LAMBDA (member_type team_member) { + int j = team_member.league_rank() / ni; + int i = team_member.league_rank() - j*ni + ib; + j += jb; + Kokkos::parallel_scan(Kokkos::TeamThreadRange<>(team_member,kb,ke), + [=] (int k, real &partial_sum, bool is_final) { + partial_sum += in(var,k,j,i)*dV(k,j,i) / (0.5*(A(k,j,i)+A(k+1,j,i))); + if(is_final) column(k,j,i) = partial_sum; + }); + }); + } + + #ifdef WITH_MPI + // Load the current sum + int dst,src; + MPI_Cart_shift(this->ColumnComm,0, 1, &src, &dst); + int size = localSum.extent(0)*localSum.extent(1); + if(MPIrank>0) { + MPI_Status status; + // Get the cumulative sum from previous processes + Kokkos::fence(); + MPI_Recv(localSum.data(), size, realMPI, src, 20, ColumnComm, &status); + // Add this to our cumulative sum + idefix_for("Addsum",kb,ke,jb,je,ib,ie, + KOKKOS_LAMBDA(int k, int j, int i) { + if(direction == IDIR) column(k,j,i) += localSum(k,j); + if(direction == JDIR) column(k,j,i) += localSum(k,i); + if(direction == KDIR) column(k,j,i) += localSum(j,i); + }); + } // rank =0 + // Get the final sum + if(MPIrank < MPIsize-1) { + if(direction==IDIR) { + idefix_for("Loadsum",kb,ke,jb,je, + KOKKOS_LAMBDA(int k, int j) { + localSum(k,j) = column(k,j,ie-1); + }); + } + if(direction==JDIR) { + idefix_for("Loadsum",kb,ke,ib,ie, + KOKKOS_LAMBDA(int k, int i) { + localSum(k,i) = column(k,je-1,i); + }); + } + if(direction==KDIR) { + idefix_for("Loadsum",jb,je,ib,ie, + KOKKOS_LAMBDA(int j, int i) { + localSum(j,i) = column(ke-1,j,i); + }); + } + // And send it + Kokkos::fence(); + MPI_Send(localSum.data(),size, realMPI, dst, 20, ColumnComm); + } // MPIrank small enough + #endif + // If we need it backwards + if(sign<0) { + // Compute total cumulative sum in the last subdomain + if(direction == IDIR) { + idefix_for("Loadsum",kb,ke,jb,je, + KOKKOS_LAMBDA(int k, int j) { + localSum(k,j) = column(k,j,ie-1) + in(var,k,j,ie-1) * dV(k,j,ie-1) / (0.5*(A(k,j,ie-1)+A(k,j,ie))); + }); + } + if(direction == JDIR) { + idefix_for("Loadsum",kb,ke,ib,ie, + KOKKOS_LAMBDA(int k, int i) { + localSum(k,i) = column(k,je-1,i) + in(var,k,je-1,i) * dV(k,je-1,i) / (0.5*(A(k,je-1,i)+A(k,je,i))); + }); + } + if(direction == KDIR) { + idefix_for("Loadsum",jb,je,ib,ie, + KOKKOS_LAMBDA(int j, int i) { + localSum(j,i) = column(ke-1,j,i) + in(var,ke-1,j,i) * dV(ke-1,j,i) / (0.5*(A(ke-1,j,i)+A(ke,j,i))); + }); + } + + #ifdef WITH_MPI + Kokkos::fence(); + MPI_Bcast(localSum.data(),size, realMPI, MPIsize-1, ColumnComm); + #endif + // All substract the local column from the full column + + idefix_for("Subsum",kb,ke,jb,je,ib,ie, + KOKKOS_LAMBDA(int k, int j, int i) { + if(direction==IDIR) column(k,j,i) = localSum(k,j)-column(k,j,i); + if(direction==JDIR) column(k,j,i) = localSum(k,i)-column(k,j,i); + if(direction==KDIR) column(k,j,i) = localSum(j,i)-column(k,j,i); + }); + } // sign <0 + // Xchange boundary elements when using MPI to ensure that column + // density in the ghost zones are coherent + #ifdef WITH_MPI + // Create a 4D array that contains our column data + this->arr4D = IdefixArray4D (column.data(), 1, this->np_tot[KDIR], + this->np_tot[JDIR], + this->np_tot[IDIR]); + + for(int dir = 0 ; dir < DIMENSIONS ; dir++) { + // MPI Exchange data when needed + if(nproc[dir]>1) { + switch(dir) { + case 0: + this->mpi.ExchangeX1(this->arr4D); + break; + case 1: + this->mpi.ExchangeX2(this->arr4D); + break; + case 2: + this->mpi.ExchangeX3(this->arr4D); + break; + } + } + } + #endif + idfx::popRegion(); +} diff --git a/test/utils/columnDensity/column.hpp b/test/utils/columnDensity/column.hpp new file mode 100644 index 000000000..b4cc5b539 --- /dev/null +++ b/test/utils/columnDensity/column.hpp @@ -0,0 +1,48 @@ +#ifndef COLUMN_HPP_ +#define COLUMN_HPP_ + +#include "idefix.hpp" +#include "input.hpp" +#include "output.hpp" +#include "grid.hpp" +#include "dataBlock.hpp" +#include "dataBlockHost.hpp" +#include +#include + + + +class Column { + public: + // Constructor from Setup arguments + Column(int dir, int sign, int variable, DataBlock *); + void ComputeColumn(IdefixArray4D in); + IdefixArray3D GetColumn() { + return (this->ColumnArray); + } + private: + IdefixArray3D ColumnArray; + int direction; // The direction along which the column is computed + int sign; // whether we integrate from the left or from the right + int variable; // The variable we use to compute the column + std::array np_tot; + std::array np_int; + std::array beg; + + IdefixArray3D Area; + IdefixArray3D Volume; + + IdefixArray2D localSum; + #ifdef WITH_MPI + Mpi mpi; // Mpi object when WITH_MPI is set + MPI_Comm ColumnComm; + int MPIrank; + int MPIsize; + + IdefixArray4D arr4D; // Dummy array 4D used for MPI xchanges + std::array nproc; // 3D size of the MPI cartesian geometry + + #endif +}; + +#endif // COLUMN_HPP_ diff --git a/test/utils/columnDensity/definitions.hpp b/test/utils/columnDensity/definitions.hpp new file mode 100644 index 000000000..3b581bd62 --- /dev/null +++ b/test/utils/columnDensity/definitions.hpp @@ -0,0 +1,4 @@ +#define COMPONENTS 3 +#define DIMENSIONS 3 + +#define GEOMETRY CARTESIAN diff --git a/test/utils/columnDensity/idefix.ini b/test/utils/columnDensity/idefix.ini new file mode 100644 index 000000000..2d994315c --- /dev/null +++ b/test/utils/columnDensity/idefix.ini @@ -0,0 +1,24 @@ +[Grid] +X1-grid 1 0.0 64 u 1.0 +X2-grid 1 0.0 64 u 1.0 +X3-grid 1 0.0 64 u 1.0 + +[TimeIntegrator] +CFL 0.8 +tstop 0.0 +nstages 2 + +[Hydro] +solver hllc +gamma 1.4 + +[Boundary] +X1-beg periodic +X1-end periodic +X2-beg periodic +X2-end periodic +X3-beg periodic +X3-end periodic + +[Output] +analysis 0.01 diff --git a/test/utils/columnDensity/setup.cpp b/test/utils/columnDensity/setup.cpp new file mode 100644 index 000000000..73d98cbae --- /dev/null +++ b/test/utils/columnDensity/setup.cpp @@ -0,0 +1,154 @@ +#include "idefix.hpp" +#include "setup.hpp" +#include "column.hpp" + + +Column *columnX1Left; +Column *columnX1Right; +Column *columnX2Left; +Column *columnX2Right; +Column *columnX3Left; +Column *columnX3Right; + +// Analyse data to check that column density works as expected +void Analysis(DataBlock & data) { + + DataBlockHost d(data); + + columnX1Left->ComputeColumn(data.hydro->Vc); + columnX1Right->ComputeColumn(data.hydro->Vc); + columnX2Left->ComputeColumn(data.hydro->Vc); + columnX2Right->ComputeColumn(data.hydro->Vc); + columnX3Left->ComputeColumn(data.hydro->Vc); + columnX3Right->ComputeColumn(data.hydro->Vc); + + IdefixArray3D columnDensityLeft, columnDensityRight; + IdefixArray3D::HostMirror columnDensityLeftHost, columnDensityRightHost; + // IDIR + columnDensityLeft = columnX1Left->GetColumn(); + columnDensityRight = columnX1Right->GetColumn(); + columnDensityLeftHost = Kokkos::create_mirror_view(columnDensityLeft); + columnDensityRightHost = Kokkos::create_mirror_view(columnDensityRight); + Kokkos::deep_copy(columnDensityLeftHost,columnDensityLeft); + Kokkos::deep_copy(columnDensityRightHost,columnDensityRight); + + real errMax = 0.0; + + for(int k = data.beg[KDIR]; k < data.end[KDIR] ; k++) { + for(int j = data.beg[JDIR]; j < data.end[JDIR] ; j++) { + for(int i = data.beg[IDIR]; i < data.end[IDIR] ; i++) { + real err = std::fabs(columnDensityLeftHost(k,j,i)-d.xr[IDIR](i)); + if(err>errMax) errMax=err; + err = std::fabs(columnDensityRightHost(k,j,i)-(1-d.xl[IDIR](i))); + if(err>errMax) errMax=err; + } + } + } + idfx::cout << "Error on column density in IDIR=" << std::scientific << errMax << std::endl; + if(errMax>1e-14) { + IDEFIX_ERROR("Error above tolerance"); + } + // JDIR + columnDensityLeft = columnX2Left->GetColumn(); + columnDensityRight = columnX2Right->GetColumn(); + columnDensityLeftHost = Kokkos::create_mirror_view(columnDensityLeft); + columnDensityRightHost = Kokkos::create_mirror_view(columnDensityRight); + Kokkos::deep_copy(columnDensityLeftHost,columnDensityLeft); + Kokkos::deep_copy(columnDensityRightHost,columnDensityRight); + + errMax = 0.0; + + for(int k = data.beg[KDIR]; k < data.end[KDIR] ; k++) { + for(int j = data.beg[JDIR]; j < data.end[JDIR] ; j++) { + for(int i = data.beg[IDIR]; i < data.end[IDIR] ; i++) { + real err = std::fabs(columnDensityLeftHost(k,j,i)-d.xr[JDIR](j)); + if(err>errMax) errMax=err; + err = std::fabs(columnDensityRightHost(k,j,i)-(1-d.xl[JDIR](j))); + if(err>errMax) errMax=err; + } + } + } + idfx::cout << "Error on column density in JDIR=" << std::scientific << errMax << std::endl; + if(errMax>1e-14) { + IDEFIX_ERROR("Error above tolerance"); + } + // KDIR + columnDensityLeft = columnX3Left->GetColumn(); + columnDensityRight = columnX3Right->GetColumn(); + columnDensityLeftHost = Kokkos::create_mirror_view(columnDensityLeft); + columnDensityRightHost = Kokkos::create_mirror_view(columnDensityRight); + Kokkos::deep_copy(columnDensityLeftHost,columnDensityLeft); + Kokkos::deep_copy(columnDensityRightHost,columnDensityRight); + + errMax = 0.0; + + for(int k = data.beg[KDIR]; k < data.end[KDIR] ; k++) { + for(int j = data.beg[JDIR]; j < data.end[JDIR] ; j++) { + for(int i = data.beg[IDIR]; i < data.end[IDIR] ; i++) { + real err = std::fabs(columnDensityLeftHost(k,j,i)-d.xr[KDIR](k)); + if(err>errMax) errMax=err; + err = std::fabs(columnDensityRightHost(k,j,i)-(1-d.xl[KDIR](k))); + if(err>errMax) errMax=err; + } + } + } + idfx::cout << "Error on column density in KDIR=" << std::scientific << errMax << std::endl; + if(errMax>1e-14) { + IDEFIX_ERROR("Error above tolerance"); + } + +} + +void InternalBoundary(Fluid * hydro, const real t) { + IdefixArray4D Vc = hydro->Vc; + idefix_for("InternalBoundary",0,hydro->data->np_tot[KDIR], + 0,hydro->data->np_tot[JDIR], + 0,hydro->data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + // Cancel any motion that could be happening + Vc(VX1,k,j,i) = 0.0; + Vc(VX2,k,j,i) = 0.0; + Vc(VX3,k,j,i) = 0.0; + }); +} + + +// Initialisation routine. Can be used to allocate +// Arrays or variables which are used later on +Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { + output.EnrollAnalysis(&Analysis); + data.hydro->EnrollInternalBoundary(&InternalBoundary); + + columnX1Left = new Column(IDIR, 1, RHO, &data); + columnX1Right = new Column(IDIR, -1, RHO, &data); + columnX2Left = new Column(JDIR, 1, RHO, &data); + columnX2Right = new Column(JDIR, -1, RHO, &data); + columnX3Left = new Column(KDIR, 1, RHO, &data); + columnX3Right = new Column(KDIR, -1, RHO, &data); + // Initialise the output file +} + +// This routine initialize the flow +// Note that data is on the device. +// One can therefore define locally +// a datahost and sync it, if needed +void Setup::InitFlow(DataBlock &data) { + // Create a host copy + DataBlockHost d(data); + + + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + + d.Vc(RHO,k,j,i) = 1.0; + d.Vc(VX1,k,j,i) = ZERO_F; + d.Vc(PRS,k,j,i) = 1.0; + + } + } + } + + // Send it all, if needed + d.SyncToDevice(); +} From 26ffc60820205069aa7c810cbd2375e42a2ac855 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Sun, 12 Jan 2025 18:15:31 +0100 Subject: [PATCH 07/82] add some documentation --- test/utils/columnDensity/column.hpp | 8 ++++++-- test/utils/columnDensity/setup.cpp | 5 +++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/test/utils/columnDensity/column.hpp b/test/utils/columnDensity/column.hpp index b4cc5b539..9defe7c6b 100644 --- a/test/utils/columnDensity/column.hpp +++ b/test/utils/columnDensity/column.hpp @@ -11,15 +11,19 @@ #include - +// A class to implement a parralel cumulative sum class Column { public: // Constructor from Setup arguments Column(int dir, int sign, int variable, DataBlock *); + // dir : direction along which the integration is performed + // sign: +1 for an integration from left to right, -1 for an integration from right to left (backwards) + // variable: index of the variable along which we do the integral (since the intput array is 4D) void ComputeColumn(IdefixArray4D in); + // Effectively compute integral from the input array in parameter IdefixArray3D GetColumn() { return (this->ColumnArray); - } + }// Return the column density private: IdefixArray3D ColumnArray; int direction; // The direction along which the column is computed diff --git a/test/utils/columnDensity/setup.cpp b/test/utils/columnDensity/setup.cpp index 73d98cbae..ebc131de7 100644 --- a/test/utils/columnDensity/setup.cpp +++ b/test/utils/columnDensity/setup.cpp @@ -34,6 +34,11 @@ void Analysis(DataBlock & data) { real errMax = 0.0; + // Because this particular setup has a constant density =1, we expect + // the column density from the left to match the cell right coordinates, + // and the column density from the right to match L-the cell left coordinate (where L is the box size) + // here we have L=1 + // This routine checks that all of the available column densities do match the expected values. for(int k = data.beg[KDIR]; k < data.end[KDIR] ; k++) { for(int j = data.beg[JDIR]; j < data.end[JDIR] ; j++) { for(int i = data.beg[IDIR]; i < data.end[IDIR] ; i++) { From a1e6e674743123cb73eee86d4833faaebc73e743 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Sun, 12 Jan 2025 18:18:40 +0100 Subject: [PATCH 08/82] add it to the tests --- .github/workflows/idefix-ci-jobs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/idefix-ci-jobs.yml b/.github/workflows/idefix-ci-jobs.yml index c0e622966..dfb89e0fb 100644 --- a/.github/workflows/idefix-ci-jobs.yml +++ b/.github/workflows/idefix-ci-jobs.yml @@ -201,3 +201,5 @@ jobs: run: scripts/ci/run-tests $IDEFIX_DIR/test/utils/lookupTable -all $TESTME_OPTIONS - name: Dump Image run: scripts/ci/run-tests $IDEFIX_DIR/test/utils/dumpImage -all $TESTME_OPTIONS + - name: Column density + run: scripts/ci/run-tests $IDEFIX_DIR/test/utils/columnDensity -all $TESTME_OPTIONS From 9c58596070646975bb7c3c3295904aa80ee34c1f Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Sun, 12 Jan 2025 18:36:28 +0100 Subject: [PATCH 09/82] imrove documentation --- src/utils/CMakeLists.txt | 2 + .../columnDensity => src/utils}/column.cpp | 30 ++++++-- src/utils/column.hpp | 69 +++++++++++++++++++ test/utils/columnDensity/column.hpp | 52 -------------- 4 files changed, 96 insertions(+), 57 deletions(-) rename {test/utils/columnDensity => src/utils}/column.cpp (89%) create mode 100644 src/utils/column.hpp delete mode 100644 test/utils/columnDensity/column.hpp diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt index b36f8a08e..afb4f57d4 100644 --- a/src/utils/CMakeLists.txt +++ b/src/utils/CMakeLists.txt @@ -5,4 +5,6 @@ target_sources(idefix PUBLIC ${CMAKE_CURRENT_LIST_DIR}/dumpImage.cpp PUBLIC ${CMAKE_CURRENT_LIST_DIR}/dumpImage.hpp PUBLIC ${CMAKE_CURRENT_LIST_DIR}/lookupTable.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/column.cpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/column.hpp ) diff --git a/test/utils/columnDensity/column.cpp b/src/utils/column.cpp similarity index 89% rename from test/utils/columnDensity/column.cpp rename to src/utils/column.cpp index 9ad4c3573..838f1d9ce 100644 --- a/test/utils/columnDensity/column.cpp +++ b/src/utils/column.cpp @@ -1,7 +1,24 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + + +#include +#include + #include "column.hpp" -#include "loop.hpp" +#include "idefix.hpp" +#include "input.hpp" +#include "output.hpp" +#include "grid.hpp" +#include "dataBlock.hpp" +#include "dataBlockHost.hpp" -Column::Column(int dir, int sign, int variable, DataBlock *data) : direction(dir), sign(sign), variable(variable) { +Column::Column(int dir, int sign, int variable, DataBlock *data) + : direction(dir), sign(sign), variable(variable) { idfx::pushRegion("Column::Column"); this->np_tot = data->np_tot; this->np_int = data->np_int; @@ -157,19 +174,22 @@ void Column::ComputeColumn(IdefixArray4D in) { if(direction == IDIR) { idefix_for("Loadsum",kb,ke,jb,je, KOKKOS_LAMBDA(int k, int j) { - localSum(k,j) = column(k,j,ie-1) + in(var,k,j,ie-1) * dV(k,j,ie-1) / (0.5*(A(k,j,ie-1)+A(k,j,ie))); + localSum(k,j) = column(k,j,ie-1) + in(var,k,j,ie-1) * dV(k,j,ie-1) + / (0.5*(A(k,j,ie-1)+A(k,j,ie))); }); } if(direction == JDIR) { idefix_for("Loadsum",kb,ke,ib,ie, KOKKOS_LAMBDA(int k, int i) { - localSum(k,i) = column(k,je-1,i) + in(var,k,je-1,i) * dV(k,je-1,i) / (0.5*(A(k,je-1,i)+A(k,je,i))); + localSum(k,i) = column(k,je-1,i) + in(var,k,je-1,i) * dV(k,je-1,i) + / (0.5*(A(k,je-1,i)+A(k,je,i))); }); } if(direction == KDIR) { idefix_for("Loadsum",jb,je,ib,ie, KOKKOS_LAMBDA(int j, int i) { - localSum(j,i) = column(ke-1,j,i) + in(var,ke-1,j,i) * dV(ke-1,j,i) / (0.5*(A(ke-1,j,i)+A(ke,j,i))); + localSum(j,i) = column(ke-1,j,i) + in(var,ke-1,j,i) * dV(ke-1,j,i) + / (0.5*(A(ke-1,j,i)+A(ke,j,i))); }); } diff --git a/src/utils/column.hpp b/src/utils/column.hpp new file mode 100644 index 000000000..feb0a8537 --- /dev/null +++ b/src/utils/column.hpp @@ -0,0 +1,69 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef UTILS_COLUMN_HPP_ +#define UTILS_COLUMN_HPP_ + +#include +#include + +#include "idefix.hpp" +#include "dataBlock.hpp" + + + +// A class to implement a parralel cumulative sum +class Column { + public: + //////////////////////////////////////////////////////////////////////////////////// + /// @brief Constructor of a cumulative sum(=column density) from setup's arguments + /// @param dir direction along which the integration is performed + /// @param sign: +1 for an integration from left to right, -1 for an integration from right to + /// left i.e (backwards) + /// @param variable: index of the variable along which we do the integral (since the + /// intput array is 4D) + /////////////////////////////////////////////////////////////////////////////////// + Column(int dir, int sign, int variable, DataBlock *); + + /////////////////////////////////////////////////////////////////////////////////// + /// @brief Effectively compute integral from the input array in argument + /////////////////////////////////////////////////////////////////////////////////// + void ComputeColumn(IdefixArray4D in); + + /////////////////////////////////////////////////////////////////////////////////// + /// @brief Get a reference to the computed column density array + /////////////////////////////////////////////////////////////////////////////////// + IdefixArray3D GetColumn() { + return (this->ColumnArray); + } + + private: + IdefixArray3D ColumnArray; + int direction; // The direction along which the column is computed + int sign; // whether we integrate from the left or from the right + int variable; // The variable we use to compute the column + std::array np_tot; + std::array np_int; + std::array beg; + + IdefixArray3D Area; + IdefixArray3D Volume; + + IdefixArray2D localSum; + #ifdef WITH_MPI + Mpi mpi; // Mpi object when WITH_MPI is set + MPI_Comm ColumnComm; + int MPIrank; + int MPIsize; + + IdefixArray4D arr4D; // Dummy array 4D used for MPI xchanges + std::array nproc; // 3D size of the MPI cartesian geometry + + #endif +}; + +#endif // UTILS_COLUMN_HPP_ diff --git a/test/utils/columnDensity/column.hpp b/test/utils/columnDensity/column.hpp deleted file mode 100644 index 9defe7c6b..000000000 --- a/test/utils/columnDensity/column.hpp +++ /dev/null @@ -1,52 +0,0 @@ -#ifndef COLUMN_HPP_ -#define COLUMN_HPP_ - -#include "idefix.hpp" -#include "input.hpp" -#include "output.hpp" -#include "grid.hpp" -#include "dataBlock.hpp" -#include "dataBlockHost.hpp" -#include -#include - - -// A class to implement a parralel cumulative sum -class Column { - public: - // Constructor from Setup arguments - Column(int dir, int sign, int variable, DataBlock *); - // dir : direction along which the integration is performed - // sign: +1 for an integration from left to right, -1 for an integration from right to left (backwards) - // variable: index of the variable along which we do the integral (since the intput array is 4D) - void ComputeColumn(IdefixArray4D in); - // Effectively compute integral from the input array in parameter - IdefixArray3D GetColumn() { - return (this->ColumnArray); - }// Return the column density - private: - IdefixArray3D ColumnArray; - int direction; // The direction along which the column is computed - int sign; // whether we integrate from the left or from the right - int variable; // The variable we use to compute the column - std::array np_tot; - std::array np_int; - std::array beg; - - IdefixArray3D Area; - IdefixArray3D Volume; - - IdefixArray2D localSum; - #ifdef WITH_MPI - Mpi mpi; // Mpi object when WITH_MPI is set - MPI_Comm ColumnComm; - int MPIrank; - int MPIsize; - - IdefixArray4D arr4D; // Dummy array 4D used for MPI xchanges - std::array nproc; // 3D size of the MPI cartesian geometry - - #endif -}; - -#endif // COLUMN_HPP_ From 6283fd5e2997d3502f81b43cfc9110c76b3216f7 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Sun, 12 Jan 2025 18:50:44 +0100 Subject: [PATCH 10/82] add doc link for column density --- doc/source/faq.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/source/faq.rst b/doc/source/faq.rst index 75a07a68b..4c7bc056b 100644 --- a/doc/source/faq.rst +++ b/doc/source/faq.rst @@ -73,3 +73,6 @@ I want to test a modification to *Idefix* source code specific to my problem wit I want to use a lookup table from a CSV file in my idefix_loop. How could I proceed? Use the ``LookupTable`` class (see :ref:`LookupTableClass`) + +I want to compute a cumulative sum (e.g a column density) on the fly. How could I proceed? + Use the ``Column`` class (see :class:`::Column`) From 8277ed0fdff7114f2c77ff8b30abfa30972c9cab Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Sun, 12 Jan 2025 20:16:30 +0100 Subject: [PATCH 11/82] missing testme file --- test/utils/columnDensity/testme.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100755 test/utils/columnDensity/testme.py diff --git a/test/utils/columnDensity/testme.py b/test/utils/columnDensity/testme.py new file mode 100755 index 000000000..6f017d7d8 --- /dev/null +++ b/test/utils/columnDensity/testme.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) +import pytools.idfx_test as tst + +test=tst.idfxTest() + +test.configure() +test.compile() +# this test succeeds if it runs successfully +test.run() + +test.mpi = True +test.configure() +test.compile() +# this test succeeds if it runs successfully +test.run() From 31ae1b8a47168fcf09cf1f4be6d6794d5a19b33e Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 14 Jan 2025 15:13:38 +0100 Subject: [PATCH 12/82] add a 3D interface for column density computation --- src/utils/column.cpp | 21 +++++++++++------- src/utils/column.hpp | 15 ++++++++----- test/utils/columnDensity/setup.cpp | 34 +++++++++++++++++++----------- 3 files changed, 45 insertions(+), 25 deletions(-) diff --git a/src/utils/column.cpp b/src/utils/column.cpp index 838f1d9ce..a774d4927 100644 --- a/src/utils/column.cpp +++ b/src/utils/column.cpp @@ -17,8 +17,8 @@ #include "dataBlock.hpp" #include "dataBlockHost.hpp" -Column::Column(int dir, int sign, int variable, DataBlock *data) - : direction(dir), sign(sign), variable(variable) { +Column::Column(int dir, int sign, DataBlock *data) + : direction(dir), sign(sign) { idfx::pushRegion("Column::Column"); this->np_tot = data->np_tot; this->np_int = data->np_int; @@ -62,7 +62,7 @@ Column::Column(int dir, int sign, int variable, DataBlock *data) idfx::popRegion(); } -void Column::ComputeColumn(IdefixArray4D in) { +void Column::ComputeColumn(IdefixArray4D in, const int var) { idfx::pushRegion("Column::ComputeColumn"); const int nk = np_int[KDIR]; const int nj = np_int[JDIR]; @@ -80,7 +80,6 @@ void Column::ComputeColumn(IdefixArray4D in) { auto A = this->Area; auto localSum = this->localSum; - const int var = this->variable; if(direction==IDIR) { // Inspired from loop.hpp Kokkos::parallel_for("ColumnX1", team_policy (nk*nj, Kokkos::AUTO), @@ -210,7 +209,7 @@ void Column::ComputeColumn(IdefixArray4D in) { // density in the ghost zones are coherent #ifdef WITH_MPI // Create a 4D array that contains our column data - this->arr4D = IdefixArray4D (column.data(), 1, this->np_tot[KDIR], + IdefixArray4D arr4D(column.data(), 1, this->np_tot[KDIR], this->np_tot[JDIR], this->np_tot[IDIR]); @@ -219,13 +218,13 @@ void Column::ComputeColumn(IdefixArray4D in) { if(nproc[dir]>1) { switch(dir) { case 0: - this->mpi.ExchangeX1(this->arr4D); + this->mpi.ExchangeX1(arr4D); break; case 1: - this->mpi.ExchangeX2(this->arr4D); + this->mpi.ExchangeX2(arr4D); break; case 2: - this->mpi.ExchangeX3(this->arr4D); + this->mpi.ExchangeX3(arr4D); break; } } @@ -233,3 +232,9 @@ void Column::ComputeColumn(IdefixArray4D in) { #endif idfx::popRegion(); } + +void Column::ComputeColumn(IdefixArray3D in) { + // 4D alias + IdefixArray4D arr4D(in.data(), 1, in.extent(2), in.extent(1), in.extent(0)); + return this->ComputeColumn(arr4D,0); +} diff --git a/src/utils/column.hpp b/src/utils/column.hpp index feb0a8537..43383cbe6 100644 --- a/src/utils/column.hpp +++ b/src/utils/column.hpp @@ -24,15 +24,22 @@ class Column { /// @param dir direction along which the integration is performed /// @param sign: +1 for an integration from left to right, -1 for an integration from right to /// left i.e (backwards) + /////////////////////////////////////////////////////////////////////////////////// + Column(int dir, int sign, DataBlock *); + + /////////////////////////////////////////////////////////////////////////////////// + /// @brief Effectively compute integral from the input array in argument + /// @param in: 4D input array /// @param variable: index of the variable along which we do the integral (since the /// intput array is 4D) /////////////////////////////////////////////////////////////////////////////////// - Column(int dir, int sign, int variable, DataBlock *); + void ComputeColumn(IdefixArray4D in, int variable); - /////////////////////////////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////////////////// /// @brief Effectively compute integral from the input array in argument + /// @param in: 3D input array /////////////////////////////////////////////////////////////////////////////////// - void ComputeColumn(IdefixArray4D in); + void ComputeColumn(IdefixArray3D in); /////////////////////////////////////////////////////////////////////////////////// /// @brief Get a reference to the computed column density array @@ -45,7 +52,6 @@ class Column { IdefixArray3D ColumnArray; int direction; // The direction along which the column is computed int sign; // whether we integrate from the left or from the right - int variable; // The variable we use to compute the column std::array np_tot; std::array np_int; std::array beg; @@ -60,7 +66,6 @@ class Column { int MPIrank; int MPIsize; - IdefixArray4D arr4D; // Dummy array 4D used for MPI xchanges std::array nproc; // 3D size of the MPI cartesian geometry #endif diff --git a/test/utils/columnDensity/setup.cpp b/test/utils/columnDensity/setup.cpp index ebc131de7..b27b6b1f3 100644 --- a/test/utils/columnDensity/setup.cpp +++ b/test/utils/columnDensity/setup.cpp @@ -15,12 +15,22 @@ void Analysis(DataBlock & data) { DataBlockHost d(data); - columnX1Left->ComputeColumn(data.hydro->Vc); - columnX1Right->ComputeColumn(data.hydro->Vc); - columnX2Left->ComputeColumn(data.hydro->Vc); - columnX2Right->ComputeColumn(data.hydro->Vc); - columnX3Left->ComputeColumn(data.hydro->Vc); - columnX3Right->ComputeColumn(data.hydro->Vc); + // Try the 4D array interface + columnX1Left->ComputeColumn(data.hydro->Vc,RHO); + columnX1Right->ComputeColumn(data.hydro->Vc,RHO); + + // Try the 3D array interface + IdefixArray3D rho("rho",data.np_tot[KDIR],data.np_tot[JDIR],data.np_tot[IDIR]); + auto Vc = data.hydro->Vc; + + idefix_for("init rho",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], + KOKKOS_LAMBDA(int k, int j, int i) { + rho(k,j,i) = Vc(RHO,k,j,i); + }); + columnX2Left->ComputeColumn(rho); + columnX2Right->ComputeColumn(rho); + columnX3Left->ComputeColumn(rho); + columnX3Right->ComputeColumn(rho); IdefixArray3D columnDensityLeft, columnDensityRight; IdefixArray3D::HostMirror columnDensityLeftHost, columnDensityRightHost; @@ -124,12 +134,12 @@ Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { output.EnrollAnalysis(&Analysis); data.hydro->EnrollInternalBoundary(&InternalBoundary); - columnX1Left = new Column(IDIR, 1, RHO, &data); - columnX1Right = new Column(IDIR, -1, RHO, &data); - columnX2Left = new Column(JDIR, 1, RHO, &data); - columnX2Right = new Column(JDIR, -1, RHO, &data); - columnX3Left = new Column(KDIR, 1, RHO, &data); - columnX3Right = new Column(KDIR, -1, RHO, &data); + columnX1Left = new Column(IDIR, 1, &data); + columnX1Right = new Column(IDIR, -1, &data); + columnX2Left = new Column(JDIR, 1, &data); + columnX2Right = new Column(JDIR, -1, &data); + columnX3Left = new Column(KDIR, 1, &data); + columnX3Right = new Column(KDIR, -1, &data); // Initialise the output file } From d6560366a1c3a3d6c8c2dabb6c61426b56aaad71 Mon Sep 17 00:00:00 2001 From: Hal Bal Date: Wed, 15 Jan 2025 10:05:44 +0100 Subject: [PATCH 13/82] Upwind scheme for cless TC with limiters + tests --- reference | 2 +- src/fluid/braginskii/bragThermalDiffusion.hpp | 97 +++++++----- test/MHD/MTI/idefix-rkl.ini | 2 +- test/MHD/MTI/idefix-sl.ini | 2 +- test/MHD/MTI/idefix.ini | 2 +- test/MHD/clessTDiffusion/CMakeLists.txt | 2 + test/MHD/clessTDiffusion/definitions.hpp | 5 + test/MHD/clessTDiffusion/idefix.ini | 41 +++++ test/MHD/clessTDiffusion/python/testidefix.py | 40 +++++ test/MHD/clessTDiffusion/setup.cpp | 149 ++++++++++++++++++ test/MHD/clessTDiffusion/testme.py | 42 +++++ test/MHD/sphBragTDiffusion/idefix.ini | 2 +- 12 files changed, 342 insertions(+), 44 deletions(-) create mode 100644 test/MHD/clessTDiffusion/CMakeLists.txt create mode 100644 test/MHD/clessTDiffusion/definitions.hpp create mode 100644 test/MHD/clessTDiffusion/idefix.ini create mode 100644 test/MHD/clessTDiffusion/python/testidefix.py create mode 100644 test/MHD/clessTDiffusion/setup.cpp create mode 100755 test/MHD/clessTDiffusion/testme.py diff --git a/reference b/reference index 609a60164..99f339c43 160000 --- a/reference +++ b/reference @@ -1 +1 @@ -Subproject commit 609a60164461fb2d4fbeffa89ce148acf1191525 +Subproject commit 99f339c43d98274a925282cc96252c5a0fa9374f diff --git a/src/fluid/braginskii/bragThermalDiffusion.hpp b/src/fluid/braginskii/bragThermalDiffusion.hpp index db01497b7..64037420f 100644 --- a/src/fluid/braginskii/bragThermalDiffusion.hpp +++ b/src/fluid/braginskii/bragThermalDiffusion.hpp @@ -122,7 +122,7 @@ BragThermalDiffusion::BragThermalDiffusion(Input &input, Grid &grid, Fluid IDEFIX_ERROR("Unknown braginskii thermal diffusion definition in idefix.ini. " "Can only be constant or userdef."); } - } else if(input.Get("Hydro","bragTDiffusion",2).compare("wcollisionless") == 0.0) { + } else if(input.Get("Hydro","bragTDiffusion",2).compare("wcless") == 0.0) { if(input.Get("Hydro","bragTDiffusion",3).compare("constant") == 0) { this->includeCollisionlessTD = true; this->kpar = input.Get("Hydro","bragTDiffusion",4); @@ -148,12 +148,12 @@ BragThermalDiffusion::BragThermalDiffusion(Input &input, Grid &grid, Fluid data->np_tot[JDIR], data->np_tot[IDIR]); } else { - IDEFIX_ERROR("Unknown braginskii/collsiionless thermal diffusion definition in idefix.ini. " + IDEFIX_ERROR("Unknown braginskii/collisionless thermal diffusion definition in idefix.ini. " "Can only be constant or userdef."); } } else { IDEFIX_ERROR("Unknown braginskii thermal diffusion saturation in idefix.ini. " - "Can only be nosat or wcollisionless."); + "Can only be nosat or wcless."); } } else { IDEFIX_ERROR("I cannot create a BragThermalDiffusion object without bragTDiffusion defined" @@ -220,7 +220,7 @@ BragThermalDiffusion::BragThermalDiffusion(Input &input, Grid &grid, Fluid //We now define spatial average macros for the magnetic field. // The magnetic field appears in the expression of the Braginskii heat flux. -// It is therefore needed at the right cell interface according to the direction of the flux. +// It is therefore needed at the left cell interface according to the direction of the flux. #define BX_I Vs(BX1s,k,j,i) #define BY_J Vs(BX2s,k,j,i) #define BZ_K Vs(BX3s,k,j,i) @@ -348,7 +348,7 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, } if(includeCollisionlessTD) { clessAlpha=HALF_F*(clessAlphaArr(k,j,i-1)+clessAlphaArr(k,j,i)); - clessBeta=HALF_F*(clessBetaArr(k,j,i-1)+clessAlphaArr(k,j,i)); + clessBeta=HALF_F*(clessBetaArr(k,j,i-1)+clessBetaArr(k,j,i)); } } else { knor = knorConstant; @@ -470,21 +470,28 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, dTn = dTi; /* For collisionless / saturated heat flux */ - if (includeCollisionlessTD) { - dvp = Vc(VX1,k,j,i+1) - Vc(VX1,k,j,i); - dvm = Vc(VX1,k,j,i) - Vc(VX1,k,j,i-1); + if (includeCollisionlessTD) { + if(haveSlopeLimiter) { + dvp = Vc(VX1,k,j,i+1) - Vc(VX1,k,j,i); + dvm = Vc(VX1,k,j,i) - Vc(VX1,k,j,i-1); - dpp = Vc(PRS,k,j,i+1) - Vc(PRS,k,j,i); - dpm = Vc(PRS,k,j,i) - Vc(PRS,k,j,i-1); + dpp = Vc(PRS,k,j,i+1) - Vc(PRS,k,j,i); + dpm = Vc(PRS,k,j,i) - Vc(PRS,k,j,i-1); - if(haveSlopeLimiter) { - Vn = Vc(VX1,k,j,i)-HALF_F*SL::PLMLim(dvp, dvm); - Pn = Vc(PRS,k,j,i)-HALF_F*SL::PLMLim(dpp, dpm); - } else { - Vn = HALF_F*(Vc(VX1,k,j,i) + Vc(VX1,k,j,i-1)); - Pn = HALF_F*(Vc(PRS,k,j,i) + Vc(PRS,k,j,i-1)); - } - } + /* Upwind scheme */ + if (Vc(VX1,k,j,i) > 0.0) { + Vn = Vc(VX1,k,j,i-1)+HALF_F*SL::PLMLim(dvm, dvp); + Pn = Vc(PRS,k,j,i-1)+HALF_F*SL::PLMLim(dpm, dpp); + } + else { + Vn = Vc(VX1,k,j,i)-HALF_F*SL::PLMLim(dvm, dvp); + Pn = Vc(PRS,k,j,i)-HALF_F*SL::PLMLim(dpm, dpp); + } + } else { + Vn = HALF_F*(Vc(VX1,k,j,i) + Vc(VX1,k,j,i-1)); + Pn = HALF_F*(Vc(PRS,k,j,i) + Vc(PRS,k,j,i-1)); + } + } } else if(dir == JDIR) { ////////////// // JDIR @@ -499,7 +506,7 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, } if(includeCollisionlessTD) { clessAlpha=HALF_F*(clessAlphaArr(k,j-1,i)+clessAlphaArr(k,j,i)); - clessBeta=HALF_F*(clessBetaArr(k,j-1,i)+clessAlphaArr(k,j,i)); + clessBeta=HALF_F*(clessBetaArr(k,j-1,i)+clessBetaArr(k,j,i)); } } else { knor = knorConstant; @@ -629,21 +636,27 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, dTn = dTj; /* For the collisionless / saturated flux */ - if(includeCollisionlessTD) { - dvp = Vc(VX2,k,j+1,i) - Vc(VX2,k,j,i); - dvm = Vc(VX2,k,j,i) - Vc(VX2,k,j-1,i); + if(includeCollisionlessTD) { + if(haveSlopeLimiter) { + dvp = Vc(VX2,k,j+1,i) - Vc(VX2,k,j,i); + dvm = Vc(VX2,k,j,i) - Vc(VX2,k,j-1,i); - dpp = Vc(PRS,k,j+1,i) - Vc(PRS,k,j,i); - dpm = Vc(PRS,k,j,i) - Vc(PRS,k,j-1,i); + dpp = Vc(PRS,k,j+1,i) - Vc(PRS,k,j,i); + dpm = Vc(PRS,k,j,i) - Vc(PRS,k,j-1,i); - if(haveSlopeLimiter) { - Vn = Vc(VX2,k,j,i)-HALF_F*SL::PLMLim(dvp, dvm); - Pn = Vc(PRS,k,j,i)-HALF_F*SL::PLMLim(dpp, dpm); - } else { - Vn = 0.5*(Vc(VX2,k,j-1,i) + Vc(VX2,k,j,i)); - Pn = 0.5*(Vc(PRS,k,j-1,i) + Vc(PRS,k,j,i)); - } - } + /* Upwind scheme */ + if (Vc(VX2,k,j,i) > 0.0) { + Vn = Vc(VX2,k,j-1,i)+HALF_F*SL::PLMLim(dvm, dvp); + Pn = Vc(PRS,k,j-1,i)+HALF_F*SL::PLMLim(dpm, dpp); + } else { + Vn = Vc(VX2,k,j,i)-HALF_F*SL::PLMLim(dvm, dvp); + Pn = Vc(PRS,k,j,i)-HALF_F*SL::PLMLim(dpm, dpp); + } + } else { + Vn = HALF_F*(Vc(VX2,k,j-1,i) + Vc(VX2,k,j,i)); + Pn = HALF_F*(Vc(PRS,k,j-1,i) + Vc(PRS,k,j,i)); + } + } } else if(dir == KDIR) { ////////////// // KDIR @@ -747,15 +760,21 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, /* For the collisionless / saturated flux */ if(includeCollisionlessTD) { - dvp = Vc(VX3,k+1,j,i) - Vc(VX3,k,j,i); - dvm = Vc(VX3,k,j,i) - Vc(VX3,k-1,j,i); + if(haveSlopeLimiter) { + dvp = Vc(VX3,k+1,j,i) - Vc(VX3,k,j,i); + dvm = Vc(VX3,k,j,i) - Vc(VX3,k-1,j,i); - dpp = Vc(PRS,k+1,j,i) - Vc(PRS,k,j,i); - dpm = Vc(PRS,k,j,i) - Vc(PRS,k-1,j,i); + dpp = Vc(PRS,k+1,j,i) - Vc(PRS,k,j,i); + dpm = Vc(PRS,k,j,i) - Vc(PRS,k-1,j,i); - if(haveSlopeLimiter) { - Vn = Vc(VX3,k,j,i)-HALF_F*SL::PLMLim(dvp, dvm); - Pn = Vc(PRS,k,j,i)-HALF_F*SL::PLMLim(dpp, dpm); + /* Upwind scheme */ + if (Vc(VX3,k,j,i) > 0.0) { + Vn = Vc(VX3,k-1,j,i)+HALF_F*SL::PLMLim(dvp, dvm); + Pn = Vc(PRS,k-1,j,i)+HALF_F*SL::PLMLim(dpp, dpm); + } else { + Vn = Vc(VX3,k,j,i)-HALF_F*SL::PLMLim(dvp, dvm); + Pn = Vc(PRS,k,j,i)-HALF_F*SL::PLMLim(dpp, dpm); + } } else { Vn = HALF_F*(Vc(VX3,k-1,j,i) + Vc(VX3,k,j,i)); Pn = HALF_F*(Vc(PRS,k-1,j,i) + Vc(PRS,k,j,i)); diff --git a/test/MHD/MTI/idefix-rkl.ini b/test/MHD/MTI/idefix-rkl.ini index 1b9b11eb2..09e9eb72d 100644 --- a/test/MHD/MTI/idefix-rkl.ini +++ b/test/MHD/MTI/idefix-rkl.ini @@ -12,7 +12,7 @@ nstages 3 [Hydro] solver hlld gamma 1.6666666666666666 -bragTDiffusion rkl nolimiter userdef +bragTDiffusion rkl nolimiter nosat userdef bragViscosity rkl nolimiter userdef [Gravity] diff --git a/test/MHD/MTI/idefix-sl.ini b/test/MHD/MTI/idefix-sl.ini index f2c9deb2b..aa707914f 100644 --- a/test/MHD/MTI/idefix-sl.ini +++ b/test/MHD/MTI/idefix-sl.ini @@ -12,7 +12,7 @@ nstages 3 [Hydro] solver hlld gamma 1.6666666666666666 -bragTDiffusion rkl mc userdef +bragTDiffusion rkl mc nosat userdef bragViscosity rkl vanleer userdef [Gravity] diff --git a/test/MHD/MTI/idefix.ini b/test/MHD/MTI/idefix.ini index 7a09c406c..a159c4f70 100644 --- a/test/MHD/MTI/idefix.ini +++ b/test/MHD/MTI/idefix.ini @@ -12,7 +12,7 @@ nstages 3 [Hydro] solver hlld gamma 1.6666666666666666 -bragTDiffusion explicit nolimiter userdef +bragTDiffusion explicit nolimiter nosat userdef bragViscosity explicit nolimiter userdef [Gravity] diff --git a/test/MHD/clessTDiffusion/CMakeLists.txt b/test/MHD/clessTDiffusion/CMakeLists.txt new file mode 100644 index 000000000..aa9abdd74 --- /dev/null +++ b/test/MHD/clessTDiffusion/CMakeLists.txt @@ -0,0 +1,2 @@ +enable_idefix_property(Idefix_MHD) +set_idefix_property(Idefix_RECONSTRUCTION Parabolic) diff --git a/test/MHD/clessTDiffusion/definitions.hpp b/test/MHD/clessTDiffusion/definitions.hpp new file mode 100644 index 000000000..21b1f8733 --- /dev/null +++ b/test/MHD/clessTDiffusion/definitions.hpp @@ -0,0 +1,5 @@ +#define COMPONENTS 3 +#define DIMENSIONS 1 +#define GEOMETRY SPHERICAL + +#define SMALL_PRESSURE_TEMPERATURE (0.1) diff --git a/test/MHD/clessTDiffusion/idefix.ini b/test/MHD/clessTDiffusion/idefix.ini new file mode 100644 index 000000000..9c444b928 --- /dev/null +++ b/test/MHD/clessTDiffusion/idefix.ini @@ -0,0 +1,41 @@ +[Grid] +X1-grid 1 1.0 256 l 40.0 +X2-grid 1 1.4708 1 u 1.6708 +X3-grid 1 0.9892 1 u 1.0108 + +[TimeIntegrator] +CFL 0.4 +CFL_max_var 1.1 # not used +tstop 300.0 +first_dt 1.e-6 +nstages 2 + +[Hydro] +solver hlld +gamma 1.6666666666666 +bragTDiffusion rkl mc wcless userdef + +[Gravity] +potential central +Mcentral 1.0 + +[Boundary] +X1-beg userdef +X1-end outflow +X2-beg periodic +X2-end periodic +X3-beg periodic +X3-end periodic + +[Setup] +UNIT_DENSITY 1.6726e-16 +UNIT_LENGTH 6.9570e10 +UNIT_VELOCITY 4.3670e7 +cs_vesc 0.26 +va_vesc 0.3 +k0 9e-7 + +[Output] +vtk 100.0 +dmp 300.0 +log 10 diff --git a/test/MHD/clessTDiffusion/python/testidefix.py b/test/MHD/clessTDiffusion/python/testidefix.py new file mode 100644 index 000000000..444816d1f --- /dev/null +++ b/test/MHD/clessTDiffusion/python/testidefix.py @@ -0,0 +1,40 @@ +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) +from pytools.vtk_io import readVTK +import numpy as np +import inifix + +conf = inifix.load("../idefix.ini") +gamma = conf["Hydro"]["gamma"] +kappa = conf["Hydro"]["bragTDiffusion"][-1] + +current_VTK = readVTK('../data.0003.vtk', geometry='spherical') + +rho = current_VTK.data['RHO'].squeeze() +prs = current_VTK.data['PRS'].squeeze() +r = current_VTK.r + +idx1=np.where(r > 20)[0][0] +idx2=np.where(r > 30)[0][0] + +def gamma_prime(gamma, beta): + return (gamma+beta*(gamma-1))/(1+beta*(gamma-1)) + +success = True +eps = 1e-2 + +gamma_eff = np.gradient(prs, r)/np.gradient(rho, r)*rho/prs + +Error=np.abs(gamma_eff[idx1:idx2]-gamma_prime(gamma, 1.5)).mean() +if Error > eps: + success=False + +if success: + print("SUCCESS") + print("Error: {0}".format(Error)) + sys.exit(0) +else: + print("Failed") + print("Error: {0}".format(Error)) + sys.exit(1) diff --git a/test/MHD/clessTDiffusion/setup.cpp b/test/MHD/clessTDiffusion/setup.cpp new file mode 100644 index 000000000..624e1d72b --- /dev/null +++ b/test/MHD/clessTDiffusion/setup.cpp @@ -0,0 +1,149 @@ +#include "idefix.hpp" +#include "setup.hpp" + +real udenGlob; +real ulenGlob; +real uvelGlob; +real gammaGlob; +real cs_vescGlob; +real va_vescGlob; +real k0_Glob; +real ParkerWind(real); + +void MyClessThermalConductivity(DataBlock &data, const real t, IdefixArray3D &kparArr, IdefixArray3D &knorArr, IdefixArray3D &alpha, IdefixArray3D &clessQ) { + IdefixArray4D Vc = data.hydro->Vc; + IdefixArray1D x1 = data.x[IDIR]; + IdefixArray1D x2 = data.x[JDIR]; + real norm = 1.6726e-24*0.5/(udenGlob*uvelGlob*ulenGlob*1.3807e-16); + real uTemp=0.5*uvelGlob*uvelGlob*1.6726e-24/1.3807e-16; + real k0 = k0_Glob*norm; + idefix_for("MyThConductivity",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + kparArr(k,j,i) = k0*pow(Vc(PRS,k,j,i)/Vc(RHO,k,j,i)*uTemp,2.5); + knorArr(k,j,i) = 0.; + alpha(k,j,i) = (1.0-tanh(x1(i)-10))/2; + clessQ(k,j,i) = -1.5; + }); +} + +// User-defined boundaries +void UserDefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + + DataBlock *data = hydro->data; + + if( (dir==IDIR) && (side == left)) { + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Vs = hydro->Vs; + IdefixArray1D x1 = data->x[IDIR]; + + real rc,vc,vwind0; + real cs=cs_vescGlob*sqrt(2.); + real va_vesc = va_vescGlob; + real PonRho; + + PonRho = cs*cs; + rc = 0.25 / (cs_vescGlob*cs_vescGlob); + vc = cs; + vwind0 = ParkerWind(1./rc) * cs; + + hydro->boundary->BoundaryFor("UserDefBoundary", dir, side, + KOKKOS_LAMBDA (int k, int j, int i) { + real r = x1(i); + real mu = va_vesc * sqrt(2.); + Vc(RHO,k,j,i) = vwind0/(vwind0 * r * r); + Vc(PRS,k,j,i) = PonRho * Vc(RHO, k, j, i); + Vc(VX1,k,j,i) = vwind0; + Vc(VX2,k,j,i) = 0.0; + Vc(VX3,k,j,i) = 0.0; + Vc(BX1,k,j,i) = mu / (r*r); + Vc(BX2,k,j,i) = 0.0; + Vc(BX3,k,j,i) = 0.0; + + }); + } +} + +// Initialisation routine. Can be used to allocate +// Arrays or variables which are used later on +Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { + // Set the function for userdefboundary + data.hydro->EnrollUserDefBoundary(&UserDefBoundary); + data.hydro->bragThermalDiffusion->EnrollClessThermalDiffusivity(&MyClessThermalConductivity); + gammaGlob=input.Get("Hydro", "gamma", 0); + udenGlob=input.Get("Setup", "UNIT_DENSITY",0); + ulenGlob=input.Get("Setup", "UNIT_LENGTH",0); + uvelGlob=input.Get("Setup", "UNIT_VELOCITY",0); + cs_vescGlob=input.Get("Setup", "cs_vesc", 0); + va_vescGlob=input.Get("Setup", "va_vesc", 0); + k0_Glob = input.Get("Setup", "k0", 0); +} + +// This routine initialize the flow +// Note that data is on the device. +// One can therefore define locally +// a datahost and sync it, if needed +void Setup::InitFlow(DataBlock &data) { + // Create a host copy + DataBlockHost d(data); + + real r,th,rl; + real PonRho, vwind0, rc, vc; + real cs=cs_vescGlob*sqrt(2.); + + + rc = 0.25 / (cs_vescGlob*cs_vescGlob); + vwind0 = ParkerWind(1./rc) * cs; + PonRho = cs*cs; + real mu = va_vescGlob * sqrt(2.); + + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + r=d.x[IDIR](i); + th=d.x[JDIR](j); + real vwind; + + vwind = ParkerWind(r/rc) * cs; + + d.Vc(RHO,k,j,i) = 1.0*vwind0/(vwind * r * r); + d.Vc(PRS,k,j,i) = PonRho * d.Vc(RHO, k, j, i); + d.Vc(VX1,k,j,i) = vwind; + d.Vc(VX2,k,j,i) = 0.0; + d.Vc(VX3,k,j,i) = 0.0; + + rl=d.xl[IDIR](i); // Radius on the left side of the cell + d.Vs(BX1s, k, j, i) = mu / (rl*rl); + + } + } + } + // Send it all, if needed + d.SyncToDevice(); +} + +// Analyse data to produce an output +void MakeAnalysis(DataBlock & data) { +} + +/**************************************************/ +real ParkerWind(real x) +/* Parker wind velocity in unit of iso sound speed + x = radius / critical radius. +**************************************************/ +{ + real v, f; + real vref; + + v = 1e-7; + f = v*v-2*log(v)-4/x-4*log(x)+3; + if (x>1) {v=10;} + while (fabs(f) > 1e-10){ + vref = v; + v = v - 0.5*f/(v-1/v); + while (v < 0){ + v = (v + vref)/2; + } + f = v*v-2*log(v)-4/x-4*log(x)+3; + } + return v; +} diff --git a/test/MHD/clessTDiffusion/testme.py b/test/MHD/clessTDiffusion/testme.py new file mode 100755 index 000000000..6ed5b1648 --- /dev/null +++ b/test/MHD/clessTDiffusion/testme.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + + +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini"] + + # loop on all the ini files for this test + name = "dump.0001.dmp" + for ini in inifiles: + test.run(inputFile=ini) + test.standardTest() + if test.init: + test.makeReference(filename=name) + test.nonRegressionTest(filename=name, tolerance=2e-15) + + +test=tst.idfxTest() + +if not test.all: + if(test.check): + test.checkOnly(filename="dump.0001.dmp") + else: + testMe(test) +else: + test.noplot = True + test.vectPot=False + test.single=False + test.reconstruction=2 + test.mpi=False + testMe(test) diff --git a/test/MHD/sphBragTDiffusion/idefix.ini b/test/MHD/sphBragTDiffusion/idefix.ini index 35ac54e8b..795ae7bb2 100644 --- a/test/MHD/sphBragTDiffusion/idefix.ini +++ b/test/MHD/sphBragTDiffusion/idefix.ini @@ -13,7 +13,7 @@ nstages 3 [Hydro] solver hlld gamma 1.4 -bragTDiffusion rkl nolimiter constant 50.0 +bragTDiffusion rkl nolimiter nosat constant 50.0 [Setup] amplitude 1e-4 From 7c10ec052d60128aea8d4dd90ee32c8a13f9b06b Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 15 Jan 2025 18:06:53 +0100 Subject: [PATCH 14/82] fix dimensions for alias --- src/utils/column.cpp | 2 +- test/utils/columnDensity/idefix.ini | 4 ++-- test/utils/columnDensity/setup.cpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/utils/column.cpp b/src/utils/column.cpp index a774d4927..34d55f56b 100644 --- a/src/utils/column.cpp +++ b/src/utils/column.cpp @@ -235,6 +235,6 @@ void Column::ComputeColumn(IdefixArray4D in, const int var) { void Column::ComputeColumn(IdefixArray3D in) { // 4D alias - IdefixArray4D arr4D(in.data(), 1, in.extent(2), in.extent(1), in.extent(0)); + IdefixArray4D arr4D(in.data(), 1, in.extent(0), in.extent(1), in.extent(2)); return this->ComputeColumn(arr4D,0); } diff --git a/test/utils/columnDensity/idefix.ini b/test/utils/columnDensity/idefix.ini index 2d994315c..5e80c8bce 100644 --- a/test/utils/columnDensity/idefix.ini +++ b/test/utils/columnDensity/idefix.ini @@ -1,7 +1,7 @@ [Grid] X1-grid 1 0.0 64 u 1.0 -X2-grid 1 0.0 64 u 1.0 -X3-grid 1 0.0 64 u 1.0 +X2-grid 1 0.0 32 u 1.0 +X3-grid 1 0.0 16 u 1.0 [TimeIntegrator] CFL 0.8 diff --git a/test/utils/columnDensity/setup.cpp b/test/utils/columnDensity/setup.cpp index b27b6b1f3..fe7c57e69 100644 --- a/test/utils/columnDensity/setup.cpp +++ b/test/utils/columnDensity/setup.cpp @@ -23,7 +23,7 @@ void Analysis(DataBlock & data) { IdefixArray3D rho("rho",data.np_tot[KDIR],data.np_tot[JDIR],data.np_tot[IDIR]); auto Vc = data.hydro->Vc; - idefix_for("init rho",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], + idefix_for("init rho",data.beg[KDIR],data.end[KDIR],data.beg[JDIR],data.end[JDIR],data.beg[IDIR],data.end[IDIR], KOKKOS_LAMBDA(int k, int j, int i) { rho(k,j,i) = Vc(RHO,k,j,i); }); From 79e70cf233f0ef0dec55e916b8c5ad723ba123ac Mon Sep 17 00:00:00 2001 From: Jean Kempf Date: Thu, 16 Jan 2025 14:12:10 +0100 Subject: [PATCH 15/82] updated ShowConfig functions from braginskii tc and viscosity to display which slope limiter is used, fixed typo in the braginskii doc --- doc/source/modules/braginskii.rst | 2 +- src/fluid/braginskii/bragThermalDiffusion.cpp | 12 ++++++++++-- src/fluid/braginskii/bragThermalDiffusion.hpp | 5 +++++ src/fluid/braginskii/bragViscosity.cpp | 10 +++++++++- src/fluid/braginskii/bragViscosity.hpp | 4 ++++ 5 files changed, 29 insertions(+), 4 deletions(-) diff --git a/doc/source/modules/braginskii.rst b/doc/source/modules/braginskii.rst index e657cedd0..ccc841f4f 100644 --- a/doc/source/modules/braginskii.rst +++ b/doc/source/modules/braginskii.rst @@ -37,7 +37,7 @@ though adapted to vector quantities. cell interface by a simple arithmetic average (Eq. (6)-(7) from Sharma & Hammett 2007). However in the same paper, the authors showed that this implementation can lead to - unphysical heat flux from high to low temperature regions. + unphysical heat flux from low to high temperature regions. So we implemented slope limiters for the computation of these transverse heat fluxes, as described in Eq. (17) from Sharma & Hammett (2007). Only the van Leer and the Monotonized Central (MC) limiters are available diff --git a/src/fluid/braginskii/bragThermalDiffusion.cpp b/src/fluid/braginskii/bragThermalDiffusion.cpp index ee1d5b996..a898ab8bd 100644 --- a/src/fluid/braginskii/bragThermalDiffusion.cpp +++ b/src/fluid/braginskii/bragThermalDiffusion.cpp @@ -42,10 +42,18 @@ void BragThermalDiffusion::ShowConfig() { IDEFIX_ERROR("Unknown time integrator for braginskii thermal diffusion."); } if(haveSlopeLimiter) { - idfx::cout << "Braginskii Thermal Diffusion: uses a slope limiter." << std::endl; + if(haveMonotizedCentral) { + idfx::cout << "Braginskii Thermal Diffusion: uses the monotonized central slope limiter." << std::endl; + } + else if(haveVanLeer) { + idfx::cout << "Braginskii Thermal Diffusion: uses the van Leer slope limiter." << std::endl; + } + else { + IDEFIX_ERROR("Unknown slope limiter for braginskii thermal diffusion."); + } } if(includeCollisionlessTD) { - idfx::cout << "Saturation with collisionless flux is enabled." << std::endl; + idfx::cout << "Braginskii Thermal Diffusion: saturation with collisionless flux is enabled." << std::endl; } } diff --git a/src/fluid/braginskii/bragThermalDiffusion.hpp b/src/fluid/braginskii/bragThermalDiffusion.hpp index 64037420f..2a103a491 100644 --- a/src/fluid/braginskii/bragThermalDiffusion.hpp +++ b/src/fluid/braginskii/bragThermalDiffusion.hpp @@ -59,6 +59,9 @@ class BragThermalDiffusion { bool includeCollisionlessTD{false}; bool haveSlopeLimiter{false}; + bool haveMonotizedCentral{false}; + bool haveVanLeer{false}; + bool haveMinmod{false}; // helper array IdefixArray4D &Vc; @@ -91,9 +94,11 @@ BragThermalDiffusion::BragThermalDiffusion(Input &input, Grid &grid, Fluid if(input.CheckEntry("Hydro","bragTDiffusion")>=0) { if(input.Get("Hydro","bragTDiffusion",1).compare("mc") == 0) { this->haveSlopeLimiter = true; + this->haveMonotizedCentral = true; limiter = PLMLimiter::McLim; } else if(input.Get("Hydro","bragTDiffusion",1).compare("vanleer") == 0) { this->haveSlopeLimiter = true; + this->haveVanLeer = true; limiter = PLMLimiter::VanLeer; } else if(input.Get("Hydro","bragTDiffusion",1).compare("minmod") == 0) { IDEFIX_ERROR("The minmod slope limiter is not available because it has been " diff --git a/src/fluid/braginskii/bragViscosity.cpp b/src/fluid/braginskii/bragViscosity.cpp index 6d5007b50..e05019a43 100644 --- a/src/fluid/braginskii/bragViscosity.cpp +++ b/src/fluid/braginskii/bragViscosity.cpp @@ -59,7 +59,15 @@ void BragViscosity::ShowConfig() { IDEFIX_ERROR("Unknown time integrator for braginskii viscosity."); } if(haveSlopeLimiter) { - idfx::cout << "Braginskii Viscosity: uses a slope limiter." << std::endl; + if(haveMonotizedCentral) { + idfx::cout << "Braginskii Viscosity: uses the monotonized central slope limiter." << std::endl; + } + else if(haveVanLeer) { + idfx::cout << "Braginskii Viscosity: uses the van Leer slope limiter." << std::endl; + } + else { + IDEFIX_ERROR("Unknown slope limiter for braginskii viscosity."); + } } #if GEOMETRY == CYLINDRICAL || GEOMETRY == POLAR diff --git a/src/fluid/braginskii/bragViscosity.hpp b/src/fluid/braginskii/bragViscosity.hpp index 7a8d2b756..5340110a8 100644 --- a/src/fluid/braginskii/bragViscosity.hpp +++ b/src/fluid/braginskii/bragViscosity.hpp @@ -52,6 +52,8 @@ class BragViscosity { DiffusivityFunc bragViscousDiffusivityFunc; bool haveSlopeLimiter{false}; + bool haveMonotizedCentral{false}; + bool haveVanLeer{false}; IdefixArray4D &Vc; IdefixArray4D &Vs; @@ -78,9 +80,11 @@ BragViscosity::BragViscosity(Input &input, Grid &grid, Fluid *hydroin): if(input.CheckEntry("Hydro","bragViscosity")>=0) { if(input.Get("Hydro","bragViscosity",1).compare("vanleer") == 0) { this->haveSlopeLimiter = true; + this->haveVanLeer = true; limiter = PLMLimiter::VanLeer; } else if(input.Get("Hydro","bragViscosity",1).compare("mc") == 0) { this->haveSlopeLimiter = true; + this->haveMonotizedCentral = true; limiter = PLMLimiter::McLim; } else if (input.Get("Hydro","bragViscosity",1).compare("nolimiter") == 0) { this->haveSlopeLimiter = false; From d0f348a823da191a9b5e1c869c0d25aefb779e50 Mon Sep 17 00:00:00 2001 From: Marc Coiffier Date: Fri, 17 Jan 2025 18:19:42 +0100 Subject: [PATCH 16/82] =?UTF-8?q?G=C3=A9n=C3=A9ration=20automatique=20de?= =?UTF-8?q?=20plots=20de=20performance=20dans=20la=20doc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- doc/python_requirements.txt | 1 + doc/source/bench-jeanzay.json | 122 ++++++++++++++++++++++++++++++++ doc/source/conf.py | 1 + doc/source/performances.rst | 20 ++++-- doc/source/plot_idefix_bench.py | 14 ++++ 5 files changed, 153 insertions(+), 5 deletions(-) create mode 100755 doc/source/bench-jeanzay.json create mode 100644 doc/source/plot_idefix_bench.py diff --git a/doc/python_requirements.txt b/doc/python_requirements.txt index 35f72dcf7..c75fa02b7 100644 --- a/doc/python_requirements.txt +++ b/doc/python_requirements.txt @@ -14,3 +14,4 @@ exhale==0.3.7 m2r2==0.3.2 sphinx-copybutton==0.5.2 #sphinxcontrib-applehelp==1.0.7 +matplotlib==3.10.0 diff --git a/doc/source/bench-jeanzay.json b/doc/source/bench-jeanzay.json new file mode 100755 index 000000000..f58d38d42 --- /dev/null +++ b/doc/source/bench-jeanzay.json @@ -0,0 +1,122 @@ +[ + { + "date": "2025-01-17_15:33:13", + "gpumodel": "v100", + "idefix_commit": "d0e82202ca90d0b664c71582962c2ae255bc6528", + "bench_commit": "a94bc00490b3c62f81f7713327f4388e44a2abea", + "results": [ + { + "nbgpu": 1, + "cell_updates": 1.192208E+8 + }, + { + "nbgpu": 2, + "cell_updates": 1.178276E+8 + }, + { + "nbgpu": 4, + "cell_updates": 1.151014E+8 + } + ] + }, + { + "date": "2025-01-17_15:35:16", + "gpumodel": "v100", + "idefix_commit": "d0e82202ca90d0b664c71582962c2ae255bc6528", + "bench_commit": "a94bc00490b3c62f81f7713327f4388e44a2abea", + "results": [ + { + "nbgpu": 1, + "cell_updates": 1.192139E+8 + }, + { + "nbgpu": 2, + "cell_updates": 1.178690E+8 + }, + { + "nbgpu": 4, + "cell_updates": 1.092092E+8 + }, + { + "nbgpu": 8, + "cell_updates": 7.464571E+7 + }, + { + "nbgpu": 16, + "cell_updates": 7.485223E+7 + }, + { + "nbgpu": 32, + "cell_updates": 6.795755E+7 + } + ] + }, + { + "date": "2025-01-17_16:57:02", + "gpumodel": "v100", + "idefix_commit": "d0e82202ca90d0b664c71582962c2ae255bc6528", + "bench_commit": "a94bc00490b3c62f81f7713327f4388e44a2abea", + "results": [ + { + "nbgpu": 1, + "cell_updates": 1.192784E+8 + }, + { + "nbgpu": 2, + "cell_updates": 1.117701E+8 + }, + { + "nbgpu": 4, + "cell_updates": 1.087580E+8 + }, + { + "nbgpu": 8, + "cell_updates": 7.472364E+7 + }, + { + "nbgpu": 16, + "cell_updates": 7.554497E+7 + }, + { + "nbgpu": 32, + "cell_updates": 6.669339E+7 + } + ] + }, + { + "date": "2025-01-17_17:09:39", + "gpumodel": "a100", + "idefix_commit": "d0e82202ca90d0b664c71582962c2ae255bc6528", + "bench_commit": "a94bc00490b3c62f81f7713327f4388e44a2abea", + "results": [ + { + "nbgpu": 1, + "cell_updates": 2.035935E+8 + }, + { + "nbgpu": 2, + "cell_updates": 1.992927E+8 + }, + { + "nbgpu": 4, + "cell_updates": 1.958386E+8 + }, + { + "nbgpu": 8, + "cell_updates": 1.929568E+8 + } + ] + }, + { + "date": "2025-01-17_17:23:13", + "gpumodel": "h100", + "idefix_commit": "d0e82202ca90d0b664c71582962c2ae255bc6528", + "bench_commit": "a94bc00490b3c62f81f7713327f4388e44a2abea", + "results": [ + { + "nbgpu": 1, + "cell_updates": 3.083581E+8 + } + ] + } +] diff --git a/doc/source/conf.py b/doc/source/conf.py index 24f9709bb..6f0cffa44 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -35,6 +35,7 @@ extensions = [ "sphinx_rtd_theme", 'sphinx_git', + 'matplotlib.sphinxext.plot_directive', "breathe", "exhale", "m2r2", diff --git a/doc/source/performances.rst b/doc/source/performances.rst index 84042685e..985991e34 100644 --- a/doc/source/performances.rst +++ b/doc/source/performances.rst @@ -33,16 +33,26 @@ CPU performances | IDRIS/Jean Zay | Intel Cascade Lake | 0.62 | +---------------------+--------------------+----------------------------------------------------+ - GPU performances ================ +.. plot:: + + import plot_idefix_bench + plot_idefix_bench.do_plot('Performance on NVidia V100 GPUs at Jean-Zay', 'bench-jeanzay.json', 'v100') + +.. plot:: + + import plot_idefix_bench + plot_idefix_bench.do_plot('Performance on NVidia A100 GPUs at Jean-Zay', 'bench-jeanzay.json', 'a100') + +.. plot:: + + import plot_idefix_bench + plot_idefix_bench.do_plot('Performance on NVidia H100 GPUs at Jean-Zay', 'bench-jeanzay.json', 'h100') + +----------------------+--------------------+----------------------------------------------------+ | Cluster name | GPU | Performances (in 10\ :sup:`6` cell/s/GPU) | +======================+====================+====================================================+ -| IDRIS/Jean Zay | NVIDIA V100 | 110 | -+----------------------+--------------------+----------------------------------------------------+ -| IDRIS/Jean Zay | NVIDIA A100 | 194 | -+----------------------+--------------------+----------------------------------------------------+ | CINES/Adastra | AMD Mi250 | 250 | +----------------------+--------------------+----------------------------------------------------+ diff --git a/doc/source/plot_idefix_bench.py b/doc/source/plot_idefix_bench.py new file mode 100644 index 000000000..cd45aa5a9 --- /dev/null +++ b/doc/source/plot_idefix_bench.py @@ -0,0 +1,14 @@ +import matplotlib.pyplot as plt +import json + +def do_plot(title, bench_file, gpumodel): + with open(bench_file, 'r') as f: + benches = json.load(f) + + select = [bench for bench in benches if bench['gpumodel'] == gpumodel][-1] + xs = [r['nbgpu'] for r in select['results']] + ys = [r['cell_updates'] for r in select['results']] + + plt.xscale("log") + plt.plot(xs, ys) + plt.title(title) From a61eb61af47c10e7caf0810a0560457af1ea809d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Victor=20R=C3=A9ville?= Date: Mon, 20 Jan 2025 12:33:43 +0100 Subject: [PATCH 17/82] Updated doc for Collisionless TC --- doc/source/modules/braginskii.rst | 63 ++++++++++++++++++++++++++++--- reference | 2 +- 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/doc/source/modules/braginskii.rst b/doc/source/modules/braginskii.rst index ccc841f4f..4c1c4dd6f 100644 --- a/doc/source/modules/braginskii.rst +++ b/doc/source/modules/braginskii.rst @@ -4,7 +4,7 @@ Braginskii module =================== Equations solved and methods ---------------------------- +---------------------------- The ``Braginskii`` module implements the anisotropic heat and momentum fluxes specific to weakly collisional, magnetised plasma like the intracluster medium @@ -72,18 +72,69 @@ of the Braginskii heat flux and viscosity. .. _braginskiiParameterSection: + +Saturation with collisionless heat flux +--------------------------------------- + +The ``Braginskii`` module can include a collisionless saturation of the Braginskii heat flux. +The heat flux is then computed as follows: + +:math:`q = \alpha (q_B + q_\perp) + (1-\alpha)\beta*p*v`, + +where :math:`\alpha \in [0,1]` controls the transition between the Braginskii heat flux and the collisionless heat flux +and :math:`\beta` controls the amplitude of the collisionless heat flux (typically :math:`\beta \in [1,4]`, see Hollweg 1976). + +.. note:: + As a result, even with :math:`\kappa_\perp = 0`, the heat flux is no longer necessarilly strictly aligned with the magnetic field. +.. note:: + The collisionless heat flux is a hyperbolic term and the diffusion coefficient is set proportional to :math:`\alpha`. +.. note:: + If selected, slope limiters are also used in the collisionless flux, where an upwind scheme has been implemented for stability. +.. note:: + This saturation has been thought to be used mostly using the uerdef function that takes four arrays as input and is enrolled through + ``data.hydro->bragThermalDiffusion->EnrollClessThermalDiffusivity()`` + Main parameters of the module ----------------------------- The ``Braginskii`` module can be enabled adding one or two lines in the ``[Hydro]`` section -starting with the keyword -`bragTDiffusion` or/and *bragViscosity*. The following table summarises the different options +starting with the keyword `bragTDiffusion` or/and *bragViscosity*. The following tables summarise the different options associated to the activation of the Braginskii module: +--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ | Column | Entry name | Parameter type | Comment | +========+=======================+=========================+=======================================================================================+ -| 0 | bragModule | string | | Activates Braginskii diffusion. Can be ``bragTDiffusion`` or ``bragViscosity``. | +| 0 | bragTDiffusion | string | | Activates Braginskii thermal diffusion. | ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ +| 1 | integration | string | | Specifies the type of scheme to be used to integrate the parabolic term. | +| | | | | Can be ``rkl`` or ``explicit``. | ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ +| 2 | slope limiter | string | | Choose the type of limiter to be used to compute anisotropic transverse flux terms. | +| | | | | Can be ``mc``, ``vanleer`` or ``nolimiter``. | ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ +| 3 | saturation mode | string | | Include or not collisionless saturation. Can be ``nosat`` or ``wcless``. | ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ +| 4 | diffusivity type | string | | Specifies the type of diffusivity wanted. Can be ``constant`` or ``userdef``. | ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ +| 5 | parallel diffusivity | real | | Mandatory if the diffusivity type is ``constant``. Not needed otherwise. | +| | | | | Value of the parallel diffusivity. Should be a real number. | ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ +| 6 | normal diffusivity | real | | When bragModule ``bragTDiffusion`` and diffusivity type ``constant``, | +| | | | | value of the normal diffusivity. Should be a real number. | ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ +| 7 | alpha collisionless | real | | If the diffusivity type is ``constant`` and saturation is ``wcless``. | +| | | | | Set to 1 if not provided. | ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ +| 8 | beta collisionless | real | | If the diffusivity type is ``constant`` and saturation is ``wcless``. | +| | | | | Set to 0 if not provided. | ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ + +for the *bragViscosity*: + ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ +| Column | Entry name | Parameter type | Comment | ++========+=======================+=========================+=======================================================================================+ +| 0 | bragViscosity | string | | Activates Braginskii viscosity. | +--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ | 1 | integration | string | | Specifies the type of scheme to be used to integrate the parabolic term. | | | | | | Can be ``rkl`` or ``explicit``. | @@ -101,7 +152,7 @@ associated to the activation of the Braginskii module: +--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ Numerical checks ---------------- +---------------- In Cartesian geometry, the ``Braginskii`` module has been tested with many setups and in all configurations of magnetic polarisation: @@ -119,3 +170,5 @@ The same goes for the anisotropic heat flux in Cylindrical/Polar geometry while the anisotropic viscosity has *never* been tested in this geometry. In spherical geometry, both ``Braginskii`` operators have been partially validated (diffusion along the polar axis has not been directly tested). + +The collisionless saturation has been tested in 1D and 2D spherical geometry. diff --git a/reference b/reference index 99f339c43..609a60164 160000 --- a/reference +++ b/reference @@ -1 +1 @@ -Subproject commit 99f339c43d98274a925282cc96252c5a0fa9374f +Subproject commit 609a60164461fb2d4fbeffa89ce148acf1191525 From 7bef7a974c409fea952dc75ea5f81e51f657e783 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Victor=20R=C3=A9ville?= Date: Mon, 20 Jan 2025 16:46:27 +0100 Subject: [PATCH 18/82] Corrected doc typo --- doc/source/modules/braginskii.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/modules/braginskii.rst b/doc/source/modules/braginskii.rst index 4c1c4dd6f..d9cb4f2d8 100644 --- a/doc/source/modules/braginskii.rst +++ b/doc/source/modules/braginskii.rst @@ -91,7 +91,7 @@ and :math:`\beta` controls the amplitude of the collisionless heat flux (typical .. note:: If selected, slope limiters are also used in the collisionless flux, where an upwind scheme has been implemented for stability. .. note:: - This saturation has been thought to be used mostly using the uerdef function that takes four arrays as input and is enrolled through + This saturation has been thought to be used mostly using the userdef function that takes four arrays as input and is enrolled through ``data.hydro->bragThermalDiffusion->EnrollClessThermalDiffusivity()`` Main parameters of the module From 8f48bc2b34f5223be8de1faa61d2c5bd05cf95b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Victor=20R=C3=A9ville?= Date: Tue, 21 Jan 2025 13:22:36 +0100 Subject: [PATCH 19/82] Corrected typo --- src/fluid/braginskii/bragThermalDiffusion.cpp | 14 +++++++------- src/fluid/braginskii/bragThermalDiffusion.hpp | 4 ++-- src/fluid/braginskii/bragViscosity.cpp | 9 ++++----- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/fluid/braginskii/bragThermalDiffusion.cpp b/src/fluid/braginskii/bragThermalDiffusion.cpp index a898ab8bd..0dd0b0331 100644 --- a/src/fluid/braginskii/bragThermalDiffusion.cpp +++ b/src/fluid/braginskii/bragThermalDiffusion.cpp @@ -42,18 +42,18 @@ void BragThermalDiffusion::ShowConfig() { IDEFIX_ERROR("Unknown time integrator for braginskii thermal diffusion."); } if(haveSlopeLimiter) { - if(haveMonotizedCentral) { - idfx::cout << "Braginskii Thermal Diffusion: uses the monotonized central slope limiter." << std::endl; - } - else if(haveVanLeer) { + if(haveMonotonizedCentral) { + idfx::cout << "Braginskii Thermal Diffusion: " + "uses the monotonized central slope limiter." << std::endl; + } else if(haveVanLeer) { idfx::cout << "Braginskii Thermal Diffusion: uses the van Leer slope limiter." << std::endl; - } - else { + } else { IDEFIX_ERROR("Unknown slope limiter for braginskii thermal diffusion."); } } if(includeCollisionlessTD) { - idfx::cout << "Braginskii Thermal Diffusion: saturation with collisionless flux is enabled." << std::endl; + idfx::cout << "Braginskii Thermal Diffusion: saturation" + " with collisionless flux is enabled." << std::endl; } } diff --git a/src/fluid/braginskii/bragThermalDiffusion.hpp b/src/fluid/braginskii/bragThermalDiffusion.hpp index 2a103a491..d05c20b4e 100644 --- a/src/fluid/braginskii/bragThermalDiffusion.hpp +++ b/src/fluid/braginskii/bragThermalDiffusion.hpp @@ -59,7 +59,7 @@ class BragThermalDiffusion { bool includeCollisionlessTD{false}; bool haveSlopeLimiter{false}; - bool haveMonotizedCentral{false}; + bool haveMonotonizedCentral{false}; bool haveVanLeer{false}; bool haveMinmod{false}; @@ -94,7 +94,7 @@ BragThermalDiffusion::BragThermalDiffusion(Input &input, Grid &grid, Fluid if(input.CheckEntry("Hydro","bragTDiffusion")>=0) { if(input.Get("Hydro","bragTDiffusion",1).compare("mc") == 0) { this->haveSlopeLimiter = true; - this->haveMonotizedCentral = true; + this->haveMonotonizedCentral = true; limiter = PLMLimiter::McLim; } else if(input.Get("Hydro","bragTDiffusion",1).compare("vanleer") == 0) { this->haveSlopeLimiter = true; diff --git a/src/fluid/braginskii/bragViscosity.cpp b/src/fluid/braginskii/bragViscosity.cpp index e05019a43..e53ab2c65 100644 --- a/src/fluid/braginskii/bragViscosity.cpp +++ b/src/fluid/braginskii/bragViscosity.cpp @@ -60,12 +60,11 @@ void BragViscosity::ShowConfig() { } if(haveSlopeLimiter) { if(haveMonotizedCentral) { - idfx::cout << "Braginskii Viscosity: uses the monotonized central slope limiter." << std::endl; - } - else if(haveVanLeer) { + idfx::cout << "Braginskii Viscosity: uses the " + "monotonized central slope limiter." << std::endl; + } else if(haveVanLeer) { idfx::cout << "Braginskii Viscosity: uses the van Leer slope limiter." << std::endl; - } - else { + } else { IDEFIX_ERROR("Unknown slope limiter for braginskii viscosity."); } } From 1f891c76289640fdd25ac27f451a1c6d7be06c3b Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 21 Jan 2025 17:34:37 +0100 Subject: [PATCH 20/82] add global routines to dump Idefix arrays (useful for debugging or checking that everything works as expected) --- src/global.hpp | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/global.hpp b/src/global.hpp index 8651f9748..0cf57e15f 100644 --- a/src/global.hpp +++ b/src/global.hpp @@ -44,6 +44,34 @@ IdefixArray1D ConvertVectorToIdefixArray(std::vector &inputVector) { return(outArr); } +///< dump Idefix array to a numpy array on disk +template +void dumpArray(std::string filename, IdefixArray3D array) { + IdefixHostArray3D hArray = Kokkos::create_mirror(array); + Kokkos::deep_copy(hArray,array); + + std::array shape; + bool fortran_order{false}; + shape[0] = array.extent(0); + shape[1] = array.extent(1); + shape[2] = array.extent(2); + npy::SaveArrayAsNumpy(filename, fortran_order, 3, shape.data(), hArray.data()); +} + +template +void dumpArray(std::string filename, IdefixArray4D array) { + IdefixHostArray4D hArray = Kokkos::create_mirror(array); + Kokkos::deep_copy(hArray,array); + + std::array shape; + bool fortran_order{false}; + shape[0] = array.extent(0); + shape[1] = array.extent(1); + shape[2] = array.extent(2); + shape[3] = array.extent(3); + npy::SaveArrayAsNumpy(filename, fortran_order, 4, shape.data(), hArray.data()); +} + } // namespace idfx class idfx::IdefixOutStream { From afc3e4dbb2cd052360cdbbe2ca4473e128491752 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 21 Jan 2025 17:37:24 +0100 Subject: [PATCH 21/82] add missing header --- src/global.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/global.hpp b/src/global.hpp index 0cf57e15f..1a8944163 100644 --- a/src/global.hpp +++ b/src/global.hpp @@ -11,6 +11,7 @@ #include #include #include "arrays.hpp" +#include "npy.hpp" namespace idfx { int initialize(); // Initialisation routine for idefix From e840e2f938f05f64ad153c2fd54db457844e68b8 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 21 Jan 2025 21:23:47 +0100 Subject: [PATCH 22/82] revert back to old reference commit sha to avoid merge conflicts --- reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reference b/reference index 609a60164..99f339c43 160000 --- a/reference +++ b/reference @@ -1 +1 @@ -Subproject commit 609a60164461fb2d4fbeffa89ce148acf1191525 +Subproject commit 99f339c43d98274a925282cc96252c5a0fa9374f From ac104701e4900c45888eda1b162483b7e240f5f4 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 22 Jan 2025 09:36:35 +0100 Subject: [PATCH 23/82] go back to last reference before fork --- reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reference b/reference index 99f339c43..b675bceaa 160000 --- a/reference +++ b/reference @@ -1 +1 @@ -Subproject commit 99f339c43d98274a925282cc96252c5a0fa9374f +Subproject commit b675bceaa6aabc01dded346e2d631857f698dc76 From 76e399aadfc86e48add05dc079b9d6131de320de Mon Sep 17 00:00:00 2001 From: Marc Coiffier Date: Fri, 24 Jan 2025 11:56:44 +0100 Subject: [PATCH 24/82] =?UTF-8?q?R=C3=A9cup=C3=A9ration=20des=20derniers?= =?UTF-8?q?=20benchmarks=20sur=20JeanZay?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- doc/source/bench-jeanzay.json | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) mode change 100755 => 100644 doc/source/bench-jeanzay.json diff --git a/doc/source/bench-jeanzay.json b/doc/source/bench-jeanzay.json old mode 100755 new mode 100644 index f58d38d42..6d5fd5ca1 --- a/doc/source/bench-jeanzay.json +++ b/doc/source/bench-jeanzay.json @@ -80,6 +80,14 @@ { "nbgpu": 32, "cell_updates": 6.669339E+7 + }, + { + "nbgpu": 64, + "cell_updates": 5.497455E+7 + }, + { + "nbgpu": 128, + "cell_updates": 5.440610E+7 } ] }, @@ -104,6 +112,22 @@ { "nbgpu": 8, "cell_updates": 1.929568E+8 + }, + { + "nbgpu": 16, + "cell_updates": 9.408132E+7 + }, + { + "nbgpu": 32, + "cell_updates": 5.937487E+7 + }, + { + "nbgpu": 64, + "cell_updates": 4.159838E+7 + }, + { + "nbgpu": 128, + "cell_updates": 5.061931E+7 } ] }, From 49a56ee5b83d0822831bcb8f41859104bee5ea41 Mon Sep 17 00:00:00 2001 From: Marc Coiffier Date: Fri, 24 Jan 2025 14:33:29 +0100 Subject: [PATCH 25/82] =?UTF-8?q?R=C3=A9cup=C3=A9ration=20de=20benchs=20po?= =?UTF-8?q?ur=20H100;=20ajout=20de=20labels=20aux=20axes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- doc/source/bench-jeanzay.json | 64 +++++++++++++++++++++++++++++++++ doc/source/plot_idefix_bench.py | 7 ++-- 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/doc/source/bench-jeanzay.json b/doc/source/bench-jeanzay.json index 6d5fd5ca1..7ded6914e 100644 --- a/doc/source/bench-jeanzay.json +++ b/doc/source/bench-jeanzay.json @@ -142,5 +142,69 @@ "cell_updates": 3.083581E+8 } ] + }, + { + "date": "2025-01-24_13:02:08", + "gpumodel": "h100", + "idefix_commit": "54482d582361f1cd05c39717902f752b3d04ae6e", + "bench_commit": "a94bc00490b3c62f81f7713327f4388e44a2abea", + "results": [ + { + "nbgpu": 1, + "cell_updates": 3.076537E+8 + } + ] + }, + { + "date": "2025-01-24_13:04:02", + "gpumodel": "h100", + "idefix_commit": "54482d582361f1cd05c39717902f752b3d04ae6e", + "bench_commit": "a94bc00490b3c62f81f7713327f4388e44a2abea", + "results": [ + { + "nbgpu": 1, + "cell_updates": 3.032686E+8 + }, + { + "nbgpu": 2, + "cell_updates": 3.007089E+8 + }, + { + "nbgpu": 4, + "cell_updates": 2.938962E+8 + } + ] + }, + { + "date": "2025-01-24_14:17:30", + "gpumodel": "h100", + "idefix_commit": "54482d582361f1cd05c39717902f752b3d04ae6e", + "bench_commit": "a94bc00490b3c62f81f7713327f4388e44a2abea", + "results": [ + { + "nbgpu": 1, + "cell_updates": 3.074363E+8 + }, + { + "nbgpu": 2, + "cell_updates": 3.009340E+8 + }, + { + "nbgpu": 4, + "cell_updates": 2.938868E+8 + }, + { + "nbgpu": 8, + "cell_updates": 2.833054E+8 + }, + { + "nbgpu": 16, + "cell_updates": 2.822483E+8 + }, + { + "nbgpu": 32, + "cell_updates": 2.817151E+8 + } + ] } ] diff --git a/doc/source/plot_idefix_bench.py b/doc/source/plot_idefix_bench.py index cd45aa5a9..ac91b787e 100644 --- a/doc/source/plot_idefix_bench.py +++ b/doc/source/plot_idefix_bench.py @@ -8,7 +8,10 @@ def do_plot(title, bench_file, gpumodel): select = [bench for bench in benches if bench['gpumodel'] == gpumodel][-1] xs = [r['nbgpu'] for r in select['results']] ys = [r['cell_updates'] for r in select['results']] - - plt.xscale("log") + + plt.xscale("log", base=2) plt.plot(xs, ys) + plt.ylim(0,max(ys)*1.1) + plt.xlabel("Number of GPUs") + plt.ylabel("Performance (cells / second / GPU)") plt.title(title) From 0cb52eaeecb730f3264344e2b5c6ccd36c18c92e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Fri, 24 Jan 2025 13:34:54 +0000 Subject: [PATCH 26/82] [pre-commit.ci lite] apply automatic fixes --- doc/source/performances.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/performances.rst b/doc/source/performances.rst index 985991e34..6f8c88b2f 100644 --- a/doc/source/performances.rst +++ b/doc/source/performances.rst @@ -50,7 +50,7 @@ GPU performances import plot_idefix_bench plot_idefix_bench.do_plot('Performance on NVidia H100 GPUs at Jean-Zay', 'bench-jeanzay.json', 'h100') - + +----------------------+--------------------+----------------------------------------------------+ | Cluster name | GPU | Performances (in 10\ :sup:`6` cell/s/GPU) | +======================+====================+====================================================+ From 0c07a8e2f309489ccc9c7e836f16271ef1783105 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Victor=20R=C3=A9ville?= Date: Wed, 29 Jan 2025 08:35:10 +0100 Subject: [PATCH 27/82] Use a std::vector of IdefixArray as input for the userdef braginskii diffusion --- doc/source/modules/braginskii.rst | 5 +- src/fluid/braginskii/bragThermalDiffusion.cpp | 13 +--- src/fluid/braginskii/bragThermalDiffusion.hpp | 65 ++++++++++--------- src/fluid/fluid_defs.hpp | 8 +-- test/MHD/MTI/setup.cpp | 6 +- test/MHD/clessTDiffusion/CMakeLists.txt | 1 - test/MHD/clessTDiffusion/python/testidefix.py | 2 +- test/MHD/clessTDiffusion/setup.cpp | 26 +++++--- 8 files changed, 64 insertions(+), 62 deletions(-) diff --git a/doc/source/modules/braginskii.rst b/doc/source/modules/braginskii.rst index d9cb4f2d8..fd5f2b8fc 100644 --- a/doc/source/modules/braginskii.rst +++ b/doc/source/modules/braginskii.rst @@ -76,7 +76,7 @@ of the Braginskii heat flux and viscosity. Saturation with collisionless heat flux --------------------------------------- -The ``Braginskii`` module can include a collisionless saturation of the Braginskii heat flux. +The ``Braginskii`` module can include a collisionless saturation of the Braginskii heat flux, typically due to supra-thermal electrons. The heat flux is then computed as follows: :math:`q = \alpha (q_B + q_\perp) + (1-\alpha)\beta*p*v`, @@ -91,8 +91,7 @@ and :math:`\beta` controls the amplitude of the collisionless heat flux (typical .. note:: If selected, slope limiters are also used in the collisionless flux, where an upwind scheme has been implemented for stability. .. note:: - This saturation has been thought to be used mostly using the userdef function that takes four arrays as input and is enrolled through - ``data.hydro->bragThermalDiffusion->EnrollClessThermalDiffusivity()`` + This saturation has been thought to be used mostly using the userdef function that takes four userdef arrays as input. Main parameters of the module ----------------------------- diff --git a/src/fluid/braginskii/bragThermalDiffusion.cpp b/src/fluid/braginskii/bragThermalDiffusion.cpp index 0dd0b0331..f251d0b0c 100644 --- a/src/fluid/braginskii/bragThermalDiffusion.cpp +++ b/src/fluid/braginskii/bragThermalDiffusion.cpp @@ -19,7 +19,6 @@ #include "eos.hpp" - void BragThermalDiffusion::ShowConfig() { if(status.status==Constant) { idfx::cout << "Braginskii Thermal Diffusion: ENABLED with constant diffusivity kpar=" @@ -27,7 +26,7 @@ void BragThermalDiffusion::ShowConfig() { } else if (status.status==UserDefFunction) { idfx::cout << "Braginskii Thermal Diffusion: ENABLED with user-defined diffusivity function." << std::endl; - if(!bragDiffusivityFunc && !clessDiffusivityFunc) { + if(!bragDiffusivityFunc) { IDEFIX_ERROR("No braginskii thermal diffusion function has been enrolled"); } } else { @@ -57,7 +56,7 @@ void BragThermalDiffusion::ShowConfig() { } } -void BragThermalDiffusion::EnrollBragThermalDiffusivity(TwoArrayDiffusivityFunc myFunc) { +void BragThermalDiffusion::EnrollBragThermalDiffusivity(BragDiffusivityFunc myFunc) { if(this->status.status != UserDefFunction) { IDEFIX_WARNING("Braginskii thermal diffusivity enrollment requires Hydro/BragThermalDiffusion " "to be set to userdef in .ini file"); @@ -65,14 +64,6 @@ void BragThermalDiffusion::EnrollBragThermalDiffusivity(TwoArrayDiffusivityFunc this->bragDiffusivityFunc = myFunc; } -void BragThermalDiffusion::EnrollClessThermalDiffusivity(FourArrayDiffusivityFunc myFunc) { - if(this->status.status != UserDefFunction) { - IDEFIX_WARNING("Collisionless/Braginskii thermal diffusivity enrollment requires " - "Hydro/BragThermalDiffusion to be set to userdef in .ini file"); - } - this->clessDiffusivityFunc = myFunc; -} - void BragThermalDiffusion::AddBragDiffusiveFlux(int dir, const real t, const IdefixArray4D &Flux) { idfx::pushRegion("BragThermalDiffusion::AddBragDiffusiveFlux"); diff --git a/src/fluid/braginskii/bragThermalDiffusion.hpp b/src/fluid/braginskii/bragThermalDiffusion.hpp index d05c20b4e..7b1875ff9 100644 --- a/src/fluid/braginskii/bragThermalDiffusion.hpp +++ b/src/fluid/braginskii/bragThermalDiffusion.hpp @@ -9,6 +9,7 @@ #define FLUID_BRAGINSKII_BRAGTHERMALDIFFUSION_HPP_ #include +#include #include "idefix.hpp" #include "input.hpp" @@ -35,8 +36,7 @@ class BragThermalDiffusion { void AddBragDiffusiveFluxLim(int, const real, const IdefixArray4D &); // Enroll user-defined thermal conductivity - void EnrollBragThermalDiffusivity(TwoArrayDiffusivityFunc); - void EnrollClessThermalDiffusivity(FourArrayDiffusivityFunc); + void EnrollBragThermalDiffusivity(BragDiffusivityFunc); IdefixArray3D heatSrc; // Source terms of the thermal operator IdefixArray3D knorArr; @@ -53,8 +53,7 @@ class BragThermalDiffusion { // status of the module ParabolicModuleStatus &status; - TwoArrayDiffusivityFunc bragDiffusivityFunc; - FourArrayDiffusivityFunc clessDiffusivityFunc; + BragDiffusivityFunc bragDiffusivityFunc; bool includeCollisionlessTD{false}; @@ -307,7 +306,8 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, if(includeCollisionlessTD == false && haveThermalDiffusion == UserDefFunction && dir == IDIR) { if(bragDiffusivityFunc) { idfx::pushRegion("UserDef::BragThermalDiffusivityFunction"); - bragDiffusivityFunc(*this->data, t, kparArr, knorArr); + std::vector> userdefArr = {kparArr, knorArr}; + bragDiffusivityFunc(*this->data, t, userdefArr); idfx::popRegion(); } else { @@ -315,9 +315,10 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, } } else if(includeCollisionlessTD == true && haveThermalDiffusion == UserDefFunction && dir == IDIR) { - if (clessDiffusivityFunc) { + if (bragDiffusivityFunc) { idfx::pushRegion("UserDef::ClessThermalDiffusivityFunction"); - clessDiffusivityFunc(*this->data, t, kparArr, knorArr, clessAlphaArr, clessBetaArr); + std::vector> userdefArr = {kparArr, knorArr, clessAlphaArr, clessBetaArr}; + bragDiffusivityFunc(*this->data, t, userdefArr); idfx::popRegion(); } else { IDEFIX_ERROR("No user-defined Braginskii/collisionless " @@ -484,14 +485,14 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, dpm = Vc(PRS,k,j,i) - Vc(PRS,k,j,i-1); /* Upwind scheme */ - if (Vc(VX1,k,j,i) > 0.0) { - Vn = Vc(VX1,k,j,i-1)+HALF_F*SL::PLMLim(dvm, dvp); - Pn = Vc(PRS,k,j,i-1)+HALF_F*SL::PLMLim(dpm, dpp); - } - else { - Vn = Vc(VX1,k,j,i)-HALF_F*SL::PLMLim(dvm, dvp); - Pn = Vc(PRS,k,j,i)-HALF_F*SL::PLMLim(dpm, dpp); - } + if (Vc(VX1,k,j,i) > 0.0) { + Vn = Vc(VX1,k,j,i-1)+HALF_F*SL::PLMLim(dvm, dvp); + Pn = Vc(PRS,k,j,i-1)+HALF_F*SL::PLMLim(dpm, dpp); + } + else { + Vn = Vc(VX1,k,j,i)-HALF_F*SL::PLMLim(dvm, dvp); + Pn = Vc(PRS,k,j,i)-HALF_F*SL::PLMLim(dpm, dpp); + } } else { Vn = HALF_F*(Vc(VX1,k,j,i) + Vc(VX1,k,j,i-1)); Pn = HALF_F*(Vc(PRS,k,j,i) + Vc(PRS,k,j,i-1)); @@ -649,14 +650,14 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, dpp = Vc(PRS,k,j+1,i) - Vc(PRS,k,j,i); dpm = Vc(PRS,k,j,i) - Vc(PRS,k,j-1,i); - /* Upwind scheme */ - if (Vc(VX2,k,j,i) > 0.0) { - Vn = Vc(VX2,k,j-1,i)+HALF_F*SL::PLMLim(dvm, dvp); - Pn = Vc(PRS,k,j-1,i)+HALF_F*SL::PLMLim(dpm, dpp); - } else { - Vn = Vc(VX2,k,j,i)-HALF_F*SL::PLMLim(dvm, dvp); - Pn = Vc(PRS,k,j,i)-HALF_F*SL::PLMLim(dpm, dpp); - } + /* Upwind scheme */ + if (Vc(VX2,k,j,i) > 0.0) { + Vn = Vc(VX2,k,j-1,i)+HALF_F*SL::PLMLim(dvm, dvp); + Pn = Vc(PRS,k,j-1,i)+HALF_F*SL::PLMLim(dpm, dpp); + } else { + Vn = Vc(VX2,k,j,i)-HALF_F*SL::PLMLim(dvm, dvp); + Pn = Vc(PRS,k,j,i)-HALF_F*SL::PLMLim(dpm, dpp); + } } else { Vn = HALF_F*(Vc(VX2,k,j-1,i) + Vc(VX2,k,j,i)); Pn = HALF_F*(Vc(PRS,k,j-1,i) + Vc(PRS,k,j,i)); @@ -772,14 +773,14 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, dpp = Vc(PRS,k+1,j,i) - Vc(PRS,k,j,i); dpm = Vc(PRS,k,j,i) - Vc(PRS,k-1,j,i); - /* Upwind scheme */ - if (Vc(VX3,k,j,i) > 0.0) { - Vn = Vc(VX3,k-1,j,i)+HALF_F*SL::PLMLim(dvp, dvm); - Pn = Vc(PRS,k-1,j,i)+HALF_F*SL::PLMLim(dpp, dpm); - } else { - Vn = Vc(VX3,k,j,i)-HALF_F*SL::PLMLim(dvp, dvm); - Pn = Vc(PRS,k,j,i)-HALF_F*SL::PLMLim(dpp, dpm); - } + /* Upwind scheme */ + if (Vc(VX3,k,j,i) > 0.0) { + Vn = Vc(VX3,k-1,j,i)+HALF_F*SL::PLMLim(dvp, dvm); + Pn = Vc(PRS,k-1,j,i)+HALF_F*SL::PLMLim(dpp, dpm); + } else { + Vn = Vc(VX3,k,j,i)-HALF_F*SL::PLMLim(dvp, dvm); + Pn = Vc(PRS,k,j,i)-HALF_F*SL::PLMLim(dpp, dpm); + } } else { Vn = HALF_F*(Vc(VX3,k-1,j,i) + Vc(VX3,k,j,i)); Pn = HALF_F*(Vc(PRS,k-1,j,i) + Vc(PRS,k,j,i)); @@ -790,8 +791,8 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, bgradT = D_EXPAND( Bi*dTi , + Bj*dTj, +Bk*dTk); Bmag = D_EXPAND( Bi*Bi , + Bj*Bj, + Bk*Bk); + // EXPAND can yield uexpected behaviour when DIMENSIONS < COMPONENTS //printf("%f , %f\n", Bmag, EXPAND(Bi*Bi, + Bj*Bj, + Bk*Bk)); - //printf("%f , %f \n", 0.5*(Vc(VX1,k,j,i) + Vc(VX1,k,j,i+1)), Vn); Bmag = sqrt(Bmag); Bmag = FMAX(1e-6*SMALL_NUMBER,Bmag); diff --git a/src/fluid/fluid_defs.hpp b/src/fluid/fluid_defs.hpp index 1d040071c..3f68c9b49 100644 --- a/src/fluid/fluid_defs.hpp +++ b/src/fluid/fluid_defs.hpp @@ -8,6 +8,7 @@ #ifndef FLUID_FLUID_DEFS_HPP_ #define FLUID_FLUID_DEFS_HPP_ +#include #include "../idefix.hpp" @@ -49,12 +50,9 @@ using SrcTermFunc = void (*) (Fluid *, const real t, const real dt); using EmfBoundaryFunc = void (*) (DataBlock &, const real t); using DiffusivityFunc = void (*) (DataBlock &, const real t, IdefixArray3D &); -using TwoArrayDiffusivityFunc = void (*) (DataBlock &, const real t, - IdefixArray3D &, IdefixArray3D &); -using FourArrayDiffusivityFunc = void (*) (DataBlock &, const real t, - IdefixArray3D &, IdefixArray3D &, - IdefixArray3D &, IdefixArray3D &); +using BragDiffusivityFunc = void (*) (DataBlock &, const real, + std::vector> &); // Deprecated signatures using SrcTermFuncOld = void (*) (DataBlock &, const real t, const real dt); diff --git a/test/MHD/MTI/setup.cpp b/test/MHD/MTI/setup.cpp index e631f7b2b..1608b5eca 100644 --- a/test/MHD/MTI/setup.cpp +++ b/test/MHD/MTI/setup.cpp @@ -39,9 +39,13 @@ void Potential(DataBlock& data, const real t, IdefixArray1D& x1, IdefixArr } -void MyBragThermalConductivity(DataBlock &data, const real t, IdefixArray3D &kparArr, IdefixArray3D &knorArr) { +void MyBragThermalConductivity(DataBlock &data, const real t, std::vector> &userdefArr) { IdefixArray4D Vc = data.hydro->Vc; IdefixArray1D x2 = data.x[JDIR]; + + IdefixArray3D kparArr = userdefArr[0]; + IdefixArray3D knorArr = userdefArr[1]; + real ksi = ksiGlob; idefix_for("MyThConductivity",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], KOKKOS_LAMBDA (int k, int j, int i) { diff --git a/test/MHD/clessTDiffusion/CMakeLists.txt b/test/MHD/clessTDiffusion/CMakeLists.txt index aa9abdd74..629aed2d1 100644 --- a/test/MHD/clessTDiffusion/CMakeLists.txt +++ b/test/MHD/clessTDiffusion/CMakeLists.txt @@ -1,2 +1 @@ enable_idefix_property(Idefix_MHD) -set_idefix_property(Idefix_RECONSTRUCTION Parabolic) diff --git a/test/MHD/clessTDiffusion/python/testidefix.py b/test/MHD/clessTDiffusion/python/testidefix.py index 444816d1f..9427aca82 100644 --- a/test/MHD/clessTDiffusion/python/testidefix.py +++ b/test/MHD/clessTDiffusion/python/testidefix.py @@ -22,7 +22,7 @@ def gamma_prime(gamma, beta): return (gamma+beta*(gamma-1))/(1+beta*(gamma-1)) success = True -eps = 1e-2 +eps = 3e-4 gamma_eff = np.gradient(prs, r)/np.gradient(rho, r)*rho/prs diff --git a/test/MHD/clessTDiffusion/setup.cpp b/test/MHD/clessTDiffusion/setup.cpp index 624e1d72b..7c6e7eb2a 100644 --- a/test/MHD/clessTDiffusion/setup.cpp +++ b/test/MHD/clessTDiffusion/setup.cpp @@ -10,19 +10,28 @@ real va_vescGlob; real k0_Glob; real ParkerWind(real); -void MyClessThermalConductivity(DataBlock &data, const real t, IdefixArray3D &kparArr, IdefixArray3D &knorArr, IdefixArray3D &alpha, IdefixArray3D &clessQ) { +const real kB{1.3807e-16}; +const real mp{1.6726e-24}; + +void MyClessThermalConductivity(DataBlock &data, const real t, std::vector> &userdefArr) { IdefixArray4D Vc = data.hydro->Vc; IdefixArray1D x1 = data.x[IDIR]; IdefixArray1D x2 = data.x[JDIR]; - real norm = 1.6726e-24*0.5/(udenGlob*uvelGlob*ulenGlob*1.3807e-16); - real uTemp=0.5*uvelGlob*uvelGlob*1.6726e-24/1.3807e-16; + + IdefixArray3D kparArr = userdefArr[0]; + IdefixArray3D knorArr = userdefArr[1]; + IdefixArray3D clessAlpha = userdefArr[2]; + IdefixArray3D clessBeta = userdefArr[3]; + + real norm = mp*0.5/(udenGlob*uvelGlob*ulenGlob*kB); + real uTemp=0.5*uvelGlob*uvelGlob*mp/kB; real k0 = k0_Glob*norm; - idefix_for("MyThConductivity",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], + idefix_for("MyClessThConductivity",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], KOKKOS_LAMBDA (int k, int j, int i) { kparArr(k,j,i) = k0*pow(Vc(PRS,k,j,i)/Vc(RHO,k,j,i)*uTemp,2.5); knorArr(k,j,i) = 0.; - alpha(k,j,i) = (1.0-tanh(x1(i)-10))/2; - clessQ(k,j,i) = -1.5; + clessAlpha(k,j,i) = (1.0-tanh(x1(i)-10))/2; + clessBeta(k,j,i) = -1.5; }); } @@ -39,6 +48,7 @@ void UserDefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { real rc,vc,vwind0; real cs=cs_vescGlob*sqrt(2.); real va_vesc = va_vescGlob; + real mu = va_vesc * sqrt(2.); real PonRho; PonRho = cs*cs; @@ -49,7 +59,7 @@ void UserDefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { hydro->boundary->BoundaryFor("UserDefBoundary", dir, side, KOKKOS_LAMBDA (int k, int j, int i) { real r = x1(i); - real mu = va_vesc * sqrt(2.); + Vc(RHO,k,j,i) = vwind0/(vwind0 * r * r); Vc(PRS,k,j,i) = PonRho * Vc(RHO, k, j, i); Vc(VX1,k,j,i) = vwind0; @@ -68,7 +78,7 @@ void UserDefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { // Set the function for userdefboundary data.hydro->EnrollUserDefBoundary(&UserDefBoundary); - data.hydro->bragThermalDiffusion->EnrollClessThermalDiffusivity(&MyClessThermalConductivity); + data.hydro->bragThermalDiffusion->EnrollBragThermalDiffusivity(&MyClessThermalConductivity); gammaGlob=input.Get("Hydro", "gamma", 0); udenGlob=input.Get("Setup", "UNIT_DENSITY",0); ulenGlob=input.Get("Setup", "UNIT_LENGTH",0); From 250610a19b14c537515ef54f09a780c39bf180f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Victor=20R=C3=A9ville?= Date: Wed, 29 Jan 2025 08:43:52 +0100 Subject: [PATCH 28/82] corrected typo --- src/fluid/braginskii/bragThermalDiffusion.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fluid/braginskii/bragThermalDiffusion.hpp b/src/fluid/braginskii/bragThermalDiffusion.hpp index 7b1875ff9..116ff9a03 100644 --- a/src/fluid/braginskii/bragThermalDiffusion.hpp +++ b/src/fluid/braginskii/bragThermalDiffusion.hpp @@ -791,7 +791,7 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, bgradT = D_EXPAND( Bi*dTi , + Bj*dTj, +Bk*dTk); Bmag = D_EXPAND( Bi*Bi , + Bj*Bj, + Bk*Bk); - // EXPAND can yield uexpected behaviour when DIMENSIONS < COMPONENTS + // EXPAND can yield unexpected behaviour when DIMENSIONS < COMPONENTS //printf("%f , %f\n", Bmag, EXPAND(Bi*Bi, + Bj*Bj, + Bk*Bk)); Bmag = sqrt(Bmag); Bmag = FMAX(1e-6*SMALL_NUMBER,Bmag); From 898fcf3e91f416395f3d9ced590ce1df8f365e08 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Sun, 9 Feb 2025 21:36:47 +0100 Subject: [PATCH 29/82] add some doc --- doc/source/programmingguide.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/doc/source/programmingguide.rst b/doc/source/programmingguide.rst index 18795b50a..4bd7c69f7 100644 --- a/doc/source/programmingguide.rst +++ b/doc/source/programmingguide.rst @@ -645,6 +645,27 @@ It may also be useful to implement debug-only safeguards with custom logic that fit `RUNTIME_CHECK_*` macros. This can be achieved by using the compiler directive `#ifdef RUNTIME_CHECKS` directly. +Dump an array to a file +----------------------- + +It is usually difficult to know what Idefix arrays effectively contains, especially when running on GPU. +To help with this difficulty, Idefix provides a global function ``DumpArray`` that can be used +to dump a ``IdefixArray`` to a numpy file (that can be read from python). This feature can be used +for debugging purpose as: + + +.. code-block:: c++ + + #include "idefix.hpp" + + IdefixArray3D myArray("debugMe",10,10,10); + + idfx::DumpArray("myFilename.npy",myArray); // Dump the array content to a numpy file named "myFilename.npy" + + +Note that the array is automatically transfered from the GPU, if needed. + + Minimal skeleton ================ From 241c69c8b524caeea33c73cdc4f20db6f0151470 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 10 Feb 2025 08:52:58 +0100 Subject: [PATCH 30/82] refactor DumpArray --- src/global.hpp | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/src/global.hpp b/src/global.hpp index 1a8944163..265699b29 100644 --- a/src/global.hpp +++ b/src/global.hpp @@ -46,31 +46,17 @@ IdefixArray1D ConvertVectorToIdefixArray(std::vector &inputVector) { } ///< dump Idefix array to a numpy array on disk -template -void dumpArray(std::string filename, IdefixArray3D array) { - IdefixHostArray3D hArray = Kokkos::create_mirror(array); - Kokkos::deep_copy(hArray,array); +template +void DumpArray(std::string filename, ArrayType array) { + auto hArray = Kokkos::create_mirror(array); + Kokkos::deep_copy(hArray, array); - std::array shape; + std::array shape; bool fortran_order{false}; - shape[0] = array.extent(0); - shape[1] = array.extent(1); - shape[2] = array.extent(2); - npy::SaveArrayAsNumpy(filename, fortran_order, 3, shape.data(), hArray.data()); -} - -template -void dumpArray(std::string filename, IdefixArray4D array) { - IdefixHostArray4D hArray = Kokkos::create_mirror(array); - Kokkos::deep_copy(hArray,array); - - std::array shape; - bool fortran_order{false}; - shape[0] = array.extent(0); - shape[1] = array.extent(1); - shape[2] = array.extent(2); - shape[3] = array.extent(3); - npy::SaveArrayAsNumpy(filename, fortran_order, 4, shape.data(), hArray.data()); + for (size_t i = 0; i < ArrayType::rank; ++i) { + shape[i] = array.extent(i); + } + npy::SaveArrayAsNumpy(filename, fortran_order, ArrayType::rank, shape.data(), hArray.data()); } } // namespace idfx From ab340ed54c34892826716dfa12d57958f76fb102 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 10 Feb 2025 17:55:33 +0100 Subject: [PATCH 31/82] implements a 1st order IMEX1 solver for polydisperse dust species --- src/dataBlock/evolveStage.cpp | 10 ++ src/fluid/drag.cpp | 194 +++++++++++++++++++++++ src/fluid/drag.hpp | 28 +++- src/fluid/evolveStage.hpp | 6 +- test/Dust/DustEnergy/idefix-implicit.ini | 36 +++++ test/Dust/DustEnergy/testme.py | 2 +- test/Dust/DustyWave/idefix-implicit.ini | 36 +++++ test/Dust/DustyWave/testme.py | 2 +- 8 files changed, 309 insertions(+), 5 deletions(-) create mode 100644 test/Dust/DustEnergy/idefix-implicit.ini create mode 100644 test/Dust/DustyWave/idefix-implicit.ini diff --git a/src/dataBlock/evolveStage.cpp b/src/dataBlock/evolveStage.cpp index 981c060ae..c97c9f713 100644 --- a/src/dataBlock/evolveStage.cpp +++ b/src/dataBlock/evolveStage.cpp @@ -19,6 +19,16 @@ void DataBlock::EvolveStage() { for(int i = 0 ; i < dust.size() ; i++) { dust[i]->EvolveStage(this->t,this->dt); } + // Add implicit term for dust drag + if(dust[0]->drag->IsImplicit()) { + for(int i = 0 ; i < dust.size() ; i++) { + dust[i]->drag->AddImplicitBackReaction(this->dt,dust[0]->drag->implicitFactor); + } + dust[0]->drag->NormalizeImplicitBackReaction(this->dt); + for(int i = 0 ; i < dust.size() ; i++) { + dust[i]->drag->AddImplicitFluidMomentum(this->dt); + } + } } idfx::popRegion(); diff --git a/src/fluid/drag.cpp b/src/fluid/drag.cpp index d24e0bea7..46542147f 100644 --- a/src/fluid/drag.cpp +++ b/src/fluid/drag.cpp @@ -21,6 +21,10 @@ void Drag::AddDragForce(const real dt) { EquationOfState eos = *(this->eos); + if(implicit) { + IDEFIX_ERROR("Add DragForce should not be called when drag is implicit"); + } + auto userGammai = this->gammai; if(type == Type::Userdef) { if(userDrag != NULL) { @@ -78,9 +82,199 @@ void Drag::AddDragForce(const real dt) { }); idfx::popRegion(); } +// +// $ p_g^{(n+1)}=p_g^{(n)}+\sum_i\frac{\rho_g\gamma_i dt}{1+\rho_g \gamma_i dt}p_i^{(n)} $ +// We accumulate in the array "prefactor" +// $ \sum_i\frac{\rho_i\gamma_i dt}{1+\rho_g\gamma_i dt} +// + +void Drag::AddImplicitBackReaction(const real dt, IdefixArray3D preFactor) { + if(!feedback) { + // no feedback, no need for anything + return; + } + idfx::pushRegion("AddImplicitFluidMomentum"); + + auto UcGas = this->UcGas; + auto VcGas = this->VcGas; + auto UcDust = this->UcDust; + auto VcDust = this->VcDust; + + const Type type = this->type; + real dragCoeff = this->dragCoeff; + bool feedback = this->feedback; + + bool isFirst = this->instanceNumber == 0; + + EquationOfState eos = *(this->eos); + + if(!implicit) { + IDEFIX_ERROR("AddImplicitGasMomentum should not be called when drag is explicit"); + } + + auto userGammai = this->gammai; + if(type == Type::Userdef) { // If feedback enabled, + // the coefficients were already computed in the gas + // momentum pass + if(userDrag != NULL) { + idfx::pushRegion("Drag::UserDrag"); + userDrag(data, dragCoeff, userGammai); + idfx::popRegion(); + } else { + IDEFIX_ERROR("No User-defined drag function has been enrolled"); + } + } + // Compute a drag force fd = - gamma*rhod*rhog*(vd-vg) + // Where gamma is computed according to the choice of drag type + idefix_for("DragForce",0,data->np_tot[KDIR],0,data->np_tot[JDIR],0,data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real gamma; // The drag coefficient + if(type == Type::Gamma) { + gamma = dragCoeff; + + } else if(type == Type::Tau) { + // In this case, the coefficient is the stopping time (assumed constant) + gamma = 1/(dragCoeff*VcGas(RHO,k,j,i)); + } else if(type == Type::Size) { + real cs; + // Assume a fixed size, hence for both Epstein or Stokes, gamma~1/rho_g/cs + // Get the sound speed + #if HAVE_ENERGY == 1 + cs = std::sqrt(eos.GetGamma(VcGas(PRS,k,j,i),VcGas(RHO,k,j,i) + *VcGas(PRS,k,j,i)/VcGas(RHO,k,j,i))); + #else + cs = eos.GetWaveSpeed(k,j,i); + #endif + gamma = cs/dragCoeff; + } else if(type == Type::Userdef) { + gamma = userGammai(k,j,i); + } + + const real factor = VcDust(RHO,k,j,i)*gamma*dt/(1+VcGas(RHO,k,j,i)*gamma*dt); + if(isFirst) { + preFactor(k,j,i) = factor; + } else { + preFactor(k,j,i) += factor; + } + + for(int n = MX1 ; n < MX1+COMPONENTS ; n++) { + UcGas(n,k,j,i) += dt * gamma * VcGas(RHO,k,j,i) * UcDust(n,k,j,i) / + (1 + VcGas(RHO,k,j,i)*dt*gamma); + } + }); + idfx::popRegion(); +} + +void Drag::NormalizeImplicitBackReaction(const real dt) { + if(!feedback) { + // no feedback, no need for anything + return; + } + idfx::pushRegion("AddImplicitFluidMomentum"); + + auto UcGas = this->UcGas; + auto preFactor = this->implicitFactor; + + if(!implicit) { + IDEFIX_ERROR("AddImplicitGasMomentum should not be called when drag is explicit"); + } + + // Compute a drag force fd = - gamma*rhod*rhog*(vd-vg) + // Where gamma is computed according to the choice of drag type + idefix_for("DragForce",0,data->np_tot[KDIR],0,data->np_tot[JDIR],0,data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + const real factor = 1+preFactor(k,j,i); + for(int n = MX1 ; n < MX1+COMPONENTS ; n++) { + UcGas(n,k,j,i) /= factor; + } + }); + idfx::popRegion(); +} + +void Drag::AddImplicitFluidMomentum(const real dt) { + idfx::pushRegion("AddImplicitFluidMomentum"); + + auto UcGas = this->UcGas; + auto VcGas = this->VcGas; + auto UcDust = this->UcDust; + auto VcDust = this->VcDust; + auto InvDt = this->InvDt; + + const Type type = this->type; + real dragCoeff = this->dragCoeff; + bool feedback = this->feedback; + + EquationOfState eos = *(this->eos); + + if(!implicit) { + IDEFIX_ERROR("AddImplicitGasMomentum should not be called when drag is explicit"); + } + + auto userGammai = this->gammai; + if(type == Type::Userdef && !feedback) { // If feedback enabled, + // the coefficients were already computed in the gas + // momentum pass + if(userDrag != NULL) { + idfx::pushRegion("Drag::UserDrag"); + userDrag(data, dragCoeff, userGammai); + idfx::popRegion(); + } else { + IDEFIX_ERROR("No User-defined drag function has been enrolled"); + } + } + // Compute a drag force fd = - gamma*rhod*rhog*(vd-vg) + // Where gamma is computed according to the choice of drag type + idefix_for("DragForce",0,data->np_tot[KDIR],0,data->np_tot[JDIR],0,data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real gamma; // The drag coefficient + if(type == Type::Gamma) { + gamma = dragCoeff; + + } else if(type == Type::Tau) { + // In this case, the coefficient is the stopping time (assumed constant) + gamma = 1/(dragCoeff*VcGas(RHO,k,j,i)); + } else if(type == Type::Size) { + real cs; + // Assume a fixed size, hence for both Epstein or Stokes, gamma~1/rho_g/cs + // Get the sound speed + #if HAVE_ENERGY == 1 + cs = std::sqrt(eos.GetGamma(VcGas(PRS,k,j,i),VcGas(RHO,k,j,i) + *VcGas(PRS,k,j,i)/VcGas(RHO,k,j,i))); + #else + cs = eos.GetWaveSpeed(k,j,i); + #endif + gamma = cs/dragCoeff; + } else if(type == Type::Userdef) { + gamma = userGammai(k,j,i); + } + + for(int n = MX1 ; n < MX1+COMPONENTS ; n++) { + real oldUc = UcDust(n,k,j,i); + UcDust(n,k,j,i) = (oldUc + dt * gamma * VcDust(RHO,k,j,i) * UcGas(n,k,j,i)) / + (1 + VcGas(RHO,k,j,i)*dt*gamma); + + #if HAVE_ENERGY == 1 + real dp = UcDust(n,k,j,i) - oldUc; + + // We add back the energy dissipated for the dust which is not accounted for + // (since there is no energy equation for dust grains) + + // TODO(GL): this should be disabled in the case of a true multifluid system where + // both fluids have a proper energy equation + UcGas(ENG,k,j,i) -= dp*VcDust(n,k,j,i); + #endif + } + }); + idfx::popRegion(); +} void Drag::ShowConfig() { idfx::cout << "Drag: Using "; + if(implicit) { + idfx::cout << " IMPLICIT "; + } else { + idfx::cout << " EXPLICIT "; + } switch(type) { case Type::Gamma: idfx::cout << "constant gamma"; diff --git a/src/fluid/drag.hpp b/src/fluid/drag.hpp index d5f7a5a73..c676a88eb 100644 --- a/src/fluid/drag.hpp +++ b/src/fluid/drag.hpp @@ -24,7 +24,16 @@ class Drag { Drag(Input &, Fluid *); void ShowConfig(); // print configuration void AddDragForce(const real); + + ////////////////////////// + // Implicit functions + void AddImplicitBackReaction(const real, IdefixArray3D); // Add the back reaction + void NormalizeImplicitBackReaction(const real); // Normalize the implicit back reaction + void AddImplicitFluidMomentum(const real); // Add the implicit drag force on dust grains + ///////////////////////// + void EnrollUserDrag(UserDefDragFunc); // User defined drag function enrollment + bool IsImplicit() const { return implicit; } // Check if the drag is implicit IdefixArray4D UcDust; // Dust conservative quantities IdefixArray4D UcGas; // Gas conservative quantities @@ -32,12 +41,17 @@ class Drag { IdefixArray4D VcGas; // Gas primitive quantities IdefixArray3D InvDt; // The InvDt of current dust specie IdefixArray3D gammai; // the drag coefficient (only used for user-defined dust grains) + IdefixArray3D implicitFactor; // The prefactor used by the implicit timestepping + Type type; private: DataBlock* data; real dragCoeff; bool feedback{false}; + bool implicit{false}; + int instanceNumber; + UserDefDragFunc userDrag{NULL}; @@ -92,11 +106,21 @@ Drag::Drag(Input &input, Fluid *hydroin): IDEFIX_ERROR(msg); } // Fetch the drag coefficient for the current specie. - const int n = hydroin->instanceNumber; - this->dragCoeff = input.Get(BlockName,"drag",n+1); + this->instanceNumber = hydroin->instanceNumber; + this->dragCoeff = input.Get(BlockName,"drag",instanceNumber+1); // Feedback is true by default, but can be switched off. this->feedback = input.GetOrSet(BlockName,"drag_feedback",0,true); + + this->implicit = input.GetOrSet(BlockName,"drag_implicit",0,false); + + if(implicit && instanceNumber == 0) { + this->implicitFactor = IdefixArray3D("ImplicitFactor", + data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); + } + } else { IDEFIX_ERROR("A [Drag] block is required in your input file to define the drag force."); } diff --git a/src/fluid/evolveStage.hpp b/src/fluid/evolveStage.hpp index 06754d0b7..31dae5b7d 100644 --- a/src/fluid/evolveStage.hpp +++ b/src/fluid/evolveStage.hpp @@ -61,7 +61,11 @@ void Fluid::EvolveStage(const real t, const real dt) { if(haveSourceTerms) AddSourceTerms(t, dt); // Step 5: add drag when needed - if(haveDrag) drag->AddDragForce(dt); + if(haveDrag) { + if(!drag->IsImplicit()) { + drag->AddDragForce(dt); + } + } if constexpr(Phys::mhd) { #if DIMENSIONS >= 2 diff --git a/test/Dust/DustEnergy/idefix-implicit.ini b/test/Dust/DustEnergy/idefix-implicit.ini new file mode 100644 index 000000000..9ba3c781a --- /dev/null +++ b/test/Dust/DustEnergy/idefix-implicit.ini @@ -0,0 +1,36 @@ +# This test checks that the total energy (thermal+dust kinetic+gas kinetic) +# is effectively conserved when drag is present + +[Grid] +X1-grid 1 0.0 500 u 1.0 +X2-grid 1 0.0 1 u 1.0 +X3-grid 1 0.0 1 u 1.0 + +[TimeIntegrator] +CFL 0.8 +tstop 1.0 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver hllc +gamma 1.4 + +[Dust] +nSpecies 1 +drag tau 1.0 +drag_feedback yes +drag_implicit yes + +[Boundary] +X1-beg periodic +X1-end periodic +X2-beg outflow +X2-end outflow +X3-beg outflow +X3-end outflow + +[Output] +dmp 1.0 +analysis 0.01 +log 1000 diff --git a/test/Dust/DustEnergy/testme.py b/test/Dust/DustEnergy/testme.py index f3b413e83..7187d299b 100755 --- a/test/Dust/DustEnergy/testme.py +++ b/test/Dust/DustEnergy/testme.py @@ -15,7 +15,7 @@ def testMe(test): test.configure() test.compile() - inifiles=["idefix.ini"] + inifiles=["idefix.ini","idefix-implicit.ini"] # loop on all the ini files for this test for ini in inifiles: diff --git a/test/Dust/DustyWave/idefix-implicit.ini b/test/Dust/DustyWave/idefix-implicit.ini new file mode 100644 index 000000000..05da6ccce --- /dev/null +++ b/test/Dust/DustyWave/idefix-implicit.ini @@ -0,0 +1,36 @@ +# This test checks the dissipation of a sound wave by a dust grains +# partially coupled to the gas (Riols & Lesur 2018, appendix A) + +[Grid] +X1-grid 1 0.0 500 u 1.0 +X2-grid 1 0.0 1 u 1.0 +X3-grid 1 0.0 1 u 1.0 + +[TimeIntegrator] +CFL 0.8 +tstop 10.0 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver hllc +csiso constant 1.0 + +[Dust] +nSpecies 1 +drag tau 1.0e-3 +drag_feedback yes +drag_implicit yes + +[Boundary] +X1-beg periodic +X1-end periodic +X2-beg outflow +X2-end outflow +X3-beg outflow +X3-end outflow + +[Output] +dmp 10.0 +analysis 0.01 +log 1000 diff --git a/test/Dust/DustyWave/testme.py b/test/Dust/DustyWave/testme.py index b6a75ce3c..2b03d1636 100755 --- a/test/Dust/DustyWave/testme.py +++ b/test/Dust/DustyWave/testme.py @@ -15,7 +15,7 @@ def testMe(test): test.configure() test.compile() - inifiles=["idefix.ini"] + inifiles=["idefix.ini","idefix-implicit.ini"] # loop on all the ini files for this test for ini in inifiles: From c0fbc84fe593b1ca599fa1c89a960d2f0c81b877 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Victor=20R=C3=A9ville?= Date: Tue, 11 Feb 2025 07:07:33 +0100 Subject: [PATCH 32/82] switch for std.vector.at() in userdef TD tests --- test/MHD/MTI/setup.cpp | 4 ++-- test/MHD/clessTDiffusion/setup.cpp | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/MHD/MTI/setup.cpp b/test/MHD/MTI/setup.cpp index 1608b5eca..552f76465 100644 --- a/test/MHD/MTI/setup.cpp +++ b/test/MHD/MTI/setup.cpp @@ -43,8 +43,8 @@ void MyBragThermalConductivity(DataBlock &data, const real t, std::vector Vc = data.hydro->Vc; IdefixArray1D x2 = data.x[JDIR]; - IdefixArray3D kparArr = userdefArr[0]; - IdefixArray3D knorArr = userdefArr[1]; + IdefixArray3D kparArr = userdefArr.at(0); + IdefixArray3D knorArr = userdefArr.at(1); real ksi = ksiGlob; idefix_for("MyThConductivity",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], diff --git a/test/MHD/clessTDiffusion/setup.cpp b/test/MHD/clessTDiffusion/setup.cpp index 7c6e7eb2a..651a81f53 100644 --- a/test/MHD/clessTDiffusion/setup.cpp +++ b/test/MHD/clessTDiffusion/setup.cpp @@ -18,10 +18,10 @@ void MyClessThermalConductivity(DataBlock &data, const real t, std::vector x1 = data.x[IDIR]; IdefixArray1D x2 = data.x[JDIR]; - IdefixArray3D kparArr = userdefArr[0]; - IdefixArray3D knorArr = userdefArr[1]; - IdefixArray3D clessAlpha = userdefArr[2]; - IdefixArray3D clessBeta = userdefArr[3]; + IdefixArray3D kparArr = userdefArr.at(0); + IdefixArray3D knorArr = userdefArr.at(1); + IdefixArray3D clessAlpha = userdefArr.at(2); + IdefixArray3D clessBeta = userdefArr.at(3); real norm = mp*0.5/(udenGlob*uvelGlob*ulenGlob*kB); real uTemp=0.5*uvelGlob*uvelGlob*mp/kB; From 22eaf5b872f26e8ec2f4a329497e2ef4805ed315 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 11 Feb 2025 10:33:14 +0100 Subject: [PATCH 33/82] refactor dust drag law --- src/fluid/drag.cpp | 188 ++++++++++-------------- src/fluid/drag.hpp | 91 +++++++----- test/Dust/DustyWave/idefix-implicit.ini | 2 +- 3 files changed, 127 insertions(+), 154 deletions(-) diff --git a/src/fluid/drag.cpp b/src/fluid/drag.cpp index 46542147f..7739d84d9 100644 --- a/src/fluid/drag.cpp +++ b/src/fluid/drag.cpp @@ -5,7 +5,12 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** #include "drag.hpp" +#include + #include "physics.hpp" + + + void Drag::AddDragForce(const real dt) { idfx::pushRegion("Drag::AddDragForce"); @@ -15,51 +20,20 @@ void Drag::AddDragForce(const real dt) { auto VcDust = this->VcDust; auto InvDt = this->InvDt; - const Type type = this->type; - real dragCoeff = this->dragCoeff; bool feedback = this->feedback; - EquationOfState eos = *(this->eos); - if(implicit) { IDEFIX_ERROR("Add DragForce should not be called when drag is implicit"); } - auto userGammai = this->gammai; - if(type == Type::Userdef) { - if(userDrag != NULL) { - idfx::pushRegion("Drag::UserDrag"); - userDrag(data, dragCoeff, userGammai); - idfx::popRegion(); - } else { - IDEFIX_ERROR("No User-defined drag function has been enrolled"); - } - } + auto gammaDrag = this->gammaDrag; + gammaDrag.RefreshUserDrag(data); + // Compute a drag force fd = - gamma*rhod*rhog*(vd-vg) // Where gamma is computed according to the choice of drag type idefix_for("DragForce",0,data->np_tot[KDIR],0,data->np_tot[JDIR],0,data->np_tot[IDIR], KOKKOS_LAMBDA (int k, int j, int i) { - real gamma; // The drag coefficient - if(type == Type::Gamma) { - gamma = dragCoeff; - - } else if(type == Type::Tau) { - // In this case, the coefficient is the stopping time (assumed constant) - gamma = 1/(dragCoeff*VcGas(RHO,k,j,i)); - } else if(type == Type::Size) { - real cs; - // Assume a fixed size, hence for both Epstein or Stokes, gamma~1/rho_g/cs - // Get the sound speed - #if HAVE_ENERGY == 1 - cs = std::sqrt(eos.GetGamma(VcGas(PRS,k,j,i),VcGas(RHO,k,j,i) - *VcGas(PRS,k,j,i)/VcGas(RHO,k,j,i))); - #else - cs = eos.GetWaveSpeed(k,j,i); - #endif - gamma = cs/dragCoeff; - } else if(type == Type::Userdef) { - gamma = userGammai(k,j,i); - } + real gamma = gammaDrag.GetGamma(k,j,i); // The drag coefficient real dp = dt * gamma * VcDust(RHO,k,j,i) * VcGas(RHO,k,j,i); for(int n = MX1 ; n < MX1+COMPONENTS ; n++) { @@ -97,58 +71,26 @@ void Drag::AddImplicitBackReaction(const real dt, IdefixArray3D preFactor) auto UcGas = this->UcGas; auto VcGas = this->VcGas; - auto UcDust = this->UcDust; auto VcDust = this->VcDust; - const Type type = this->type; - real dragCoeff = this->dragCoeff; bool feedback = this->feedback; bool isFirst = this->instanceNumber == 0; - EquationOfState eos = *(this->eos); if(!implicit) { IDEFIX_ERROR("AddImplicitGasMomentum should not be called when drag is explicit"); } - auto userGammai = this->gammai; - if(type == Type::Userdef) { // If feedback enabled, - // the coefficients were already computed in the gas - // momentum pass - if(userDrag != NULL) { - idfx::pushRegion("Drag::UserDrag"); - userDrag(data, dragCoeff, userGammai); - idfx::popRegion(); - } else { - IDEFIX_ERROR("No User-defined drag function has been enrolled"); - } - } + auto gammaDrag = this->gammaDrag; + gammaDrag.RefreshUserDrag(data); + // Compute a drag force fd = - gamma*rhod*rhog*(vd-vg) // Where gamma is computed according to the choice of drag type idefix_for("DragForce",0,data->np_tot[KDIR],0,data->np_tot[JDIR],0,data->np_tot[IDIR], KOKKOS_LAMBDA (int k, int j, int i) { - real gamma; // The drag coefficient - if(type == Type::Gamma) { - gamma = dragCoeff; - - } else if(type == Type::Tau) { - // In this case, the coefficient is the stopping time (assumed constant) - gamma = 1/(dragCoeff*VcGas(RHO,k,j,i)); - } else if(type == Type::Size) { - real cs; - // Assume a fixed size, hence for both Epstein or Stokes, gamma~1/rho_g/cs - // Get the sound speed - #if HAVE_ENERGY == 1 - cs = std::sqrt(eos.GetGamma(VcGas(PRS,k,j,i),VcGas(RHO,k,j,i) - *VcGas(PRS,k,j,i)/VcGas(RHO,k,j,i))); - #else - cs = eos.GetWaveSpeed(k,j,i); - #endif - gamma = cs/dragCoeff; - } else if(type == Type::Userdef) { - gamma = userGammai(k,j,i); - } + real gamma = gammaDrag.GetGamma(k,j,i); // The drag coefficient + const real factor = VcDust(RHO,k,j,i)*gamma*dt/(1+VcGas(RHO,k,j,i)*gamma*dt); if(isFirst) { @@ -200,53 +142,20 @@ void Drag::AddImplicitFluidMomentum(const real dt) { auto VcDust = this->VcDust; auto InvDt = this->InvDt; - const Type type = this->type; - real dragCoeff = this->dragCoeff; bool feedback = this->feedback; - EquationOfState eos = *(this->eos); - if(!implicit) { IDEFIX_ERROR("AddImplicitGasMomentum should not be called when drag is explicit"); } - auto userGammai = this->gammai; - if(type == Type::Userdef && !feedback) { // If feedback enabled, - // the coefficients were already computed in the gas - // momentum pass - if(userDrag != NULL) { - idfx::pushRegion("Drag::UserDrag"); - userDrag(data, dragCoeff, userGammai); - idfx::popRegion(); - } else { - IDEFIX_ERROR("No User-defined drag function has been enrolled"); - } - } + auto gammaDrag = this->gammaDrag; + if(!feedback) gammaDrag.RefreshUserDrag(data); + // Compute a drag force fd = - gamma*rhod*rhog*(vd-vg) // Where gamma is computed according to the choice of drag type idefix_for("DragForce",0,data->np_tot[KDIR],0,data->np_tot[JDIR],0,data->np_tot[IDIR], KOKKOS_LAMBDA (int k, int j, int i) { - real gamma; // The drag coefficient - if(type == Type::Gamma) { - gamma = dragCoeff; - - } else if(type == Type::Tau) { - // In this case, the coefficient is the stopping time (assumed constant) - gamma = 1/(dragCoeff*VcGas(RHO,k,j,i)); - } else if(type == Type::Size) { - real cs; - // Assume a fixed size, hence for both Epstein or Stokes, gamma~1/rho_g/cs - // Get the sound speed - #if HAVE_ENERGY == 1 - cs = std::sqrt(eos.GetGamma(VcGas(PRS,k,j,i),VcGas(RHO,k,j,i) - *VcGas(PRS,k,j,i)/VcGas(RHO,k,j,i))); - #else - cs = eos.GetWaveSpeed(k,j,i); - #endif - gamma = cs/dragCoeff; - } else if(type == Type::Userdef) { - gamma = userGammai(k,j,i); - } + real gamma = gammaDrag.GetGamma(k,j,i); // The drag coefficient for(int n = MX1 ; n < MX1+COMPONENTS ; n++) { real oldUc = UcDust(n,k,j,i); @@ -275,17 +184,17 @@ void Drag::ShowConfig() { } else { idfx::cout << " EXPLICIT "; } - switch(type) { - case Type::Gamma: + switch(gammaDrag.type) { + case GammaDrag::Type::Gamma: idfx::cout << "constant gamma"; break; - case Type::Tau: + case GammaDrag::Type::Tau: idfx::cout << "constant stopping time"; break; - case Type::Size: + case GammaDrag::Type::Size: idfx::cout << "constant dust size"; break; - case Type::Userdef: + case GammaDrag::Type::Userdef: idfx::cout << "user-defined"; break; } @@ -299,8 +208,59 @@ void Drag::ShowConfig() { } void Drag::EnrollUserDrag(UserDefDragFunc func) { + gammaDrag.EnrollUserDrag(func); +} + +//////////////////////////////////////////// +// GammaDrag function definitions +//////////////////////////////////////////// + +void GammaDrag::RefreshUserDrag(DataBlock *data) { + if(type == Type::Userdef) { + if(userDrag != NULL) { + idfx::pushRegion("GammaDrag::UserDrag"); + userDrag(data, dragCoeff, gammai); + idfx::popRegion(); + } else { + IDEFIX_ERROR("No User-defined drag function has been enrolled"); + } + } +} + +void GammaDrag::EnrollUserDrag(UserDefDragFunc func) { if(type != Type::Userdef) { IDEFIX_ERROR("User-defined drag function requires drag entry to be set to \"userdef\""); } this->userDrag = func; } + +GammaDrag::GammaDrag(Input &input, std::string BlockName, int instanceNumber, DataBlock *data) { + if(input.CheckEntry(BlockName,"drag")>=0) { + std::string dragType = input.Get(BlockName,"drag",0); + if(dragType.compare("gamma") == 0) { + this->type = Type::Gamma; + } else if(dragType.compare("tau") == 0) { + this->type = Type::Tau; + this->VcGas = data->hydro->Vc; + } else if(dragType.compare("size") == 0) { + this->type = Type::Size; + this->eos = *(data->hydro->eos.get()); + this->VcGas = data->hydro->Vc; + } else if(dragType.compare("userdef") == 0) { + this->type = Type::Userdef; + this->gammai = IdefixArray3D("UserDrag", + data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); + } else { + std::stringstream msg; + msg << "Unknown drag type \"" << dragType + << "\" in your input file." << std::endl + << "Allowed values are: gamma, tau, epstein, stokes, userdef." << std::endl; + + IDEFIX_ERROR(msg); + } + } + // Fetch the drag coefficient for the current specie. + this->dragCoeff = input.Get(BlockName,"drag",instanceNumber+1); +} diff --git a/src/fluid/drag.hpp b/src/fluid/drag.hpp index c676a88eb..26332eff8 100644 --- a/src/fluid/drag.hpp +++ b/src/fluid/drag.hpp @@ -15,10 +15,52 @@ using UserDefDragFunc = void (*) (DataBlock *, real beta, IdefixArray3D &gammai); +/* A class that holds the kind of drag law we intend to use */ +class GammaDrag { + public: + enum class Type{Gamma, Tau, Size, Userdef}; + GammaDrag() = default; + GammaDrag(Input &, std::string BlockName, int instanceNumber, DataBlock *data); + void EnrollUserDrag(UserDefDragFunc); + void RefreshUserDrag(DataBlock *); + + KOKKOS_INLINE_FUNCTION real GetGamma(const int k, const int j, const int i) const { + real gamma; // The drag coefficient + if(type == Type::Gamma) { + gamma = dragCoeff; + + } else if(type == Type::Tau) { + // In this case, the coefficient is the stopping time (assumed constant) + gamma = 1/(dragCoeff*VcGas(RHO,k,j,i)); + } else if(type == Type::Size) { + real cs; + // Assume a fixed size, hence for both Epstein or Stokes, gamma~1/rho_g/cs + // Get the sound speed + #if HAVE_ENERGY == 1 + cs = std::sqrt(eos.GetGamma(VcGas(PRS,k,j,i),VcGas(RHO,k,j,i) + *VcGas(PRS,k,j,i)/VcGas(RHO,k,j,i))); + #else + cs = eos.GetWaveSpeed(k,j,i); + #endif + gamma = cs/dragCoeff; + } else if(type == Type::Userdef) { + gamma = gammai(k,j,i); + } + return gamma; + } + + Type type; + real dragCoeff; + EquationOfState eos; + IdefixArray3D gammai; + IdefixArray4D VcGas; + + UserDefDragFunc userDrag{NULL}; +}; + class Drag { public: - enum class Type{Gamma, Tau, Size, Userdef}; // Different types of implementation for the drag force. template Drag(Input &, Fluid *); @@ -40,24 +82,20 @@ class Drag { IdefixArray4D VcDust; // Gas primitive quantities IdefixArray4D VcGas; // Gas primitive quantities IdefixArray3D InvDt; // The InvDt of current dust specie - IdefixArray3D gammai; // the drag coefficient (only used for user-defined dust grains) IdefixArray3D implicitFactor; // The prefactor used by the implicit timestepping - Type type; + GammaDrag gammaDrag; // The drag law private: DataBlock* data; - real dragCoeff; bool feedback{false}; bool implicit{false}; int instanceNumber; +}; + - UserDefDragFunc userDrag{NULL}; - // Sound speed computation - EquationOfState *eos; -}; #include "fluid.hpp" @@ -72,47 +110,22 @@ Drag::Drag(Input &input, Fluid *hydroin): // Save the parent hydro object this->data = hydroin->data; - // We use the EOS of the basic fluid instance, as there is no EOS for dust! - this->eos = hydroin->data->hydro->eos.get(); // Check in which block we should fetch our information - std::string BlockName; + std::string blockName; if(Phys::dust) { - BlockName = "Dust"; + blockName = "Dust"; } else { IDEFIX_ERROR("Drag is currently implemented only for dusty fuids"); } - if(input.CheckEntry(BlockName,"drag")>=0) { - std::string dragType = input.Get(BlockName,"drag",0); - if(dragType.compare("gamma") == 0) { - this->type = Type::Gamma; - } else if(dragType.compare("tau") == 0) { - this->type = Type::Tau; - } else if(dragType.compare("size") == 0) { - this->type = Type::Size; - } else if(dragType.compare("userdef") == 0) { - this->type = Type::Userdef; - this->gammai = IdefixArray3D("UserDrag", - data->np_tot[KDIR], - data->np_tot[JDIR], - data->np_tot[IDIR]); - } else { - std::stringstream msg; - msg << "Unknown drag type \"" << dragType - << "\" in your input file." << std::endl - << "Allowed values are: gamma, tau, epstein, stokes, userdef." << std::endl; - - IDEFIX_ERROR(msg); - } - // Fetch the drag coefficient for the current specie. + if(input.CheckEntry(blockName,"drag")>=0) { this->instanceNumber = hydroin->instanceNumber; - this->dragCoeff = input.Get(BlockName,"drag",instanceNumber+1); + this->gammaDrag = GammaDrag(input,blockName,instanceNumber,data); // Feedback is true by default, but can be switched off. - this->feedback = input.GetOrSet(BlockName,"drag_feedback",0,true); - - this->implicit = input.GetOrSet(BlockName,"drag_implicit",0,false); + this->feedback = input.GetOrSet(blockName,"drag_feedback",0,true); + this->implicit = input.GetOrSet(blockName,"drag_implicit",0,false); if(implicit && instanceNumber == 0) { this->implicitFactor = IdefixArray3D("ImplicitFactor", diff --git a/test/Dust/DustyWave/idefix-implicit.ini b/test/Dust/DustyWave/idefix-implicit.ini index 05da6ccce..726585e0b 100644 --- a/test/Dust/DustyWave/idefix-implicit.ini +++ b/test/Dust/DustyWave/idefix-implicit.ini @@ -18,7 +18,7 @@ csiso constant 1.0 [Dust] nSpecies 1 -drag tau 1.0e-3 +drag tau 1.0 drag_feedback yes drag_implicit yes From cbcc63fffd13e8f3ec7cbc58bcd88a2ec0112e31 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 11 Feb 2025 10:35:35 +0100 Subject: [PATCH 34/82] clean up --- src/fluid/drag.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/fluid/drag.cpp b/src/fluid/drag.cpp index 7739d84d9..791cdf889 100644 --- a/src/fluid/drag.cpp +++ b/src/fluid/drag.cpp @@ -6,11 +6,8 @@ // *********************************************************************************** #include "drag.hpp" #include - #include "physics.hpp" - - void Drag::AddDragForce(const real dt) { idfx::pushRegion("Drag::AddDragForce"); From 05ca488b9b069397532f5a9f3970b5a45c0d29d6 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 11 Feb 2025 14:11:04 +0100 Subject: [PATCH 35/82] change signature of userdef drag function --- src/fluid/drag.cpp | 3 ++- src/fluid/drag.hpp | 6 ++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/fluid/drag.cpp b/src/fluid/drag.cpp index 791cdf889..bbbf11b20 100644 --- a/src/fluid/drag.cpp +++ b/src/fluid/drag.cpp @@ -216,7 +216,7 @@ void GammaDrag::RefreshUserDrag(DataBlock *data) { if(type == Type::Userdef) { if(userDrag != NULL) { idfx::pushRegion("GammaDrag::UserDrag"); - userDrag(data, dragCoeff, gammai); + userDrag(data, instanceNumber, dragCoeff, gammai); idfx::popRegion(); } else { IDEFIX_ERROR("No User-defined drag function has been enrolled"); @@ -260,4 +260,5 @@ GammaDrag::GammaDrag(Input &input, std::string BlockName, int instanceNumber, Da } // Fetch the drag coefficient for the current specie. this->dragCoeff = input.Get(BlockName,"drag",instanceNumber+1); + this->instanceNumber = instanceNumber; } diff --git a/src/fluid/drag.hpp b/src/fluid/drag.hpp index 26332eff8..29eca3097 100644 --- a/src/fluid/drag.hpp +++ b/src/fluid/drag.hpp @@ -13,7 +13,7 @@ #include "fluid_defs.hpp" #include "eos.hpp" -using UserDefDragFunc = void (*) (DataBlock *, real beta, IdefixArray3D &gammai); +using UserDefDragFunc = void (*) (DataBlock *, int n, real beta, IdefixArray3D &gammai); /* A class that holds the kind of drag law we intend to use */ class GammaDrag { @@ -55,6 +55,7 @@ class GammaDrag { IdefixArray3D gammai; IdefixArray4D VcGas; + int instanceNumber; UserDefDragFunc userDrag{NULL}; }; @@ -94,9 +95,6 @@ class Drag { }; - - - #include "fluid.hpp" template From 6fbd722439179c67bf22518098d1b210ebede845 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 11 Feb 2025 14:11:57 +0100 Subject: [PATCH 36/82] add the dustyShock test from Benitez-Llambay 2019 --- test/Dust/DustyShock/definitions.hpp | 5 ++ test/Dust/DustyShock/idefix-implicit.ini | 36 +++++++++ test/Dust/DustyShock/idefix.ini | 35 ++++++++ test/Dust/DustyShock/python/testidefix.py | 97 ++++++++++++++++++++++ test/Dust/DustyShock/setup.cpp | 99 +++++++++++++++++++++++ test/Dust/DustyShock/testme.py | 39 +++++++++ 6 files changed, 311 insertions(+) create mode 100644 test/Dust/DustyShock/definitions.hpp create mode 100644 test/Dust/DustyShock/idefix-implicit.ini create mode 100644 test/Dust/DustyShock/idefix.ini create mode 100755 test/Dust/DustyShock/python/testidefix.py create mode 100644 test/Dust/DustyShock/setup.cpp create mode 100755 test/Dust/DustyShock/testme.py diff --git a/test/Dust/DustyShock/definitions.hpp b/test/Dust/DustyShock/definitions.hpp new file mode 100644 index 000000000..cdbe23d6e --- /dev/null +++ b/test/Dust/DustyShock/definitions.hpp @@ -0,0 +1,5 @@ +#define COMPONENTS 1 +#define DIMENSIONS 1 +#define ISOTHERMAL + +#define GEOMETRY CARTESIAN diff --git a/test/Dust/DustyShock/idefix-implicit.ini b/test/Dust/DustyShock/idefix-implicit.ini new file mode 100644 index 000000000..e83d5bf44 --- /dev/null +++ b/test/Dust/DustyShock/idefix-implicit.ini @@ -0,0 +1,36 @@ +# This test checks the behaviour of a dust sound shock +# following the 4 fluids test of Benitez-Llambay+ 2019 + +[Grid] +X1-grid 1 0.0 400 u 40.0 +X2-grid 1 0.0 1 u 1.0 +X3-grid 1 0.0 1 u 1.0 + +[TimeIntegrator] +CFL 0.8 +tstop 500.0 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver hllc +csiso constant 1.0 + +[Dust] +nSpecies 3 +drag userdef 1.0 3.0 5.0 +drag_feedback yes +drag_implicit yes + +[Boundary] +X1-beg userdef +X1-end userdef +X2-beg outflow +X2-end outflow +X3-beg outflow +X3-end outflow + +[Output] +# dmp 10.0 +vtk 500.0 +log 1000 diff --git a/test/Dust/DustyShock/idefix.ini b/test/Dust/DustyShock/idefix.ini new file mode 100644 index 000000000..1b9c9ea2e --- /dev/null +++ b/test/Dust/DustyShock/idefix.ini @@ -0,0 +1,35 @@ +# This test checks the behaviour of a dust sound shock +# following the 4 fluids test of Benitez-Llambay+ 2019 + +[Grid] +X1-grid 1 0.0 400 u 40.0 +X2-grid 1 0.0 1 u 1.0 +X3-grid 1 0.0 1 u 1.0 + +[TimeIntegrator] +CFL 0.8 +tstop 500.0 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver hllc +csiso constant 1.0 + +[Dust] +nSpecies 3 +drag userdef 1.0 3.0 5.0 +drag_feedback yes + +[Boundary] +X1-beg userdef +X1-end userdef +X2-beg outflow +X2-end outflow +X3-beg outflow +X3-end outflow + +[Output] +# dmp 10.0 +vtk 500.0 +log 1000 diff --git a/test/Dust/DustyShock/python/testidefix.py b/test/Dust/DustyShock/python/testidefix.py new file mode 100755 index 000000000..8fca15a3f --- /dev/null +++ b/test/Dust/DustyShock/python/testidefix.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Thu Mar 5 11:29:41 2020 + +@author: glesur +""" + +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) +from pytools.vtk_io import readVTK +import argparse +import numpy as np +import matplotlib.pyplot as plt +from scipy.integrate import solve_ivp + +parser = argparse.ArgumentParser() +parser.add_argument("-noplot", + default=False, + help="disable plotting", + action="store_true") + + +args, unknown=parser.parse_known_args() + +V=readVTK('../data.0001.vtk', geometry='cartesian') + +# Find where the shock starts +istart = np.argwhere(V.data['RHO'][:,0,0]>2.0)[0][0]-1 + +# Compute analytical solution +M=2 # Mach number +K=np.asarray([1,3,5])/M # Drag coefficients + +# (56) from Benitez-Llambay 2018 +def get_wg(Mach, wi): + coeff=[1,np.sum(wi-1)-1/Mach**2-1,1/Mach**2] + solution=np.roots(coeff) + return(np.min(solution)) + +# (55) from Benitez-Llambay 2018 +def rhs(t,wi,Mach,K): + wg = get_wg(Mach,wi) + return K*(wg-wi) + +wi0=np.asarray([1.0,1.0,1.0]) +x=V.x[istart:] +arguments=M,K + +sol = solve_ivp(rhs, t_span=[x[0],x[-1]], y0=wi0, t_eval=x,args=arguments) + +# compute gas velocity +wg=np.zeros(len(x)) +for i in range(0,len(wg)): + wg[i]=get_wg(M,sol.y[:,i]) + + +if(not args.noplot): + plt.figure(1) + plt.plot(V.x,V.data['RHO'][:,0,0],'o',markersize=4,color='tab:purple') + plt.plot(x,1/wg,color='tab:purple') + plt.plot(V.x,V.data['Dust0_RHO'][:,0,0],'o',markersize=4,color='tab:orange') + plt.plot(x,1/sol.y[0,:],color='tab:orange') + plt.plot(V.x,V.data['Dust1_RHO'][:,0,0],'o',markersize=4,color='tab:blue') + plt.plot(x,1/sol.y[1,:],color='tab:blue') + plt.plot(V.x,V.data['Dust2_RHO'][:,0,0],'o',markersize=4,color='tab:green') + plt.plot(x,1/sol.y[2,:],color='tab:green') + plt.xlim([0,10]) + plt.title('Density') + + plt.figure(2) + plt.plot(V.x,V.data['VX1'][:,0,0],'o',markersize=4,color='tab:purple') + plt.plot(x,M*wg,color='tab:purple') + plt.plot(V.x[::4],V.data['Dust0_VX1'][::4,0,0],'o',markersize=4,color='tab:orange') + plt.plot(x,M*sol.y[0,:],color='tab:orange') + plt.plot(V.x[::4],V.data['Dust1_VX1'][::4,0,0],'o',markersize=4,color='tab:blue') + plt.plot(x,M*sol.y[1,:],color='tab:blue') + plt.plot(V.x[::4],V.data['Dust2_VX1'][::4,0,0],'o',markersize=4,color='tab:green') + plt.plot(x,M*sol.y[2,:],color='tab:green') + + plt.xlim([0,10]) + plt.title('Velocity') + + plt.ioff() + plt.show() + +# Compute the error on the dust densities + +error=np.mean(np.fabs(V.data['Dust0_RHO'][istart:,0,0]-1/sol.y[0,:]) + np.fabs(V.data['Dust1_RHO'][istart:,0,0]-1/sol.y[1,:]) + np.fabs(V.data['Dust2_RHO'][istart:,0,0]-1/sol.y[2,:])) / 3 +print("Error=%e"%error) +if error<3.6e-2: + print("SUCCESS!") + sys.exit(0) +else: + print("FAILURE!") + sys.exit(1) diff --git a/test/Dust/DustyShock/setup.cpp b/test/Dust/DustyShock/setup.cpp new file mode 100644 index 000000000..f8632248a --- /dev/null +++ b/test/Dust/DustyShock/setup.cpp @@ -0,0 +1,99 @@ +#include "idefix.hpp" +#include "setup.hpp" + +#define FILENAME "timevol.dat" + +void MyDrag(DataBlock *data, int n, real beta, IdefixArray3D &gamma) { + // Compute the drag coefficient gamma from the input beta + auto VcGas = data->hydro->Vc; + auto VcDust = data->dust[n]->Vc; + + idefix_for("MyDrag",0,data->np_tot[KDIR],0,data->np_tot[JDIR],0,data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + gamma(k,j,i) = beta/VcGas(RHO,k,j,i)/VcDust(RHO,k,j,i); + }); +} + + +void ApplyBoundary(DataBlock *data, IdefixArray4D Vc, int dir, BoundarySide side) { + if(dir==IDIR) { + int iref,ibeg,iend; + if(side == left) { + iref = data->beg[IDIR]; + ibeg = 0; + iend = data->beg[IDIR]; + } else { + iref =data->end[IDIR] - 1; + ibeg=data->end[IDIR]; + iend=data->np_tot[IDIR]; + } + int nvar = Vc.extent(0); + idefix_for("UserDefBoundary",0,data->np_tot[KDIR],0,data->np_tot[JDIR],ibeg,iend, + KOKKOS_LAMBDA (int k, int j, int i) { + for(int n = 0 ; n < nvar ; n++) { + Vc(n,k,j,i) = Vc(n,k,j,iref ); + } + }); + } +} + +void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + ApplyBoundary(hydro->data, hydro->Vc, dir, side); +} + +void UserdefBoundaryDust(Fluid *dust, int dir, BoundarySide side, real t) { + ApplyBoundary(dust->data, dust->Vc, dir, side); +} + + +// Initialisation routine. Can be used to allocate +// Arrays or variables which are used later on +Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { + + data.hydro->EnrollUserDefBoundary(&UserdefBoundary); + if(data.haveDust) { + int nSpecies = data.dust.size(); + for(int n = 0 ; n < nSpecies ; n++) { + data.dust[n]->EnrollUserDefBoundary(&UserdefBoundaryDust); + data.dust[n]->drag->EnrollUserDrag(&MyDrag); + } + } + +} + +// This routine initialize the flow +// Note that data is on the device. +// One can therefore define locally +// a datahost and sync it, if needed +void Setup::InitFlow(DataBlock &data) { + // Create a host copy + DataBlockHost d(data); + + const real x0 = 4.0; + + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + real x = d.x[IDIR](i); + + d.Vc(RHO,k,j,i) = (x < x0) ? 1.0 : 16.0; + d.Vc(VX1,k,j,i) = (x < x0) ? 2.0 : 0.125; + + for(int n = 0 ; n < data.dust.size(); n++) { + d.dustVc[n](RHO,k,j,i) = (x < x0) ? 1.0 : 16.0; + d.dustVc[n](VX1,k,j,i) = (x < x0) ? 2.0 : 0.125; + } + + + } + } + } + + // Send it all, if needed + d.SyncToDevice(); +} + +// Analyse data to produce an output +void MakeAnalysis(DataBlock & data) { + +} diff --git a/test/Dust/DustyShock/testme.py b/test/Dust/DustyShock/testme.py new file mode 100755 index 000000000..2b03d1636 --- /dev/null +++ b/test/Dust/DustyShock/testme.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + +name="dump.0001.dmp" + +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini","idefix-implicit.ini"] + + # loop on all the ini files for this test + for ini in inifiles: + test.run(inputFile=ini) + if test.init: + test.makeReference(filename=name) + test.standardTest() + test.nonRegressionTest(filename=name,tolerance=1e-14) + + +test=tst.idfxTest() + +if not test.all: + if(test.check): + test.checkOnly(filename=name) + else: + testMe(test) +else: + test.noplot = True + test.reconstruction=2 + testMe(test) From 60740e78eecd2e70ea5bd0768ec100f017a8356d Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 11 Feb 2025 14:20:18 +0100 Subject: [PATCH 37/82] add back dump files for reference --- test/Dust/DustyShock/idefix-implicit.ini | 2 +- test/Dust/DustyShock/idefix.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Dust/DustyShock/idefix-implicit.ini b/test/Dust/DustyShock/idefix-implicit.ini index e83d5bf44..2d92a0a52 100644 --- a/test/Dust/DustyShock/idefix-implicit.ini +++ b/test/Dust/DustyShock/idefix-implicit.ini @@ -31,6 +31,6 @@ X3-beg outflow X3-end outflow [Output] -# dmp 10.0 +dmp 500.0 vtk 500.0 log 1000 diff --git a/test/Dust/DustyShock/idefix.ini b/test/Dust/DustyShock/idefix.ini index 1b9c9ea2e..a98ebd097 100644 --- a/test/Dust/DustyShock/idefix.ini +++ b/test/Dust/DustyShock/idefix.ini @@ -30,6 +30,6 @@ X3-beg outflow X3-end outflow [Output] -# dmp 10.0 +dmp 500.0 vtk 500.0 log 1000 From 4b8e277e51e78709642cb8cdd3a2f2d510afd240 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 11 Feb 2025 14:21:02 +0100 Subject: [PATCH 38/82] update reference files for new dusyShock --- reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reference b/reference index b675bceaa..3e1555090 160000 --- a/reference +++ b/reference @@ -1 +1 @@ -Subproject commit b675bceaa6aabc01dded346e2d631857f698dc76 +Subproject commit 3e1555090230ceb21ea6428d82c032734fef7488 From e844bc19c14b497b4e590b4a32feeff38d605308 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 11 Feb 2025 17:03:09 +0100 Subject: [PATCH 39/82] add documentation for implicit dust module --- doc/source/modules/dust.rst | 41 +++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/doc/source/modules/dust.rst b/doc/source/modules/dust.rst index ca371dade..0cdac49ba 100644 --- a/doc/source/modules/dust.rst +++ b/doc/source/modules/dust.rst @@ -39,15 +39,45 @@ which guarantees total momentum conservation. -Drag CFL condition -------------------- -*Idefix* computes the drag terms with a time-explicit scheme. Hence, an addition CFL constraint arrises because of the drag: +Time Integration +---------------- + +*Idefix* can compute the drag with a 2nd order time-explicit (default) or 1st order time-implicit scheme. Each scheme has its pros and cons. The choice +is made by the user in the input file. + +Explicit scheme: drag CFL condition ++++++++++++++++++++++++++++++++++++ + +When using the time explicit scheme, an addition CFL constraint arrises because of the drag: .. math:: dt < \min(\frac{1}{\sum_i\gamma_i(\rho_i+\rho)}) -*Idefix* automatically adjusts the CFL to satisfy this inequality, in addition to the usual CFL condition. +*Idefix* automatically adjusts the CFL to satisfy this inequality, in addition to the usual CFL condition. This can severly limit the time step, +especially when the gas density is high. + +Implicit scheme ++++++++++++++++ + +When the drag is applied with a 1st order implicit scheme, the drag force is applied at the end of each step. In order to avoid +the complete inversion of the system, we follow a simplified inversion procedure, where we first update the gas momentum as: + +.. math:: + + v_g^{(n+1)}=\left(v_g^{(n)}+\sum_i\frac{\rho_i\gamma_i dt}{1+\rho \gamma_i dt}v_i^{(n)}\right)/\left(1+\sum_i\frac{\rho_i\gamma_i dt}{1+\rho\gamma_idt}\right) + +And then update the dust momentum as: + +.. math:: + + v_i^{(n+1)}=\left(v_i^{(n)}+\rho\gamma_i v_g^{(n+1)}dt\right)/(1+\rho\gamma_i dt) + +Note that the latter equation relies on the *updated* gas velocity. + +.. warning:: + While the implicit scheme is more stable than the explicit one, and it does not require any additional CFL condition, it is less accurate and + possibly lead to inacurrate dust velocities when :math:`dt\gg (\gamma_i\rho)^{-1}`. Use it at your own risk. Dust parameters --------------- @@ -66,6 +96,9 @@ The dust module can be enabled adding a block `[Dust]` in your input .ini file. +----------------+-------------------------+---------------------------------------------------------------------------------------------+ | drag_feedback | bool | | (optionnal) whether the gas feedback is enabled (default true). | +----------------+-------------------------+---------------------------------------------------------------------------------------------+ +| drag_implicit | bool | | (optionnal) whether the drag uses a 1st order implicit method. Otherwise use the | +| | | | 2nd order time-explicit scheme (default is false=time explicit) | ++----------------+-------------------------+---------------------------------------------------------------------------------------------+ The drag parameter :math:`\beta_i` above sets the functional form of :math:`\gamma_i(\rho, \rho_i, c_s)` depending on the drag type: From 78fae024a76ee95b94940ffacb8ad36ec0fd2a90 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 13 Feb 2025 13:57:46 +0100 Subject: [PATCH 40/82] use proper update variables istead of pre-stage ones --- src/fluid/drag.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/fluid/drag.cpp b/src/fluid/drag.cpp index bbbf11b20..b6eb1b3d3 100644 --- a/src/fluid/drag.cpp +++ b/src/fluid/drag.cpp @@ -89,7 +89,7 @@ void Drag::AddImplicitBackReaction(const real dt, IdefixArray3D preFactor) real gamma = gammaDrag.GetGamma(k,j,i); // The drag coefficient - const real factor = VcDust(RHO,k,j,i)*gamma*dt/(1+VcGas(RHO,k,j,i)*gamma*dt); + const real factor = UcDust(RHO,k,j,i)*gamma*dt/(1+UcGas(RHO,k,j,i)*gamma*dt); if(isFirst) { preFactor(k,j,i) = factor; } else { @@ -97,8 +97,8 @@ void Drag::AddImplicitBackReaction(const real dt, IdefixArray3D preFactor) } for(int n = MX1 ; n < MX1+COMPONENTS ; n++) { - UcGas(n,k,j,i) += dt * gamma * VcGas(RHO,k,j,i) * UcDust(n,k,j,i) / - (1 + VcGas(RHO,k,j,i)*dt*gamma); + UcGas(n,k,j,i) += dt * gamma * UcGas(RHO,k,j,i) * UcDust(n,k,j,i) / + (1 + UcGas(RHO,k,j,i)*dt*gamma); } }); idfx::popRegion(); @@ -156,8 +156,8 @@ void Drag::AddImplicitFluidMomentum(const real dt) { for(int n = MX1 ; n < MX1+COMPONENTS ; n++) { real oldUc = UcDust(n,k,j,i); - UcDust(n,k,j,i) = (oldUc + dt * gamma * VcDust(RHO,k,j,i) * UcGas(n,k,j,i)) / - (1 + VcGas(RHO,k,j,i)*dt*gamma); + UcDust(n,k,j,i) = (oldUc + dt * gamma * UcDust(RHO,k,j,i) * UcGas(n,k,j,i)) / + (1 + UcGas(RHO,k,j,i)*dt*gamma); #if HAVE_ENERGY == 1 real dp = UcDust(n,k,j,i) - oldUc; @@ -167,7 +167,7 @@ void Drag::AddImplicitFluidMomentum(const real dt) { // TODO(GL): this should be disabled in the case of a true multifluid system where // both fluids have a proper energy equation - UcGas(ENG,k,j,i) -= dp*VcDust(n,k,j,i); + UcGas(ENG,k,j,i) -= dp*UcDust(n,k,j,i)/UcDust(RHO,k,j,i); #endif } }); From 59a91d3cea976f8de290ee461a609cc865ee0fa2 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 13 Feb 2025 14:07:17 +0100 Subject: [PATCH 41/82] switch off heating terms when feedback is disabled (feedback assumes rho_d is negligible, so this makes sense anyway) --- src/fluid/drag.cpp | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/fluid/drag.cpp b/src/fluid/drag.cpp index b6eb1b3d3..a1aae441c 100644 --- a/src/fluid/drag.cpp +++ b/src/fluid/drag.cpp @@ -36,16 +36,17 @@ void Drag::AddDragForce(const real dt) { for(int n = MX1 ; n < MX1+COMPONENTS ; n++) { real dv = VcDust(n,k,j,i) - VcGas(n,k,j,i); UcDust(n,k,j,i) -= dp*dv; - if(feedback) UcGas(n,k,j,i) += dp*dv; - #if HAVE_ENERGY == 1 - // We add back the energy dissipated for the dust which is not accounted for - // (since there is no energy equation for dust grains) - - // TODO(GL): this should be disabled in the case of a true multifluid system where - // both fluids have a proper energy equation - UcGas(ENG,k,j,i) += dp*dv*VcDust(n,k,j,i); - #endif - } + if(feedback) { + UcGas(n,k,j,i) += dp*dv; + #if HAVE_ENERGY == 1 + // We add back the energy dissipated for the dust which is not accounted for + // (since there is no energy equation for dust grains) + + // TODO(GL): this should be disabled in the case of a true multifluid system where + // both fluids have a proper energy equation + UcGas(ENG,k,j,i) += dp*dv*VcDust(n,k,j,i); + #endif + } // feedback // Cfl constraint real idt = gamma*VcGas(RHO,k,j,i); if(feedback) idt += gamma*VcDust(RHO,k,j,i); @@ -167,7 +168,7 @@ void Drag::AddImplicitFluidMomentum(const real dt) { // TODO(GL): this should be disabled in the case of a true multifluid system where // both fluids have a proper energy equation - UcGas(ENG,k,j,i) -= dp*UcDust(n,k,j,i)/UcDust(RHO,k,j,i); + if(feedback) UcGas(ENG,k,j,i) -= dp*UcDust(n,k,j,i)/UcDust(RHO,k,j,i); #endif } }); From f1141eef04e8955cfa9a0d336b389ab87f9d2790 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 13 Feb 2025 14:09:21 +0100 Subject: [PATCH 42/82] typo --- src/fluid/drag.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/fluid/drag.cpp b/src/fluid/drag.cpp index a1aae441c..8bc765160 100644 --- a/src/fluid/drag.cpp +++ b/src/fluid/drag.cpp @@ -47,6 +47,7 @@ void Drag::AddDragForce(const real dt) { UcGas(ENG,k,j,i) += dp*dv*VcDust(n,k,j,i); #endif } // feedback + } // Cfl constraint real idt = gamma*VcGas(RHO,k,j,i); if(feedback) idt += gamma*VcDust(RHO,k,j,i); From 9b2afc546fae579c34be795b4e26ab4fbb6d2c1d Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 13 Feb 2025 14:13:00 +0100 Subject: [PATCH 43/82] restore energy conservation in implicit scheme with feedback --- src/fluid/drag.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fluid/drag.cpp b/src/fluid/drag.cpp index 8bc765160..d1b5de954 100644 --- a/src/fluid/drag.cpp +++ b/src/fluid/drag.cpp @@ -169,7 +169,7 @@ void Drag::AddImplicitFluidMomentum(const real dt) { // TODO(GL): this should be disabled in the case of a true multifluid system where // both fluids have a proper energy equation - if(feedback) UcGas(ENG,k,j,i) -= dp*UcDust(n,k,j,i)/UcDust(RHO,k,j,i); + if(feedback) UcGas(ENG,k,j,i) -= dp*VcDust(n,k,j,i); #endif } }); From e8d34bb2b16671598f3fb20914368a3f53a1baea Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 13 Feb 2025 21:02:41 +0100 Subject: [PATCH 44/82] fix label of dust tracers with non-isothermal equation of state fix #320 --- src/fluid/fluid.hpp | 91 +++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 44 deletions(-) diff --git a/src/fluid/fluid.hpp b/src/fluid/fluid.hpp index ae809b489..44cace6bf 100644 --- a/src/fluid/fluid.hpp +++ b/src/fluid/fluid.hpp @@ -585,52 +585,55 @@ Fluid::Fluid(Grid &grid, Input &input, DataBlock *datain, int n) { } for(int i = 0 ; i < Phys::nvar+nTracer ; i++) { - switch(i) { - case RHO: - VcName.push_back("RHO"); - data->dump->RegisterVariable(Vc, outputPrefix+"Vc-RHO", RHO); - break; - case VX1: - VcName.push_back("VX1"); - data->dump->RegisterVariable(Vc, outputPrefix+"Vc-VX1", VX1, IDIR); - break; - case VX2: - VcName.push_back("VX2"); - data->dump->RegisterVariable(Vc, outputPrefix+"Vc-VX2", VX2, JDIR); - break; - case VX3: - VcName.push_back("VX3"); - data->dump->RegisterVariable(Vc, outputPrefix+"Vc-VX3", VX3, KDIR); - break; - case BX1: - VcName.push_back("BX1"); - // never save cell-centered BX1 in dumps - break; - case BX2: - VcName.push_back("BX2"); - #if DIMENSIONS < 2 - data->dump->RegisterVariable(Vc, outputPrefix+"Vc-BX2", BX2, JDIR); - #endif - break; - case BX3: - VcName.push_back("BX3"); - #if DIMENSIONS < 3 - data->dump->RegisterVariable(Vc, outputPrefix+"Vc-BX3", BX3, KDIR); - #endif - break; - case PRS: - VcName.push_back("PRS"); - data->dump->RegisterVariable(Vc, outputPrefix+"Vc-PRS", PRS); - break; - default: - if(i>=Phys::nvar) { - std::string tracerLabel = std::string("TR")+std::to_string(i-Phys::nvar); // ="TRn" - VcName.push_back(tracerLabel); - data->dump->RegisterVariable(Vc, outputPrefix+"Vc-"+tracerLabel, i); - } else { + if(i < Phys::nvar) { + // These are standard variable names + switch(i) { + case RHO: + VcName.push_back("RHO"); + data->dump->RegisterVariable(Vc, outputPrefix+"Vc-RHO", RHO); + break; + case VX1: + VcName.push_back("VX1"); + data->dump->RegisterVariable(Vc, outputPrefix+"Vc-VX1", VX1, IDIR); + break; + case VX2: + VcName.push_back("VX2"); + data->dump->RegisterVariable(Vc, outputPrefix+"Vc-VX2", VX2, JDIR); + break; + case VX3: + VcName.push_back("VX3"); + data->dump->RegisterVariable(Vc, outputPrefix+"Vc-VX3", VX3, KDIR); + break; + case BX1: + VcName.push_back("BX1"); + // never save cell-centered BX1 in dumps + break; + case BX2: + VcName.push_back("BX2"); + #if DIMENSIONS < 2 + data->dump->RegisterVariable(Vc, outputPrefix+"Vc-BX2", BX2, JDIR); + #endif + break; + case BX3: + VcName.push_back("BX3"); + #if DIMENSIONS < 3 + data->dump->RegisterVariable(Vc, outputPrefix+"Vc-BX3", BX3, KDIR); + #endif + break; + case PRS: + VcName.push_back("PRS"); + data->dump->RegisterVariable(Vc, outputPrefix+"Vc-PRS", PRS); + break; + default: + // unknown variable name (this should never happen, but who knows...) VcName.push_back("Vc-"+std::to_string(i)); data->vtk->RegisterVariable(Vc, outputPrefix+"Vc-"+std::to_string(i), i); - } + break; + } + } else { //if(i>=Phys::nvar) + std::string tracerLabel = std::string("TR")+std::to_string(i-Phys::nvar); // ="TRn" + VcName.push_back(tracerLabel); + data->dump->RegisterVariable(Vc, outputPrefix+"Vc-"+tracerLabel, i); } data->vtk->RegisterVariable(Vc, outputPrefix+VcName[i], i); #ifdef WITH_HDF5 From 34d9366527d8870651f268735a2874e9bc38c875 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 19 Feb 2025 21:37:54 +0100 Subject: [PATCH 45/82] fix a bug that could lead to Nans when using the RKL scheme on grids with small grid spacing (<1e-5) --- src/rkl/rkl.hpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/rkl/rkl.hpp b/src/rkl/rkl.hpp index a2f6734f9..6f6796209 100644 --- a/src/rkl/rkl.hpp +++ b/src/rkl/rkl.hpp @@ -763,11 +763,6 @@ void RKLegendre::CalcParabolicRHS(real t) { KOKKOS_LAMBDA (int n, int k, int j, int i) { real Ax = A(k,j,i); -#if GEOMETRY != CARTESIAN - if(Ax Date: Wed, 19 Feb 2025 22:22:19 +0100 Subject: [PATCH 46/82] change the way polar singularities are handled --- src/fluid/calcRightHandSide.hpp | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/fluid/calcRightHandSide.hpp b/src/fluid/calcRightHandSide.hpp index c8738b4b2..b2e964ee4 100644 --- a/src/fluid/calcRightHandSide.hpp +++ b/src/fluid/calcRightHandSide.hpp @@ -162,9 +162,7 @@ struct Fluid_CorrectFluxFunctor { // Conserve angular momentum, hence flux is R*Bphi Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(x1m(i)); if constexpr(Phys::mhd) { - if(Ax0) Flux(iBPHI,k,j,i) = Flux(iBPHI,k,j,i) / Ax; //avoid singularity around poles } } #endif // GEOMETRY==POLAR OR CYLINDRICAL @@ -175,17 +173,17 @@ struct Fluid_CorrectFluxFunctor { Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(x1m(i)); #endif // COMPONENTS == 3 if constexpr(Phys::mhd) { - if(Ax0) { // avoid singularity around poles + EXPAND( , + Flux(iBTH,k,j,i) = Flux(iBTH,k,j,i) * x1m(i) / Ax; , + Flux(iBPHI,k,j,i) = Flux(iBPHI,k,j,i) * x1m(i) / Ax; ) + } } } else if constexpr (dir==JDIR) { #if COMPONENTS == 3 Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(sinx2m(j)); if constexpr(Phys::mhd) { - if(Ax0) Flux(iBPHI,k,j,i) = Flux(iBPHI,k,j,i) / Ax; // avoid singularity around poles } #endif // COMPONENTS = 3 } From 2874fda4bd24157089d1b6940562a8c4a96d35ff Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 19 Feb 2025 23:18:46 +0100 Subject: [PATCH 47/82] -refactor curvature terms without using a small number on all Areas -properly handles curvature on Bphi in 2.5D spherical with RKL --- src/rkl/rkl.hpp | 64 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 14 deletions(-) diff --git a/src/rkl/rkl.hpp b/src/rkl/rkl.hpp index 6f6796209..0bc220c9e 100644 --- a/src/rkl/rkl.hpp +++ b/src/rkl/rkl.hpp @@ -770,19 +770,35 @@ void RKLegendre::CalcParabolicRHS(real t) { // Curvature terms #if (GEOMETRY == POLAR && COMPONENTS >= 2) \ || (GEOMETRY == CYLINDRICAL && COMPONENTS == 3) - if(dir==IDIR && nv==iMPHI) { - // Conserve angular momentum, hence flux is R*Vphi - Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(x1m(i)); + if(dir==IDIR) { + if(nv==iMPHI) { + Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(x1(i)); + } + if constexpr(Phys::mhd) { + if(nv==iBPHI) { + // No area for this one + if(Ax > 0) Flux(iBPHI,k,j,i) = Flux(iBPHI,k,j,i) / Ax;//avoid singularity around poles + } + } } #endif // GEOMETRY==POLAR OR CYLINDRICAL -#if GEOMETRY == SPHERICAL && COMPONENTS == 3 - if(dir==IDIR && nv==iMPHI) { - Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(x1m(i)); - } else if(dir==JDIR && nv==iMPHI) { - Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(sm(j)); +#if GEOMETRY == SPHERICAL + if(dir==IDIR && nv==VX3) { + Flux(nv,k,j,i) = Flux(nv,k,j,i) * FABS(x1m(i)); + } else if(dir==JDIR && nv==VX3) { + Flux(nv,k,j,i) = Flux(nv,k,j,i) * FABS(sm(j)); + } + + if constexpr(Phys::mhd) { + if(dir == IDIR && Ax > 0 && (nv==BX3 || nv == BX2)) { + Flux(nv,k,j,i) = Flux(nv,k,j,i) * x1m(i) / Ax; + } + if(dir==JDIR && Ax > 0 && nv==BX3) { + Flux(nv,k,j,i) = Flux(nv,k,j,i) / Ax; + } } -#endif // GEOMETRY == SPHERICAL && COMPONENTS == 3 +#endif // GEOMETRY == SPHERICAL$ } ); @@ -813,17 +829,37 @@ void RKLegendre::CalcParabolicRHS(real t) { } #if GEOMETRY != CARTESIAN - #ifdef iMPHI - if((dir==IDIR) && (nv == iMPHI)) { + if(dir==IDIR) { + #ifdef iMPHI + if((nv == iMPHI)) { rhs /= x1(i); } + #endif // iMPHI + if constexpr(Phys::mhd) { + #if (GEOMETRY == POLAR || GEOMETRY == CYLINDRICAL) && (defined iBPHI) + if(nv==iBPHI) rhs = - 1 / dx(i) * (Flux(iBPHI, k, j, i+1) - Flux(iBPHI, k, j, i) ); + + #elif (GEOMETRY == SPHERICAL) + real q = 1 / (x1(i)*dx(i)); + if(nv == BX2 || nv == BX3) { + rhs = -q * ((Flux(nv, k, j, i+1) - Flux(nv, k, j, i) )); + } + #endif // GEOMETRY + } // MHD + } // dir==IDIR + if(dir==JDIR) { #if (GEOMETRY == SPHERICAL) && (COMPONENTS == 3) - if((dir==JDIR) && (nv == iMPHI)) { + if(nv == iMPHI) { rhs /= FABS(s(j)); } + if constexpr(Phys::mhd) { + if(nv == iBPHI) { + rhs = - 1 / (rt(i)*dx(j)) * (Flux(nv, k, j+1, i) - Flux(nv, k, j, i)); + } + } #endif // GEOMETRY - // Nothing for KDIR - #endif // iMPHI + } // dir==JDIR + // Nothing for KDIR #endif // GEOMETRY != CARTESIAN // store the field components From 461ae58a13c2a6734ae0e0faece47e828cc2d32b Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 20 Feb 2025 09:14:14 +0100 Subject: [PATCH 48/82] fix typo --- src/rkl/rkl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rkl/rkl.hpp b/src/rkl/rkl.hpp index 0bc220c9e..1ebebe5ad 100644 --- a/src/rkl/rkl.hpp +++ b/src/rkl/rkl.hpp @@ -772,7 +772,7 @@ void RKLegendre::CalcParabolicRHS(real t) { || (GEOMETRY == CYLINDRICAL && COMPONENTS == 3) if(dir==IDIR) { if(nv==iMPHI) { - Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(x1(i)); + Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(x1m(i)); } if constexpr(Phys::mhd) { if(nv==iBPHI) { From a4ed690b6acf4f4e23351a0cda7dcf2270291b15 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 20 Feb 2025 09:45:34 +0100 Subject: [PATCH 49/82] fix capture on Cuda --- src/rkl/rkl.hpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/rkl/rkl.hpp b/src/rkl/rkl.hpp index 1ebebe5ad..3a7d37670 100644 --- a/src/rkl/rkl.hpp +++ b/src/rkl/rkl.hpp @@ -835,12 +835,15 @@ void RKLegendre::CalcParabolicRHS(real t) { rhs /= x1(i); } #endif // iMPHI + real dx_ = dx(i); + real x1_ = x1(i); + if constexpr(Phys::mhd) { #if (GEOMETRY == POLAR || GEOMETRY == CYLINDRICAL) && (defined iBPHI) - if(nv==iBPHI) rhs = - 1 / dx(i) * (Flux(iBPHI, k, j, i+1) - Flux(iBPHI, k, j, i) ); + if(nv==iBPHI) rhs = - 1 / dx_ * (Flux(iBPHI, k, j, i+1) - Flux(iBPHI, k, j, i) ); #elif (GEOMETRY == SPHERICAL) - real q = 1 / (x1(i)*dx(i)); + real q = 1 / (x1_*dx_); if(nv == BX2 || nv == BX3) { rhs = -q * ((Flux(nv, k, j, i+1) - Flux(nv, k, j, i) )); } @@ -852,9 +855,11 @@ void RKLegendre::CalcParabolicRHS(real t) { if(nv == iMPHI) { rhs /= FABS(s(j)); } + real dx_ = dx(j); + real rt_ = rt(i); if constexpr(Phys::mhd) { if(nv == iBPHI) { - rhs = - 1 / (rt(i)*dx(j)) * (Flux(nv, k, j+1, i) - Flux(nv, k, j, i)); + rhs = - 1 / (rt_*dx_) * (Flux(nv, k, j+1, i) - Flux(nv, k, j, i)); } } #endif // GEOMETRY From b3481f52365ce515c733f7b6e66a36c1214f451f Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 20 Feb 2025 13:02:33 +0100 Subject: [PATCH 50/82] do not make an exception when Area=0, as this artificiallykills off one of the EMF component when axisymmetric --- src/fluid/calcRightHandSide.hpp | 66 +++++++++++++++++---------------- src/rkl/rkl.hpp | 57 ++++++++++++++-------------- 2 files changed, 63 insertions(+), 60 deletions(-) diff --git a/src/fluid/calcRightHandSide.hpp b/src/fluid/calcRightHandSide.hpp index b2e964ee4..38ef76d5d 100644 --- a/src/fluid/calcRightHandSide.hpp +++ b/src/fluid/calcRightHandSide.hpp @@ -149,45 +149,49 @@ struct Fluid_CorrectFluxFunctor { Flux(MX1+meanDir,k,j,i) += meanV * Flux(RHO,k,j,i); } // Fargo & Rotation corrections - real Ax = A(k,j,i); + ////////////////////////////////////////////// + // Define correction factor for the fluxes + ////////////////////////////////////////////// - for(int nv = 0 ; nv < Phys::nvar ; nv++) { - Flux(nv,k,j,i) = Flux(nv,k,j,i) * Ax; - } + real Ax[Phys::nvar]; // corrected Area + for(int nv = 0 ; nv < Phys::nvar ; nv++) Ax[nv] = A(k,j,i); // Curvature terms -#if (GEOMETRY == POLAR && COMPONENTS >= 2) \ - || (GEOMETRY == CYLINDRICAL && COMPONENTS == 3) - if constexpr (dir==IDIR) { - // Conserve angular momentum, hence flux is R*Bphi - Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(x1m(i)); - if constexpr(Phys::mhd) { - if(Ax>0) Flux(iBPHI,k,j,i) = Flux(iBPHI,k,j,i) / Ax; //avoid singularity around poles + #if (GEOMETRY == POLAR && COMPONENTS >= 2) \ + || (GEOMETRY == CYLINDRICAL && COMPONENTS == 3) + if constexpr (dir==IDIR) { + // Conserve angular momentum, hence flux is R*Bphi + Ax[iMPHI] *= FABS(x1m(i)); + if constexpr(Phys::mhd) { + Ax[iBPHI] = 1.0; // corrected Flux is simply Bphi + } } - } -#endif // GEOMETRY==POLAR OR CYLINDRICAL + #endif // GEOMETRY==POLAR OR CYLINDRICAL -#if GEOMETRY == SPHERICAL - if constexpr(dir==IDIR) { - #if COMPONENTS == 3 - Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(x1m(i)); - #endif // COMPONENTS == 3 - if constexpr(Phys::mhd) { - if(Ax>0) { // avoid singularity around poles + #if GEOMETRY == SPHERICAL + if constexpr(dir==IDIR) { + #if COMPONENTS == 3 + Ax[iMPHI] *= FABS(x1m(i)); + #endif // COMPONENTS == 3 + if constexpr(Phys::mhd) { EXPAND( , - Flux(iBTH,k,j,i) = Flux(iBTH,k,j,i) * x1m(i) / Ax; , - Flux(iBPHI,k,j,i) = Flux(iBPHI,k,j,i) * x1m(i) / Ax; ) - } - } - } else if constexpr (dir==JDIR) { - #if COMPONENTS == 3 - Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(sinx2m(j)); - if constexpr(Phys::mhd) { - if(Ax>0) Flux(iBPHI,k,j,i) = Flux(iBPHI,k,j,i) / Ax; // avoid singularity around poles + Ax[iBTH] = x1m(i); , + Ax[iBPHI] = x1m(i); ) + }// Phys::mhd + } else if constexpr (dir==JDIR) { + #if COMPONENTS == 3 + Ax[iMPHI] *= FABS(sinx2m(j)); + if constexpr(Phys::mhd) { + Ax[iBPHI] = 1.0; // corrected Flux is simply Bphi + } + #endif // COMPONENTS = 3 } - #endif // COMPONENTS = 3 + #endif // GEOMETRY == SPHERICAL + + // Finally correct the flux + for(int nv = 0 ; nv < Phys::nvar ; nv++) { + Flux(nv,k,j,i) = Flux(nv,k,j,i) * Ax[nv]; } -#endif // GEOMETRY == SPHERICAL } }; diff --git a/src/rkl/rkl.hpp b/src/rkl/rkl.hpp index 3a7d37670..c8dd8cb96 100644 --- a/src/rkl/rkl.hpp +++ b/src/rkl/rkl.hpp @@ -762,43 +762,42 @@ void RKLegendre::CalcParabolicRHS(real t) { data->beg[IDIR],data->end[IDIR]+ioffset, KOKKOS_LAMBDA (int n, int k, int j, int i) { real Ax = A(k,j,i); - const int nv = varList(n); - Flux(nv,k,j,i) = Flux(nv,k,j,i) * Ax; - // Curvature terms -#if (GEOMETRY == POLAR && COMPONENTS >= 2) \ - || (GEOMETRY == CYLINDRICAL && COMPONENTS == 3) - if(dir==IDIR) { - if(nv==iMPHI) { - Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(x1m(i)); - } - if constexpr(Phys::mhd) { - if(nv==iBPHI) { - // No area for this one - if(Ax > 0) Flux(iBPHI,k,j,i) = Flux(iBPHI,k,j,i) / Ax;//avoid singularity around poles + #if (GEOMETRY == POLAR && COMPONENTS >= 2) \ + || (GEOMETRY == CYLINDRICAL && COMPONENTS == 3) + if(dir==IDIR) { + if(nv==iMPHI) { + Ax *= FABS(x1m(i)); + } + if constexpr(Phys::mhd) { + if(nv==iBPHI) { + // No area for this one + Ax = 1; + } } } - } -#endif // GEOMETRY==POLAR OR CYLINDRICAL - -#if GEOMETRY == SPHERICAL - if(dir==IDIR && nv==VX3) { - Flux(nv,k,j,i) = Flux(nv,k,j,i) * FABS(x1m(i)); - } else if(dir==JDIR && nv==VX3) { - Flux(nv,k,j,i) = Flux(nv,k,j,i) * FABS(sm(j)); - } + #endif // GEOMETRY==POLAR OR CYLINDRICAL - if constexpr(Phys::mhd) { - if(dir == IDIR && Ax > 0 && (nv==BX3 || nv == BX2)) { - Flux(nv,k,j,i) = Flux(nv,k,j,i) * x1m(i) / Ax; + #if GEOMETRY == SPHERICAL + if(dir==IDIR && nv==VX3) { + Ax *= FABS(x1m(i)); + } else if(dir==JDIR && nv==VX3) { + Ax *= FABS(sm(j)); } - if(dir==JDIR && Ax > 0 && nv==BX3) { - Flux(nv,k,j,i) = Flux(nv,k,j,i) / Ax; + + if constexpr(Phys::mhd) { + if(dir == IDIR && (nv==BX3 || nv == BX2)) { + Ax = x1m(i); + } + if(dir==JDIR && nv==BX3) { + Ax = 1.0; + } } - } -#endif // GEOMETRY == SPHERICAL$ + #endif // GEOMETRY == SPHERICAL + + Flux(nv,k,j,i) = Flux(nv,k,j,i) * Ax; } ); From af6b3b7064000de48d69c5034635c520186e4670 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 20 Feb 2025 21:08:11 +0100 Subject: [PATCH 51/82] update reference test change tolerance for dustywave to cope with the implicit drag --- reference | 2 +- test/Dust/DustyWave/python/testidefix.py | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/reference b/reference index 3e1555090..5ecb95884 160000 --- a/reference +++ b/reference @@ -1 +1 @@ -Subproject commit 3e1555090230ceb21ea6428d82c032734fef7488 +Subproject commit 5ecb95884f845a23fa0baf45492770c30e22b4fc diff --git a/test/Dust/DustyWave/python/testidefix.py b/test/Dust/DustyWave/python/testidefix.py index c98bc8213..ad52d3306 100755 --- a/test/Dust/DustyWave/python/testidefix.py +++ b/test/Dust/DustyWave/python/testidefix.py @@ -35,9 +35,6 @@ # Get the minimal decay rate (this should be the one that pops up) tau=np.amax(np.imag(sol)) - - - # load the dat file produced by the setup raw=np.loadtxt('../timevol.dat',skiprows=1) t=raw[:,0] @@ -58,10 +55,10 @@ # Compute decay rate: tau_measured=t[-1]/(2*np.log(etot[-1]/etot[0])) # error on the decay rate: -error=(tau_measured-tau)/tau +error=np.abs((tau_measured-tau)/tau) print("error=%f"%error) -if(error<0.065): +if(error<0.07): print("Success!") else: print("Failure!") From 9ea85b0edcd1d4d7dd83ce9194dd79ba96f72d45 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 20 Feb 2025 21:15:35 +0100 Subject: [PATCH 52/82] do not perform implicit capture --- src/fluid/drag.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/fluid/drag.cpp b/src/fluid/drag.cpp index d1b5de954..06a88f025 100644 --- a/src/fluid/drag.cpp +++ b/src/fluid/drag.cpp @@ -71,6 +71,7 @@ void Drag::AddImplicitBackReaction(const real dt, IdefixArray3D preFactor) auto UcGas = this->UcGas; auto VcGas = this->VcGas; auto VcDust = this->VcDust; + auto UcDust = this->UcDust; bool feedback = this->feedback; From ff459c0d8691caf55d222779e5b8face02a5f243 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 20 Feb 2025 21:21:37 +0100 Subject: [PATCH 53/82] remove reference to useless feedback --- src/fluid/drag.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/fluid/drag.cpp b/src/fluid/drag.cpp index 06a88f025..4cf879cdd 100644 --- a/src/fluid/drag.cpp +++ b/src/fluid/drag.cpp @@ -73,8 +73,6 @@ void Drag::AddImplicitBackReaction(const real dt, IdefixArray3D preFactor) auto VcDust = this->VcDust; auto UcDust = this->UcDust; - bool feedback = this->feedback; - bool isFirst = this->instanceNumber == 0; From 3c9f26df2dddbe6d5c7f92b2a388d4b595e3950e Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Fri, 21 Feb 2025 10:42:14 +0100 Subject: [PATCH 54/82] update signature of fargo+planet+dust drag function --- test/Dust/FargoPlanet/setup.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Dust/FargoPlanet/setup.cpp b/test/Dust/FargoPlanet/setup.cpp index af611911d..8447efe0b 100644 --- a/test/Dust/FargoPlanet/setup.cpp +++ b/test/Dust/FargoPlanet/setup.cpp @@ -42,7 +42,7 @@ void MySoundSpeed(DataBlock &data, const real t, IdefixArray3D &cs) { // a = 20 cm * beta * (Sigma_phys/100 g.cm^-2) / (rho_s / 2 g.cm^-3) // NB: checked against A. Johansen+ (2007) -void MyDrag(DataBlock *data, real beta, IdefixArray3D &gamma) { +void MyDrag(DataBlock *data, int nSpecie, real beta, IdefixArray3D &gamma) { // Compute the drag coefficient gamma from the input beta auto x1 = data->x[IDIR]; real sigma0 = sigma0Glob; From 902c0032ecda2391819dda0d234612338b75a582 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Fri, 21 Feb 2025 10:35:00 +0100 Subject: [PATCH 55/82] add the job to the test suite --- .github/workflows/idefix-ci-jobs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/idefix-ci-jobs.yml b/.github/workflows/idefix-ci-jobs.yml index dfb89e0fb..b95385e40 100644 --- a/.github/workflows/idefix-ci-jobs.yml +++ b/.github/workflows/idefix-ci-jobs.yml @@ -177,6 +177,8 @@ jobs: run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/sphBragTDiffusion -all $TESTME_OPTIONS - name: Spherical anisotropic viscosity run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/sphBragViscosity -all $TESTME_OPTIONS + - name: Collisionless thermal conduction + run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/clessTDiffusion -all $TESTME_OPTIONS Examples: needs: [Fargo, Dust, Planet, ShearingBox, SelfGravity] From 24a62e49eba5c49af1393490dacef8918497ef16 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Fri, 21 Feb 2025 10:37:17 +0100 Subject: [PATCH 56/82] update reference commit # --- reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reference b/reference index 5ecb95884..c32894208 160000 --- a/reference +++ b/reference @@ -1 +1 @@ -Subproject commit 5ecb95884f845a23fa0baf45492770c30e22b4fc +Subproject commit c328942083b618a85b84eb16ce9fd35e67c00597 From c34b8190030ffb12ae6598597fe2d364e0a0bf57 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Fri, 21 Feb 2025 23:21:28 +0100 Subject: [PATCH 57/82] clean up test --- test/MHD/clessTDiffusion/setup.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/MHD/clessTDiffusion/setup.cpp b/test/MHD/clessTDiffusion/setup.cpp index 651a81f53..dceafd125 100644 --- a/test/MHD/clessTDiffusion/setup.cpp +++ b/test/MHD/clessTDiffusion/setup.cpp @@ -45,7 +45,7 @@ void UserDefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { IdefixArray4D Vs = hydro->Vs; IdefixArray1D x1 = data->x[IDIR]; - real rc,vc,vwind0; + real rc,vwind0; real cs=cs_vescGlob*sqrt(2.); real va_vesc = va_vescGlob; real mu = va_vesc * sqrt(2.); @@ -96,8 +96,8 @@ void Setup::InitFlow(DataBlock &data) { // Create a host copy DataBlockHost d(data); - real r,th,rl; - real PonRho, vwind0, rc, vc; + real r,rl; + real PonRho, vwind0, rc; real cs=cs_vescGlob*sqrt(2.); @@ -110,7 +110,7 @@ void Setup::InitFlow(DataBlock &data) { for(int j = 0; j < d.np_tot[JDIR] ; j++) { for(int i = 0; i < d.np_tot[IDIR] ; i++) { r=d.x[IDIR](i); - th=d.x[JDIR](j); + real vwind; vwind = ParkerWind(r/rc) * cs; From 6f3bedd7fc7b425ad6cc86a0a537c8a7027b952c Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Sat, 22 Feb 2025 11:33:27 +0100 Subject: [PATCH 58/82] too much cleaning --- test/MHD/clessTDiffusion/setup.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/MHD/clessTDiffusion/setup.cpp b/test/MHD/clessTDiffusion/setup.cpp index dceafd125..4e369b2d3 100644 --- a/test/MHD/clessTDiffusion/setup.cpp +++ b/test/MHD/clessTDiffusion/setup.cpp @@ -45,7 +45,7 @@ void UserDefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { IdefixArray4D Vs = hydro->Vs; IdefixArray1D x1 = data->x[IDIR]; - real rc,vwind0; + real rc,vc,vwind0; real cs=cs_vescGlob*sqrt(2.); real va_vesc = va_vescGlob; real mu = va_vesc * sqrt(2.); From dbdd28d60a7f9f7ce605ba381b55a74c0f7e87ed Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Sat, 22 Feb 2025 16:13:33 +0100 Subject: [PATCH 59/82] final cleanup --- test/MHD/clessTDiffusion/setup.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/MHD/clessTDiffusion/setup.cpp b/test/MHD/clessTDiffusion/setup.cpp index 4e369b2d3..c540c0def 100644 --- a/test/MHD/clessTDiffusion/setup.cpp +++ b/test/MHD/clessTDiffusion/setup.cpp @@ -45,7 +45,7 @@ void UserDefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { IdefixArray4D Vs = hydro->Vs; IdefixArray1D x1 = data->x[IDIR]; - real rc,vc,vwind0; + real rc,vwind0; real cs=cs_vescGlob*sqrt(2.); real va_vesc = va_vescGlob; real mu = va_vesc * sqrt(2.); @@ -53,7 +53,6 @@ void UserDefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { PonRho = cs*cs; rc = 0.25 / (cs_vescGlob*cs_vescGlob); - vc = cs; vwind0 = ParkerWind(1./rc) * cs; hydro->boundary->BoundaryFor("UserDefBoundary", dir, side, From da2702497ade86da469624f7745ecad686e47b41 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 3 Mar 2025 13:06:59 +0100 Subject: [PATCH 60/82] check coarsening inside ghost zone, and detect Nan. Reduce memory footprint by copying only what's needed --- src/dataBlock/coarsen.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/dataBlock/coarsen.cpp b/src/dataBlock/coarsen.cpp index 2da3049cb..a922f226a 100644 --- a/src/dataBlock/coarsen.cpp +++ b/src/dataBlock/coarsen.cpp @@ -66,15 +66,18 @@ void DataBlock::CheckCoarseningLevels() { idfx::pushRegion("DataBlock::CheckCoarseningLevels()"); // Check that the coarsening levels we have are valid // NB: this is a costly procedure, we can't repeat it at each loop! - DataBlockHost d(*this); - d.SyncFromDevice(); for(int dir = 0 ; dir < DIMENSIONS ; dir++) { if(mygrid->coarseningDirection[dir]) { - IdefixHostArray2D arr = d.coarseningLevel[dir]; - const int Xt = (dir == IDIR ? JDIR : IDIR); - const int Xb = (dir == KDIR ? JDIR : KDIR); - for(int i = beg[Xt] ; i < end[Xt] ; i++) { - for(int j = beg[Xb] ; j < end[Xb] ; j++) { + IdefixHostArray2D arr = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), + coarseningLevel[dir]); + for(int i = 0 ; i < arr.extent(1) ; i++) { + for(int j = 0 ; j < arr.extent(0) ; j++) { + if(std::isnan(arr(j,i))) { + std::stringstream str; + str << "Incorrect grid coarsening levels" << std::endl; + str << "at (i,j)=("<< i << "," << j << "): Coarsening level is NaN!" << std::endl; + IDEFIX_ERROR(str); + } if(arr(j,i) < 1) { std::stringstream str; str << "Incorrect grid coarsening levels" << std::endl; From 8bed32b1e3835afa5c99285c441c482d54e37f00 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 3 Mar 2025 13:10:59 +0100 Subject: [PATCH 61/82] fix loop unrolling --- src/dataBlock/coarsen.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dataBlock/coarsen.cpp b/src/dataBlock/coarsen.cpp index a922f226a..28d7b6690 100644 --- a/src/dataBlock/coarsen.cpp +++ b/src/dataBlock/coarsen.cpp @@ -70,8 +70,8 @@ void DataBlock::CheckCoarseningLevels() { if(mygrid->coarseningDirection[dir]) { IdefixHostArray2D arr = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), coarseningLevel[dir]); - for(int i = 0 ; i < arr.extent(1) ; i++) { - for(int j = 0 ; j < arr.extent(0) ; j++) { + for(int j = 0 ; j < arr.extent(0) ; j++) { + for(int i = 0 ; i < arr.extent(1) ; i++) { if(std::isnan(arr(j,i))) { std::stringstream str; str << "Incorrect grid coarsening levels" << std::endl; From 3d96f4104ff97bfde4dd1c446c880866300bfbaa Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 3 Mar 2025 17:39:43 +0100 Subject: [PATCH 62/82] add back axis flux tube test --- .github/workflows/idefix-ci-jobs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/idefix-ci-jobs.yml b/.github/workflows/idefix-ci-jobs.yml index b95385e40..275ea1ce2 100644 --- a/.github/workflows/idefix-ci-jobs.yml +++ b/.github/workflows/idefix-ci-jobs.yml @@ -65,6 +65,8 @@ jobs: run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/OrszagTang -all $TESTME_OPTIONS - name: Orszag Tang 3D+restart tests run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/OrszagTang3D -all $TESTME_OPTIONS + - name: Axis Flux tube + run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/AxisFluxTube -all $TESTME_OPTIONS ParabolicMHD: runs-on: self-hosted From f283f7834b38172cdb3d34bc5edb78df88376e90 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 4 Mar 2025 10:24:50 +0100 Subject: [PATCH 63/82] remove analysis from axisfluxtube --- test/MHD/AxisFluxTube/setup.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/MHD/AxisFluxTube/setup.cpp b/test/MHD/AxisFluxTube/setup.cpp index 34009da85..4c30506d3 100644 --- a/test/MHD/AxisFluxTube/setup.cpp +++ b/test/MHD/AxisFluxTube/setup.cpp @@ -112,7 +112,7 @@ void CoarsenFunction(DataBlock &data) { Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { output.EnrollUserDefVariables(&ComputeUserVars); data.hydro->EnrollUserDefBoundary(&UserdefBoundary); - output.EnrollAnalysis(&Analysis); + //output.EnrollAnalysis(&Analysis); Rtorus = input.Get("Setup","Rtorus",0); Ztorus = input.Get("Setup","Ztorus",0); Rin = input.Get("Setup","Rin",0); From 4c9985fbdd64d8923626dc7b7c10218d10d14e2d Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 4 Mar 2025 15:18:05 +0100 Subject: [PATCH 64/82] add environement to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 422647c3f..4de7fb76e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ doc/source/_static/* doc/source/_public/* doc/source/api/* doc/source/xml/* +doc/env/* # compiled files **/__pycache__ From 3fb2236fd461f139c23fe12dbbed847cd4e03daf Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 4 Mar 2025 15:29:49 +0100 Subject: [PATCH 65/82] final improvement to error message on grid coarsening levels --- src/dataBlock/coarsen.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/dataBlock/coarsen.cpp b/src/dataBlock/coarsen.cpp index 28d7b6690..4c6b60ccd 100644 --- a/src/dataBlock/coarsen.cpp +++ b/src/dataBlock/coarsen.cpp @@ -74,22 +74,25 @@ void DataBlock::CheckCoarseningLevels() { for(int i = 0 ; i < arr.extent(1) ; i++) { if(std::isnan(arr(j,i))) { std::stringstream str; - str << "Incorrect grid coarsening levels" << std::endl; + str << "Nan in grid coarsening levels" << std::endl; str << "at (i,j)=("<< i << "," << j << "): Coarsening level is NaN!" << std::endl; IDEFIX_ERROR(str); } if(arr(j,i) < 1) { std::stringstream str; - str << "Incorrect grid coarsening levels" << std::endl; - str << "at (i,j)=("<< i << "," << j << "): Coarsening level < 1!" << std::endl; + str << "Coarsening level < 1!" << std::endl; + str << "at (i,j)=("<< i << "," << j << "): "; + str << "coarsening level= " << arr(j,i) << std::endl; IDEFIX_ERROR(str); } const int factor = 1 << (arr(j,i) - 1); if(np_int[dir] % factor != 0) { std::stringstream str; - str << "local grid size not divisible by coarsening level" << std::endl; - str << "at (i,j)=("<< i << "," << j << "): Coarsening level: "; - str << arr(j,i) << std::endl; + str << "Local grid size not divisible by coarsening level." << std::endl; + str << "at (i,j)=("<< i << "," << j << "): "; + str << "coarsening level= " << arr(j,i) << std::endl; + str << np_int[dir] << " cannot be divided by 2^" << arr(j,i)-1; + str << " = " << factor << std::endl; IDEFIX_ERROR(str); } } From 1d9d101860f1a6561030c2f59e04b27e544f5afe Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 4 Mar 2025 16:09:46 +0100 Subject: [PATCH 66/82] use one single plot update performance page --- doc/source/{bench-jeanzay.json => bench.json} | 0 doc/source/performances.rst | 17 +++----------- doc/source/plot_idefix_bench.py | 23 +++++++++++++------ 3 files changed, 19 insertions(+), 21 deletions(-) rename doc/source/{bench-jeanzay.json => bench.json} (100%) diff --git a/doc/source/bench-jeanzay.json b/doc/source/bench.json similarity index 100% rename from doc/source/bench-jeanzay.json rename to doc/source/bench.json diff --git a/doc/source/performances.rst b/doc/source/performances.rst index 6f8c88b2f..0cf8f862c 100644 --- a/doc/source/performances.rst +++ b/doc/source/performances.rst @@ -6,9 +6,8 @@ We report below the performances obtained on various architectures using Idefix. is the 3D MHD Orszag-Tang test problem with 2nd order reconstruction and uct_contact EMFS bundled in Idefix test suite, disabling passive tracers. The test is computed with a 128\ :sup:`3` resolution per MPI sub-domain on GPUs or 32\ :sup:`3` per MPI sub-domain on CPUs. All of the performances measures -have been obtained enabling MPI on *one full node*, but we report here the performance *per GPU* -(i.e. with 2 GCDs on AMD Mi250) or *per core* (on CPU), i.e. dividing the node performance by the number of GPU/core -to simplify the comparison with other clusters. +have been obtained enabling MPI and we reporte here the performance *per GPU*, *per GCD* (on Mi250) + or *per core* (on CPU). The complete scalability tests are available in Idefix `method paper `_. The performances mentionned below are updated for each major revision of Idefix, so they might slightly differ from the method paper. @@ -39,17 +38,7 @@ GPU performances .. plot:: import plot_idefix_bench - plot_idefix_bench.do_plot('Performance on NVidia V100 GPUs at Jean-Zay', 'bench-jeanzay.json', 'v100') - -.. plot:: - - import plot_idefix_bench - plot_idefix_bench.do_plot('Performance on NVidia A100 GPUs at Jean-Zay', 'bench-jeanzay.json', 'a100') - -.. plot:: - - import plot_idefix_bench - plot_idefix_bench.do_plot('Performance on NVidia H100 GPUs at Jean-Zay', 'bench-jeanzay.json', 'h100') + plot_idefix_bench.do_plot('Performance on NVidia and AMD GPUs', 'bench.json', ['v100','a100','h100']) +----------------------+--------------------+----------------------------------------------------+ | Cluster name | GPU | Performances (in 10\ :sup:`6` cell/s/GPU) | diff --git a/doc/source/plot_idefix_bench.py b/doc/source/plot_idefix_bench.py index ac91b787e..43482eb04 100644 --- a/doc/source/plot_idefix_bench.py +++ b/doc/source/plot_idefix_bench.py @@ -1,17 +1,26 @@ import matplotlib.pyplot as plt import json -def do_plot(title, bench_file, gpumodel): +def do_plot(title, bench_file, gpumodels): with open(bench_file, 'r') as f: benches = json.load(f) - select = [bench for bench in benches if bench['gpumodel'] == gpumodel][-1] - xs = [r['nbgpu'] for r in select['results']] - ys = [r['cell_updates'] for r in select['results']] + plt.figure() + xmax=0 + ymax=0 + for gpumodel in gpumodels: + select = [bench for bench in benches if bench['gpumodel'] == gpumodel][-1] + + xs = [r['nbgpu'] for r in select['results']] + ys = [r['cell_updates'] for r in select['results']] + plt.plot(xs, ys,'o-',label=gpumodel) + xmax=max(xmax,max(xs)) + ymax=max(ymax,max(ys)) plt.xscale("log", base=2) - plt.plot(xs, ys) - plt.ylim(0,max(ys)*1.1) - plt.xlabel("Number of GPUs") + plt.ylim(0,ymax*1.1) + plt.xlim(1,xmax*1.1) + plt.legend() + plt.xlabel("Number of GPUs/GCDs") plt.ylabel("Performance (cells / second / GPU)") plt.title(title) From 4771bde1c29468d6084b8773a46dda11d4fdf5be Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 6 Mar 2025 10:13:29 +0100 Subject: [PATCH 67/82] Update Adastra Mi250 to lock Rocm version --- doc/source/reference/makefile.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/reference/makefile.rst b/doc/source/reference/makefile.rst index aadc205c3..23052d92c 100644 --- a/doc/source/reference/makefile.rst +++ b/doc/source/reference/makefile.rst @@ -111,7 +111,8 @@ We recommend the following modules and environement variables on AdAstra: module load cpe/24.07 module load craype-accel-amd-gfx90a craype-x86-trento module load PrgEnv-cray - module load amd-mixed + module load amd-mixed/6.1.2 + module load rocm/6.1.2 module load cray-python/3.11.7 Finally, *Idefix* can be configured to run on Mi250 by enabling HIP and the desired architecture with the following options to ccmake: From b2168dfb0c10b87f04a6d7ce34243e3fdd323a47 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 10 Mar 2025 16:13:46 +0100 Subject: [PATCH 68/82] add benchmark data --- doc/source/bench.json | 162 +++++++++++++----------------------- doc/source/performances.rst | 12 ++- 2 files changed, 62 insertions(+), 112 deletions(-) diff --git a/doc/source/bench.json b/doc/source/bench.json index 7ded6914e..08a3c8040 100644 --- a/doc/source/bench.json +++ b/doc/source/bench.json @@ -1,209 +1,161 @@ [ { - "date": "2025-01-17_15:33:13", + "date": "2025-03-04_12:57:12", "gpumodel": "v100", - "idefix_commit": "d0e82202ca90d0b664c71582962c2ae255bc6528", - "bench_commit": "a94bc00490b3c62f81f7713327f4388e44a2abea", + "idefix_commit": "2bc09a0d218459f278e2b28506a09e4591b103ae", + "bench_commit": "37161676db15115c38fed3f35c94fa447cbac7bd", "results": [ { "nbgpu": 1, - "cell_updates": 1.192208E+8 + "cell_updates": 1.193720E+8 }, { "nbgpu": 2, - "cell_updates": 1.178276E+8 + "cell_updates": 1.178864E+8 }, { "nbgpu": 4, - "cell_updates": 1.151014E+8 - } - ] - }, - { - "date": "2025-01-17_15:35:16", - "gpumodel": "v100", - "idefix_commit": "d0e82202ca90d0b664c71582962c2ae255bc6528", - "bench_commit": "a94bc00490b3c62f81f7713327f4388e44a2abea", - "results": [ - { - "nbgpu": 1, - "cell_updates": 1.192139E+8 + "cell_updates": 1.155336E+8 }, { - "nbgpu": 2, - "cell_updates": 1.178690E+8 + "nbgpu": 8, + "cell_updates": 1.014338E+8 }, { - "nbgpu": 4, - "cell_updates": 1.092092E+8 + "nbgpu": 16, + "cell_updates": 9.855007E+7 }, { - "nbgpu": 8, - "cell_updates": 7.464571E+7 + "nbgpu": 32, + "cell_updates": 9.012061E+7 }, { - "nbgpu": 16, - "cell_updates": 7.485223E+7 + "nbgpu": 64, + "cell_updates": 8.538461E+7 }, { - "nbgpu": 32, - "cell_updates": 6.795755E+7 + "nbgpu": 128, + "cell_updates": 8.531021E+7 } ] }, { - "date": "2025-01-17_16:57:02", - "gpumodel": "v100", - "idefix_commit": "d0e82202ca90d0b664c71582962c2ae255bc6528", - "bench_commit": "a94bc00490b3c62f81f7713327f4388e44a2abea", + "date": "2025-03-04_13:07:10", + "gpumodel": "a100", + "idefix_commit": "2bc09a0d218459f278e2b28506a09e4591b103ae", + "bench_commit": "b536949200e50fac68d8a46d5db38fc8e3f02da5", "results": [ { "nbgpu": 1, - "cell_updates": 1.192784E+8 + "cell_updates": 2.044728E+8 }, { "nbgpu": 2, - "cell_updates": 1.117701E+8 + "cell_updates": 2.003563E+8 }, { "nbgpu": 4, - "cell_updates": 1.087580E+8 + "cell_updates": 1.963512E+8 }, { "nbgpu": 8, - "cell_updates": 7.472364E+7 + "cell_updates": 1.933039E+8 }, { "nbgpu": 16, - "cell_updates": 7.554497E+7 + "cell_updates": 9.759154E+7 }, { "nbgpu": 32, - "cell_updates": 6.669339E+7 + "cell_updates": 6.369645E+7 }, { "nbgpu": 64, - "cell_updates": 5.497455E+7 + "cell_updates": 4.629474E+7 }, { "nbgpu": 128, - "cell_updates": 5.440610E+7 + "cell_updates": 4.580281E+7 } ] }, { - "date": "2025-01-17_17:09:39", - "gpumodel": "a100", - "idefix_commit": "d0e82202ca90d0b664c71582962c2ae255bc6528", - "bench_commit": "a94bc00490b3c62f81f7713327f4388e44a2abea", + "date": "2025-03-04_13:16:01", + "gpumodel": "h100", + "idefix_commit": "2bc09a0d218459f278e2b28506a09e4591b103ae", + "bench_commit": "b536949200e50fac68d8a46d5db38fc8e3f02da5", "results": [ { "nbgpu": 1, - "cell_updates": 2.035935E+8 + "cell_updates": 3.079643E+8 }, { "nbgpu": 2, - "cell_updates": 1.992927E+8 + "cell_updates": 3.012300E+8 }, { "nbgpu": 4, - "cell_updates": 1.958386E+8 + "cell_updates": 2.944091E+8 }, { "nbgpu": 8, - "cell_updates": 1.929568E+8 + "cell_updates": 2.837224E+8 }, { "nbgpu": 16, - "cell_updates": 9.408132E+7 + "cell_updates": 2.827778E+8 }, { "nbgpu": 32, - "cell_updates": 5.937487E+7 + "cell_updates": 2.822657E+8 }, { "nbgpu": 64, - "cell_updates": 4.159838E+7 + "cell_updates": 2.767820E+8 }, { "nbgpu": 128, - "cell_updates": 5.061931E+7 + "cell_updates": 2.767322E+8 } ] }, { - "date": "2025-01-17_17:23:13", - "gpumodel": "h100", - "idefix_commit": "d0e82202ca90d0b664c71582962c2ae255bc6528", - "bench_commit": "a94bc00490b3c62f81f7713327f4388e44a2abea", + "date": "2025-03-06_11:21:56", + "gpumodel": "mi250x", + "idefix_commit": "2bc09a0d218459f278e2b28506a09e4591b103ae", + "bench_commit": "868be0a87c6fcda665cbb62db7020aeff70dc62d", "results": [ { "nbgpu": 1, - "cell_updates": 3.083581E+8 - } - ] - }, - { - "date": "2025-01-24_13:02:08", - "gpumodel": "h100", - "idefix_commit": "54482d582361f1cd05c39717902f752b3d04ae6e", - "bench_commit": "a94bc00490b3c62f81f7713327f4388e44a2abea", - "results": [ - { - "nbgpu": 1, - "cell_updates": 3.076537E+8 - } - ] - }, - { - "date": "2025-01-24_13:04:02", - "gpumodel": "h100", - "idefix_commit": "54482d582361f1cd05c39717902f752b3d04ae6e", - "bench_commit": "a94bc00490b3c62f81f7713327f4388e44a2abea", - "results": [ - { - "nbgpu": 1, - "cell_updates": 3.032686E+8 + "cell_updates": 1.436580E+8 }, { "nbgpu": 2, - "cell_updates": 3.007089E+8 + "cell_updates": 1.372499E+8 }, { "nbgpu": 4, - "cell_updates": 2.938962E+8 - } - ] - }, - { - "date": "2025-01-24_14:17:30", - "gpumodel": "h100", - "idefix_commit": "54482d582361f1cd05c39717902f752b3d04ae6e", - "bench_commit": "a94bc00490b3c62f81f7713327f4388e44a2abea", - "results": [ - { - "nbgpu": 1, - "cell_updates": 3.074363E+8 + "cell_updates": 1.344528E+8 }, { - "nbgpu": 2, - "cell_updates": 3.009340E+8 + "nbgpu": 8, + "cell_updates": 1.293602E+8 }, { - "nbgpu": 4, - "cell_updates": 2.938868E+8 + "nbgpu": 16, + "cell_updates": 1.260359E+8 }, { - "nbgpu": 8, - "cell_updates": 2.833054E+8 + "nbgpu": 32, + "cell_updates": 1.204980E+8 }, { - "nbgpu": 16, - "cell_updates": 2.822483E+8 + "nbgpu": 64, + "cell_updates": 1.163099E+8 }, { - "nbgpu": 32, - "cell_updates": 2.817151E+8 + "nbgpu": 128, + "cell_updates": 1.192343E+8 } ] } diff --git a/doc/source/performances.rst b/doc/source/performances.rst index 0cf8f862c..deae1194e 100644 --- a/doc/source/performances.rst +++ b/doc/source/performances.rst @@ -7,7 +7,7 @@ is the 3D MHD Orszag-Tang test problem with 2nd order reconstruction and uct_con Idefix test suite, disabling passive tracers. The test is computed with a 128\ :sup:`3` resolution per MPI sub-domain on GPUs or 32\ :sup:`3` per MPI sub-domain on CPUs. All of the performances measures have been obtained enabling MPI and we reporte here the performance *per GPU*, *per GCD* (on Mi250) - or *per core* (on CPU). +or *per core* (on CPU). The complete scalability tests are available in Idefix `method paper `_. The performances mentionned below are updated for each major revision of Idefix, so they might slightly differ from the method paper. @@ -38,10 +38,8 @@ GPU performances .. plot:: import plot_idefix_bench - plot_idefix_bench.do_plot('Performance on NVidia and AMD GPUs', 'bench.json', ['v100','a100','h100']) + plot_idefix_bench.do_plot('Performance on NVidia and AMD GPUs', 'bench.json', ['v100','a100','h100','mi250x']) -+----------------------+--------------------+----------------------------------------------------+ -| Cluster name | GPU | Performances (in 10\ :sup:`6` cell/s/GPU) | -+======================+====================+====================================================+ -| CINES/Adastra | AMD Mi250 | 250 | -+----------------------+--------------------+----------------------------------------------------+ +.. note:: + + The inter-node communication on Jean Zay is not optimal on A100 nodes. A ticket is opened with IDRIS support to fix this issue. From d59ecf0eb5d89abd974cadd87fc7ef5997271dde Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 11 Mar 2025 13:15:35 +0100 Subject: [PATCH 69/82] attempt at fixing Kokkos SYCL loop configuration (a proper benchmark would be needed). --- src/loop.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/loop.hpp b/src/loop.hpp index 8ad345dea..73afe53e1 100644 --- a/src/loop.hpp +++ b/src/loop.hpp @@ -51,6 +51,8 @@ typedef Kokkos::TeamPolicy<>::member_type member_type; constexpr LoopPattern defaultLoop = LoopPattern::RANGE; #elif defined(KOKKOS_ENABLE_HIP) constexpr LoopPattern defaultLoop = LoopPattern::RANGE; + #elif defined(KOKKOS_ENABLE_SYCL) + constexpr LoopPattern defaultLoop = LoopPattern::RANGE; #elif defined(KOKKOS_ENABLE_SERIAL) constexpr LoopPattern defaultLoop = LoopPattern::SIMDFOR; #else From ca9c8b0302c424f0f0b6439c4568979d9c21ef56 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 11 Mar 2025 13:22:26 +0100 Subject: [PATCH 70/82] tabs --- src/loop.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/loop.hpp b/src/loop.hpp index 73afe53e1..262c6ac12 100644 --- a/src/loop.hpp +++ b/src/loop.hpp @@ -51,7 +51,7 @@ typedef Kokkos::TeamPolicy<>::member_type member_type; constexpr LoopPattern defaultLoop = LoopPattern::RANGE; #elif defined(KOKKOS_ENABLE_HIP) constexpr LoopPattern defaultLoop = LoopPattern::RANGE; - #elif defined(KOKKOS_ENABLE_SYCL) + #elif defined(KOKKOS_ENABLE_SYCL) constexpr LoopPattern defaultLoop = LoopPattern::RANGE; #elif defined(KOKKOS_ENABLE_SERIAL) constexpr LoopPattern defaultLoop = LoopPattern::SIMDFOR; From dd702ab3e6d8a54b13e632773a00f78ab4b7247b Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 12 Mar 2025 14:27:16 -0400 Subject: [PATCH 71/82] add a static_assert so that the error observed on syscl doesn't go unnoticed when a new architecture is tried --- src/loop.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/loop.hpp b/src/loop.hpp index 262c6ac12..b6a5a9a1f 100644 --- a/src/loop.hpp +++ b/src/loop.hpp @@ -310,6 +310,12 @@ inline void idefix_for(const std::string & NAME, // SIMD FOR loops } else if constexpr(defaultLoop == LoopPattern::SIMDFOR) { + // Check that Idefix Arrays can be assigned from SIMD loop + static_assert( + Kokkos::SpaceAccessibility::accessible, + "Idefix Arrays cannot be accessed from SIMD loop. You should try another loop pattern." + ); for (auto n = NB; n < NE; n++) for (auto k = KB; k < KE; k++) for (auto j = JB; j < JE; j++) From 9d7c589b43f935ac10768cf49072cea597fb2e4d Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 12 Mar 2025 14:32:08 -0400 Subject: [PATCH 72/82] do this for all loops --- src/loop.hpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/loop.hpp b/src/loop.hpp index b6a5a9a1f..062380118 100644 --- a/src/loop.hpp +++ b/src/loop.hpp @@ -128,6 +128,12 @@ inline void idefix_for(const std::string & NAME, // SIMD FOR loops } else if constexpr(defaultLoop == LoopPattern::SIMDFOR) { + // Check that Idefix Arrays can be assigned from SIMD loop + static_assert( + Kokkos::SpaceAccessibility::accessible, + "Idefix Arrays cannot be accessed from SIMD loop. You should try another loop pattern." + ); for (auto j = JB; j < JE; j++) #pragma omp simd for (auto i = IB; i < IE; i++) @@ -212,6 +218,12 @@ inline void idefix_for(const std::string & NAME, // SIMD FOR loops } else if constexpr(defaultLoop == LoopPattern::SIMDFOR) { + // Check that Idefix Arrays can be assigned from SIMD loop + static_assert( + Kokkos::SpaceAccessibility::accessible, + "Idefix Arrays cannot be accessed from SIMD loop. You should try another loop pattern." + ); for (auto k = KB; k < KE; k++) for (auto j = JB; j < JE; j++) #pragma omp simd From 217a812a9a2bf82b676004ecee5b21eaf043796b Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 12 Mar 2025 14:46:13 -0400 Subject: [PATCH 73/82] catch misbehaved loops earlier (fix cuda!), and be more explicit. --- src/loop.hpp | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/src/loop.hpp b/src/loop.hpp index 062380118..20933ec91 100644 --- a/src/loop.hpp +++ b/src/loop.hpp @@ -34,6 +34,12 @@ typedef Kokkos::TeamPolicy<>::member_type member_type; // Check if the user requested a specific loop unrolling strategy #if defined(LOOP_PATTERN_SIMD) + // Check that Idefix Arrays can be assigned from SIMD loop + static_assert( + Kokkos::SpaceAccessibility::accessible, + "Idefix Arrays cannot be accessed from SIMD loop. You should try another loop pattern." + ); constexpr LoopPattern defaultLoop = LoopPattern::SIMDFOR; #elif defined(LOOP_PATTERN_1DRANGE) constexpr LoopPattern defaultLoop = LoopPattern::RANGE; @@ -54,6 +60,13 @@ typedef Kokkos::TeamPolicy<>::member_type member_type; #elif defined(KOKKOS_ENABLE_SYCL) constexpr LoopPattern defaultLoop = LoopPattern::RANGE; #elif defined(KOKKOS_ENABLE_SERIAL) + // Check that Idefix Arrays can be assigned from SIMD loop + static_assert( + Kokkos::SpaceAccessibility::accessible, + "Idefix Arrays cannot be accessed from Host, but Device is unknown/untested. " + "Ask the developers to add support." + ); constexpr LoopPattern defaultLoop = LoopPattern::SIMDFOR; #else #warning "Unknown target architeture: default to MDrange" @@ -129,11 +142,6 @@ inline void idefix_for(const std::string & NAME, // SIMD FOR loops } else if constexpr(defaultLoop == LoopPattern::SIMDFOR) { // Check that Idefix Arrays can be assigned from SIMD loop - static_assert( - Kokkos::SpaceAccessibility::accessible, - "Idefix Arrays cannot be accessed from SIMD loop. You should try another loop pattern." - ); for (auto j = JB; j < JE; j++) #pragma omp simd for (auto i = IB; i < IE; i++) @@ -219,11 +227,6 @@ inline void idefix_for(const std::string & NAME, // SIMD FOR loops } else if constexpr(defaultLoop == LoopPattern::SIMDFOR) { // Check that Idefix Arrays can be assigned from SIMD loop - static_assert( - Kokkos::SpaceAccessibility::accessible, - "Idefix Arrays cannot be accessed from SIMD loop. You should try another loop pattern." - ); for (auto k = KB; k < KE; k++) for (auto j = JB; j < JE; j++) #pragma omp simd @@ -322,12 +325,6 @@ inline void idefix_for(const std::string & NAME, // SIMD FOR loops } else if constexpr(defaultLoop == LoopPattern::SIMDFOR) { - // Check that Idefix Arrays can be assigned from SIMD loop - static_assert( - Kokkos::SpaceAccessibility::accessible, - "Idefix Arrays cannot be accessed from SIMD loop. You should try another loop pattern." - ); for (auto n = NB; n < NE; n++) for (auto k = KB; k < KE; k++) for (auto j = JB; j < JE; j++) From e58f72452254ef5650fd3529452af56eae1337f1 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 12 Mar 2025 16:32:48 -0400 Subject: [PATCH 74/82] cleaning up --- src/loop.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/loop.hpp b/src/loop.hpp index 20933ec91..42192fa84 100644 --- a/src/loop.hpp +++ b/src/loop.hpp @@ -141,7 +141,6 @@ inline void idefix_for(const std::string & NAME, // SIMD FOR loops } else if constexpr(defaultLoop == LoopPattern::SIMDFOR) { - // Check that Idefix Arrays can be assigned from SIMD loop for (auto j = JB; j < JE; j++) #pragma omp simd for (auto i = IB; i < IE; i++) @@ -226,7 +225,6 @@ inline void idefix_for(const std::string & NAME, // SIMD FOR loops } else if constexpr(defaultLoop == LoopPattern::SIMDFOR) { - // Check that Idefix Arrays can be assigned from SIMD loop for (auto k = KB; k < KE; k++) for (auto j = JB; j < JE; j++) #pragma omp simd From 939f0cdf362498f371acab02cc62b0a135c8d66f Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 31 Mar 2025 16:07:53 +0200 Subject: [PATCH 75/82] fix axis current regularisation along the south pole --- src/fluid/boundary/axis.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fluid/boundary/axis.cpp b/src/fluid/boundary/axis.cpp index b77fe3b5f..3ad67f0a0 100644 --- a/src/fluid/boundary/axis.cpp +++ b/src/fluid/boundary/axis.cpp @@ -128,12 +128,12 @@ void Axis::RegularizeCurrentSide(int side) { real deltaPhi = data->mygrid->xend[KDIR] - data->mygrid->xbeg[KDIR]; // Use the circulation around the pole of Bphi to determine Jr on the pole: - // Delta phi r^2(1-cos theta) Jr = int r sin(theta) Bphi dphi + // 1/2*Delta phi r^2 sin theta^2 Jr = int r sin(theta) Bphi dphi idefix_for("fixJ",0,data->np_tot[KDIR],0,data->np_tot[IDIR], KOKKOS_LAMBDA(int k,int i) { real th = x2(jc); - real fact = sign*sin(th)/(deltaPhi*x1(i)*(1-cos(th))); + real fact = 2*sign/(deltaPhi*x1(i)*sin(th)); J(IDIR, k,js,i) = BAvg(i)*fact; }); From 8e0481e93e07c94b9d6e2e1706b552058257cb12 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 14 Apr 2025 21:01:52 +0200 Subject: [PATCH 76/82] make divB check diemsionless --- src/fluid/checkDivB.hpp | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/fluid/checkDivB.hpp b/src/fluid/checkDivB.hpp index f9aac45c2..c78f6b3e0 100644 --- a/src/fluid/checkDivB.hpp +++ b/src/fluid/checkDivB.hpp @@ -30,14 +30,30 @@ real Fluid::CheckDivB() { data->beg[IDIR], data->end[IDIR], KOKKOS_LAMBDA (int k, int j, int i, real &divBmax) { [[maybe_unused]] real dB1,dB2,dB3; + [[maybe_unused]] real d1, d2, d3; + [[maybe_unused]] real B1,B2,B3; dB1=dB2=dB3=ZERO_F; + d1=d2=d3=ZERO_F; + B1=B2=B3=ZERO_F; + D_EXPAND( dB1=(Ax1(k,j,i+1)*Vs(BX1s,k,j,i+1)-Ax1(k,j,i)*Vs(BX1s,k,j,i)); , dB2=(Ax2(k,j+1,i)*Vs(BX2s,k,j+1,i)-Ax2(k,j,i)*Vs(BX2s,k,j,i)); , dB3=(Ax3(k+1,j,i)*Vs(BX3s,k+1,j,i)-Ax3(k,j,i)*Vs(BX3s,k,j,i)); ) - divBmax=FMAX(FABS(D_EXPAND(dB1, +dB2, +dB3))/dV(k,j,i),divBmax); + D_EXPAND( d1=0.5*(Ax1(k,j,i+1) + Ax1(k,j,i)); , + d2=0.5*(Ax2(k,j+1,i) + Ax2(k,j,i)); , + d3=0.5*(Ax3(k+1,j,i) + Ax3(k,j,i)); ) + + D_EXPAND( B1=0.5*(Vs(BX1s,k,j,i+1) + Vs(BX1s,k,j,i)); , + B2=0.5*(Vs(BX2s,k,j+1,i) + Vs(BX2s,k,j,i)); , + B3=0.5*(Vs(BX3s,k+1,j,i) + Vs(BX3s,k,j,i)); ) + + real amplitude = 1e-40; + amplitude += D_EXPAND( std::fabs(B1)*d1, + std::fabs(B2)*d2, + std::fabs(B3)*d3 ); + + divBmax=FMAX(FABS(D_EXPAND(dB1, +dB2, +dB3))/amplitude,divBmax); }, Kokkos::Max(divB) // reduction ); From d894627578db1f80322c9daadbcee26ef161dd78 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 16 Apr 2025 11:22:03 +0200 Subject: [PATCH 77/82] use one single extra parameter for dump time output --- src/output/output.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/output/output.cpp b/src/output/output.cpp index a67cb1666..9df188926 100644 --- a/src/output/output.cpp +++ b/src/output/output.cpp @@ -55,8 +55,13 @@ Output::Output(Input &input, DataBlock &data) dumpPeriod = input.Get("Output","dmp",0); dumpLast = data.t - dumpPeriod; // dump something in the next CheckForWrite() if(input.CheckEntry("Output","dmp")>1) { - dumpTimePeriod = input.Get("Output","dmp",1); - std::string dumpTimeExtension = input.Get("Output","dmp",2); + std::string dumpString = input.Get("Output","dmp",1); + try { + dumpTimePeriod = std::stod(dumpString.substr(0, dumpString.size()-1), NULL); + } catch(const std::exception& e) { + IDEFIX_ERROR("The dump time period should be a number followed by a unit (s, m, h or d)"); + } + std::string dumpTimeExtension = dumpString.substr(dumpString.size()-1,1); if(dumpTimeExtension.compare("s")==0) { dumpTimePeriod *= 1.0; // Dump time period is in seconds by default } else if (dumpTimeExtension.compare("m")==0) { From 83986d0efd681945bae6da2e5fba852d6014fa5d Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 16 Apr 2025 11:28:57 +0200 Subject: [PATCH 78/82] add doc for wallclock dump outputs --- doc/source/reference/idefix.ini.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/source/reference/idefix.ini.rst b/doc/source/reference/idefix.ini.rst index c9f9c5785..1fa05cc67 100644 --- a/doc/source/reference/idefix.ini.rst +++ b/doc/source/reference/idefix.ini.rst @@ -388,8 +388,10 @@ This section describes the outputs *Idefix* produces. For more details about eac +================+=========================+==================================================================================================+ | log | integer | | Time interval between log outputs, in code steps (default 100). | +----------------+-------------------------+--------------------------------------------------------------------------------------------------+ -| dmp | float | | Time interval between dump outputs, in code units. | -| | | | If negative, periodic dump outputs are disabled. | +| dmp | float, float+char | | 1st parameter: Code time interval between dump outputs, in code units. | +| | | | If negative, the first parameter is ignored. | +| | | | 2nd parameter (optional): Wallclock time interval between two dumps. The ending character | +| | | | can be "s" (seconds) "m" (minutes) "h" (hours) or "d" (days) | +----------------+-------------------------+--------------------------------------------------------------------------------------------------+ | dmp_dir | string | | directory for dump file outputs. Default to "./" | | | | | The directory is automatically created if it does not exist. | From 0a739f426933280273210daa0cb04b5be4e418ad Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 16 Apr 2025 10:13:49 +0200 Subject: [PATCH 79/82] update changelog --- CHANGELOG.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66f01f7be..c5cba24e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,25 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.2.01] 2025-04-18 +### Changed + +- Fix a bug that led to instabilities in the RKL scheme with very small grid spacings (#323) +- Fix a bug that prevented Idefix from running with Sycl backend (required on Intel GPUs) (#331) +- Fix an error that led to incorrect electrical current regularisation around the polar axis in non-Ideal MHD (#333) +- Improve div(B) checks with a dimensionless implementation, avoiding too large divB errors in grids with large stretch factors (#334) + +### Added + +- Time-Implicit drag for multiple dust species, preventing small time steps for strongly coupled dust grains (#321) +- Collisionless heat flux added to the Braginskii module (#317) +- New global `idfx::DumpArray` debugging function to dump any Idefix array into a numpy array that can read from python (#318) +- Automatic benchmark plots in the documentation (#319) +- More CI tests of grid coarsening (#329) + +### Removed + + ## [2.2.00] 2025-01-17 ### Changed From d406c2b555d15d98c83a4adcf2e87dd1f56f6bd6 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 16 Apr 2025 10:14:51 +0200 Subject: [PATCH 80/82] nothing was removed --- CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5cba24e9..d2b6c4a99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,8 +20,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Automatic benchmark plots in the documentation (#319) - More CI tests of grid coarsening (#329) -### Removed - ## [2.2.00] 2025-01-17 ### Changed From 2fe1a9a14662457f1eb326ea4e19e49c4a1d51d8 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 16 Apr 2025 10:32:04 +0200 Subject: [PATCH 81/82] update version number --- CMakeLists.txt | 4 ++-- doc/source/index.rst | 7 +++++-- src/main.cpp | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a8c7ff94e..6b62b7542 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,9 +6,9 @@ set (CMAKE_CXX_STANDARD 17) set(Idefix_VERSION_MAJOR 2) set(Idefix_VERSION_MINOR 2) -set(Idefix_VERSION_PATCH 00) +set(Idefix_VERSION_PATCH 01) -project (idefix VERSION 2.2.00) +project (idefix VERSION 2.2.01) option(Idefix_MHD "enable MHD" OFF) option(Idefix_MPI "enable Message Passing Interface parallelisation" OFF) option(Idefix_HIGH_ORDER_FARGO "Force Fargo to use a PPM reconstruction scheme" OFF) diff --git a/doc/source/index.rst b/doc/source/index.rst index 128d591ae..411d73d6d 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -84,8 +84,11 @@ Jonah Mauxion Clément Robert gitlab integration, linter -Jean Kempf & François Rincon - anisotropic diffusion +Jean Kempf, Victor Reville, François Rincon + anisotropic diffusion and collisionless thermal conduction + +Marc Coiffier + Continuous integration, automatic benchmarking ======================== About this documentation diff --git a/src/main.cpp b/src/main.cpp index 6b65c4f1c..d21793d1d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -9,7 +9,7 @@ //@HEADER // ************************************************************************ // -// IDEFIX v 2.2.00 +// IDEFIX v 2.2.01 // // ************************************************************************ //@HEADER From 5bf399e3bfbdb8f066b2784e109714d3d783c606 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 16 Apr 2025 14:27:06 +0200 Subject: [PATCH 82/82] update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2b6c4a99..b0732bb58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [2.2.01] 2025-04-18 +## [2.2.01] 2025-04-16 ### Changed - Fix a bug that led to instabilities in the RKL scheme with very small grid spacings (#323) @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - New global `idfx::DumpArray` debugging function to dump any Idefix array into a numpy array that can read from python (#318) - Automatic benchmark plots in the documentation (#319) - More CI tests of grid coarsening (#329) +- Dump outputs based on wallclock time (#335) ## [2.2.00] 2025-01-17