From da2702497ade86da469624f7745ecad686e47b41 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 3 Mar 2025 13:06:59 +0100 Subject: [PATCH 1/5] check coarsening inside ghost zone, and detect Nan. Reduce memory footprint by copying only what's needed --- src/dataBlock/coarsen.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/dataBlock/coarsen.cpp b/src/dataBlock/coarsen.cpp index 2da3049c..a922f226 100644 --- a/src/dataBlock/coarsen.cpp +++ b/src/dataBlock/coarsen.cpp @@ -66,15 +66,18 @@ void DataBlock::CheckCoarseningLevels() { idfx::pushRegion("DataBlock::CheckCoarseningLevels()"); // Check that the coarsening levels we have are valid // NB: this is a costly procedure, we can't repeat it at each loop! - DataBlockHost d(*this); - d.SyncFromDevice(); for(int dir = 0 ; dir < DIMENSIONS ; dir++) { if(mygrid->coarseningDirection[dir]) { - IdefixHostArray2D arr = d.coarseningLevel[dir]; - const int Xt = (dir == IDIR ? JDIR : IDIR); - const int Xb = (dir == KDIR ? JDIR : KDIR); - for(int i = beg[Xt] ; i < end[Xt] ; i++) { - for(int j = beg[Xb] ; j < end[Xb] ; j++) { + IdefixHostArray2D arr = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), + coarseningLevel[dir]); + for(int i = 0 ; i < arr.extent(1) ; i++) { + for(int j = 0 ; j < arr.extent(0) ; j++) { + if(std::isnan(arr(j,i))) { + std::stringstream str; + str << "Incorrect grid coarsening levels" << std::endl; + str << "at (i,j)=("<< i << "," << j << "): Coarsening level is NaN!" << std::endl; + IDEFIX_ERROR(str); + } if(arr(j,i) < 1) { std::stringstream str; str << "Incorrect grid coarsening levels" << std::endl; From 8bed32b1e3835afa5c99285c441c482d54e37f00 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 3 Mar 2025 13:10:59 +0100 Subject: [PATCH 2/5] fix loop unrolling --- src/dataBlock/coarsen.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dataBlock/coarsen.cpp b/src/dataBlock/coarsen.cpp index a922f226..28d7b669 100644 --- a/src/dataBlock/coarsen.cpp +++ b/src/dataBlock/coarsen.cpp @@ -70,8 +70,8 @@ void DataBlock::CheckCoarseningLevels() { if(mygrid->coarseningDirection[dir]) { IdefixHostArray2D arr = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), coarseningLevel[dir]); - for(int i = 0 ; i < arr.extent(1) ; i++) { - for(int j = 0 ; j < arr.extent(0) ; j++) { + for(int j = 0 ; j < arr.extent(0) ; j++) { + for(int i = 0 ; i < arr.extent(1) ; i++) { if(std::isnan(arr(j,i))) { std::stringstream str; str << "Incorrect grid coarsening levels" << std::endl; From 3d96f4104ff97bfde4dd1c446c880866300bfbaa Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 3 Mar 2025 17:39:43 +0100 Subject: [PATCH 3/5] add back axis flux tube test --- .github/workflows/idefix-ci-jobs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/idefix-ci-jobs.yml b/.github/workflows/idefix-ci-jobs.yml index b95385e4..275ea1ce 100644 --- a/.github/workflows/idefix-ci-jobs.yml +++ b/.github/workflows/idefix-ci-jobs.yml @@ -65,6 +65,8 @@ jobs: run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/OrszagTang -all $TESTME_OPTIONS - name: Orszag Tang 3D+restart tests run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/OrszagTang3D -all $TESTME_OPTIONS + - name: Axis Flux tube + run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/AxisFluxTube -all $TESTME_OPTIONS ParabolicMHD: runs-on: self-hosted From f283f7834b38172cdb3d34bc5edb78df88376e90 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 4 Mar 2025 10:24:50 +0100 Subject: [PATCH 4/5] remove analysis from axisfluxtube --- test/MHD/AxisFluxTube/setup.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/MHD/AxisFluxTube/setup.cpp b/test/MHD/AxisFluxTube/setup.cpp index 34009da8..4c30506d 100644 --- a/test/MHD/AxisFluxTube/setup.cpp +++ b/test/MHD/AxisFluxTube/setup.cpp @@ -112,7 +112,7 @@ void CoarsenFunction(DataBlock &data) { Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { output.EnrollUserDefVariables(&ComputeUserVars); data.hydro->EnrollUserDefBoundary(&UserdefBoundary); - output.EnrollAnalysis(&Analysis); + //output.EnrollAnalysis(&Analysis); Rtorus = input.Get("Setup","Rtorus",0); Ztorus = input.Get("Setup","Ztorus",0); Rin = input.Get("Setup","Rin",0); From 3fb2236fd461f139c23fe12dbbed847cd4e03daf Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 4 Mar 2025 15:29:49 +0100 Subject: [PATCH 5/5] final improvement to error message on grid coarsening levels --- src/dataBlock/coarsen.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/dataBlock/coarsen.cpp b/src/dataBlock/coarsen.cpp index 28d7b669..4c6b60cc 100644 --- a/src/dataBlock/coarsen.cpp +++ b/src/dataBlock/coarsen.cpp @@ -74,22 +74,25 @@ void DataBlock::CheckCoarseningLevels() { for(int i = 0 ; i < arr.extent(1) ; i++) { if(std::isnan(arr(j,i))) { std::stringstream str; - str << "Incorrect grid coarsening levels" << std::endl; + str << "Nan in grid coarsening levels" << std::endl; str << "at (i,j)=("<< i << "," << j << "): Coarsening level is NaN!" << std::endl; IDEFIX_ERROR(str); } if(arr(j,i) < 1) { std::stringstream str; - str << "Incorrect grid coarsening levels" << std::endl; - str << "at (i,j)=("<< i << "," << j << "): Coarsening level < 1!" << std::endl; + str << "Coarsening level < 1!" << std::endl; + str << "at (i,j)=("<< i << "," << j << "): "; + str << "coarsening level= " << arr(j,i) << std::endl; IDEFIX_ERROR(str); } const int factor = 1 << (arr(j,i) - 1); if(np_int[dir] % factor != 0) { std::stringstream str; - str << "local grid size not divisible by coarsening level" << std::endl; - str << "at (i,j)=("<< i << "," << j << "): Coarsening level: "; - str << arr(j,i) << std::endl; + str << "Local grid size not divisible by coarsening level." << std::endl; + str << "at (i,j)=("<< i << "," << j << "): "; + str << "coarsening level= " << arr(j,i) << std::endl; + str << np_int[dir] << " cannot be divided by 2^" << arr(j,i)-1; + str << " = " << factor << std::endl; IDEFIX_ERROR(str); } }