From 36e8c24a6999d0051ffd26cf4782f12a4b05fb0a Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 9 Jul 2024 17:33:08 +0200 Subject: [PATCH 01/83] ensure that Fargo can't be used with too low DIMENSIONS (#250) Following problems encountered by T. Jannaud on 1D spherical tests. --- src/dataBlock/fargo.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/dataBlock/fargo.cpp b/src/dataBlock/fargo.cpp index e1878d37..a509ef88 100644 --- a/src/dataBlock/fargo.cpp +++ b/src/dataBlock/fargo.cpp @@ -65,6 +65,9 @@ Fargo::Fargo(Input &input, int nmax, DataBlock *data) { } #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR + #if DIMENSIONS < 2 + IDEFIX_ERROR("Fargo should be used with DIMENSIONS >= 2 in Cartesian or Polar geometries"); + #endif // Check if there is a domain decomposition in the intended fargo direction if(data->mygrid->nproc[JDIR]>1) { haveDomainDecomposition = true; @@ -80,6 +83,9 @@ Fargo::Fargo(Input &input, int nmax, DataBlock *data) { this->meanVelocity = IdefixArray2D("FargoVelocity",data->np_tot[KDIR], data->np_tot[IDIR]); #elif GEOMETRY == SPHERICAL + #if DIMENSIONS < 3 + IDEFIX_ERROR("Fargo should be used with DIMENSIONS == 3 in Spherical geometry"); + #endif // Check if there is a domain decomposition in the intended fargo direction if(data->mygrid->nproc[KDIR]>1) { haveDomainDecomposition = true; From cb5117cd50963a52931147ff8e7e3f93e556a6d4 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 9 Jul 2024 17:35:57 +0200 Subject: [PATCH 02/83] Doc fix (#251) * benchmarks have tracers disabled * fix typo in AdAstra configuration options --- doc/source/performances.rst | 6 +++--- doc/source/reference/makefile.rst | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/source/performances.rst b/doc/source/performances.rst index 2222c2e1..84042685 100644 --- a/doc/source/performances.rst +++ b/doc/source/performances.rst @@ -4,9 +4,9 @@ Performances We report below the performances obtained on various architectures using Idefix. The reference test is the 3D MHD Orszag-Tang test problem with 2nd order reconstruction and uct_contact EMFS bundled in -Idefix test suite, computed with a 128\ :sup:`3` resolution per MPI sub-domain on GPUs or 32\ :sup:`3` -per MPI sub-domain on CPUs. All of the performances measures have been obtained enabling MPI on -*one full node*, but we report here the performance *per GPU* +Idefix test suite, disabling passive tracers. The test is computed with a 128\ :sup:`3` resolution per +MPI sub-domain on GPUs or 32\ :sup:`3` per MPI sub-domain on CPUs. All of the performances measures +have been obtained enabling MPI on *one full node*, but we report here the performance *per GPU* (i.e. with 2 GCDs on AMD Mi250) or *per core* (on CPU), i.e. dividing the node performance by the number of GPU/core to simplify the comparison with other clusters. diff --git a/doc/source/reference/makefile.rst b/doc/source/reference/makefile.rst index b015b682..bf8ecdad 100644 --- a/doc/source/reference/makefile.rst +++ b/doc/source/reference/makefile.rst @@ -125,7 +125,7 @@ Finally, *Idefix* can be configured to run on Mi250 by enabling HIP and the desi .. code-block:: bash - -DKokkos_ENABLE_HIP=ON -DKokkos_ENABLE_HIP_MULTIPLE_KERNEL_INSTANTIATION=ON -DKokkos_ARCH_VEGA90A=ON + -DKokkos_ENABLE_HIP=ON -DKokkos_ENABLE_HIP_MULTIPLE_KERNEL_INSTANTIATIONS=ON -DKokkos_ARCH_VEGA90A=ON MPI (multi-GPU) can be enabled by adding ``-DIdefix_MPI=ON`` as usual. From 040592bae192be5c1f20c0d39b2d01f199036bdd Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Fri, 19 Jul 2024 17:00:40 +0200 Subject: [PATCH 03/83] fix #252 (#253) --- src/fluid/boundary/boundary.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fluid/boundary/boundary.hpp b/src/fluid/boundary/boundary.hpp index 4a9b34ee..86909936 100644 --- a/src/fluid/boundary/boundary.hpp +++ b/src/fluid/boundary/boundary.hpp @@ -199,7 +199,7 @@ void Boundary::EnrollFluxBoundary(UserDefBoundaryFuncOld myFunc) { IDEFIX_WARNING(msg); this->fluxBoundaryFuncOld = myFunc; - this->haveUserDefBoundary = true; + this->haveFluxBoundary = true; } template From 0a6193d8ad7a7f7cd3f96c72fc3b5b8acfda136d Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Fri, 26 Jul 2024 17:05:46 +0200 Subject: [PATCH 04/83] avoid implicit conversion to double in VTK I/O routines (#255) * avoid implicit conversion to double in VTK I/O routines * consolidated implementation of BigEndian - works for any type size - checks that we are dealing with numbers - remove pointers in favor of more elegant c++ unions * use proper lower case for variables and functions ("bigEndian") --- src/output/vtk.cpp | 52 ++++++++++++++++----------------- src/output/vtk.hpp | 31 +++----------------- src/utils/CMakeLists.txt | 1 + src/utils/bigEndian.hpp | 63 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 94 insertions(+), 53 deletions(-) create mode 100644 src/utils/bigEndian.hpp diff --git a/src/output/vtk.cpp b/src/output/vtk.cpp index 559f00c8..13b47d30 100644 --- a/src/output/vtk.cpp +++ b/src/output/vtk.cpp @@ -124,21 +124,21 @@ Vtk::Vtk(Input &input, DataBlock *datain, std::string filebase) { for (int32_t i = 0; i < nx1 + ioffset; i++) { if(grid.np_tot[IDIR] == 1) // only one dimension in this direction - xnode[i] = BigEndian(static_cast(grid.x[IDIR](i))); + xnode[i] = bigEndian(static_cast(grid.x[IDIR](i))); else - xnode[i] = BigEndian(static_cast(grid.xl[IDIR](i + grid.nghost[IDIR]))); + xnode[i] = bigEndian(static_cast(grid.xl[IDIR](i + grid.nghost[IDIR]))); } for (int32_t j = 0; j < nx2 + joffset; j++) { if(grid.np_tot[JDIR] == 1) // only one dimension in this direction - ynode[j] = BigEndian(static_cast(grid.x[JDIR](j))); + ynode[j] = bigEndian(static_cast(grid.x[JDIR](j))); else - ynode[j] = BigEndian(static_cast(grid.xl[JDIR](j + grid.nghost[JDIR]))); + ynode[j] = bigEndian(static_cast(grid.xl[JDIR](j + grid.nghost[JDIR]))); } for (int32_t k = 0; k < nx3 + koffset; k++) { if(grid.np_tot[KDIR] == 1) - znode[k] = BigEndian(static_cast(grid.x[KDIR](k))); + znode[k] = bigEndian(static_cast(grid.x[KDIR](k))); else - znode[k] = BigEndian(static_cast(grid.xl[KDIR](k + grid.nghost[KDIR]))); + znode[k] = bigEndian(static_cast(grid.xl[KDIR](k + grid.nghost[KDIR]))); } #if VTK_FORMAT == VTK_STRUCTURED_GRID // VTK_FORMAT /* -- Allocate memory for node_coord which is later used -- */ @@ -191,35 +191,35 @@ Vtk::Vtk(Input &input, DataBlock *datain, std::string filebase) { for (int32_t k = 0; k < nodesubsize[0]; k++) { for (int32_t j = 0; j < nodesubsize[1]; j++) { for (int32_t i = 0; i < nodesubsize[2]; i++) { - // BigEndian allows us to get back to little endian when needed + // bigEndian allows us to get back to little endian when needed x1 = grid.xl[IDIR](i + data->gbeg[IDIR]); x2 = grid.xl[JDIR](j + data->gbeg[JDIR]); x3 = grid.xl[KDIR](k + data->gbeg[KDIR]); #if (GEOMETRY == CARTESIAN) || (GEOMETRY == CYLINDRICAL) - node_coord(k,j,i,0) = BigEndian(x1); - node_coord(k,j,i,1) = BigEndian(x2); - node_coord(k,j,i,2) = BigEndian(x3); + node_coord(k,j,i,0) = bigEndian(x1); + node_coord(k,j,i,1) = bigEndian(x2); + node_coord(k,j,i,2) = bigEndian(x3); #elif GEOMETRY == POLAR - node_coord(k,j,i,0) = BigEndian(x1 * cos(x2)); - node_coord(k,j,i,1) = BigEndian(x1 * sin(x2)); - node_coord(k,j,i,2) = BigEndian(x3); + node_coord(k,j,i,0) = bigEndian(x1 * std::cos(x2)); + node_coord(k,j,i,1) = bigEndian(x1 * std::sin(x2)); + node_coord(k,j,i,2) = bigEndian(x3); #elif GEOMETRY == SPHERICAL #if DIMENSIONS == 1 - node_coord(k,j,i,0) = BigEndian(x1); - node_coord(k,j,i,1) = BigEndian(0.0); - node_coord(k,j,i,2) = BigEndian(0.0); + node_coord(k,j,i,0) = bigEndian(x1); + node_coord(k,j,i,1) = bigEndian(0.0f); + node_coord(k,j,i,2) = bigEndian(0.0f); #elif DIMENSIONS == 2 - node_coord(k,j,i,0) = BigEndian(x1 * sin(x2)); - node_coord(k,j,i,1) = BigEndian(x1 * cos(x2)); - node_coord(k,j,i,2) = BigEndian(0.0); + node_coord(k,j,i,0) = bigEndian(x1 * std::sin(x2)); + node_coord(k,j,i,1) = bigEndian(x1 * std::cos(x2)); + node_coord(k,j,i,2) = bigEndian(0.0f); #elif DIMENSIONS == 3 - node_coord(k,j,i,0) = BigEndian(x1 * sin(x2) * cos(x3)); - node_coord(k,j,i,1) = BigEndian(x1 * sin(x2) * sin(x3)); - node_coord(k,j,i,2) = BigEndian(x1 * cos(x2)); + node_coord(k,j,i,0) = bigEndian(x1 * std::sin(x2) * std::cos(x3)); + node_coord(k,j,i,1) = bigEndian(x1 * std::sin(x2) * std::sin(x3)); + node_coord(k,j,i,2) = bigEndian(x1 * std::cos(x2)); #endif // DIMENSIONS #endif // GEOMETRY } @@ -301,7 +301,7 @@ int Vtk::Write() { for(int j = data->beg[JDIR]; j < data->end[JDIR] ; j++ ) { for(int i = data->beg[IDIR]; i < data->end[IDIR] ; i++ ) { vect3D[i-data->beg[IDIR] + (j-data->beg[JDIR])*nx1loc + (k-data->beg[KDIR])*nx1loc*nx2loc] - = BigEndian(static_cast(Vcin(k,j,i))); + = bigEndian(static_cast(Vcin(k,j,i))); } } } @@ -379,7 +379,7 @@ void Vtk::WriteHeader(IdfxFileHandler fvtk, real time) { ssheader.str(std::string()); // convert time to single precision big endian - int32_t geoBig = BigEndian(this->geometry); + int32_t geoBig = bigEndian(this->geometry); WriteHeaderBinary(&geoBig, 1, fvtk); // Done, add cariage return for next ascii write @@ -396,7 +396,7 @@ void Vtk::WriteHeader(IdfxFileHandler fvtk, real time) { int32_t perBig{-1}; for (int dir=0; dir<3; dir++) { - perBig = BigEndian(this->periodicity[dir]); + perBig = bigEndian(this->periodicity[dir]); WriteHeaderBinary(&perBig, 1, fvtk); } // Done, add cariage return for next ascii write @@ -410,7 +410,7 @@ void Vtk::WriteHeader(IdfxFileHandler fvtk, real time) { ssheader.str(std::string()); // convert time to single precision big endian - float timeBE = BigEndian(static_cast(time)); + float timeBE = bigEndian(static_cast(time)); WriteHeaderBinary(&timeBE, 1, fvtk); // Done, add cariage return for next ascii write diff --git a/src/output/vtk.hpp b/src/output/vtk.hpp index 4857f19d..c33e0d05 100644 --- a/src/output/vtk.hpp +++ b/src/output/vtk.hpp @@ -21,25 +21,18 @@ #include "idefix.hpp" #include "input.hpp" #include "dataBlock.hpp" +#include "bigEndian.hpp" #include "scalarField.hpp" + // Forward class declaration class Output; class DataBlock; class BaseVtk { - private: - // Endianness swaping function and variable - bool shouldSwapEndian {true}; - protected: BaseVtk() { - // Test endianness - int tmp1 = 1; - unsigned char *tmp2 = (unsigned char *) &tmp1; - if (*tmp2 == 0) - this->shouldSwapEndian = false; // Initialise the root tag (used for MPI non-collective I/Os) this->isRoot = idfx::prank == 0; } @@ -68,25 +61,9 @@ class BaseVtk { // DataBlock parent DataBlock *data; - /* ****************************************************************************/ - /** Determines if the machine is little-endian. If so, - it will force the data to be big-endian. - @param in_number floating point number to be converted in big endian */ - /* *************************************************************************** */ + // BigEndian conversion + BigEndian bigEndian; - template - T BigEndian(T in_number) { - if (shouldSwapEndian) { - unsigned char *bytes = (unsigned char*) &in_number; - unsigned char tmp = bytes[0]; - bytes[0] = bytes[3]; - bytes[3] = tmp; - tmp = bytes[1]; - bytes[1] = bytes[2]; - bytes[2] = tmp; - } - return(in_number); - } void WriteHeaderString(const char* header, IdfxFileHandler fvtk) { #ifdef WITH_MPI diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt index ff971569..b36f8a08 100644 --- a/src/utils/CMakeLists.txt +++ b/src/utils/CMakeLists.txt @@ -1,6 +1,7 @@ add_subdirectory(iterativesolver) target_sources(idefix + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/bigEndian.hpp PUBLIC ${CMAKE_CURRENT_LIST_DIR}/dumpImage.cpp PUBLIC ${CMAKE_CURRENT_LIST_DIR}/dumpImage.hpp PUBLIC ${CMAKE_CURRENT_LIST_DIR}/lookupTable.hpp diff --git a/src/utils/bigEndian.hpp b/src/utils/bigEndian.hpp new file mode 100644 index 00000000..08bd73c0 --- /dev/null +++ b/src/utils/bigEndian.hpp @@ -0,0 +1,63 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef UTILS_BIGENDIAN_HPP_ +#define UTILS_BIGENDIAN_HPP_ + +#include +#include +#include "idefix.hpp" + +/* ****************************************************************************/ +/** Determines if the machine is little-endian. If so, + it will force the data to be big-endian. +@param in_number floating point number to be converted in big endian */ +/* *************************************************************************** */ + +class BigEndian { + public: + BigEndian() { + // Test endianness + union { + uint32_t i; + char c[4]; + } bint = {0x01020304}; + + if(bint.c[0] == 1) + this->shouldSwapEndian = false; + else + this->shouldSwapEndian = true; + } + + // Swap when needed + template + T operator() (T in_number) { + static_assert(std::is_arithmetic_v == true); + T out_number; + if (this->shouldSwapEndian) { + constexpr int size = sizeof(T); + union { + T u; + unsigned char byte[size]; + } in, out; + in.u = in_number; + for(int n = 0 ; n < size ; n++) { + out.byte[size-n-1] = in.byte[n]; + } + out_number = out.u; + } else { + out_number = in_number; + } + return(out_number); + } + + private: + // Endianness swaping flag + bool shouldSwapEndian; +}; + +#endif // UTILS_BIGENDIAN_HPP_ From 1859ad502e5a97e4d39443fbcb49693b6ea143ba Mon Sep 17 00:00:00 2001 From: Nicolas Aunai Date: Wed, 7 Aug 2024 23:26:30 +0200 Subject: [PATCH 05/83] set built type as default (allow user override) (#256) --- CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b7b1d88a..2ef5e686 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,7 @@ cmake_minimum_required(VERSION 3.16) -set(CMAKE_BUILD_TYPE Release) +if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE "Release" CACHE STRING "" FORCE) +endif() set (CMAKE_CXX_STANDARD 17) set(Idefix_VERSION_MAJOR 2) From e5ca2b51ec602e4bcd7da8f610eae7881c02a52c Mon Sep 17 00:00:00 2001 From: vdbma <93188557+vdbma@users.noreply.github.com> Date: Mon, 9 Sep 2024 08:09:51 +0200 Subject: [PATCH 06/83] Make shockFlattening public (#257) Co-authored-by: Marc Van den Bossche --- src/fluid/RiemannSolver/riemannSolver.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fluid/RiemannSolver/riemannSolver.hpp b/src/fluid/RiemannSolver/riemannSolver.hpp index 4b306f38..abb70eed 100644 --- a/src/fluid/RiemannSolver/riemannSolver.hpp +++ b/src/fluid/RiemannSolver/riemannSolver.hpp @@ -62,6 +62,8 @@ class RiemannSolver { template ExtrapolateToFaces* GetExtrapolator(); + std::unique_ptr> shockFlattening; + private: template friend class ExtrapolateToFaces; @@ -75,8 +77,6 @@ class RiemannSolver { Solver mySolver; - std::unique_ptr> shockFlattening; - // Because each direction is a different template, we can't use std::unique_ptr> slopeLimIDIR; std::unique_ptr> slopeLimJDIR; From a020a3fe844e1f6e451e314f6a0074b7a5205544 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Fri, 13 Sep 2024 22:34:02 +0200 Subject: [PATCH 07/83] Check for successful write in serial (#260) --- src/output/dump.cpp | 42 +++++++++++++++++++++++++++++++++--------- src/output/vtk.cpp | 15 +++++++++++++-- src/output/vtk.hpp | 9 +++++++-- 3 files changed, 53 insertions(+), 13 deletions(-) diff --git a/src/output/dump.cpp b/src/output/dump.cpp index 5a27ecc2..73b0df99 100644 --- a/src/output/dump.cpp +++ b/src/output/dump.cpp @@ -248,7 +248,9 @@ void Dump::WriteString(IdfxFileHandler fileHdl, char *str, int size) { } offset=offset+size; #else - fwrite (str, sizeof(char), size, fileHdl); + if(fwrite (str, sizeof(char), size, fileHdl) != size) { + IDEFIX_ERROR("Unable to write to file. Check your filesystem permissions and disk quota."); + } #endif } @@ -312,15 +314,23 @@ void Dump::WriteSerial(IdfxFileHandler fileHdl, int ndim, int *dim, #else // Write type of data - fwrite(&type, 1, sizeof(int), fileHdl); + if(fwrite(&type, sizeof(int), 1, fileHdl) != 1) { + IDEFIX_ERROR("Unable to write to file. Check your filesystem permissions and disk quota."); + } // Write dimensions of array - fwrite(&ndim, 1, sizeof(int), fileHdl); + if(fwrite(&ndim, sizeof(int), 1, fileHdl) != 1) { + IDEFIX_ERROR("Unable to write to file. Check your filesystem permissions and disk quota."); + } for(int n = 0 ; n < ndim ; n++) { - fwrite(dim+n, 1, sizeof(int), fileHdl); + if(fwrite(dim+n, sizeof(int), 1, fileHdl) != 1) { + IDEFIX_ERROR("Unable to write to file. Check your filesystem permissions and disk quota."); + } ntot = ntot * dim[n]; } // Write raw data - fwrite(data, ntot, size, fileHdl); + if(fwrite(data, size, ntot, fileHdl) != ntot) { + IDEFIX_ERROR("Unable to write to file. Check your filesystem permissions and disk quota."); + } #endif } @@ -384,18 +394,26 @@ void Dump::WriteDistributed(IdfxFileHandler fileHdl, int ndim, int *dim, int *gd #else // Write type of data - fwrite(&type, 1, sizeof(int), fileHdl); + if(fwrite(&type, sizeof(int), 1, fileHdl) != 1) { + IDEFIX_ERROR("Unable to write to file. Check your filesystem permissions and disk quota."); + } // Write dimensions of array // (in serial, dim and gdim are identical, so no need to differentiate) - fwrite(&ndim, 1, sizeof(int), fileHdl); + if(fwrite(&ndim, sizeof(int), 1, fileHdl) != 1) { + IDEFIX_ERROR("Unable to write to file. Check your filesystem permissions and disk quota."); + } for(int n = 0 ; n < ndim ; n++) { - fwrite(dim+n, 1, sizeof(int), fileHdl); + if(fwrite(dim+n, sizeof(int), 1, fileHdl) != 1) { + IDEFIX_ERROR("Unable to write to file. Check your filesystem permissions and disk quota."); + } ntot = ntot * dim[n]; } // Write raw data - fwrite(data, ntot, sizeof(real), fileHdl); + if(fwrite(data, sizeof(real), ntot, fileHdl) != ntot) { + IDEFIX_ERROR("Unable to write to file. Check your filesystem permissions and disk quota."); + } #endif } @@ -836,6 +854,12 @@ int Dump::Write(Output& output) { this->offset = 0; #else fileHdl = fopen(filename.c_str(),"wb"); + if(fileHdl == NULL) { + std::stringstream msg; + msg << "Unable to open file " << filename << std::endl; + msg << "Check that you have write access and that you don't exceed your quota." << std::endl; + IDEFIX_ERROR(msg); + } #endif // File is open // First thing we need are coordinates: init a host mirror and sync it diff --git a/src/output/vtk.cpp b/src/output/vtk.cpp index 13b47d30..0f52f37c 100644 --- a/src/output/vtk.cpp +++ b/src/output/vtk.cpp @@ -53,7 +53,9 @@ void Vtk::WriteHeaderNodes(IdfxFileHandler fvtk) { MPI_FLOAT, MPI_STATUS_IGNORE)); this->offset += sizeof(float)*(nx1+ioffset)*(nx2+joffset)*(nx3+koffset)*3; #else - fwrite(node_coord.data(),sizeof(float),size,fvtk); + if(fwrite(node_coord.data(),sizeof(float),size,fvtk) != size) { + IDEFIX_ERROR("Unable to write to file. Check your filesystem permissions and disk quota."); + } #endif } @@ -290,6 +292,13 @@ int Vtk::Write() { this->offset = 0; #else fileHdl = fopen(filename.c_str(),"wb"); + + if(fileHdl == NULL) { + std::stringstream msg; + msg << "Unable to open file " << filename << std::endl; + msg << "Check that you have write access and that you don't exceed your quota." << std::endl; + IDEFIX_ERROR(msg); + } #endif WriteHeader(fileHdl, this->data->t); @@ -502,6 +511,8 @@ void Vtk::WriteScalar(IdfxFileHandler fvtk, float* Vin, const std::string &var_ this->offset = this->offset + sizeof(float)*nx1*nx2*nx3; #else - fwrite(Vin,sizeof(float),nx1loc*nx2loc*nx3loc,fvtk); + if(fwrite(Vin,sizeof(float),nx1loc*nx2loc*nx3loc,fvtk) != nx1loc*nx2loc*nx3loc) { + IDEFIX_ERROR("Unable to write to file. Check your filesystem permissions and disk quota."); + } #endif } diff --git a/src/output/vtk.hpp b/src/output/vtk.hpp index c33e0d05..dcda3618 100644 --- a/src/output/vtk.hpp +++ b/src/output/vtk.hpp @@ -75,7 +75,10 @@ class BaseVtk { } offset=offset+strlen(header); #else - fprintf (fvtk, "%s", header); + int rc = fprintf (fvtk, "%s", header); + if(rc<0) { + IDEFIX_ERROR("Unable to write to file. Check your filesystem permissions and disk quota."); + } #endif } @@ -90,7 +93,9 @@ class BaseVtk { } offset=offset+nelem*sizeof(T); #else - fwrite(buffer, sizeof(T), nelem, fvtk); + if(fwrite(buffer, sizeof(T), nelem, fvtk) != nelem) { + IDEFIX_ERROR("Unable to write to file. Check your filesystem permissions and disk quota."); + } #endif } }; From 362969517d55fbc8b5307382338e7252a01fc542 Mon Sep 17 00:00:00 2001 From: Alankar Dutta Date: Mon, 16 Sep 2024 02:43:36 -0600 Subject: [PATCH 08/83] [FIX] xdmf datatype agnostic dump (#261) Co-authored-by: Alankar Dutta --- src/output/xdmf.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/output/xdmf.cpp b/src/output/xdmf.cpp index 7f2baf7d..6ad4e22a 100644 --- a/src/output/xdmf.cpp +++ b/src/output/xdmf.cpp @@ -165,7 +165,7 @@ Xdmf::Xdmf(Input &input, DataBlock *datain) { cellsubsize[3]); */ // Temporary storage on host for 3D arrays - this->vect3D = new float[nx1loc*nx2loc*nx3loc]; + this->vect3D = new DUMP_DATATYPE[nx1loc*nx2loc*nx3loc]; // fill the node_coord array DUMP_DATATYPE x1 = 0.0; From dcaefdf6a9a0ad859ab7b8b099013db593ff84d8 Mon Sep 17 00:00:00 2001 From: ThomasJannaudCAM <159052976+ThomasJannaudCAM@users.noreply.github.com> Date: Fri, 20 Sep 2024 13:18:25 +0200 Subject: [PATCH 09/83] Update setup.cpp.rst (#264) Corrected a typo on "restart from a dump", arrow that should have been a point --- doc/source/reference/setup.cpp.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/reference/setup.cpp.rst b/doc/source/reference/setup.cpp.rst index 2662e903..0ebe76d3 100644 --- a/doc/source/reference/setup.cpp.rst +++ b/doc/source/reference/setup.cpp.rst @@ -369,7 +369,7 @@ assuming we want to create a dump from ``mydump.dmp``: int iglob=i-2*d.beg[IDIR]+d.gbeg[IDIR]; int jglob=j-2*d.beg[JDIR]+d.gbeg[JDIR]; int kglob=k-2*d.beg[KDIR]+d.gbeg[KDIR]; - d.Vs(BX1s,k,j,i) = image->arrays["Vs-BX1s"](kglob,jglob,iglob); + d.Vs(BX1s,k,j,i) = image.arrays["Vs-BX1s"](kglob,jglob,iglob); }}} // And so on for the other components From 6657a0a52ee708a8d9d8f13f263949e1e4d75475 Mon Sep 17 00:00:00 2001 From: Alankar Dutta Date: Fri, 27 Sep 2024 15:31:49 +0200 Subject: [PATCH 10/83] Minor correction to have the filesystem header same as dump and vtk (#269) * [FIX] xdmf datatype agnostic dump * [ENH] xdmf io now supports experimental filesystems like dump and vtk --------- Co-authored-by: Alankar Dutta --- src/output/xdmf.cpp | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/output/xdmf.cpp b/src/output/xdmf.cpp index 6ad4e22a..237f4ca3 100644 --- a/src/output/xdmf.cpp +++ b/src/output/xdmf.cpp @@ -10,6 +10,16 @@ #include #include #include +#if __has_include() + #include + namespace fs = std::filesystem; +#elif __has_include() + #include + namespace fs = std::experimental::filesystem; +#else + #error "Missing the header." +#endif + #include "xdmf.hpp" #include "version.hpp" #include "idefix.hpp" @@ -39,9 +49,9 @@ Xdmf::Xdmf(Input &input, DataBlock *datain) { } if(idfx::prank==0) { - if(!std::filesystem::is_directory(outputDirectory)) { + if(!fs::is_directory(outputDirectory)) { try { - if(!std::filesystem::create_directory(outputDirectory)) { + if(!fs::create_directory(outputDirectory)) { std::stringstream msg; msg << "Cannot create directory " << outputDirectory << std::endl; IDEFIX_ERROR(msg); @@ -274,8 +284,8 @@ Xdmf::Xdmf(Input &input, DataBlock *datain) { int Xdmf::Write() { idfx::pushRegion("Xdmf::Write"); - std::filesystem::path filename; - std::filesystem::path filename_xmf; + fs::path filename; + fs::path filename_xmf; hid_t err; idfx::cout << "Xdmf: Write file n " << xdmfFileNumber << "..." << std::flush; timer.reset(); From c0974592ef2016d7be8ab7bebd487f2c9448b29b Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 30 Oct 2023 22:58:16 +0100 Subject: [PATCH 11/83] add Python binding to cmake --- CMakeLists.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2ef5e686..e1e20d4c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,6 +15,7 @@ option(Idefix_HIGH_ORDER_FARGO "Force Fargo to use a PPM reconstruction scheme" option(Idefix_DEBUG "Enable Idefix debug features (makes the code very slow)" OFF) option(Idefix_RUNTIME_CHECKS "Enable runtime sanity checks" OFF) option(Idefix_WERROR "Treat compiler warnings as errors" OFF) +option(Idefix_PYTHON "Enable python bindings (requires pybind11)" OFF) set(Idefix_CXX_FLAGS "" CACHE STRING "Additional compiler/linker flag") set(Idefix_DEFS "definitions.hpp" CACHE FILEPATH "Problem definition header file") option(Idefix_CUSTOM_EOS "Use custom equation of state" OFF) @@ -118,6 +119,12 @@ else() set(Idefix_HDF5 OFF) endif() +if(Idefix_PYTHON) + add_compile_definitions("WITH_PYTHON") + find_package(pybind11 REQUIRED) + target_link_libraries(idefix PRIVATE pybind11::embed) +endif() + if(Idefix_DEBUG) add_compile_definitions("DEBUG") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g") @@ -229,6 +236,7 @@ else() endif() message(STATUS " MPI: ${Idefix_MPI}") message(STATUS " HDF5: ${Idefix_HDF5}") +message(STATUS " Python: ${Idefix_PYTHON}") message(STATUS " Reconstruction: ${Idefix_RECONSTRUCTION}") message(STATUS " Precision: ${Idefix_PRECISION}") message(STATUS " Version: ${Idefix_VERSION}") From 8a07eafa4f57ada729b85fd34dbe53bc351e1982 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 31 Oct 2023 09:20:48 +0100 Subject: [PATCH 12/83] working version with pybind11 --- CMakeLists.txt | 8 ++++++-- src/output/output.hpp | 7 +++++++ src/pydefix.cpp | 33 +++++++++++++++++++++++++++++++++ src/pydefix.hpp | 21 +++++++++++++++++++++ 4 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 src/pydefix.cpp create mode 100644 src/pydefix.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index e1e20d4c..3a0e2d3a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -113,7 +113,7 @@ if(Idefix_HDF5) ) find_package(HDF5 REQUIRED) target_link_libraries(idefix "${HDF5_LIBRARIES}") - target_include_directories(idefix PUBLIC "${HDF5_INCLUDE_DIRS}") + target_include_directories(idefix "${HDF5_INCLUDE_DIRS}") message(STATUS "XDMF (hdf5+xmf) dumps enabled") else() set(Idefix_HDF5 OFF) @@ -122,7 +122,11 @@ endif() if(Idefix_PYTHON) add_compile_definitions("WITH_PYTHON") find_package(pybind11 REQUIRED) - target_link_libraries(idefix PRIVATE pybind11::embed) + target_link_libraries(idefix pybind11::embed) + target_sources(idefix + PUBLIC src/pydefix.cpp + PUBLIC src/pydefix.hpp + ) endif() if(Idefix_DEBUG) diff --git a/src/output/output.hpp b/src/output/output.hpp index 54cb97af..331ae60c 100644 --- a/src/output/output.hpp +++ b/src/output/output.hpp @@ -18,6 +18,9 @@ #ifdef WITH_HDF5 #include "xdmf.hpp" #endif +#ifdef WITH_PYTHON +#include "pydefix.hpp" +#endif #include "dump.hpp" #include "slice.hpp" @@ -81,5 +84,9 @@ class Output { Kokkos::Timer timer; double elapsedTime{0.0}; + + #ifdef WITH_PYTHON + Pydefix pydefix; + #endif }; #endif // OUTPUT_OUTPUT_HPP_ diff --git a/src/pydefix.cpp b/src/pydefix.cpp new file mode 100644 index 00000000..97599a1b --- /dev/null +++ b/src/pydefix.cpp @@ -0,0 +1,33 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#include // everything needed for embedding +#include "idefix.hpp" +#include "pydefix.hpp" + +namespace py = pybind11; + +int Pydefix::ninstance = 0; + +Pydefix::Pydefix() { + ninstance++; + if(ninstance==1) { + idfx::cout << "Pydefix: start interpreter." << std::endl; + + py::initialize_interpreter(); + } + py::print("Hello,from python"); // use the Python API +} + +Pydefix::~Pydefix() { + if(ninstance == 1) { + py::finalize_interpreter(); + idfx::cout << "Pydefix: shutdown interpreter." << std::endl; + } + ninstance--; + idfx::cout << "Pydefix: destroyed." << std::endl; +} diff --git a/src/pydefix.hpp b/src/pydefix.hpp new file mode 100644 index 00000000..37289fe4 --- /dev/null +++ b/src/pydefix.hpp @@ -0,0 +1,21 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef PYDEFIX_HPP_ +#define PYDEFIX_HPP_ + +#include "idefix.hpp" +class Pydefix { + public: + Pydefix(); + ~Pydefix(); + + private: + static int ninstance; +}; + +#endif // PYDEFIX_HPP_ From 18811827a0a468a547dc9dc36a1fc008ab4c3360 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 2 Nov 2023 22:26:16 +0100 Subject: [PATCH 13/83] tentative automatic cast of IdefixArray @ --- src/output/output.cpp | 58 ++++++++++++++++++++++++++++++++++ src/output/output.hpp | 13 ++++++-- src/pydefix.cpp | 11 +++++++ src/pydefix.hpp | 73 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 152 insertions(+), 3 deletions(-) diff --git a/src/output/output.cpp b/src/output/output.cpp index 30a85039..966f6ed0 100644 --- a/src/output/output.cpp +++ b/src/output/output.cpp @@ -9,6 +9,12 @@ #include "dataBlock.hpp" #include "fluid.hpp" #include "slice.hpp" +#include "pydefix.hpp" + +PYBIND11_EMBEDDED_MODULE(embeded, module){ + + py::class_> animal(module, "IdefixArray"); +} Output::Output(Input &input, DataBlock &data) { idfx::pushRegion("Output::Output"); @@ -115,6 +121,22 @@ Output::Output(Input &input, DataBlock &data) { } } + // Initialise python script outputs + if(input.CheckEntry("Output","python")>0) { + #ifndef WITH_PYTHON + IDEFIX_ERROR("Usage of python outputs requires Idefix to be compiled with Python capabilities";) + #else + pythonPeriod = input.Get("Output","python",0); + if(pythonPeriod>=0.0) { // backward compatibility (negative value means no file) + pythonLast = data.t - pythonPeriod; // write something in the next CheckForWrite() + pythonScript = input.Get("Output","python",1); + pythonFunction = input.Get("Output","python",2); + pythonEnabled = true; + pythonNcall = 0; + } + #endif + } + // Register variables that are needed in restart dumps data.dump->RegisterVariable(&dumpLast, "dumpLast"); data.dump->RegisterVariable(&analysisLast, "analysisLast"); @@ -122,6 +144,9 @@ Output::Output(Input &input, DataBlock &data) { #ifdef WITH_HDF5 data.dump->RegisterVariable(&xdmfLast, "xdmfLast"); #endif + #ifdef WITH_PYTHON + data.dump->RegisterVariable(&pythonLast, "pythonLast"); + #endif idfx::popRegion(); } @@ -227,6 +252,39 @@ int Output::CheckForWrites(DataBlock &data) { slices[i]->CheckForWrite(data); } } + + #ifdef WITH_PYTHON + if(pythonEnabled) { + if(data.t >= pythonLast + pythonPeriod) { + elapsedTime -= timer.seconds(); + DataBlockHost d(data); + //try { + auto Vc = pydefix.toNumpyArray(d.Vc); + py::module_ script = py::module_::import(pythonScript.c_str()); + py::module_ embeded = py::module_::import("embeded"); + py::object myV = py::cast(d.Vc); + py::object result = script.attr(pythonFunction.c_str())(pythonNcall, myV); + /*} catch(std::exception &e) { + std::stringstream message; + message << "An exception occured while calling the Python interpreter in" + << "file \"" << pythonScript << "\" function \"" << pythonFunction << "\"" + << std::endl + << e.what() << std::endl; + IDEFIX_ERROR(message); + }*/ + elapsedTime += timer.seconds(); + pythonNcall++; + // Check if our next predicted output should already have happened + if((pythonLast+pythonPeriod <= data.t) && pythonPeriod>0.0) { + // Move forward analysisLast + while(pythonLast <= data.t - pythonPeriod) { + pythonLast += pythonPeriod; + } + } + + } + } + #endif // Do we need a restart dump? if(dumpEnabled) { bool haveClockDump = false; diff --git a/src/output/output.hpp b/src/output/output.hpp index 331ae60c..1fe5e000 100644 --- a/src/output/output.hpp +++ b/src/output/output.hpp @@ -82,11 +82,18 @@ class Output { bool haveSlices = false; std::vector> slices; + #ifdef WITH_PYTHON + Pydefix pydefix; + bool pythonEnabled = false; + real pythonPeriod; + real pythonLast; + int pythonNcall; + std::string pythonScript; + std::string pythonFunction; + #endif + Kokkos::Timer timer; double elapsedTime{0.0}; - #ifdef WITH_PYTHON - Pydefix pydefix; - #endif }; #endif // OUTPUT_OUTPUT_HPP_ diff --git a/src/pydefix.cpp b/src/pydefix.cpp index 97599a1b..966881b1 100644 --- a/src/pydefix.cpp +++ b/src/pydefix.cpp @@ -6,6 +6,7 @@ // *********************************************************************************** #include // everything needed for embedding +#include // for numpy arrays #include "idefix.hpp" #include "pydefix.hpp" @@ -31,3 +32,13 @@ Pydefix::~Pydefix() { ninstance--; idfx::cout << "Pydefix: destroyed." << std::endl; } + +py::array_t Pydefix::toNumpyArray(const IdefixHostArray3D& in) { + py::array_t array({in.extent(0),in.extent(1),in.extent(2)},in.data()); + return array; +} + +py::array_t Pydefix::toNumpyArray(const IdefixHostArray4D& in) { + py::array_t array({in.extent(0),in.extent(1),in.extent(2),in.extent(3)},in.data()); + return array; +} diff --git a/src/pydefix.hpp b/src/pydefix.hpp index 37289fe4..d6587ef2 100644 --- a/src/pydefix.hpp +++ b/src/pydefix.hpp @@ -8,14 +8,87 @@ #ifndef PYDEFIX_HPP_ #define PYDEFIX_HPP_ + +#define PYBIND11_DETAILED_ERROR_MESSAGES +#include +#include +#include #include "idefix.hpp" +#include "pydefix.hpp" + +namespace py = pybind11; + class Pydefix { public: Pydefix(); ~Pydefix(); + py::array_t toNumpyArray(const IdefixHostArray3D&); + py::array_t toNumpyArray(const IdefixHostArray4D&); private: static int ninstance; }; +// Caster for IdefixArray4D +namespace pybind11 { namespace detail { + template struct type_caster> + { + public: + + PYBIND11_TYPE_CASTER(IdefixHostArray4D, _("IdefixHostArray4D")); + + // Conversion part 1 (Python -> C++) + bool load(py::handle src, bool convert) + { + if ( !convert and !py::array_t::check_(src) ) + return false; + + auto buf = py::array_t::ensure(src); + if ( !buf ) + return false; + + auto dims = buf.ndim(); + if ( dims != 4 ) + return false; + + std::vector shape(4); + + for ( int i = 0 ; i < 4 ; ++i ) + shape[i] = buf.shape()[i]; + + + value = IdefixHostArray4D("pyArray",shape[0], shape[1], shape[2], shape[3]); + + // Still need to fill in with buf.data()+buf.size() + + return true; + } + + //Conversion part 2 (C++ -> Python) + static py::handle cast(const IdefixHostArray4D& src, py::return_value_policy policy, py::handle parent) + { + std::cout << "coucou2" << std::endl; + std::vector shape (4); + std::vector strides(4); + + for ( int i = 0 ; i < 4 ; ++i ) { + shape [i] = src.extent [i]; + strides[i] = src.strides[i]*sizeof(T); + } + + //py::array a(std::move(shape), std::move(strides), src.data() ); + py::array_t a({src.extent(0),src.extent(1),src.extent(2),src.extent(3)},src.data()); + return a.release(); + } + }; +}} // namespace pybind11::detail + + + +/* +PYBIND11_PLUGIN(example) { + py::module m("example", "Module description"); + m.def("func", &func, "Function description" ); + return m.ptr(); +}*/ #endif // PYDEFIX_HPP_ From 76bee8c4a995e0ba8df0c688f53d47c6d83e4fae Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 2 Nov 2023 22:26:48 +0100 Subject: [PATCH 14/83] missing files --- test/HD/sod/idefix_py.ini | 27 +++++++++++++++++++++++++++ test/HD/sod/output.py | 6 ++++++ 2 files changed, 33 insertions(+) create mode 100644 test/HD/sod/idefix_py.ini create mode 100644 test/HD/sod/output.py diff --git a/test/HD/sod/idefix_py.ini b/test/HD/sod/idefix_py.ini new file mode 100644 index 00000000..977f2da1 --- /dev/null +++ b/test/HD/sod/idefix_py.ini @@ -0,0 +1,27 @@ +[Grid] +X1-grid 1 0.0 500 u 1.0 +X2-grid 1 0.0 1 u 1.0 +X3-grid 1 0.0 1 u 1.0 + +[TimeIntegrator] +CFL 0.8 +tstop 0.2 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver roe +gamma 1.4 + +[Boundary] +X1-beg outflow +X1-end outflow +X2-beg outflow +X2-end outflow +X3-beg outflow +X3-end outflow + +[Output] +vtk 0.1 +dmp 0.2 +python 0.1 output myfunc diff --git a/test/HD/sod/output.py b/test/HD/sod/output.py new file mode 100644 index 00000000..d514e389 --- /dev/null +++ b/test/HD/sod/output.py @@ -0,0 +1,6 @@ +import numpy as np + +def myfunc(n, array): + print("Python n=%d"%n) + print("Dimensions="+str(array.shape)) + print(array[0,0,0,:]) From 10dc6bf9c5749f42b92217391167a5b53e69a496 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 6 Nov 2023 15:13:41 +0100 Subject: [PATCH 15/83] Fully working python output routines --- src/dataBlock/dataBlockHost.hpp | 32 ++--- src/output/output.cpp | 31 ++--- src/output/output.hpp | 2 - src/pydefix.cpp | 64 +++++++++- src/pydefix.hpp | 207 ++++++++++++++++++++++++-------- test/HD/sod/output.py | 11 +- 6 files changed, 249 insertions(+), 98 deletions(-) diff --git a/src/dataBlock/dataBlockHost.hpp b/src/dataBlock/dataBlockHost.hpp index 86c8c570..4a1e8e5e 100644 --- a/src/dataBlock/dataBlockHost.hpp +++ b/src/dataBlock/dataBlockHost.hpp @@ -25,33 +25,33 @@ class DataBlockHost { public: // Local grid information - std::array::HostMirror,3> x; ///< geometrical central points - std::array::HostMirror,3> xr; ///< cell right interface - std::array::HostMirror,3> xl; ///< cell left interface - std::array::HostMirror,3> dx; ///< cell width + std::array,3> x; ///< geometrical central points + std::array,3> xr; ///< cell right interface + std::array,3> xl; ///< cell left interface + std::array,3> dx; ///< cell width - IdefixArray3D::HostMirror dV; ///< cell volume - std::array::HostMirror,3> A; ///< cell right interface area + IdefixHostArray3D dV; ///< cell volume + std::array,3> A; ///< cell right interface area - IdefixArray4D::HostMirror Vc; ///< Main cell-centered primitive variables index + IdefixHostArray4D Vc; ///< Main cell-centered primitive variables index bool haveDust{false}; std::vector> dustVc; ///< Cell-centered primitive variables index for dust #if MHD == YES - IdefixArray4D::HostMirror Vs; ///< Main face-centered primitive variables index - IdefixArray4D::HostMirror Ve; ///< Main edge-centered primitive variables index - IdefixArray4D::HostMirror J; ///< Current (only when haveCurrent is enabled) + IdefixHostArray4D Vs; ///< Main face-centered primitive variables index + IdefixHostArray4D Ve; ///< Main edge-centered primitive variables index + IdefixHostArray4D J; ///< Current (only when haveCurrent is enabled) - IdefixArray3D::HostMirror Ex1; ///< x1 electric field - IdefixArray3D::HostMirror Ex2; ///< x2 electric field - IdefixArray3D::HostMirror Ex3; ///< x3 electric field + IdefixHostArray3D Ex1; ///< x1 electric field + IdefixHostArray3D Ex2; ///< x2 electric field + IdefixHostArray3D Ex3; ///< x3 electric field #endif - IdefixArray4D::HostMirror Uc; ///< Main cell-centered conservative variables - IdefixArray3D::HostMirror InvDt; ///< Inverse of maximum timestep in each cell + IdefixHostArray4D Uc; ///< Main cell-centered conservative variables + IdefixHostArray3D InvDt; ///< Inverse of maximum timestep in each cell - std::array::HostMirror,3> coarseningLevel; ///< Grid coarsening level + std::array,3> coarseningLevel; ///< Grid coarsening level ///< (only defined when coarsening ///< is enabled) diff --git a/src/output/output.cpp b/src/output/output.cpp index 966f6ed0..90717fb1 100644 --- a/src/output/output.cpp +++ b/src/output/output.cpp @@ -9,12 +9,13 @@ #include "dataBlock.hpp" #include "fluid.hpp" #include "slice.hpp" +#include "dataBlockHost.hpp" + +#ifdef WITH_PYTHON #include "pydefix.hpp" -PYBIND11_EMBEDDED_MODULE(embeded, module){ - py::class_> animal(module, "IdefixArray"); -} +#endif Output::Output(Input &input, DataBlock &data) { idfx::pushRegion("Output::Output"); @@ -124,15 +125,17 @@ Output::Output(Input &input, DataBlock &data) { // Initialise python script outputs if(input.CheckEntry("Output","python")>0) { #ifndef WITH_PYTHON - IDEFIX_ERROR("Usage of python outputs requires Idefix to be compiled with Python capabilities";) + IDEFIX_ERROR("Usage of python outputs requires Idefix to be compiled with Python capabilities"); #else pythonPeriod = input.Get("Output","python",0); if(pythonPeriod>=0.0) { // backward compatibility (negative value means no file) pythonLast = data.t - pythonPeriod; // write something in the next CheckForWrite() pythonScript = input.Get("Output","python",1); + if(pythonScript.substr(pythonScript.length() - 3, 3).compare(".py")==0) { + IDEFIX_ERROR("You should not include the python script .py extension in your input file"); + } pythonFunction = input.Get("Output","python",2); pythonEnabled = true; - pythonNcall = 0; } #endif } @@ -257,23 +260,8 @@ int Output::CheckForWrites(DataBlock &data) { if(pythonEnabled) { if(data.t >= pythonLast + pythonPeriod) { elapsedTime -= timer.seconds(); - DataBlockHost d(data); - //try { - auto Vc = pydefix.toNumpyArray(d.Vc); - py::module_ script = py::module_::import(pythonScript.c_str()); - py::module_ embeded = py::module_::import("embeded"); - py::object myV = py::cast(d.Vc); - py::object result = script.attr(pythonFunction.c_str())(pythonNcall, myV); - /*} catch(std::exception &e) { - std::stringstream message; - message << "An exception occured while calling the Python interpreter in" - << "file \"" << pythonScript << "\" function \"" << pythonFunction << "\"" - << std::endl - << e.what() << std::endl; - IDEFIX_ERROR(message); - }*/ + pydefix.CallScript(&data, pythonScript,pythonFunction); elapsedTime += timer.seconds(); - pythonNcall++; // Check if our next predicted output should already have happened if((pythonLast+pythonPeriod <= data.t) && pythonPeriod>0.0) { // Move forward analysisLast @@ -281,7 +269,6 @@ int Output::CheckForWrites(DataBlock &data) { pythonLast += pythonPeriod; } } - } } #endif diff --git a/src/output/output.hpp b/src/output/output.hpp index 1fe5e000..1be44372 100644 --- a/src/output/output.hpp +++ b/src/output/output.hpp @@ -87,13 +87,11 @@ class Output { bool pythonEnabled = false; real pythonPeriod; real pythonLast; - int pythonNcall; std::string pythonScript; std::string pythonFunction; #endif Kokkos::Timer timer; double elapsedTime{0.0}; - }; #endif // OUTPUT_OUTPUT_HPP_ diff --git a/src/pydefix.cpp b/src/pydefix.cpp index 966881b1..55ce5ef8 100644 --- a/src/pydefix.cpp +++ b/src/pydefix.cpp @@ -5,34 +5,87 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#include "pydefix.hpp" #include // everything needed for embedding #include // for numpy arrays +#include // For STL vectors and containers +#include +#include #include "idefix.hpp" -#include "pydefix.hpp" +#include "dataBlock.hpp" +#include "dataBlockHost.hpp" namespace py = pybind11; int Pydefix::ninstance = 0; +/************************************ + * DataBlockHost Python binding + * **********************************/ + +PYBIND11_EMBEDDED_MODULE(pydefix, m) { + py::class_(m, "DataBlockHost") + .def(py::init<>()) + .def_readwrite("x", &DataBlockHost::x) + .def_readwrite("xr", &DataBlockHost::x) + .def_readwrite("xl", &DataBlockHost::x) + .def_readwrite("dx", &DataBlockHost::x) + .def_readwrite("dV", &DataBlockHost::x) + .def_readwrite("A", &DataBlockHost::A) + .def_readwrite("Vc", &DataBlockHost::Vc) + .def_readwrite("dustVc", &DataBlockHost::dustVc) + #if MHD == YES + .def_readwrite("Vs", &DataBlockHost::Vs) + .def_readwrite("Ve", &DataBlockHost::Ve) + .def_readwrite("J", &DataBlockHost::J) + .def_readwrite("Ex1", &DataBlockHost::Ex1) + .def_readwrite("Ex2", &DataBlockHost::Ex2) + .def_readwrite("Ex3", &DataBlockHost::Ex3) + #endif + .def_readwrite("InvDt", &DataBlockHost::InvDt); +} + + Pydefix::Pydefix() { ninstance++; if(ninstance==1) { - idfx::cout << "Pydefix: start interpreter." << std::endl; + idfx::cout << "Pydefix: start Python interpreter." << std::endl; py::initialize_interpreter(); } - py::print("Hello,from python"); // use the Python API } Pydefix::~Pydefix() { if(ninstance == 1) { py::finalize_interpreter(); - idfx::cout << "Pydefix: shutdown interpreter." << std::endl; + idfx::cout << "Pydefix: shutdown Python interpreter." << std::endl; } ninstance--; - idfx::cout << "Pydefix: destroyed." << std::endl; } +void Pydefix::CallScript(DataBlock *data, std::string scriptName, std::string funcName) { + idfx::pushRegion("Pydefix::CallScript"); + DataBlockHost d(*data); + d.SyncFromDevice(); + try { + //auto Vc = pydefix.toNumpyArray(d.Vc); + py::module_ script = py::module_::import(scriptName.c_str()); + + //py::module_ embeded = py::module_::import("embeded"); + //py::object myV = py::cast(Vc); + py::object result = script.attr(funcName.c_str())(nCalls, d); + } catch(std::exception &e) { + std::stringstream message; + message << "An exception occured while calling the Python interpreter" << std::endl + << "in file \"" << scriptName << ".py\" function \"" << funcName << "\":" + << std::endl + << e.what() << std::endl; + IDEFIX_ERROR(message); + } + nCalls++; + idfx::popRegion(); +} +/* py::array_t Pydefix::toNumpyArray(const IdefixHostArray3D& in) { py::array_t array({in.extent(0),in.extent(1),in.extent(2)},in.data()); return array; @@ -42,3 +95,4 @@ py::array_t Pydefix::toNumpyArray(const IdefixHostArray4D& in) { py::array_t array({in.extent(0),in.extent(1),in.extent(2),in.extent(3)},in.data()); return array; } +*/ diff --git a/src/pydefix.hpp b/src/pydefix.hpp index d6587ef2..b76ea3bb 100644 --- a/src/pydefix.hpp +++ b/src/pydefix.hpp @@ -9,86 +9,195 @@ #define PYDEFIX_HPP_ -#define PYBIND11_DETAILED_ERROR_MESSAGES +//#define PYBIND11_DETAILED_ERROR_MESSAGES #include #include #include +#include +#include #include "idefix.hpp" -#include "pydefix.hpp" namespace py = pybind11; +class DataBlock; + class Pydefix { public: Pydefix(); ~Pydefix(); - py::array_t toNumpyArray(const IdefixHostArray3D&); - py::array_t toNumpyArray(const IdefixHostArray4D&); + void CallScript(DataBlock *, std::string, std::string); private: static int ninstance; + int nCalls{0}; }; -// Caster for IdefixArray4D namespace pybind11 { namespace detail { - template struct type_caster> - { - public: +// Caster for IdefixArray4D +template struct type_caster> { + public: + PYBIND11_TYPE_CASTER(IdefixHostArray4D, _("IdefixHostArray4D")); + + // Conversion part 1 (Python -> C++) + bool load(py::handle src, bool convert) { + if ( !convert && !py::array_t::check_(src) ) + return false; - PYBIND11_TYPE_CASTER(IdefixHostArray4D, _("IdefixHostArray4D")); + auto buf = py::array_t::ensure(src); + if ( !buf ) + return false; - // Conversion part 1 (Python -> C++) - bool load(py::handle src, bool convert) - { - if ( !convert and !py::array_t::check_(src) ) - return false; + auto dims = buf.ndim(); + if ( dims != 4 ) + return false; - auto buf = py::array_t::ensure(src); - if ( !buf ) - return false; + std::vector shape(4); - auto dims = buf.ndim(); - if ( dims != 4 ) - return false; + for ( int i = 0 ; i < 4 ; ++i ) + shape[i] = buf.shape()[i]; - std::vector shape(4); - for ( int i = 0 ; i < 4 ; ++i ) - shape[i] = buf.shape()[i]; + value = IdefixHostArray4D("pyArray",shape[0], shape[1], shape[2], shape[3]); + // Still need to fill in with buf.data()+buf.size() + IDEFIX_ERROR("Python->Idefix Not implemented"); + + return true; + } + + //Conversion part 2 (C++ -> Python) + static py::handle cast(const IdefixHostArray4D& src, + py::return_value_policy policy, + py::handle parent) { + py::array_t a({src.extent(0), + src.extent(1), + src.extent(2), + src.extent(3)}, + src.data()); + return a.release(); + } +}; +// Caster for IdefixArray3D +template struct type_caster> { + public: + PYBIND11_TYPE_CASTER(IdefixHostArray3D, _("IdefixHostArray3D")); - value = IdefixHostArray4D("pyArray",shape[0], shape[1], shape[2], shape[3]); + // Conversion part 1 (Python -> C++) + bool load(py::handle src, bool convert) { + if ( !convert && !py::array_t::check_(src) ) + return false; - // Still need to fill in with buf.data()+buf.size() + auto buf = py::array_t::ensure(src); + if ( !buf ) + return false; - return true; - } + auto dims = buf.ndim(); + if ( dims != 3 ) + return false; + + std::vector shape(3); + + for ( int i = 0 ; i < 3 ; ++i ) + shape[i] = buf.shape()[i]; + + + value = IdefixHostArray3D("pyArray",shape[0], shape[1], shape[2]); + + // Still need to fill in with buf.data()+buf.size() + IDEFIX_ERROR("Python->Idefix Not implemented"); + return true; + } + + //Conversion part 2 (C++ -> Python) + static py::handle cast(const IdefixHostArray3D& src, + py::return_value_policy policy, + py::handle parent) { + py::array_t a({src.extent(0), + src.extent(1), + src.extent(2)}, + src.data()); + return a.release(); + } +}; +// Caster for IdefixArray2D +template struct type_caster> { + public: + PYBIND11_TYPE_CASTER(IdefixHostArray2D, _("IdefixHostArray2D")); - //Conversion part 2 (C++ -> Python) - static py::handle cast(const IdefixHostArray4D& src, py::return_value_policy policy, py::handle parent) - { - std::cout << "coucou2" << std::endl; - std::vector shape (4); - std::vector strides(4); + // Conversion part 1 (Python -> C++) + bool load(py::handle src, bool convert) { + if ( !convert && !py::array_t::check_(src) ) + return false; - for ( int i = 0 ; i < 4 ; ++i ) { - shape [i] = src.extent [i]; - strides[i] = src.strides[i]*sizeof(T); - } + auto buf = py::array_t::ensure(src); + if ( !buf ) + return false; - //py::array a(std::move(shape), std::move(strides), src.data() ); - py::array_t a({src.extent(0),src.extent(1),src.extent(2),src.extent(3)},src.data()); - return a.release(); - } - }; -}} // namespace pybind11::detail + auto dims = buf.ndim(); + if ( dims != 2 ) + return false; + std::vector shape(2); + for ( int i = 0 ; i < 2 ; ++i ) + shape[i] = buf.shape()[i]; + + + value = IdefixHostArray2D("pyArray",shape[0], shape[1]); + + // Still need to fill in with buf.data()+buf.size() + IDEFIX_ERROR("Python->Idefix Not implemented"); + return true; + } + + //Conversion part 2 (C++ -> Python) + static py::handle cast(const IdefixHostArray2D& src, + py::return_value_policy policy, + py::handle parent) { + py::array_t a({src.extent(0),src.extent(1)},src.data()); + return a.release(); + } +}; +// Caster for IdefixArray1D +template struct type_caster> { + public: + PYBIND11_TYPE_CASTER(IdefixHostArray1D, _("IdefixHostArray1D")); + + // Conversion part 1 (Python -> C++) + bool load(py::handle src, bool convert) { + if ( !convert && !py::array_t::check_(src) ) + return false; + + auto buf = py::array_t::ensure(src); + if ( !buf ) + return false; + + auto dims = buf.ndim(); + if ( dims != 1 ) + return false; + + std::vector shape(1); + + for ( int i = 0 ; i < 1 ; ++i ) + shape[i] = buf.shape()[i]; + + + value = IdefixHostArray1D("pyArray",shape[0]); + + // Still need to fill in with buf.data()+buf.size() + IDEFIX_ERROR("Python->Idefix Not implemented"); + return true; + } + + //Conversion part 2 (C++ -> Python) + static py::handle cast(const IdefixHostArray1D& src, + py::return_value_policy policy, + py::handle parent) { + py::array_t a(src.extent(0),src.data()); + return a.release(); + } +}; +} // namespace detail +} // namespace pybind11 -/* -PYBIND11_PLUGIN(example) { - py::module m("example", "Module description"); - m.def("func", &func, "Function description" ); - return m.ptr(); -}*/ #endif // PYDEFIX_HPP_ diff --git a/test/HD/sod/output.py b/test/HD/sod/output.py index d514e389..79f05327 100644 --- a/test/HD/sod/output.py +++ b/test/HD/sod/output.py @@ -1,6 +1,9 @@ -import numpy as np -def myfunc(n, array): +def myfunc(n, data): print("Python n=%d"%n) - print("Dimensions="+str(array.shape)) - print(array[0,0,0,:]) + print(dir(data)) + print("Dimensions="+str(data.Vc.shape)) + print("x1=") + print(data.x[0][:]) + print("data=") + print(data.Vc[0,0,0,:]) From 227e3751c30b308dc4f6f0d18c37f090d99c5a50 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 3 Sep 2024 11:42:54 +0200 Subject: [PATCH 16/83] working now with initial condition (but copies won't work!) --- src/main.cpp | 41 +++++++++++-- src/output/output.cpp | 15 ++--- src/output/output.hpp | 2 - src/pydefix.cpp | 106 +++++++++++++++++++++++++++------ src/pydefix.hpp | 20 +++++-- test/HD/sod/idefix_py.ini | 11 +++- test/HD/sod/output.py | 9 --- test/HD/sod/pydefix_example.py | 14 +++++ 8 files changed, 166 insertions(+), 52 deletions(-) delete mode 100644 test/HD/sod/output.py create mode 100644 test/HD/sod/pydefix_example.py diff --git a/src/main.cpp b/src/main.cpp index 5a7f6f73..6294dcff 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -90,8 +90,12 @@ int main( int argc, char* argv[] ) { // instantiate required objects. DataBlock data(grid, input); TimeIntegrator Tint(input,data); + #ifdef WITH_PYTHON + Pydefix pydefix(input); + #endif Output output(input, data); Setup mysetup(input, grid, data, output); + idfx::cout << "Main: initialisation finished." << std::endl; char host[1024]; @@ -110,6 +114,9 @@ int main( int argc, char* argv[] ) { grid.ShowConfig(); data.ShowConfig(); Tint.ShowConfig(); + #ifdef WITH_PYTHON + pydefix.ShowConfig(); + #endif /////////////////////////////// // Initial conditions (or restart) @@ -117,10 +124,22 @@ int main( int argc, char* argv[] ) { // Are we restarting? if(input.restartRequested) { if(input.forceInitRequested) { - idfx::pushRegion("Setup::Initflow"); - mysetup.InitFlow(data); - data.DeriveVectorPotential(); - idfx::popRegion(); + #ifdef WITH_PYTHON + if(pydefix.haveInitflow) { + idfx::pushRegion("Pydefix::Initflow"); + pydefix.InitFlow(data); + } else { + idfx::pushRegion("Setup::Initflow"); + mysetup.InitFlow(data); + } + data.DeriveVectorPotential(); + idfx::popRegion(); + #else + idfx::pushRegion("Setup::Initflow"); + mysetup.InitFlow(data); + data.DeriveVectorPotential(); + idfx::popRegion(); + #endif } idfx::cout << "Main: Restarting from dump file." << std::endl; bool restartSuccess = output.RestartFromDump(data,input.restartFileNumber); @@ -133,8 +152,18 @@ int main( int argc, char* argv[] ) { } if(!input.restartRequested) { idfx::cout << "Main: Creating initial conditions." << std::endl; - idfx::pushRegion("Setup::Initflow"); - mysetup.InitFlow(data); + #ifdef WITH_PYTHON + if(pydefix.haveInitflow) { + idfx::pushRegion("Pydefix::Initflow"); + pydefix.InitFlow(data); + } else { + idfx::pushRegion("Setup::Initflow"); + mysetup.InitFlow(data); + } + #else + idfx::pushRegion("Setup::Initflow"); + mysetup.InitFlow(data); + #endif idfx::popRegion(); data.DeriveVectorPotential(); // This does something only when evolveVectorPotential is on data.SetBoundaries(); diff --git a/src/output/output.cpp b/src/output/output.cpp index 90717fb1..75d75124 100644 --- a/src/output/output.cpp +++ b/src/output/output.cpp @@ -13,11 +13,13 @@ #ifdef WITH_PYTHON #include "pydefix.hpp" - - #endif -Output::Output(Input &input, DataBlock &data) { +Output::Output(Input &input, DataBlock &data): +#ifdef WITH_PYTHON +pydefix(input) +#endif +{ idfx::pushRegion("Output::Output"); // initialise the output objects for each format @@ -130,11 +132,6 @@ Output::Output(Input &input, DataBlock &data) { pythonPeriod = input.Get("Output","python",0); if(pythonPeriod>=0.0) { // backward compatibility (negative value means no file) pythonLast = data.t - pythonPeriod; // write something in the next CheckForWrite() - pythonScript = input.Get("Output","python",1); - if(pythonScript.substr(pythonScript.length() - 3, 3).compare(".py")==0) { - IDEFIX_ERROR("You should not include the python script .py extension in your input file"); - } - pythonFunction = input.Get("Output","python",2); pythonEnabled = true; } #endif @@ -260,7 +257,7 @@ int Output::CheckForWrites(DataBlock &data) { if(pythonEnabled) { if(data.t >= pythonLast + pythonPeriod) { elapsedTime -= timer.seconds(); - pydefix.CallScript(&data, pythonScript,pythonFunction); + pydefix.Output(data); elapsedTime += timer.seconds(); // Check if our next predicted output should already have happened if((pythonLast+pythonPeriod <= data.t) && pythonPeriod>0.0) { diff --git a/src/output/output.hpp b/src/output/output.hpp index 1be44372..981716b7 100644 --- a/src/output/output.hpp +++ b/src/output/output.hpp @@ -87,8 +87,6 @@ class Output { bool pythonEnabled = false; real pythonPeriod; real pythonLast; - std::string pythonScript; - std::string pythonFunction; #endif Kokkos::Timer timer; diff --git a/src/pydefix.cpp b/src/pydefix.cpp index 55ce5ef8..d2a69928 100644 --- a/src/pydefix.cpp +++ b/src/pydefix.cpp @@ -5,6 +5,8 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#define PYBIND11_DETAILED_ERROR_MESSAGES + #include "pydefix.hpp" #include // everything needed for embedding #include // for numpy arrays @@ -15,6 +17,7 @@ #include "dataBlock.hpp" #include "dataBlockHost.hpp" + namespace py = pybind11; int Pydefix::ninstance = 0; @@ -27,10 +30,10 @@ PYBIND11_EMBEDDED_MODULE(pydefix, m) { py::class_(m, "DataBlockHost") .def(py::init<>()) .def_readwrite("x", &DataBlockHost::x) - .def_readwrite("xr", &DataBlockHost::x) - .def_readwrite("xl", &DataBlockHost::x) - .def_readwrite("dx", &DataBlockHost::x) - .def_readwrite("dV", &DataBlockHost::x) + .def_readwrite("xr", &DataBlockHost::xr) + .def_readwrite("xl", &DataBlockHost::xl) + .def_readwrite("dx", &DataBlockHost::dx) + .def_readwrite("dV", &DataBlockHost::dV) .def_readwrite("A", &DataBlockHost::A) .def_readwrite("Vc", &DataBlockHost::Vc) .def_readwrite("dustVc", &DataBlockHost::dustVc) @@ -46,34 +49,102 @@ PYBIND11_EMBEDDED_MODULE(pydefix, m) { } -Pydefix::Pydefix() { - ninstance++; - if(ninstance==1) { - idfx::cout << "Pydefix: start Python interpreter." << std::endl; +Pydefix::Pydefix(Input &input) { + // Check that the input has a [python] block + if(input.CheckBlock("Python")) { + this->isActive = true; + ninstance++; + // Check whether we need to start an interpreter + if(ninstance==1) { + idfx::cout << "Pydefix: start Python interpreter." << std::endl; + + py::initialize_interpreter(); + } + this->scriptFilename = input.Get("Python","script",0); + if(scriptFilename.substr(scriptFilename.length() - 3, 3).compare(".py")==0) { + IDEFIX_ERROR("You should not include the python script .py extension in your input file"); + } + if(input.CheckEntry("Python","output_function")>0) { + this->haveOutput = true; + this->outputFunctionName = input.Get("Python","output_function",0); + } + if(input.CheckEntry("Python","initflow_function")>0) { + this->haveInitflow = true; + this->initflowFunctionName = input.Get("Python","initflow_function",0); + } + } + this->ShowConfig(); +} + +void Pydefix::Output(DataBlock &data) { + idfx::pushRegion("Pydefix::Output"); + if(!this->isActive) { + IDEFIX_ERROR("Python Outputs requires the [python] block to be defined in your input file."); + } + if(!this->haveOutput) { + IDEFIX_ERROR("No python output function has been defined " + "in your input file [python]:output_function"); + } + DataBlockHost dataHost(data); + dataHost.SyncFromDevice(); + this->CallScript(&dataHost,this->scriptFilename,this->outputFunctionName); + idfx::popRegion(); +} + +void Pydefix::InitFlow(DataBlock &data) { + idfx::pushRegion("Pydefix::InitFlow"); + if(!this->isActive) { + IDEFIX_ERROR("Python Initflow requires the [python] block to be defined in your input file."); + } + if(!this->haveOutput) { + IDEFIX_ERROR("No python initflow function has been defined " + "in your input file [python]:initflow_function"); + } + DataBlockHost dataHost(data); + dataHost.SyncFromDevice(); + this->CallScript(&dataHost,this->scriptFilename,this->initflowFunctionName); + dataHost.SyncToDevice(); + idfx::popRegion(); +} - py::initialize_interpreter(); +void Pydefix::ShowConfig() { + if(isActive == false) { + idfx::cout << "Pydefix: DISABLED." << std::endl; + } else { + idfx::cout << "Pydefix: ENABLED." << std::endl; + if(haveOutput) { + idfx::cout << "Pydefix: output function ENABLED." << std::endl; + } else { + idfx::cout << "Pydefix: output function DISABLED." << std::endl; + } + if(haveInitflow) { + idfx::cout << "Pydefix: initflow function ENABLED." << std::endl; + } else { + idfx::cout << "Pydefix: initflow function DISABLED." << std::endl; + } } } Pydefix::~Pydefix() { - if(ninstance == 1) { - py::finalize_interpreter(); - idfx::cout << "Pydefix: shutdown Python interpreter." << std::endl; + if(isActive) { + if(ninstance == 1) { + py::finalize_interpreter(); + idfx::cout << "Pydefix: shutdown Python interpreter." << std::endl; + } + ninstance--; + isActive = false; } - ninstance--; } -void Pydefix::CallScript(DataBlock *data, std::string scriptName, std::string funcName) { +void Pydefix::CallScript(DataBlockHost *data, std::string scriptName, std::string funcName) { idfx::pushRegion("Pydefix::CallScript"); - DataBlockHost d(*data); - d.SyncFromDevice(); try { //auto Vc = pydefix.toNumpyArray(d.Vc); py::module_ script = py::module_::import(scriptName.c_str()); //py::module_ embeded = py::module_::import("embeded"); //py::object myV = py::cast(Vc); - py::object result = script.attr(funcName.c_str())(nCalls, d); + py::object result = script.attr(funcName.c_str())(data); } catch(std::exception &e) { std::stringstream message; message << "An exception occured while calling the Python interpreter" << std::endl @@ -82,7 +153,6 @@ void Pydefix::CallScript(DataBlock *data, std::string scriptName, std::string fu << e.what() << std::endl; IDEFIX_ERROR(message); } - nCalls++; idfx::popRegion(); } /* diff --git a/src/pydefix.hpp b/src/pydefix.hpp index b76ea3bb..bd985c67 100644 --- a/src/pydefix.hpp +++ b/src/pydefix.hpp @@ -9,27 +9,36 @@ #define PYDEFIX_HPP_ -//#define PYBIND11_DETAILED_ERROR_MESSAGES +#define PYBIND11_DETAILED_ERROR_MESSAGES #include #include #include #include #include #include "idefix.hpp" +#include "input.hpp" namespace py = pybind11; class DataBlock; +class DataBlockHost; class Pydefix { public: - Pydefix(); + explicit Pydefix(Input&); ~Pydefix(); - void CallScript(DataBlock *, std::string, std::string); - + void Output(DataBlock &); + void InitFlow(DataBlock &); + void ShowConfig(); + bool isActive{false}; + bool haveOutput{false}; + bool haveInitflow{false}; private: + void CallScript(DataBlockHost *, std::string, std::string); static int ninstance; - int nCalls{0}; + std::string scriptFilename; + std::string outputFunctionName; + std::string initflowFunctionName; }; namespace pybind11 { namespace detail { @@ -74,6 +83,7 @@ template struct type_caster> { src.extent(2), src.extent(3)}, src.data()); + idfx::cout << "Coucou @ cast" << std::endl; return a.release(); } }; diff --git a/test/HD/sod/idefix_py.ini b/test/HD/sod/idefix_py.ini index 977f2da1..378b5a85 100644 --- a/test/HD/sod/idefix_py.ini +++ b/test/HD/sod/idefix_py.ini @@ -21,7 +21,12 @@ X2-end outflow X3-beg outflow X3-end outflow +[Python] +script pydefix_example +output_function output +initflow_function initflow + [Output] -vtk 0.1 -dmp 0.2 -python 0.1 output myfunc +vtk 0.1 +dmp 0.2 +python 0.1 output myfunc diff --git a/test/HD/sod/output.py b/test/HD/sod/output.py deleted file mode 100644 index 79f05327..00000000 --- a/test/HD/sod/output.py +++ /dev/null @@ -1,9 +0,0 @@ - -def myfunc(n, data): - print("Python n=%d"%n) - print(dir(data)) - print("Dimensions="+str(data.Vc.shape)) - print("x1=") - print(data.x[0][:]) - print("data=") - print(data.Vc[0,0,0,:]) diff --git a/test/HD/sod/pydefix_example.py b/test/HD/sod/pydefix_example.py new file mode 100644 index 00000000..8230bc23 --- /dev/null +++ b/test/HD/sod/pydefix_example.py @@ -0,0 +1,14 @@ + +def output(data): + print("data=") + print(data.Vc[0,0,0,:]) + data.Vc[0,0,0,10] = 2.0 + +def initflow(data): + # Initialize the flow + print(data.Vc.shape) + data.Vc[0,0,0,:] = 1+0*data.x[0][:] + data.Vc[0,0,0,1] = 1.0 + data.Vc[1,0,0,:] = data.x[0][:] + print(data.Vc[0,0,0,:]) + print(data.Vc[1,0,0,:]) From c2d73e63824d8a486d6fce8cd9f145e153353b66 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Fri, 6 Sep 2024 13:37:15 +0200 Subject: [PATCH 17/83] working version with possibility to initialize the flow from python --- src/pydefix.cpp | 21 ++++++++++++++++++--- src/pydefix.hpp | 16 ++++++++++------ test/HD/sod/idefix_py.ini | 2 +- test/HD/sod/pydefix_example.py | 23 ++++++++++++++--------- 4 files changed, 43 insertions(+), 19 deletions(-) diff --git a/src/pydefix.cpp b/src/pydefix.cpp index d2a69928..2da45161 100644 --- a/src/pydefix.cpp +++ b/src/pydefix.cpp @@ -46,6 +46,20 @@ PYBIND11_EMBEDDED_MODULE(pydefix, m) { .def_readwrite("Ex3", &DataBlockHost::Ex3) #endif .def_readwrite("InvDt", &DataBlockHost::InvDt); + + m.attr("RHO") = RHO; + m.attr("VX1") = VX1; + m.attr("VX2") = VX2; + m.attr("VX3") = VX3; + m.attr("PRS") = PRS; + #if MHD == YES + m.attr("BX1") = BX1; + m.attr("BX2") = BX2; + m.attr("BX3") = BX3; + #endif + m.attr("IDIR") = IDIR; + m.attr("JDIR") = JDIR; + m.attr("KDIR") = KDIR; } @@ -87,7 +101,7 @@ void Pydefix::Output(DataBlock &data) { } DataBlockHost dataHost(data); dataHost.SyncFromDevice(); - this->CallScript(&dataHost,this->scriptFilename,this->outputFunctionName); + this->CallScript(dataHost,this->scriptFilename,this->outputFunctionName); idfx::popRegion(); } @@ -102,7 +116,7 @@ void Pydefix::InitFlow(DataBlock &data) { } DataBlockHost dataHost(data); dataHost.SyncFromDevice(); - this->CallScript(&dataHost,this->scriptFilename,this->initflowFunctionName); + this->CallScript(dataHost,this->scriptFilename,this->initflowFunctionName); dataHost.SyncToDevice(); idfx::popRegion(); } @@ -136,7 +150,7 @@ Pydefix::~Pydefix() { } } -void Pydefix::CallScript(DataBlockHost *data, std::string scriptName, std::string funcName) { +void Pydefix::CallScript(DataBlockHost &data, std::string scriptName, std::string funcName) { idfx::pushRegion("Pydefix::CallScript"); try { //auto Vc = pydefix.toNumpyArray(d.Vc); @@ -144,6 +158,7 @@ void Pydefix::CallScript(DataBlockHost *data, std::string scriptName, std::strin //py::module_ embeded = py::module_::import("embeded"); //py::object myV = py::cast(Vc); + //py::object result = script.attr(funcName.c_str())(data); py::object result = script.attr(funcName.c_str())(data); } catch(std::exception &e) { std::stringstream message; diff --git a/src/pydefix.hpp b/src/pydefix.hpp index bd985c67..e8ac8879 100644 --- a/src/pydefix.hpp +++ b/src/pydefix.hpp @@ -34,7 +34,7 @@ class Pydefix { bool haveOutput{false}; bool haveInitflow{false}; private: - void CallScript(DataBlockHost *, std::string, std::string); + void CallScript(DataBlockHost &, std::string, std::string); static int ninstance; std::string scriptFilename; std::string outputFunctionName; @@ -78,12 +78,13 @@ template struct type_caster> { static py::handle cast(const IdefixHostArray4D& src, py::return_value_policy policy, py::handle parent) { + py::none dummyDataOwner; py::array_t a({src.extent(0), src.extent(1), src.extent(2), src.extent(3)}, - src.data()); - idfx::cout << "Coucou @ cast" << std::endl; + src.data(), dummyDataOwner); + return a.release(); } }; @@ -122,10 +123,11 @@ template struct type_caster> { static py::handle cast(const IdefixHostArray3D& src, py::return_value_policy policy, py::handle parent) { + py::none dummyDataOwner; py::array_t a({src.extent(0), src.extent(1), src.extent(2)}, - src.data()); + src.data(),dummyDataOwner); return a.release(); } }; @@ -164,7 +166,8 @@ template struct type_caster> { static py::handle cast(const IdefixHostArray2D& src, py::return_value_policy policy, py::handle parent) { - py::array_t a({src.extent(0),src.extent(1)},src.data()); + py::none dummyOwner; + py::array_t a({src.extent(0),src.extent(1)},src.data(),dummyOwner); return a.release(); } }; @@ -203,7 +206,8 @@ template struct type_caster> { static py::handle cast(const IdefixHostArray1D& src, py::return_value_policy policy, py::handle parent) { - py::array_t a(src.extent(0),src.data()); + py::none dummyDataOwner; + py::array_t a(src.extent(0),src.data(),dummyDataOwner); return a.release(); } }; diff --git a/test/HD/sod/idefix_py.ini b/test/HD/sod/idefix_py.ini index 378b5a85..59c58f64 100644 --- a/test/HD/sod/idefix_py.ini +++ b/test/HD/sod/idefix_py.ini @@ -29,4 +29,4 @@ initflow_function initflow [Output] vtk 0.1 dmp 0.2 -python 0.1 output myfunc +python 0.01 diff --git a/test/HD/sod/pydefix_example.py b/test/HD/sod/pydefix_example.py index 8230bc23..58d73642 100644 --- a/test/HD/sod/pydefix_example.py +++ b/test/HD/sod/pydefix_example.py @@ -1,14 +1,19 @@ +import pydefix as pdfx +import numpy as np +import matplotlib.pyplot as plt def output(data): - print("data=") - print(data.Vc[0,0,0,:]) - data.Vc[0,0,0,10] = 2.0 + + plt.figure() + plt.plot(data.x[pdfx.IDIR],data.Vc[pdfx.VX1,0,0,:],label='VX1') + plt.plot(data.x[pdfx.IDIR],data.Vc[pdfx.RHO,0,0,:],label='RHO') + plt.legend() + plt.show() + #data.Vc[0,0,0,10] = 2.0 def initflow(data): + # Initialize the flow - print(data.Vc.shape) - data.Vc[0,0,0,:] = 1+0*data.x[0][:] - data.Vc[0,0,0,1] = 1.0 - data.Vc[1,0,0,:] = data.x[0][:] - print(data.Vc[0,0,0,:]) - print(data.Vc[1,0,0,:]) + data.Vc[pdfx.RHO,0,0,:] = 1.0 + data.Vc[pdfx.PRS,0,0,:] = 2.0 + data.Vc[pdfx.VX1,0,0,:] = np.sin(2.0*np.pi*data.x[0][:]) From 7dc75538c87a5935e96024e6c9b2d197e2c9c9d2 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 18 Sep 2024 15:24:38 +0200 Subject: [PATCH 18/83] add BXs to python bindings --- src/pydefix.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/pydefix.cpp b/src/pydefix.cpp index 2da45161..e62f371e 100644 --- a/src/pydefix.cpp +++ b/src/pydefix.cpp @@ -56,6 +56,10 @@ PYBIND11_EMBEDDED_MODULE(pydefix, m) { m.attr("BX1") = BX1; m.attr("BX2") = BX2; m.attr("BX3") = BX3; + D_EXPAND( + m.attr("BX1s") = BX1; , + m.attr("BX2s") = BX2; , + m.attr("BX3s") = BX3; ) #endif m.attr("IDIR") = IDIR; m.attr("JDIR") = JDIR; From 8aa7e93f976c8ab1d84ab66cf6deba921643fe6f Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Sun, 22 Sep 2024 17:13:00 +0200 Subject: [PATCH 19/83] fix BXs indices add time in datablockHost --- src/dataBlock/dataBlockHost.cpp | 5 +++++ src/dataBlock/dataBlockHost.hpp | 3 +++ src/pydefix.cpp | 11 ++++++----- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/dataBlock/dataBlockHost.cpp b/src/dataBlock/dataBlockHost.cpp index ca26e5cb..f5e8c269 100644 --- a/src/dataBlock/dataBlockHost.cpp +++ b/src/dataBlock/dataBlockHost.cpp @@ -101,6 +101,8 @@ DataBlockHost::DataBlockHost(DataBlock& datain) { void DataBlockHost::SyncToDevice() { idfx::pushRegion("DataBlockHost::SyncToDevice()"); + data->t = this->t; + data->dt = this->dt; Kokkos::deep_copy(data->hydro->Vc,Vc); Kokkos::deep_copy(data->hydro->InvDt,InvDt); @@ -137,6 +139,9 @@ void DataBlockHost::SyncToDevice() { void DataBlockHost::SyncFromDevice() { idfx::pushRegion("DataBlockHost::SyncFromDevice()"); + this->t = data->t; + this->dt = data->dt; + Kokkos::deep_copy(Vc,data->hydro->Vc); Kokkos::deep_copy(InvDt,data->hydro->InvDt); diff --git a/src/dataBlock/dataBlockHost.hpp b/src/dataBlock/dataBlockHost.hpp index 4a1e8e5e..2a732a32 100644 --- a/src/dataBlock/dataBlockHost.hpp +++ b/src/dataBlock/dataBlockHost.hpp @@ -75,6 +75,9 @@ class DataBlockHost { std::array gbeg; ///< Begining of local block in the grid (internal) std::array gend; ///< End of local block in the grid (internal) + real dt; ///< Current timestep + real t; ///< Current time + explicit DataBlockHost(DataBlock &); ///< Constructor from a device datablock ///< (NB: does not sync any data) diff --git a/src/pydefix.cpp b/src/pydefix.cpp index e62f371e..5662dce5 100644 --- a/src/pydefix.cpp +++ b/src/pydefix.cpp @@ -45,7 +45,9 @@ PYBIND11_EMBEDDED_MODULE(pydefix, m) { .def_readwrite("Ex2", &DataBlockHost::Ex2) .def_readwrite("Ex3", &DataBlockHost::Ex3) #endif - .def_readwrite("InvDt", &DataBlockHost::InvDt); + .def_readwrite("InvDt", &DataBlockHost::InvDt) + .def_readwrite("t",&DataBlockHost::t) + .def_readwrite("dt",&DataBlockHost::dt); m.attr("RHO") = RHO; m.attr("VX1") = VX1; @@ -57,9 +59,9 @@ PYBIND11_EMBEDDED_MODULE(pydefix, m) { m.attr("BX2") = BX2; m.attr("BX3") = BX3; D_EXPAND( - m.attr("BX1s") = BX1; , - m.attr("BX2s") = BX2; , - m.attr("BX3s") = BX3; ) + m.attr("BX1s") = BX1s; , + m.attr("BX2s") = BX2s; , + m.attr("BX3s") = BX3s; ) #endif m.attr("IDIR") = IDIR; m.attr("JDIR") = JDIR; @@ -91,7 +93,6 @@ Pydefix::Pydefix(Input &input) { this->initflowFunctionName = input.Get("Python","initflow_function",0); } } - this->ShowConfig(); } void Pydefix::Output(DataBlock &data) { From 134ca6b2c6b688097b814ffdac9c8f4e81c6dc22 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Sun, 22 Sep 2024 17:13:39 +0200 Subject: [PATCH 20/83] change init list to work without python --- src/output/output.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/output/output.cpp b/src/output/output.cpp index 75d75124..a6596490 100644 --- a/src/output/output.cpp +++ b/src/output/output.cpp @@ -15,9 +15,9 @@ #include "pydefix.hpp" #endif -Output::Output(Input &input, DataBlock &data): +Output::Output(Input &input, DataBlock &data) #ifdef WITH_PYTHON -pydefix(input) +:pydefix(input) #endif { idfx::pushRegion("Output::Output"); From 8f6cf5d09333e3a4f32c59cdc37a589a5ccd2c4f Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 3 Oct 2024 10:30:09 +0200 Subject: [PATCH 21/83] generalised call script with variadic template parameters add the output number n to the list of parameters passed to the output python routine --- src/output/output.cpp | 5 ++++- src/output/output.hpp | 1 + src/pydefix.cpp | 43 +++++++++++++++++++++---------------------- src/pydefix.hpp | 6 ++++-- 4 files changed, 30 insertions(+), 25 deletions(-) diff --git a/src/output/output.cpp b/src/output/output.cpp index a6596490..a482f66a 100644 --- a/src/output/output.cpp +++ b/src/output/output.cpp @@ -133,6 +133,7 @@ Output::Output(Input &input, DataBlock &data) if(pythonPeriod>=0.0) { // backward compatibility (negative value means no file) pythonLast = data.t - pythonPeriod; // write something in the next CheckForWrite() pythonEnabled = true; + pythonNumber = 0; } #endif } @@ -146,6 +147,7 @@ Output::Output(Input &input, DataBlock &data) #endif #ifdef WITH_PYTHON data.dump->RegisterVariable(&pythonLast, "pythonLast"); + data.dump->RegisterVariable(&pythonNumber,"pythonNumber"); #endif idfx::popRegion(); @@ -257,7 +259,8 @@ int Output::CheckForWrites(DataBlock &data) { if(pythonEnabled) { if(data.t >= pythonLast + pythonPeriod) { elapsedTime -= timer.seconds(); - pydefix.Output(data); + pydefix.Output(data,pythonNumber); + pythonNumber++; elapsedTime += timer.seconds(); // Check if our next predicted output should already have happened if((pythonLast+pythonPeriod <= data.t) && pythonPeriod>0.0) { diff --git a/src/output/output.hpp b/src/output/output.hpp index 981716b7..039ab157 100644 --- a/src/output/output.hpp +++ b/src/output/output.hpp @@ -87,6 +87,7 @@ class Output { bool pythonEnabled = false; real pythonPeriod; real pythonLast; + real pythonNumber; #endif Kokkos::Timer timer; diff --git a/src/pydefix.cpp b/src/pydefix.cpp index 5662dce5..6756d0a2 100644 --- a/src/pydefix.cpp +++ b/src/pydefix.cpp @@ -69,6 +69,24 @@ PYBIND11_EMBEDDED_MODULE(pydefix, m) { } +template +void Pydefix::CallScript(std::string scriptName, std::string funcName, Ts... args) { + idfx::pushRegion("Pydefix::CallScript"); + try { + py::module_ script = py::module_::import(scriptName.c_str()); + py::object result = script.attr(funcName.c_str())(&args...); + } catch(std::exception &e) { + std::stringstream message; + message << "An exception occured while calling the Python interpreter" << std::endl + << "in file \"" << scriptName << ".py\" function \"" << funcName << "\":" + << std::endl + << e.what() << std::endl; + IDEFIX_ERROR(message); + } + idfx::popRegion(); +} + + Pydefix::Pydefix(Input &input) { // Check that the input has a [python] block if(input.CheckBlock("Python")) { @@ -95,7 +113,7 @@ Pydefix::Pydefix(Input &input) { } } -void Pydefix::Output(DataBlock &data) { +void Pydefix::Output(DataBlock &data, int n) { idfx::pushRegion("Pydefix::Output"); if(!this->isActive) { IDEFIX_ERROR("Python Outputs requires the [python] block to be defined in your input file."); @@ -106,7 +124,7 @@ void Pydefix::Output(DataBlock &data) { } DataBlockHost dataHost(data); dataHost.SyncFromDevice(); - this->CallScript(dataHost,this->scriptFilename,this->outputFunctionName); + this->CallScript(this->scriptFilename,this->outputFunctionName,dataHost, n); idfx::popRegion(); } @@ -121,7 +139,7 @@ void Pydefix::InitFlow(DataBlock &data) { } DataBlockHost dataHost(data); dataHost.SyncFromDevice(); - this->CallScript(dataHost,this->scriptFilename,this->initflowFunctionName); + this->CallScript(this->scriptFilename,this->initflowFunctionName,dataHost); dataHost.SyncToDevice(); idfx::popRegion(); } @@ -155,26 +173,7 @@ Pydefix::~Pydefix() { } } -void Pydefix::CallScript(DataBlockHost &data, std::string scriptName, std::string funcName) { - idfx::pushRegion("Pydefix::CallScript"); - try { - //auto Vc = pydefix.toNumpyArray(d.Vc); - py::module_ script = py::module_::import(scriptName.c_str()); - //py::module_ embeded = py::module_::import("embeded"); - //py::object myV = py::cast(Vc); - //py::object result = script.attr(funcName.c_str())(data); - py::object result = script.attr(funcName.c_str())(data); - } catch(std::exception &e) { - std::stringstream message; - message << "An exception occured while calling the Python interpreter" << std::endl - << "in file \"" << scriptName << ".py\" function \"" << funcName << "\":" - << std::endl - << e.what() << std::endl; - IDEFIX_ERROR(message); - } - idfx::popRegion(); -} /* py::array_t Pydefix::toNumpyArray(const IdefixHostArray3D& in) { py::array_t array({in.extent(0),in.extent(1),in.extent(2)},in.data()); diff --git a/src/pydefix.hpp b/src/pydefix.hpp index e8ac8879..2f782d89 100644 --- a/src/pydefix.hpp +++ b/src/pydefix.hpp @@ -27,20 +27,22 @@ class Pydefix { public: explicit Pydefix(Input&); ~Pydefix(); - void Output(DataBlock &); + void Output(DataBlock &, int); void InitFlow(DataBlock &); void ShowConfig(); bool isActive{false}; bool haveOutput{false}; bool haveInitflow{false}; private: - void CallScript(DataBlockHost &, std::string, std::string); + template + void CallScript(std::string, std::string, Ts...); static int ninstance; std::string scriptFilename; std::string outputFunctionName; std::string initflowFunctionName; }; + namespace pybind11 { namespace detail { // Caster for IdefixArray4D template struct type_caster> { From 6985443b56911c4128e544834585d5e536f66b2c Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 3 Oct 2024 11:32:16 +0200 Subject: [PATCH 22/83] -fix warning in cmake -add a readme --- CMakeLists.txt | 3 +- test/IO/pydefix/CMakeLists.txt | 2 ++ test/IO/pydefix/README | 43 +++++++++++++++++++++++++ test/IO/pydefix/definitions.hpp | 4 +++ test/IO/pydefix/idefix.ini | 30 +++++++++++++++++ test/IO/pydefix/pydefix_example.py | 29 +++++++++++++++++ test/IO/pydefix/python_requirements.txt | 6 ++++ 7 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 test/IO/pydefix/CMakeLists.txt create mode 100644 test/IO/pydefix/README create mode 100644 test/IO/pydefix/definitions.hpp create mode 100644 test/IO/pydefix/idefix.ini create mode 100644 test/IO/pydefix/pydefix_example.py create mode 100644 test/IO/pydefix/python_requirements.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a0e2d3a..ca152646 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -75,7 +75,7 @@ add_subdirectory(src build) if(EXISTS ${PROJECT_BINARY_DIR}/setup.cpp) target_sources(idefix PUBLIC ${PROJECT_BINARY_DIR}/setup.cpp) else() - message(WARNING "No specific setup.cpp found in the problem directory") + message(WARNING "No specific setup.cpp found in the problem directory (this message can be ignored if using python to define your problem)") endif() # If a CMakeLists.txt is in the problem dir (for problem-specific source files) @@ -121,6 +121,7 @@ endif() if(Idefix_PYTHON) add_compile_definitions("WITH_PYTHON") + set(PYBIND11_FINDPYTHON ON CACHE BOOL "Idefix requires python" FORCE) find_package(pybind11 REQUIRED) target_link_libraries(idefix pybind11::embed) target_sources(idefix diff --git a/test/IO/pydefix/CMakeLists.txt b/test/IO/pydefix/CMakeLists.txt new file mode 100644 index 00000000..83d16a52 --- /dev/null +++ b/test/IO/pydefix/CMakeLists.txt @@ -0,0 +1,2 @@ +enable_idefix_property(Idefix_MHD) +enable_idefix_property(Idefix_PYTHON) diff --git a/test/IO/pydefix/README b/test/IO/pydefix/README new file mode 100644 index 00000000..6cc5678a --- /dev/null +++ b/test/IO/pydefix/README @@ -0,0 +1,43 @@ +this directory shows how to use Idefix with pure python initial condtions and outputs. It reproduced the 2D OrszagTang vortex available in MHD/OrszagTang +The python script initialize the initial condition of the OT test and produces a series of PNG files through matplotlib + +# Python modules installation + +In order to use pydefix, you need to be working in a python environement that includes pybind11. The simplest way is to install the suggested list of packages (that you can deploy in dedicated venv environement) + +```bash +pip install -r python_requirements.txt +``` + +# Configuration + +In order to use Pydefix, you need to switch the `Idefix_PYTHON` to ON in cmake. In this particular setup, it is automatically done for you in CMakeLists.txt to avoid mistakes. + +# Running + +Just run idefix as usual. + +# Troubleshooting + +It during configuration stage, you get: + +` +CMake Error at CMakeLists.txt:122 (find_package): + By not providing "Findpybind11.cmake" in CMAKE_MODULE_PATH this project has + asked CMake to find a package configuration file provided by "pybind11", + but CMake did not find one.` + +It means that cmake cannot find the location of pybind11 (this typically happens on MacOs). In order to locate pybind11, open a python interpreter, and get pybind11 install dir through: + +```python +import pybind11 +print(pybind11.__file__) +``` + +You can then exit the interpreter and set the pybind11_DIR environement variable to the right path: + +```bash +export pybind11_DIR=env/lib/python3.10/site-packages/pybind11 +``` + +you can then run cmake which should be able to find pybind11, and compile the code. diff --git a/test/IO/pydefix/definitions.hpp b/test/IO/pydefix/definitions.hpp new file mode 100644 index 00000000..69f0c49e --- /dev/null +++ b/test/IO/pydefix/definitions.hpp @@ -0,0 +1,4 @@ +#define COMPONENTS 2 +#define DIMENSIONS 2 + +#define GEOMETRY CARTESIAN diff --git a/test/IO/pydefix/idefix.ini b/test/IO/pydefix/idefix.ini new file mode 100644 index 00000000..57fdcfc4 --- /dev/null +++ b/test/IO/pydefix/idefix.ini @@ -0,0 +1,30 @@ +[Grid] +X1-grid 1 0.0 256 u 1.0 +X2-grid 1 0.0 256 u 1.0 +X3-grid 1 0.0 1 u 1.0 + +[TimeIntegrator] +CFL 0.6 +tstop 0.5 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver roe + +[Python] +script pydefix_example +output_function output +initflow_function initflow + +[Boundary] +X1-beg periodic +X1-end periodic +X2-beg periodic +X2-end periodic +X3-beg outflow +X3-end outflow + +[Output] +log 10 +python 0.02 diff --git a/test/IO/pydefix/pydefix_example.py b/test/IO/pydefix/pydefix_example.py new file mode 100644 index 00000000..258f674f --- /dev/null +++ b/test/IO/pydefix/pydefix_example.py @@ -0,0 +1,29 @@ +import pydefix as pdfx +import numpy as np +import matplotlib.pyplot as plt + +# The output function +# the only argument is dataBlockHost python object, wrapping a dataBlockHost Idefix object +def output(data,n): + plt.close() + plt.figure() + plt.pcolormesh(data.x[pdfx.IDIR],data.x[pdfx.JDIR],data.Vc[pdfx.PRS,0,:,:],label='PRS',vmin=0.02,vmax=0.5,cmap='plasma') + plt.title("t=%.2f"%data.t) + plt.colorbar() + plt.savefig("PRS.%.4d.png"%n) + + +def initflow(data): + # Field amplitude + B0 = 1/np.sqrt(4*np.pi) + + [z,y,x] = np.meshgrid(data.x[pdfx.KDIR], data.x[pdfx.JDIR], data.x[pdfx.IDIR], indexing='ij') + + # Initialize the flow + data.Vc[pdfx.RHO,:,:,:] = 25/(36*np.pi) + data.Vc[pdfx.PRS,:,:,:] = 5/(12*np.pi) + data.Vc[pdfx.VX1,:,:,:] = -np.sin(2*np.pi*y) + data.Vc[pdfx.VX2,:,:,:] = np.sin(2*np.pi*x) + + data.Vs[pdfx.BX1s,:,:-1,:-1] = -B0*np.sin(2*np.pi*y) + data.Vs[pdfx.BX2s,:,:-1,:-1] = B0*np.sin(4*np.pi*x) diff --git a/test/IO/pydefix/python_requirements.txt b/test/IO/pydefix/python_requirements.txt new file mode 100644 index 00000000..6afd3019 --- /dev/null +++ b/test/IO/pydefix/python_requirements.txt @@ -0,0 +1,6 @@ +# note: version requirements are indicative and tests # should be able to run with +# older versions of our dependencies, though it is not guaranteed. + +numpy>=1.16.6 +matplotlib>=2.2.5 +pybind11>=2.12.0 From d3b3bb73adf82f0ea87350863e350632baf5c01b Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 3 Oct 2024 11:45:42 +0200 Subject: [PATCH 23/83] typos --- test/IO/pydefix/README | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/IO/pydefix/README b/test/IO/pydefix/README index 6cc5678a..820a0958 100644 --- a/test/IO/pydefix/README +++ b/test/IO/pydefix/README @@ -1,5 +1,6 @@ -this directory shows how to use Idefix with pure python initial condtions and outputs. It reproduced the 2D OrszagTang vortex available in MHD/OrszagTang -The python script initialize the initial condition of the OT test and produces a series of PNG files through matplotlib +This directory shows how to use Idefix with pure python initial conditions and outputs. It reproduces the 2D OrszagTang vortex available in MHD/OrszagTang without requiring any single line of C++ code from the user. + +The python script `pydefix_example.py` initializes the initial condition of the OT test (`initflow`) and produces a series of PNG files through matplotlib (`output`). # Python modules installation From a92732f3683af37bc3e55a9e8954867b1ae43967 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 3 Oct 2024 11:46:07 +0200 Subject: [PATCH 24/83] change naming --- test/IO/pydefix/{README => README.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/IO/pydefix/{README => README.md} (100%) diff --git a/test/IO/pydefix/README b/test/IO/pydefix/README.md similarity index 100% rename from test/IO/pydefix/README rename to test/IO/pydefix/README.md From 1c29ebbfee5a27d7d1388fdb259eebf02e478013 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 9 Oct 2024 20:16:38 +0200 Subject: [PATCH 25/83] fix lack of initialisation of datablockHost's dt and t after construction fix #276 --- src/dataBlock/dataBlockHost.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/dataBlock/dataBlockHost.cpp b/src/dataBlock/dataBlockHost.cpp index f5e8c269..3443dbb5 100644 --- a/src/dataBlock/dataBlockHost.cpp +++ b/src/dataBlock/dataBlockHost.cpp @@ -94,6 +94,9 @@ DataBlockHost::DataBlockHost(DataBlock& datain) { this->haveplanetarySystem = data->haveplanetarySystem; this->planetarySystem = data->planetarySystem.get(); + this->t = data->t; + this->dt = data->dt; + idfx::popRegion(); } @@ -141,7 +144,7 @@ void DataBlockHost::SyncFromDevice() { idfx::pushRegion("DataBlockHost::SyncFromDevice()"); this->t = data->t; this->dt = data->dt; - + Kokkos::deep_copy(Vc,data->hydro->Vc); Kokkos::deep_copy(InvDt,data->hydro->InvDt); From 427982a3e4ca98365cc8be9048d64ed47b48faaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Fri, 11 Oct 2024 22:38:07 +0200 Subject: [PATCH 26/83] MNT: fix linting with cpplint 2.0.0 (#279) --- CPPLINT.cfg | 2 ++ src/dataBlock/dataBlock.cpp | 2 ++ src/dataBlock/dataBlock.hpp | 1 + src/dataBlock/dataBlockHost.cpp | 1 + src/dataBlock/dumpToFile.cpp | 2 ++ src/fluid/boundary/axis.cpp | 1 + src/fluid/checkNan.hpp | 1 + src/global.cpp | 1 + src/global.hpp | 1 + src/idefix.hpp | 1 + src/macros.hpp | 1 + src/main.cpp | 1 + src/mpi.cpp | 1 + src/output/dump.cpp | 4 +++- src/output/dump.hpp | 2 +- src/output/output.cpp | 1 + src/output/slice.cpp | 1 + src/output/vtk.cpp | 3 ++- src/output/vtk.hpp | 3 ++- src/output/xdmf.hpp | 10 +++++++++- src/utils/dumpImage.cpp | 2 ++ src/utils/npy.hpp | 1 + 22 files changed, 38 insertions(+), 5 deletions(-) diff --git a/CPPLINT.cfg b/CPPLINT.cfg index 4f3e42cb..b02d056c 100644 --- a/CPPLINT.cfg +++ b/CPPLINT.cfg @@ -16,6 +16,8 @@ filter=-whitespace/comma # 6027 errors filter=-whitespace/comments # 881 errors filter=-whitespace/operators # 5240 errors filter=-whitespace/parens # 413 error +filter=-whitespace/newline +filter=-whitespace/indent_namespace filter=-readability/multiline_string filter=-build/include_subdir # 296 errors #filter=-whitespace/end_of_line diff --git a/src/dataBlock/dataBlock.cpp b/src/dataBlock/dataBlock.cpp index c0648f92..81874cef 100644 --- a/src/dataBlock/dataBlock.cpp +++ b/src/dataBlock/dataBlock.cpp @@ -6,6 +6,8 @@ // *********************************************************************************** #include +#include +#include #include "idefix.hpp" #include "dataBlock.hpp" #include "fluid.hpp" diff --git a/src/dataBlock/dataBlock.hpp b/src/dataBlock/dataBlock.hpp index 12f0847e..89b745fa 100644 --- a/src/dataBlock/dataBlock.hpp +++ b/src/dataBlock/dataBlock.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include "idefix.hpp" #include "grid.hpp" diff --git a/src/dataBlock/dataBlockHost.cpp b/src/dataBlock/dataBlockHost.cpp index ca26e5cb..b5dff915 100644 --- a/src/dataBlock/dataBlockHost.cpp +++ b/src/dataBlock/dataBlockHost.cpp @@ -5,6 +5,7 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#include #include "idefix.hpp" #include "dataBlockHost.hpp" #include "fluid.hpp" diff --git a/src/dataBlock/dumpToFile.cpp b/src/dataBlock/dumpToFile.cpp index 9980d09c..98b4a474 100644 --- a/src/dataBlock/dumpToFile.cpp +++ b/src/dataBlock/dumpToFile.cpp @@ -5,6 +5,8 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#include +#include #include "../idefix.hpp" #include "dataBlock.hpp" #include "version.hpp" diff --git a/src/fluid/boundary/axis.cpp b/src/fluid/boundary/axis.cpp index ead51f4f..b77fe3b5 100644 --- a/src/fluid/boundary/axis.cpp +++ b/src/fluid/boundary/axis.cpp @@ -5,6 +5,7 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#include #include "axis.hpp" #include "boundary.hpp" diff --git a/src/fluid/checkNan.hpp b/src/fluid/checkNan.hpp index 71911c58..88e206e5 100644 --- a/src/fluid/checkNan.hpp +++ b/src/fluid/checkNan.hpp @@ -7,6 +7,7 @@ #ifndef FLUID_CHECKNAN_HPP_ #define FLUID_CHECKNAN_HPP_ +#include #include "dataBlock.hpp" #include "dataBlockHost.hpp" #include "fluid.hpp" diff --git a/src/global.cpp b/src/global.cpp index 1e9c11ee..4d9499d9 100644 --- a/src/global.cpp +++ b/src/global.cpp @@ -5,6 +5,7 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#include #include #include #include "idefix.hpp" diff --git a/src/global.hpp b/src/global.hpp index 47fca42e..8651f974 100644 --- a/src/global.hpp +++ b/src/global.hpp @@ -7,6 +7,7 @@ #ifndef GLOBAL_HPP_ #define GLOBAL_HPP_ +#include #include #include #include "arrays.hpp" diff --git a/src/idefix.hpp b/src/idefix.hpp index a44bf4c2..f547c5d7 100644 --- a/src/idefix.hpp +++ b/src/idefix.hpp @@ -9,6 +9,7 @@ #define IDEFIX_HPP_ #include #include +#include #include // #include // do we still need this? #ifdef WITH_MPI diff --git a/src/macros.hpp b/src/macros.hpp index 9a484c3d..2c7bda12 100644 --- a/src/macros.hpp +++ b/src/macros.hpp @@ -10,6 +10,7 @@ #ifndef MACROS_HPP_ #define MACROS_HPP_ +#include #if COMPONENTS == 1 #define EXPAND(a,b,c) a diff --git a/src/main.cpp b/src/main.cpp index 5a7f6f73..c242ccef 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include diff --git a/src/mpi.cpp b/src/mpi.cpp index ee2fdb5c..d25056fa 100644 --- a/src/mpi.cpp +++ b/src/mpi.cpp @@ -12,6 +12,7 @@ #include // NOLINT [build/c++11] #include // NOLINT [build/c++11] #include +#include #include "idefix.hpp" #include "dataBlock.hpp" diff --git a/src/output/dump.cpp b/src/output/dump.cpp index 73b0df99..955e06e7 100644 --- a/src/output/dump.cpp +++ b/src/output/dump.cpp @@ -8,7 +8,7 @@ #include #include #if __has_include() - #include + #include // NOLINT [build/c++17] namespace fs = std::filesystem; #elif __has_include() #include @@ -17,6 +17,8 @@ #error "Missing the header." #endif #include +#include +#include #include "dump.hpp" #include "version.hpp" #include "dataBlockHost.hpp" diff --git a/src/output/dump.hpp b/src/output/dump.hpp index a9a52554..d823ddca 100644 --- a/src/output/dump.hpp +++ b/src/output/dump.hpp @@ -11,7 +11,7 @@ #include #include #if __has_include() - #include + #include // NOLINT [build/c++17] namespace fs = std::filesystem; #elif __has_include() #include diff --git a/src/output/output.cpp b/src/output/output.cpp index 30a85039..658e1cdd 100644 --- a/src/output/output.cpp +++ b/src/output/output.cpp @@ -5,6 +5,7 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#include #include "output.hpp" #include "dataBlock.hpp" #include "fluid.hpp" diff --git a/src/output/slice.cpp b/src/output/slice.cpp index 3f971932..98caa5c2 100644 --- a/src/output/slice.cpp +++ b/src/output/slice.cpp @@ -7,6 +7,7 @@ #include #include +#include #include "slice.hpp" #include "input.hpp" #include "physics.hpp" diff --git a/src/output/vtk.cpp b/src/output/vtk.cpp index 0f52f37c..ae4a4c60 100644 --- a/src/output/vtk.cpp +++ b/src/output/vtk.cpp @@ -8,10 +8,11 @@ #include "vtk.hpp" #include #include +#include #include #include #if __has_include() - #include + #include // NOLINT [build/c++17] namespace fs = std::filesystem; #elif __has_include() #include diff --git a/src/output/vtk.hpp b/src/output/vtk.hpp index dcda3618..a9503a5a 100644 --- a/src/output/vtk.hpp +++ b/src/output/vtk.hpp @@ -10,7 +10,7 @@ #include #include #if __has_include() - #include + #include // NOLINT [build/c++17] namespace fs = std::filesystem; #elif __has_include() #include @@ -18,6 +18,7 @@ #else error "Missing the header." #endif +#include #include "idefix.hpp" #include "input.hpp" #include "dataBlock.hpp" diff --git a/src/output/xdmf.hpp b/src/output/xdmf.hpp index b2fb6628..6c6fc15c 100644 --- a/src/output/xdmf.hpp +++ b/src/output/xdmf.hpp @@ -8,7 +8,15 @@ #ifndef OUTPUT_XDMF_HPP_ #define OUTPUT_XDMF_HPP_ #include -#include +#if __has_include() + #include // NOLINT [build/c++17] + namespace fs = std::filesystem; +#elif __has_include() + #include + namespace fs = std::experimental::filesystem; +#else + error "Missing the header." +#endif #include #include "idefix.hpp" #include "input.hpp" diff --git a/src/utils/dumpImage.cpp b/src/utils/dumpImage.cpp index 03a456e1..04f413a7 100644 --- a/src/utils/dumpImage.cpp +++ b/src/utils/dumpImage.cpp @@ -5,6 +5,8 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#include +#include #include "dumpImage.hpp" #include "dataBlock.hpp" #include "idefix.hpp" diff --git a/src/utils/npy.hpp b/src/utils/npy.hpp index 34b7c724..edae82ee 100644 --- a/src/utils/npy.hpp +++ b/src/utils/npy.hpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include From 84c1763483c5fe7324e25bcb59ede92643f9fc78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sat, 12 Oct 2024 07:57:36 +0200 Subject: [PATCH 27/83] STY: locally disable linting around #include (#281) --- src/output/xdmf.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/output/xdmf.cpp b/src/output/xdmf.cpp index 237f4ca3..23ddaa03 100644 --- a/src/output/xdmf.cpp +++ b/src/output/xdmf.cpp @@ -11,7 +11,7 @@ #include #include #if __has_include() - #include + #include // NOLINT [build/c++17] namespace fs = std::filesystem; #elif __has_include() #include From d0ace44b3985d196dbb585a8b603e6f3ed83e1c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Fri, 11 Oct 2024 22:38:07 +0200 Subject: [PATCH 28/83] MNT: fix linting with cpplint 2.0.0 (#279) --- CPPLINT.cfg | 2 ++ src/dataBlock/dataBlock.cpp | 2 ++ src/dataBlock/dataBlock.hpp | 1 + src/dataBlock/dataBlockHost.cpp | 1 + src/dataBlock/dumpToFile.cpp | 2 ++ src/fluid/boundary/axis.cpp | 1 + src/fluid/checkNan.hpp | 1 + src/global.cpp | 1 + src/global.hpp | 1 + src/idefix.hpp | 1 + src/macros.hpp | 1 + src/main.cpp | 1 + src/mpi.cpp | 1 + src/output/dump.cpp | 4 +++- src/output/dump.hpp | 2 +- src/output/output.cpp | 1 + src/output/slice.cpp | 1 + src/output/vtk.cpp | 3 ++- src/output/vtk.hpp | 3 ++- src/output/xdmf.hpp | 10 +++++++++- src/utils/dumpImage.cpp | 2 ++ src/utils/npy.hpp | 1 + 22 files changed, 38 insertions(+), 5 deletions(-) diff --git a/CPPLINT.cfg b/CPPLINT.cfg index 4f3e42cb..b02d056c 100644 --- a/CPPLINT.cfg +++ b/CPPLINT.cfg @@ -16,6 +16,8 @@ filter=-whitespace/comma # 6027 errors filter=-whitespace/comments # 881 errors filter=-whitespace/operators # 5240 errors filter=-whitespace/parens # 413 error +filter=-whitespace/newline +filter=-whitespace/indent_namespace filter=-readability/multiline_string filter=-build/include_subdir # 296 errors #filter=-whitespace/end_of_line diff --git a/src/dataBlock/dataBlock.cpp b/src/dataBlock/dataBlock.cpp index c0648f92..81874cef 100644 --- a/src/dataBlock/dataBlock.cpp +++ b/src/dataBlock/dataBlock.cpp @@ -6,6 +6,8 @@ // *********************************************************************************** #include +#include +#include #include "idefix.hpp" #include "dataBlock.hpp" #include "fluid.hpp" diff --git a/src/dataBlock/dataBlock.hpp b/src/dataBlock/dataBlock.hpp index 12f0847e..89b745fa 100644 --- a/src/dataBlock/dataBlock.hpp +++ b/src/dataBlock/dataBlock.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include "idefix.hpp" #include "grid.hpp" diff --git a/src/dataBlock/dataBlockHost.cpp b/src/dataBlock/dataBlockHost.cpp index 3443dbb5..76f797c7 100644 --- a/src/dataBlock/dataBlockHost.cpp +++ b/src/dataBlock/dataBlockHost.cpp @@ -5,6 +5,7 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#include #include "idefix.hpp" #include "dataBlockHost.hpp" #include "fluid.hpp" diff --git a/src/dataBlock/dumpToFile.cpp b/src/dataBlock/dumpToFile.cpp index 9980d09c..98b4a474 100644 --- a/src/dataBlock/dumpToFile.cpp +++ b/src/dataBlock/dumpToFile.cpp @@ -5,6 +5,8 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#include +#include #include "../idefix.hpp" #include "dataBlock.hpp" #include "version.hpp" diff --git a/src/fluid/boundary/axis.cpp b/src/fluid/boundary/axis.cpp index ead51f4f..b77fe3b5 100644 --- a/src/fluid/boundary/axis.cpp +++ b/src/fluid/boundary/axis.cpp @@ -5,6 +5,7 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#include #include "axis.hpp" #include "boundary.hpp" diff --git a/src/fluid/checkNan.hpp b/src/fluid/checkNan.hpp index 71911c58..88e206e5 100644 --- a/src/fluid/checkNan.hpp +++ b/src/fluid/checkNan.hpp @@ -7,6 +7,7 @@ #ifndef FLUID_CHECKNAN_HPP_ #define FLUID_CHECKNAN_HPP_ +#include #include "dataBlock.hpp" #include "dataBlockHost.hpp" #include "fluid.hpp" diff --git a/src/global.cpp b/src/global.cpp index 1e9c11ee..4d9499d9 100644 --- a/src/global.cpp +++ b/src/global.cpp @@ -5,6 +5,7 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#include #include #include #include "idefix.hpp" diff --git a/src/global.hpp b/src/global.hpp index 47fca42e..8651f974 100644 --- a/src/global.hpp +++ b/src/global.hpp @@ -7,6 +7,7 @@ #ifndef GLOBAL_HPP_ #define GLOBAL_HPP_ +#include #include #include #include "arrays.hpp" diff --git a/src/idefix.hpp b/src/idefix.hpp index a44bf4c2..f547c5d7 100644 --- a/src/idefix.hpp +++ b/src/idefix.hpp @@ -9,6 +9,7 @@ #define IDEFIX_HPP_ #include #include +#include #include // #include // do we still need this? #ifdef WITH_MPI diff --git a/src/macros.hpp b/src/macros.hpp index 9a484c3d..2c7bda12 100644 --- a/src/macros.hpp +++ b/src/macros.hpp @@ -10,6 +10,7 @@ #ifndef MACROS_HPP_ #define MACROS_HPP_ +#include #if COMPONENTS == 1 #define EXPAND(a,b,c) a diff --git a/src/main.cpp b/src/main.cpp index 6294dcff..06f408ff 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include diff --git a/src/mpi.cpp b/src/mpi.cpp index ee2fdb5c..d25056fa 100644 --- a/src/mpi.cpp +++ b/src/mpi.cpp @@ -12,6 +12,7 @@ #include // NOLINT [build/c++11] #include // NOLINT [build/c++11] #include +#include #include "idefix.hpp" #include "dataBlock.hpp" diff --git a/src/output/dump.cpp b/src/output/dump.cpp index 73b0df99..955e06e7 100644 --- a/src/output/dump.cpp +++ b/src/output/dump.cpp @@ -8,7 +8,7 @@ #include #include #if __has_include() - #include + #include // NOLINT [build/c++17] namespace fs = std::filesystem; #elif __has_include() #include @@ -17,6 +17,8 @@ #error "Missing the header." #endif #include +#include +#include #include "dump.hpp" #include "version.hpp" #include "dataBlockHost.hpp" diff --git a/src/output/dump.hpp b/src/output/dump.hpp index a9a52554..d823ddca 100644 --- a/src/output/dump.hpp +++ b/src/output/dump.hpp @@ -11,7 +11,7 @@ #include #include #if __has_include() - #include + #include // NOLINT [build/c++17] namespace fs = std::filesystem; #elif __has_include() #include diff --git a/src/output/output.cpp b/src/output/output.cpp index a482f66a..a67cb166 100644 --- a/src/output/output.cpp +++ b/src/output/output.cpp @@ -5,6 +5,7 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#include #include "output.hpp" #include "dataBlock.hpp" #include "fluid.hpp" diff --git a/src/output/slice.cpp b/src/output/slice.cpp index 3f971932..98caa5c2 100644 --- a/src/output/slice.cpp +++ b/src/output/slice.cpp @@ -7,6 +7,7 @@ #include #include +#include #include "slice.hpp" #include "input.hpp" #include "physics.hpp" diff --git a/src/output/vtk.cpp b/src/output/vtk.cpp index 0f52f37c..ae4a4c60 100644 --- a/src/output/vtk.cpp +++ b/src/output/vtk.cpp @@ -8,10 +8,11 @@ #include "vtk.hpp" #include #include +#include #include #include #if __has_include() - #include + #include // NOLINT [build/c++17] namespace fs = std::filesystem; #elif __has_include() #include diff --git a/src/output/vtk.hpp b/src/output/vtk.hpp index dcda3618..a9503a5a 100644 --- a/src/output/vtk.hpp +++ b/src/output/vtk.hpp @@ -10,7 +10,7 @@ #include #include #if __has_include() - #include + #include // NOLINT [build/c++17] namespace fs = std::filesystem; #elif __has_include() #include @@ -18,6 +18,7 @@ #else error "Missing the header." #endif +#include #include "idefix.hpp" #include "input.hpp" #include "dataBlock.hpp" diff --git a/src/output/xdmf.hpp b/src/output/xdmf.hpp index b2fb6628..6c6fc15c 100644 --- a/src/output/xdmf.hpp +++ b/src/output/xdmf.hpp @@ -8,7 +8,15 @@ #ifndef OUTPUT_XDMF_HPP_ #define OUTPUT_XDMF_HPP_ #include -#include +#if __has_include() + #include // NOLINT [build/c++17] + namespace fs = std::filesystem; +#elif __has_include() + #include + namespace fs = std::experimental::filesystem; +#else + error "Missing the header." +#endif #include #include "idefix.hpp" #include "input.hpp" diff --git a/src/utils/dumpImage.cpp b/src/utils/dumpImage.cpp index 03a456e1..04f413a7 100644 --- a/src/utils/dumpImage.cpp +++ b/src/utils/dumpImage.cpp @@ -5,6 +5,8 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#include +#include #include "dumpImage.hpp" #include "dataBlock.hpp" #include "idefix.hpp" diff --git a/src/utils/npy.hpp b/src/utils/npy.hpp index 34b7c724..edae82ee 100644 --- a/src/utils/npy.hpp +++ b/src/utils/npy.hpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include From 4f48ac6153076723166314e49fea396540e62061 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sat, 12 Oct 2024 07:57:36 +0200 Subject: [PATCH 29/83] STY: locally disable linting around #include (#281) --- src/output/xdmf.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/output/xdmf.cpp b/src/output/xdmf.cpp index 237f4ca3..23ddaa03 100644 --- a/src/output/xdmf.cpp +++ b/src/output/xdmf.cpp @@ -11,7 +11,7 @@ #include #include #if __has_include() - #include + #include // NOLINT [build/c++17] namespace fs = std::filesystem; #elif __has_include() #include From 9d92bfe69a677ddb0de888097e74dc0e7e03e960 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Sun, 13 Oct 2024 10:15:59 +0200 Subject: [PATCH 30/83] Simplify input file for DIMENSIONS < 3 setups (#274) --- doc/source/reference/idefix.ini.rst | 6 ++-- .../planetarySystem/planetarySystem.cpp | 14 +++++---- src/gravity/laplacian.hpp | 9 +++++- src/gravity/selfGravity.cpp | 6 +++- src/grid.cpp | 15 +++++++++- src/gridHost.cpp | 30 +++++++++++-------- src/idefix.hpp | 2 +- test/HD/FargoPlanet/setup.cpp | 3 -- test/HD/VSI/idefix.ini | 3 -- test/HD/ViscousDisk/idefix-rkl.ini | 3 -- test/HD/ViscousDisk/idefix.ini | 3 -- .../HD/ViscousFlowPastCylinder/idefix-rkl.ini | 5 ---- test/HD/ViscousFlowPastCylinder/idefix.ini | 5 ---- test/HD/sod-iso/idefix-hll.ini | 6 ---- test/HD/sod-iso/idefix-hllc-rk3.ini | 6 ---- test/HD/sod-iso/idefix-hllc.ini | 6 ---- test/HD/sod-iso/idefix-rk3.ini | 6 ---- test/HD/sod-iso/idefix-tvdlf.ini | 6 ---- test/HD/sod-iso/idefix.ini | 6 ---- test/HD/sod/idefix-hll.ini | 6 ---- test/HD/sod/idefix-hllc-rk3.ini | 6 ---- test/HD/sod/idefix-hllc.ini | 6 ---- test/HD/sod/idefix-rk3.ini | 6 ---- test/HD/sod/idefix-tvdlf.ini | 6 ---- test/HD/sod/idefix.ini | 6 ---- test/MHD/AmbipolarCshock/idefix-rkl.ini | 6 ---- test/MHD/AmbipolarWind/idefix.ini | 3 -- test/MHD/MTI/idefix-rkl.ini | 3 -- test/MHD/MTI/idefix-sl.ini | 3 -- test/MHD/MTI/idefix.ini | 3 -- test/MHD/OrszagTang/idefix-hll.ini | 3 -- .../MHD/OrszagTang/idefix-hlld-arithmetic.ini | 3 -- test/MHD/OrszagTang/idefix-hlld-hll.ini | 3 -- test/MHD/OrszagTang/idefix-hlld-hlld.ini | 3 -- test/MHD/OrszagTang/idefix-hlld-uct0.ini | 3 -- test/MHD/OrszagTang/idefix-hlld.ini | 3 -- test/MHD/OrszagTang/idefix-tvdlf.ini | 3 -- test/MHD/OrszagTang/idefix.ini | 3 -- test/MHD/ResistiveAlfvenWave/idefix-rkl.ini | 6 ---- test/MHD/ResistiveAlfvenWave/idefix.ini | 6 ---- test/MHD/RotorCartesian/idefix.ini | 3 -- test/MHD/RotorPolar/idefix.ini | 3 -- test/MHD/sod-iso/idefix-hll.ini | 6 ---- test/MHD/sod-iso/idefix-hlld-rk3.ini | 6 ---- test/MHD/sod-iso/idefix-hlld.ini | 6 ---- test/MHD/sod-iso/idefix-rk3.ini | 6 ---- test/MHD/sod-iso/idefix-tvdlf.ini | 6 ---- test/MHD/sod-iso/idefix.ini | 6 ---- test/MHD/sod/idefix-hll.ini | 6 ---- test/MHD/sod/idefix-hlld-rk3.ini | 6 ---- test/MHD/sod/idefix-hlld.ini | 6 ---- test/MHD/sod/idefix-rk3.ini | 6 ---- test/MHD/sod/idefix-tvdlf.ini | 6 ---- test/MHD/sod/idefix.ini | 6 ---- test/Planet/Planet3Body/idefix.ini | 7 ++--- test/Planet/PlanetMigration2D/idefix.ini | 7 ++--- test/Planet/PlanetPlanetRK42D/idefix.ini | 7 ++--- test/Planet/PlanetSpiral2D/idefix.ini | 7 ++--- .../PlanetsIsActiveRK52D/idefix-rk4.ini | 7 ++--- .../PlanetsIsActiveRK52D/idefix-rk5.ini | 7 ++--- test/SelfGravity/DustyCollapse/idefix.ini | 10 ------- .../JeansInstability/idefix-cg.ini | 10 ------- test/SelfGravity/JeansInstability/idefix.ini | 10 ------- test/SelfGravity/UniformCollapse/idefix.ini | 10 ------- 64 files changed, 70 insertions(+), 320 deletions(-) diff --git a/doc/source/reference/idefix.ini.rst b/doc/source/reference/idefix.ini.rst index 066db997..a7309f10 100644 --- a/doc/source/reference/idefix.ini.rst +++ b/doc/source/reference/idefix.ini.rst @@ -16,7 +16,7 @@ allows for comments, which should start with ``#``. ``Grid`` section -------------------- -The grid section defines the grid total dimension. It consists of 3 entries ``X1-grid``, ``X2-grid`` and ``X3-grid``. Each entry defines the repartition of the grid points in the corresponding direction (the grid is always rectilinear). +The grid section defines the grid total dimension. It consists of 3 entries ``X1-grid``, ``X2-grid`` (when DIMENSIONS>=2) and ``X3-grid`` (when DIMENSIONS=3). Each entry defines the repartition of the grid points in the corresponding direction (the grid is always rectilinear). Each entry defines a series of grid blocks which are concatenated along the direction. Each block in a direction can have a different spacing rule (uniform, log or stretched). The definition of the Grid entries is as follows +----------------------------+-------------------------+------------------------------+ @@ -332,8 +332,8 @@ this block is simply ignored. ------------------------ This section describes the boundary conditions used by the code. There are 6 entries -which need to be defined: ``X1-beg``, ``X2-beg``, ``X3-beg`` for the left boundaries in the direction X1, X2, X3, -and ``X1-end``, ``X2-end``, ``X3-end`` for the right boundaries. Each boundary can be assigned the following types of conditions +that need to be defined: ``X1-beg``, ``X2-beg``, ``X3-beg`` for the left boundaries in the direction X1, X2, X3, +and ``X1-end``, ``X2-end``, ``X3-end`` for the right boundaries. ``X2`` boundaries are mandatory only when DIMENSIONS>=2 and ``X3`` when DIMENSIONS=3. Each boundary can be assigned the following types of conditions +----------------+------------------------------------------------------------------------------------------------------------------+ | Boundary type | Comment | diff --git a/src/dataBlock/planetarySystem/planetarySystem.cpp b/src/dataBlock/planetarySystem/planetarySystem.cpp index 3d18e69d..8d01118d 100644 --- a/src/dataBlock/planetarySystem/planetarySystem.cpp +++ b/src/dataBlock/planetarySystem/planetarySystem.cpp @@ -83,12 +83,16 @@ indirect term, with multiple planets that don't feel each others."); IDEFIX_ERROR("need to define a planet-to-primary mass ratio via planetToPrimary"); } #if GEOMETRY == POLAR || GEOMETRY == CARTESIAN - if ((this->data->mygrid->xbeg[KDIR] == 0) - || (this->data->mygrid->xend[KDIR] == 0)) { - this->halfdisk = true; - } else { + #if DIMENSIONS == 3 + if ((this->data->mygrid->xbeg[KDIR] == 0) + || (this->data->mygrid->xend[KDIR] == 0)) { + this->halfdisk = true; + } else { + this->halfdisk = false; + } + #else // DIMENSIONS == 3 this->halfdisk = false; - } + #endif // DIMENSIONS == 3 #elif GEOMETRY == SPHERICAL if ( (this->data->mygrid->xbeg[JDIR] == M_PI/2.0) || (this->data->mygrid->xend[JDIR] == M_PI/2.0)) { diff --git a/src/gravity/laplacian.hpp b/src/gravity/laplacian.hpp index 4c41842b..28572439 100644 --- a/src/gravity/laplacian.hpp +++ b/src/gravity/laplacian.hpp @@ -19,7 +19,14 @@ class DataBlock; class Laplacian { public: // Types of boundary which can be treated - enum LaplacianBoundaryType {internalgrav, periodic, nullgrad, nullpot, userdef, axis, origin}; + enum LaplacianBoundaryType {internalgrav, + periodic, + nullgrad, + nullpot, + userdef, + axis, + origin, + undefined}; Laplacian() = default; Laplacian(DataBlock *, std::array, diff --git a/src/gravity/selfGravity.cpp b/src/gravity/selfGravity.cpp index 8efef800..43f76471 100644 --- a/src/gravity/selfGravity.cpp +++ b/src/gravity/selfGravity.cpp @@ -42,7 +42,11 @@ void SelfGravity::Init(Input &input, DataBlock *datain) { } // Get the gravity-related boundary conditions - for(int dir = 0 ; dir < 3 ; dir++) { + for (int dir = 0 ; dir < 3 ; dir++) { + this->lbound[dir] = Laplacian::LaplacianBoundaryType::undefined; + this->rbound[dir] = Laplacian::LaplacianBoundaryType::undefined; + } + for(int dir = 0 ; dir < DIMENSIONS ; dir++) { std::string label = std::string("boundary-X")+std::to_string(dir+1)+std::string("-beg"); std::string boundary = input.Get("SelfGravity",label,0); diff --git a/src/grid.cpp b/src/grid.cpp index 14a1e174..e4c6b7b4 100644 --- a/src/grid.cpp +++ b/src/grid.cpp @@ -55,7 +55,7 @@ Grid::Grid(Input &input) { npoints[dir] = 1; nghost[dir] = 0; std::string label = std::string("X")+std::to_string(dir+1)+std::string("-grid"); - int numPatch = input.Get("Grid",label,0); + if(dir("Grid",label,0); for(int patch = 0; patch < numPatch ; patch++) { npoints[dir] += input.Get("Grid",label,2+3*patch ); } @@ -73,7 +74,14 @@ Grid::Grid(Input &input) { for(int dir = 0 ; dir < 3 ; dir++) { np_tot[dir] = npoints[dir] + 2*nghost[dir]; np_int[dir] = npoints[dir]; + lbound[dir] = undefined; + rbound[dir] = undefined; + } + + // Default boundary conditions on each axis + + for(int dir = 0 ; dir < DIMENSIONS ; dir++) { std::string label = std::string("X")+std::to_string(dir+1)+std::string("-beg"); std::string boundary = input.Get("Boundary",label,0); @@ -337,6 +345,9 @@ void Grid::ShowConfig() { case userdef: lboundString="userdef"; break; + case undefined: + lboundString="undefined"; + break; default: lboundString="unknown"; } @@ -362,6 +373,8 @@ void Grid::ShowConfig() { case userdef: rboundString="userdef"; break; + case undefined: + lboundString="undefined"; default: rboundString="unknown"; } diff --git a/src/gridHost.cpp b/src/gridHost.cpp index 1c48b53c..d31a232c 100644 --- a/src/gridHost.cpp +++ b/src/gridHost.cpp @@ -50,17 +50,17 @@ void GridHost::MakeGrid(Input &input) { // Get grid parameters from input file, block [Grid] for(int dir = 0 ; dir < 3 ; dir++) { - std::string label = std::string("X")+std::to_string(dir+1)+std::string("-grid"); - int numPatch = input.Get("Grid",label,0); + // These are extra dimensions that are not being used. - xstart[dir] = input.Get("Grid",label,1); - xend[dir] = input.Get("Grid",label,4+(numPatch-1)*3); - - this->xbeg[dir] = xstart[dir]; - this->xend[dir] = xend[dir]; + if(dir("Grid",label,0); + xstart[dir] = input.Get("Grid",label,1); + xend[dir] = input.Get("Grid",label,4+(numPatch-1)*3); - if(dirxbeg[dir] = xstart[dir]; + this->xend[dir] = xend[dir]; // First, we fill cells for any non strecthed patch // Loop on all the patches int idxstart = nghost[dir]; @@ -200,10 +200,16 @@ void GridHost::MakeGrid(Input &input) { } else { // dir >= DIMENSIONS/ Init simple uniform grid for(int i = 0 ; i < np_tot[dir] ; i++) { - dx[dir](i) = (xend[dir]-xstart[dir])/(np_int[dir]); - x[dir](i)=xstart[dir] + (i-nghost[dir]+HALF_F)*dx[dir](i); - xl[dir](i)=xstart[dir] + (i-nghost[dir])*dx[dir](i); - xr[dir](i)=xstart[dir] + (i-nghost[dir]+1)*dx[dir](i); + // Initialize to default values + xstart[dir] = -0.5; + xend[dir] = 0.5; + + this->xbeg[dir] = xstart[dir]; + this->xend[dir] = xend[dir]; + dx[dir](i) = 1.0; + x[dir](i)=0.0; + xl[dir](i)=-0.5; + xr[dir](i)=0.5; } } } diff --git a/src/idefix.hpp b/src/idefix.hpp index f547c5d7..7485f95f 100644 --- a/src/idefix.hpp +++ b/src/idefix.hpp @@ -217,7 +217,7 @@ using IdfxFileHandler = FILE*; #endif // Types of boundary which can be treated -enum BoundaryType { internal, periodic, reflective, outflow, shearingbox, axis, userdef}; +enum BoundaryType { internal, periodic, reflective, outflow, shearingbox, axis, userdef, undefined}; enum BoundarySide { left, right}; enum class SliceType {Cut, Average}; diff --git a/test/HD/FargoPlanet/setup.cpp b/test/HD/FargoPlanet/setup.cpp index 125834a4..314858ba 100644 --- a/test/HD/FargoPlanet/setup.cpp +++ b/test/HD/FargoPlanet/setup.cpp @@ -43,7 +43,6 @@ void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { IdefixArray4D Vc = hydro->Vc; auto *data = hydro->data; IdefixArray1D x1 = data->x[IDIR]; - IdefixArray1D x3 = data->x[KDIR]; if(dir==IDIR) { int ighost,ibeg,iend; if(side == left) { @@ -56,7 +55,6 @@ void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { ibeg, iend, KOKKOS_LAMBDA (int k, int j, int i) { real R=x1(i); - real z=x3(k); real Vk = 1.0/sqrt(R); Vc(RHO,k,j,i) = Vc(RHO,k,j,2*ighost - i +1); @@ -75,7 +73,6 @@ void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { ibeg, iend, KOKKOS_LAMBDA (int k, int j, int i) { real R=x1(i); - real z=x3(k); real Vk = 1.0/sqrt(R); Vc(RHO,k,j,i) = Vc(RHO,k,j,ighost); diff --git a/test/HD/VSI/idefix.ini b/test/HD/VSI/idefix.ini index bc728eac..f0ce6a4d 100644 --- a/test/HD/VSI/idefix.ini +++ b/test/HD/VSI/idefix.ini @@ -1,7 +1,6 @@ [Grid] X1-grid 1 1.0 1024 l 3.0 X2-grid 1 1.2707963267948965 512 u 1.8707963267948966 -X3-grid 1 0.0 1 u 6.283185307179586 [TimeIntegrator] CFL 0.8 @@ -22,8 +21,6 @@ X1-beg userdef X1-end outflow X2-beg outflow X2-end outflow -X3-beg periodic -X3-end periodic [Setup] epsilon 0.1 diff --git a/test/HD/ViscousDisk/idefix-rkl.ini b/test/HD/ViscousDisk/idefix-rkl.ini index 63888492..207f3abe 100644 --- a/test/HD/ViscousDisk/idefix-rkl.ini +++ b/test/HD/ViscousDisk/idefix-rkl.ini @@ -1,7 +1,6 @@ [Grid] X1-grid 1 1.0 64 u 3.0 X2-grid 1 1.2707963267948965 64 u 1.8707963267948966 -X3-grid 1 0.0 1 u 6.283185307179586 [TimeIntegrator] CFL 0.5 @@ -23,8 +22,6 @@ X1-beg userdef X1-end userdef X2-beg userdef X2-end userdef -X3-beg periodic -X3-end periodic [Setup] epsilon 0.1 diff --git a/test/HD/ViscousDisk/idefix.ini b/test/HD/ViscousDisk/idefix.ini index 673d4be7..dbe8a499 100644 --- a/test/HD/ViscousDisk/idefix.ini +++ b/test/HD/ViscousDisk/idefix.ini @@ -1,7 +1,6 @@ [Grid] X1-grid 1 1.0 64 u 3.0 X2-grid 1 1.2707963267948965 64 u 1.8707963267948966 -X3-grid 1 0.0 1 u 6.283185307179586 [TimeIntegrator] CFL 0.5 @@ -23,8 +22,6 @@ X1-beg userdef X1-end userdef X2-beg userdef X2-end userdef -X3-beg periodic -X3-end periodic [Setup] epsilon 0.1 diff --git a/test/HD/ViscousFlowPastCylinder/idefix-rkl.ini b/test/HD/ViscousFlowPastCylinder/idefix-rkl.ini index 8b8b33e3..73b8722b 100644 --- a/test/HD/ViscousFlowPastCylinder/idefix-rkl.ini +++ b/test/HD/ViscousFlowPastCylinder/idefix-rkl.ini @@ -1,9 +1,6 @@ [Grid] X1-grid 1 1.0 128 l 10.0 -# X2-grid 3 -3.141592653589793 64 s+ -0.2 128 u 0.2 64 s- 3.141592653589793 -# X2-grid 1 -3.141592653589793 256 u 3.141592653589793 X2-grid 1 0.0 64 u 6.28318530717958 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.4 @@ -21,8 +18,6 @@ X1-beg userdef X1-end userdef X2-beg periodic X2-end periodic -X3-beg outflow -X3-end outflow [Output] vtk 1.0 diff --git a/test/HD/ViscousFlowPastCylinder/idefix.ini b/test/HD/ViscousFlowPastCylinder/idefix.ini index 33793490..de1cd417 100644 --- a/test/HD/ViscousFlowPastCylinder/idefix.ini +++ b/test/HD/ViscousFlowPastCylinder/idefix.ini @@ -1,9 +1,6 @@ [Grid] X1-grid 1 1.0 128 l 10.0 -# X2-grid 3 -3.141592653589793 64 s+ -0.2 128 u 0.2 64 s- 3.141592653589793 -# X2-grid 1 -3.141592653589793 256 u 3.141592653589793 X2-grid 1 0.0 64 u 6.28318530717958 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.4 @@ -21,8 +18,6 @@ X1-beg userdef X1-end userdef X2-beg periodic X2-end periodic -X3-beg outflow -X3-end outflow [Output] vtk 1.0 diff --git a/test/HD/sod-iso/idefix-hll.ini b/test/HD/sod-iso/idefix-hll.ini index cb6ab432..f7b36503 100644 --- a/test/HD/sod-iso/idefix-hll.ini +++ b/test/HD/sod-iso/idefix-hll.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 500 u 1.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 @@ -16,10 +14,6 @@ csiso constant 1.0 [Boundary] X1-beg outflow X1-end outflow -X2-beg outflow -X2-end outflow -X3-beg outflow -X3-end outflow [Output] vtk 0.1 diff --git a/test/HD/sod-iso/idefix-hllc-rk3.ini b/test/HD/sod-iso/idefix-hllc-rk3.ini index 7606c7fc..161237da 100644 --- a/test/HD/sod-iso/idefix-hllc-rk3.ini +++ b/test/HD/sod-iso/idefix-hllc-rk3.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 500 u 1.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 @@ -16,10 +14,6 @@ csiso constant 1.0 [Boundary] X1-beg outflow X1-end outflow -X2-beg outflow -X2-end outflow -X3-beg outflow -X3-end outflow [Output] vtk 0.1 diff --git a/test/HD/sod-iso/idefix-hllc.ini b/test/HD/sod-iso/idefix-hllc.ini index 751f5e86..a5d3eea8 100644 --- a/test/HD/sod-iso/idefix-hllc.ini +++ b/test/HD/sod-iso/idefix-hllc.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 500 u 1.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 @@ -16,10 +14,6 @@ csiso constant 1.0 [Boundary] X1-beg outflow X1-end outflow -X2-beg outflow -X2-end outflow -X3-beg outflow -X3-end outflow [Output] vtk 0.1 diff --git a/test/HD/sod-iso/idefix-rk3.ini b/test/HD/sod-iso/idefix-rk3.ini index e913683d..498a24f3 100644 --- a/test/HD/sod-iso/idefix-rk3.ini +++ b/test/HD/sod-iso/idefix-rk3.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 500 u 1.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 @@ -16,10 +14,6 @@ csiso constant 1.0 [Boundary] X1-beg outflow X1-end outflow -X2-beg outflow -X2-end outflow -X3-beg outflow -X3-end outflow [Output] vtk 0.1 diff --git a/test/HD/sod-iso/idefix-tvdlf.ini b/test/HD/sod-iso/idefix-tvdlf.ini index f6c7e57d..ecb0b07d 100644 --- a/test/HD/sod-iso/idefix-tvdlf.ini +++ b/test/HD/sod-iso/idefix-tvdlf.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 500 u 1.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 @@ -16,10 +14,6 @@ csiso constant 1.0 [Boundary] X1-beg outflow X1-end outflow -X2-beg outflow -X2-end outflow -X3-beg outflow -X3-end outflow [Output] vtk 0.1 diff --git a/test/HD/sod-iso/idefix.ini b/test/HD/sod-iso/idefix.ini index 564286e3..2477fa6d 100644 --- a/test/HD/sod-iso/idefix.ini +++ b/test/HD/sod-iso/idefix.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 500 u 1.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 @@ -16,10 +14,6 @@ csiso constant 1.0 [Boundary] X1-beg outflow X1-end outflow -X2-beg outflow -X2-end outflow -X3-beg outflow -X3-end outflow [Output] vtk 0.1 diff --git a/test/HD/sod/idefix-hll.ini b/test/HD/sod/idefix-hll.ini index 0bfb03ce..e515254d 100644 --- a/test/HD/sod/idefix-hll.ini +++ b/test/HD/sod/idefix-hll.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 500 u 1.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 @@ -16,10 +14,6 @@ gamma 1.4 [Boundary] X1-beg outflow X1-end outflow -X2-beg outflow -X2-end outflow -X3-beg outflow -X3-end outflow [Output] vtk 0.1 diff --git a/test/HD/sod/idefix-hllc-rk3.ini b/test/HD/sod/idefix-hllc-rk3.ini index 68a97c55..0c73d838 100644 --- a/test/HD/sod/idefix-hllc-rk3.ini +++ b/test/HD/sod/idefix-hllc-rk3.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 500 u 1.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 @@ -16,10 +14,6 @@ gamma 1.4 [Boundary] X1-beg outflow X1-end outflow -X2-beg outflow -X2-end outflow -X3-beg outflow -X3-end outflow [Output] vtk 0.1 diff --git a/test/HD/sod/idefix-hllc.ini b/test/HD/sod/idefix-hllc.ini index 3db50c3c..1acedec1 100644 --- a/test/HD/sod/idefix-hllc.ini +++ b/test/HD/sod/idefix-hllc.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 500 u 1.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 @@ -16,10 +14,6 @@ gamma 1.4 [Boundary] X1-beg outflow X1-end outflow -X2-beg outflow -X2-end outflow -X3-beg outflow -X3-end outflow [Output] vtk 0.1 diff --git a/test/HD/sod/idefix-rk3.ini b/test/HD/sod/idefix-rk3.ini index 7fe738ed..96675389 100644 --- a/test/HD/sod/idefix-rk3.ini +++ b/test/HD/sod/idefix-rk3.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 500 u 1.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 @@ -16,10 +14,6 @@ gamma 1.4 [Boundary] X1-beg outflow X1-end outflow -X2-beg outflow -X2-end outflow -X3-beg outflow -X3-end outflow [Output] vtk 0.1 diff --git a/test/HD/sod/idefix-tvdlf.ini b/test/HD/sod/idefix-tvdlf.ini index 36133b9b..ef465164 100644 --- a/test/HD/sod/idefix-tvdlf.ini +++ b/test/HD/sod/idefix-tvdlf.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 500 u 1.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 @@ -16,10 +14,6 @@ gamma 1.4 [Boundary] X1-beg outflow X1-end outflow -X2-beg outflow -X2-end outflow -X3-beg outflow -X3-end outflow [Output] vtk 0.1 diff --git a/test/HD/sod/idefix.ini b/test/HD/sod/idefix.ini index b34a23c3..8e089ef6 100644 --- a/test/HD/sod/idefix.ini +++ b/test/HD/sod/idefix.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 500 u 1.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 @@ -16,10 +14,6 @@ gamma 1.4 [Boundary] X1-beg outflow X1-end outflow -X2-beg outflow -X2-end outflow -X3-beg outflow -X3-end outflow [Output] vtk 0.1 diff --git a/test/MHD/AmbipolarCshock/idefix-rkl.ini b/test/MHD/AmbipolarCshock/idefix-rkl.ini index e02b3415..76a21a44 100644 --- a/test/MHD/AmbipolarCshock/idefix-rkl.ini +++ b/test/MHD/AmbipolarCshock/idefix-rkl.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 100 u 50.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.9 @@ -17,10 +15,6 @@ csiso constant 0.1 [Boundary] X1-beg userdef X1-end userdef -X2-beg periodic -X2-end periodic -X3-beg periodic -X3-end periodic [Output] vtk 100.0 diff --git a/test/MHD/AmbipolarWind/idefix.ini b/test/MHD/AmbipolarWind/idefix.ini index 22d8006a..24dd839f 100644 --- a/test/MHD/AmbipolarWind/idefix.ini +++ b/test/MHD/AmbipolarWind/idefix.ini @@ -1,7 +1,6 @@ [Grid] X1-grid 1 1.0 768 l 100.0 X2-grid 3 0.0 64 s+ 1.28 96 u 1.861592653589 64 s- 3.141592653589793 -X3-grid 1 0.0 0 u 1.0 [TimeIntegrator] CFL 0.9 @@ -25,8 +24,6 @@ X1-beg userdef X1-end userdef X2-beg axis X2-end axis -X3-beg periodic -X3-end periodic [Setup] epsilon 0.05 diff --git a/test/MHD/MTI/idefix-rkl.ini b/test/MHD/MTI/idefix-rkl.ini index 2eeb7615..1b9b11eb 100644 --- a/test/MHD/MTI/idefix-rkl.ini +++ b/test/MHD/MTI/idefix-rkl.ini @@ -1,7 +1,6 @@ [Grid] X1-grid 1 0. 32 u 0.1 X2-grid 1 0. 32 u 0.1 -X3-grid 1 0. 1 u 1. [TimeIntegrator] CFL 0.7 @@ -24,8 +23,6 @@ X1-beg periodic X1-end periodic X2-beg userdef X2-end userdef -X3-beg userdef -X3-end userdef [Setup] ksi 5e-4 diff --git a/test/MHD/MTI/idefix-sl.ini b/test/MHD/MTI/idefix-sl.ini index de020034..f2c9deb2 100644 --- a/test/MHD/MTI/idefix-sl.ini +++ b/test/MHD/MTI/idefix-sl.ini @@ -1,7 +1,6 @@ [Grid] X1-grid 1 0. 32 u 0.1 X2-grid 1 0. 32 u 0.1 -X3-grid 1 0. 1 u 1. [TimeIntegrator] CFL 0.7 @@ -24,8 +23,6 @@ X1-beg periodic X1-end periodic X2-beg userdef X2-end userdef -X3-beg userdef -X3-end userdef [Setup] ksi 5e-4 diff --git a/test/MHD/MTI/idefix.ini b/test/MHD/MTI/idefix.ini index b54bedb0..7a09c406 100644 --- a/test/MHD/MTI/idefix.ini +++ b/test/MHD/MTI/idefix.ini @@ -1,7 +1,6 @@ [Grid] X1-grid 1 0. 32 u 0.1 X2-grid 1 0. 32 u 0.1 -X3-grid 1 0. 1 u 1. [TimeIntegrator] CFL 0.7 @@ -24,8 +23,6 @@ X1-beg periodic X1-end periodic X2-beg userdef X2-end userdef -X3-beg userdef -X3-end userdef [Setup] ksi 5e-4 diff --git a/test/MHD/OrszagTang/idefix-hll.ini b/test/MHD/OrszagTang/idefix-hll.ini index 3dd96a58..4f13fd97 100644 --- a/test/MHD/OrszagTang/idefix-hll.ini +++ b/test/MHD/OrszagTang/idefix-hll.ini @@ -1,7 +1,6 @@ [Grid] X1-grid 1 0.0 128 u 1.0 X2-grid 1 0.0 128 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.6 @@ -17,8 +16,6 @@ X1-beg periodic X1-end periodic X2-beg periodic X2-end periodic -X3-beg outflow -X3-end outflow [Output] vtk 0.5 diff --git a/test/MHD/OrszagTang/idefix-hlld-arithmetic.ini b/test/MHD/OrszagTang/idefix-hlld-arithmetic.ini index b8549db5..8a656d7b 100644 --- a/test/MHD/OrszagTang/idefix-hlld-arithmetic.ini +++ b/test/MHD/OrszagTang/idefix-hlld-arithmetic.ini @@ -1,7 +1,6 @@ [Grid] X1-grid 1 0.0 128 u 1.0 X2-grid 1 0.0 128 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.6 @@ -18,8 +17,6 @@ X1-beg periodic X1-end periodic X2-beg periodic X2-end periodic -X3-beg outflow -X3-end outflow [Output] vtk 0.5 diff --git a/test/MHD/OrszagTang/idefix-hlld-hll.ini b/test/MHD/OrszagTang/idefix-hlld-hll.ini index 8c3e4ef9..b396e991 100644 --- a/test/MHD/OrszagTang/idefix-hlld-hll.ini +++ b/test/MHD/OrszagTang/idefix-hlld-hll.ini @@ -1,7 +1,6 @@ [Grid] X1-grid 1 0.0 128 u 1.0 X2-grid 1 0.0 128 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.6 @@ -18,8 +17,6 @@ X1-beg periodic X1-end periodic X2-beg periodic X2-end periodic -X3-beg outflow -X3-end outflow [Output] vtk 0.5 diff --git a/test/MHD/OrszagTang/idefix-hlld-hlld.ini b/test/MHD/OrszagTang/idefix-hlld-hlld.ini index 50e15897..5e158b93 100644 --- a/test/MHD/OrszagTang/idefix-hlld-hlld.ini +++ b/test/MHD/OrszagTang/idefix-hlld-hlld.ini @@ -1,7 +1,6 @@ [Grid] X1-grid 1 0.0 128 u 1.0 X2-grid 1 0.0 128 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.6 @@ -18,8 +17,6 @@ X1-beg periodic X1-end periodic X2-beg periodic X2-end periodic -X3-beg outflow -X3-end outflow [Output] vtk 0.5 diff --git a/test/MHD/OrszagTang/idefix-hlld-uct0.ini b/test/MHD/OrszagTang/idefix-hlld-uct0.ini index fc32e81b..23646b96 100644 --- a/test/MHD/OrszagTang/idefix-hlld-uct0.ini +++ b/test/MHD/OrszagTang/idefix-hlld-uct0.ini @@ -1,7 +1,6 @@ [Grid] X1-grid 1 0.0 128 u 1.0 X2-grid 1 0.0 128 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.6 @@ -18,8 +17,6 @@ X1-beg periodic X1-end periodic X2-beg periodic X2-end periodic -X3-beg outflow -X3-end outflow [Output] vtk 0.5 diff --git a/test/MHD/OrszagTang/idefix-hlld.ini b/test/MHD/OrszagTang/idefix-hlld.ini index cf0b0c09..cf6d5cbf 100644 --- a/test/MHD/OrszagTang/idefix-hlld.ini +++ b/test/MHD/OrszagTang/idefix-hlld.ini @@ -1,7 +1,6 @@ [Grid] X1-grid 1 0.0 128 u 1.0 X2-grid 1 0.0 128 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.6 @@ -17,8 +16,6 @@ X1-beg periodic X1-end periodic X2-beg periodic X2-end periodic -X3-beg outflow -X3-end outflow [Output] vtk 0.5 diff --git a/test/MHD/OrszagTang/idefix-tvdlf.ini b/test/MHD/OrszagTang/idefix-tvdlf.ini index 47a05bb9..ec9ffe85 100644 --- a/test/MHD/OrszagTang/idefix-tvdlf.ini +++ b/test/MHD/OrszagTang/idefix-tvdlf.ini @@ -1,7 +1,6 @@ [Grid] X1-grid 1 0.0 128 u 1.0 X2-grid 1 0.0 128 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.6 @@ -17,8 +16,6 @@ X1-beg periodic X1-end periodic X2-beg periodic X2-end periodic -X3-beg outflow -X3-end outflow [Output] vtk 0.5 diff --git a/test/MHD/OrszagTang/idefix.ini b/test/MHD/OrszagTang/idefix.ini index e3afb65b..0e742184 100644 --- a/test/MHD/OrszagTang/idefix.ini +++ b/test/MHD/OrszagTang/idefix.ini @@ -1,7 +1,6 @@ [Grid] X1-grid 1 0.0 128 u 1.0 X2-grid 1 0.0 128 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.6 @@ -17,8 +16,6 @@ X1-beg periodic X1-end periodic X2-beg periodic X2-end periodic -X3-beg outflow -X3-end outflow [Output] vtk 0.5 diff --git a/test/MHD/ResistiveAlfvenWave/idefix-rkl.ini b/test/MHD/ResistiveAlfvenWave/idefix-rkl.ini index 29299b06..eed2139f 100644 --- a/test/MHD/ResistiveAlfvenWave/idefix-rkl.ini +++ b/test/MHD/ResistiveAlfvenWave/idefix-rkl.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 128 u 1.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.9 @@ -16,10 +14,6 @@ resistivity rkl constant 0.05 [Boundary] X1-beg periodic X1-end periodic -X2-beg periodic -X2-end periodic -X3-beg periodic -X3-end periodic [Output] # vtk 0.1 diff --git a/test/MHD/ResistiveAlfvenWave/idefix.ini b/test/MHD/ResistiveAlfvenWave/idefix.ini index 60873036..b09cb0a6 100644 --- a/test/MHD/ResistiveAlfvenWave/idefix.ini +++ b/test/MHD/ResistiveAlfvenWave/idefix.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 128 u 1.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.9 @@ -16,10 +14,6 @@ resistivity explicit constant 0.05 [Boundary] X1-beg periodic X1-end periodic -X2-beg periodic -X2-end periodic -X3-beg periodic -X3-end periodic [Output] # vtk 0.1 diff --git a/test/MHD/RotorCartesian/idefix.ini b/test/MHD/RotorCartesian/idefix.ini index ecac9205..54d785ab 100644 --- a/test/MHD/RotorCartesian/idefix.ini +++ b/test/MHD/RotorCartesian/idefix.ini @@ -1,7 +1,6 @@ [Grid] X1-grid 1 -0.5 512 u 0.5 X2-grid 1 -0.5 512 u 0.5 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.2 @@ -19,8 +18,6 @@ X1-beg outflow X1-end outflow X2-beg outflow X2-end outflow -X3-beg outflow -X3-end outflow [Output] vtk 0.01 -1 single_file diff --git a/test/MHD/RotorPolar/idefix.ini b/test/MHD/RotorPolar/idefix.ini index b893ed12..ca88369b 100644 --- a/test/MHD/RotorPolar/idefix.ini +++ b/test/MHD/RotorPolar/idefix.ini @@ -1,7 +1,6 @@ [Grid] X1-grid 1 0.05 256 u 0.5 X2-grid 1 0.0 1024 u 6.283185307179586 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.2 @@ -19,8 +18,6 @@ X1-beg userdef X1-end outflow X2-beg periodic X2-end periodic -X3-beg outflow -X3-end outflow [Output] uservar Vx Vy diff --git a/test/MHD/sod-iso/idefix-hll.ini b/test/MHD/sod-iso/idefix-hll.ini index be20a405..52af4cfe 100644 --- a/test/MHD/sod-iso/idefix-hll.ini +++ b/test/MHD/sod-iso/idefix-hll.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 800 u 100.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 @@ -16,10 +14,6 @@ csiso constant 1.0 [Boundary] X1-beg outflow X1-end outflow -X2-beg periodic -X2-end periodic -X3-beg periodic -X3-end periodic [Output] vtk 10.0 diff --git a/test/MHD/sod-iso/idefix-hlld-rk3.ini b/test/MHD/sod-iso/idefix-hlld-rk3.ini index aa0723e7..2390b88f 100644 --- a/test/MHD/sod-iso/idefix-hlld-rk3.ini +++ b/test/MHD/sod-iso/idefix-hlld-rk3.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 800 u 100.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 @@ -16,10 +14,6 @@ csiso constant 1.0 [Boundary] X1-beg outflow X1-end outflow -X2-beg periodic -X2-end periodic -X3-beg periodic -X3-end periodic [Output] vtk 10.0 diff --git a/test/MHD/sod-iso/idefix-hlld.ini b/test/MHD/sod-iso/idefix-hlld.ini index f9acb7aa..1a8cfee9 100644 --- a/test/MHD/sod-iso/idefix-hlld.ini +++ b/test/MHD/sod-iso/idefix-hlld.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 800 u 100.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 @@ -16,10 +14,6 @@ csiso constant 1.0 [Boundary] X1-beg outflow X1-end outflow -X2-beg periodic -X2-end periodic -X3-beg periodic -X3-end periodic [Output] vtk 10.0 diff --git a/test/MHD/sod-iso/idefix-rk3.ini b/test/MHD/sod-iso/idefix-rk3.ini index 6e6cd525..602f3492 100644 --- a/test/MHD/sod-iso/idefix-rk3.ini +++ b/test/MHD/sod-iso/idefix-rk3.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 800 u 100.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 @@ -16,10 +14,6 @@ csiso constant 1.0 [Boundary] X1-beg outflow X1-end outflow -X2-beg periodic -X2-end periodic -X3-beg periodic -X3-end periodic [Output] vtk 10.0 diff --git a/test/MHD/sod-iso/idefix-tvdlf.ini b/test/MHD/sod-iso/idefix-tvdlf.ini index 32a33959..97195411 100644 --- a/test/MHD/sod-iso/idefix-tvdlf.ini +++ b/test/MHD/sod-iso/idefix-tvdlf.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 800 u 100.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 @@ -16,10 +14,6 @@ csiso constant 1.0 [Boundary] X1-beg outflow X1-end outflow -X2-beg periodic -X2-end periodic -X3-beg periodic -X3-end periodic [Output] vtk 10.0 diff --git a/test/MHD/sod-iso/idefix.ini b/test/MHD/sod-iso/idefix.ini index 075eb2ae..134359d5 100644 --- a/test/MHD/sod-iso/idefix.ini +++ b/test/MHD/sod-iso/idefix.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 800 u 100.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 @@ -16,10 +14,6 @@ csiso constant 1.0 [Boundary] X1-beg outflow X1-end outflow -X2-beg periodic -X2-end periodic -X3-beg periodic -X3-end periodic [Output] vtk 10.0 diff --git a/test/MHD/sod/idefix-hll.ini b/test/MHD/sod/idefix-hll.ini index c6c5dbb1..e353fa75 100644 --- a/test/MHD/sod/idefix-hll.ini +++ b/test/MHD/sod/idefix-hll.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 800 u 100.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 @@ -18,10 +16,6 @@ gamma 1.66666666666667 # not used X1-beg outflow X1-end outflow -X2-beg periodic -X2-end periodic -X3-beg periodic -X3-end periodic [Output] vtk 10.0 diff --git a/test/MHD/sod/idefix-hlld-rk3.ini b/test/MHD/sod/idefix-hlld-rk3.ini index 68aeee11..02925c52 100644 --- a/test/MHD/sod/idefix-hlld-rk3.ini +++ b/test/MHD/sod/idefix-hlld-rk3.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 800 u 100.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 @@ -18,10 +16,6 @@ gamma 1.66666666666667 # not used X1-beg outflow X1-end outflow -X2-beg periodic -X2-end periodic -X3-beg periodic -X3-end periodic [Output] vtk 10.0 diff --git a/test/MHD/sod/idefix-hlld.ini b/test/MHD/sod/idefix-hlld.ini index cfbd486e..d8111420 100644 --- a/test/MHD/sod/idefix-hlld.ini +++ b/test/MHD/sod/idefix-hlld.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 800 u 100.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 @@ -18,10 +16,6 @@ gamma 1.66666666666667 # not used X1-beg outflow X1-end outflow -X2-beg periodic -X2-end periodic -X3-beg periodic -X3-end periodic [Output] vtk 10.0 diff --git a/test/MHD/sod/idefix-rk3.ini b/test/MHD/sod/idefix-rk3.ini index c1926956..b074218e 100644 --- a/test/MHD/sod/idefix-rk3.ini +++ b/test/MHD/sod/idefix-rk3.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 800 u 100.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 @@ -16,10 +14,6 @@ gamma 1.66666666666667 [Boundary] X1-beg outflow X1-end outflow -X2-beg periodic -X2-end periodic -X3-beg periodic -X3-end periodic [Output] vtk 10.0 diff --git a/test/MHD/sod/idefix-tvdlf.ini b/test/MHD/sod/idefix-tvdlf.ini index 4bbc10c4..305cc2fc 100644 --- a/test/MHD/sod/idefix-tvdlf.ini +++ b/test/MHD/sod/idefix-tvdlf.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 800 u 100.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 @@ -18,10 +16,6 @@ gamma 1.66666666666667 # not used X1-beg outflow X1-end outflow -X2-beg periodic -X2-end periodic -X3-beg periodic -X3-end periodic [Output] vtk 10.0 diff --git a/test/MHD/sod/idefix.ini b/test/MHD/sod/idefix.ini index 67741314..39c846dc 100644 --- a/test/MHD/sod/idefix.ini +++ b/test/MHD/sod/idefix.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 800 u 100.0 -X2-grid 1 0.0 1 u 1.0 -X3-grid 1 0.0 1 u 1.0 [TimeIntegrator] CFL 0.8 @@ -16,10 +14,6 @@ gamma 1.66666666666667 [Boundary] X1-beg outflow X1-end outflow -X2-beg periodic -X2-end periodic -X3-beg periodic -X3-end periodic [Output] vtk 10.0 diff --git a/test/Planet/Planet3Body/idefix.ini b/test/Planet/Planet3Body/idefix.ini index a8b7bea0..4e354bf9 100644 --- a/test/Planet/Planet3Body/idefix.ini +++ b/test/Planet/Planet3Body/idefix.ini @@ -1,7 +1,6 @@ [Grid] -X1-grid 1 0.042 64 u 2.3 -X2-grid 1 0.0 32 u 6.283185307179586 -X3-grid 1 -0.00125 1 u 0.00125 +X1-grid 1 0.042 64 u 2.3 +X2-grid 1 0.0 32 u 6.283185307179586 [TimeIntegrator] CFL 0.5 @@ -26,8 +25,6 @@ X1-beg userdef X1-end userdef X2-beg periodic X2-end periodic -X3-beg outflow -X3-end outflow [Setup] sigma0 0.001 diff --git a/test/Planet/PlanetMigration2D/idefix.ini b/test/Planet/PlanetMigration2D/idefix.ini index 2be56b1a..04259b9e 100644 --- a/test/Planet/PlanetMigration2D/idefix.ini +++ b/test/Planet/PlanetMigration2D/idefix.ini @@ -1,7 +1,6 @@ [Grid] -X1-grid 1 0.42 192 u 2.14 -X2-grid 1 0.0 768 u 6.283185307179586 -X3-grid 1 -0.0125 1 u 0.0125 +X1-grid 1 0.42 192 u 2.14 +X2-grid 1 0.0 768 u 6.283185307179586 [TimeIntegrator] CFL 0.5 @@ -29,8 +28,6 @@ X1-beg userdef X1-end userdef X2-beg periodic X2-end periodic -X3-beg outflow -X3-end outflow [Setup] sigma0 0.001 diff --git a/test/Planet/PlanetPlanetRK42D/idefix.ini b/test/Planet/PlanetPlanetRK42D/idefix.ini index cbcf3c29..bf9a6bda 100644 --- a/test/Planet/PlanetPlanetRK42D/idefix.ini +++ b/test/Planet/PlanetPlanetRK42D/idefix.ini @@ -1,7 +1,6 @@ [Grid] -X1-grid 1 0.42 96 u 2.14 -X2-grid 1 0.0 384 u 6.283185307179586 -X3-grid 1 -0.0125 1 u 0.0125 +X1-grid 1 0.42 96 u 2.14 +X2-grid 1 0.0 384 u 6.283185307179586 [TimeIntegrator] CFL 0.5 @@ -24,8 +23,6 @@ X1-beg userdef X1-end userdef X2-beg periodic X2-end periodic -X3-beg outflow -X3-end outflow [Setup] sigma0 0.001 diff --git a/test/Planet/PlanetSpiral2D/idefix.ini b/test/Planet/PlanetSpiral2D/idefix.ini index 9864a223..b030e4a4 100644 --- a/test/Planet/PlanetSpiral2D/idefix.ini +++ b/test/Planet/PlanetSpiral2D/idefix.ini @@ -1,7 +1,6 @@ [Grid] -X1-grid 1 0.42 128 u 2.14 -X2-grid 1 0.0 512 u 6.283185307179586 -X3-grid 1 -0.0125 1 u 0.0125 +X1-grid 1 0.42 128 u 2.14 +X2-grid 1 0.0 512 u 6.283185307179586 [TimeIntegrator] CFL 0.5 @@ -28,8 +27,6 @@ X1-beg userdef X1-end userdef X2-beg periodic X2-end periodic -X3-beg outflow -X3-end outflow [Setup] sigma0 1.0 diff --git a/test/Planet/PlanetsIsActiveRK52D/idefix-rk4.ini b/test/Planet/PlanetsIsActiveRK52D/idefix-rk4.ini index c714362f..7fbedc65 100644 --- a/test/Planet/PlanetsIsActiveRK52D/idefix-rk4.ini +++ b/test/Planet/PlanetsIsActiveRK52D/idefix-rk4.ini @@ -1,7 +1,6 @@ [Grid] -X1-grid 1 0.42 48 u 2.14 -X2-grid 1 0.0 192 u 6.283185307179586 -X3-grid 1 -0.0125 1 u 0.0125 +X1-grid 1 0.42 48 u 2.14 +X2-grid 1 0.0 192 u 6.283185307179586 [TimeIntegrator] CFL 0.5 @@ -24,8 +23,6 @@ X1-beg userdef X1-end userdef X2-beg periodic X2-end periodic -X3-beg outflow -X3-end outflow [Setup] sigma0 0.001 diff --git a/test/Planet/PlanetsIsActiveRK52D/idefix-rk5.ini b/test/Planet/PlanetsIsActiveRK52D/idefix-rk5.ini index 8f1f8a11..44b22327 100644 --- a/test/Planet/PlanetsIsActiveRK52D/idefix-rk5.ini +++ b/test/Planet/PlanetsIsActiveRK52D/idefix-rk5.ini @@ -1,7 +1,6 @@ [Grid] -X1-grid 1 0.42 48 u 2.14 -X2-grid 1 0.0 192 u 6.283185307179586 -X3-grid 1 -0.0125 1 u 0.0125 +X1-grid 1 0.42 48 u 2.14 +X2-grid 1 0.0 192 u 6.283185307179586 [TimeIntegrator] CFL 0.5 @@ -24,8 +23,6 @@ X1-beg userdef X1-end userdef X2-beg periodic X2-end periodic -X3-beg outflow -X3-end outflow [Setup] sigma0 0.001 diff --git a/test/SelfGravity/DustyCollapse/idefix.ini b/test/SelfGravity/DustyCollapse/idefix.ini index 37976368..d650cc88 100644 --- a/test/SelfGravity/DustyCollapse/idefix.ini +++ b/test/SelfGravity/DustyCollapse/idefix.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.1 4096 l 3e4 -X2-grid 1 0 64 u 3.14159265358979 -X3-grid 1 0 64 u 6.28318530717958 [TimeIntegrator] CFL 0.2 @@ -25,18 +23,10 @@ targetError 1e-7 maxIter 10000 boundary-X1-beg nullgrad boundary-X1-end nullpot -boundary-X2-beg axis -boundary-X2-end axis -boundary-X3-beg periodic -boundary-X3-end periodic [Boundary] X1-beg outflow X1-end outflow -X2-beg periodic -X2-end periodic -X3-beg periodic -X3-end periodic [Output] dmp 1 diff --git a/test/SelfGravity/JeansInstability/idefix-cg.ini b/test/SelfGravity/JeansInstability/idefix-cg.ini index 97f1e6c9..3b40ba82 100644 --- a/test/SelfGravity/JeansInstability/idefix-cg.ini +++ b/test/SelfGravity/JeansInstability/idefix-cg.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 1000 u 10.0 -X2-grid 1 0.0 100 u 10.0 -X3-grid 1 0.0 100 u 10.0 [TimeIntegrator] CFL 0.8 @@ -24,18 +22,10 @@ solver CG targetError 1e-6 boundary-X1-beg periodic boundary-X1-end periodic -boundary-X2-beg periodic -boundary-X2-end periodic -boundary-X3-beg periodic -boundary-X3-end periodic [Boundary] X1-beg periodic X1-end periodic -X2-beg periodic -X2-end periodic -X3-beg periodic -X3-end periodic [Output] vtk 0.1 diff --git a/test/SelfGravity/JeansInstability/idefix.ini b/test/SelfGravity/JeansInstability/idefix.ini index 47101406..c21b52b5 100644 --- a/test/SelfGravity/JeansInstability/idefix.ini +++ b/test/SelfGravity/JeansInstability/idefix.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 0.0 1000 u 10.0 -X2-grid 1 0.0 100 u 10.0 -X3-grid 1 0.0 100 u 10.0 [TimeIntegrator] CFL 0.8 @@ -24,18 +22,10 @@ targetError 1e-6 # skip 2 boundary-X1-beg periodic boundary-X1-end periodic -boundary-X2-beg periodic -boundary-X2-end periodic -boundary-X3-beg periodic -boundary-X3-end periodic [Boundary] X1-beg periodic X1-end periodic -X2-beg periodic -X2-end periodic -X3-beg periodic -X3-end periodic [Output] vtk 0.1 diff --git a/test/SelfGravity/UniformCollapse/idefix.ini b/test/SelfGravity/UniformCollapse/idefix.ini index d5f8e4a4..e0e74a36 100644 --- a/test/SelfGravity/UniformCollapse/idefix.ini +++ b/test/SelfGravity/UniformCollapse/idefix.ini @@ -1,7 +1,5 @@ [Grid] X1-grid 1 .01 100 l 1000. -X2-grid 1 0.0 20 u 3.141592653589793 -X3-grid 1 0.0 20 u 6.283185307179586 [TimeIntegrator] CFL 0.8 @@ -25,18 +23,10 @@ skip 5 targetError 1e-6 boundary-X1-beg origin boundary-X1-end nullpot -boundary-X2-beg periodic -boundary-X2-end periodic -boundary-X3-beg periodic -boundary-X3-end periodic [Boundary] X1-beg userdef X1-end outflow -X2-beg periodic -X2-end periodic -X3-beg periodic -X3-end periodic [Output] analysis 10. From 0b79b28fe6525f298c78b2ad8316c145b2ef52c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sat, 19 Oct 2024 22:38:17 +0200 Subject: [PATCH 31/83] MNT: auto-upgrade pre-commit hooks (#278) --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 06fe7210..3d351a7c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ exclude: "^(src/kokkos)" repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v5.0.0 hooks: - id: trailing-whitespace # auto-fix trailing whitespaces - id: end-of-file-fixer # add EOF "\n" if missing @@ -23,7 +23,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.278 + rev: v0.6.9 hooks: - id: ruff args: @@ -35,12 +35,12 @@ repos: - I # isort - repo: https://github.com/neutrinoceros/inifix - rev: v4.4.0 + rev: v5.0.2 hooks: - id: inifix-format - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: v1.5.1 + rev: v1.5.5 hooks: - id: remove-tabs # auto-fix tab/space mixing - id: insert-license From 0c3ce8a06f8837b5dd48c2f9e330e29f3c18a1b2 Mon Sep 17 00:00:00 2001 From: marc Date: Fri, 18 Oct 2024 15:01:33 +0200 Subject: [PATCH 32/83] Update to numpy 2+ --- pytools/idfx_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytools/idfx_io.py b/pytools/idfx_io.py index 480e713d..17dc6272 100644 --- a/pytools/idfx_io.py +++ b/pytools/idfx_io.py @@ -29,7 +29,7 @@ def __init__(self, fh, byteorder="little"): dims = [] for dim in range(self.ndims): dims.append(int.from_bytes(fh.read(INT_SIZE), byteorder)) - ntot = int(np.product(dims)) + ntot = int(np.prod(dims)) raw = struct.unpack(str(ntot) + "d", fh.read(DOUBLE_SIZE * ntot)) self.array = np.asarray(raw).reshape(dims[::-1]) From 6c95183e17142a014ab6916e7edc68c8f6c9ffb8 Mon Sep 17 00:00:00 2001 From: marc Date: Mon, 21 Oct 2024 10:46:43 +0200 Subject: [PATCH 33/83] Included numpy to ruff --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3d351a7c..726f0045 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,6 +33,7 @@ repos: - F # pyflakes - B # flake8-bugbear - I # isort + - NPY # numpy-specific rules - repo: https://github.com/neutrinoceros/inifix rev: v5.0.2 From 45fcfd6646d3e9958b07a217b880cdaf731d09fc Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 22 Oct 2024 09:50:48 +0200 Subject: [PATCH 34/83] fix arch for nvidia architectures --- doc/source/reference/makefile.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/reference/makefile.rst b/doc/source/reference/makefile.rst index bf8ecdad..094b7533 100644 --- a/doc/source/reference/makefile.rst +++ b/doc/source/reference/makefile.rst @@ -146,13 +146,13 @@ We recommend the following modules and environement variables on Jean Zay: .. code-block:: bash - -DKokkos_ENABLE_CUDA=ON -DKokkos_ENABLE_VOLTA70=ON -DKokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC=OFF + -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_VOLTA70=ON -DKokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC=OFF While Ampere A100 GPUs are enabled with .. code-block:: bash - -DKokkos_ENABLE_CUDA=ON -DKokkos_ENABLE_AMPERE80=ON -DKokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC=OFF + -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_AMPERE80=ON -DKokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC=OFF MPI (multi-GPU) can be enabled by adding ``-DIdefix_MPI=ON`` as usual. The malloc async option is here to prevent a bug when using PSM2 with async Cuda malloc possibly leading to openmpi crash or hangs on Jean Zay. From d9466abaee07cba22f4fcd1da3216bc304b5e2d6 Mon Sep 17 00:00:00 2001 From: Antonin Borderies <89980449+Anto6453@users.noreply.github.com> Date: Tue, 22 Oct 2024 23:00:45 +0200 Subject: [PATCH 35/83] add central mass in the restart dump. (#287) --- src/gravity/gravity.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/gravity/gravity.cpp b/src/gravity/gravity.cpp index 2492ce09..c0096c51 100644 --- a/src/gravity/gravity.cpp +++ b/src/gravity/gravity.cpp @@ -10,6 +10,7 @@ #include "gravity.hpp" #include "planetarySystem.hpp" #include "dataBlock.hpp" +#include "output.hpp" #include "input.hpp" Gravity::Gravity(Input &input, DataBlock *datain) { @@ -29,6 +30,7 @@ Gravity::Gravity(Input &input, DataBlock *datain) { } else if (potentialString.compare("central") == 0) { this->haveCentralMassPotential = true; this->centralMass = input.GetOrSet("Gravity","Mcentral",0, 1.0); + data->dump->RegisterVariable(&this->centralMass, "centralMass",1); } else if (potentialString.compare("selfgravity") == 0) { this->haveSelfGravityPotential = true; } else if (potentialString.compare("planet") == 0) { From ec0594df4b9dfc694d789b990084f53c8652eb2d Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 23 Oct 2024 09:47:38 +0200 Subject: [PATCH 36/83] Reduce roundoff error on div(B) when using grid coarsening (#284) * reduce roundoff error when using coarsening * update reference tag for new grid coarsening --- reference | 2 +- src/fluid/coarsenFlow.hpp | 42 ++++++++++++++++++++++++++++++--------- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/reference b/reference index 99f339c4..609a6016 160000 --- a/reference +++ b/reference @@ -1 +1 @@ -Subproject commit 99f339c43d98274a925282cc96252c5a0fa9374f +Subproject commit 609a60164461fb2d4fbeffa89ce148acf1191525 diff --git a/src/fluid/coarsenFlow.hpp b/src/fluid/coarsenFlow.hpp index 217c7711..b1bf79ea 100644 --- a/src/fluid/coarsenFlow.hpp +++ b/src/fluid/coarsenFlow.hpp @@ -290,28 +290,52 @@ void Fluid::CoarsenMagField(IdefixArray4D &Vsin) { // We reconstruct the parrallel field component with divB=0 int factor = 1 << (coarsening-1); + + if( (index-begDir)%factor == 0) { + real qt = 0; + real qb = 0; + + // Loop forward for(int shift = 0 ; shift < factor-1 ; shift++) { - real qt = Vsin(BXt,k+kt+shift*koffset, j+jt+shift*joffset, i+it+shift*ioffset) + qt += Vsin(BXt,k+kt+shift*koffset, j+jt+shift*joffset, i+it+shift*ioffset) * At(k+kt+shift*koffset, j+jt+shift*joffset, i+it+shift*ioffset) - Vsin(BXt,k+shift*koffset, j+shift*joffset, i+shift*ioffset) * At(k+shift*koffset, j+shift*joffset, i+shift*ioffset); #if DIMENSIONS == 3 - real qb = Vsin(BXb,k+kb+shift*koffset, j+jb+shift*joffset, i+ib+shift*ioffset) + qb += Vsin(BXb,k+kb+shift*koffset, j+jb+shift*joffset, i+ib+shift*ioffset) * Ab(k+kb+shift*koffset, j+jb+shift*joffset, i+ib+shift*ioffset) - Vsin(BXb,k+shift*koffset, j+shift*joffset, i+shift*ioffset) * Ab(k+shift*koffset, j+shift*joffset, i+shift*ioffset); - #else - real qb = 0.0; #endif Vsin(BXn, k+(shift+1)*koffset, j+(shift+1)*joffset, i+(shift+1)*ioffset) = + ((real) (factor-(shift+1))) / ((real)factor) * 1.0/An(k+(shift+1)*koffset, j+(shift+1)*joffset, i+(shift+1)*ioffset) * - ( - Vsin(BXn, k+shift*koffset, j+shift*joffset, i+shift*ioffset) * - An(k+shift*koffset, j+shift*joffset, i+shift*ioffset) - - qt - qb - ); + (Vsin(BXn, k, j, i) * An(k, j, i) - qt - qb); + } + // Loop backward + qt = 0; + qb = 0; + + // Loop forward + for(int shift = factor-1 ; shift >=1 ; shift--) { + qt += Vsin(BXt,k+kt+shift*koffset, j+jt+shift*joffset, i+it+shift*ioffset) + * At(k+kt+shift*koffset, j+jt+shift*joffset, i+it+shift*ioffset) + - Vsin(BXt,k+shift*koffset, j+shift*joffset, i+shift*ioffset) + * At(k+shift*koffset, j+shift*joffset, i+shift*ioffset); + #if DIMENSIONS == 3 + qb += Vsin(BXb,k+kb+shift*koffset, j+jb+shift*joffset, i+ib+shift*ioffset) + * Ab(k+kb+shift*koffset, j+jb+shift*joffset, i+ib+shift*ioffset) + - Vsin(BXb,k+shift*koffset, j+shift*joffset, i+shift*ioffset) + * Ab(k+shift*koffset, j+shift*joffset, i+shift*ioffset); + #endif + + Vsin(BXn, k+shift*koffset, j+shift*joffset, i+shift*ioffset) += + ((real) shift) / ((real)factor) * + 1.0/An(k+shift*koffset, j+shift*joffset, i+shift*ioffset) * + (Vsin(BXn, k+factor*koffset, j+factor*joffset, i+factor*ioffset) + * An( k+factor*koffset, j+factor*joffset, i+factor*ioffset) + qt + qb); } } } From 46d19732f2a021acfc527765fb350c2e4abf60a2 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 24 Oct 2024 09:36:46 +0200 Subject: [PATCH 37/83] update JZ documentation --- doc/source/reference/makefile.rst | 40 ++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/doc/source/reference/makefile.rst b/doc/source/reference/makefile.rst index 094b7533..4f948e5d 100644 --- a/doc/source/reference/makefile.rst +++ b/doc/source/reference/makefile.rst @@ -130,32 +130,56 @@ Finally, *Idefix* can be configured to run on Mi250 by enabling HIP and the desi MPI (multi-GPU) can be enabled by adding ``-DIdefix_MPI=ON`` as usual. -Jean Zay at IDRIS, Nvidia V100 and A100 GPUs --------------------------------------------- +Jean Zay at IDRIS, Nvidia V100/A100/H100 GPUs +--------------------------------------------- -We recommend the following modules and environement variables on Jean Zay: +We recommend the following modules and environement variables on Jean Zay V100/A100: .. code-block:: bash + module load arch/a100 # ONLY forA100 module load cuda/12.1.0 module load gcc/12.2.0 module load openmpi/4.1.1-cuda - module load cmake/3.18.0 + module load cmake/3.25.2 + +While for H100: + +.. code-block:: bash + + module load arch/h100 + module load cmake/3.30.1 + module load cuda/12.1.0 + module load openmpi/4.1.5-cuda *Idefix* can then be configured to run on Nvidia V100 with the following options to ccmake: .. code-block:: bash - -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_VOLTA70=ON -DKokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC=OFF + -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_VOLTA70=ON While Ampere A100 GPUs are enabled with .. code-block:: bash - -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_AMPERE80=ON -DKokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC=OFF + -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_AMPERE80=ON + +And for H100 GPUS: + +.. code-block:: bash + + -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_HOPPER90=ON + + +MPI (multi-GPU) can be enabled by adding ``-DIdefix_MPI=ON`` as usual. + + +.. warning:: + + As of *Idefix* 2.1.02, we automatically disable Cuda Malloc async (``-DKokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC=OFF``). However, earlier versions of + *Idefix* requires this flag when calling cmake to prevent a bug when using PSM2 with async Cuda malloc possibly leading to openmpi crash or hangs on Jean Zay. + -MPI (multi-GPU) can be enabled by adding ``-DIdefix_MPI=ON`` as usual. The malloc async option is here to prevent a bug when using PSM2 with async -Cuda malloc possibly leading to openmpi crash or hangs on Jean Zay. .. _setupSpecificOptions: From a2a0189d554c5fce1f43c7702b6995342097d65c Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 24 Oct 2024 09:37:45 +0200 Subject: [PATCH 38/83] -move to Kokkos 4.4 (#289) -prevent Async malloc automatically (cf issues with JZ) -update include dir for Kokkos as recommended in the documentation --- CMakeLists.txt | 2 ++ src/kokkos | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2ef5e686..d960102b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,10 +45,12 @@ include(SetRequiredBuildSettingsForGCC8) #Idefix requires Cuda Lambdas (experimental) if(Kokkos_ENABLE_CUDA) set(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "Idefix requires lambdas on Cuda" FORCE) + set(Kokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC OFF CACHE BOOL "Disable Async malloc to avoid bugs on PSM2" FORCE) endif() # Add kokkos CMAKE files (required early since these set compiler options) add_subdirectory(src/kokkos build/kokkos) +include_directories(${Kokkos_INCLUDE_DIRS_RET}) # Add Idefix CXX Flags add_compile_options(${Idefix_CXX_FLAGS}) diff --git a/src/kokkos b/src/kokkos index 6ecdf605..15dc143e 160000 --- a/src/kokkos +++ b/src/kokkos @@ -1 +1 @@ -Subproject commit 6ecdf605e0f7639adec599d25cf0e206d7b8f9f5 +Subproject commit 15dc143e5f39949eece972a798e175c4b463d4b8 From 2efed9c7b5d001eacecf6140bd36cb650c9ffa0b Mon Sep 17 00:00:00 2001 From: vdbma <93188557+vdbma@users.noreply.github.com> Date: Fri, 25 Oct 2024 13:48:16 +0200 Subject: [PATCH 39/83] Adding SG potential to idfx files (#285) * Added SG Pot to idfx files * consistent var names --- src/dataBlock/dumpToFile.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/dataBlock/dumpToFile.cpp b/src/dataBlock/dumpToFile.cpp index 98b4a474..35b60b81 100644 --- a/src/dataBlock/dumpToFile.cpp +++ b/src/dataBlock/dumpToFile.cpp @@ -92,6 +92,16 @@ void DataBlock::DumpToFile(std::string filebase) { WriteVariable(fileHdl, 4, dims, fieldName, locVc.data()); + if (this->gravity->haveSelfGravityPotential) { + IdefixArray3D::HostMirror locPot = Kokkos::create_mirror_view(this->gravity->phiP); + Kokkos::deep_copy(locPot, this->gravity->phiP); + + dims[3] = 1; + std::snprintf(fieldName,NAMESIZE,"Pot"); + + WriteVariable(fileHdl, 4, dims, fieldName, locPot.data()); + } + // Write Flux /* nx1=this->np_tot[IDIR]; From 9a7716d29d09d93c9e2e646103caa3b37ecd8142 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 6 Nov 2024 09:29:12 +0100 Subject: [PATCH 40/83] fix MakeVsFromAmag in spherical-non axisymmetric runs (#293) fix field computation for non-axisymmetric vector potential in spherical geometry that led to non-zero divergence --- src/dataBlock/dataBlockHost.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dataBlock/dataBlockHost.cpp b/src/dataBlock/dataBlockHost.cpp index b5dff915..cfc83a3a 100644 --- a/src/dataBlock/dataBlockHost.cpp +++ b/src/dataBlock/dataBlockHost.cpp @@ -223,8 +223,8 @@ void DataBlockHost::MakeVsFromAmag(IdefixHostArray4D &Ain) { + 1/(x1m(i)*(cos(x2m(j)) - cos(x2m(j+1)))) * (sin(x2m(j+1))*Ain(KDIR,k,j+1,i) - sin(x2m(j))*Ain(KDIR,k,j,i) ) , - - 1/(x1m(i)*sin(x2(j))*dx3(k)) * (Ain(JDIR,k+1,j,i) - - Ain(JDIR,k,j,i) ) ); + - dx2(j)/(x1m(i)*((cos(x2m(j))- cos(x2m(j+1))))*dx3(k)) + * (Ain(JDIR,k+1,j,i) - Ain(JDIR,k,j,i) ) ); real Ax2m = fabs(sin(x2m(j))); // Regularisation along the axis From 1087e858d72fc99f1e37aa2276bfdb79b3007ac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Tue, 5 Nov 2024 07:48:18 +0100 Subject: [PATCH 41/83] ENH: include native coordinates in vtk outputs Co-authored-by: volodia99 --- pytools/vtk_io.py | 34 +++++++++++++- src/output/vtk.cpp | 108 +++++++++++++++++++++++++++++++++++++++++---- src/output/vtk.hpp | 1 + 3 files changed, 133 insertions(+), 10 deletions(-) diff --git a/pytools/vtk_io.py b/pytools/vtk_io.py index 1f693318..eb5cf412 100644 --- a/pytools/vtk_io.py +++ b/pytools/vtk_io.py @@ -7,7 +7,7 @@ import warnings import numpy as np import os - +import re # restrict what's included with `import *` to public API __all__ = [ @@ -21,6 +21,8 @@ dt = np.dtype(">f") # Big endian single precision floats dint = np.dtype(">i4") # Big endian integer +NATIVE_COORDINATE_REGEXP = re.compile(r"X(1|2|3)(L|C)_NATIVE_COORDINATES") + KNOWN_GEOMETRIES = { 0: "cartesian", 1: "polar", @@ -33,10 +35,14 @@ class VTKDataset(object): def __init__(self, filename, geometry=None): self.filename = os.path.abspath(filename) self.data = {} + self.native_coordinates = {} with open(filename, "rb") as fh: self._load_header(fh, geometry=geometry) self._load(fh) + if self.native_coordinates: + self._setup_coordinates_from_native() + def _load(self, fh): if self._dataset_type in ("RECTILINEAR_GRID", "STRUCTURED_GRID"): self._load_hydro(fh) @@ -88,6 +94,9 @@ def _load_header(self, fh, geometry=None): self.t = np.fromfile(fh, dt, 1) elif d.startswith("PERIODICITY"): self.periodicity = np.fromfile(fh, dtype=dint, count=3).astype(bool) + elif NATIVE_COORDINATE_REGEXP.match(d): + native_name, _ncomp, native_dim, _dtype = d.split() + self.native_coordinates[native_name] = np.fromfile(fh, dtype=dt, count=int(native_dim)) else: warnings.warn("Found unknown field %s" % d) fh.readline() # skip extra linefeed (empty line) @@ -341,6 +350,29 @@ def _load_hydro(self, fh): def _load_particles(self, fh): raise NotImplementedError("Particles vtk are not supported yet !") + def _setup_coordinates_from_native(self): + if self.geometry == "spherical": + native2attr = { + "X1L_NATIVE_COORDINATES": "rl", + "X1C_NATIVE_COORDINATES": "r", + "X2L_NATIVE_COORDINATES": "thetal", + "X2C_NATIVE_COORDINATES": "theta", + "X3L_NATIVE_COORDINATES": "phil", + "X3C_NATIVE_COORDINATES": "phi", + } + elif self.geometry in ("cartesian", "cylindrical", "polar"): + native2attr = { + "X1L_NATIVE_COORDINATES": "xl", + "X1C_NATIVE_COORDINATES": "x", + "X2L_NATIVE_COORDINATES": "yl", + "X2C_NATIVE_COORDINATES": "y", + "X3L_NATIVE_COORDINATES": "zl", + "X3C_NATIVE_COORDINATES": "z", + } + + for native_field, attr in native2attr.items(): + setattr(self, attr, self.native_coordinates[native_field]) + def __repr__(self): return "VTKDataset('%s')" % self.filename diff --git a/src/output/vtk.cpp b/src/output/vtk.cpp index ae4a4c60..f7460654 100644 --- a/src/output/vtk.cpp +++ b/src/output/vtk.cpp @@ -124,24 +124,51 @@ Vtk::Vtk(Input &input, DataBlock *datain, std::string filebase) { this->xnode = new float[nx1+ioffset]; this->ynode = new float[nx2+joffset]; this->znode = new float[nx3+koffset]; + this->xcenter = new float[nx1]; + this->ycenter = new float[nx2]; + this->zcenter = new float[nx3]; for (int32_t i = 0; i < nx1 + ioffset; i++) { - if(grid.np_tot[IDIR] == 1) // only one dimension in this direction + if(grid.np_tot[IDIR] == 1) { // only one dimension in this direction xnode[i] = bigEndian(static_cast(grid.x[IDIR](i))); - else + } else { xnode[i] = bigEndian(static_cast(grid.xl[IDIR](i + grid.nghost[IDIR]))); + } } for (int32_t j = 0; j < nx2 + joffset; j++) { - if(grid.np_tot[JDIR] == 1) // only one dimension in this direction + if(grid.np_tot[JDIR] == 1) { // only one dimension in this direction ynode[j] = bigEndian(static_cast(grid.x[JDIR](j))); - else + } else { ynode[j] = bigEndian(static_cast(grid.xl[JDIR](j + grid.nghost[JDIR]))); + } } for (int32_t k = 0; k < nx3 + koffset; k++) { - if(grid.np_tot[KDIR] == 1) + if(grid.np_tot[KDIR] == 1) { znode[k] = bigEndian(static_cast(grid.x[KDIR](k))); - else + } else { znode[k] = bigEndian(static_cast(grid.xl[KDIR](k + grid.nghost[KDIR]))); + } + } + for (int32_t i = 0; i < nx1; i++) { + if(grid.np_tot[IDIR] == 1) { // only one dimension in this direction + xcenter[i] = xnode[i]; + } else { + xcenter[i] = bigEndian(static_cast(grid.x[IDIR](i + grid.nghost[IDIR]))); + } + } + for (int32_t j = 0; j < nx2; j++) { + if(grid.np_tot[JDIR] == 1) { // only one dimension in this direction + ycenter[j] = ynode[j]; + } else { + ycenter[j] = bigEndian(static_cast(grid.x[JDIR](j + grid.nghost[JDIR]))); + } + } + for (int32_t k = 0; k < nx3; k++) { + if(grid.np_tot[KDIR] == 1) { + zcenter[k] = znode[k]; + } else { + zcenter[k] = bigEndian(static_cast(grid.x[KDIR](k + grid.nghost[KDIR]))); + } } #if VTK_FORMAT == VTK_STRUCTURED_GRID // VTK_FORMAT /* -- Allocate memory for node_coord which is later used -- */ @@ -375,8 +402,8 @@ void Vtk::WriteHeader(IdfxFileHandler fvtk, real time) { #elif VTK_FORMAT == VTK_STRUCTURED_GRID ssheader << "DATASET STRUCTURED_GRID" << std::endl; #endif - // fields: geometry, periodicity, time - int nfields = 3; + // fields: geometry, periodicity, time, 6 NativeCoordinates (x1l, x2l, x3l, x1, x2, x3) + int nfields = 9; // Write grid geometry in the VTK file ssheader << "FIELD FieldData " << nfields << std::endl; @@ -426,7 +453,71 @@ void Vtk::WriteHeader(IdfxFileHandler fvtk, real time) { // Done, add cariage return for next ascii write ssheader << std::endl; + // write x1l native coordinates + ssheader << "X1L_NATIVE_COORDINATES 1 " << (nx1 + ioffset) << " float" << std::endl; + // Flush the ascii header + header = ssheader.str(); + WriteHeaderString(header.c_str(), fvtk); + // reset the string stream + ssheader.str(std::string()); + WriteHeaderBinary(this->xnode, nx1 + ioffset, fvtk); + // Done, add cariage return for next ascii write + ssheader << std::endl; + + // write x2l native coordinates + ssheader << "X2L_NATIVE_COORDINATES 1 " << (nx2 + joffset) << " float" << std::endl; + // Flush the ascii header + header = ssheader.str(); + WriteHeaderString(header.c_str(), fvtk); + // reset the string stream + ssheader.str(std::string()); + WriteHeaderBinary(this->ynode, nx2 + joffset, fvtk); + // Done, add cariage return for next ascii write + ssheader << std::endl; + + // write x3l native coordinates + ssheader << "X3L_NATIVE_COORDINATES 1 " << (nx3 + koffset) << " float" << std::endl; + // Flush the ascii header + header = ssheader.str(); + WriteHeaderString(header.c_str(), fvtk); + // reset the string stream + ssheader.str(std::string()); + WriteHeaderBinary(this->znode, nx3 + koffset, fvtk); + // Done, add cariage return for next ascii write + ssheader << std::endl; + + // write x1 native coordinates + ssheader << "X1C_NATIVE_COORDINATES 1 " << nx1 << " float" << std::endl; + // Flush the ascii header + header = ssheader.str(); + WriteHeaderString(header.c_str(), fvtk); + // reset the string stream + ssheader.str(std::string()); + WriteHeaderBinary(this->xcenter, nx1, fvtk); + // Done, add cariage return for next ascii write + ssheader << std::endl; + + // write x2 native coordinates + ssheader << "X2C_NATIVE_COORDINATES 1 " << nx2 << " float" << std::endl; + // Flush the ascii header + header = ssheader.str(); + WriteHeaderString(header.c_str(), fvtk); + // reset the string stream + ssheader.str(std::string()); + WriteHeaderBinary(this->ycenter, nx2, fvtk); + // Done, add cariage return for next ascii write + ssheader << std::endl; + // write x3 native coordinates + ssheader << "X3C_NATIVE_COORDINATES 1 " << nx3 << " float" << std::endl; + // Flush the ascii header + header = ssheader.str(); + WriteHeaderString(header.c_str(), fvtk); + // reset the string stream + ssheader.str(std::string()); + WriteHeaderBinary(this->zcenter, nx3, fvtk); + // Done, add cariage return for next ascii write + ssheader << std::endl; ssheader << "DIMENSIONS " << nx1 + ioffset << " " << nx2 + joffset << " " << nx3 + koffset << std::endl; @@ -486,7 +577,6 @@ void Vtk::WriteHeader(IdfxFileHandler fvtk, real time) { #undef VTK_STRUCTURED_GRID #undef VTK_RECTILINEAR_GRID - /* ********************************************************************* */ void Vtk::WriteScalar(IdfxFileHandler fvtk, float* Vin, const std::string &var_name) { /*! diff --git a/src/output/vtk.hpp b/src/output/vtk.hpp index a9503a5a..1a55f67f 100644 --- a/src/output/vtk.hpp +++ b/src/output/vtk.hpp @@ -129,6 +129,7 @@ class Vtk : public BaseVtk { // Coordinates needed by VTK outputs float *xnode, *ynode, *znode; + float *xcenter, *ycenter, *zcenter; IdefixHostArray4D node_coord; From f68a652f6b4665fa005a228a4354e289a5f9aee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Thu, 7 Nov 2024 20:45:37 +0100 Subject: [PATCH 42/83] TST: handle a deprecation warning from numpy 1.25 (#294) --- test/MHD/AmbipolarCshock/python/testidefix.py | 2 +- test/MHD/AmbipolarCshock3D/python/testidefix.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/MHD/AmbipolarCshock/python/testidefix.py b/test/MHD/AmbipolarCshock/python/testidefix.py index ed622fdf..8c95dbb1 100755 --- a/test/MHD/AmbipolarCshock/python/testidefix.py +++ b/test/MHD/AmbipolarCshock/python/testidefix.py @@ -72,7 +72,7 @@ def f(x,D): Dth=np.zeros(x.shape) for i in range(x.size): r.set_initial_value(DSim[iref],x[iref]) - Dth[i]=r.integrate(x[i]) + Dth[i]=r.integrate(x[i]).item() #print("Dth[%d]=%g"%(i,Dth[i])) bTh=np.sqrt(b0**2+2*A**2*(Dth-1)*(1/Dth-1/M**2)) diff --git a/test/MHD/AmbipolarCshock3D/python/testidefix.py b/test/MHD/AmbipolarCshock3D/python/testidefix.py index 52f10b63..a7354fba 100755 --- a/test/MHD/AmbipolarCshock3D/python/testidefix.py +++ b/test/MHD/AmbipolarCshock3D/python/testidefix.py @@ -75,7 +75,7 @@ def f(x,D): Dth=np.zeros(x.shape) for i in range(x.size): r.set_initial_value(DSim[iref],x[iref]) - Dth[i]=r.integrate(x[i]) + Dth[i]=r.integrate(x[i]).item() #print("Dth[%d]=%g"%(i,Dth[i])) bTh=np.sqrt(b0**2+2*A**2*(Dth-1)*(1/Dth-1/M**2)) From 573512ff4458d8ff0501fcf9171adee2e95a5009 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Fri, 8 Nov 2024 10:06:32 +0100 Subject: [PATCH 43/83] Update src/output/vtk.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Robert --- src/output/vtk.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/output/vtk.cpp b/src/output/vtk.cpp index f7460654..98e4c257 100644 --- a/src/output/vtk.cpp +++ b/src/output/vtk.cpp @@ -402,7 +402,7 @@ void Vtk::WriteHeader(IdfxFileHandler fvtk, real time) { #elif VTK_FORMAT == VTK_STRUCTURED_GRID ssheader << "DATASET STRUCTURED_GRID" << std::endl; #endif - // fields: geometry, periodicity, time, 6 NativeCoordinates (x1l, x2l, x3l, x1, x2, x3) + // fields: geometry, periodicity, time, 6 NativeCoordinates (x1l, x2l, x3l, x1c, x2c, x3c) int nfields = 9; // Write grid geometry in the VTK file From f83e56006a5a0f0686e3e452218a7688fbf02e37 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Fri, 15 Nov 2024 09:32:22 +0100 Subject: [PATCH 44/83] Bug: MHD+Fargo+MPI (#295) fix a bug that lead to segfault when MHD+Fargo+MPI (with X3 domain decomposition) are all activated This went undetected for a while as the test FargoMHDSpherical was actually testing for small fargo shifts (less than a cell). --- reference | 2 +- src/dataBlock/fargo.hpp | 16 ++++++++-------- test/MHD/FargoMHDSpherical/idefix.ini | 6 +++--- test/MHD/FargoMHDSpherical/testme.py | 4 ++++ 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/reference b/reference index 609a6016..b675bcea 160000 --- a/reference +++ b/reference @@ -1 +1 @@ -Subproject commit 609a60164461fb2d4fbeffa89ce148acf1191525 +Subproject commit b675bceaa6aabc01dded346e2d631857f698dc76 diff --git a/src/dataBlock/fargo.hpp b/src/dataBlock/fargo.hpp index 3c3a9401..099c9935 100644 --- a/src/dataBlock/fargo.hpp +++ b/src/dataBlock/fargo.hpp @@ -508,7 +508,7 @@ void Fargo::ShiftFluid(const real t, const real dt, Fluid* hydro) { for(int ss = s-m ; ss < s ; ss++) { int sc; if(haveDomainDecomposition) { - sc = ss; + sc = ss + maxShift; } else { sc = sbeg + modPositive(ss-sbeg,n); } @@ -518,7 +518,7 @@ void Fargo::ShiftFluid(const real t, const real dt, Fluid* hydro) { for(int ss = s ; ss < s-m ; ss++) { int sc; if(haveDomainDecomposition) { - sc = ss; + sc = ss + maxShift; } else { sc = sbeg + modPositive(ss-sbeg,n); } @@ -544,7 +544,7 @@ void Fargo::ShiftFluid(const real t, const real dt, Fluid* hydro) { for(int ss = s-m ; ss < s ; ss++) { int sc; if(haveDomainDecomposition) { - sc = ss; + sc = ss + maxShift; } else { sc = sbeg + modPositive(ss-sbeg,n); } @@ -554,7 +554,7 @@ void Fargo::ShiftFluid(const real t, const real dt, Fluid* hydro) { for(int ss = s ; ss < s-m ; ss++) { int sc; if(haveDomainDecomposition) { - sc = ss; + sc = ss + maxShift; } else { sc = sbeg + modPositive(ss-sbeg,n); } @@ -637,7 +637,7 @@ void Fargo::ShiftFluid(const real t, const real dt, Fluid* hydro) { for(int ss = s-m ; ss < s ; ss++) { int sc; if(haveDomainDecomposition) { - sc = ss; + sc = ss + maxShift; } else { sc = sbeg + modPositive(ss-sbeg,n); } @@ -647,7 +647,7 @@ void Fargo::ShiftFluid(const real t, const real dt, Fluid* hydro) { for(int ss = s ; ss < s-m ; ss++) { int sc; if(haveDomainDecomposition) { - sc = ss; + sc = ss + maxShift; } else { sc = sbeg + modPositive(ss-sbeg,n); } @@ -672,7 +672,7 @@ void Fargo::ShiftFluid(const real t, const real dt, Fluid* hydro) { for(int ss = s-m ; ss < s ; ss++) { int sc; if(haveDomainDecomposition) { - sc = ss; + sc = ss + maxShift; } else { sc = sbeg + modPositive(ss-sbeg,n); } @@ -682,7 +682,7 @@ void Fargo::ShiftFluid(const real t, const real dt, Fluid* hydro) { for(int ss = s ; ss < s-m ; ss++) { int sc; if(haveDomainDecomposition) { - sc = ss; + sc = ss + maxShift; } else { sc = sbeg + modPositive(ss-sbeg,n); } diff --git a/test/MHD/FargoMHDSpherical/idefix.ini b/test/MHD/FargoMHDSpherical/idefix.ini index 4165ebc7..06bfcdf7 100644 --- a/test/MHD/FargoMHDSpherical/idefix.ini +++ b/test/MHD/FargoMHDSpherical/idefix.ini @@ -1,7 +1,7 @@ [Grid] -X1-grid 1 1.0 16 l 2.0 -X2-grid 1 1.2707963267948965 64 u 1.8707963267948966 -X3-grid 1 0.0 64 u 6.283185307179586 +X1-grid 1 1.0 16 l 2.0 +X2-grid 1 1.2707963267948965 16 u 1.8707963267948966 +X3-grid 1 0.0 128 u 6.283185307179586 [TimeIntegrator] CFL 0.5 diff --git a/test/MHD/FargoMHDSpherical/testme.py b/test/MHD/FargoMHDSpherical/testme.py index a09d3eeb..7ee3100b 100755 --- a/test/MHD/FargoMHDSpherical/testme.py +++ b/test/MHD/FargoMHDSpherical/testme.py @@ -53,3 +53,7 @@ def testMe(test): test.vectPot=False test.mpi=True testMe(test) + + test.vectPot=True + test.mpi=True + testMe(test) From 9143e77e913e21633a50d9b2cef2bb398e837e6e Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 26 Nov 2024 11:06:54 +0100 Subject: [PATCH 45/83] assume full sphere symmetry in SPHERICAL with 1 dimension and Components>1 assume full sphere symmetry when SPHERICAL coordinates with 1 dimension (#291) fix #290 --- src/gridHost.cpp | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/gridHost.cpp b/src/gridHost.cpp index d31a232c..ad61f043 100644 --- a/src/gridHost.cpp +++ b/src/gridHost.cpp @@ -201,15 +201,25 @@ void GridHost::MakeGrid(Input &input) { // dir >= DIMENSIONS/ Init simple uniform grid for(int i = 0 ; i < np_tot[dir] ; i++) { // Initialize to default values - xstart[dir] = -0.5; - xend[dir] = 0.5; - + #if GEOMETRY != SPHERICAL + xstart[dir] = -0.5; + xend[dir] = 0.5; + #elif GEOMETRY == SPHERICAL + if(dir == JDIR) { + xstart[dir] = 0; + xend[dir] = M_PI; + } + if(dir == KDIR) { + xstart[dir] = -0.5; + xend[dir] = 0.5; + } + #endif this->xbeg[dir] = xstart[dir]; this->xend[dir] = xend[dir]; - dx[dir](i) = 1.0; - x[dir](i)=0.0; - xl[dir](i)=-0.5; - xr[dir](i)=0.5; + dx[dir](i) = xend[dir] - xstart[dir]; + x[dir](i)=0.5*(xstart[dir]+xend[dir]); + xl[dir](i)=xstart[dir]; + xr[dir](i)=xend[dir]; } } } From 579ceaa22422dace5eb38726e14a685154f9bad2 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 4 Dec 2024 14:25:06 +0100 Subject: [PATCH 46/83] remove spurious EMF_AVERAGE flags --- src/fluid/constrainedTransport/calcRiemannEmf.hpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/fluid/constrainedTransport/calcRiemannEmf.hpp b/src/fluid/constrainedTransport/calcRiemannEmf.hpp index 3c9479c2..2779ffba 100644 --- a/src/fluid/constrainedTransport/calcRiemannEmf.hpp +++ b/src/fluid/constrainedTransport/calcRiemannEmf.hpp @@ -22,8 +22,6 @@ template void ConstrainedTransport::CalcRiemannAverage() { idfx::pushRegion("ConstrainedTransport::calcRiemannAverage"); -#if EMF_AVERAGE == UCT_HLLD || EMF_AVERAGE == UCT_HLL - // Corned EMFs IdefixArray3D ez = this->ez; #if DIMENSIONS == 3 @@ -428,7 +426,6 @@ void ConstrainedTransport::CalcRiemannAverage() { #endif } ); -#endif // EMF_AVERAGE idfx::popRegion(); } From f4b2d4645b9d0f049d2278f26dd76b1ac4c0262e Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 4 Dec 2024 15:02:21 +0100 Subject: [PATCH 47/83] bump up to kokkos4.5 --- src/kokkos | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kokkos b/src/kokkos index 15dc143e..09e775bf 160000 --- a/src/kokkos +++ b/src/kokkos @@ -1 +1 @@ -Subproject commit 15dc143e5f39949eece972a798e175c4b463d4b8 +Subproject commit 09e775bfc585840bb9ab1156cbd8d7d1c9e0cc6d From 4447932d6ce99ac17473c16631c045adfa7b34c5 Mon Sep 17 00:00:00 2001 From: nscepi Date: Fri, 13 Dec 2024 15:17:08 +0100 Subject: [PATCH 48/83] Add error message when total grid size is not a multiple of nb mpi procs --- src/grid.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/grid.cpp b/src/grid.cpp index e4c6b7b4..fab84816 100644 --- a/src/grid.cpp +++ b/src/grid.cpp @@ -158,8 +158,16 @@ Grid::Grid(Input &input) { for(int i=0 ; i < 3 ; i++) period[i] = 0; - // Check if the dec option has been passed when number of procs > 1 + // Check that number of procs > 1 if(idfx::psize>1) { + int ngridtot=1; + for(int dir=0 ; dir < DIMENSIONS; dir++) { + ngridtot *= np_int[dir]; + } + // Check that the total grid dimension is effectively divisible by number of procs + if(ngridtot % idfx::psize) + IDEFIX_ERROR("Total grid size must be a multiple of the number of mpi process"); + // Check that dec option has been passed if(input.CheckEntry("CommandLine","dec") != DIMENSIONS) { // No command line decomposition, make auto-decomposition if possible // (only when nproc and dimensions are powers of 2, and in 1D) @@ -185,7 +193,7 @@ Grid::Grid(Input &input) { int ntot=1; for(int dir=0 ; dir < DIMENSIONS; dir++) { nproc[dir] = input.Get("CommandLine","dec",dir); - // Check that the dimension is effectively divisible by number of procs + // Check that the dimension is effectively divisible by number of procs along each direction if(np_int[dir] % nproc[dir]) IDEFIX_ERROR("Grid size must be a multiple of the domain decomposition"); // Count the total number of procs we'll need for the specified domain decomposition From 97423b9214ce3fd4bf62ee805517eec421ba2657 Mon Sep 17 00:00:00 2001 From: Marc Coiffier Date: Fri, 29 Nov 2024 14:30:54 +0100 Subject: [PATCH 49/83] Tentative de modularisation de la CI --- .github/workflows/idefix-ci-jobs.yml | 291 ++++++++++++++++++++++++++ .github/workflows/idefix-ci.yml | 296 ++------------------------- 2 files changed, 303 insertions(+), 284 deletions(-) create mode 100644 .github/workflows/idefix-ci-jobs.yml diff --git a/.github/workflows/idefix-ci-jobs.yml b/.github/workflows/idefix-ci-jobs.yml new file mode 100644 index 00000000..7462d11e --- /dev/null +++ b/.github/workflows/idefix-ci-jobs.yml @@ -0,0 +1,291 @@ +on: + workflow_call: + inputs: + TESTME_OPTIONS: + required: true + type: string + +# concurrency: +# # auto-cancel any concurrent job *in the same context* +# # see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#concurrency +# # see https://docs.github.com/en/actions/learn-github-actions/contexts#github-context +# group: ${{ github.workflow }}-${{ github.ref }} +# cancel-in-progress: true + +env: + TESTME_OPTIONS: ${{ inputs.TESTME_OPTIONS }} + PYTHONPATH: ${{ github.workspace }} + IDEFIX_DIR: ${{ github.workspace }} + +jobs: + Linter: + # Don't do this in forks + if: ${{ github.repository == 'idefix-code/idefix' || github.repository == 'glesur/idefix' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: 3.x + - uses: pre-commit/action@v3.0.0 + - uses: pre-commit-ci/lite-action@v1.0.0 + if: always() + + ShocksHydro: + needs: Linter + runs-on: self-hosted + steps: + - name: Check out repo + uses: actions/checkout@v3 + with: + submodules: recursive + - name: Sod test + run: | + cd $IDEFIX_DIR/test/HD/sod + ./testme.py -all $TESTME_OPTIONS + - name: Isothermal Sod test + run: | + cd $IDEFIX_DIR/test/HD/sod-iso + ./testme.py -all $TESTME_OPTIONS + - name: Mach reflection test + run: | + cd $IDEFIX_DIR/test/HD//MachReflection + ./testme.py -all $TESTME_OPTIONS + + ParabolicHydro: + needs: Linter + runs-on: self-hosted + steps: + - name: Check out repo + uses: actions/checkout@v3 + with: + submodules: recursive + - name: Viscous flow past cylinder + run: | + cd $IDEFIX_DIR/test/HD/ViscousFlowPastCylinder + ./testme.py -all $TESTME_OPTIONS + - name: Viscous disk + run: | + cd $IDEFIX_DIR/test/HD/ViscousDisk + ./testme.py -all $TESTME_OPTIONS + - name: Thermal diffusion + run: | + cd $IDEFIX_DIR/test/HD/thermalDiffusion + ./testme.py -all $TESTME_OPTIONS + + ShocksMHD: + needs: Linter + runs-on: self-hosted + steps: + - name: Check out repo + uses: actions/checkout@v3 + with: + submodules: recursive + - name: MHD Sod test + run: | + cd $IDEFIX_DIR/test/MHD/sod + ./testme.py -all $TESTME_OPTIONS + - name: MHD Isothermal Sod test + run: | + cd $IDEFIX_DIR/test/MHD/sod-iso + ./testme.py -all $TESTME_OPTIONS + - name: Orszag Tang + run: | + cd $IDEFIX_DIR/test/MHD/OrszagTang + ./testme.py -all $TESTME_OPTIONS + - name: Orszag Tang 3D+restart tests + run: | + cd $IDEFIX_DIR/test/MHD/OrszagTang3D + ./testme.py -all $TESTME_OPTIONS + + + ParabolicMHD: + needs: Linter + runs-on: self-hosted + steps: + - name: Check out repo + uses: actions/checkout@v3 + with: + submodules: recursive + - name: Ambipolar C Shock + run: | + cd $IDEFIX_DIR/test/MHD/AmbipolarCshock + ./testme.py -all $TESTME_OPTIONS + - name: Ambipolar C Shock 3D + run: | + cd $IDEFIX_DIR/test/MHD/AmbipolarCshock3D + ./testme.py -all $TESTME_OPTIONS + - name: Resistive Alfvén wave + run: | + cd $IDEFIX_DIR/test/MHD/ResistiveAlfvenWave + ./testme.py -all $TESTME_OPTIONS + - name: Grid coarsening diffusion + run: | + cd $IDEFIX_DIR/test/MHD/Coarsening + ./testme.py -all $TESTME_OPTIONS + - name: Hall whistler waves + run: | + cd $IDEFIX_DIR/test/MHD/HallWhistler + ./testme.py -all $TESTME_OPTIONS + + Fargo: + needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] + runs-on: self-hosted + steps: + - name: Check out repo + uses: actions/checkout@v3 + with: + submodules: recursive + - name: Fargo + planet + run: | + cd $IDEFIX_DIR/test/HD/FargoPlanet + ./testme.py -all $TESTME_OPTIONS + - name: Fargo MHD spherical + run: | + cd $IDEFIX_DIR/test/MHD/FargoMHDSpherical + ./testme.py -all $TESTME_OPTIONS + + ShearingBox: + needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] + runs-on: self-hosted + steps: + - name: Check out repo + uses: actions/checkout@v3 + with: + submodules: recursive + - name: Hydro shearing box + run: | + cd $IDEFIX_DIR/test/HD/ShearingBox + ./testme.py -all $TESTME_OPTIONS + - name: MHD shearing box + run: | + cd $IDEFIX_DIR/test/MHD/ShearingBox + ./testme.py -all $TESTME_OPTIONS + + SelfGravity: + needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] + runs-on: self-hosted + steps: + - name: Check out repo + uses: actions/checkout@v3 + with: + submodules: recursive + - name: Jeans Instability + run: | + cd $IDEFIX_DIR/test/SelfGravity/JeansInstability + ./testme.py -all $TESTME_OPTIONS + - name: Random sphere spherical + run: | + cd $IDEFIX_DIR/test/SelfGravity/RandomSphere + ./testme.py -all $TESTME_OPTIONS + - name: Random sphere cartesian + run: | + cd $IDEFIX_DIR/test/SelfGravity/RandomSphereCartesian + ./testme.py -all $TESTME_OPTIONS + - name: Uniform spherical collapse + run: | + cd $IDEFIX_DIR/test/SelfGravity/UniformCollapse + ./testme.py -all $TESTME_OPTIONS + - name: Dusty spherical collapse + run: | + cd $IDEFIX_DIR/test/SelfGravity/DustyCollapse + ./testme.py -all $TESTME_OPTIONS + + Planet: + needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] + runs-on: self-hosted + steps: + - name: Check out repo + uses: actions/checkout@v3 + with: + submodules: recursive + - name: 3 body + run: | + cd $IDEFIX_DIR/test/Planet/Planet3Body + ./testme.py -all $TESTME_OPTIONS + - name: migration + run: | + cd $IDEFIX_DIR/test/Planet/PlanetMigration2D + ./testme.py -all $TESTME_OPTIONS + - name: planet-planet + run: | + cd $IDEFIX_DIR/test/Planet/PlanetPlanetRK42D + ./testme.py -all $TESTME_OPTIONS + - name: spiral wake + run: | + cd $IDEFIX_DIR/test/Planet/PlanetSpiral2D + ./testme.py -all $TESTME_OPTIONS + - name: torques + run: | + cd $IDEFIX_DIR/test/Planet/PlanetTorque3D + ./testme.py -all $TESTME_OPTIONS + - name: RK5 + run: | + cd $IDEFIX_DIR/test/Planet/PlanetsIsActiveRK52D + ./testme.py -all $TESTME_OPTIONS + + Dust: + needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] + runs-on: self-hosted + steps: + - name: Check out repo + uses: actions/checkout@v3 + with: + submodules: recursive + - name: Energy conservation + run: | + cd $IDEFIX_DIR/test/Dust/DustEnergy + ./testme.py -all $TESTME_OPTIONS + - name: Dusty wave + run: | + cd $IDEFIX_DIR/test/Dust/DustyWave + ./testme.py -all $TESTME_OPTIONS + + Braginskii: + needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] + runs-on: self-hosted + steps: + - name: Check out repo + uses: actions/checkout@v3 + with: + submodules: recursive + - name: MTI + run: | + cd $IDEFIX_DIR/test/MHD/MTI + ./testme.py -all $TESTME_OPTIONS + - name: Spherical anisotropic diffusion + run: | + cd $IDEFIX_DIR/test/MHD/sphBragTDiffusion + ./testme.py -all $TESTME_OPTIONS + - name: Spherical anisotropic viscosity + run: | + cd $IDEFIX_DIR/test/MHD/sphBragViscosity + ./testme.py -all $TESTME_OPTIONS + + Examples: + needs: [Fargo, Dust, Planet, ShearingBox, SelfGravity] + runs-on: self-hosted + steps: + - name: Check out repo + uses: actions/checkout@v3 + with: + submodules: recursive + - name: Run examples test + run: cd test && ./checks_examples.sh $TEST_OPTIONS + + Utils: + needs: [Fargo, Dust, Planet, ShearingBox, SelfGravity] + runs-on: self-hosted + steps: + - name: Check out repo + uses: actions/checkout@v3 + with: + submodules: recursive + - name: Lookup table + run: | + cd $IDEFIX_DIR/test/utils/lookupTable + ./testme.py -all $TESTME_OPTIONS + - name: Dump Image + run: | + cd $IDEFIX_DIR/test/utils/dumpImage + ./testme.py -all $TESTME_OPTIONS diff --git a/.github/workflows/idefix-ci.yml b/.github/workflows/idefix-ci.yml index e8201828..c83635b8 100644 --- a/.github/workflows/idefix-ci.yml +++ b/.github/workflows/idefix-ci.yml @@ -4,292 +4,20 @@ on: push: branches: - master - - v2.0 + - test-ci pull_request: paths-ignore: - '.github/ISSUE_TEMPLATE/*' -concurrency: - # auto-cancel any concurrent job *in the same context* - # see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#concurrency - # see https://docs.github.com/en/actions/learn-github-actions/contexts#github-context - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - TESTME_OPTIONS: -cuda -Werror - PYTHONPATH: ${{ github.workspace }} - IDEFIX_DIR: ${{ github.workspace }} - jobs: - Linter: - # Don't do this in forks - if: ${{ github.repository == 'idefix-code/idefix' || github.repository == 'glesur/idefix' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: 3.x - - uses: pre-commit/action@v3.0.0 - - uses: pre-commit-ci/lite-action@v1.0.0 - if: always() - - ShocksHydro: - needs: Linter - runs-on: self-hosted - steps: - - name: Check out repo - uses: actions/checkout@v3 - with: - submodules: recursive - - name: Sod test - run: | - cd $IDEFIX_DIR/test/HD/sod - ./testme.py -all $TESTME_OPTIONS - - name: Isothermal Sod test - run: | - cd $IDEFIX_DIR/test/HD/sod-iso - ./testme.py -all $TESTME_OPTIONS - - name: Mach reflection test - run: | - cd $IDEFIX_DIR/test/HD//MachReflection - ./testme.py -all $TESTME_OPTIONS - - ParabolicHydro: - needs: Linter - runs-on: self-hosted - steps: - - name: Check out repo - uses: actions/checkout@v3 - with: - submodules: recursive - - name: Viscous flow past cylinder - run: | - cd $IDEFIX_DIR/test/HD/ViscousFlowPastCylinder - ./testme.py -all $TESTME_OPTIONS - - name: Viscous disk - run: | - cd $IDEFIX_DIR/test/HD/ViscousDisk - ./testme.py -all $TESTME_OPTIONS - - name: Thermal diffusion - run: | - cd $IDEFIX_DIR/test/HD/thermalDiffusion - ./testme.py -all $TESTME_OPTIONS - - ShocksMHD: - needs: Linter - runs-on: self-hosted - steps: - - name: Check out repo - uses: actions/checkout@v3 - with: - submodules: recursive - - name: MHD Sod test - run: | - cd $IDEFIX_DIR/test/MHD/sod - ./testme.py -all $TESTME_OPTIONS - - name: MHD Isothermal Sod test - run: | - cd $IDEFIX_DIR/test/MHD/sod-iso - ./testme.py -all $TESTME_OPTIONS - - name: Orszag Tang - run: | - cd $IDEFIX_DIR/test/MHD/OrszagTang - ./testme.py -all $TESTME_OPTIONS - - name: Orszag Tang 3D+restart tests - run: | - cd $IDEFIX_DIR/test/MHD/OrszagTang3D - ./testme.py -all $TESTME_OPTIONS - - - ParabolicMHD: - needs: Linter - runs-on: self-hosted - steps: - - name: Check out repo - uses: actions/checkout@v3 - with: - submodules: recursive - - name: Ambipolar C Shock - run: | - cd $IDEFIX_DIR/test/MHD/AmbipolarCshock - ./testme.py -all $TESTME_OPTIONS - - name: Ambipolar C Shock 3D - run: | - cd $IDEFIX_DIR/test/MHD/AmbipolarCshock3D - ./testme.py -all $TESTME_OPTIONS - - name: Resistive Alfvén wave - run: | - cd $IDEFIX_DIR/test/MHD/ResistiveAlfvenWave - ./testme.py -all $TESTME_OPTIONS - - name: Grid coarsening diffusion - run: | - cd $IDEFIX_DIR/test/MHD/Coarsening - ./testme.py -all $TESTME_OPTIONS - - name: Hall whistler waves - run: | - cd $IDEFIX_DIR/test/MHD/HallWhistler - ./testme.py -all $TESTME_OPTIONS - - Fargo: - needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] - runs-on: self-hosted - steps: - - name: Check out repo - uses: actions/checkout@v3 - with: - submodules: recursive - - name: Fargo + planet - run: | - cd $IDEFIX_DIR/test/HD/FargoPlanet - ./testme.py -all $TESTME_OPTIONS - - name: Fargo MHD spherical - run: | - cd $IDEFIX_DIR/test/MHD/FargoMHDSpherical - ./testme.py -all $TESTME_OPTIONS - - ShearingBox: - needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] - runs-on: self-hosted - steps: - - name: Check out repo - uses: actions/checkout@v3 - with: - submodules: recursive - - name: Hydro shearing box - run: | - cd $IDEFIX_DIR/test/HD/ShearingBox - ./testme.py -all $TESTME_OPTIONS - - name: MHD shearing box - run: | - cd $IDEFIX_DIR/test/MHD/ShearingBox - ./testme.py -all $TESTME_OPTIONS - - SelfGravity: - needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] - runs-on: self-hosted - steps: - - name: Check out repo - uses: actions/checkout@v3 - with: - submodules: recursive - - name: Jeans Instability - run: | - cd $IDEFIX_DIR/test/SelfGravity/JeansInstability - ./testme.py -all $TESTME_OPTIONS - - name: Random sphere spherical - run: | - cd $IDEFIX_DIR/test/SelfGravity/RandomSphere - ./testme.py -all $TESTME_OPTIONS - - name: Random sphere cartesian - run: | - cd $IDEFIX_DIR/test/SelfGravity/RandomSphereCartesian - ./testme.py -all $TESTME_OPTIONS - - name: Uniform spherical collapse - run: | - cd $IDEFIX_DIR/test/SelfGravity/UniformCollapse - ./testme.py -all $TESTME_OPTIONS - - name: Dusty spherical collapse - run: | - cd $IDEFIX_DIR/test/SelfGravity/DustyCollapse - ./testme.py -all $TESTME_OPTIONS - - Planet: - needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] - runs-on: self-hosted - steps: - - name: Check out repo - uses: actions/checkout@v3 - with: - submodules: recursive - - name: 3 body - run: | - cd $IDEFIX_DIR/test/Planet/Planet3Body - ./testme.py -all $TESTME_OPTIONS - - name: migration - run: | - cd $IDEFIX_DIR/test/Planet/PlanetMigration2D - ./testme.py -all $TESTME_OPTIONS - - name: planet-planet - run: | - cd $IDEFIX_DIR/test/Planet/PlanetPlanetRK42D - ./testme.py -all $TESTME_OPTIONS - - name: spiral wake - run: | - cd $IDEFIX_DIR/test/Planet/PlanetSpiral2D - ./testme.py -all $TESTME_OPTIONS - - name: torques - run: | - cd $IDEFIX_DIR/test/Planet/PlanetTorque3D - ./testme.py -all $TESTME_OPTIONS - - name: RK5 - run: | - cd $IDEFIX_DIR/test/Planet/PlanetsIsActiveRK52D - ./testme.py -all $TESTME_OPTIONS - - Dust: - needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] - runs-on: self-hosted - steps: - - name: Check out repo - uses: actions/checkout@v3 - with: - submodules: recursive - - name: Energy conservation - run: | - cd $IDEFIX_DIR/test/Dust/DustEnergy - ./testme.py -all $TESTME_OPTIONS - - name: Dusty wave - run: | - cd $IDEFIX_DIR/test/Dust/DustyWave - ./testme.py -all $TESTME_OPTIONS - - Braginskii: - needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] - runs-on: self-hosted - steps: - - name: Check out repo - uses: actions/checkout@v3 - with: - submodules: recursive - - name: MTI - run: | - cd $IDEFIX_DIR/test/MHD/MTI - ./testme.py -all $TESTME_OPTIONS - - name: Spherical anisotropic diffusion - run: | - cd $IDEFIX_DIR/test/MHD/sphBragTDiffusion - ./testme.py -all $TESTME_OPTIONS - - name: Spherical anisotropic viscosity - run: | - cd $IDEFIX_DIR/test/MHD/sphBragViscosity - ./testme.py -all $TESTME_OPTIONS - - Examples: - needs: [Fargo, Dust, Planet, ShearingBox, SelfGravity] - runs-on: self-hosted - steps: - - name: Check out repo - uses: actions/checkout@v3 - with: - submodules: recursive - - name: Run examples test - run: cd test && ./checks_examples.sh $TEST_OPTIONS - - Utils: - needs: [Fargo, Dust, Planet, ShearingBox, SelfGravity] - runs-on: self-hosted - steps: - - name: Check out repo - uses: actions/checkout@v3 - with: - submodules: recursive - - name: Lookup table - run: | - cd $IDEFIX_DIR/test/utils/lookupTable - ./testme.py -all $TESTME_OPTIONS - - name: Dump Image - run: | - cd $IDEFIX_DIR/test/utils/dumpImage - ./testme.py -all $TESTME_OPTIONS + cuda-jobs: + name: CUDA Jobs + uses: ./.github/workflows/idefix-ci-jobs.yml + with: + TESTME_OPTIONS: -cuda -Werror + + cpu-jobs: + name: CPU Jobs + uses: ./.github/workflows/idefix-ci-jobs.yml + with: + TESTME_OPTIONS: -Werror From 603bbf07ea881a6d38e27353482bfe67ea8991e8 Mon Sep 17 00:00:00 2001 From: Marc Coiffier Date: Wed, 4 Dec 2024 16:24:34 +0100 Subject: [PATCH 50/83] Tentative de compilation avec icc --- .github/workflows/idefix-ci-jobs.yml | 153 +++++++-------------------- .github/workflows/idefix-ci.yml | 13 ++- scripts/ci/run-tests | 9 ++ 3 files changed, 61 insertions(+), 114 deletions(-) create mode 100755 scripts/ci/run-tests diff --git a/.github/workflows/idefix-ci-jobs.yml b/.github/workflows/idefix-ci-jobs.yml index 7462d11e..b31bf190 100644 --- a/.github/workflows/idefix-ci-jobs.yml +++ b/.github/workflows/idefix-ci-jobs.yml @@ -4,6 +4,9 @@ on: TESTME_OPTIONS: required: true type: string + IDEFIX_TARGET: + required: true + type: string # concurrency: # # auto-cancel any concurrent job *in the same context* @@ -13,6 +16,7 @@ on: # cancel-in-progress: true env: + IDEFIX_COMPILER: ${{ inputs.IDEFIX_COMPILER }} TESTME_OPTIONS: ${{ inputs.TESTME_OPTIONS }} PYTHONPATH: ${{ github.workspace }} IDEFIX_DIR: ${{ github.workspace }} @@ -40,17 +44,11 @@ jobs: with: submodules: recursive - name: Sod test - run: | - cd $IDEFIX_DIR/test/HD/sod - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/HD/sod -all $TESTME_OPTIONS - name: Isothermal Sod test - run: | - cd $IDEFIX_DIR/test/HD/sod-iso - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/HD/sod-iso -all $TESTME_OPTIONS - name: Mach reflection test - run: | - cd $IDEFIX_DIR/test/HD//MachReflection - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/HD//MachReflection -all $TESTME_OPTIONS ParabolicHydro: needs: Linter @@ -61,17 +59,11 @@ jobs: with: submodules: recursive - name: Viscous flow past cylinder - run: | - cd $IDEFIX_DIR/test/HD/ViscousFlowPastCylinder - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/HD/ViscousFlowPastCylinder -all $TESTME_OPTIONS - name: Viscous disk - run: | - cd $IDEFIX_DIR/test/HD/ViscousDisk - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/HD/ViscousDisk -all $TESTME_OPTIONS - name: Thermal diffusion - run: | - cd $IDEFIX_DIR/test/HD/thermalDiffusion - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/HD/thermalDiffusion -all $TESTME_OPTIONS ShocksMHD: needs: Linter @@ -82,22 +74,13 @@ jobs: with: submodules: recursive - name: MHD Sod test - run: | - cd $IDEFIX_DIR/test/MHD/sod - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/sod -all $TESTME_OPTIONS - name: MHD Isothermal Sod test - run: | - cd $IDEFIX_DIR/test/MHD/sod-iso - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/sod-iso -all $TESTME_OPTIONS - name: Orszag Tang - run: | - cd $IDEFIX_DIR/test/MHD/OrszagTang - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/OrszagTang -all $TESTME_OPTIONS - name: Orszag Tang 3D+restart tests - run: | - cd $IDEFIX_DIR/test/MHD/OrszagTang3D - ./testme.py -all $TESTME_OPTIONS - + run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/OrszagTang3D -all $TESTME_OPTIONS ParabolicMHD: needs: Linter @@ -108,25 +91,15 @@ jobs: with: submodules: recursive - name: Ambipolar C Shock - run: | - cd $IDEFIX_DIR/test/MHD/AmbipolarCshock - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/AmbipolarCshock -all $TESTME_OPTIONS - name: Ambipolar C Shock 3D - run: | - cd $IDEFIX_DIR/test/MHD/AmbipolarCshock3D - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/AmbipolarCshock3D -all $TESTME_OPTIONS - name: Resistive Alfvén wave - run: | - cd $IDEFIX_DIR/test/MHD/ResistiveAlfvenWave - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/ResistiveAlfvenWave -all $TESTME_OPTIONS - name: Grid coarsening diffusion - run: | - cd $IDEFIX_DIR/test/MHD/Coarsening - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/Coarsening -all $TESTME_OPTIONS - name: Hall whistler waves - run: | - cd $IDEFIX_DIR/test/MHD/HallWhistler - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/HallWhistler -all $TESTME_OPTIONS Fargo: needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] @@ -137,13 +110,9 @@ jobs: with: submodules: recursive - name: Fargo + planet - run: | - cd $IDEFIX_DIR/test/HD/FargoPlanet - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/HD/FargoPlanet -all $TESTME_OPTIONS - name: Fargo MHD spherical - run: | - cd $IDEFIX_DIR/test/MHD/FargoMHDSpherical - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/FargoMHDSpherical -all $TESTME_OPTIONS ShearingBox: needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] @@ -154,13 +123,9 @@ jobs: with: submodules: recursive - name: Hydro shearing box - run: | - cd $IDEFIX_DIR/test/HD/ShearingBox - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/HD/ShearingBox -all $TESTME_OPTIONS - name: MHD shearing box - run: | - cd $IDEFIX_DIR/test/MHD/ShearingBox - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/ShearingBox -all $TESTME_OPTIONS SelfGravity: needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] @@ -171,25 +136,15 @@ jobs: with: submodules: recursive - name: Jeans Instability - run: | - cd $IDEFIX_DIR/test/SelfGravity/JeansInstability - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/SelfGravity/JeansInstability -all $TESTME_OPTIONS - name: Random sphere spherical - run: | - cd $IDEFIX_DIR/test/SelfGravity/RandomSphere - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/SelfGravity/RandomSphere -all $TESTME_OPTIONS - name: Random sphere cartesian - run: | - cd $IDEFIX_DIR/test/SelfGravity/RandomSphereCartesian - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/SelfGravity/RandomSphereCartesian -all $TESTME_OPTIONS - name: Uniform spherical collapse - run: | - cd $IDEFIX_DIR/test/SelfGravity/UniformCollapse - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/SelfGravity/UniformCollapse -all $TESTME_OPTIONS - name: Dusty spherical collapse - run: | - cd $IDEFIX_DIR/test/SelfGravity/DustyCollapse - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/SelfGravity/DustyCollapse -all $TESTME_OPTIONS Planet: needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] @@ -200,29 +155,17 @@ jobs: with: submodules: recursive - name: 3 body - run: | - cd $IDEFIX_DIR/test/Planet/Planet3Body - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/Planet/Planet3Body -all $TESTME_OPTIONS - name: migration - run: | - cd $IDEFIX_DIR/test/Planet/PlanetMigration2D - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/Planet/PlanetMigration2D -all $TESTME_OPTIONS - name: planet-planet - run: | - cd $IDEFIX_DIR/test/Planet/PlanetPlanetRK42D - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/Planet/PlanetPlanetRK42D -all $TESTME_OPTIONS - name: spiral wake - run: | - cd $IDEFIX_DIR/test/Planet/PlanetSpiral2D - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/Planet/PlanetSpiral2D -all $TESTME_OPTIONS - name: torques - run: | - cd $IDEFIX_DIR/test/Planet/PlanetTorque3D - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/Planet/PlanetTorque3D -all $TESTME_OPTIONS - name: RK5 - run: | - cd $IDEFIX_DIR/test/Planet/PlanetsIsActiveRK52D - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/Planet/PlanetsIsActiveRK52D -all $TESTME_OPTIONS Dust: needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] @@ -233,13 +176,9 @@ jobs: with: submodules: recursive - name: Energy conservation - run: | - cd $IDEFIX_DIR/test/Dust/DustEnergy - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/Dust/DustEnergy -all $TESTME_OPTIONS - name: Dusty wave - run: | - cd $IDEFIX_DIR/test/Dust/DustyWave - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/Dust/DustyWave -all $TESTME_OPTIONS Braginskii: needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] @@ -250,17 +189,11 @@ jobs: with: submodules: recursive - name: MTI - run: | - cd $IDEFIX_DIR/test/MHD/MTI - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/MTI -all $TESTME_OPTIONS - name: Spherical anisotropic diffusion - run: | - cd $IDEFIX_DIR/test/MHD/sphBragTDiffusion - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/sphBragTDiffusion -all $TESTME_OPTIONS - name: Spherical anisotropic viscosity - run: | - cd $IDEFIX_DIR/test/MHD/sphBragViscosity - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/sphBragViscosity -all $TESTME_OPTIONS Examples: needs: [Fargo, Dust, Planet, ShearingBox, SelfGravity] @@ -282,10 +215,6 @@ jobs: with: submodules: recursive - name: Lookup table - run: | - cd $IDEFIX_DIR/test/utils/lookupTable - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/utils/lookupTable -all $TESTME_OPTIONS - name: Dump Image - run: | - cd $IDEFIX_DIR/test/utils/dumpImage - ./testme.py -all $TESTME_OPTIONS + run: scripts/ci/run-tests $IDEFIX_DIR/test/utils/dumpImage -all $TESTME_OPTIONS diff --git a/.github/workflows/idefix-ci.yml b/.github/workflows/idefix-ci.yml index c83635b8..6bf49643 100644 --- a/.github/workflows/idefix-ci.yml +++ b/.github/workflows/idefix-ci.yml @@ -15,9 +15,18 @@ jobs: uses: ./.github/workflows/idefix-ci-jobs.yml with: TESTME_OPTIONS: -cuda -Werror + IDEFIX_COMPILER: cuda - cpu-jobs: - name: CPU Jobs + gcc-jobs: + name: CPU Jobs (gcc) uses: ./.github/workflows/idefix-ci-jobs.yml with: TESTME_OPTIONS: -Werror + IDEFIX_COMPILER: gcc + + icc-jobs: + name: CPU Jobs (icc) + uses: ./.github/workflows/idefix-ci-jobs.yml + with: + TESTME_OPTIONS: -Werror + IDEFIX_COMPILER: icc diff --git a/scripts/ci/run-tests b/scripts/ci/run-tests new file mode 100755 index 00000000..f4565d53 --- /dev/null +++ b/scripts/ci/run-tests @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -ue +if [ "$IDEFIX_COMPILER" == icc ]; then + source /opt/intel/oneapi/setvars.sh + export CC=icx + export CXX=icpx +fi +cd "$1" +./testme.py "${@:2}" From 1eff9a3863f7cd6257e1851eb38531e2dcee3cfd Mon Sep 17 00:00:00 2001 From: Marc Coiffier Date: Wed, 4 Dec 2024 16:26:00 +0100 Subject: [PATCH 51/83] Renommage de IDEFIX_TARGET en IDEFIX_COMPILER dans les inputs --- .github/workflows/idefix-ci-jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/idefix-ci-jobs.yml b/.github/workflows/idefix-ci-jobs.yml index b31bf190..ea98e1c5 100644 --- a/.github/workflows/idefix-ci-jobs.yml +++ b/.github/workflows/idefix-ci-jobs.yml @@ -4,7 +4,7 @@ on: TESTME_OPTIONS: required: true type: string - IDEFIX_TARGET: + IDEFIX_COMPILER: required: true type: string From 0fcf26811cb0988f5be640a41132ec9ec46b3dfc Mon Sep 17 00:00:00 2001 From: Marc Coiffier Date: Wed, 4 Dec 2024 16:33:55 +0100 Subject: [PATCH 52/83] =?UTF-8?q?R=C3=A9ordonne=20les=20tests=20pour=20que?= =?UTF-8?q?=20'icc'=20se=20fasse=20en=20premier=20(vu=20que=20c'est=20?= =?UTF-8?q?=C3=A7a=20qu'on=20teste)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/idefix-ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/idefix-ci.yml b/.github/workflows/idefix-ci.yml index 6bf49643..3a188c93 100644 --- a/.github/workflows/idefix-ci.yml +++ b/.github/workflows/idefix-ci.yml @@ -10,12 +10,12 @@ on: - '.github/ISSUE_TEMPLATE/*' jobs: - cuda-jobs: - name: CUDA Jobs + icc-jobs: + name: CPU Jobs (icc) uses: ./.github/workflows/idefix-ci-jobs.yml with: - TESTME_OPTIONS: -cuda -Werror - IDEFIX_COMPILER: cuda + TESTME_OPTIONS: -Werror + IDEFIX_COMPILER: icc gcc-jobs: name: CPU Jobs (gcc) @@ -24,9 +24,9 @@ jobs: TESTME_OPTIONS: -Werror IDEFIX_COMPILER: gcc - icc-jobs: - name: CPU Jobs (icc) + cuda-jobs: + name: CUDA Jobs uses: ./.github/workflows/idefix-ci-jobs.yml with: - TESTME_OPTIONS: -Werror - IDEFIX_COMPILER: icc + TESTME_OPTIONS: -cuda -Werror + IDEFIX_COMPILER: cuda From a1cbc4b7db735257c688b7fbe8544d3c2a40956d Mon Sep 17 00:00:00 2001 From: Marc Coiffier Date: Wed, 4 Dec 2024 17:55:24 +0100 Subject: [PATCH 53/83] =?UTF-8?q?D=C3=A9placement=20du=20'set=20-ue'=20apr?= =?UTF-8?q?=C3=A8s=20l'import=20des=20variables=20Intel=20(pour=20cause=20?= =?UTF-8?q?d'erreur=20dans=20le=20script=20setvars.sh...)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/ci/run-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/run-tests b/scripts/ci/run-tests index f4565d53..3be11bd9 100755 --- a/scripts/ci/run-tests +++ b/scripts/ci/run-tests @@ -1,9 +1,9 @@ #!/usr/bin/env bash -set -ue if [ "$IDEFIX_COMPILER" == icc ]; then source /opt/intel/oneapi/setvars.sh export CC=icx export CXX=icpx fi +set -ue cd "$1" ./testme.py "${@:2}" From 2dca8b138b484ccc865bc56bc7f33e3d03142383 Mon Sep 17 00:00:00 2001 From: Marc Coiffier Date: Thu, 5 Dec 2024 13:42:23 +0100 Subject: [PATCH 54/83] =?UTF-8?q?Ajout=20de=20-fp-model=3Dstrict=20lors=20?= =?UTF-8?q?de=20la=20compilation=20avec=20icc=20pour=20=C3=A9viter=20les?= =?UTF-8?q?=20optimisation=20impr=C3=A9cises?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/ci/run-tests | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/ci/run-tests b/scripts/ci/run-tests index 3be11bd9..899eff34 100755 --- a/scripts/ci/run-tests +++ b/scripts/ci/run-tests @@ -3,6 +3,8 @@ if [ "$IDEFIX_COMPILER" == icc ]; then source /opt/intel/oneapi/setvars.sh export CC=icx export CXX=icpx + export CFLAGS="-fp-model=strict" + export CXXFLAGS="-fp-model=strict" fi set -ue cd "$1" From e590e4316850de67bd555751d7fe001f9265ce01 Mon Sep 17 00:00:00 2001 From: Marc Coiffier Date: Fri, 6 Dec 2024 09:22:59 +0100 Subject: [PATCH 55/83] =?UTF-8?q?S=C3=A9oaration=20du=20linter=20et=20des?= =?UTF-8?q?=20jobs=20de=20compilation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/idefix-ci-jobs.yml | 29 ++++++---------------------- .github/workflows/idefix-ci.yml | 16 +++++++++++++++ 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/.github/workflows/idefix-ci-jobs.yml b/.github/workflows/idefix-ci-jobs.yml index ea98e1c5..c0e62296 100644 --- a/.github/workflows/idefix-ci-jobs.yml +++ b/.github/workflows/idefix-ci-jobs.yml @@ -22,21 +22,7 @@ env: IDEFIX_DIR: ${{ github.workspace }} jobs: - Linter: - # Don't do this in forks - if: ${{ github.repository == 'idefix-code/idefix' || github.repository == 'glesur/idefix' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: 3.x - - uses: pre-commit/action@v3.0.0 - - uses: pre-commit-ci/lite-action@v1.0.0 - if: always() - ShocksHydro: - needs: Linter runs-on: self-hosted steps: - name: Check out repo @@ -51,7 +37,6 @@ jobs: run: scripts/ci/run-tests $IDEFIX_DIR/test/HD//MachReflection -all $TESTME_OPTIONS ParabolicHydro: - needs: Linter runs-on: self-hosted steps: - name: Check out repo @@ -66,7 +51,6 @@ jobs: run: scripts/ci/run-tests $IDEFIX_DIR/test/HD/thermalDiffusion -all $TESTME_OPTIONS ShocksMHD: - needs: Linter runs-on: self-hosted steps: - name: Check out repo @@ -83,7 +67,6 @@ jobs: run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/OrszagTang3D -all $TESTME_OPTIONS ParabolicMHD: - needs: Linter runs-on: self-hosted steps: - name: Check out repo @@ -102,7 +85,7 @@ jobs: run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/HallWhistler -all $TESTME_OPTIONS Fargo: - needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] + needs: [ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] runs-on: self-hosted steps: - name: Check out repo @@ -115,7 +98,7 @@ jobs: run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/FargoMHDSpherical -all $TESTME_OPTIONS ShearingBox: - needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] + needs: [ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] runs-on: self-hosted steps: - name: Check out repo @@ -128,7 +111,7 @@ jobs: run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/ShearingBox -all $TESTME_OPTIONS SelfGravity: - needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] + needs: [ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] runs-on: self-hosted steps: - name: Check out repo @@ -147,7 +130,7 @@ jobs: run: scripts/ci/run-tests $IDEFIX_DIR/test/SelfGravity/DustyCollapse -all $TESTME_OPTIONS Planet: - needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] + needs: [ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] runs-on: self-hosted steps: - name: Check out repo @@ -168,7 +151,7 @@ jobs: run: scripts/ci/run-tests $IDEFIX_DIR/test/Planet/PlanetsIsActiveRK52D -all $TESTME_OPTIONS Dust: - needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] + needs: [ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] runs-on: self-hosted steps: - name: Check out repo @@ -181,7 +164,7 @@ jobs: run: scripts/ci/run-tests $IDEFIX_DIR/test/Dust/DustyWave -all $TESTME_OPTIONS Braginskii: - needs: [Linter, ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] + needs: [ShocksHydro, ParabolicHydro, ShocksMHD, ParabolicMHD] runs-on: self-hosted steps: - name: Check out repo diff --git a/.github/workflows/idefix-ci.yml b/.github/workflows/idefix-ci.yml index 3a188c93..eb53abf5 100644 --- a/.github/workflows/idefix-ci.yml +++ b/.github/workflows/idefix-ci.yml @@ -10,7 +10,21 @@ on: - '.github/ISSUE_TEMPLATE/*' jobs: + Linter: + # Don't do this in forks + if: ${{ github.repository == 'idefix-code/idefix' || github.repository == 'glesur/idefix' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: 3.x + - uses: pre-commit/action@v3.0.0 + - uses: pre-commit-ci/lite-action@v1.0.0 + if: always() + icc-jobs: + needs: Linter name: CPU Jobs (icc) uses: ./.github/workflows/idefix-ci-jobs.yml with: @@ -18,6 +32,7 @@ jobs: IDEFIX_COMPILER: icc gcc-jobs: + needs: Linter name: CPU Jobs (gcc) uses: ./.github/workflows/idefix-ci-jobs.yml with: @@ -25,6 +40,7 @@ jobs: IDEFIX_COMPILER: gcc cuda-jobs: + needs: Linter name: CUDA Jobs uses: ./.github/workflows/idefix-ci-jobs.yml with: From 8f4f47727e39d2c89000c1cd58c9d2ded36eff8b Mon Sep 17 00:00:00 2001 From: Marc Coiffier Date: Fri, 6 Dec 2024 09:27:58 +0100 Subject: [PATCH 56/83] =?UTF-8?q?Suppression=20des=20espaces=20qui=20font?= =?UTF-8?q?=20=C3=A9chouer=20le=20linter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/idefix-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/idefix-ci.yml b/.github/workflows/idefix-ci.yml index eb53abf5..d3fb4b0d 100644 --- a/.github/workflows/idefix-ci.yml +++ b/.github/workflows/idefix-ci.yml @@ -22,7 +22,7 @@ jobs: - uses: pre-commit/action@v3.0.0 - uses: pre-commit-ci/lite-action@v1.0.0 if: always() - + icc-jobs: needs: Linter name: CPU Jobs (icc) From 1befc52991daeec9557ee68bea195e6ed757fb67 Mon Sep 17 00:00:00 2001 From: Marc Coiffier Date: Fri, 13 Dec 2024 14:41:12 +0100 Subject: [PATCH 57/83] Move some Intel compiler options to the idfx_test module --- .github/workflows/idefix-ci.yml | 4 ++-- pytools/idfx_test.py | 10 ++++++++++ scripts/ci/run-tests | 4 ---- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/idefix-ci.yml b/.github/workflows/idefix-ci.yml index d3fb4b0d..267d65ff 100644 --- a/.github/workflows/idefix-ci.yml +++ b/.github/workflows/idefix-ci.yml @@ -28,7 +28,7 @@ jobs: name: CPU Jobs (icc) uses: ./.github/workflows/idefix-ci-jobs.yml with: - TESTME_OPTIONS: -Werror + TESTME_OPTIONS: -intel -Werror IDEFIX_COMPILER: icc gcc-jobs: @@ -45,4 +45,4 @@ jobs: uses: ./.github/workflows/idefix-ci-jobs.yml with: TESTME_OPTIONS: -cuda -Werror - IDEFIX_COMPILER: cuda + IDEFIX_COMPILER: nvcc diff --git a/pytools/idfx_test.py b/pytools/idfx_test.py index dc2da0a0..25bac76b 100644 --- a/pytools/idfx_test.py +++ b/pytools/idfx_test.py @@ -58,6 +58,10 @@ def __init__ (self): help="Test on Nvidia GPU using CUDA", action="store_true") + parser.add_argument("-intel", + help="Test compiling with Intel OneAPI", + action="store_true") + parser.add_argument("-hip", help="Test on AMD GPU using HIP", action="store_true") @@ -119,6 +123,12 @@ def configure(self,definitionFile=""): # disable Async cuda malloc for tests performed on old UCX implementations comm.append("-DKokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC=OFF") + if self.intel: + # disable fmad operations on Cuda to make it compatible with CPU arithmetics + comm.append("-DIdefix_CXX_FLAGS=-fp-model=strict") + comm.append("-DCMAKE_CXX_COMPILER=icpx") + comm.append("-DCMAKE_C_COMPILER=icx") + if self.hip: comm.append("-DKokkos_ENABLE_HIP=ON") # disable fmad operations on HIP to make it compatible with CPU arithmetics diff --git a/scripts/ci/run-tests b/scripts/ci/run-tests index 899eff34..32983102 100755 --- a/scripts/ci/run-tests +++ b/scripts/ci/run-tests @@ -1,10 +1,6 @@ #!/usr/bin/env bash if [ "$IDEFIX_COMPILER" == icc ]; then source /opt/intel/oneapi/setvars.sh - export CC=icx - export CXX=icpx - export CFLAGS="-fp-model=strict" - export CXXFLAGS="-fp-model=strict" fi set -ue cd "$1" From a8513f85db06911a0de24ece28ea8e266105f87f Mon Sep 17 00:00:00 2001 From: Marc Coiffier Date: Fri, 13 Dec 2024 15:32:58 +0100 Subject: [PATCH 58/83] Change 'test-ci' to 'develop' as the target branch for CI --- .github/workflows/idefix-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/idefix-ci.yml b/.github/workflows/idefix-ci.yml index 267d65ff..adceb5c9 100644 --- a/.github/workflows/idefix-ci.yml +++ b/.github/workflows/idefix-ci.yml @@ -4,7 +4,7 @@ on: push: branches: - master - - test-ci + - develop pull_request: paths-ignore: - '.github/ISSUE_TEMPLATE/*' From 144777830a566877049349177bbd201d21fecb25 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 16:41:45 +0000 Subject: [PATCH 59/83] [pre-commit.ci lite] apply automatic fixes --- src/grid.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/grid.cpp b/src/grid.cpp index fab84816..ace5eb0c 100644 --- a/src/grid.cpp +++ b/src/grid.cpp @@ -167,7 +167,7 @@ Grid::Grid(Input &input) { // Check that the total grid dimension is effectively divisible by number of procs if(ngridtot % idfx::psize) IDEFIX_ERROR("Total grid size must be a multiple of the number of mpi process"); - // Check that dec option has been passed + // Check that dec option has been passed if(input.CheckEntry("CommandLine","dec") != DIMENSIONS) { // No command line decomposition, make auto-decomposition if possible // (only when nproc and dimensions are powers of 2, and in 1D) From 13c34d32074788a96c22bd157ee27799d87d4773 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Fri, 13 Dec 2024 20:34:56 +0100 Subject: [PATCH 60/83] ICC is outdated --- .github/workflows/idefix-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/idefix-ci.yml b/.github/workflows/idefix-ci.yml index adceb5c9..cb946143 100644 --- a/.github/workflows/idefix-ci.yml +++ b/.github/workflows/idefix-ci.yml @@ -25,7 +25,7 @@ jobs: icc-jobs: needs: Linter - name: CPU Jobs (icc) + name: CPU Jobs (intel OneApi) uses: ./.github/workflows/idefix-ci-jobs.yml with: TESTME_OPTIONS: -intel -Werror From 1453cca029725d1d6e8ac9105ee9a184b96f05a6 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 16 Dec 2024 08:45:38 +0100 Subject: [PATCH 61/83] remove force disable cuda malloc async following update to 4.5 https://kokkos.org/kokkos-core-wiki/known-issues.html#cuda --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8df0de73..8b503fad 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,7 +45,8 @@ include(SetRequiredBuildSettingsForGCC8) #Idefix requires Cuda Lambdas (experimental) if(Kokkos_ENABLE_CUDA) set(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "Idefix requires lambdas on Cuda" FORCE) - set(Kokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC OFF CACHE BOOL "Disable Async malloc to avoid bugs on PSM2" FORCE) + # CUDA_MALLOC_ASYNC disbaled by default in Kokkos 4.5, so not required here + #set(Kokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC OFF CACHE BOOL "Disable Async malloc to avoid bugs on PSM2" FORCE) endif() # Add kokkos CMAKE files (required early since these set compiler options) From 003b28c47d6dd42a0ade19a5a7fc6fbeda6551a4 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 16 Dec 2024 09:18:25 +0100 Subject: [PATCH 62/83] fix #275 --- src/pydefix.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pydefix.cpp b/src/pydefix.cpp index 6756d0a2..824254aa 100644 --- a/src/pydefix.cpp +++ b/src/pydefix.cpp @@ -133,7 +133,7 @@ void Pydefix::InitFlow(DataBlock &data) { if(!this->isActive) { IDEFIX_ERROR("Python Initflow requires the [python] block to be defined in your input file."); } - if(!this->haveOutput) { + if(!this->haveInitflow) { IDEFIX_ERROR("No python initflow function has been defined " "in your input file [python]:initflow_function"); } From 63f6a4ab73130ba316b3ed3fd5472200def2b39a Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Fri, 20 Dec 2024 16:47:26 +0100 Subject: [PATCH 63/83] test the documentation generation workflow --- .github/workflows/idefix-ci-doc.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/idefix-ci-doc.yml diff --git a/.github/workflows/idefix-ci-doc.yml b/.github/workflows/idefix-ci-doc.yml new file mode 100644 index 00000000..66e75dda --- /dev/null +++ b/.github/workflows/idefix-ci-doc.yml @@ -0,0 +1,26 @@ +name: Idefix CIs +on: + workflow_dispatch: + push: + branches: + - master + - develop + - docCI + pull_request: + paths-ignore: + - '.github/ISSUE_TEMPLATE/*' + + +jobs: + ReadTheDocs: + runs-on: ubuntu-latest + steps: + - name: Check out repo + uses: actions/checkout@v3 + with: + submodules: recursive + - name: install dependencies + run: python -m pip install --exists-action=w --no-cache-dir -r doc/python_requirements.txt + - name: compile documentation + run: python -m sphinx -T -b html -d _build/doctrees -D language=en . ./html + From af3d563803258b6e3a319252a54cf4e06928a3a0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Fri, 20 Dec 2024 15:50:09 +0000 Subject: [PATCH 64/83] [pre-commit.ci lite] apply automatic fixes --- .github/workflows/idefix-ci-doc.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/idefix-ci-doc.yml b/.github/workflows/idefix-ci-doc.yml index 66e75dda..3f0a622c 100644 --- a/.github/workflows/idefix-ci-doc.yml +++ b/.github/workflows/idefix-ci-doc.yml @@ -23,4 +23,3 @@ jobs: run: python -m pip install --exists-action=w --no-cache-dir -r doc/python_requirements.txt - name: compile documentation run: python -m sphinx -T -b html -d _build/doctrees -D language=en . ./html - From c060c419da184a31c0e3fc38bd28d09a4ae027f9 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Fri, 20 Dec 2024 16:54:59 +0100 Subject: [PATCH 65/83] update path --- .github/workflows/idefix-ci-doc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/idefix-ci-doc.yml b/.github/workflows/idefix-ci-doc.yml index 3f0a622c..77f48fd9 100644 --- a/.github/workflows/idefix-ci-doc.yml +++ b/.github/workflows/idefix-ci-doc.yml @@ -22,4 +22,4 @@ jobs: - name: install dependencies run: python -m pip install --exists-action=w --no-cache-dir -r doc/python_requirements.txt - name: compile documentation - run: python -m sphinx -T -b html -d _build/doctrees -D language=en . ./html + run: python -m sphinx -T -b html -d _build/doctrees -D language=en doc/sources doc/html From c08fc76320f909f5a0e41ff30bd782ae9f1e22d3 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Fri, 20 Dec 2024 16:58:14 +0100 Subject: [PATCH 66/83] typo --- .github/workflows/idefix-ci-doc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/idefix-ci-doc.yml b/.github/workflows/idefix-ci-doc.yml index 77f48fd9..de8d8ffb 100644 --- a/.github/workflows/idefix-ci-doc.yml +++ b/.github/workflows/idefix-ci-doc.yml @@ -22,4 +22,4 @@ jobs: - name: install dependencies run: python -m pip install --exists-action=w --no-cache-dir -r doc/python_requirements.txt - name: compile documentation - run: python -m sphinx -T -b html -d _build/doctrees -D language=en doc/sources doc/html + run: python -m sphinx -T -b html -d _build/doctrees -D language=en doc/source doc/html From 3eaa696db6b2b4cb96dff1d477a142f983148040 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Fri, 20 Dec 2024 16:59:58 +0100 Subject: [PATCH 67/83] add doxygen to the build --- .github/workflows/idefix-ci-doc.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/idefix-ci-doc.yml b/.github/workflows/idefix-ci-doc.yml index de8d8ffb..43e7aff8 100644 --- a/.github/workflows/idefix-ci-doc.yml +++ b/.github/workflows/idefix-ci-doc.yml @@ -20,6 +20,7 @@ jobs: with: submodules: recursive - name: install dependencies + run: sudo apt-get install -y doxygen run: python -m pip install --exists-action=w --no-cache-dir -r doc/python_requirements.txt - name: compile documentation run: python -m sphinx -T -b html -d _build/doctrees -D language=en doc/source doc/html From dee00e54bd96c2f322ed685841ce0b0d3ee39ce2 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Fri, 20 Dec 2024 17:02:18 +0100 Subject: [PATCH 68/83] doxygen again --- .github/workflows/idefix-ci-doc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/idefix-ci-doc.yml b/.github/workflows/idefix-ci-doc.yml index 43e7aff8..b6ae7133 100644 --- a/.github/workflows/idefix-ci-doc.yml +++ b/.github/workflows/idefix-ci-doc.yml @@ -5,7 +5,6 @@ on: branches: - master - develop - - docCI pull_request: paths-ignore: - '.github/ISSUE_TEMPLATE/*' @@ -21,6 +20,7 @@ jobs: submodules: recursive - name: install dependencies run: sudo apt-get install -y doxygen + - name: install doxygen run: python -m pip install --exists-action=w --no-cache-dir -r doc/python_requirements.txt - name: compile documentation run: python -m sphinx -T -b html -d _build/doctrees -D language=en doc/source doc/html From bfa54f724e3dc94cfba1a97c65f32539c7f6b354 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Fri, 20 Dec 2024 17:09:37 +0100 Subject: [PATCH 69/83] clean naming no submodule (experimental) --- .github/workflows/idefix-ci-doc.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/idefix-ci-doc.yml b/.github/workflows/idefix-ci-doc.yml index b6ae7133..72b5dd0b 100644 --- a/.github/workflows/idefix-ci-doc.yml +++ b/.github/workflows/idefix-ci-doc.yml @@ -16,11 +16,9 @@ jobs: steps: - name: Check out repo uses: actions/checkout@v3 - with: - submodules: recursive - - name: install dependencies - run: sudo apt-get install -y doxygen - name: install doxygen + run: sudo apt-get install -y doxygen + - name: install python dependencies run: python -m pip install --exists-action=w --no-cache-dir -r doc/python_requirements.txt - name: compile documentation run: python -m sphinx -T -b html -d _build/doctrees -D language=en doc/source doc/html From 66be68534e50a60e21a2c671ca8095ab0affd278 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Mon, 23 Dec 2024 13:15:23 +0100 Subject: [PATCH 70/83] MNT: give a unique name to documentation build workflows --- .github/workflows/idefix-ci-doc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/idefix-ci-doc.yml b/.github/workflows/idefix-ci-doc.yml index 72b5dd0b..03640465 100644 --- a/.github/workflows/idefix-ci-doc.yml +++ b/.github/workflows/idefix-ci-doc.yml @@ -1,4 +1,4 @@ -name: Idefix CIs +name: Build the docs on: workflow_dispatch: push: From 90653658b3b241fddb5f527f21374d5e5b10ab38 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 24 Dec 2024 08:37:19 +0100 Subject: [PATCH 71/83] implement MPi data gathering for pydefix Working implementation but: - should write the type caster for 1D,2D and 3D following the 4D one - should implement a proper python type for the grid - check boundary conditions --- CMakeLists.txt | 2 +- src/dataBlock/dataBlockHost.hpp | 1 - src/pydefix.cpp | 105 ++++++++++++++++++++++++++++++++ src/pydefix.hpp | 35 +++++++---- 4 files changed, 128 insertions(+), 15 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 79f6d30e..f7095d2c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -124,7 +124,7 @@ endif() if(Idefix_PYTHON) add_compile_definitions("WITH_PYTHON") - set(PYBIND11_FINDPYTHON ON CACHE BOOL "Idefix requires python" FORCE) + #set(PYBIND11_FINDPYTHON ON CACHE BOOL "Idefix requires python" FORCE) find_package(pybind11 REQUIRED) target_link_libraries(idefix pybind11::embed) target_sources(idefix diff --git a/src/dataBlock/dataBlockHost.hpp b/src/dataBlock/dataBlockHost.hpp index 2a732a32..f7dc4041 100644 --- a/src/dataBlock/dataBlockHost.hpp +++ b/src/dataBlock/dataBlockHost.hpp @@ -98,7 +98,6 @@ class DataBlockHost { GridCoarsening haveGridCoarsening{GridCoarsening::disabled}; ///< Is grid coarsening enabled? - private: // Data object to which we are the mirror DataBlock *data; }; diff --git a/src/pydefix.cpp b/src/pydefix.cpp index 824254aa..d3204e9d 100644 --- a/src/pydefix.cpp +++ b/src/pydefix.cpp @@ -26,6 +26,106 @@ int Pydefix::ninstance = 0; * DataBlockHost Python binding * **********************************/ +IdefixHostArray4D GatherIdefixArray(IdefixHostArray4D in, DataBlockHost dataHost) { + idfx::cout << "I am being called" << std::endl; + idfx::cout << "My dims=" << in.extent(0) << " " << in.extent(1) << " " << in.extent(2) << " " << in.extent(3) << std::endl; + idfx::cout << "First element=" << in(0,0,0,0) << std::endl; + Grid *grid = dataHost.data->mygrid; + IdefixHostArray4D out; + #ifdef WITH_MPI + if(idfx::psize > 1) { + const int nvar = in.extent(0); + out = IdefixHostArray4D("pydefix::GatheredArray",nvar,grid->np_tot[KDIR],grid->np_tot[JDIR],grid->np_tot[IDIR]); + if(idfx::prank == 0) { + for(int rank = 0 ; rank < idfx::psize ; rank++) { + // np_tot: total size of the incoming array + // np_int: size that should be copied into global + // beg: offset in the incoming array where copy should begin + // gbeg: offset in the global array where copy should be begin + MPI_Status status; + std::array np_int,np_tot, beg, gbeg; + IdefixHostArray4D buf; + + if(rank==0) { + np_int = dataHost.np_int; + np_tot = dataHost.np_tot; + gbeg = dataHost.gbeg; + beg = dataHost.beg; + + // Add back boundaries + for(int dir = 0 ; dir < DIMENSIONS ; dir++) { + if(dataHost.lbound[dir] != internal) { + np_int[dir] += dataHost.nghost[dir]; + gbeg[dir] -= dataHost.nghost[dir]; + beg[dir] -= dataHost.nghost[dir]; + } + if(dataHost.rbound[dir] != internal) { + np_int[dir] += dataHost.nghost[dir]; + } + } + buf = in; + } else { // target rank >0 + // Fetch the local array size + MPI_Recv(np_int.data(), 3, MPI_INT, rank, 010, MPI_COMM_WORLD, &status); + MPI_Recv(np_tot.data(), 3, MPI_INT, rank, 011, MPI_COMM_WORLD, &status); + MPI_Recv(beg.data(), 3, MPI_INT, rank, 012, MPI_COMM_WORLD, &status); + MPI_Recv(gbeg.data(), 3, MPI_INT, rank, 013, MPI_COMM_WORLD, &status); + + buf = IdefixHostArray4D("pydefix::tempArray",nvar,np_tot[KDIR],np_tot[JDIR],np_tot[IDIR]); + // Fetch data + MPI_Recv(buf.data(), nvar*np_tot[IDIR]*np_tot[JDIR]*np_tot[KDIR], realMPI, rank, 014, MPI_COMM_WORLD,&status); + } // target rank + // Copy data from the buffer + for(int n = 0 ; n < nvar ; n++) { + for(int k = 0 ; k < np_int[KDIR] ; k++) { + const int kt = k+gbeg[KDIR]; + for(int j = 0 ; j < np_int[JDIR] ; j++) { + const int jt = j+gbeg[JDIR]; + for(int i = 0 ; i < np_int[IDIR] ; i++) { + const int it = i+gbeg[IDIR]; + out(n,kt,jt,it) = buf(n,k+beg[KDIR],j+beg[JDIR],i+beg[IDIR]); + } + } + } + }// End for + }// End loop on target rank for root process + } else { // MPI prank >0 + std::array np_int = dataHost.np_int; + std::array np_tot = dataHost.np_tot; + std::array gbeg = dataHost.gbeg; + std::array beg = dataHost.beg; + + // Add back boundaries + for(int dir = 0 ; dir < DIMENSIONS ; dir++) { + if(dataHost.lbound[dir] != internal) { + np_int[dir] += dataHost.nghost[dir]; + gbeg[dir] -= dataHost.nghost[dir]; + beg[dir] -= dataHost.nghost[dir]; + } + if(dataHost.rbound[dir] != internal) { + np_int[dir] += dataHost.nghost[dir]; + } + } + + // send the local array size + MPI_Send(np_int.data(), 3, MPI_INT, 0, 010, MPI_COMM_WORLD); + MPI_Send(np_tot.data(), 3, MPI_INT, 0, 011, MPI_COMM_WORLD); + MPI_Send(beg.data(), 3, MPI_INT, 0, 012, MPI_COMM_WORLD); + MPI_Send(gbeg.data(), 3, MPI_INT, 0, 013, MPI_COMM_WORLD); + MPI_Send(in.data(), nvar*np_tot[IDIR]*np_tot[JDIR]*np_tot[KDIR], realMPI, 0, 014, MPI_COMM_WORLD); // Allocate array + } + // All is transfered + MPI_Bcast(out.data(), nvar*grid->np_tot[KDIR]*grid->np_tot[JDIR]*grid->np_tot[IDIR], realMPI, 0, MPI_COMM_WORLD); + } else { // MPI with a single proc + out = in; + } + #else // No MPI + out = in; + #endif + idfx::cout << "final dims=" << out.extent(0) << " " << out.extent(1) << " " << out.extent(2) << " " << out.extent(3) << std::endl; + return out; +} + PYBIND11_EMBEDDED_MODULE(pydefix, m) { py::class_(m, "DataBlockHost") .def(py::init<>()) @@ -66,6 +166,11 @@ PYBIND11_EMBEDDED_MODULE(pydefix, m) { m.attr("IDIR") = IDIR; m.attr("JDIR") = JDIR; m.attr("KDIR") = KDIR; + + m.attr("prank") = idfx::prank; + m.attr("psize") = idfx::psize; + + m.def("GatherIdefixArray",&GatherIdefixArray, "Gather arrays from domain decomposition"); } diff --git a/src/pydefix.hpp b/src/pydefix.hpp index 2f782d89..b0c27ffc 100644 --- a/src/pydefix.hpp +++ b/src/pydefix.hpp @@ -54,24 +54,33 @@ template struct type_caster> { if ( !convert && !py::array_t::check_(src) ) return false; - auto buf = py::array_t::ensure(src); - if ( !buf ) + auto array = py::array_t::ensure(src); + if ( !array ) return false; - auto dims = buf.ndim(); + auto dims = array.ndim(); if ( dims != 4 ) return false; - std::vector shape(4); - - for ( int i = 0 ; i < 4 ; ++i ) - shape[i] = buf.shape()[i]; - - - value = IdefixHostArray4D("pyArray",shape[0], shape[1], shape[2], shape[3]); - - // Still need to fill in with buf.data()+buf.size() - IDEFIX_ERROR("Python->Idefix Not implemented"); + auto buf = array.request(); + + + + value = Kokkos::View> ((T*)buf.ptr, + array.shape()[0], + array.shape()[1], + array.shape()[2], + array.shape()[3]); + + /* + value = Kokkos::View> ((T*)buf.ptr, + array.shape()[0]);*/ return true; } From a37b5f1044729d9050d5ce08aa6c3a979eda8fcd Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Sat, 28 Dec 2024 18:27:19 +0100 Subject: [PATCH 72/83] better gathering with/without boundaries --- src/gridHost.hpp | 8 ++--- src/pydefix.cpp | 93 +++++++++++++++++++++++++++++++++--------------- 2 files changed, 69 insertions(+), 32 deletions(-) diff --git a/src/gridHost.hpp b/src/gridHost.hpp index 065c65af..55283a5c 100644 --- a/src/gridHost.hpp +++ b/src/gridHost.hpp @@ -22,10 +22,10 @@ class GridHost { public: - std::array::HostMirror,3> x; ///< geometrical central points - std::array::HostMirror,3> xr; ///< cell right interface - std::array::HostMirror,3> xl; ///< cell left interface - std::array::HostMirror,3> dx; ///< cell width + std::array,3> x; ///< geometrical central points + std::array,3> xr; ///< cell right interface + std::array,3> xl; ///< cell left interface + std::array,3> dx; ///< cell width std::array xbeg; ///< Beginning of grid std::array xend; ///< End of grid diff --git a/src/pydefix.cpp b/src/pydefix.cpp index d3204e9d..37ddbc01 100644 --- a/src/pydefix.cpp +++ b/src/pydefix.cpp @@ -22,11 +22,9 @@ namespace py = pybind11; int Pydefix::ninstance = 0; -/************************************ - * DataBlockHost Python binding - * **********************************/ -IdefixHostArray4D GatherIdefixArray(IdefixHostArray4D in, DataBlockHost dataHost) { +namespace PydefixTools { +IdefixHostArray4D GatherIdefixArray(IdefixHostArray4D in, DataBlockHost dataHost, bool keepBoundaries = true) { idfx::cout << "I am being called" << std::endl; idfx::cout << "My dims=" << in.extent(0) << " " << in.extent(1) << " " << in.extent(2) << " " << in.extent(3) << std::endl; idfx::cout << "First element=" << in(0,0,0,0) << std::endl; @@ -35,7 +33,11 @@ IdefixHostArray4D GatherIdefixArray(IdefixHostArray4D in, DataBlockH #ifdef WITH_MPI if(idfx::psize > 1) { const int nvar = in.extent(0); - out = IdefixHostArray4D("pydefix::GatheredArray",nvar,grid->np_tot[KDIR],grid->np_tot[JDIR],grid->np_tot[IDIR]); + if(keepBoundaries) { + out = IdefixHostArray4D("pydefix::GatheredArray",nvar,grid->np_tot[KDIR],grid->np_tot[JDIR],grid->np_tot[IDIR]); + } else { + out = IdefixHostArray4D("pydefix::GatheredArray",nvar,grid->np_int[KDIR],grid->np_int[JDIR],grid->np_int[IDIR]); + } if(idfx::prank == 0) { for(int rank = 0 ; rank < idfx::psize ; rank++) { // np_tot: total size of the incoming array @@ -53,14 +55,16 @@ IdefixHostArray4D GatherIdefixArray(IdefixHostArray4D in, DataBlockH beg = dataHost.beg; // Add back boundaries - for(int dir = 0 ; dir < DIMENSIONS ; dir++) { - if(dataHost.lbound[dir] != internal) { - np_int[dir] += dataHost.nghost[dir]; - gbeg[dir] -= dataHost.nghost[dir]; - beg[dir] -= dataHost.nghost[dir]; - } - if(dataHost.rbound[dir] != internal) { - np_int[dir] += dataHost.nghost[dir]; + if(keepBoundaries) { + for(int dir = 0 ; dir < DIMENSIONS ; dir++) { + if(dataHost.lbound[dir] != internal) { + np_int[dir] += dataHost.nghost[dir]; + gbeg[dir] -= dataHost.nghost[dir]; + beg[dir] -= dataHost.nghost[dir]; + } + if(dataHost.rbound[dir] != internal) { + np_int[dir] += dataHost.nghost[dir]; + } } } buf = in; @@ -76,13 +80,17 @@ IdefixHostArray4D GatherIdefixArray(IdefixHostArray4D in, DataBlockH MPI_Recv(buf.data(), nvar*np_tot[IDIR]*np_tot[JDIR]*np_tot[KDIR], realMPI, rank, 014, MPI_COMM_WORLD,&status); } // target rank // Copy data from the buffer + for(int n = 0 ; n < nvar ; n++) { for(int k = 0 ; k < np_int[KDIR] ; k++) { - const int kt = k+gbeg[KDIR]; + int kt = k+gbeg[KDIR]; + if(!keepBoundaries) kt -= dataHost.nghost[KDIR]; for(int j = 0 ; j < np_int[JDIR] ; j++) { - const int jt = j+gbeg[JDIR]; + int jt = j+gbeg[JDIR]; + if(!keepBoundaries) jt -= dataHost.nghost[JDIR]; for(int i = 0 ; i < np_int[IDIR] ; i++) { - const int it = i+gbeg[IDIR]; + int it = i+gbeg[IDIR]; + if(!keepBoundaries) it -= dataHost.nghost[IDIR]; out(n,kt,jt,it) = buf(n,k+beg[KDIR],j+beg[JDIR],i+beg[IDIR]); } } @@ -96,14 +104,16 @@ IdefixHostArray4D GatherIdefixArray(IdefixHostArray4D in, DataBlockH std::array beg = dataHost.beg; // Add back boundaries - for(int dir = 0 ; dir < DIMENSIONS ; dir++) { - if(dataHost.lbound[dir] != internal) { - np_int[dir] += dataHost.nghost[dir]; - gbeg[dir] -= dataHost.nghost[dir]; - beg[dir] -= dataHost.nghost[dir]; - } - if(dataHost.rbound[dir] != internal) { - np_int[dir] += dataHost.nghost[dir]; + if(keepBoundaries) { + for(int dir = 0 ; dir < DIMENSIONS ; dir++) { + if(dataHost.lbound[dir] != internal) { + np_int[dir] += dataHost.nghost[dir]; + gbeg[dir] -= dataHost.nghost[dir]; + beg[dir] -= dataHost.nghost[dir]; + } + if(dataHost.rbound[dir] != internal) { + np_int[dir] += dataHost.nghost[dir]; + } } } @@ -122,9 +132,13 @@ IdefixHostArray4D GatherIdefixArray(IdefixHostArray4D in, DataBlockH #else // No MPI out = in; #endif - idfx::cout << "final dims=" << out.extent(0) << " " << out.extent(1) << " " << out.extent(2) << " " << out.extent(3) << std::endl; return out; -} +} +}// PydefixTools + +/************************************ + * DataBlockHost Python binding + * **********************************/ PYBIND11_EMBEDDED_MODULE(pydefix, m) { py::class_(m, "DataBlockHost") @@ -145,10 +159,31 @@ PYBIND11_EMBEDDED_MODULE(pydefix, m) { .def_readwrite("Ex2", &DataBlockHost::Ex2) .def_readwrite("Ex3", &DataBlockHost::Ex3) #endif + .def_readwrite("xbeg", &DataBlockHost::xbeg) + .def_readwrite("xend", &DataBlockHost::xend) + .def_readwrite("gbeg", &DataBlockHost::gbeg) + .def_readwrite("gend", &DataBlockHost::gend) + .def_readwrite("beg", &DataBlockHost::beg) + .def_readwrite("end", &DataBlockHost::end) + .def_readwrite("np_tot", &DataBlockHost::np_tot) + .def_readwrite("np_int", &DataBlockHost::np_int) + .def_readwrite("nghost", &DataBlockHost::nghost) .def_readwrite("InvDt", &DataBlockHost::InvDt) .def_readwrite("t",&DataBlockHost::t) .def_readwrite("dt",&DataBlockHost::dt); + py::class_(m, "GridHost") + .def(py::init<>()) + .def_readwrite("x", &GridHost::x) + .def_readwrite("xr", &GridHost::xr) + .def_readwrite("xl", &GridHost::xl) + .def_readwrite("dx", &GridHost::dx) + .def_readwrite("xbeg", &GridHost::xbeg) + .def_readwrite("xend", &GridHost::xend) + .def_readwrite("np_tot", &GridHost::np_tot) + .def_readwrite("np_int", &GridHost::np_int) + .def_readwrite("nghost", &GridHost::nghost); + m.attr("RHO") = RHO; m.attr("VX1") = VX1; m.attr("VX2") = VX2; @@ -170,7 +205,7 @@ PYBIND11_EMBEDDED_MODULE(pydefix, m) { m.attr("prank") = idfx::prank; m.attr("psize") = idfx::psize; - m.def("GatherIdefixArray",&GatherIdefixArray, "Gather arrays from domain decomposition"); + m.def("GatherIdefixArray",&PydefixTools::GatherIdefixArray, "Gather arrays from domain decomposition"); } @@ -228,8 +263,10 @@ void Pydefix::Output(DataBlock &data, int n) { "in your input file [python]:output_function"); } DataBlockHost dataHost(data); + GridHost gridHost(*data.mygrid); + gridHost.SyncFromDevice(); dataHost.SyncFromDevice(); - this->CallScript(this->scriptFilename,this->outputFunctionName,dataHost, n); + this->CallScript(this->scriptFilename,this->outputFunctionName,dataHost, gridHost, n); idfx::popRegion(); } From 8b89c246613cb9fff0ee3b7d5f23acc36a9fb311 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 30 Dec 2024 08:49:44 +0100 Subject: [PATCH 73/83] use numpy array to avoid ownership errors --- src/pydefix.cpp | 220 ++++++++++++++++++++++++++++-------------------- src/pydefix.hpp | 42 ++++----- 2 files changed, 150 insertions(+), 112 deletions(-) diff --git a/src/pydefix.cpp b/src/pydefix.cpp index 37ddbc01..7aedbe76 100644 --- a/src/pydefix.cpp +++ b/src/pydefix.cpp @@ -24,84 +24,58 @@ int Pydefix::ninstance = 0; namespace PydefixTools { -IdefixHostArray4D GatherIdefixArray(IdefixHostArray4D in, DataBlockHost dataHost, bool keepBoundaries = true) { - idfx::cout << "I am being called" << std::endl; - idfx::cout << "My dims=" << in.extent(0) << " " << in.extent(1) << " " << in.extent(2) << " " << in.extent(3) << std::endl; - idfx::cout << "First element=" << in(0,0,0,0) << std::endl; +// Functions provided by Idefix in Pydefix for user convenience +py::array_t GatherIdefixArray(IdefixHostArray3D in, + DataBlockHost dataHost, + bool keepBoundaries = true, + bool broadcast = true) { + idfx::pushRegion("PydefixTools::GatherIdefixArray"); Grid *grid = dataHost.data->mygrid; - IdefixHostArray4D out; - #ifdef WITH_MPI - if(idfx::psize > 1) { - const int nvar = in.extent(0); - if(keepBoundaries) { - out = IdefixHostArray4D("pydefix::GatheredArray",nvar,grid->np_tot[KDIR],grid->np_tot[JDIR],grid->np_tot[IDIR]); - } else { - out = IdefixHostArray4D("pydefix::GatheredArray",nvar,grid->np_int[KDIR],grid->np_int[JDIR],grid->np_int[IDIR]); - } - if(idfx::prank == 0) { - for(int rank = 0 ; rank < idfx::psize ; rank++) { - // np_tot: total size of the incoming array - // np_int: size that should be copied into global - // beg: offset in the incoming array where copy should begin - // gbeg: offset in the global array where copy should be begin - MPI_Status status; - std::array np_int,np_tot, beg, gbeg; - IdefixHostArray4D buf; - - if(rank==0) { - np_int = dataHost.np_int; - np_tot = dataHost.np_tot; - gbeg = dataHost.gbeg; - beg = dataHost.beg; - - // Add back boundaries - if(keepBoundaries) { - for(int dir = 0 ; dir < DIMENSIONS ; dir++) { - if(dataHost.lbound[dir] != internal) { - np_int[dir] += dataHost.nghost[dir]; - gbeg[dir] -= dataHost.nghost[dir]; - beg[dir] -= dataHost.nghost[dir]; - } - if(dataHost.rbound[dir] != internal) { - np_int[dir] += dataHost.nghost[dir]; - } - } - } - buf = in; - } else { // target rank >0 - // Fetch the local array size - MPI_Recv(np_int.data(), 3, MPI_INT, rank, 010, MPI_COMM_WORLD, &status); - MPI_Recv(np_tot.data(), 3, MPI_INT, rank, 011, MPI_COMM_WORLD, &status); - MPI_Recv(beg.data(), 3, MPI_INT, rank, 012, MPI_COMM_WORLD, &status); - MPI_Recv(gbeg.data(), 3, MPI_INT, rank, 013, MPI_COMM_WORLD, &status); - - buf = IdefixHostArray4D("pydefix::tempArray",nvar,np_tot[KDIR],np_tot[JDIR],np_tot[IDIR]); - // Fetch data - MPI_Recv(buf.data(), nvar*np_tot[IDIR]*np_tot[JDIR]*np_tot[KDIR], realMPI, rank, 014, MPI_COMM_WORLD,&status); - } // target rank - // Copy data from the buffer - - for(int n = 0 ; n < nvar ; n++) { - for(int k = 0 ; k < np_int[KDIR] ; k++) { - int kt = k+gbeg[KDIR]; - if(!keepBoundaries) kt -= dataHost.nghost[KDIR]; - for(int j = 0 ; j < np_int[JDIR] ; j++) { - int jt = j+gbeg[JDIR]; - if(!keepBoundaries) jt -= dataHost.nghost[JDIR]; - for(int i = 0 ; i < np_int[IDIR] ; i++) { - int it = i+gbeg[IDIR]; - if(!keepBoundaries) it -= dataHost.nghost[IDIR]; - out(n,kt,jt,it) = buf(n,k+beg[KDIR],j+beg[JDIR],i+beg[IDIR]); - } - } - } - }// End for - }// End loop on target rank for root process - } else { // MPI prank >0 - std::array np_int = dataHost.np_int; - std::array np_tot = dataHost.np_tot; - std::array gbeg = dataHost.gbeg; - std::array beg = dataHost.beg; + IdefixHostArray3D out; + py::array_t pyOut; + if(broadcast || idfx::prank==0) { + if(keepBoundaries) { + // Create a python-managed array, with memory accessible from Kokkos + pyOut = py::array_t({grid->np_tot[KDIR], + grid->np_tot[JDIR], + grid->np_tot[IDIR]}); + out = Kokkos::View> + (reinterpret_cast(pyOut.request().ptr), + grid->np_tot[KDIR], + grid->np_tot[JDIR], + grid->np_tot[IDIR]); + + } else { + pyOut = py::array_t({grid->np_int[KDIR], + grid->np_int[JDIR], + grid->np_int[IDIR]}); + out = Kokkos::View> + (reinterpret_cast(pyOut.request().ptr), + grid->np_int[KDIR], + grid->np_int[JDIR], + grid->np_int[IDIR]); + } + } + if(idfx::prank == 0) { + for(int rank = 0 ; rank < idfx::psize ; rank++) { + // np_tot: total size of the incoming array + // np_int: size that should be copied into global + // beg: offset in the incoming array where copy should begin + // gbeg: offset in the global array where copy should be begin + std::array np_int,np_tot, beg, gbeg; + IdefixHostArray3D buf; + + if(rank==0) { + np_int = dataHost.np_int; + np_tot = dataHost.np_tot; + gbeg = dataHost.gbeg; + beg = dataHost.beg; // Add back boundaries if(keepBoundaries) { @@ -116,25 +90,82 @@ IdefixHostArray4D GatherIdefixArray(IdefixHostArray4D in, DataBlockH } } } + buf = in; + } else { // target rank >0 + #ifdef WITH_MPI + MPI_Status status; + // Fetch the local array size + MPI_Recv(np_int.data(), 3, MPI_INT, rank, 010, MPI_COMM_WORLD, &status); + MPI_Recv(np_tot.data(), 3, MPI_INT, rank, 011, MPI_COMM_WORLD, &status); + MPI_Recv(beg.data(), 3, MPI_INT, rank, 012, MPI_COMM_WORLD, &status); + MPI_Recv(gbeg.data(), 3, MPI_INT, rank, 013, MPI_COMM_WORLD, &status); + + buf = IdefixHostArray3D("pydefix::tempArray", + np_tot[KDIR],np_tot[JDIR],np_tot[IDIR]); + // Fetch data + MPI_Recv(buf.data(), np_tot[IDIR]*np_tot[JDIR]*np_tot[KDIR], + realMPI, rank, 014, MPI_COMM_WORLD,&status); + #else + IDEFIX_ERROR("Can't deal with psize>1 without MPI."); + #endif + } // target rank + // Copy data from the buffer - // send the local array size - MPI_Send(np_int.data(), 3, MPI_INT, 0, 010, MPI_COMM_WORLD); - MPI_Send(np_tot.data(), 3, MPI_INT, 0, 011, MPI_COMM_WORLD); - MPI_Send(beg.data(), 3, MPI_INT, 0, 012, MPI_COMM_WORLD); - MPI_Send(gbeg.data(), 3, MPI_INT, 0, 013, MPI_COMM_WORLD); - MPI_Send(in.data(), nvar*np_tot[IDIR]*np_tot[JDIR]*np_tot[KDIR], realMPI, 0, 014, MPI_COMM_WORLD); // Allocate array + for(int k = 0 ; k < np_int[KDIR] ; k++) { + int kt = k+gbeg[KDIR]; + if(!keepBoundaries) kt -= dataHost.nghost[KDIR]; + for(int j = 0 ; j < np_int[JDIR] ; j++) { + int jt = j+gbeg[JDIR]; + if(!keepBoundaries) jt -= dataHost.nghost[JDIR]; + for(int i = 0 ; i < np_int[IDIR] ; i++) { + int it = i+gbeg[IDIR]; + if(!keepBoundaries) it -= dataHost.nghost[IDIR]; + out(kt,jt,it) = buf(k+beg[KDIR],j+beg[JDIR],i+beg[IDIR]); + } + } + }// End for + }// End loop on target rank for root process + } else { // MPI prank >0 + std::array np_int = dataHost.np_int; + std::array np_tot = dataHost.np_tot; + std::array gbeg = dataHost.gbeg; + std::array beg = dataHost.beg; + + // Add back boundaries + if(keepBoundaries) { + for(int dir = 0 ; dir < DIMENSIONS ; dir++) { + if(dataHost.lbound[dir] != internal) { + np_int[dir] += dataHost.nghost[dir]; + gbeg[dir] -= dataHost.nghost[dir]; + beg[dir] -= dataHost.nghost[dir]; + } + if(dataHost.rbound[dir] != internal) { + np_int[dir] += dataHost.nghost[dir]; + } } - // All is transfered - MPI_Bcast(out.data(), nvar*grid->np_tot[KDIR]*grid->np_tot[JDIR]*grid->np_tot[IDIR], realMPI, 0, MPI_COMM_WORLD); - } else { // MPI with a single proc - out = in; } - #else // No MPI - out = in; + #ifdef WITH_MPI + // send the local array size + MPI_Send(np_int.data(), 3, MPI_INT, 0, 010, MPI_COMM_WORLD); + MPI_Send(np_tot.data(), 3, MPI_INT, 0, 011, MPI_COMM_WORLD); + MPI_Send(beg.data(), 3, MPI_INT, 0, 012, MPI_COMM_WORLD); + MPI_Send(gbeg.data(), 3, MPI_INT, 0, 013, MPI_COMM_WORLD); + MPI_Send(in.data(), np_tot[IDIR]*np_tot[JDIR]*np_tot[KDIR], realMPI, 0, 014, MPI_COMM_WORLD); + #else + IDEFIX_ERROR("Can't deal with psize>1 without MPI."); + #endif + } + // All is transfered + #ifdef WITH_MPI + if(broadcast) { + MPI_Bcast(out.data(), out.extent(0)*out.extent(1)*out.extent(2), realMPI, 0, MPI_COMM_WORLD); + } #endif - return out; + + idfx::popRegion(); + return pyOut; } -}// PydefixTools +// namespace PydefixTools /************************************ * DataBlockHost Python binding @@ -205,7 +236,12 @@ PYBIND11_EMBEDDED_MODULE(pydefix, m) { m.attr("prank") = idfx::prank; m.attr("psize") = idfx::psize; - m.def("GatherIdefixArray",&PydefixTools::GatherIdefixArray, "Gather arrays from domain decomposition"); + m.def("GatherIdefixArray",&PydefixTools::GatherIdefixArray, + py::arg("in"), + py::arg("data"), + py::arg("keepBoundaries") = true, + py::arg("broadcast") = true, + "Gather arrays from MPI domain decomposition"); } diff --git a/src/pydefix.hpp b/src/pydefix.hpp index b0c27ffc..572a92c5 100644 --- a/src/pydefix.hpp +++ b/src/pydefix.hpp @@ -64,23 +64,17 @@ template struct type_caster> { auto buf = array.request(); - - value = Kokkos::View> ((T*)buf.ptr, + + value = Kokkos::View> (reinterpret_cast(buf.ptr), array.shape()[0], array.shape()[1], array.shape()[2], array.shape()[3]); - /* - value = Kokkos::View> ((T*)buf.ptr, - array.shape()[0]);*/ return true; } @@ -106,27 +100,31 @@ template struct type_caster> { // Conversion part 1 (Python -> C++) bool load(py::handle src, bool convert) { + idfx::pushRegion("Pydefix::TypeCaster3D Python->C"); if ( !convert && !py::array_t::check_(src) ) return false; - auto buf = py::array_t::ensure(src); - if ( !buf ) + auto array = py::array_t::ensure(src); + if ( !array ) return false; - auto dims = buf.ndim(); + auto dims = array.ndim(); if ( dims != 3 ) return false; - std::vector shape(3); + auto buf = array.request(); - for ( int i = 0 ; i < 3 ; ++i ) - shape[i] = buf.shape()[i]; - value = IdefixHostArray3D("pyArray",shape[0], shape[1], shape[2]); + value = Kokkos::View> (reinterpret_cast(buf.ptr), + array.shape()[0], + array.shape()[1], + array.shape()[2]); - // Still need to fill in with buf.data()+buf.size() - IDEFIX_ERROR("Python->Idefix Not implemented"); + idfx::popRegion(); return true; } @@ -134,11 +132,15 @@ template struct type_caster> { static py::handle cast(const IdefixHostArray3D& src, py::return_value_policy policy, py::handle parent) { + idfx::pushRegion("Pydefix::TypeCaster3D C->Python"); + // Keep a local reference + auto arr = src; py::none dummyDataOwner; py::array_t a({src.extent(0), src.extent(1), src.extent(2)}, src.data(),dummyDataOwner); + idfx::popRegion(); return a.release(); } }; From f00986cbc9e6c5004b5650cef9ca9e15787ac26b Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 30 Dec 2024 08:53:31 +0100 Subject: [PATCH 74/83] missplaced namespace --- src/pydefix.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pydefix.cpp b/src/pydefix.cpp index 7aedbe76..f0d8bc78 100644 --- a/src/pydefix.cpp +++ b/src/pydefix.cpp @@ -165,7 +165,7 @@ py::array_t GatherIdefixArray(IdefixHostArray3D idfx::popRegion(); return pyOut; } -// namespace PydefixTools +}// namespace PydefixTools /************************************ * DataBlockHost Python binding From 30f8433074b3c85a5d5d8a58d3e7ba8e8afc890c Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 6 Jan 2025 09:32:38 +0100 Subject: [PATCH 75/83] make python auto detection work on MacOs. No need to specify pybind11 path anymore --- CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f7095d2c..d1a2e3d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -124,7 +124,10 @@ endif() if(Idefix_PYTHON) add_compile_definitions("WITH_PYTHON") - #set(PYBIND11_FINDPYTHON ON CACHE BOOL "Idefix requires python" FORCE) + if (NOT DEFINED Python_FIND_FRAMEWORK) + set(Python_FIND_FRAMEWORK "LAST") # Use Apple's python only at last resort on Macos + endif () + set(PYBIND11_FINDPYTHON ON CACHE BOOL "Idefix requires python" FORCE) find_package(pybind11 REQUIRED) target_link_libraries(idefix pybind11::embed) target_sources(idefix From a541532195b4a2db07686807c23d2cba5b82ff6e Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 6 Jan 2025 09:42:06 +0100 Subject: [PATCH 76/83] type caster for 2D and 1D numpy->Idefix Array (not used as of now) --- src/pydefix.hpp | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/src/pydefix.hpp b/src/pydefix.hpp index 572a92c5..eda6a70e 100644 --- a/src/pydefix.hpp +++ b/src/pydefix.hpp @@ -114,8 +114,6 @@ template struct type_caster> { auto buf = array.request(); - - value = Kokkos::View struct type_caster> { if ( dims != 2 ) return false; - std::vector shape(2); - - for ( int i = 0 ; i < 2 ; ++i ) - shape[i] = buf.shape()[i]; - + auto buf = array.request(); - value = IdefixHostArray2D("pyArray",shape[0], shape[1]); + value = Kokkos::View> (reinterpret_cast(buf.ptr), + array.shape()[0], + array.shape()[1]); - // Still need to fill in with buf.data()+buf.size() - IDEFIX_ERROR("Python->Idefix Not implemented"); + idfx::popRegion(); return true; } @@ -202,16 +200,15 @@ template struct type_caster> { if ( dims != 1 ) return false; - std::vector shape(1); - - for ( int i = 0 ; i < 1 ; ++i ) - shape[i] = buf.shape()[i]; - + auto buf = array.request(); - value = IdefixHostArray1D("pyArray",shape[0]); + value = Kokkos::View> (reinterpret_cast(buf.ptr), + array.shape()[0]); - // Still need to fill in with buf.data()+buf.size() - IDEFIX_ERROR("Python->Idefix Not implemented"); + idfx::popRegion(); return true; } From ed7de3c416822a6b6ed376b0c335fa587556aa10 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 6 Jan 2025 10:52:56 +0100 Subject: [PATCH 77/83] proper example with MPI reduction --- .pre-commit-config.yaml | 4 ++++ src/pydefix.hpp | 12 +++++------ test/IO/pydefix/pydefix_example.py | 34 +++++++++++++++++------------- 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 726f0045..72360011 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,6 +34,10 @@ repos: - B # flake8-bugbear - I # isort - NPY # numpy-specific rules + - --ignore + - F405 + - --ignore + - F403 # ignore import * - repo: https://github.com/neutrinoceros/inifix rev: v5.0.2 diff --git a/src/pydefix.hpp b/src/pydefix.hpp index eda6a70e..a12086f8 100644 --- a/src/pydefix.hpp +++ b/src/pydefix.hpp @@ -152,11 +152,11 @@ template struct type_caster> { if ( !convert && !py::array_t::check_(src) ) return false; - auto buf = py::array_t::ensure(src); - if ( !buf ) + auto array = py::array_t::ensure(src); + if ( !array ) return false; - auto dims = buf.ndim(); + auto dims = array.ndim(); if ( dims != 2 ) return false; @@ -192,11 +192,11 @@ template struct type_caster> { if ( !convert && !py::array_t::check_(src) ) return false; - auto buf = py::array_t::ensure(src); - if ( !buf ) + auto array = py::array_t::ensure(src); + if ( !array ) return false; - auto dims = buf.ndim(); + auto dims = array.ndim(); if ( dims != 1 ) return false; diff --git a/test/IO/pydefix/pydefix_example.py b/test/IO/pydefix/pydefix_example.py index 258f674f..a08e8646 100644 --- a/test/IO/pydefix/pydefix_example.py +++ b/test/IO/pydefix/pydefix_example.py @@ -1,29 +1,33 @@ -import pydefix as pdfx +from pydefix import * import numpy as np import matplotlib.pyplot as plt # The output function # the only argument is dataBlockHost python object, wrapping a dataBlockHost Idefix object -def output(data,n): - plt.close() - plt.figure() - plt.pcolormesh(data.x[pdfx.IDIR],data.x[pdfx.JDIR],data.Vc[pdfx.PRS,0,:,:],label='PRS',vmin=0.02,vmax=0.5,cmap='plasma') - plt.title("t=%.2f"%data.t) - plt.colorbar() - plt.savefig("PRS.%.4d.png"%n) +def output(data,grid,n): + # Note: if MPI is not enabled, GatherIdefixArray still works (but merely does a local copy) + pressure = GatherIdefixArray(data.Vc[PRS,:,:,:],data,broadcast=False,keepBoundaries=True) + # only process #0 performs the output + if prank==0: + plt.close() + plt.figure() + plt.pcolormesh(grid.x[IDIR],grid.x[JDIR],pressure[0,:,:],label='PRS',vmin=0.02,vmax=0.5,cmap='plasma') + plt.title("t=%.2f"%data.t) + plt.colorbar() + plt.savefig("PRS.%.4d.png"%n) def initflow(data): # Field amplitude B0 = 1/np.sqrt(4*np.pi) - [z,y,x] = np.meshgrid(data.x[pdfx.KDIR], data.x[pdfx.JDIR], data.x[pdfx.IDIR], indexing='ij') + [z,y,x] = np.meshgrid(data.x[KDIR], data.x[JDIR], data.x[IDIR], indexing='ij') # Initialize the flow - data.Vc[pdfx.RHO,:,:,:] = 25/(36*np.pi) - data.Vc[pdfx.PRS,:,:,:] = 5/(12*np.pi) - data.Vc[pdfx.VX1,:,:,:] = -np.sin(2*np.pi*y) - data.Vc[pdfx.VX2,:,:,:] = np.sin(2*np.pi*x) + data.Vc[RHO,:,:,:] = 25/(36*np.pi) + data.Vc[PRS,:,:,:] = 5/(12*np.pi) + data.Vc[VX1,:,:,:] = -np.sin(2*np.pi*y) + data.Vc[VX2,:,:,:] = np.sin(2*np.pi*x) - data.Vs[pdfx.BX1s,:,:-1,:-1] = -B0*np.sin(2*np.pi*y) - data.Vs[pdfx.BX2s,:,:-1,:-1] = B0*np.sin(4*np.pi*x) + data.Vs[BX1s,:,:-1,:-1] = -B0*np.sin(2*np.pi*y) + data.Vs[BX2s,:,:-1,:-1] = B0*np.sin(4*np.pi*x) From a7d809a12800140a857c89a28eb373c0ffea5c4d Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 6 Jan 2025 14:49:32 +0100 Subject: [PATCH 78/83] add documentation for pydefix --- doc/source/index.rst | 4 + doc/source/modules.rst | 4 + doc/source/modules/pydefix.rst | 126 ++++++++++++++++++++++++++++ doc/source/reference/idefix.ini.rst | 20 +++++ doc/source/reference/outputs.rst | 11 ++- 5 files changed, 161 insertions(+), 4 deletions(-) create mode 100644 doc/source/modules/pydefix.rst diff --git a/doc/source/index.rst b/doc/source/index.rst index fa8dadb3..128d591a 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -33,6 +33,10 @@ MPI library When using MPI parallelisation, *Idefix* relies on an external MPI library. *Idefix* has been tested successfully with OpenMPI and IntelMPI libraries. When used on GPU architectures, *Idefix* assumes that the MPI library is GPU-Aware. If unsure, check this last point with your system administrator. +Python + When using *Idefix* with its python interface through the module `Pydefix`, *Idefix* relies on an external python>=3.8 interpreter with the module `pybind11 `_ + installed. + ================ Features ================ diff --git a/doc/source/modules.rst b/doc/source/modules.rst index 13287ece..2f1f3787 100644 --- a/doc/source/modules.rst +++ b/doc/source/modules.rst @@ -28,6 +28,9 @@ In this section, you will find a more detailed documentation about each module t :ref:`gridCoarseningModule` The grid coarsening module, that allows to derefine the grid in particular locations to speed up the computation. +:ref:`pydefixModule` + The Python interaction module, that allows Idefix to interact directly with a python interpreter. + .. toctree:: :maxdepth: 2 @@ -40,3 +43,4 @@ In this section, you will find a more detailed documentation about each module t modules/selfGravity.rst modules/braginskii.rst modules/gridCoarsening.rst + modules/pydefix.rst diff --git a/doc/source/modules/pydefix.rst b/doc/source/modules/pydefix.rst new file mode 100644 index 00000000..f6828420 --- /dev/null +++ b/doc/source/modules/pydefix.rst @@ -0,0 +1,126 @@ +.. _pydefixModule: + +Pydefix module +============== + +The Pydefix module allows Idefix to talk directly to a Python interpreter while running. It can be used to create your own initial condition +and/or for custom outputs produced from Python. Pydefix relies on the pybind11 python package + +The essence of Pydefix is to allows the user to have a direct access to Idefix data structure from Python without writing/accessing any file. In particular, IdefixArrays are viewed as numpy arrays in Python. +Note however that to keep things simple, Pydefix works on the host memory space only, and hence sync data to/from the GPU (if used) before invoking Python functions. Hence, using Pydefix for outputs induces +a significant loss of performances. + + +Before you start +---------------- +Pybind11 installation ++++++++++++++++++++++ + +In order to use pydefix, you need to be working in a python>=3.8 environement that includes `pybind11 `_. Follow the instruction of your package manager to install pybind11>=2.12. + +Pydefix usage +------------- +Idefix Configuration +++++++++++++++++++++ + +In order to use Pydefix, you need to switch on ``Idefix_PYTHON`` in cmake. This will auto-detect Python and check that pybind11 can be used effectively. + + +Run Idefix with Pydefix ++++++++++++++++++++++++ + +Pydefix is typically enabled from your input file `idefix.ini` in the block ``[Python]``. The following parameters are available: + ++------------------------+-----------------------+-----------------------------------------------------------------------------------------------------------+ +| Entry name | Parameter type | Comment | ++========================+=======================+===========================================================================================================+ +| script | string | | (Mandatory) Filename (*without ".py"!*) of the python script that Idefix should use. | +| | | | The script should be in the same location as the Idefix executable file. | ++------------------------+-----------------------+-----------------------------------------------------------------------------------------------------------+ +| output_function | string | | (Optional) Name of the function that will be called for each output event (the function should be | +| | | | defined in the python script above). When ommited, pydefix output functions are disabled. | +| | | | The periodicity of the pydefix output routine is set in the block:entry `[Output]:python` | ++------------------------+-----------------------+-----------------------------------------------------------------------------------------------------------+ +| initflow_function | string | | (Optional) Name of the python function that will be called to initialize the flow in place of the C++ | +| | | | function ``Setup::InitFlow``. Revert to ``Setup::Initflow`` when ommited. | ++------------------------+-----------------------+-----------------------------------------------------------------------------------------------------------+ + +Python script ++++++++++++++ + +When using Pydefix, idefix expects a python script to be specified in the input file (see ``script`` parameter above). To be fully functionnal, you should import the ``pydefix`` module at the beginning +of your python script (you can also import other python module, as any python script). + +Your python script should define functions that will be called while Idefix is running: +* The signature of the ``initflow`` function should be ``(data)`` where ``data`` is a python structure matching Idefix's ``DataBlockHost`` class. +* The signature of the ``output`` function should be ``(data,grid,n)`` where ``data`` is a python structure matching Idefix's ``DataBlockHost`` class, ``grid`` is Idefix's ``GridHost`` class, and ``n`` is an integer representing the current number of the output + +MPI parallelism ++++++++++++++++ +When Idefix runs with MPI parallelism enabled, a python interpreter and script is launched by each MPI process. Each of these script is independent +and have access to its local ``dataBlockHost``. The `pydefix` module however gives access to the local rank ``prank`` and total MPI size ``psize``. In addition, +pydefix provides the function ``GatherIdefixArray`` to gather the data distributed among each process without invoking MPI directly in python. This function +expects a 3D distributed IdefixArray in entry following the signature + +.. code-block:: c++ + + GatherIdefixArray(IdefixHostArray3D in, // 3D distributed array + DataBlockHost dataHost, // dataBlock structure + bool keepBoundaries = true, // Whether we keep the ghost zones in the returned array + bool broadcast = true) // Whether the returned array is available only in proc #0 or in every proc (caution! possibly requires lots of memory) + +This function is used as follows: + +.. code-block:: python + + import pydefix as pdfx # mandatory + import numpy as np + import matplotlib.pyplot as plt + + def output(data,grid,n): + # Gather pressure field from every process in process #0 (set broadcast to True to distribute the result to all processes) + prs = pdfx.GatherIdefixArray(data.Vc[pdfx.PRS,:,:,:],data,broadcast=False) + + # Only root process performs this + if pdfx.prank==0: + x=grid.x[pdfx.IDIR] # The grid contains the full domain size, while the datablock contains only local information + y=grid.x[pdfx.JDIR] + plt.figure() + plt.pcolormesh(x,y,prs[0,:,:],cmap='plasma') + + +.. note:: + For more advanced usage, it is also possibly to directly call MPI routines from python using the `Mpi4py `_ module. + +Example ++++++++ + +An example is provided in the directory `test/IO/python`. This example shows how to use Idefix with pure python initial conditions and outputs. +It reproduces the 2D OrszagTang vortex available in MHD/OrszagTang without requiring any single line of C++ code from the user. + +The python script `pydefix_example.py` initializes the initial condition of the OT test (``initflow``) and produces a series of PNG files through matplotlib (`output`). + +Troubleshooting +--------------- + +It during configuration stage, you get:: + + CMake Error at CMakeLists.txt:122 (find_package): + By not providing "Findpybind11.cmake" in CMAKE_MODULE_PATH this project has + asked CMake to find a package configuration file provided by "pybind11", + but CMake did not find one. + +It means that cmake cannot find the location of pybind11 (this typically happens on MacOs). In order to locate pybind11, open a python interpreter, and get pybind11 install dir through: + +.. code-block:: python + + import pybind11 + print(pybind11.__file__) + +You can then exit the interpreter and set the pybind11_DIR environement variable to the right path: + +.. code-block:: bash + + export pybind11_DIR=env/lib/python3.10/site-packages/pybind11 + +you can then run cmake which should be able to find pybind11, and compile the code. diff --git a/doc/source/reference/idefix.ini.rst b/doc/source/reference/idefix.ini.rst index a7309f10..c9f9c578 100644 --- a/doc/source/reference/idefix.ini.rst +++ b/doc/source/reference/idefix.ini.rst @@ -358,6 +358,23 @@ and ``X1-end``, ``X2-end``, ``X3-end`` for the right boundaries. ``X2`` boundari | | | (see :ref:`userdefBoundaries`) | +----------------+------------------------------------------------------------------------------------------------------------------+ +``Python`` section +------------------ + +This section describes the python script and function that can interact with Idefix while running using the Pydefix module (see :ref:`pydefixModule`) + ++------------------------+-----------------------+-----------------------------------------------------------------------------------------------------------+ +| Entry name | Parameter type | Comment | ++========================+=======================+===========================================================================================================+ +| script | string | | (Mandatory) Filename (*without ".py"!*) of the python script that Idefix should use. | +| | | | The script should be in location of Idefix executable file | ++------------------------+-----------------------+-----------------------------------------------------------------------------------------------------------+ +| output_function | string | | (Optional) Name of the function that will be called for each output event (the function should be | +| | | | defined in the python script above). When ommited, pydefix output functions are disabled. | ++------------------------+-----------------------+-----------------------------------------------------------------------------------------------------------+ +| initflow_function | string | | (optional) Name of the python function that will be called to initialize the flow in place of the C++ | +| | | | function `Setup::InitFlow`. Revert to `Setup::Initflow`` when ommited. | ++------------------------+-----------------------+-----------------------------------------------------------------------------------------------------------+ .. _outputSection: @@ -411,6 +428,9 @@ This section describes the outputs *Idefix* produces. For more details about eac | | | | (see :ref:`functionEnrollment`). The user-defined variables defined by this function | | | | | are then written as new variables in vtk and/or xdmf outputs. | +----------------+-------------------------+--------------------------------------------------------------------------------------------------+ +| python | float | | Time interval between pydefix outputs, in code units. | +| | | | If negative, periodic pydefix outputs are disabled. | ++----------------+-------------------------+--------------------------------------------------------------------------------------------------+ .. note:: Even if dumps are not mentionned in your input file (and are therefore disabled), dump files are still produced when *Idefix* captures a signal diff --git a/doc/source/reference/outputs.rst b/doc/source/reference/outputs.rst index d531b497..2604d9ab 100644 --- a/doc/source/reference/outputs.rst +++ b/doc/source/reference/outputs.rst @@ -7,7 +7,7 @@ Output formats -------------- *Idefix* uses several types of outputs you may want for your setup. By default, *Idefix* allows -for 4 kinds of outputs: +for 5 kinds of outputs: * logs which essentially tells the user what *Idefix* is currently doing. When running in serial, logs are sent to stdout, but when MPI is enabled, only the logs of the rank 0 process is sent to stdout, and each process (including rank 0) simultaneously writes a @@ -21,17 +21,20 @@ for 4 kinds of outputs: or `Visit `_. The XDMF format relies on the HDF5 format and therefore requires *Idefix* to be configured with HDF5 support. * user-defined analysis files. These are totally left to the user. They usually consist of ascii tables defined by the user, but they can be anything. +* python script, that relies on the :ref:`pydefixModule`. This launches a user-defined python function fed with Idefix data. One can then directly plot or interact with Idefix outputs from python. The output periodicity and the userdef variables should all be declared in the input file, as described in :ref:`outputSection`. Defining your own outputs ------------------------- -*Idefix* provides two ways to define your own outputs: analysis, which are used to make your -own output file (e.g. an ascii-tabulated file); and user variables, which are written by *Idefix* output routines. +*Idefix* provides three ways to define your own outputs: analysis, which are used to make your +own output file (e.g. an ascii-tabulated file); user variables, which are written by *Idefix* output routines, and python user-defined +functions that process Idefix's data. Both analysis and uservar requires the definition of a user function which needs to be enrolled following the procedure described -in :ref:`functionEnrollment` and using the function signatures declared in `output.hpp`. +in :ref:`functionEnrollment` and using the function signatures declared in `output.hpp`. The python outputs are described +in the :ref:`pydefixModule`. We provide below an example of a setup using both analysis outputs and uservar outputs From 682f93bb632f76c5d8aa4a4e528479a39a45ec15 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 8 Jan 2025 09:42:32 +0100 Subject: [PATCH 79/83] missing initialisation of boundary conditions in dataBlockHost --- src/dataBlock/dataBlockHost.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/dataBlock/dataBlockHost.cpp b/src/dataBlock/dataBlockHost.cpp index 91f599c2..d570648f 100644 --- a/src/dataBlock/dataBlockHost.cpp +++ b/src/dataBlock/dataBlockHost.cpp @@ -34,6 +34,9 @@ DataBlockHost::DataBlockHost(DataBlock& datain) { nghost = data->nghost; + lbound = data->lbound; + rbound = data->rbound; + xbeg = data->xbeg; xend = data->xend; beg = data->beg; From f89fc65af2072233d0167d5c0f1e84bbc4959da3 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 8 Jan 2025 14:09:53 +0100 Subject: [PATCH 80/83] update recommended configuration on Adastra following discussion with Paul Segretain to be compatible with pydefix --- doc/source/reference/makefile.rst | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/doc/source/reference/makefile.rst b/doc/source/reference/makefile.rst index 4f948e5d..aadc205c 100644 --- a/doc/source/reference/makefile.rst +++ b/doc/source/reference/makefile.rst @@ -108,18 +108,11 @@ We recommend the following modules and environement variables on AdAstra: .. code-block:: bash - module load cpe/23.12 + module load cpe/24.07 module load craype-accel-amd-gfx90a craype-x86-trento module load PrgEnv-cray - module load amd-mixed/5.7.1 - module load rocm/5.7.1 # nécessaire a cause d'un bug de path pas encore fix.. - export HIPCC_COMPILE_FLAGS_APPEND="-isystem ${CRAY_MPICH_PREFIX}/include" - export HIPCC_LINK_FLAGS_APPEND="-L${CRAY_MPICH_PREFIX}/lib -lmpi ${PE_MPICH_GTL_DIR_amd_gfx90a} ${PE_MPICH_GTL_LIBS_amd_gfx90a} -lstdc++fs" - export CXX=hipcc - export CC=hipcc - -The `-lstdc++fs` option being there to guarantee the link to the HIP library and the access to specific -C++17 functions. + module load amd-mixed + module load cray-python/3.11.7 Finally, *Idefix* can be configured to run on Mi250 by enabling HIP and the desired architecture with the following options to ccmake: From 4ab8212cf520bb0b2b888fdb4f4d7153f15bd374 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 15 Jan 2025 11:39:02 +0100 Subject: [PATCH 81/83] test MPI using Send/Recv instead of reduction --- src/mpi.cpp | 113 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 70 insertions(+), 43 deletions(-) diff --git a/src/mpi.cpp b/src/mpi.cpp index d25056fa..4de7739c 100644 --- a/src/mpi.cpp +++ b/src/mpi.cpp @@ -860,54 +860,81 @@ void Mpi::CheckConfig() { IdefixArray1D src("MPIChecksrc",1); IdefixArray1D::HostMirror srcHost = Kokkos::create_mirror_view(src); - srcHost(0) = idfx::prank; - Kokkos::deep_copy(src, srcHost); - - IdefixArray1D dst("MPICheckdst",1); - IdefixArray1D::HostMirror dstHost = Kokkos::create_mirror_view(dst); - - MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN); - - // Capture segfaults - struct sigaction newHandler; - struct sigaction oldHandler; - memset(&newHandler, 0, sizeof(newHandler)); - newHandler.sa_flags = SA_SIGINFO; - newHandler.sa_sigaction = Mpi::SigErrorHandler; - sigaction(SIGSEGV, &newHandler, &oldHandler); - - try { - int ierr = MPI_Allreduce(src.data(), dst.data(), 1, MPI_INT64_T, MPI_SUM, MPI_COMM_WORLD); - if(ierr != 0) { - char MPImsg[MPI_MAX_ERROR_STRING]; - int MPImsgLen; - MPI_Error_string(ierr, MPImsg, &MPImsgLen); - throw std::runtime_error(std::string(MPImsg, MPImsgLen)); + if(idfx::prank == 0) { + srcHost(0) = 0; + Kokkos::deep_copy(src, srcHost); + } + + if(idfx::psize > 1) { + MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN); + + // Capture segfaults + struct sigaction newHandler; + struct sigaction oldHandler; + memset(&newHandler, 0, sizeof(newHandler)); + newHandler.sa_flags = SA_SIGINFO; + newHandler.sa_sigaction = Mpi::SigErrorHandler; + sigaction(SIGSEGV, &newHandler, &oldHandler); + try { + // We next circulate the info round-robin accross all the nodes to check that + // MPI can exchange buffers in idefix arrays + + MPI_Status status; + int ierrSend, ierrRecv; + if(idfx::prank == 0) { + ierrSend = MPI_Send(src.data(), 1, MPI_INT64_T, idfx::prank+1, 1, MPI_COMM_WORLD); + ierrRecv = MPI_Recv(src.data(), 1, MPI_INT64_T, idfx::psize-1, 1, MPI_COMM_WORLD, &status); + } else { + ierrRecv = MPI_Recv(src.data(), 1, MPI_INT64_T, idfx::prank-1, 1, MPI_COMM_WORLD, &status); + // Add our own rank to the data + Kokkos::deep_copy(srcHost, src); + srcHost(0) += idfx::prank; + Kokkos::deep_copy(src, srcHost); + ierrSend = MPI_Send(src.data(), 1, MPI_INT64_T, (idfx::prank+1)%idfx::psize, 1, + MPI_COMM_WORLD); + } + + if(ierrSend != 0) { + char MPImsg[MPI_MAX_ERROR_STRING]; + int MPImsgLen; + MPI_Error_string(ierrSend, MPImsg, &MPImsgLen); + throw std::runtime_error(std::string(MPImsg, MPImsgLen)); + } + if(ierrRecv != 0) { + char MPImsg[MPI_MAX_ERROR_STRING]; + int MPImsgLen; + MPI_Error_string(ierrSend, MPImsg, &MPImsgLen); + throw std::runtime_error(std::string(MPImsg, MPImsgLen)); + } + } catch(std::exception &e) { + std::stringstream errmsg; + errmsg << "Your MPI library is unable to perform Send/Recv on Idefix arrays."; + errmsg << std::endl; + #ifdef KOKKOS_ENABLE_CUDA + errmsg << "Check that your MPI library is CUDA aware." << std::endl; + #elif defined(KOKKOS_ENABLE_HIP) + errmsg << "Check that your MPI library is RocM aware." << std::endl; + #else + errmsg << "Check your MPI library configuration." << std::endl; + #endif + errmsg << "Error: " << e.what() << std::endl; + IDEFIX_ERROR(errmsg); } - } catch(std::exception &e) { - std::stringstream errmsg; - errmsg << "Your MPI library is unable to perform reductions on Idefix arrays."; - errmsg << std::endl; - #ifdef KOKKOS_ENABLE_CUDA - errmsg << "Check that your MPI library is CUDA aware." << std::endl; - #elif defined(KOKKOS_ENABLE_HIP) - errmsg << "Check that your MPI library is RocM aware." << std::endl; - #else - errmsg << "Check your MPI library configuration." << std::endl; - #endif - errmsg << "Error: " << e.what() << std::endl; - IDEFIX_ERROR(errmsg); + // Restore old handlers + sigaction(SIGSEGV, &oldHandler, NULL ); + MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_ARE_FATAL); } - // Restore old handlers - sigaction(SIGSEGV, &oldHandler, NULL ); - MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_ARE_FATAL); - // Check that we have the proper end result - Kokkos::deep_copy(dstHost, dst); + // Check that we have the proper end result + Kokkos::deep_copy(srcHost, src); int64_t size = static_cast(idfx::psize); - if(dstHost(0) != size*(size-1)/2) { + int64_t rank = static_cast(idfx::prank); + int64_t result = rank == 0 ? size*(size-1)/2 : rank*(rank+1)/2; + + if(srcHost(0) != result) { + idfx::cout << "got " << srcHost(0) << " expected " << result << std::endl; std::stringstream errmsg; - errmsg << "Your MPI library managed to perform reduction on Idefix Arrays, but the result "; + errmsg << "Your MPI library managed to perform MPI exchanges on Idefix Arrays, but the result "; errmsg << "is incorrect. " << std::endl; errmsg << "Check your MPI library configuration." << std::endl; IDEFIX_ERROR(errmsg); From ee6dd0bc67ae06d5446df016f26f69c207c6606c Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 15 Jan 2025 13:49:14 +0100 Subject: [PATCH 82/83] update version number --- CHANGELOG.md | 12 ++++++++++++ CMakeLists.txt | 6 +++--- doc/source/conf.py | 2 +- src/main.cpp | 2 +- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d791f8f..a675e176 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.2.00] 2025-01-17 +### Changed + +- + +### Added + +- + +### Removed + + ## [2.1.02] 2024-10-24 ### Changed diff --git a/CMakeLists.txt b/CMakeLists.txt index d1a2e3d3..a8c7ff94 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,10 +5,10 @@ endif() set (CMAKE_CXX_STANDARD 17) set(Idefix_VERSION_MAJOR 2) -set(Idefix_VERSION_MINOR 1) -set(Idefix_VERSION_PATCH 02) +set(Idefix_VERSION_MINOR 2) +set(Idefix_VERSION_PATCH 00) -project (idefix VERSION 2.1.02) +project (idefix VERSION 2.2.00) option(Idefix_MHD "enable MHD" OFF) option(Idefix_MPI "enable Message Passing Interface parallelisation" OFF) option(Idefix_HIGH_ORDER_FARGO "Force Fargo to use a PPM reconstruction scheme" OFF) diff --git a/doc/source/conf.py b/doc/source/conf.py index b712d026..24f9709b 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -23,7 +23,7 @@ author = 'Geoffroy Lesur' # The full version, including alpha/beta/rc tags -release = '2.1.02' +release = '2.2.00' diff --git a/src/main.cpp b/src/main.cpp index 06f408ff..6b65c4f1 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -9,7 +9,7 @@ //@HEADER // ************************************************************************ // -// IDEFIX v 2.1.00 +// IDEFIX v 2.2.00 // // ************************************************************************ //@HEADER From 5d1a15da396f4a0f8b7cd1a6d6579a2f868e8a83 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 15 Jan 2025 14:07:03 +0100 Subject: [PATCH 83/83] update changelog --- CHANGELOG.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a675e176..66f01f7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,13 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [2.2.00] 2025-01-17 ### Changed -- +- Fix a bug that could lead to a segmentation fault when MHD+Fargo+MPI (with X3 domain decomposition) are all enabled (#295) +- Fix a bug that could result in an incorrect magnetic field when initialising B from the vector potential in non-axisymmetric spherical geometry (#293) +- Fix a bug that could result in Idefix believing the MPI library is not Cuda aware for some versions of OpenMPI (#310) +- Ensure that the behaviour in 1D Spherical geometry is identical to Pluto (#291) ### Added -- - -### Removed +- Add the python interface "pydefix", allowing users to initialise and analyse Idefix simulations live from Python without writing any file (#277) +- Add the native Idefix coordinates in VTK file to simplify postprocessing (#292) +- Add code testing on CPU targets using gcc and Intel Oneapi (#300) ## [2.1.02] 2024-10-24