From c0974592ef2016d7be8ab7bebd487f2c9448b29b Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 30 Oct 2023 22:58:16 +0100 Subject: [PATCH 01/28] add Python binding to cmake --- CMakeLists.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2ef5e686..e1e20d4c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,6 +15,7 @@ option(Idefix_HIGH_ORDER_FARGO "Force Fargo to use a PPM reconstruction scheme" option(Idefix_DEBUG "Enable Idefix debug features (makes the code very slow)" OFF) option(Idefix_RUNTIME_CHECKS "Enable runtime sanity checks" OFF) option(Idefix_WERROR "Treat compiler warnings as errors" OFF) +option(Idefix_PYTHON "Enable python bindings (requires pybind11)" OFF) set(Idefix_CXX_FLAGS "" CACHE STRING "Additional compiler/linker flag") set(Idefix_DEFS "definitions.hpp" CACHE FILEPATH "Problem definition header file") option(Idefix_CUSTOM_EOS "Use custom equation of state" OFF) @@ -118,6 +119,12 @@ else() set(Idefix_HDF5 OFF) endif() +if(Idefix_PYTHON) + add_compile_definitions("WITH_PYTHON") + find_package(pybind11 REQUIRED) + target_link_libraries(idefix PRIVATE pybind11::embed) +endif() + if(Idefix_DEBUG) add_compile_definitions("DEBUG") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g") @@ -229,6 +236,7 @@ else() endif() message(STATUS " MPI: ${Idefix_MPI}") message(STATUS " HDF5: ${Idefix_HDF5}") +message(STATUS " Python: ${Idefix_PYTHON}") message(STATUS " Reconstruction: ${Idefix_RECONSTRUCTION}") message(STATUS " Precision: ${Idefix_PRECISION}") message(STATUS " Version: ${Idefix_VERSION}") From 8a07eafa4f57ada729b85fd34dbe53bc351e1982 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 31 Oct 2023 09:20:48 +0100 Subject: [PATCH 02/28] working version with pybind11 --- CMakeLists.txt | 8 ++++++-- src/output/output.hpp | 7 +++++++ src/pydefix.cpp | 33 +++++++++++++++++++++++++++++++++ src/pydefix.hpp | 21 +++++++++++++++++++++ 4 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 src/pydefix.cpp create mode 100644 src/pydefix.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index e1e20d4c..3a0e2d3a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -113,7 +113,7 @@ if(Idefix_HDF5) ) find_package(HDF5 REQUIRED) target_link_libraries(idefix "${HDF5_LIBRARIES}") - target_include_directories(idefix PUBLIC "${HDF5_INCLUDE_DIRS}") + target_include_directories(idefix "${HDF5_INCLUDE_DIRS}") message(STATUS "XDMF (hdf5+xmf) dumps enabled") else() set(Idefix_HDF5 OFF) @@ -122,7 +122,11 @@ endif() if(Idefix_PYTHON) add_compile_definitions("WITH_PYTHON") find_package(pybind11 REQUIRED) - target_link_libraries(idefix PRIVATE pybind11::embed) + target_link_libraries(idefix pybind11::embed) + target_sources(idefix + PUBLIC src/pydefix.cpp + PUBLIC src/pydefix.hpp + ) endif() if(Idefix_DEBUG) diff --git a/src/output/output.hpp b/src/output/output.hpp index 54cb97af..331ae60c 100644 --- a/src/output/output.hpp +++ b/src/output/output.hpp @@ -18,6 +18,9 @@ #ifdef WITH_HDF5 #include "xdmf.hpp" #endif +#ifdef WITH_PYTHON +#include "pydefix.hpp" +#endif #include "dump.hpp" #include "slice.hpp" @@ -81,5 +84,9 @@ class Output { Kokkos::Timer timer; double elapsedTime{0.0}; + + #ifdef WITH_PYTHON + Pydefix pydefix; + #endif }; #endif // OUTPUT_OUTPUT_HPP_ diff --git a/src/pydefix.cpp b/src/pydefix.cpp new file mode 100644 index 00000000..97599a1b --- /dev/null +++ b/src/pydefix.cpp @@ -0,0 +1,33 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#include // everything needed for embedding +#include "idefix.hpp" +#include "pydefix.hpp" + +namespace py = pybind11; + +int Pydefix::ninstance = 0; + +Pydefix::Pydefix() { + ninstance++; + if(ninstance==1) { + idfx::cout << "Pydefix: start interpreter." << std::endl; + + py::initialize_interpreter(); + } + py::print("Hello,from python"); // use the Python API +} + +Pydefix::~Pydefix() { + if(ninstance == 1) { + py::finalize_interpreter(); + idfx::cout << "Pydefix: shutdown interpreter." << std::endl; + } + ninstance--; + idfx::cout << "Pydefix: destroyed." << std::endl; +} diff --git a/src/pydefix.hpp b/src/pydefix.hpp new file mode 100644 index 00000000..37289fe4 --- /dev/null +++ b/src/pydefix.hpp @@ -0,0 +1,21 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef PYDEFIX_HPP_ +#define PYDEFIX_HPP_ + +#include "idefix.hpp" +class Pydefix { + public: + Pydefix(); + ~Pydefix(); + + private: + static int ninstance; +}; + +#endif // PYDEFIX_HPP_ From 18811827a0a468a547dc9dc36a1fc008ab4c3360 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 2 Nov 2023 22:26:16 +0100 Subject: [PATCH 03/28] tentative automatic cast of IdefixArray @ --- src/output/output.cpp | 58 ++++++++++++++++++++++++++++++++++ src/output/output.hpp | 13 ++++++-- src/pydefix.cpp | 11 +++++++ src/pydefix.hpp | 73 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 152 insertions(+), 3 deletions(-) diff --git a/src/output/output.cpp b/src/output/output.cpp index 30a85039..966f6ed0 100644 --- a/src/output/output.cpp +++ b/src/output/output.cpp @@ -9,6 +9,12 @@ #include "dataBlock.hpp" #include "fluid.hpp" #include "slice.hpp" +#include "pydefix.hpp" + +PYBIND11_EMBEDDED_MODULE(embeded, module){ + + py::class_> animal(module, "IdefixArray"); +} Output::Output(Input &input, DataBlock &data) { idfx::pushRegion("Output::Output"); @@ -115,6 +121,22 @@ Output::Output(Input &input, DataBlock &data) { } } + // Initialise python script outputs + if(input.CheckEntry("Output","python")>0) { + #ifndef WITH_PYTHON + IDEFIX_ERROR("Usage of python outputs requires Idefix to be compiled with Python capabilities";) + #else + pythonPeriod = input.Get("Output","python",0); + if(pythonPeriod>=0.0) { // backward compatibility (negative value means no file) + pythonLast = data.t - pythonPeriod; // write something in the next CheckForWrite() + pythonScript = input.Get("Output","python",1); + pythonFunction = input.Get("Output","python",2); + pythonEnabled = true; + pythonNcall = 0; + } + #endif + } + // Register variables that are needed in restart dumps data.dump->RegisterVariable(&dumpLast, "dumpLast"); data.dump->RegisterVariable(&analysisLast, "analysisLast"); @@ -122,6 +144,9 @@ Output::Output(Input &input, DataBlock &data) { #ifdef WITH_HDF5 data.dump->RegisterVariable(&xdmfLast, "xdmfLast"); #endif + #ifdef WITH_PYTHON + data.dump->RegisterVariable(&pythonLast, "pythonLast"); + #endif idfx::popRegion(); } @@ -227,6 +252,39 @@ int Output::CheckForWrites(DataBlock &data) { slices[i]->CheckForWrite(data); } } + + #ifdef WITH_PYTHON + if(pythonEnabled) { + if(data.t >= pythonLast + pythonPeriod) { + elapsedTime -= timer.seconds(); + DataBlockHost d(data); + //try { + auto Vc = pydefix.toNumpyArray(d.Vc); + py::module_ script = py::module_::import(pythonScript.c_str()); + py::module_ embeded = py::module_::import("embeded"); + py::object myV = py::cast(d.Vc); + py::object result = script.attr(pythonFunction.c_str())(pythonNcall, myV); + /*} catch(std::exception &e) { + std::stringstream message; + message << "An exception occured while calling the Python interpreter in" + << "file \"" << pythonScript << "\" function \"" << pythonFunction << "\"" + << std::endl + << e.what() << std::endl; + IDEFIX_ERROR(message); + }*/ + elapsedTime += timer.seconds(); + pythonNcall++; + // Check if our next predicted output should already have happened + if((pythonLast+pythonPeriod <= data.t) && pythonPeriod>0.0) { + // Move forward analysisLast + while(pythonLast <= data.t - pythonPeriod) { + pythonLast += pythonPeriod; + } + } + + } + } + #endif // Do we need a restart dump? if(dumpEnabled) { bool haveClockDump = false; diff --git a/src/output/output.hpp b/src/output/output.hpp index 331ae60c..1fe5e000 100644 --- a/src/output/output.hpp +++ b/src/output/output.hpp @@ -82,11 +82,18 @@ class Output { bool haveSlices = false; std::vector> slices; + #ifdef WITH_PYTHON + Pydefix pydefix; + bool pythonEnabled = false; + real pythonPeriod; + real pythonLast; + int pythonNcall; + std::string pythonScript; + std::string pythonFunction; + #endif + Kokkos::Timer timer; double elapsedTime{0.0}; - #ifdef WITH_PYTHON - Pydefix pydefix; - #endif }; #endif // OUTPUT_OUTPUT_HPP_ diff --git a/src/pydefix.cpp b/src/pydefix.cpp index 97599a1b..966881b1 100644 --- a/src/pydefix.cpp +++ b/src/pydefix.cpp @@ -6,6 +6,7 @@ // *********************************************************************************** #include // everything needed for embedding +#include // for numpy arrays #include "idefix.hpp" #include "pydefix.hpp" @@ -31,3 +32,13 @@ Pydefix::~Pydefix() { ninstance--; idfx::cout << "Pydefix: destroyed." << std::endl; } + +py::array_t Pydefix::toNumpyArray(const IdefixHostArray3D& in) { + py::array_t array({in.extent(0),in.extent(1),in.extent(2)},in.data()); + return array; +} + +py::array_t Pydefix::toNumpyArray(const IdefixHostArray4D& in) { + py::array_t array({in.extent(0),in.extent(1),in.extent(2),in.extent(3)},in.data()); + return array; +} diff --git a/src/pydefix.hpp b/src/pydefix.hpp index 37289fe4..d6587ef2 100644 --- a/src/pydefix.hpp +++ b/src/pydefix.hpp @@ -8,14 +8,87 @@ #ifndef PYDEFIX_HPP_ #define PYDEFIX_HPP_ + +#define PYBIND11_DETAILED_ERROR_MESSAGES +#include +#include +#include #include "idefix.hpp" +#include "pydefix.hpp" + +namespace py = pybind11; + class Pydefix { public: Pydefix(); ~Pydefix(); + py::array_t toNumpyArray(const IdefixHostArray3D&); + py::array_t toNumpyArray(const IdefixHostArray4D&); private: static int ninstance; }; +// Caster for IdefixArray4D +namespace pybind11 { namespace detail { + template struct type_caster> + { + public: + + PYBIND11_TYPE_CASTER(IdefixHostArray4D, _("IdefixHostArray4D")); + + // Conversion part 1 (Python -> C++) + bool load(py::handle src, bool convert) + { + if ( !convert and !py::array_t::check_(src) ) + return false; + + auto buf = py::array_t::ensure(src); + if ( !buf ) + return false; + + auto dims = buf.ndim(); + if ( dims != 4 ) + return false; + + std::vector shape(4); + + for ( int i = 0 ; i < 4 ; ++i ) + shape[i] = buf.shape()[i]; + + + value = IdefixHostArray4D("pyArray",shape[0], shape[1], shape[2], shape[3]); + + // Still need to fill in with buf.data()+buf.size() + + return true; + } + + //Conversion part 2 (C++ -> Python) + static py::handle cast(const IdefixHostArray4D& src, py::return_value_policy policy, py::handle parent) + { + std::cout << "coucou2" << std::endl; + std::vector shape (4); + std::vector strides(4); + + for ( int i = 0 ; i < 4 ; ++i ) { + shape [i] = src.extent [i]; + strides[i] = src.strides[i]*sizeof(T); + } + + //py::array a(std::move(shape), std::move(strides), src.data() ); + py::array_t a({src.extent(0),src.extent(1),src.extent(2),src.extent(3)},src.data()); + return a.release(); + } + }; +}} // namespace pybind11::detail + + + +/* +PYBIND11_PLUGIN(example) { + py::module m("example", "Module description"); + m.def("func", &func, "Function description" ); + return m.ptr(); +}*/ #endif // PYDEFIX_HPP_ From 76bee8c4a995e0ba8df0c688f53d47c6d83e4fae Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 2 Nov 2023 22:26:48 +0100 Subject: [PATCH 04/28] missing files --- test/HD/sod/idefix_py.ini | 27 +++++++++++++++++++++++++++ test/HD/sod/output.py | 6 ++++++ 2 files changed, 33 insertions(+) create mode 100644 test/HD/sod/idefix_py.ini create mode 100644 test/HD/sod/output.py diff --git a/test/HD/sod/idefix_py.ini b/test/HD/sod/idefix_py.ini new file mode 100644 index 00000000..977f2da1 --- /dev/null +++ b/test/HD/sod/idefix_py.ini @@ -0,0 +1,27 @@ +[Grid] +X1-grid 1 0.0 500 u 1.0 +X2-grid 1 0.0 1 u 1.0 +X3-grid 1 0.0 1 u 1.0 + +[TimeIntegrator] +CFL 0.8 +tstop 0.2 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver roe +gamma 1.4 + +[Boundary] +X1-beg outflow +X1-end outflow +X2-beg outflow +X2-end outflow +X3-beg outflow +X3-end outflow + +[Output] +vtk 0.1 +dmp 0.2 +python 0.1 output myfunc diff --git a/test/HD/sod/output.py b/test/HD/sod/output.py new file mode 100644 index 00000000..d514e389 --- /dev/null +++ b/test/HD/sod/output.py @@ -0,0 +1,6 @@ +import numpy as np + +def myfunc(n, array): + print("Python n=%d"%n) + print("Dimensions="+str(array.shape)) + print(array[0,0,0,:]) From 10dc6bf9c5749f42b92217391167a5b53e69a496 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 6 Nov 2023 15:13:41 +0100 Subject: [PATCH 05/28] Fully working python output routines --- src/dataBlock/dataBlockHost.hpp | 32 ++--- src/output/output.cpp | 31 ++--- src/output/output.hpp | 2 - src/pydefix.cpp | 64 +++++++++- src/pydefix.hpp | 207 ++++++++++++++++++++++++-------- test/HD/sod/output.py | 11 +- 6 files changed, 249 insertions(+), 98 deletions(-) diff --git a/src/dataBlock/dataBlockHost.hpp b/src/dataBlock/dataBlockHost.hpp index 86c8c570..4a1e8e5e 100644 --- a/src/dataBlock/dataBlockHost.hpp +++ b/src/dataBlock/dataBlockHost.hpp @@ -25,33 +25,33 @@ class DataBlockHost { public: // Local grid information - std::array::HostMirror,3> x; ///< geometrical central points - std::array::HostMirror,3> xr; ///< cell right interface - std::array::HostMirror,3> xl; ///< cell left interface - std::array::HostMirror,3> dx; ///< cell width + std::array,3> x; ///< geometrical central points + std::array,3> xr; ///< cell right interface + std::array,3> xl; ///< cell left interface + std::array,3> dx; ///< cell width - IdefixArray3D::HostMirror dV; ///< cell volume - std::array::HostMirror,3> A; ///< cell right interface area + IdefixHostArray3D dV; ///< cell volume + std::array,3> A; ///< cell right interface area - IdefixArray4D::HostMirror Vc; ///< Main cell-centered primitive variables index + IdefixHostArray4D Vc; ///< Main cell-centered primitive variables index bool haveDust{false}; std::vector> dustVc; ///< Cell-centered primitive variables index for dust #if MHD == YES - IdefixArray4D::HostMirror Vs; ///< Main face-centered primitive variables index - IdefixArray4D::HostMirror Ve; ///< Main edge-centered primitive variables index - IdefixArray4D::HostMirror J; ///< Current (only when haveCurrent is enabled) + IdefixHostArray4D Vs; ///< Main face-centered primitive variables index + IdefixHostArray4D Ve; ///< Main edge-centered primitive variables index + IdefixHostArray4D J; ///< Current (only when haveCurrent is enabled) - IdefixArray3D::HostMirror Ex1; ///< x1 electric field - IdefixArray3D::HostMirror Ex2; ///< x2 electric field - IdefixArray3D::HostMirror Ex3; ///< x3 electric field + IdefixHostArray3D Ex1; ///< x1 electric field + IdefixHostArray3D Ex2; ///< x2 electric field + IdefixHostArray3D Ex3; ///< x3 electric field #endif - IdefixArray4D::HostMirror Uc; ///< Main cell-centered conservative variables - IdefixArray3D::HostMirror InvDt; ///< Inverse of maximum timestep in each cell + IdefixHostArray4D Uc; ///< Main cell-centered conservative variables + IdefixHostArray3D InvDt; ///< Inverse of maximum timestep in each cell - std::array::HostMirror,3> coarseningLevel; ///< Grid coarsening level + std::array,3> coarseningLevel; ///< Grid coarsening level ///< (only defined when coarsening ///< is enabled) diff --git a/src/output/output.cpp b/src/output/output.cpp index 966f6ed0..90717fb1 100644 --- a/src/output/output.cpp +++ b/src/output/output.cpp @@ -9,12 +9,13 @@ #include "dataBlock.hpp" #include "fluid.hpp" #include "slice.hpp" +#include "dataBlockHost.hpp" + +#ifdef WITH_PYTHON #include "pydefix.hpp" -PYBIND11_EMBEDDED_MODULE(embeded, module){ - py::class_> animal(module, "IdefixArray"); -} +#endif Output::Output(Input &input, DataBlock &data) { idfx::pushRegion("Output::Output"); @@ -124,15 +125,17 @@ Output::Output(Input &input, DataBlock &data) { // Initialise python script outputs if(input.CheckEntry("Output","python")>0) { #ifndef WITH_PYTHON - IDEFIX_ERROR("Usage of python outputs requires Idefix to be compiled with Python capabilities";) + IDEFIX_ERROR("Usage of python outputs requires Idefix to be compiled with Python capabilities"); #else pythonPeriod = input.Get("Output","python",0); if(pythonPeriod>=0.0) { // backward compatibility (negative value means no file) pythonLast = data.t - pythonPeriod; // write something in the next CheckForWrite() pythonScript = input.Get("Output","python",1); + if(pythonScript.substr(pythonScript.length() - 3, 3).compare(".py")==0) { + IDEFIX_ERROR("You should not include the python script .py extension in your input file"); + } pythonFunction = input.Get("Output","python",2); pythonEnabled = true; - pythonNcall = 0; } #endif } @@ -257,23 +260,8 @@ int Output::CheckForWrites(DataBlock &data) { if(pythonEnabled) { if(data.t >= pythonLast + pythonPeriod) { elapsedTime -= timer.seconds(); - DataBlockHost d(data); - //try { - auto Vc = pydefix.toNumpyArray(d.Vc); - py::module_ script = py::module_::import(pythonScript.c_str()); - py::module_ embeded = py::module_::import("embeded"); - py::object myV = py::cast(d.Vc); - py::object result = script.attr(pythonFunction.c_str())(pythonNcall, myV); - /*} catch(std::exception &e) { - std::stringstream message; - message << "An exception occured while calling the Python interpreter in" - << "file \"" << pythonScript << "\" function \"" << pythonFunction << "\"" - << std::endl - << e.what() << std::endl; - IDEFIX_ERROR(message); - }*/ + pydefix.CallScript(&data, pythonScript,pythonFunction); elapsedTime += timer.seconds(); - pythonNcall++; // Check if our next predicted output should already have happened if((pythonLast+pythonPeriod <= data.t) && pythonPeriod>0.0) { // Move forward analysisLast @@ -281,7 +269,6 @@ int Output::CheckForWrites(DataBlock &data) { pythonLast += pythonPeriod; } } - } } #endif diff --git a/src/output/output.hpp b/src/output/output.hpp index 1fe5e000..1be44372 100644 --- a/src/output/output.hpp +++ b/src/output/output.hpp @@ -87,13 +87,11 @@ class Output { bool pythonEnabled = false; real pythonPeriod; real pythonLast; - int pythonNcall; std::string pythonScript; std::string pythonFunction; #endif Kokkos::Timer timer; double elapsedTime{0.0}; - }; #endif // OUTPUT_OUTPUT_HPP_ diff --git a/src/pydefix.cpp b/src/pydefix.cpp index 966881b1..55ce5ef8 100644 --- a/src/pydefix.cpp +++ b/src/pydefix.cpp @@ -5,34 +5,87 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#include "pydefix.hpp" #include // everything needed for embedding #include // for numpy arrays +#include // For STL vectors and containers +#include +#include #include "idefix.hpp" -#include "pydefix.hpp" +#include "dataBlock.hpp" +#include "dataBlockHost.hpp" namespace py = pybind11; int Pydefix::ninstance = 0; +/************************************ + * DataBlockHost Python binding + * **********************************/ + +PYBIND11_EMBEDDED_MODULE(pydefix, m) { + py::class_(m, "DataBlockHost") + .def(py::init<>()) + .def_readwrite("x", &DataBlockHost::x) + .def_readwrite("xr", &DataBlockHost::x) + .def_readwrite("xl", &DataBlockHost::x) + .def_readwrite("dx", &DataBlockHost::x) + .def_readwrite("dV", &DataBlockHost::x) + .def_readwrite("A", &DataBlockHost::A) + .def_readwrite("Vc", &DataBlockHost::Vc) + .def_readwrite("dustVc", &DataBlockHost::dustVc) + #if MHD == YES + .def_readwrite("Vs", &DataBlockHost::Vs) + .def_readwrite("Ve", &DataBlockHost::Ve) + .def_readwrite("J", &DataBlockHost::J) + .def_readwrite("Ex1", &DataBlockHost::Ex1) + .def_readwrite("Ex2", &DataBlockHost::Ex2) + .def_readwrite("Ex3", &DataBlockHost::Ex3) + #endif + .def_readwrite("InvDt", &DataBlockHost::InvDt); +} + + Pydefix::Pydefix() { ninstance++; if(ninstance==1) { - idfx::cout << "Pydefix: start interpreter." << std::endl; + idfx::cout << "Pydefix: start Python interpreter." << std::endl; py::initialize_interpreter(); } - py::print("Hello,from python"); // use the Python API } Pydefix::~Pydefix() { if(ninstance == 1) { py::finalize_interpreter(); - idfx::cout << "Pydefix: shutdown interpreter." << std::endl; + idfx::cout << "Pydefix: shutdown Python interpreter." << std::endl; } ninstance--; - idfx::cout << "Pydefix: destroyed." << std::endl; } +void Pydefix::CallScript(DataBlock *data, std::string scriptName, std::string funcName) { + idfx::pushRegion("Pydefix::CallScript"); + DataBlockHost d(*data); + d.SyncFromDevice(); + try { + //auto Vc = pydefix.toNumpyArray(d.Vc); + py::module_ script = py::module_::import(scriptName.c_str()); + + //py::module_ embeded = py::module_::import("embeded"); + //py::object myV = py::cast(Vc); + py::object result = script.attr(funcName.c_str())(nCalls, d); + } catch(std::exception &e) { + std::stringstream message; + message << "An exception occured while calling the Python interpreter" << std::endl + << "in file \"" << scriptName << ".py\" function \"" << funcName << "\":" + << std::endl + << e.what() << std::endl; + IDEFIX_ERROR(message); + } + nCalls++; + idfx::popRegion(); +} +/* py::array_t Pydefix::toNumpyArray(const IdefixHostArray3D& in) { py::array_t array({in.extent(0),in.extent(1),in.extent(2)},in.data()); return array; @@ -42,3 +95,4 @@ py::array_t Pydefix::toNumpyArray(const IdefixHostArray4D& in) { py::array_t array({in.extent(0),in.extent(1),in.extent(2),in.extent(3)},in.data()); return array; } +*/ diff --git a/src/pydefix.hpp b/src/pydefix.hpp index d6587ef2..b76ea3bb 100644 --- a/src/pydefix.hpp +++ b/src/pydefix.hpp @@ -9,86 +9,195 @@ #define PYDEFIX_HPP_ -#define PYBIND11_DETAILED_ERROR_MESSAGES +//#define PYBIND11_DETAILED_ERROR_MESSAGES #include #include #include +#include +#include #include "idefix.hpp" -#include "pydefix.hpp" namespace py = pybind11; +class DataBlock; + class Pydefix { public: Pydefix(); ~Pydefix(); - py::array_t toNumpyArray(const IdefixHostArray3D&); - py::array_t toNumpyArray(const IdefixHostArray4D&); + void CallScript(DataBlock *, std::string, std::string); private: static int ninstance; + int nCalls{0}; }; -// Caster for IdefixArray4D namespace pybind11 { namespace detail { - template struct type_caster> - { - public: +// Caster for IdefixArray4D +template struct type_caster> { + public: + PYBIND11_TYPE_CASTER(IdefixHostArray4D, _("IdefixHostArray4D")); + + // Conversion part 1 (Python -> C++) + bool load(py::handle src, bool convert) { + if ( !convert && !py::array_t::check_(src) ) + return false; - PYBIND11_TYPE_CASTER(IdefixHostArray4D, _("IdefixHostArray4D")); + auto buf = py::array_t::ensure(src); + if ( !buf ) + return false; - // Conversion part 1 (Python -> C++) - bool load(py::handle src, bool convert) - { - if ( !convert and !py::array_t::check_(src) ) - return false; + auto dims = buf.ndim(); + if ( dims != 4 ) + return false; - auto buf = py::array_t::ensure(src); - if ( !buf ) - return false; + std::vector shape(4); - auto dims = buf.ndim(); - if ( dims != 4 ) - return false; + for ( int i = 0 ; i < 4 ; ++i ) + shape[i] = buf.shape()[i]; - std::vector shape(4); - for ( int i = 0 ; i < 4 ; ++i ) - shape[i] = buf.shape()[i]; + value = IdefixHostArray4D("pyArray",shape[0], shape[1], shape[2], shape[3]); + // Still need to fill in with buf.data()+buf.size() + IDEFIX_ERROR("Python->Idefix Not implemented"); + + return true; + } + + //Conversion part 2 (C++ -> Python) + static py::handle cast(const IdefixHostArray4D& src, + py::return_value_policy policy, + py::handle parent) { + py::array_t a({src.extent(0), + src.extent(1), + src.extent(2), + src.extent(3)}, + src.data()); + return a.release(); + } +}; +// Caster for IdefixArray3D +template struct type_caster> { + public: + PYBIND11_TYPE_CASTER(IdefixHostArray3D, _("IdefixHostArray3D")); - value = IdefixHostArray4D("pyArray",shape[0], shape[1], shape[2], shape[3]); + // Conversion part 1 (Python -> C++) + bool load(py::handle src, bool convert) { + if ( !convert && !py::array_t::check_(src) ) + return false; - // Still need to fill in with buf.data()+buf.size() + auto buf = py::array_t::ensure(src); + if ( !buf ) + return false; - return true; - } + auto dims = buf.ndim(); + if ( dims != 3 ) + return false; + + std::vector shape(3); + + for ( int i = 0 ; i < 3 ; ++i ) + shape[i] = buf.shape()[i]; + + + value = IdefixHostArray3D("pyArray",shape[0], shape[1], shape[2]); + + // Still need to fill in with buf.data()+buf.size() + IDEFIX_ERROR("Python->Idefix Not implemented"); + return true; + } + + //Conversion part 2 (C++ -> Python) + static py::handle cast(const IdefixHostArray3D& src, + py::return_value_policy policy, + py::handle parent) { + py::array_t a({src.extent(0), + src.extent(1), + src.extent(2)}, + src.data()); + return a.release(); + } +}; +// Caster for IdefixArray2D +template struct type_caster> { + public: + PYBIND11_TYPE_CASTER(IdefixHostArray2D, _("IdefixHostArray2D")); - //Conversion part 2 (C++ -> Python) - static py::handle cast(const IdefixHostArray4D& src, py::return_value_policy policy, py::handle parent) - { - std::cout << "coucou2" << std::endl; - std::vector shape (4); - std::vector strides(4); + // Conversion part 1 (Python -> C++) + bool load(py::handle src, bool convert) { + if ( !convert && !py::array_t::check_(src) ) + return false; - for ( int i = 0 ; i < 4 ; ++i ) { - shape [i] = src.extent [i]; - strides[i] = src.strides[i]*sizeof(T); - } + auto buf = py::array_t::ensure(src); + if ( !buf ) + return false; - //py::array a(std::move(shape), std::move(strides), src.data() ); - py::array_t a({src.extent(0),src.extent(1),src.extent(2),src.extent(3)},src.data()); - return a.release(); - } - }; -}} // namespace pybind11::detail + auto dims = buf.ndim(); + if ( dims != 2 ) + return false; + std::vector shape(2); + for ( int i = 0 ; i < 2 ; ++i ) + shape[i] = buf.shape()[i]; + + + value = IdefixHostArray2D("pyArray",shape[0], shape[1]); + + // Still need to fill in with buf.data()+buf.size() + IDEFIX_ERROR("Python->Idefix Not implemented"); + return true; + } + + //Conversion part 2 (C++ -> Python) + static py::handle cast(const IdefixHostArray2D& src, + py::return_value_policy policy, + py::handle parent) { + py::array_t a({src.extent(0),src.extent(1)},src.data()); + return a.release(); + } +}; +// Caster for IdefixArray1D +template struct type_caster> { + public: + PYBIND11_TYPE_CASTER(IdefixHostArray1D, _("IdefixHostArray1D")); + + // Conversion part 1 (Python -> C++) + bool load(py::handle src, bool convert) { + if ( !convert && !py::array_t::check_(src) ) + return false; + + auto buf = py::array_t::ensure(src); + if ( !buf ) + return false; + + auto dims = buf.ndim(); + if ( dims != 1 ) + return false; + + std::vector shape(1); + + for ( int i = 0 ; i < 1 ; ++i ) + shape[i] = buf.shape()[i]; + + + value = IdefixHostArray1D("pyArray",shape[0]); + + // Still need to fill in with buf.data()+buf.size() + IDEFIX_ERROR("Python->Idefix Not implemented"); + return true; + } + + //Conversion part 2 (C++ -> Python) + static py::handle cast(const IdefixHostArray1D& src, + py::return_value_policy policy, + py::handle parent) { + py::array_t a(src.extent(0),src.data()); + return a.release(); + } +}; +} // namespace detail +} // namespace pybind11 -/* -PYBIND11_PLUGIN(example) { - py::module m("example", "Module description"); - m.def("func", &func, "Function description" ); - return m.ptr(); -}*/ #endif // PYDEFIX_HPP_ diff --git a/test/HD/sod/output.py b/test/HD/sod/output.py index d514e389..79f05327 100644 --- a/test/HD/sod/output.py +++ b/test/HD/sod/output.py @@ -1,6 +1,9 @@ -import numpy as np -def myfunc(n, array): +def myfunc(n, data): print("Python n=%d"%n) - print("Dimensions="+str(array.shape)) - print(array[0,0,0,:]) + print(dir(data)) + print("Dimensions="+str(data.Vc.shape)) + print("x1=") + print(data.x[0][:]) + print("data=") + print(data.Vc[0,0,0,:]) From 227e3751c30b308dc4f6f0d18c37f090d99c5a50 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 3 Sep 2024 11:42:54 +0200 Subject: [PATCH 06/28] working now with initial condition (but copies won't work!) --- src/main.cpp | 41 +++++++++++-- src/output/output.cpp | 15 ++--- src/output/output.hpp | 2 - src/pydefix.cpp | 106 +++++++++++++++++++++++++++------ src/pydefix.hpp | 20 +++++-- test/HD/sod/idefix_py.ini | 11 +++- test/HD/sod/output.py | 9 --- test/HD/sod/pydefix_example.py | 14 +++++ 8 files changed, 166 insertions(+), 52 deletions(-) delete mode 100644 test/HD/sod/output.py create mode 100644 test/HD/sod/pydefix_example.py diff --git a/src/main.cpp b/src/main.cpp index 5a7f6f73..6294dcff 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -90,8 +90,12 @@ int main( int argc, char* argv[] ) { // instantiate required objects. DataBlock data(grid, input); TimeIntegrator Tint(input,data); + #ifdef WITH_PYTHON + Pydefix pydefix(input); + #endif Output output(input, data); Setup mysetup(input, grid, data, output); + idfx::cout << "Main: initialisation finished." << std::endl; char host[1024]; @@ -110,6 +114,9 @@ int main( int argc, char* argv[] ) { grid.ShowConfig(); data.ShowConfig(); Tint.ShowConfig(); + #ifdef WITH_PYTHON + pydefix.ShowConfig(); + #endif /////////////////////////////// // Initial conditions (or restart) @@ -117,10 +124,22 @@ int main( int argc, char* argv[] ) { // Are we restarting? if(input.restartRequested) { if(input.forceInitRequested) { - idfx::pushRegion("Setup::Initflow"); - mysetup.InitFlow(data); - data.DeriveVectorPotential(); - idfx::popRegion(); + #ifdef WITH_PYTHON + if(pydefix.haveInitflow) { + idfx::pushRegion("Pydefix::Initflow"); + pydefix.InitFlow(data); + } else { + idfx::pushRegion("Setup::Initflow"); + mysetup.InitFlow(data); + } + data.DeriveVectorPotential(); + idfx::popRegion(); + #else + idfx::pushRegion("Setup::Initflow"); + mysetup.InitFlow(data); + data.DeriveVectorPotential(); + idfx::popRegion(); + #endif } idfx::cout << "Main: Restarting from dump file." << std::endl; bool restartSuccess = output.RestartFromDump(data,input.restartFileNumber); @@ -133,8 +152,18 @@ int main( int argc, char* argv[] ) { } if(!input.restartRequested) { idfx::cout << "Main: Creating initial conditions." << std::endl; - idfx::pushRegion("Setup::Initflow"); - mysetup.InitFlow(data); + #ifdef WITH_PYTHON + if(pydefix.haveInitflow) { + idfx::pushRegion("Pydefix::Initflow"); + pydefix.InitFlow(data); + } else { + idfx::pushRegion("Setup::Initflow"); + mysetup.InitFlow(data); + } + #else + idfx::pushRegion("Setup::Initflow"); + mysetup.InitFlow(data); + #endif idfx::popRegion(); data.DeriveVectorPotential(); // This does something only when evolveVectorPotential is on data.SetBoundaries(); diff --git a/src/output/output.cpp b/src/output/output.cpp index 90717fb1..75d75124 100644 --- a/src/output/output.cpp +++ b/src/output/output.cpp @@ -13,11 +13,13 @@ #ifdef WITH_PYTHON #include "pydefix.hpp" - - #endif -Output::Output(Input &input, DataBlock &data) { +Output::Output(Input &input, DataBlock &data): +#ifdef WITH_PYTHON +pydefix(input) +#endif +{ idfx::pushRegion("Output::Output"); // initialise the output objects for each format @@ -130,11 +132,6 @@ Output::Output(Input &input, DataBlock &data) { pythonPeriod = input.Get("Output","python",0); if(pythonPeriod>=0.0) { // backward compatibility (negative value means no file) pythonLast = data.t - pythonPeriod; // write something in the next CheckForWrite() - pythonScript = input.Get("Output","python",1); - if(pythonScript.substr(pythonScript.length() - 3, 3).compare(".py")==0) { - IDEFIX_ERROR("You should not include the python script .py extension in your input file"); - } - pythonFunction = input.Get("Output","python",2); pythonEnabled = true; } #endif @@ -260,7 +257,7 @@ int Output::CheckForWrites(DataBlock &data) { if(pythonEnabled) { if(data.t >= pythonLast + pythonPeriod) { elapsedTime -= timer.seconds(); - pydefix.CallScript(&data, pythonScript,pythonFunction); + pydefix.Output(data); elapsedTime += timer.seconds(); // Check if our next predicted output should already have happened if((pythonLast+pythonPeriod <= data.t) && pythonPeriod>0.0) { diff --git a/src/output/output.hpp b/src/output/output.hpp index 1be44372..981716b7 100644 --- a/src/output/output.hpp +++ b/src/output/output.hpp @@ -87,8 +87,6 @@ class Output { bool pythonEnabled = false; real pythonPeriod; real pythonLast; - std::string pythonScript; - std::string pythonFunction; #endif Kokkos::Timer timer; diff --git a/src/pydefix.cpp b/src/pydefix.cpp index 55ce5ef8..d2a69928 100644 --- a/src/pydefix.cpp +++ b/src/pydefix.cpp @@ -5,6 +5,8 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#define PYBIND11_DETAILED_ERROR_MESSAGES + #include "pydefix.hpp" #include // everything needed for embedding #include // for numpy arrays @@ -15,6 +17,7 @@ #include "dataBlock.hpp" #include "dataBlockHost.hpp" + namespace py = pybind11; int Pydefix::ninstance = 0; @@ -27,10 +30,10 @@ PYBIND11_EMBEDDED_MODULE(pydefix, m) { py::class_(m, "DataBlockHost") .def(py::init<>()) .def_readwrite("x", &DataBlockHost::x) - .def_readwrite("xr", &DataBlockHost::x) - .def_readwrite("xl", &DataBlockHost::x) - .def_readwrite("dx", &DataBlockHost::x) - .def_readwrite("dV", &DataBlockHost::x) + .def_readwrite("xr", &DataBlockHost::xr) + .def_readwrite("xl", &DataBlockHost::xl) + .def_readwrite("dx", &DataBlockHost::dx) + .def_readwrite("dV", &DataBlockHost::dV) .def_readwrite("A", &DataBlockHost::A) .def_readwrite("Vc", &DataBlockHost::Vc) .def_readwrite("dustVc", &DataBlockHost::dustVc) @@ -46,34 +49,102 @@ PYBIND11_EMBEDDED_MODULE(pydefix, m) { } -Pydefix::Pydefix() { - ninstance++; - if(ninstance==1) { - idfx::cout << "Pydefix: start Python interpreter." << std::endl; +Pydefix::Pydefix(Input &input) { + // Check that the input has a [python] block + if(input.CheckBlock("Python")) { + this->isActive = true; + ninstance++; + // Check whether we need to start an interpreter + if(ninstance==1) { + idfx::cout << "Pydefix: start Python interpreter." << std::endl; + + py::initialize_interpreter(); + } + this->scriptFilename = input.Get("Python","script",0); + if(scriptFilename.substr(scriptFilename.length() - 3, 3).compare(".py")==0) { + IDEFIX_ERROR("You should not include the python script .py extension in your input file"); + } + if(input.CheckEntry("Python","output_function")>0) { + this->haveOutput = true; + this->outputFunctionName = input.Get("Python","output_function",0); + } + if(input.CheckEntry("Python","initflow_function")>0) { + this->haveInitflow = true; + this->initflowFunctionName = input.Get("Python","initflow_function",0); + } + } + this->ShowConfig(); +} + +void Pydefix::Output(DataBlock &data) { + idfx::pushRegion("Pydefix::Output"); + if(!this->isActive) { + IDEFIX_ERROR("Python Outputs requires the [python] block to be defined in your input file."); + } + if(!this->haveOutput) { + IDEFIX_ERROR("No python output function has been defined " + "in your input file [python]:output_function"); + } + DataBlockHost dataHost(data); + dataHost.SyncFromDevice(); + this->CallScript(&dataHost,this->scriptFilename,this->outputFunctionName); + idfx::popRegion(); +} + +void Pydefix::InitFlow(DataBlock &data) { + idfx::pushRegion("Pydefix::InitFlow"); + if(!this->isActive) { + IDEFIX_ERROR("Python Initflow requires the [python] block to be defined in your input file."); + } + if(!this->haveOutput) { + IDEFIX_ERROR("No python initflow function has been defined " + "in your input file [python]:initflow_function"); + } + DataBlockHost dataHost(data); + dataHost.SyncFromDevice(); + this->CallScript(&dataHost,this->scriptFilename,this->initflowFunctionName); + dataHost.SyncToDevice(); + idfx::popRegion(); +} - py::initialize_interpreter(); +void Pydefix::ShowConfig() { + if(isActive == false) { + idfx::cout << "Pydefix: DISABLED." << std::endl; + } else { + idfx::cout << "Pydefix: ENABLED." << std::endl; + if(haveOutput) { + idfx::cout << "Pydefix: output function ENABLED." << std::endl; + } else { + idfx::cout << "Pydefix: output function DISABLED." << std::endl; + } + if(haveInitflow) { + idfx::cout << "Pydefix: initflow function ENABLED." << std::endl; + } else { + idfx::cout << "Pydefix: initflow function DISABLED." << std::endl; + } } } Pydefix::~Pydefix() { - if(ninstance == 1) { - py::finalize_interpreter(); - idfx::cout << "Pydefix: shutdown Python interpreter." << std::endl; + if(isActive) { + if(ninstance == 1) { + py::finalize_interpreter(); + idfx::cout << "Pydefix: shutdown Python interpreter." << std::endl; + } + ninstance--; + isActive = false; } - ninstance--; } -void Pydefix::CallScript(DataBlock *data, std::string scriptName, std::string funcName) { +void Pydefix::CallScript(DataBlockHost *data, std::string scriptName, std::string funcName) { idfx::pushRegion("Pydefix::CallScript"); - DataBlockHost d(*data); - d.SyncFromDevice(); try { //auto Vc = pydefix.toNumpyArray(d.Vc); py::module_ script = py::module_::import(scriptName.c_str()); //py::module_ embeded = py::module_::import("embeded"); //py::object myV = py::cast(Vc); - py::object result = script.attr(funcName.c_str())(nCalls, d); + py::object result = script.attr(funcName.c_str())(data); } catch(std::exception &e) { std::stringstream message; message << "An exception occured while calling the Python interpreter" << std::endl @@ -82,7 +153,6 @@ void Pydefix::CallScript(DataBlock *data, std::string scriptName, std::string fu << e.what() << std::endl; IDEFIX_ERROR(message); } - nCalls++; idfx::popRegion(); } /* diff --git a/src/pydefix.hpp b/src/pydefix.hpp index b76ea3bb..bd985c67 100644 --- a/src/pydefix.hpp +++ b/src/pydefix.hpp @@ -9,27 +9,36 @@ #define PYDEFIX_HPP_ -//#define PYBIND11_DETAILED_ERROR_MESSAGES +#define PYBIND11_DETAILED_ERROR_MESSAGES #include #include #include #include #include #include "idefix.hpp" +#include "input.hpp" namespace py = pybind11; class DataBlock; +class DataBlockHost; class Pydefix { public: - Pydefix(); + explicit Pydefix(Input&); ~Pydefix(); - void CallScript(DataBlock *, std::string, std::string); - + void Output(DataBlock &); + void InitFlow(DataBlock &); + void ShowConfig(); + bool isActive{false}; + bool haveOutput{false}; + bool haveInitflow{false}; private: + void CallScript(DataBlockHost *, std::string, std::string); static int ninstance; - int nCalls{0}; + std::string scriptFilename; + std::string outputFunctionName; + std::string initflowFunctionName; }; namespace pybind11 { namespace detail { @@ -74,6 +83,7 @@ template struct type_caster> { src.extent(2), src.extent(3)}, src.data()); + idfx::cout << "Coucou @ cast" << std::endl; return a.release(); } }; diff --git a/test/HD/sod/idefix_py.ini b/test/HD/sod/idefix_py.ini index 977f2da1..378b5a85 100644 --- a/test/HD/sod/idefix_py.ini +++ b/test/HD/sod/idefix_py.ini @@ -21,7 +21,12 @@ X2-end outflow X3-beg outflow X3-end outflow +[Python] +script pydefix_example +output_function output +initflow_function initflow + [Output] -vtk 0.1 -dmp 0.2 -python 0.1 output myfunc +vtk 0.1 +dmp 0.2 +python 0.1 output myfunc diff --git a/test/HD/sod/output.py b/test/HD/sod/output.py deleted file mode 100644 index 79f05327..00000000 --- a/test/HD/sod/output.py +++ /dev/null @@ -1,9 +0,0 @@ - -def myfunc(n, data): - print("Python n=%d"%n) - print(dir(data)) - print("Dimensions="+str(data.Vc.shape)) - print("x1=") - print(data.x[0][:]) - print("data=") - print(data.Vc[0,0,0,:]) diff --git a/test/HD/sod/pydefix_example.py b/test/HD/sod/pydefix_example.py new file mode 100644 index 00000000..8230bc23 --- /dev/null +++ b/test/HD/sod/pydefix_example.py @@ -0,0 +1,14 @@ + +def output(data): + print("data=") + print(data.Vc[0,0,0,:]) + data.Vc[0,0,0,10] = 2.0 + +def initflow(data): + # Initialize the flow + print(data.Vc.shape) + data.Vc[0,0,0,:] = 1+0*data.x[0][:] + data.Vc[0,0,0,1] = 1.0 + data.Vc[1,0,0,:] = data.x[0][:] + print(data.Vc[0,0,0,:]) + print(data.Vc[1,0,0,:]) From c2d73e63824d8a486d6fce8cd9f145e153353b66 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Fri, 6 Sep 2024 13:37:15 +0200 Subject: [PATCH 07/28] working version with possibility to initialize the flow from python --- src/pydefix.cpp | 21 ++++++++++++++++++--- src/pydefix.hpp | 16 ++++++++++------ test/HD/sod/idefix_py.ini | 2 +- test/HD/sod/pydefix_example.py | 23 ++++++++++++++--------- 4 files changed, 43 insertions(+), 19 deletions(-) diff --git a/src/pydefix.cpp b/src/pydefix.cpp index d2a69928..2da45161 100644 --- a/src/pydefix.cpp +++ b/src/pydefix.cpp @@ -46,6 +46,20 @@ PYBIND11_EMBEDDED_MODULE(pydefix, m) { .def_readwrite("Ex3", &DataBlockHost::Ex3) #endif .def_readwrite("InvDt", &DataBlockHost::InvDt); + + m.attr("RHO") = RHO; + m.attr("VX1") = VX1; + m.attr("VX2") = VX2; + m.attr("VX3") = VX3; + m.attr("PRS") = PRS; + #if MHD == YES + m.attr("BX1") = BX1; + m.attr("BX2") = BX2; + m.attr("BX3") = BX3; + #endif + m.attr("IDIR") = IDIR; + m.attr("JDIR") = JDIR; + m.attr("KDIR") = KDIR; } @@ -87,7 +101,7 @@ void Pydefix::Output(DataBlock &data) { } DataBlockHost dataHost(data); dataHost.SyncFromDevice(); - this->CallScript(&dataHost,this->scriptFilename,this->outputFunctionName); + this->CallScript(dataHost,this->scriptFilename,this->outputFunctionName); idfx::popRegion(); } @@ -102,7 +116,7 @@ void Pydefix::InitFlow(DataBlock &data) { } DataBlockHost dataHost(data); dataHost.SyncFromDevice(); - this->CallScript(&dataHost,this->scriptFilename,this->initflowFunctionName); + this->CallScript(dataHost,this->scriptFilename,this->initflowFunctionName); dataHost.SyncToDevice(); idfx::popRegion(); } @@ -136,7 +150,7 @@ Pydefix::~Pydefix() { } } -void Pydefix::CallScript(DataBlockHost *data, std::string scriptName, std::string funcName) { +void Pydefix::CallScript(DataBlockHost &data, std::string scriptName, std::string funcName) { idfx::pushRegion("Pydefix::CallScript"); try { //auto Vc = pydefix.toNumpyArray(d.Vc); @@ -144,6 +158,7 @@ void Pydefix::CallScript(DataBlockHost *data, std::string scriptName, std::strin //py::module_ embeded = py::module_::import("embeded"); //py::object myV = py::cast(Vc); + //py::object result = script.attr(funcName.c_str())(data); py::object result = script.attr(funcName.c_str())(data); } catch(std::exception &e) { std::stringstream message; diff --git a/src/pydefix.hpp b/src/pydefix.hpp index bd985c67..e8ac8879 100644 --- a/src/pydefix.hpp +++ b/src/pydefix.hpp @@ -34,7 +34,7 @@ class Pydefix { bool haveOutput{false}; bool haveInitflow{false}; private: - void CallScript(DataBlockHost *, std::string, std::string); + void CallScript(DataBlockHost &, std::string, std::string); static int ninstance; std::string scriptFilename; std::string outputFunctionName; @@ -78,12 +78,13 @@ template struct type_caster> { static py::handle cast(const IdefixHostArray4D& src, py::return_value_policy policy, py::handle parent) { + py::none dummyDataOwner; py::array_t a({src.extent(0), src.extent(1), src.extent(2), src.extent(3)}, - src.data()); - idfx::cout << "Coucou @ cast" << std::endl; + src.data(), dummyDataOwner); + return a.release(); } }; @@ -122,10 +123,11 @@ template struct type_caster> { static py::handle cast(const IdefixHostArray3D& src, py::return_value_policy policy, py::handle parent) { + py::none dummyDataOwner; py::array_t a({src.extent(0), src.extent(1), src.extent(2)}, - src.data()); + src.data(),dummyDataOwner); return a.release(); } }; @@ -164,7 +166,8 @@ template struct type_caster> { static py::handle cast(const IdefixHostArray2D& src, py::return_value_policy policy, py::handle parent) { - py::array_t a({src.extent(0),src.extent(1)},src.data()); + py::none dummyOwner; + py::array_t a({src.extent(0),src.extent(1)},src.data(),dummyOwner); return a.release(); } }; @@ -203,7 +206,8 @@ template struct type_caster> { static py::handle cast(const IdefixHostArray1D& src, py::return_value_policy policy, py::handle parent) { - py::array_t a(src.extent(0),src.data()); + py::none dummyDataOwner; + py::array_t a(src.extent(0),src.data(),dummyDataOwner); return a.release(); } }; diff --git a/test/HD/sod/idefix_py.ini b/test/HD/sod/idefix_py.ini index 378b5a85..59c58f64 100644 --- a/test/HD/sod/idefix_py.ini +++ b/test/HD/sod/idefix_py.ini @@ -29,4 +29,4 @@ initflow_function initflow [Output] vtk 0.1 dmp 0.2 -python 0.1 output myfunc +python 0.01 diff --git a/test/HD/sod/pydefix_example.py b/test/HD/sod/pydefix_example.py index 8230bc23..58d73642 100644 --- a/test/HD/sod/pydefix_example.py +++ b/test/HD/sod/pydefix_example.py @@ -1,14 +1,19 @@ +import pydefix as pdfx +import numpy as np +import matplotlib.pyplot as plt def output(data): - print("data=") - print(data.Vc[0,0,0,:]) - data.Vc[0,0,0,10] = 2.0 + + plt.figure() + plt.plot(data.x[pdfx.IDIR],data.Vc[pdfx.VX1,0,0,:],label='VX1') + plt.plot(data.x[pdfx.IDIR],data.Vc[pdfx.RHO,0,0,:],label='RHO') + plt.legend() + plt.show() + #data.Vc[0,0,0,10] = 2.0 def initflow(data): + # Initialize the flow - print(data.Vc.shape) - data.Vc[0,0,0,:] = 1+0*data.x[0][:] - data.Vc[0,0,0,1] = 1.0 - data.Vc[1,0,0,:] = data.x[0][:] - print(data.Vc[0,0,0,:]) - print(data.Vc[1,0,0,:]) + data.Vc[pdfx.RHO,0,0,:] = 1.0 + data.Vc[pdfx.PRS,0,0,:] = 2.0 + data.Vc[pdfx.VX1,0,0,:] = np.sin(2.0*np.pi*data.x[0][:]) From 7dc75538c87a5935e96024e6c9b2d197e2c9c9d2 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 18 Sep 2024 15:24:38 +0200 Subject: [PATCH 08/28] add BXs to python bindings --- src/pydefix.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/pydefix.cpp b/src/pydefix.cpp index 2da45161..e62f371e 100644 --- a/src/pydefix.cpp +++ b/src/pydefix.cpp @@ -56,6 +56,10 @@ PYBIND11_EMBEDDED_MODULE(pydefix, m) { m.attr("BX1") = BX1; m.attr("BX2") = BX2; m.attr("BX3") = BX3; + D_EXPAND( + m.attr("BX1s") = BX1; , + m.attr("BX2s") = BX2; , + m.attr("BX3s") = BX3; ) #endif m.attr("IDIR") = IDIR; m.attr("JDIR") = JDIR; From 8aa7e93f976c8ab1d84ab66cf6deba921643fe6f Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Sun, 22 Sep 2024 17:13:00 +0200 Subject: [PATCH 09/28] fix BXs indices add time in datablockHost --- src/dataBlock/dataBlockHost.cpp | 5 +++++ src/dataBlock/dataBlockHost.hpp | 3 +++ src/pydefix.cpp | 11 ++++++----- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/dataBlock/dataBlockHost.cpp b/src/dataBlock/dataBlockHost.cpp index ca26e5cb..f5e8c269 100644 --- a/src/dataBlock/dataBlockHost.cpp +++ b/src/dataBlock/dataBlockHost.cpp @@ -101,6 +101,8 @@ DataBlockHost::DataBlockHost(DataBlock& datain) { void DataBlockHost::SyncToDevice() { idfx::pushRegion("DataBlockHost::SyncToDevice()"); + data->t = this->t; + data->dt = this->dt; Kokkos::deep_copy(data->hydro->Vc,Vc); Kokkos::deep_copy(data->hydro->InvDt,InvDt); @@ -137,6 +139,9 @@ void DataBlockHost::SyncToDevice() { void DataBlockHost::SyncFromDevice() { idfx::pushRegion("DataBlockHost::SyncFromDevice()"); + this->t = data->t; + this->dt = data->dt; + Kokkos::deep_copy(Vc,data->hydro->Vc); Kokkos::deep_copy(InvDt,data->hydro->InvDt); diff --git a/src/dataBlock/dataBlockHost.hpp b/src/dataBlock/dataBlockHost.hpp index 4a1e8e5e..2a732a32 100644 --- a/src/dataBlock/dataBlockHost.hpp +++ b/src/dataBlock/dataBlockHost.hpp @@ -75,6 +75,9 @@ class DataBlockHost { std::array gbeg; ///< Begining of local block in the grid (internal) std::array gend; ///< End of local block in the grid (internal) + real dt; ///< Current timestep + real t; ///< Current time + explicit DataBlockHost(DataBlock &); ///< Constructor from a device datablock ///< (NB: does not sync any data) diff --git a/src/pydefix.cpp b/src/pydefix.cpp index e62f371e..5662dce5 100644 --- a/src/pydefix.cpp +++ b/src/pydefix.cpp @@ -45,7 +45,9 @@ PYBIND11_EMBEDDED_MODULE(pydefix, m) { .def_readwrite("Ex2", &DataBlockHost::Ex2) .def_readwrite("Ex3", &DataBlockHost::Ex3) #endif - .def_readwrite("InvDt", &DataBlockHost::InvDt); + .def_readwrite("InvDt", &DataBlockHost::InvDt) + .def_readwrite("t",&DataBlockHost::t) + .def_readwrite("dt",&DataBlockHost::dt); m.attr("RHO") = RHO; m.attr("VX1") = VX1; @@ -57,9 +59,9 @@ PYBIND11_EMBEDDED_MODULE(pydefix, m) { m.attr("BX2") = BX2; m.attr("BX3") = BX3; D_EXPAND( - m.attr("BX1s") = BX1; , - m.attr("BX2s") = BX2; , - m.attr("BX3s") = BX3; ) + m.attr("BX1s") = BX1s; , + m.attr("BX2s") = BX2s; , + m.attr("BX3s") = BX3s; ) #endif m.attr("IDIR") = IDIR; m.attr("JDIR") = JDIR; @@ -91,7 +93,6 @@ Pydefix::Pydefix(Input &input) { this->initflowFunctionName = input.Get("Python","initflow_function",0); } } - this->ShowConfig(); } void Pydefix::Output(DataBlock &data) { From 134ca6b2c6b688097b814ffdac9c8f4e81c6dc22 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Sun, 22 Sep 2024 17:13:39 +0200 Subject: [PATCH 10/28] change init list to work without python --- src/output/output.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/output/output.cpp b/src/output/output.cpp index 75d75124..a6596490 100644 --- a/src/output/output.cpp +++ b/src/output/output.cpp @@ -15,9 +15,9 @@ #include "pydefix.hpp" #endif -Output::Output(Input &input, DataBlock &data): +Output::Output(Input &input, DataBlock &data) #ifdef WITH_PYTHON -pydefix(input) +:pydefix(input) #endif { idfx::pushRegion("Output::Output"); From 8f6cf5d09333e3a4f32c59cdc37a589a5ccd2c4f Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 3 Oct 2024 10:30:09 +0200 Subject: [PATCH 11/28] generalised call script with variadic template parameters add the output number n to the list of parameters passed to the output python routine --- src/output/output.cpp | 5 ++++- src/output/output.hpp | 1 + src/pydefix.cpp | 43 +++++++++++++++++++++---------------------- src/pydefix.hpp | 6 ++++-- 4 files changed, 30 insertions(+), 25 deletions(-) diff --git a/src/output/output.cpp b/src/output/output.cpp index a6596490..a482f66a 100644 --- a/src/output/output.cpp +++ b/src/output/output.cpp @@ -133,6 +133,7 @@ Output::Output(Input &input, DataBlock &data) if(pythonPeriod>=0.0) { // backward compatibility (negative value means no file) pythonLast = data.t - pythonPeriod; // write something in the next CheckForWrite() pythonEnabled = true; + pythonNumber = 0; } #endif } @@ -146,6 +147,7 @@ Output::Output(Input &input, DataBlock &data) #endif #ifdef WITH_PYTHON data.dump->RegisterVariable(&pythonLast, "pythonLast"); + data.dump->RegisterVariable(&pythonNumber,"pythonNumber"); #endif idfx::popRegion(); @@ -257,7 +259,8 @@ int Output::CheckForWrites(DataBlock &data) { if(pythonEnabled) { if(data.t >= pythonLast + pythonPeriod) { elapsedTime -= timer.seconds(); - pydefix.Output(data); + pydefix.Output(data,pythonNumber); + pythonNumber++; elapsedTime += timer.seconds(); // Check if our next predicted output should already have happened if((pythonLast+pythonPeriod <= data.t) && pythonPeriod>0.0) { diff --git a/src/output/output.hpp b/src/output/output.hpp index 981716b7..039ab157 100644 --- a/src/output/output.hpp +++ b/src/output/output.hpp @@ -87,6 +87,7 @@ class Output { bool pythonEnabled = false; real pythonPeriod; real pythonLast; + real pythonNumber; #endif Kokkos::Timer timer; diff --git a/src/pydefix.cpp b/src/pydefix.cpp index 5662dce5..6756d0a2 100644 --- a/src/pydefix.cpp +++ b/src/pydefix.cpp @@ -69,6 +69,24 @@ PYBIND11_EMBEDDED_MODULE(pydefix, m) { } +template +void Pydefix::CallScript(std::string scriptName, std::string funcName, Ts... args) { + idfx::pushRegion("Pydefix::CallScript"); + try { + py::module_ script = py::module_::import(scriptName.c_str()); + py::object result = script.attr(funcName.c_str())(&args...); + } catch(std::exception &e) { + std::stringstream message; + message << "An exception occured while calling the Python interpreter" << std::endl + << "in file \"" << scriptName << ".py\" function \"" << funcName << "\":" + << std::endl + << e.what() << std::endl; + IDEFIX_ERROR(message); + } + idfx::popRegion(); +} + + Pydefix::Pydefix(Input &input) { // Check that the input has a [python] block if(input.CheckBlock("Python")) { @@ -95,7 +113,7 @@ Pydefix::Pydefix(Input &input) { } } -void Pydefix::Output(DataBlock &data) { +void Pydefix::Output(DataBlock &data, int n) { idfx::pushRegion("Pydefix::Output"); if(!this->isActive) { IDEFIX_ERROR("Python Outputs requires the [python] block to be defined in your input file."); @@ -106,7 +124,7 @@ void Pydefix::Output(DataBlock &data) { } DataBlockHost dataHost(data); dataHost.SyncFromDevice(); - this->CallScript(dataHost,this->scriptFilename,this->outputFunctionName); + this->CallScript(this->scriptFilename,this->outputFunctionName,dataHost, n); idfx::popRegion(); } @@ -121,7 +139,7 @@ void Pydefix::InitFlow(DataBlock &data) { } DataBlockHost dataHost(data); dataHost.SyncFromDevice(); - this->CallScript(dataHost,this->scriptFilename,this->initflowFunctionName); + this->CallScript(this->scriptFilename,this->initflowFunctionName,dataHost); dataHost.SyncToDevice(); idfx::popRegion(); } @@ -155,26 +173,7 @@ Pydefix::~Pydefix() { } } -void Pydefix::CallScript(DataBlockHost &data, std::string scriptName, std::string funcName) { - idfx::pushRegion("Pydefix::CallScript"); - try { - //auto Vc = pydefix.toNumpyArray(d.Vc); - py::module_ script = py::module_::import(scriptName.c_str()); - //py::module_ embeded = py::module_::import("embeded"); - //py::object myV = py::cast(Vc); - //py::object result = script.attr(funcName.c_str())(data); - py::object result = script.attr(funcName.c_str())(data); - } catch(std::exception &e) { - std::stringstream message; - message << "An exception occured while calling the Python interpreter" << std::endl - << "in file \"" << scriptName << ".py\" function \"" << funcName << "\":" - << std::endl - << e.what() << std::endl; - IDEFIX_ERROR(message); - } - idfx::popRegion(); -} /* py::array_t Pydefix::toNumpyArray(const IdefixHostArray3D& in) { py::array_t array({in.extent(0),in.extent(1),in.extent(2)},in.data()); diff --git a/src/pydefix.hpp b/src/pydefix.hpp index e8ac8879..2f782d89 100644 --- a/src/pydefix.hpp +++ b/src/pydefix.hpp @@ -27,20 +27,22 @@ class Pydefix { public: explicit Pydefix(Input&); ~Pydefix(); - void Output(DataBlock &); + void Output(DataBlock &, int); void InitFlow(DataBlock &); void ShowConfig(); bool isActive{false}; bool haveOutput{false}; bool haveInitflow{false}; private: - void CallScript(DataBlockHost &, std::string, std::string); + template + void CallScript(std::string, std::string, Ts...); static int ninstance; std::string scriptFilename; std::string outputFunctionName; std::string initflowFunctionName; }; + namespace pybind11 { namespace detail { // Caster for IdefixArray4D template struct type_caster> { From 6985443b56911c4128e544834585d5e536f66b2c Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 3 Oct 2024 11:32:16 +0200 Subject: [PATCH 12/28] -fix warning in cmake -add a readme --- CMakeLists.txt | 3 +- test/IO/pydefix/CMakeLists.txt | 2 ++ test/IO/pydefix/README | 43 +++++++++++++++++++++++++ test/IO/pydefix/definitions.hpp | 4 +++ test/IO/pydefix/idefix.ini | 30 +++++++++++++++++ test/IO/pydefix/pydefix_example.py | 29 +++++++++++++++++ test/IO/pydefix/python_requirements.txt | 6 ++++ 7 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 test/IO/pydefix/CMakeLists.txt create mode 100644 test/IO/pydefix/README create mode 100644 test/IO/pydefix/definitions.hpp create mode 100644 test/IO/pydefix/idefix.ini create mode 100644 test/IO/pydefix/pydefix_example.py create mode 100644 test/IO/pydefix/python_requirements.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a0e2d3a..ca152646 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -75,7 +75,7 @@ add_subdirectory(src build) if(EXISTS ${PROJECT_BINARY_DIR}/setup.cpp) target_sources(idefix PUBLIC ${PROJECT_BINARY_DIR}/setup.cpp) else() - message(WARNING "No specific setup.cpp found in the problem directory") + message(WARNING "No specific setup.cpp found in the problem directory (this message can be ignored if using python to define your problem)") endif() # If a CMakeLists.txt is in the problem dir (for problem-specific source files) @@ -121,6 +121,7 @@ endif() if(Idefix_PYTHON) add_compile_definitions("WITH_PYTHON") + set(PYBIND11_FINDPYTHON ON CACHE BOOL "Idefix requires python" FORCE) find_package(pybind11 REQUIRED) target_link_libraries(idefix pybind11::embed) target_sources(idefix diff --git a/test/IO/pydefix/CMakeLists.txt b/test/IO/pydefix/CMakeLists.txt new file mode 100644 index 00000000..83d16a52 --- /dev/null +++ b/test/IO/pydefix/CMakeLists.txt @@ -0,0 +1,2 @@ +enable_idefix_property(Idefix_MHD) +enable_idefix_property(Idefix_PYTHON) diff --git a/test/IO/pydefix/README b/test/IO/pydefix/README new file mode 100644 index 00000000..6cc5678a --- /dev/null +++ b/test/IO/pydefix/README @@ -0,0 +1,43 @@ +this directory shows how to use Idefix with pure python initial condtions and outputs. It reproduced the 2D OrszagTang vortex available in MHD/OrszagTang +The python script initialize the initial condition of the OT test and produces a series of PNG files through matplotlib + +# Python modules installation + +In order to use pydefix, you need to be working in a python environement that includes pybind11. The simplest way is to install the suggested list of packages (that you can deploy in dedicated venv environement) + +```bash +pip install -r python_requirements.txt +``` + +# Configuration + +In order to use Pydefix, you need to switch the `Idefix_PYTHON` to ON in cmake. In this particular setup, it is automatically done for you in CMakeLists.txt to avoid mistakes. + +# Running + +Just run idefix as usual. + +# Troubleshooting + +It during configuration stage, you get: + +` +CMake Error at CMakeLists.txt:122 (find_package): + By not providing "Findpybind11.cmake" in CMAKE_MODULE_PATH this project has + asked CMake to find a package configuration file provided by "pybind11", + but CMake did not find one.` + +It means that cmake cannot find the location of pybind11 (this typically happens on MacOs). In order to locate pybind11, open a python interpreter, and get pybind11 install dir through: + +```python +import pybind11 +print(pybind11.__file__) +``` + +You can then exit the interpreter and set the pybind11_DIR environement variable to the right path: + +```bash +export pybind11_DIR=env/lib/python3.10/site-packages/pybind11 +``` + +you can then run cmake which should be able to find pybind11, and compile the code. diff --git a/test/IO/pydefix/definitions.hpp b/test/IO/pydefix/definitions.hpp new file mode 100644 index 00000000..69f0c49e --- /dev/null +++ b/test/IO/pydefix/definitions.hpp @@ -0,0 +1,4 @@ +#define COMPONENTS 2 +#define DIMENSIONS 2 + +#define GEOMETRY CARTESIAN diff --git a/test/IO/pydefix/idefix.ini b/test/IO/pydefix/idefix.ini new file mode 100644 index 00000000..57fdcfc4 --- /dev/null +++ b/test/IO/pydefix/idefix.ini @@ -0,0 +1,30 @@ +[Grid] +X1-grid 1 0.0 256 u 1.0 +X2-grid 1 0.0 256 u 1.0 +X3-grid 1 0.0 1 u 1.0 + +[TimeIntegrator] +CFL 0.6 +tstop 0.5 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver roe + +[Python] +script pydefix_example +output_function output +initflow_function initflow + +[Boundary] +X1-beg periodic +X1-end periodic +X2-beg periodic +X2-end periodic +X3-beg outflow +X3-end outflow + +[Output] +log 10 +python 0.02 diff --git a/test/IO/pydefix/pydefix_example.py b/test/IO/pydefix/pydefix_example.py new file mode 100644 index 00000000..258f674f --- /dev/null +++ b/test/IO/pydefix/pydefix_example.py @@ -0,0 +1,29 @@ +import pydefix as pdfx +import numpy as np +import matplotlib.pyplot as plt + +# The output function +# the only argument is dataBlockHost python object, wrapping a dataBlockHost Idefix object +def output(data,n): + plt.close() + plt.figure() + plt.pcolormesh(data.x[pdfx.IDIR],data.x[pdfx.JDIR],data.Vc[pdfx.PRS,0,:,:],label='PRS',vmin=0.02,vmax=0.5,cmap='plasma') + plt.title("t=%.2f"%data.t) + plt.colorbar() + plt.savefig("PRS.%.4d.png"%n) + + +def initflow(data): + # Field amplitude + B0 = 1/np.sqrt(4*np.pi) + + [z,y,x] = np.meshgrid(data.x[pdfx.KDIR], data.x[pdfx.JDIR], data.x[pdfx.IDIR], indexing='ij') + + # Initialize the flow + data.Vc[pdfx.RHO,:,:,:] = 25/(36*np.pi) + data.Vc[pdfx.PRS,:,:,:] = 5/(12*np.pi) + data.Vc[pdfx.VX1,:,:,:] = -np.sin(2*np.pi*y) + data.Vc[pdfx.VX2,:,:,:] = np.sin(2*np.pi*x) + + data.Vs[pdfx.BX1s,:,:-1,:-1] = -B0*np.sin(2*np.pi*y) + data.Vs[pdfx.BX2s,:,:-1,:-1] = B0*np.sin(4*np.pi*x) diff --git a/test/IO/pydefix/python_requirements.txt b/test/IO/pydefix/python_requirements.txt new file mode 100644 index 00000000..6afd3019 --- /dev/null +++ b/test/IO/pydefix/python_requirements.txt @@ -0,0 +1,6 @@ +# note: version requirements are indicative and tests # should be able to run with +# older versions of our dependencies, though it is not guaranteed. + +numpy>=1.16.6 +matplotlib>=2.2.5 +pybind11>=2.12.0 From d3b3bb73adf82f0ea87350863e350632baf5c01b Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 3 Oct 2024 11:45:42 +0200 Subject: [PATCH 13/28] typos --- test/IO/pydefix/README | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/IO/pydefix/README b/test/IO/pydefix/README index 6cc5678a..820a0958 100644 --- a/test/IO/pydefix/README +++ b/test/IO/pydefix/README @@ -1,5 +1,6 @@ -this directory shows how to use Idefix with pure python initial condtions and outputs. It reproduced the 2D OrszagTang vortex available in MHD/OrszagTang -The python script initialize the initial condition of the OT test and produces a series of PNG files through matplotlib +This directory shows how to use Idefix with pure python initial conditions and outputs. It reproduces the 2D OrszagTang vortex available in MHD/OrszagTang without requiring any single line of C++ code from the user. + +The python script `pydefix_example.py` initializes the initial condition of the OT test (`initflow`) and produces a series of PNG files through matplotlib (`output`). # Python modules installation From a92732f3683af37bc3e55a9e8954867b1ae43967 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Thu, 3 Oct 2024 11:46:07 +0200 Subject: [PATCH 14/28] change naming --- test/IO/pydefix/{README => README.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/IO/pydefix/{README => README.md} (100%) diff --git a/test/IO/pydefix/README b/test/IO/pydefix/README.md similarity index 100% rename from test/IO/pydefix/README rename to test/IO/pydefix/README.md From 1c29ebbfee5a27d7d1388fdb259eebf02e478013 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 9 Oct 2024 20:16:38 +0200 Subject: [PATCH 15/28] fix lack of initialisation of datablockHost's dt and t after construction fix #276 --- src/dataBlock/dataBlockHost.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/dataBlock/dataBlockHost.cpp b/src/dataBlock/dataBlockHost.cpp index f5e8c269..3443dbb5 100644 --- a/src/dataBlock/dataBlockHost.cpp +++ b/src/dataBlock/dataBlockHost.cpp @@ -94,6 +94,9 @@ DataBlockHost::DataBlockHost(DataBlock& datain) { this->haveplanetarySystem = data->haveplanetarySystem; this->planetarySystem = data->planetarySystem.get(); + this->t = data->t; + this->dt = data->dt; + idfx::popRegion(); } @@ -141,7 +144,7 @@ void DataBlockHost::SyncFromDevice() { idfx::pushRegion("DataBlockHost::SyncFromDevice()"); this->t = data->t; this->dt = data->dt; - + Kokkos::deep_copy(Vc,data->hydro->Vc); Kokkos::deep_copy(InvDt,data->hydro->InvDt); From d0ace44b3985d196dbb585a8b603e6f3ed83e1c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Fri, 11 Oct 2024 22:38:07 +0200 Subject: [PATCH 16/28] MNT: fix linting with cpplint 2.0.0 (#279) --- CPPLINT.cfg | 2 ++ src/dataBlock/dataBlock.cpp | 2 ++ src/dataBlock/dataBlock.hpp | 1 + src/dataBlock/dataBlockHost.cpp | 1 + src/dataBlock/dumpToFile.cpp | 2 ++ src/fluid/boundary/axis.cpp | 1 + src/fluid/checkNan.hpp | 1 + src/global.cpp | 1 + src/global.hpp | 1 + src/idefix.hpp | 1 + src/macros.hpp | 1 + src/main.cpp | 1 + src/mpi.cpp | 1 + src/output/dump.cpp | 4 +++- src/output/dump.hpp | 2 +- src/output/output.cpp | 1 + src/output/slice.cpp | 1 + src/output/vtk.cpp | 3 ++- src/output/vtk.hpp | 3 ++- src/output/xdmf.hpp | 10 +++++++++- src/utils/dumpImage.cpp | 2 ++ src/utils/npy.hpp | 1 + 22 files changed, 38 insertions(+), 5 deletions(-) diff --git a/CPPLINT.cfg b/CPPLINT.cfg index 4f3e42cb..b02d056c 100644 --- a/CPPLINT.cfg +++ b/CPPLINT.cfg @@ -16,6 +16,8 @@ filter=-whitespace/comma # 6027 errors filter=-whitespace/comments # 881 errors filter=-whitespace/operators # 5240 errors filter=-whitespace/parens # 413 error +filter=-whitespace/newline +filter=-whitespace/indent_namespace filter=-readability/multiline_string filter=-build/include_subdir # 296 errors #filter=-whitespace/end_of_line diff --git a/src/dataBlock/dataBlock.cpp b/src/dataBlock/dataBlock.cpp index c0648f92..81874cef 100644 --- a/src/dataBlock/dataBlock.cpp +++ b/src/dataBlock/dataBlock.cpp @@ -6,6 +6,8 @@ // *********************************************************************************** #include +#include +#include #include "idefix.hpp" #include "dataBlock.hpp" #include "fluid.hpp" diff --git a/src/dataBlock/dataBlock.hpp b/src/dataBlock/dataBlock.hpp index 12f0847e..89b745fa 100644 --- a/src/dataBlock/dataBlock.hpp +++ b/src/dataBlock/dataBlock.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include "idefix.hpp" #include "grid.hpp" diff --git a/src/dataBlock/dataBlockHost.cpp b/src/dataBlock/dataBlockHost.cpp index 3443dbb5..76f797c7 100644 --- a/src/dataBlock/dataBlockHost.cpp +++ b/src/dataBlock/dataBlockHost.cpp @@ -5,6 +5,7 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#include #include "idefix.hpp" #include "dataBlockHost.hpp" #include "fluid.hpp" diff --git a/src/dataBlock/dumpToFile.cpp b/src/dataBlock/dumpToFile.cpp index 9980d09c..98b4a474 100644 --- a/src/dataBlock/dumpToFile.cpp +++ b/src/dataBlock/dumpToFile.cpp @@ -5,6 +5,8 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#include +#include #include "../idefix.hpp" #include "dataBlock.hpp" #include "version.hpp" diff --git a/src/fluid/boundary/axis.cpp b/src/fluid/boundary/axis.cpp index ead51f4f..b77fe3b5 100644 --- a/src/fluid/boundary/axis.cpp +++ b/src/fluid/boundary/axis.cpp @@ -5,6 +5,7 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#include #include "axis.hpp" #include "boundary.hpp" diff --git a/src/fluid/checkNan.hpp b/src/fluid/checkNan.hpp index 71911c58..88e206e5 100644 --- a/src/fluid/checkNan.hpp +++ b/src/fluid/checkNan.hpp @@ -7,6 +7,7 @@ #ifndef FLUID_CHECKNAN_HPP_ #define FLUID_CHECKNAN_HPP_ +#include #include "dataBlock.hpp" #include "dataBlockHost.hpp" #include "fluid.hpp" diff --git a/src/global.cpp b/src/global.cpp index 1e9c11ee..4d9499d9 100644 --- a/src/global.cpp +++ b/src/global.cpp @@ -5,6 +5,7 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#include #include #include #include "idefix.hpp" diff --git a/src/global.hpp b/src/global.hpp index 47fca42e..8651f974 100644 --- a/src/global.hpp +++ b/src/global.hpp @@ -7,6 +7,7 @@ #ifndef GLOBAL_HPP_ #define GLOBAL_HPP_ +#include #include #include #include "arrays.hpp" diff --git a/src/idefix.hpp b/src/idefix.hpp index a44bf4c2..f547c5d7 100644 --- a/src/idefix.hpp +++ b/src/idefix.hpp @@ -9,6 +9,7 @@ #define IDEFIX_HPP_ #include #include +#include #include // #include // do we still need this? #ifdef WITH_MPI diff --git a/src/macros.hpp b/src/macros.hpp index 9a484c3d..2c7bda12 100644 --- a/src/macros.hpp +++ b/src/macros.hpp @@ -10,6 +10,7 @@ #ifndef MACROS_HPP_ #define MACROS_HPP_ +#include #if COMPONENTS == 1 #define EXPAND(a,b,c) a diff --git a/src/main.cpp b/src/main.cpp index 6294dcff..06f408ff 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include diff --git a/src/mpi.cpp b/src/mpi.cpp index ee2fdb5c..d25056fa 100644 --- a/src/mpi.cpp +++ b/src/mpi.cpp @@ -12,6 +12,7 @@ #include // NOLINT [build/c++11] #include // NOLINT [build/c++11] #include +#include #include "idefix.hpp" #include "dataBlock.hpp" diff --git a/src/output/dump.cpp b/src/output/dump.cpp index 73b0df99..955e06e7 100644 --- a/src/output/dump.cpp +++ b/src/output/dump.cpp @@ -8,7 +8,7 @@ #include #include #if __has_include() - #include + #include // NOLINT [build/c++17] namespace fs = std::filesystem; #elif __has_include() #include @@ -17,6 +17,8 @@ #error "Missing the header." #endif #include +#include +#include #include "dump.hpp" #include "version.hpp" #include "dataBlockHost.hpp" diff --git a/src/output/dump.hpp b/src/output/dump.hpp index a9a52554..d823ddca 100644 --- a/src/output/dump.hpp +++ b/src/output/dump.hpp @@ -11,7 +11,7 @@ #include #include #if __has_include() - #include + #include // NOLINT [build/c++17] namespace fs = std::filesystem; #elif __has_include() #include diff --git a/src/output/output.cpp b/src/output/output.cpp index a482f66a..a67cb166 100644 --- a/src/output/output.cpp +++ b/src/output/output.cpp @@ -5,6 +5,7 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#include #include "output.hpp" #include "dataBlock.hpp" #include "fluid.hpp" diff --git a/src/output/slice.cpp b/src/output/slice.cpp index 3f971932..98caa5c2 100644 --- a/src/output/slice.cpp +++ b/src/output/slice.cpp @@ -7,6 +7,7 @@ #include #include +#include #include "slice.hpp" #include "input.hpp" #include "physics.hpp" diff --git a/src/output/vtk.cpp b/src/output/vtk.cpp index 0f52f37c..ae4a4c60 100644 --- a/src/output/vtk.cpp +++ b/src/output/vtk.cpp @@ -8,10 +8,11 @@ #include "vtk.hpp" #include #include +#include #include #include #if __has_include() - #include + #include // NOLINT [build/c++17] namespace fs = std::filesystem; #elif __has_include() #include diff --git a/src/output/vtk.hpp b/src/output/vtk.hpp index dcda3618..a9503a5a 100644 --- a/src/output/vtk.hpp +++ b/src/output/vtk.hpp @@ -10,7 +10,7 @@ #include #include #if __has_include() - #include + #include // NOLINT [build/c++17] namespace fs = std::filesystem; #elif __has_include() #include @@ -18,6 +18,7 @@ #else error "Missing the header." #endif +#include #include "idefix.hpp" #include "input.hpp" #include "dataBlock.hpp" diff --git a/src/output/xdmf.hpp b/src/output/xdmf.hpp index b2fb6628..6c6fc15c 100644 --- a/src/output/xdmf.hpp +++ b/src/output/xdmf.hpp @@ -8,7 +8,15 @@ #ifndef OUTPUT_XDMF_HPP_ #define OUTPUT_XDMF_HPP_ #include -#include +#if __has_include() + #include // NOLINT [build/c++17] + namespace fs = std::filesystem; +#elif __has_include() + #include + namespace fs = std::experimental::filesystem; +#else + error "Missing the header." +#endif #include #include "idefix.hpp" #include "input.hpp" diff --git a/src/utils/dumpImage.cpp b/src/utils/dumpImage.cpp index 03a456e1..04f413a7 100644 --- a/src/utils/dumpImage.cpp +++ b/src/utils/dumpImage.cpp @@ -5,6 +5,8 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#include +#include #include "dumpImage.hpp" #include "dataBlock.hpp" #include "idefix.hpp" diff --git a/src/utils/npy.hpp b/src/utils/npy.hpp index 34b7c724..edae82ee 100644 --- a/src/utils/npy.hpp +++ b/src/utils/npy.hpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include From 4f48ac6153076723166314e49fea396540e62061 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sat, 12 Oct 2024 07:57:36 +0200 Subject: [PATCH 17/28] STY: locally disable linting around #include (#281) --- src/output/xdmf.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/output/xdmf.cpp b/src/output/xdmf.cpp index 237f4ca3..23ddaa03 100644 --- a/src/output/xdmf.cpp +++ b/src/output/xdmf.cpp @@ -11,7 +11,7 @@ #include #include #if __has_include() - #include + #include // NOLINT [build/c++17] namespace fs = std::filesystem; #elif __has_include() #include From 003b28c47d6dd42a0ade19a5a7fc6fbeda6551a4 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 16 Dec 2024 09:18:25 +0100 Subject: [PATCH 18/28] fix #275 --- src/pydefix.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pydefix.cpp b/src/pydefix.cpp index 6756d0a2..824254aa 100644 --- a/src/pydefix.cpp +++ b/src/pydefix.cpp @@ -133,7 +133,7 @@ void Pydefix::InitFlow(DataBlock &data) { if(!this->isActive) { IDEFIX_ERROR("Python Initflow requires the [python] block to be defined in your input file."); } - if(!this->haveOutput) { + if(!this->haveInitflow) { IDEFIX_ERROR("No python initflow function has been defined " "in your input file [python]:initflow_function"); } From 90653658b3b241fddb5f527f21374d5e5b10ab38 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 24 Dec 2024 08:37:19 +0100 Subject: [PATCH 19/28] implement MPi data gathering for pydefix Working implementation but: - should write the type caster for 1D,2D and 3D following the 4D one - should implement a proper python type for the grid - check boundary conditions --- CMakeLists.txt | 2 +- src/dataBlock/dataBlockHost.hpp | 1 - src/pydefix.cpp | 105 ++++++++++++++++++++++++++++++++ src/pydefix.hpp | 35 +++++++---- 4 files changed, 128 insertions(+), 15 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 79f6d30e..f7095d2c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -124,7 +124,7 @@ endif() if(Idefix_PYTHON) add_compile_definitions("WITH_PYTHON") - set(PYBIND11_FINDPYTHON ON CACHE BOOL "Idefix requires python" FORCE) + #set(PYBIND11_FINDPYTHON ON CACHE BOOL "Idefix requires python" FORCE) find_package(pybind11 REQUIRED) target_link_libraries(idefix pybind11::embed) target_sources(idefix diff --git a/src/dataBlock/dataBlockHost.hpp b/src/dataBlock/dataBlockHost.hpp index 2a732a32..f7dc4041 100644 --- a/src/dataBlock/dataBlockHost.hpp +++ b/src/dataBlock/dataBlockHost.hpp @@ -98,7 +98,6 @@ class DataBlockHost { GridCoarsening haveGridCoarsening{GridCoarsening::disabled}; ///< Is grid coarsening enabled? - private: // Data object to which we are the mirror DataBlock *data; }; diff --git a/src/pydefix.cpp b/src/pydefix.cpp index 824254aa..d3204e9d 100644 --- a/src/pydefix.cpp +++ b/src/pydefix.cpp @@ -26,6 +26,106 @@ int Pydefix::ninstance = 0; * DataBlockHost Python binding * **********************************/ +IdefixHostArray4D GatherIdefixArray(IdefixHostArray4D in, DataBlockHost dataHost) { + idfx::cout << "I am being called" << std::endl; + idfx::cout << "My dims=" << in.extent(0) << " " << in.extent(1) << " " << in.extent(2) << " " << in.extent(3) << std::endl; + idfx::cout << "First element=" << in(0,0,0,0) << std::endl; + Grid *grid = dataHost.data->mygrid; + IdefixHostArray4D out; + #ifdef WITH_MPI + if(idfx::psize > 1) { + const int nvar = in.extent(0); + out = IdefixHostArray4D("pydefix::GatheredArray",nvar,grid->np_tot[KDIR],grid->np_tot[JDIR],grid->np_tot[IDIR]); + if(idfx::prank == 0) { + for(int rank = 0 ; rank < idfx::psize ; rank++) { + // np_tot: total size of the incoming array + // np_int: size that should be copied into global + // beg: offset in the incoming array where copy should begin + // gbeg: offset in the global array where copy should be begin + MPI_Status status; + std::array np_int,np_tot, beg, gbeg; + IdefixHostArray4D buf; + + if(rank==0) { + np_int = dataHost.np_int; + np_tot = dataHost.np_tot; + gbeg = dataHost.gbeg; + beg = dataHost.beg; + + // Add back boundaries + for(int dir = 0 ; dir < DIMENSIONS ; dir++) { + if(dataHost.lbound[dir] != internal) { + np_int[dir] += dataHost.nghost[dir]; + gbeg[dir] -= dataHost.nghost[dir]; + beg[dir] -= dataHost.nghost[dir]; + } + if(dataHost.rbound[dir] != internal) { + np_int[dir] += dataHost.nghost[dir]; + } + } + buf = in; + } else { // target rank >0 + // Fetch the local array size + MPI_Recv(np_int.data(), 3, MPI_INT, rank, 010, MPI_COMM_WORLD, &status); + MPI_Recv(np_tot.data(), 3, MPI_INT, rank, 011, MPI_COMM_WORLD, &status); + MPI_Recv(beg.data(), 3, MPI_INT, rank, 012, MPI_COMM_WORLD, &status); + MPI_Recv(gbeg.data(), 3, MPI_INT, rank, 013, MPI_COMM_WORLD, &status); + + buf = IdefixHostArray4D("pydefix::tempArray",nvar,np_tot[KDIR],np_tot[JDIR],np_tot[IDIR]); + // Fetch data + MPI_Recv(buf.data(), nvar*np_tot[IDIR]*np_tot[JDIR]*np_tot[KDIR], realMPI, rank, 014, MPI_COMM_WORLD,&status); + } // target rank + // Copy data from the buffer + for(int n = 0 ; n < nvar ; n++) { + for(int k = 0 ; k < np_int[KDIR] ; k++) { + const int kt = k+gbeg[KDIR]; + for(int j = 0 ; j < np_int[JDIR] ; j++) { + const int jt = j+gbeg[JDIR]; + for(int i = 0 ; i < np_int[IDIR] ; i++) { + const int it = i+gbeg[IDIR]; + out(n,kt,jt,it) = buf(n,k+beg[KDIR],j+beg[JDIR],i+beg[IDIR]); + } + } + } + }// End for + }// End loop on target rank for root process + } else { // MPI prank >0 + std::array np_int = dataHost.np_int; + std::array np_tot = dataHost.np_tot; + std::array gbeg = dataHost.gbeg; + std::array beg = dataHost.beg; + + // Add back boundaries + for(int dir = 0 ; dir < DIMENSIONS ; dir++) { + if(dataHost.lbound[dir] != internal) { + np_int[dir] += dataHost.nghost[dir]; + gbeg[dir] -= dataHost.nghost[dir]; + beg[dir] -= dataHost.nghost[dir]; + } + if(dataHost.rbound[dir] != internal) { + np_int[dir] += dataHost.nghost[dir]; + } + } + + // send the local array size + MPI_Send(np_int.data(), 3, MPI_INT, 0, 010, MPI_COMM_WORLD); + MPI_Send(np_tot.data(), 3, MPI_INT, 0, 011, MPI_COMM_WORLD); + MPI_Send(beg.data(), 3, MPI_INT, 0, 012, MPI_COMM_WORLD); + MPI_Send(gbeg.data(), 3, MPI_INT, 0, 013, MPI_COMM_WORLD); + MPI_Send(in.data(), nvar*np_tot[IDIR]*np_tot[JDIR]*np_tot[KDIR], realMPI, 0, 014, MPI_COMM_WORLD); // Allocate array + } + // All is transfered + MPI_Bcast(out.data(), nvar*grid->np_tot[KDIR]*grid->np_tot[JDIR]*grid->np_tot[IDIR], realMPI, 0, MPI_COMM_WORLD); + } else { // MPI with a single proc + out = in; + } + #else // No MPI + out = in; + #endif + idfx::cout << "final dims=" << out.extent(0) << " " << out.extent(1) << " " << out.extent(2) << " " << out.extent(3) << std::endl; + return out; +} + PYBIND11_EMBEDDED_MODULE(pydefix, m) { py::class_(m, "DataBlockHost") .def(py::init<>()) @@ -66,6 +166,11 @@ PYBIND11_EMBEDDED_MODULE(pydefix, m) { m.attr("IDIR") = IDIR; m.attr("JDIR") = JDIR; m.attr("KDIR") = KDIR; + + m.attr("prank") = idfx::prank; + m.attr("psize") = idfx::psize; + + m.def("GatherIdefixArray",&GatherIdefixArray, "Gather arrays from domain decomposition"); } diff --git a/src/pydefix.hpp b/src/pydefix.hpp index 2f782d89..b0c27ffc 100644 --- a/src/pydefix.hpp +++ b/src/pydefix.hpp @@ -54,24 +54,33 @@ template struct type_caster> { if ( !convert && !py::array_t::check_(src) ) return false; - auto buf = py::array_t::ensure(src); - if ( !buf ) + auto array = py::array_t::ensure(src); + if ( !array ) return false; - auto dims = buf.ndim(); + auto dims = array.ndim(); if ( dims != 4 ) return false; - std::vector shape(4); - - for ( int i = 0 ; i < 4 ; ++i ) - shape[i] = buf.shape()[i]; - - - value = IdefixHostArray4D("pyArray",shape[0], shape[1], shape[2], shape[3]); - - // Still need to fill in with buf.data()+buf.size() - IDEFIX_ERROR("Python->Idefix Not implemented"); + auto buf = array.request(); + + + + value = Kokkos::View> ((T*)buf.ptr, + array.shape()[0], + array.shape()[1], + array.shape()[2], + array.shape()[3]); + + /* + value = Kokkos::View> ((T*)buf.ptr, + array.shape()[0]);*/ return true; } From a37b5f1044729d9050d5ce08aa6c3a979eda8fcd Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Sat, 28 Dec 2024 18:27:19 +0100 Subject: [PATCH 20/28] better gathering with/without boundaries --- src/gridHost.hpp | 8 ++--- src/pydefix.cpp | 93 +++++++++++++++++++++++++++++++++--------------- 2 files changed, 69 insertions(+), 32 deletions(-) diff --git a/src/gridHost.hpp b/src/gridHost.hpp index 065c65af..55283a5c 100644 --- a/src/gridHost.hpp +++ b/src/gridHost.hpp @@ -22,10 +22,10 @@ class GridHost { public: - std::array::HostMirror,3> x; ///< geometrical central points - std::array::HostMirror,3> xr; ///< cell right interface - std::array::HostMirror,3> xl; ///< cell left interface - std::array::HostMirror,3> dx; ///< cell width + std::array,3> x; ///< geometrical central points + std::array,3> xr; ///< cell right interface + std::array,3> xl; ///< cell left interface + std::array,3> dx; ///< cell width std::array xbeg; ///< Beginning of grid std::array xend; ///< End of grid diff --git a/src/pydefix.cpp b/src/pydefix.cpp index d3204e9d..37ddbc01 100644 --- a/src/pydefix.cpp +++ b/src/pydefix.cpp @@ -22,11 +22,9 @@ namespace py = pybind11; int Pydefix::ninstance = 0; -/************************************ - * DataBlockHost Python binding - * **********************************/ -IdefixHostArray4D GatherIdefixArray(IdefixHostArray4D in, DataBlockHost dataHost) { +namespace PydefixTools { +IdefixHostArray4D GatherIdefixArray(IdefixHostArray4D in, DataBlockHost dataHost, bool keepBoundaries = true) { idfx::cout << "I am being called" << std::endl; idfx::cout << "My dims=" << in.extent(0) << " " << in.extent(1) << " " << in.extent(2) << " " << in.extent(3) << std::endl; idfx::cout << "First element=" << in(0,0,0,0) << std::endl; @@ -35,7 +33,11 @@ IdefixHostArray4D GatherIdefixArray(IdefixHostArray4D in, DataBlockH #ifdef WITH_MPI if(idfx::psize > 1) { const int nvar = in.extent(0); - out = IdefixHostArray4D("pydefix::GatheredArray",nvar,grid->np_tot[KDIR],grid->np_tot[JDIR],grid->np_tot[IDIR]); + if(keepBoundaries) { + out = IdefixHostArray4D("pydefix::GatheredArray",nvar,grid->np_tot[KDIR],grid->np_tot[JDIR],grid->np_tot[IDIR]); + } else { + out = IdefixHostArray4D("pydefix::GatheredArray",nvar,grid->np_int[KDIR],grid->np_int[JDIR],grid->np_int[IDIR]); + } if(idfx::prank == 0) { for(int rank = 0 ; rank < idfx::psize ; rank++) { // np_tot: total size of the incoming array @@ -53,14 +55,16 @@ IdefixHostArray4D GatherIdefixArray(IdefixHostArray4D in, DataBlockH beg = dataHost.beg; // Add back boundaries - for(int dir = 0 ; dir < DIMENSIONS ; dir++) { - if(dataHost.lbound[dir] != internal) { - np_int[dir] += dataHost.nghost[dir]; - gbeg[dir] -= dataHost.nghost[dir]; - beg[dir] -= dataHost.nghost[dir]; - } - if(dataHost.rbound[dir] != internal) { - np_int[dir] += dataHost.nghost[dir]; + if(keepBoundaries) { + for(int dir = 0 ; dir < DIMENSIONS ; dir++) { + if(dataHost.lbound[dir] != internal) { + np_int[dir] += dataHost.nghost[dir]; + gbeg[dir] -= dataHost.nghost[dir]; + beg[dir] -= dataHost.nghost[dir]; + } + if(dataHost.rbound[dir] != internal) { + np_int[dir] += dataHost.nghost[dir]; + } } } buf = in; @@ -76,13 +80,17 @@ IdefixHostArray4D GatherIdefixArray(IdefixHostArray4D in, DataBlockH MPI_Recv(buf.data(), nvar*np_tot[IDIR]*np_tot[JDIR]*np_tot[KDIR], realMPI, rank, 014, MPI_COMM_WORLD,&status); } // target rank // Copy data from the buffer + for(int n = 0 ; n < nvar ; n++) { for(int k = 0 ; k < np_int[KDIR] ; k++) { - const int kt = k+gbeg[KDIR]; + int kt = k+gbeg[KDIR]; + if(!keepBoundaries) kt -= dataHost.nghost[KDIR]; for(int j = 0 ; j < np_int[JDIR] ; j++) { - const int jt = j+gbeg[JDIR]; + int jt = j+gbeg[JDIR]; + if(!keepBoundaries) jt -= dataHost.nghost[JDIR]; for(int i = 0 ; i < np_int[IDIR] ; i++) { - const int it = i+gbeg[IDIR]; + int it = i+gbeg[IDIR]; + if(!keepBoundaries) it -= dataHost.nghost[IDIR]; out(n,kt,jt,it) = buf(n,k+beg[KDIR],j+beg[JDIR],i+beg[IDIR]); } } @@ -96,14 +104,16 @@ IdefixHostArray4D GatherIdefixArray(IdefixHostArray4D in, DataBlockH std::array beg = dataHost.beg; // Add back boundaries - for(int dir = 0 ; dir < DIMENSIONS ; dir++) { - if(dataHost.lbound[dir] != internal) { - np_int[dir] += dataHost.nghost[dir]; - gbeg[dir] -= dataHost.nghost[dir]; - beg[dir] -= dataHost.nghost[dir]; - } - if(dataHost.rbound[dir] != internal) { - np_int[dir] += dataHost.nghost[dir]; + if(keepBoundaries) { + for(int dir = 0 ; dir < DIMENSIONS ; dir++) { + if(dataHost.lbound[dir] != internal) { + np_int[dir] += dataHost.nghost[dir]; + gbeg[dir] -= dataHost.nghost[dir]; + beg[dir] -= dataHost.nghost[dir]; + } + if(dataHost.rbound[dir] != internal) { + np_int[dir] += dataHost.nghost[dir]; + } } } @@ -122,9 +132,13 @@ IdefixHostArray4D GatherIdefixArray(IdefixHostArray4D in, DataBlockH #else // No MPI out = in; #endif - idfx::cout << "final dims=" << out.extent(0) << " " << out.extent(1) << " " << out.extent(2) << " " << out.extent(3) << std::endl; return out; -} +} +}// PydefixTools + +/************************************ + * DataBlockHost Python binding + * **********************************/ PYBIND11_EMBEDDED_MODULE(pydefix, m) { py::class_(m, "DataBlockHost") @@ -145,10 +159,31 @@ PYBIND11_EMBEDDED_MODULE(pydefix, m) { .def_readwrite("Ex2", &DataBlockHost::Ex2) .def_readwrite("Ex3", &DataBlockHost::Ex3) #endif + .def_readwrite("xbeg", &DataBlockHost::xbeg) + .def_readwrite("xend", &DataBlockHost::xend) + .def_readwrite("gbeg", &DataBlockHost::gbeg) + .def_readwrite("gend", &DataBlockHost::gend) + .def_readwrite("beg", &DataBlockHost::beg) + .def_readwrite("end", &DataBlockHost::end) + .def_readwrite("np_tot", &DataBlockHost::np_tot) + .def_readwrite("np_int", &DataBlockHost::np_int) + .def_readwrite("nghost", &DataBlockHost::nghost) .def_readwrite("InvDt", &DataBlockHost::InvDt) .def_readwrite("t",&DataBlockHost::t) .def_readwrite("dt",&DataBlockHost::dt); + py::class_(m, "GridHost") + .def(py::init<>()) + .def_readwrite("x", &GridHost::x) + .def_readwrite("xr", &GridHost::xr) + .def_readwrite("xl", &GridHost::xl) + .def_readwrite("dx", &GridHost::dx) + .def_readwrite("xbeg", &GridHost::xbeg) + .def_readwrite("xend", &GridHost::xend) + .def_readwrite("np_tot", &GridHost::np_tot) + .def_readwrite("np_int", &GridHost::np_int) + .def_readwrite("nghost", &GridHost::nghost); + m.attr("RHO") = RHO; m.attr("VX1") = VX1; m.attr("VX2") = VX2; @@ -170,7 +205,7 @@ PYBIND11_EMBEDDED_MODULE(pydefix, m) { m.attr("prank") = idfx::prank; m.attr("psize") = idfx::psize; - m.def("GatherIdefixArray",&GatherIdefixArray, "Gather arrays from domain decomposition"); + m.def("GatherIdefixArray",&PydefixTools::GatherIdefixArray, "Gather arrays from domain decomposition"); } @@ -228,8 +263,10 @@ void Pydefix::Output(DataBlock &data, int n) { "in your input file [python]:output_function"); } DataBlockHost dataHost(data); + GridHost gridHost(*data.mygrid); + gridHost.SyncFromDevice(); dataHost.SyncFromDevice(); - this->CallScript(this->scriptFilename,this->outputFunctionName,dataHost, n); + this->CallScript(this->scriptFilename,this->outputFunctionName,dataHost, gridHost, n); idfx::popRegion(); } From 8b89c246613cb9fff0ee3b7d5f23acc36a9fb311 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 30 Dec 2024 08:49:44 +0100 Subject: [PATCH 21/28] use numpy array to avoid ownership errors --- src/pydefix.cpp | 220 ++++++++++++++++++++++++++++-------------------- src/pydefix.hpp | 42 ++++----- 2 files changed, 150 insertions(+), 112 deletions(-) diff --git a/src/pydefix.cpp b/src/pydefix.cpp index 37ddbc01..7aedbe76 100644 --- a/src/pydefix.cpp +++ b/src/pydefix.cpp @@ -24,84 +24,58 @@ int Pydefix::ninstance = 0; namespace PydefixTools { -IdefixHostArray4D GatherIdefixArray(IdefixHostArray4D in, DataBlockHost dataHost, bool keepBoundaries = true) { - idfx::cout << "I am being called" << std::endl; - idfx::cout << "My dims=" << in.extent(0) << " " << in.extent(1) << " " << in.extent(2) << " " << in.extent(3) << std::endl; - idfx::cout << "First element=" << in(0,0,0,0) << std::endl; +// Functions provided by Idefix in Pydefix for user convenience +py::array_t GatherIdefixArray(IdefixHostArray3D in, + DataBlockHost dataHost, + bool keepBoundaries = true, + bool broadcast = true) { + idfx::pushRegion("PydefixTools::GatherIdefixArray"); Grid *grid = dataHost.data->mygrid; - IdefixHostArray4D out; - #ifdef WITH_MPI - if(idfx::psize > 1) { - const int nvar = in.extent(0); - if(keepBoundaries) { - out = IdefixHostArray4D("pydefix::GatheredArray",nvar,grid->np_tot[KDIR],grid->np_tot[JDIR],grid->np_tot[IDIR]); - } else { - out = IdefixHostArray4D("pydefix::GatheredArray",nvar,grid->np_int[KDIR],grid->np_int[JDIR],grid->np_int[IDIR]); - } - if(idfx::prank == 0) { - for(int rank = 0 ; rank < idfx::psize ; rank++) { - // np_tot: total size of the incoming array - // np_int: size that should be copied into global - // beg: offset in the incoming array where copy should begin - // gbeg: offset in the global array where copy should be begin - MPI_Status status; - std::array np_int,np_tot, beg, gbeg; - IdefixHostArray4D buf; - - if(rank==0) { - np_int = dataHost.np_int; - np_tot = dataHost.np_tot; - gbeg = dataHost.gbeg; - beg = dataHost.beg; - - // Add back boundaries - if(keepBoundaries) { - for(int dir = 0 ; dir < DIMENSIONS ; dir++) { - if(dataHost.lbound[dir] != internal) { - np_int[dir] += dataHost.nghost[dir]; - gbeg[dir] -= dataHost.nghost[dir]; - beg[dir] -= dataHost.nghost[dir]; - } - if(dataHost.rbound[dir] != internal) { - np_int[dir] += dataHost.nghost[dir]; - } - } - } - buf = in; - } else { // target rank >0 - // Fetch the local array size - MPI_Recv(np_int.data(), 3, MPI_INT, rank, 010, MPI_COMM_WORLD, &status); - MPI_Recv(np_tot.data(), 3, MPI_INT, rank, 011, MPI_COMM_WORLD, &status); - MPI_Recv(beg.data(), 3, MPI_INT, rank, 012, MPI_COMM_WORLD, &status); - MPI_Recv(gbeg.data(), 3, MPI_INT, rank, 013, MPI_COMM_WORLD, &status); - - buf = IdefixHostArray4D("pydefix::tempArray",nvar,np_tot[KDIR],np_tot[JDIR],np_tot[IDIR]); - // Fetch data - MPI_Recv(buf.data(), nvar*np_tot[IDIR]*np_tot[JDIR]*np_tot[KDIR], realMPI, rank, 014, MPI_COMM_WORLD,&status); - } // target rank - // Copy data from the buffer - - for(int n = 0 ; n < nvar ; n++) { - for(int k = 0 ; k < np_int[KDIR] ; k++) { - int kt = k+gbeg[KDIR]; - if(!keepBoundaries) kt -= dataHost.nghost[KDIR]; - for(int j = 0 ; j < np_int[JDIR] ; j++) { - int jt = j+gbeg[JDIR]; - if(!keepBoundaries) jt -= dataHost.nghost[JDIR]; - for(int i = 0 ; i < np_int[IDIR] ; i++) { - int it = i+gbeg[IDIR]; - if(!keepBoundaries) it -= dataHost.nghost[IDIR]; - out(n,kt,jt,it) = buf(n,k+beg[KDIR],j+beg[JDIR],i+beg[IDIR]); - } - } - } - }// End for - }// End loop on target rank for root process - } else { // MPI prank >0 - std::array np_int = dataHost.np_int; - std::array np_tot = dataHost.np_tot; - std::array gbeg = dataHost.gbeg; - std::array beg = dataHost.beg; + IdefixHostArray3D out; + py::array_t pyOut; + if(broadcast || idfx::prank==0) { + if(keepBoundaries) { + // Create a python-managed array, with memory accessible from Kokkos + pyOut = py::array_t({grid->np_tot[KDIR], + grid->np_tot[JDIR], + grid->np_tot[IDIR]}); + out = Kokkos::View> + (reinterpret_cast(pyOut.request().ptr), + grid->np_tot[KDIR], + grid->np_tot[JDIR], + grid->np_tot[IDIR]); + + } else { + pyOut = py::array_t({grid->np_int[KDIR], + grid->np_int[JDIR], + grid->np_int[IDIR]}); + out = Kokkos::View> + (reinterpret_cast(pyOut.request().ptr), + grid->np_int[KDIR], + grid->np_int[JDIR], + grid->np_int[IDIR]); + } + } + if(idfx::prank == 0) { + for(int rank = 0 ; rank < idfx::psize ; rank++) { + // np_tot: total size of the incoming array + // np_int: size that should be copied into global + // beg: offset in the incoming array where copy should begin + // gbeg: offset in the global array where copy should be begin + std::array np_int,np_tot, beg, gbeg; + IdefixHostArray3D buf; + + if(rank==0) { + np_int = dataHost.np_int; + np_tot = dataHost.np_tot; + gbeg = dataHost.gbeg; + beg = dataHost.beg; // Add back boundaries if(keepBoundaries) { @@ -116,25 +90,82 @@ IdefixHostArray4D GatherIdefixArray(IdefixHostArray4D in, DataBlockH } } } + buf = in; + } else { // target rank >0 + #ifdef WITH_MPI + MPI_Status status; + // Fetch the local array size + MPI_Recv(np_int.data(), 3, MPI_INT, rank, 010, MPI_COMM_WORLD, &status); + MPI_Recv(np_tot.data(), 3, MPI_INT, rank, 011, MPI_COMM_WORLD, &status); + MPI_Recv(beg.data(), 3, MPI_INT, rank, 012, MPI_COMM_WORLD, &status); + MPI_Recv(gbeg.data(), 3, MPI_INT, rank, 013, MPI_COMM_WORLD, &status); + + buf = IdefixHostArray3D("pydefix::tempArray", + np_tot[KDIR],np_tot[JDIR],np_tot[IDIR]); + // Fetch data + MPI_Recv(buf.data(), np_tot[IDIR]*np_tot[JDIR]*np_tot[KDIR], + realMPI, rank, 014, MPI_COMM_WORLD,&status); + #else + IDEFIX_ERROR("Can't deal with psize>1 without MPI."); + #endif + } // target rank + // Copy data from the buffer - // send the local array size - MPI_Send(np_int.data(), 3, MPI_INT, 0, 010, MPI_COMM_WORLD); - MPI_Send(np_tot.data(), 3, MPI_INT, 0, 011, MPI_COMM_WORLD); - MPI_Send(beg.data(), 3, MPI_INT, 0, 012, MPI_COMM_WORLD); - MPI_Send(gbeg.data(), 3, MPI_INT, 0, 013, MPI_COMM_WORLD); - MPI_Send(in.data(), nvar*np_tot[IDIR]*np_tot[JDIR]*np_tot[KDIR], realMPI, 0, 014, MPI_COMM_WORLD); // Allocate array + for(int k = 0 ; k < np_int[KDIR] ; k++) { + int kt = k+gbeg[KDIR]; + if(!keepBoundaries) kt -= dataHost.nghost[KDIR]; + for(int j = 0 ; j < np_int[JDIR] ; j++) { + int jt = j+gbeg[JDIR]; + if(!keepBoundaries) jt -= dataHost.nghost[JDIR]; + for(int i = 0 ; i < np_int[IDIR] ; i++) { + int it = i+gbeg[IDIR]; + if(!keepBoundaries) it -= dataHost.nghost[IDIR]; + out(kt,jt,it) = buf(k+beg[KDIR],j+beg[JDIR],i+beg[IDIR]); + } + } + }// End for + }// End loop on target rank for root process + } else { // MPI prank >0 + std::array np_int = dataHost.np_int; + std::array np_tot = dataHost.np_tot; + std::array gbeg = dataHost.gbeg; + std::array beg = dataHost.beg; + + // Add back boundaries + if(keepBoundaries) { + for(int dir = 0 ; dir < DIMENSIONS ; dir++) { + if(dataHost.lbound[dir] != internal) { + np_int[dir] += dataHost.nghost[dir]; + gbeg[dir] -= dataHost.nghost[dir]; + beg[dir] -= dataHost.nghost[dir]; + } + if(dataHost.rbound[dir] != internal) { + np_int[dir] += dataHost.nghost[dir]; + } } - // All is transfered - MPI_Bcast(out.data(), nvar*grid->np_tot[KDIR]*grid->np_tot[JDIR]*grid->np_tot[IDIR], realMPI, 0, MPI_COMM_WORLD); - } else { // MPI with a single proc - out = in; } - #else // No MPI - out = in; + #ifdef WITH_MPI + // send the local array size + MPI_Send(np_int.data(), 3, MPI_INT, 0, 010, MPI_COMM_WORLD); + MPI_Send(np_tot.data(), 3, MPI_INT, 0, 011, MPI_COMM_WORLD); + MPI_Send(beg.data(), 3, MPI_INT, 0, 012, MPI_COMM_WORLD); + MPI_Send(gbeg.data(), 3, MPI_INT, 0, 013, MPI_COMM_WORLD); + MPI_Send(in.data(), np_tot[IDIR]*np_tot[JDIR]*np_tot[KDIR], realMPI, 0, 014, MPI_COMM_WORLD); + #else + IDEFIX_ERROR("Can't deal with psize>1 without MPI."); + #endif + } + // All is transfered + #ifdef WITH_MPI + if(broadcast) { + MPI_Bcast(out.data(), out.extent(0)*out.extent(1)*out.extent(2), realMPI, 0, MPI_COMM_WORLD); + } #endif - return out; + + idfx::popRegion(); + return pyOut; } -}// PydefixTools +// namespace PydefixTools /************************************ * DataBlockHost Python binding @@ -205,7 +236,12 @@ PYBIND11_EMBEDDED_MODULE(pydefix, m) { m.attr("prank") = idfx::prank; m.attr("psize") = idfx::psize; - m.def("GatherIdefixArray",&PydefixTools::GatherIdefixArray, "Gather arrays from domain decomposition"); + m.def("GatherIdefixArray",&PydefixTools::GatherIdefixArray, + py::arg("in"), + py::arg("data"), + py::arg("keepBoundaries") = true, + py::arg("broadcast") = true, + "Gather arrays from MPI domain decomposition"); } diff --git a/src/pydefix.hpp b/src/pydefix.hpp index b0c27ffc..572a92c5 100644 --- a/src/pydefix.hpp +++ b/src/pydefix.hpp @@ -64,23 +64,17 @@ template struct type_caster> { auto buf = array.request(); - - value = Kokkos::View> ((T*)buf.ptr, + + value = Kokkos::View> (reinterpret_cast(buf.ptr), array.shape()[0], array.shape()[1], array.shape()[2], array.shape()[3]); - /* - value = Kokkos::View> ((T*)buf.ptr, - array.shape()[0]);*/ return true; } @@ -106,27 +100,31 @@ template struct type_caster> { // Conversion part 1 (Python -> C++) bool load(py::handle src, bool convert) { + idfx::pushRegion("Pydefix::TypeCaster3D Python->C"); if ( !convert && !py::array_t::check_(src) ) return false; - auto buf = py::array_t::ensure(src); - if ( !buf ) + auto array = py::array_t::ensure(src); + if ( !array ) return false; - auto dims = buf.ndim(); + auto dims = array.ndim(); if ( dims != 3 ) return false; - std::vector shape(3); + auto buf = array.request(); - for ( int i = 0 ; i < 3 ; ++i ) - shape[i] = buf.shape()[i]; - value = IdefixHostArray3D("pyArray",shape[0], shape[1], shape[2]); + value = Kokkos::View> (reinterpret_cast(buf.ptr), + array.shape()[0], + array.shape()[1], + array.shape()[2]); - // Still need to fill in with buf.data()+buf.size() - IDEFIX_ERROR("Python->Idefix Not implemented"); + idfx::popRegion(); return true; } @@ -134,11 +132,15 @@ template struct type_caster> { static py::handle cast(const IdefixHostArray3D& src, py::return_value_policy policy, py::handle parent) { + idfx::pushRegion("Pydefix::TypeCaster3D C->Python"); + // Keep a local reference + auto arr = src; py::none dummyDataOwner; py::array_t a({src.extent(0), src.extent(1), src.extent(2)}, src.data(),dummyDataOwner); + idfx::popRegion(); return a.release(); } }; From f00986cbc9e6c5004b5650cef9ca9e15787ac26b Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 30 Dec 2024 08:53:31 +0100 Subject: [PATCH 22/28] missplaced namespace --- src/pydefix.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pydefix.cpp b/src/pydefix.cpp index 7aedbe76..f0d8bc78 100644 --- a/src/pydefix.cpp +++ b/src/pydefix.cpp @@ -165,7 +165,7 @@ py::array_t GatherIdefixArray(IdefixHostArray3D idfx::popRegion(); return pyOut; } -// namespace PydefixTools +}// namespace PydefixTools /************************************ * DataBlockHost Python binding From 30f8433074b3c85a5d5d8a58d3e7ba8e8afc890c Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 6 Jan 2025 09:32:38 +0100 Subject: [PATCH 23/28] make python auto detection work on MacOs. No need to specify pybind11 path anymore --- CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f7095d2c..d1a2e3d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -124,7 +124,10 @@ endif() if(Idefix_PYTHON) add_compile_definitions("WITH_PYTHON") - #set(PYBIND11_FINDPYTHON ON CACHE BOOL "Idefix requires python" FORCE) + if (NOT DEFINED Python_FIND_FRAMEWORK) + set(Python_FIND_FRAMEWORK "LAST") # Use Apple's python only at last resort on Macos + endif () + set(PYBIND11_FINDPYTHON ON CACHE BOOL "Idefix requires python" FORCE) find_package(pybind11 REQUIRED) target_link_libraries(idefix pybind11::embed) target_sources(idefix From a541532195b4a2db07686807c23d2cba5b82ff6e Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 6 Jan 2025 09:42:06 +0100 Subject: [PATCH 24/28] type caster for 2D and 1D numpy->Idefix Array (not used as of now) --- src/pydefix.hpp | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/src/pydefix.hpp b/src/pydefix.hpp index 572a92c5..eda6a70e 100644 --- a/src/pydefix.hpp +++ b/src/pydefix.hpp @@ -114,8 +114,6 @@ template struct type_caster> { auto buf = array.request(); - - value = Kokkos::View struct type_caster> { if ( dims != 2 ) return false; - std::vector shape(2); - - for ( int i = 0 ; i < 2 ; ++i ) - shape[i] = buf.shape()[i]; - + auto buf = array.request(); - value = IdefixHostArray2D("pyArray",shape[0], shape[1]); + value = Kokkos::View> (reinterpret_cast(buf.ptr), + array.shape()[0], + array.shape()[1]); - // Still need to fill in with buf.data()+buf.size() - IDEFIX_ERROR("Python->Idefix Not implemented"); + idfx::popRegion(); return true; } @@ -202,16 +200,15 @@ template struct type_caster> { if ( dims != 1 ) return false; - std::vector shape(1); - - for ( int i = 0 ; i < 1 ; ++i ) - shape[i] = buf.shape()[i]; - + auto buf = array.request(); - value = IdefixHostArray1D("pyArray",shape[0]); + value = Kokkos::View> (reinterpret_cast(buf.ptr), + array.shape()[0]); - // Still need to fill in with buf.data()+buf.size() - IDEFIX_ERROR("Python->Idefix Not implemented"); + idfx::popRegion(); return true; } From ed7de3c416822a6b6ed376b0c335fa587556aa10 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 6 Jan 2025 10:52:56 +0100 Subject: [PATCH 25/28] proper example with MPI reduction --- .pre-commit-config.yaml | 4 ++++ src/pydefix.hpp | 12 +++++------ test/IO/pydefix/pydefix_example.py | 34 +++++++++++++++++------------- 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 726f0045..72360011 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,6 +34,10 @@ repos: - B # flake8-bugbear - I # isort - NPY # numpy-specific rules + - --ignore + - F405 + - --ignore + - F403 # ignore import * - repo: https://github.com/neutrinoceros/inifix rev: v5.0.2 diff --git a/src/pydefix.hpp b/src/pydefix.hpp index eda6a70e..a12086f8 100644 --- a/src/pydefix.hpp +++ b/src/pydefix.hpp @@ -152,11 +152,11 @@ template struct type_caster> { if ( !convert && !py::array_t::check_(src) ) return false; - auto buf = py::array_t::ensure(src); - if ( !buf ) + auto array = py::array_t::ensure(src); + if ( !array ) return false; - auto dims = buf.ndim(); + auto dims = array.ndim(); if ( dims != 2 ) return false; @@ -192,11 +192,11 @@ template struct type_caster> { if ( !convert && !py::array_t::check_(src) ) return false; - auto buf = py::array_t::ensure(src); - if ( !buf ) + auto array = py::array_t::ensure(src); + if ( !array ) return false; - auto dims = buf.ndim(); + auto dims = array.ndim(); if ( dims != 1 ) return false; diff --git a/test/IO/pydefix/pydefix_example.py b/test/IO/pydefix/pydefix_example.py index 258f674f..a08e8646 100644 --- a/test/IO/pydefix/pydefix_example.py +++ b/test/IO/pydefix/pydefix_example.py @@ -1,29 +1,33 @@ -import pydefix as pdfx +from pydefix import * import numpy as np import matplotlib.pyplot as plt # The output function # the only argument is dataBlockHost python object, wrapping a dataBlockHost Idefix object -def output(data,n): - plt.close() - plt.figure() - plt.pcolormesh(data.x[pdfx.IDIR],data.x[pdfx.JDIR],data.Vc[pdfx.PRS,0,:,:],label='PRS',vmin=0.02,vmax=0.5,cmap='plasma') - plt.title("t=%.2f"%data.t) - plt.colorbar() - plt.savefig("PRS.%.4d.png"%n) +def output(data,grid,n): + # Note: if MPI is not enabled, GatherIdefixArray still works (but merely does a local copy) + pressure = GatherIdefixArray(data.Vc[PRS,:,:,:],data,broadcast=False,keepBoundaries=True) + # only process #0 performs the output + if prank==0: + plt.close() + plt.figure() + plt.pcolormesh(grid.x[IDIR],grid.x[JDIR],pressure[0,:,:],label='PRS',vmin=0.02,vmax=0.5,cmap='plasma') + plt.title("t=%.2f"%data.t) + plt.colorbar() + plt.savefig("PRS.%.4d.png"%n) def initflow(data): # Field amplitude B0 = 1/np.sqrt(4*np.pi) - [z,y,x] = np.meshgrid(data.x[pdfx.KDIR], data.x[pdfx.JDIR], data.x[pdfx.IDIR], indexing='ij') + [z,y,x] = np.meshgrid(data.x[KDIR], data.x[JDIR], data.x[IDIR], indexing='ij') # Initialize the flow - data.Vc[pdfx.RHO,:,:,:] = 25/(36*np.pi) - data.Vc[pdfx.PRS,:,:,:] = 5/(12*np.pi) - data.Vc[pdfx.VX1,:,:,:] = -np.sin(2*np.pi*y) - data.Vc[pdfx.VX2,:,:,:] = np.sin(2*np.pi*x) + data.Vc[RHO,:,:,:] = 25/(36*np.pi) + data.Vc[PRS,:,:,:] = 5/(12*np.pi) + data.Vc[VX1,:,:,:] = -np.sin(2*np.pi*y) + data.Vc[VX2,:,:,:] = np.sin(2*np.pi*x) - data.Vs[pdfx.BX1s,:,:-1,:-1] = -B0*np.sin(2*np.pi*y) - data.Vs[pdfx.BX2s,:,:-1,:-1] = B0*np.sin(4*np.pi*x) + data.Vs[BX1s,:,:-1,:-1] = -B0*np.sin(2*np.pi*y) + data.Vs[BX2s,:,:-1,:-1] = B0*np.sin(4*np.pi*x) From a7d809a12800140a857c89a28eb373c0ffea5c4d Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Mon, 6 Jan 2025 14:49:32 +0100 Subject: [PATCH 26/28] add documentation for pydefix --- doc/source/index.rst | 4 + doc/source/modules.rst | 4 + doc/source/modules/pydefix.rst | 126 ++++++++++++++++++++++++++++ doc/source/reference/idefix.ini.rst | 20 +++++ doc/source/reference/outputs.rst | 11 ++- 5 files changed, 161 insertions(+), 4 deletions(-) create mode 100644 doc/source/modules/pydefix.rst diff --git a/doc/source/index.rst b/doc/source/index.rst index fa8dadb3..128d591a 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -33,6 +33,10 @@ MPI library When using MPI parallelisation, *Idefix* relies on an external MPI library. *Idefix* has been tested successfully with OpenMPI and IntelMPI libraries. When used on GPU architectures, *Idefix* assumes that the MPI library is GPU-Aware. If unsure, check this last point with your system administrator. +Python + When using *Idefix* with its python interface through the module `Pydefix`, *Idefix* relies on an external python>=3.8 interpreter with the module `pybind11 `_ + installed. + ================ Features ================ diff --git a/doc/source/modules.rst b/doc/source/modules.rst index 13287ece..2f1f3787 100644 --- a/doc/source/modules.rst +++ b/doc/source/modules.rst @@ -28,6 +28,9 @@ In this section, you will find a more detailed documentation about each module t :ref:`gridCoarseningModule` The grid coarsening module, that allows to derefine the grid in particular locations to speed up the computation. +:ref:`pydefixModule` + The Python interaction module, that allows Idefix to interact directly with a python interpreter. + .. toctree:: :maxdepth: 2 @@ -40,3 +43,4 @@ In this section, you will find a more detailed documentation about each module t modules/selfGravity.rst modules/braginskii.rst modules/gridCoarsening.rst + modules/pydefix.rst diff --git a/doc/source/modules/pydefix.rst b/doc/source/modules/pydefix.rst new file mode 100644 index 00000000..f6828420 --- /dev/null +++ b/doc/source/modules/pydefix.rst @@ -0,0 +1,126 @@ +.. _pydefixModule: + +Pydefix module +============== + +The Pydefix module allows Idefix to talk directly to a Python interpreter while running. It can be used to create your own initial condition +and/or for custom outputs produced from Python. Pydefix relies on the pybind11 python package + +The essence of Pydefix is to allows the user to have a direct access to Idefix data structure from Python without writing/accessing any file. In particular, IdefixArrays are viewed as numpy arrays in Python. +Note however that to keep things simple, Pydefix works on the host memory space only, and hence sync data to/from the GPU (if used) before invoking Python functions. Hence, using Pydefix for outputs induces +a significant loss of performances. + + +Before you start +---------------- +Pybind11 installation ++++++++++++++++++++++ + +In order to use pydefix, you need to be working in a python>=3.8 environement that includes `pybind11 `_. Follow the instruction of your package manager to install pybind11>=2.12. + +Pydefix usage +------------- +Idefix Configuration +++++++++++++++++++++ + +In order to use Pydefix, you need to switch on ``Idefix_PYTHON`` in cmake. This will auto-detect Python and check that pybind11 can be used effectively. + + +Run Idefix with Pydefix ++++++++++++++++++++++++ + +Pydefix is typically enabled from your input file `idefix.ini` in the block ``[Python]``. The following parameters are available: + ++------------------------+-----------------------+-----------------------------------------------------------------------------------------------------------+ +| Entry name | Parameter type | Comment | ++========================+=======================+===========================================================================================================+ +| script | string | | (Mandatory) Filename (*without ".py"!*) of the python script that Idefix should use. | +| | | | The script should be in the same location as the Idefix executable file. | ++------------------------+-----------------------+-----------------------------------------------------------------------------------------------------------+ +| output_function | string | | (Optional) Name of the function that will be called for each output event (the function should be | +| | | | defined in the python script above). When ommited, pydefix output functions are disabled. | +| | | | The periodicity of the pydefix output routine is set in the block:entry `[Output]:python` | ++------------------------+-----------------------+-----------------------------------------------------------------------------------------------------------+ +| initflow_function | string | | (Optional) Name of the python function that will be called to initialize the flow in place of the C++ | +| | | | function ``Setup::InitFlow``. Revert to ``Setup::Initflow`` when ommited. | ++------------------------+-----------------------+-----------------------------------------------------------------------------------------------------------+ + +Python script ++++++++++++++ + +When using Pydefix, idefix expects a python script to be specified in the input file (see ``script`` parameter above). To be fully functionnal, you should import the ``pydefix`` module at the beginning +of your python script (you can also import other python module, as any python script). + +Your python script should define functions that will be called while Idefix is running: +* The signature of the ``initflow`` function should be ``(data)`` where ``data`` is a python structure matching Idefix's ``DataBlockHost`` class. +* The signature of the ``output`` function should be ``(data,grid,n)`` where ``data`` is a python structure matching Idefix's ``DataBlockHost`` class, ``grid`` is Idefix's ``GridHost`` class, and ``n`` is an integer representing the current number of the output + +MPI parallelism ++++++++++++++++ +When Idefix runs with MPI parallelism enabled, a python interpreter and script is launched by each MPI process. Each of these script is independent +and have access to its local ``dataBlockHost``. The `pydefix` module however gives access to the local rank ``prank`` and total MPI size ``psize``. In addition, +pydefix provides the function ``GatherIdefixArray`` to gather the data distributed among each process without invoking MPI directly in python. This function +expects a 3D distributed IdefixArray in entry following the signature + +.. code-block:: c++ + + GatherIdefixArray(IdefixHostArray3D in, // 3D distributed array + DataBlockHost dataHost, // dataBlock structure + bool keepBoundaries = true, // Whether we keep the ghost zones in the returned array + bool broadcast = true) // Whether the returned array is available only in proc #0 or in every proc (caution! possibly requires lots of memory) + +This function is used as follows: + +.. code-block:: python + + import pydefix as pdfx # mandatory + import numpy as np + import matplotlib.pyplot as plt + + def output(data,grid,n): + # Gather pressure field from every process in process #0 (set broadcast to True to distribute the result to all processes) + prs = pdfx.GatherIdefixArray(data.Vc[pdfx.PRS,:,:,:],data,broadcast=False) + + # Only root process performs this + if pdfx.prank==0: + x=grid.x[pdfx.IDIR] # The grid contains the full domain size, while the datablock contains only local information + y=grid.x[pdfx.JDIR] + plt.figure() + plt.pcolormesh(x,y,prs[0,:,:],cmap='plasma') + + +.. note:: + For more advanced usage, it is also possibly to directly call MPI routines from python using the `Mpi4py `_ module. + +Example ++++++++ + +An example is provided in the directory `test/IO/python`. This example shows how to use Idefix with pure python initial conditions and outputs. +It reproduces the 2D OrszagTang vortex available in MHD/OrszagTang without requiring any single line of C++ code from the user. + +The python script `pydefix_example.py` initializes the initial condition of the OT test (``initflow``) and produces a series of PNG files through matplotlib (`output`). + +Troubleshooting +--------------- + +It during configuration stage, you get:: + + CMake Error at CMakeLists.txt:122 (find_package): + By not providing "Findpybind11.cmake" in CMAKE_MODULE_PATH this project has + asked CMake to find a package configuration file provided by "pybind11", + but CMake did not find one. + +It means that cmake cannot find the location of pybind11 (this typically happens on MacOs). In order to locate pybind11, open a python interpreter, and get pybind11 install dir through: + +.. code-block:: python + + import pybind11 + print(pybind11.__file__) + +You can then exit the interpreter and set the pybind11_DIR environement variable to the right path: + +.. code-block:: bash + + export pybind11_DIR=env/lib/python3.10/site-packages/pybind11 + +you can then run cmake which should be able to find pybind11, and compile the code. diff --git a/doc/source/reference/idefix.ini.rst b/doc/source/reference/idefix.ini.rst index a7309f10..c9f9c578 100644 --- a/doc/source/reference/idefix.ini.rst +++ b/doc/source/reference/idefix.ini.rst @@ -358,6 +358,23 @@ and ``X1-end``, ``X2-end``, ``X3-end`` for the right boundaries. ``X2`` boundari | | | (see :ref:`userdefBoundaries`) | +----------------+------------------------------------------------------------------------------------------------------------------+ +``Python`` section +------------------ + +This section describes the python script and function that can interact with Idefix while running using the Pydefix module (see :ref:`pydefixModule`) + ++------------------------+-----------------------+-----------------------------------------------------------------------------------------------------------+ +| Entry name | Parameter type | Comment | ++========================+=======================+===========================================================================================================+ +| script | string | | (Mandatory) Filename (*without ".py"!*) of the python script that Idefix should use. | +| | | | The script should be in location of Idefix executable file | ++------------------------+-----------------------+-----------------------------------------------------------------------------------------------------------+ +| output_function | string | | (Optional) Name of the function that will be called for each output event (the function should be | +| | | | defined in the python script above). When ommited, pydefix output functions are disabled. | ++------------------------+-----------------------+-----------------------------------------------------------------------------------------------------------+ +| initflow_function | string | | (optional) Name of the python function that will be called to initialize the flow in place of the C++ | +| | | | function `Setup::InitFlow`. Revert to `Setup::Initflow`` when ommited. | ++------------------------+-----------------------+-----------------------------------------------------------------------------------------------------------+ .. _outputSection: @@ -411,6 +428,9 @@ This section describes the outputs *Idefix* produces. For more details about eac | | | | (see :ref:`functionEnrollment`). The user-defined variables defined by this function | | | | | are then written as new variables in vtk and/or xdmf outputs. | +----------------+-------------------------+--------------------------------------------------------------------------------------------------+ +| python | float | | Time interval between pydefix outputs, in code units. | +| | | | If negative, periodic pydefix outputs are disabled. | ++----------------+-------------------------+--------------------------------------------------------------------------------------------------+ .. note:: Even if dumps are not mentionned in your input file (and are therefore disabled), dump files are still produced when *Idefix* captures a signal diff --git a/doc/source/reference/outputs.rst b/doc/source/reference/outputs.rst index d531b497..2604d9ab 100644 --- a/doc/source/reference/outputs.rst +++ b/doc/source/reference/outputs.rst @@ -7,7 +7,7 @@ Output formats -------------- *Idefix* uses several types of outputs you may want for your setup. By default, *Idefix* allows -for 4 kinds of outputs: +for 5 kinds of outputs: * logs which essentially tells the user what *Idefix* is currently doing. When running in serial, logs are sent to stdout, but when MPI is enabled, only the logs of the rank 0 process is sent to stdout, and each process (including rank 0) simultaneously writes a @@ -21,17 +21,20 @@ for 4 kinds of outputs: or `Visit `_. The XDMF format relies on the HDF5 format and therefore requires *Idefix* to be configured with HDF5 support. * user-defined analysis files. These are totally left to the user. They usually consist of ascii tables defined by the user, but they can be anything. +* python script, that relies on the :ref:`pydefixModule`. This launches a user-defined python function fed with Idefix data. One can then directly plot or interact with Idefix outputs from python. The output periodicity and the userdef variables should all be declared in the input file, as described in :ref:`outputSection`. Defining your own outputs ------------------------- -*Idefix* provides two ways to define your own outputs: analysis, which are used to make your -own output file (e.g. an ascii-tabulated file); and user variables, which are written by *Idefix* output routines. +*Idefix* provides three ways to define your own outputs: analysis, which are used to make your +own output file (e.g. an ascii-tabulated file); user variables, which are written by *Idefix* output routines, and python user-defined +functions that process Idefix's data. Both analysis and uservar requires the definition of a user function which needs to be enrolled following the procedure described -in :ref:`functionEnrollment` and using the function signatures declared in `output.hpp`. +in :ref:`functionEnrollment` and using the function signatures declared in `output.hpp`. The python outputs are described +in the :ref:`pydefixModule`. We provide below an example of a setup using both analysis outputs and uservar outputs From 682f93bb632f76c5d8aa4a4e528479a39a45ec15 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 8 Jan 2025 09:42:32 +0100 Subject: [PATCH 27/28] missing initialisation of boundary conditions in dataBlockHost --- src/dataBlock/dataBlockHost.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/dataBlock/dataBlockHost.cpp b/src/dataBlock/dataBlockHost.cpp index 91f599c2..d570648f 100644 --- a/src/dataBlock/dataBlockHost.cpp +++ b/src/dataBlock/dataBlockHost.cpp @@ -34,6 +34,9 @@ DataBlockHost::DataBlockHost(DataBlock& datain) { nghost = data->nghost; + lbound = data->lbound; + rbound = data->rbound; + xbeg = data->xbeg; xend = data->xend; beg = data->beg; From f89fc65af2072233d0167d5c0f1e84bbc4959da3 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 8 Jan 2025 14:09:53 +0100 Subject: [PATCH 28/28] update recommended configuration on Adastra following discussion with Paul Segretain to be compatible with pydefix --- doc/source/reference/makefile.rst | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/doc/source/reference/makefile.rst b/doc/source/reference/makefile.rst index 4f948e5d..aadc205c 100644 --- a/doc/source/reference/makefile.rst +++ b/doc/source/reference/makefile.rst @@ -108,18 +108,11 @@ We recommend the following modules and environement variables on AdAstra: .. code-block:: bash - module load cpe/23.12 + module load cpe/24.07 module load craype-accel-amd-gfx90a craype-x86-trento module load PrgEnv-cray - module load amd-mixed/5.7.1 - module load rocm/5.7.1 # nécessaire a cause d'un bug de path pas encore fix.. - export HIPCC_COMPILE_FLAGS_APPEND="-isystem ${CRAY_MPICH_PREFIX}/include" - export HIPCC_LINK_FLAGS_APPEND="-L${CRAY_MPICH_PREFIX}/lib -lmpi ${PE_MPICH_GTL_DIR_amd_gfx90a} ${PE_MPICH_GTL_LIBS_amd_gfx90a} -lstdc++fs" - export CXX=hipcc - export CC=hipcc - -The `-lstdc++fs` option being there to guarantee the link to the HIP library and the access to specific -C++17 functions. + module load amd-mixed + module load cray-python/3.11.7 Finally, *Idefix* can be configured to run on Mi250 by enabling HIP and the desired architecture with the following options to ccmake: