From c08e4626f218c4ddf2f5d5eb40152fd3fe145007 Mon Sep 17 00:00:00 2001 From: maorz1998 Date: Wed, 28 Jun 2023 17:42:05 +0800 Subject: [PATCH 1/4] update configure and installation for GPU solver --- Allwclean | 1 + applications/solvers/dfLowMachFoam/EEqn.H | 66 +++++++------------ .../solvers/dfLowMachFoam/Make/options | 13 ++-- applications/solvers/dfLowMachFoam/UEqn.H | 10 --- applications/solvers/dfLowMachFoam/YEqn.H | 25 ++++--- .../solvers/dfLowMachFoam/dfLowMachFoam.C | 15 +++-- bashrc.in | 5 +- configure.sh | 21 +++++- install.sh | 25 ++++--- 9 files changed, 87 insertions(+), 94 deletions(-) diff --git a/Allwclean b/Allwclean index 2dc13b3ef..569262a61 100755 --- a/Allwclean +++ b/Allwclean @@ -25,3 +25,4 @@ wclean ./applications/solvers/dfHighSpeedFoam rm -rf src_orig/ rm -rf bin/ rm -rf lib/ +rm -rf src_gpu/build diff --git a/applications/solvers/dfLowMachFoam/EEqn.H b/applications/solvers/dfLowMachFoam/EEqn.H index a2e351074..6ebf7ca31 100644 --- a/applications/solvers/dfLowMachFoam/EEqn.H +++ b/applications/solvers/dfLowMachFoam/EEqn.H @@ -1,47 +1,5 @@ { volScalarField& he = thermo.he(); - -#ifdef CPUSolver_ - start1 = std::clock(); - //debug - // { - // const fvPatchScalarField& hew = he.boundaryField()[5]; - // const basicThermo& bThermo = basicThermo::lookupThermo(hew); - // const scalarField& pw = bThermo.p().boundaryField()[5]; - // fvPatchScalarField& Tw = - // const_cast(bThermo.T().boundaryField()[5]); - // scalarField& Tw_v = Tw; - - // Tw.evaluate(); - - // Info << "internal field" <alphaEff(), he) - + diffAlphaD - == - fvc::div(hDiffCorrFlux) - ); - end1 = std::clock(); - time_monitor_EEqn += double(end1 - start1) / double(CLOCKS_PER_SEC); - time_monitor_EEqn_mtxAssembly += double(end1 - start1) / double(CLOCKS_PER_SEC); - - // EEqn.relax(); - start1 = std::clock(); - EEqn.solve(); - end1 = std::clock(); - time_monitor_EEqn += double(end1 - start1) / double(CLOCKS_PER_SEC); - time_monitor_EEqn_solve += double(end1 - start1) / double(CLOCKS_PER_SEC); -#endif - #ifdef GPUSolver_ start1 = std::clock(); UEqn_GPU.updatePsi(&U[0][0]); @@ -51,9 +9,7 @@ end1 = std::clock(); time_monitor_UEqn += double(end1 - start1) / double(CLOCKS_PER_SEC); time_monitor_UEqn_correctBC += double(end1 - start1) / double(CLOCKS_PER_SEC); -#endif -#ifdef GPUSolver_ // prepare data on CPU start1 = std::clock(); start2 = std::clock(); @@ -148,5 +104,27 @@ end1 = std::clock(); time_monitor_EEqn += double(end1 - start1) / double(CLOCKS_PER_SEC); time_monitor_EEqn_correctBC += double(end1 - start1) / double(CLOCKS_PER_SEC); +#else + start1 = std::clock(); + fvScalarMatrix EEqn + ( + fvm::ddt(rho, he) + fvm::div(phi, he) + + fvc::ddt(rho, K) + fvc::div(phi, K) + - dpdt + - fvm::laplacian(turbulence->alphaEff(), he) + + diffAlphaD + == + fvc::div(hDiffCorrFlux) + ); + end1 = std::clock(); + time_monitor_EEqn += double(end1 - start1) / double(CLOCKS_PER_SEC); + time_monitor_EEqn_mtxAssembly += double(end1 - start1) / double(CLOCKS_PER_SEC); + + // EEqn.relax(); + start1 = std::clock(); + EEqn.solve(); + end1 = std::clock(); + time_monitor_EEqn += double(end1 - start1) / double(CLOCKS_PER_SEC); + time_monitor_EEqn_solve += double(end1 - start1) / double(CLOCKS_PER_SEC); #endif } diff --git a/applications/solvers/dfLowMachFoam/Make/options b/applications/solvers/dfLowMachFoam/Make/options index 506f0d90a..e2a57bd00 100644 --- a/applications/solvers/dfLowMachFoam/Make/options +++ b/applications/solvers/dfLowMachFoam/Make/options @@ -9,6 +9,7 @@ EXE_INC = -std=c++14 \ $(PFLAGS) $(PINC) \ $(if $(LIBTORCH_ROOT),-DUSE_LIBTORCH,) \ $(if $(PYTHON_INC_DIR),-DUSE_PYTORCH,) \ + $(if $(AMGX_DIR),-DGPUSolver_,) \ -I$(LIB_SRC)/transportModels/compressible/lnInclude \ -I$(LIB_SRC)/thermophysicalModels/basic/lnInclude \ -I$(LIB_SRC)/TurbulenceModels/turbulenceModels/lnInclude \ @@ -26,9 +27,9 @@ EXE_INC = -std=c++14 \ $(if $(LIBTORCH_ROOT),-I$(LIBTORCH_ROOT)/include,) \ $(if $(LIBTORCH_ROOT),-I$(LIBTORCH_ROOT)/include/torch/csrc/api/include,) \ $(PYTHON_INC_DIR) \ - -I$(DF_ROOT)/src_gpu \ - -I/usr/local/cuda-11.6/include \ - -I$(AMGX_DIR)/include + $(if $(AMGX_DIR), -I$(DF_ROOT)/src_gpu,) \ + $(if $(AMGX_DIR), -I/usr/local/cuda-11.6/include,) \ + $(if $(AMGX_DIR), -I$(AMGX_DIR)/include,) EXE_LIBS = \ -lcompressibleTransportModels \ @@ -50,7 +51,7 @@ EXE_LIBS = \ $(if $(LIBTORCH_ROOT),$(DF_SRC)/dfChemistryModel/DNNInferencer/build/libDNNInferencer.so,) \ $(if $(PYTHON_LIB_DIR),-L$(PYTHON_LIB_DIR),) \ $(if $(PYTHON_LIB_DIR),-lpython3.8,) \ - $(DF_ROOT)/src_gpu/build/libdfMatrix.so \ - /usr/local/cuda-11.6/lib64/libcudart.so \ - $(AMGX_DIR)/build/libamgxsh.so + $(if $(AMGX_DIR), /usr/local/cuda-11.6/lib64/libcudart.so,) \ + $(if $(AMGX_DIR), $(DF_ROOT)/src_gpu/build/libdfMatrix.so,) \ + $(if $(AMGX_DIR), $(AMGX_DIR)/build/libamgxsh.so,) diff --git a/applications/solvers/dfLowMachFoam/UEqn.H b/applications/solvers/dfLowMachFoam/UEqn.H index 65bc7e788..cac4218c0 100644 --- a/applications/solvers/dfLowMachFoam/UEqn.H +++ b/applications/solvers/dfLowMachFoam/UEqn.H @@ -79,16 +79,6 @@ // } // } // } - // const tmp tgradU(fvc::grad(U)); - // const volTensorField& gradU = tgradU(); - // Pout << "gradU_of[1]\n" << gradU[1] << nl; - // Pout << "gradU_of[0][64]\n" << gradU.boundaryField()[0][64] << nl; - // Pout << "gradU_of[1][64]\n" << gradU.boundaryField()[1][64] << nl; - // Pout << "gradU_of[5][1]\n" << gradU.boundaryField()[5][1] << nl; - // Pout << "U[1][1]\n" << U[1] << nl; - // Pout << "Ubou[0][64]\n" << U.boundaryField()[0][64] << nl; - // Pout << "Ubou[1][64]\n" << U.boundaryField()[1][64] << nl; - // Pout << "Ubou[5][1]\n" << U.boundaryField()[5][1] << nl; // if (pimple.momentumPredictor()) // { // solve(UEqn); diff --git a/applications/solvers/dfLowMachFoam/YEqn.H b/applications/solvers/dfLowMachFoam/YEqn.H index 78a20397a..aff3fdbd6 100644 --- a/applications/solvers/dfLowMachFoam/YEqn.H +++ b/applications/solvers/dfLowMachFoam/YEqn.H @@ -12,20 +12,6 @@ tmp> mvConvection mesh.divScheme("div(phi,Yi_h)") ) ); - -#ifdef CPUSolver_ -start1 = std::clock(); -forAll(Y, i) -{ - sumYDiffError += chemistry->rhoD(i)*fvc::grad(Y[i]); -} -// Info << "sumYDiffError\n" << sumYDiffError << endl; -const surfaceScalarField phiUc = linearInterpolate(sumYDiffError) & mesh.Sf(); -start1 = std::clock(); -time_monitor_YEqn += double(end1 - start1) / double(CLOCKS_PER_SEC); -time_monitor_YEqn_solve += double(end1 - start1) / double(CLOCKS_PER_SEC); -#endif - #ifdef GPUSolver_ start1 = std::clock(); UEqn_GPU.solve(); @@ -124,6 +110,17 @@ time_monitor_YEqn_solve += double(end1 - start1) / double(CLOCKS_PER_SEC); end1 = std::clock(); time_monitor_YEqn += double(end1 - start1) / double(CLOCKS_PER_SEC); time_monitor_YEqn_solve += double(end1 - start1) / double(CLOCKS_PER_SEC); +#else + start1 = std::clock(); + forAll(Y, i) + { + sumYDiffError += chemistry->rhoD(i)*fvc::grad(Y[i]); + } + // Info << "sumYDiffError\n" << sumYDiffError << endl; + const surfaceScalarField phiUc = linearInterpolate(sumYDiffError) & mesh.Sf(); + start1 = std::clock(); + time_monitor_YEqn += double(end1 - start1) / double(CLOCKS_PER_SEC); + time_monitor_YEqn_solve += double(end1 - start1) / double(CLOCKS_PER_SEC); #endif //MPI_Barrier(PstreamGlobals::MPI_COMM_FOAM); diff --git a/applications/solvers/dfLowMachFoam/dfLowMachFoam.C b/applications/solvers/dfLowMachFoam/dfLowMachFoam.C index 284dc88f0..34fa85ed3 100644 --- a/applications/solvers/dfLowMachFoam/dfLowMachFoam.C +++ b/applications/solvers/dfLowMachFoam/dfLowMachFoam.C @@ -59,6 +59,7 @@ Description #include "basicThermo.H" #include "CombustionModel.H" +#ifdef GPUSolver_ #include "dfUEqn.H" #include "dfYEqn.H" #include "dfRhoEqn.H" @@ -66,9 +67,7 @@ Description #include #include #include "upwind.H" - -#define GPUSolver_ -// #define CPUSolver_ +#endif // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // @@ -79,10 +78,10 @@ int main(int argc, char *argv[]) #endif #include "postProcess.H" - unsigned int flags = 0; - checkCudaErrors(cudaGetDeviceFlags(&flags)); - flags |= cudaDeviceScheduleYield; - checkCudaErrors(cudaSetDeviceFlags(flags)); + // unsigned int flags = 0; + // checkCudaErrors(cudaGetDeviceFlags(&flags)); + // flags |= cudaDeviceScheduleYield; + // checkCudaErrors(cudaSetDeviceFlags(flags)); // #include "setRootCaseLists.H" #include "listOptions.H" @@ -158,7 +157,9 @@ int main(int argc, char *argv[]) } start1 = std::clock(); + #ifdef GPUSolver_ #include "createdfSolver.H" + #endif end1 = std::clock(); time_monitor_init += double(end1 - start1) / double(CLOCKS_PER_SEC); diff --git a/bashrc.in b/bashrc.in index 6d3b75da9..0aa02ce1a 100644 --- a/bashrc.in +++ b/bashrc.in @@ -8,12 +8,11 @@ export CANTERA_ROOT=LIBCANTERA_DIR export CANTERA_DATA=$CANTERA_ROOT/share/cantera/data export LD_LIBRARY_PATH=$LIBTORCH_ROOT/lib:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=$CANTERA_ROOT/lib:$LD_LIBRARY_PATH +export AMGX_DIR=@AMGX_DIR@ export DF_APPBIN=pwd/bin export DF_LIBBIN=pwd/lib export PATH=$DF_APPBIN:$PATH export LD_LIBRARY_PATH=$DF_LIBBIN:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=$DF_ROOT/src_gpu/build:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=path-to-amgx/build:$LD_LIBRARY_PATH -export AMGX_DIR=path-to-amgx - +export LD_LIBRARY_PATH=$AMGX_DIR/build:$LD_LIBRARY_PATH \ No newline at end of file diff --git a/configure.sh b/configure.sh index 1cf9254a9..4947df660 100644 --- a/configure.sh +++ b/configure.sh @@ -7,15 +7,19 @@ unset LIBTORCH_DIR unset LIBCANTERA_DIR unset PYTORCH_INC unset PYTORCH_LIB +unset USE_GPUSOLVER +unset AMGX_DIR print_usage() { - echo "Usage: . install.sh --libtorch_no (default) | --libtorch_dir _path_to_libtorch | --libtorch_autodownload | --use_pytorch | --libcantera_dir _path_to_libcantera" + echo "Usage: . install.sh --libtorch_no (default) | --libtorch_dir _path_to_libtorch | --libtorch_autodownload | --use_pytorch | --libcantera_dir _path_to_libcantera + | --amgx_dir _path_to_amgx" } # default LIBTORCH_AUTO=false USE_LIBTORCH=false USE_PYTORCH=false +USE_GPUSOLVER=false while test $# -gt 0; do case "$1" in @@ -60,6 +64,17 @@ while test $# -gt 0; do fi shift ;; + --amgx_dir) + shift + if test $# -gt 0; then + AMGX_DIR=$1 + USE_GPUSOLVER=true + else + print_usage + return + fi + shift + ;; -h|--help) shift print_usage @@ -148,6 +163,9 @@ if [ $USE_PYTORCH = true ]; then echo PYTORCH_LIB=$PYTORCH_LIB echo LIBTORCH_DIR="" fi +if [ $USE_GPUSOLVER = true ]; then + echo AMGX_DIR=$AMGX_DIR +fi cp bashrc.in bashrc sed -i "s#pwd#$PWD#g" ./bashrc @@ -155,6 +173,7 @@ sed -i "s#LIBTORCH_DIR#$LIBTORCH_DIR#g" ./bashrc sed -i "s#PYTORCH_INC#$PYTORCH_INC#g" ./bashrc sed -i "s#PYTORCH_LIB#$PYTORCH_LIB#g" ./bashrc sed -i "s#LIBCANTERA_DIR#$LIBCANTERA_DIR#g" ./bashrc +sed -i "s#@AMGX_DIR@#$AMGX_DIR#g" ./bashrc diff --git a/install.sh b/install.sh index 17b3d2097..7a699bb25 100755 --- a/install.sh +++ b/install.sh @@ -4,18 +4,17 @@ print_finish() { if [ ! -z "$LIBTORCH_ROOT" ]; then echo " = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =" echo "| deepflame (linked with libcantera and libtorch) compiled successfully! Enjoy!! |" + elif [ ! -z "$PYTHON_LIB_DIR" ]; then echo " = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =" - return + echo "| deepflame (linked with libcantera and pytorch) compiled successfully! Enjoy!! |" + else + echo " = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =" + echo "| deepflame (linked with libcantera) compiled successfully! Enjoy!! |" fi - if [ ! -z "$PYTHON_LIB_DIR" ]; then - echo " = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =" - echo "| deepflame (linked with libcantera and pytorch) compiled successfully! Enjoy!! | " - echo " = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =" - return + if [ ! -z "$AMGX_DIR" ]; then + echo "| select the GPU solver coupled with AMGx library to solve PDE |" fi - echo " = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =" - echo "| deepflame (linked with libcantera) compiled successfully! Enjoy!! |" - echo " = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =" + echo " = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =" } if [ $USE_LIBTORCH = true ]; then cd "$DF_SRC/dfChemistryModel/DNNInferencer" @@ -25,5 +24,13 @@ if [ $USE_LIBTORCH = true ]; then make export LD_LIBRARY_PATH=$DF_SRC/dfChemistryModel/DNNInferencer/build:$LD_LIBRARY_PATH fi +if [ $USE_GPUSOLVER = true ]; then + cd "$DF_ROOT/src_gpu" + mkdir build + cd build + cmake .. + make + export LD_LIBRARY_PATH=$DF_ROOT/src_gpu/build:$LD_LIBRARY_PATH +fi cd $DF_ROOT ./Allwmake -j && print_finish From e419f718432ef6554c1e4e0dbdf55801f255836c Mon Sep 17 00:00:00 2001 From: maorz1998 Date: Wed, 28 Jun 2023 17:52:56 +0800 Subject: [PATCH 2/4] modify docs --- docs/source/qs/install.rst | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/docs/source/qs/install.rst b/docs/source/qs/install.rst index 28a3ad9e4..8bf129840 100644 --- a/docs/source/qs/install.rst +++ b/docs/source/qs/install.rst @@ -142,27 +142,25 @@ If ``df-notorch`` not activated (or you have a self-complied libcantera), specif . install.sh -**3. If you wish to employ the AMGX library for accelerating PDE solving using GPU:** +**3. If you wish to employ dfMatrix and the AMGX library for accelerating PDE solving using GPU:** To begin, you will need to install AMGX. You can find the instructions for installing AMGX on its official website. Follow the instructions provided to install AMGX on your system. Once you have installed AMGX, navigate to the DeepFlame directory and follow the commands below. .. code-block:: bash - cd ${DF_ROOT}/src/dfMatrix/solver/amgx/ - export AMGX_DIR=/your/path/to/AMGX/ - cmake -B build - cd build - make - -After this, two libraries for enabling DeepFlame with AMGX are available in ``${DF_ROOT}/src/dfMatrix/solver/amgx/build``. -Beforing using AMGX, run: - -.. code-block:: bash - - export LD_LIBRARY_PATH=${DF_ROOT}/src/dfMatrix/solver/amgx/build:$LD_LIBRARY_PATH + cd deepflame-dev + . configure.sh --amgx_dir /your/path/to/AMGX/ + source ./bashrc + . install.sh +Also, you will need to add configuration files for AMGX for each euqation under ``system`` folder and name them in the pattern of ``amgxpOptions``, ``amgxUOptions`` . Please refer to the AMGX official website to find out detailed instructions. +**If you have compiled DeepFlame with GPU solver successfully, you should see the print message in your terminal:** +.. code-block:: -If you want to use AMGX, you will need to add configuration files for AMGX for each euqation under ``system`` folder and name them in the pattern of ``amgxpOptions``, ``amgxUOptions`` . Please refer to the AMGX official website to find out detailed instructions. + = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = + | deepflame (linked with libcantera) compiled successfully! Enjoy!! | + | select the GPU solver coupled with AMGx library to solve PDE | + = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = **4. If you wish to install DeepFlame with CMake** From 416b4a540152f83b408cfb4e9a7c63f7e44cfbfe Mon Sep 17 00:00:00 2001 From: maorz1998 Date: Wed, 28 Jun 2023 17:53:57 +0800 Subject: [PATCH 3/4] modify docs --- docs/source/qs/install.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/qs/install.rst b/docs/source/qs/install.rst index 8bf129840..4235ae50f 100644 --- a/docs/source/qs/install.rst +++ b/docs/source/qs/install.rst @@ -154,7 +154,9 @@ To begin, you will need to install AMGX. You can find the instructions for insta . install.sh Also, you will need to add configuration files for AMGX for each euqation under ``system`` folder and name them in the pattern of ``amgxpOptions``, ``amgxUOptions`` . Please refer to the AMGX official website to find out detailed instructions. + **If you have compiled DeepFlame with GPU solver successfully, you should see the print message in your terminal:** + .. code-block:: = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = From 6c53c42856010221ae03e6caf5dc361d0bfd410b Mon Sep 17 00:00:00 2001 From: Runze Date: Wed, 28 Jun 2023 17:55:15 +0800 Subject: [PATCH 4/4] Update install.rst --- docs/source/qs/install.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/qs/install.rst b/docs/source/qs/install.rst index 4235ae50f..cd0f1ec96 100644 --- a/docs/source/qs/install.rst +++ b/docs/source/qs/install.rst @@ -144,6 +144,8 @@ If ``df-notorch`` not activated (or you have a self-complied libcantera), specif **3. If you wish to employ dfMatrix and the AMGX library for accelerating PDE solving using GPU:** +.. Note:: This is still under developement. + To begin, you will need to install AMGX. You can find the instructions for installing AMGX on its official website. Follow the instructions provided to install AMGX on your system. Once you have installed AMGX, navigate to the DeepFlame directory and follow the commands below. .. code-block:: bash