From 299f25626fab0c6933b8acf2c511abc1e25a6e04 Mon Sep 17 00:00:00 2001 From: Guanlin Date: Fri, 1 Mar 2024 17:46:07 +0800 Subject: [PATCH] ODE GPU solver is accessed by DeepFlame! --- .../solvers/dfLowMachFoam/Make/options | 7 +- applications/solvers/dfLowMachFoam/YEqn.H | 55 ++++++++++++++ .../solvers/dfLowMachFoam/createFields_GPU.H | 71 +++++++++++++++++++ .../solvers/dfLowMachFoam/dfLowMachFoam.C | 8 +++ bashrc.in | 3 +- configure.sh | 14 +++- 6 files changed, 153 insertions(+), 5 deletions(-) create mode 100755 applications/solvers/dfLowMachFoam/createFields_GPU.H diff --git a/applications/solvers/dfLowMachFoam/Make/options b/applications/solvers/dfLowMachFoam/Make/options index 668a3133a..a0dc9ba1c 100644 --- a/applications/solvers/dfLowMachFoam/Make/options +++ b/applications/solvers/dfLowMachFoam/Make/options @@ -28,7 +28,9 @@ EXE_INC = -std=c++14 \ $(PYTHON_INC_DIR) \ $(if $(AMGX_DIR), -I$(DF_ROOT)/src_gpu,) \ $(if $(AMGX_DIR), -I/usr/local/cuda/include,) \ - $(if $(AMGX_DIR), -I$(AMGX_DIR)/include,) + $(if $(AMGX_DIR), -I$(AMGX_DIR)/include,) \ + $(if $(ODE_GPU_SOLVER), -I$(OPENCC_PATH)/include,) \ + $(if $(ODE_GPU_SOLVER), -DODE_GPU_SOLVER,) EXE_LIBS = \ -lcompressibleTransportModels \ @@ -52,4 +54,5 @@ EXE_LIBS = \ $(if $(AMGX_DIR), /usr/local/cuda/lib64/libcudart.so,) \ $(if $(AMGX_DIR), /usr/local/cuda/lib64/libnccl.so,) \ $(if $(AMGX_DIR), $(DF_ROOT)/src_gpu/build/libdfMatrix.so,) \ - $(if $(AMGX_DIR), $(AMGX_DIR)/build/libamgxsh.so,) \ No newline at end of file + $(if $(AMGX_DIR), $(AMGX_DIR)/build/libamgxsh.so,) \ + $(if $(ODE_GPU_SOLVER), $(ODE_GPU_SOLVER)/lib/libopencc.so,) diff --git a/applications/solvers/dfLowMachFoam/YEqn.H b/applications/solvers/dfLowMachFoam/YEqn.H index 2120408d0..01dd085ba 100644 --- a/applications/solvers/dfLowMachFoam/YEqn.H +++ b/applications/solvers/dfLowMachFoam/YEqn.H @@ -85,6 +85,24 @@ if(flag_mpi_init) MPI_Barrier(PstreamGlobals::MPI_COMM_FOAM); start1 = std::clock(); tmp DEff = chemistry->rhoD(i) + turbulence->mut()/Sct; + #ifdef ODE_GPU_SOLVER + fvScalarMatrix YiEqn + ( + fvm::ddt(rho, Yi) + + + ( + turbName == "laminar" + ? (mvConvection->fvmDiv(phi, Yi) + mvConvection->fvmDiv(phiUc, Yi)) + : mvConvection->fvmDiv(phi, Yi) + ) + == + ( + splitting + ? fvm::laplacian(DEff(), Yi) + : (fvm::laplacian(DEff(), Yi) + RR_GPU[i]) + ) + ); + #else fvScalarMatrix YiEqn ( fvm::ddt(rho, Yi) @@ -101,6 +119,7 @@ if(flag_mpi_init) MPI_Barrier(PstreamGlobals::MPI_COMM_FOAM); : (fvm::laplacian(DEff(), Yi) + combustion->R(Yi)) ) ); + #endif end1 = std::clock(); time_monitor_YEqn_mtxAssembly += double(end1 - start1) / double(CLOCKS_PER_SEC); @@ -264,7 +283,43 @@ if(flag_mpi_init) MPI_Barrier(PstreamGlobals::MPI_COMM_FOAM); if (!splitting) { std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now(); + + #ifdef ODE_GPU_SOLVER + scalar dt = runTime.deltaTValue(); + + memcpy(h_T, &T[0], num_cells * sizeof(double)); + memcpy(h_p, &p[0], num_cells * sizeof(double)); + + forAll(Y, speciesI) { + volScalarField& Yi = Y[speciesI]; + memcpy(h_y + speciesI * num_cells, &Yi[0], num_cells * sizeof(double)); + } + + for (int i = 0; i < num_cells; i++) { + for (int j = 0; j < sp_num; j++) { + h_y_t[j + i*sp_num] = h_y[i + j*num_cells]; + } + } + + opencc_ode_all(h_T, h_p, h_y_t, 1e-10, dt, CPU); + + for (int i = 0; i < num_cells; i++) { + for (int j = 0; j < sp_num; j++) { + Ynew[i + j*num_cells] = h_y_t[j + i*sp_num]; + } + } + + QdotGPU = Zero; + forAll(QdotGPU,celli) + { + for (int sp = 0; sp < sp_num; sp++) + { + RRGPU[sp][celli] = (Ynew[sp*num_cells+celli]-Y[sp][celli])*rho[celli]/dt; + } + } + #else combustion->correct(); + #endif //label flag_mpi_init; //MPI_Initialized(&flag_mpi_init); if(flag_mpi_init) MPI_Barrier(PstreamGlobals::MPI_COMM_FOAM); diff --git a/applications/solvers/dfLowMachFoam/createFields_GPU.H b/applications/solvers/dfLowMachFoam/createFields_GPU.H new file mode 100755 index 000000000..fc115f80f --- /dev/null +++ b/applications/solvers/dfLowMachFoam/createFields_GPU.H @@ -0,0 +1,71 @@ +/******************************************************************************************* + * This file is used to add OpenCC calls to OpenFoam, + * initialize OpenCC scopes and request GPU space. + * + * @author Lynn Dang + ******************************************************************************************/ + +volScalarField QdotGPU +( + IOobject + ( + "QdotGPU", + mesh.time().timeName(), + mesh, + IOobject::READ_IF_PRESENT, + IOobject::AUTO_WRITE + ), + mesh, + dimensionedScalar(dimEnergy/dimVolume/dimTime, 0) +); + +PtrList RRGPU(Y.size()); +forAll(RRGPU, fieldi) +{ + RRGPU.set + ( + fieldi, + new volScalarField::Internal + ( + IOobject + ( + "RRGPU." + Y[fieldi].name(), + mesh.time().timeName(), + mesh, + IOobject::NO_READ, + IOobject::NO_WRITE + ), + mesh, + dimensionedScalar(dimMass/dimVolume/dimTime, 0) + ) + ); +} + +int num_cells = T.size(); +int num_species = Y.size(); + +double* h_T = new double[num_cells]; +double* h_p = new double[num_cells]; +double* h_y = new double[num_cells * num_species]; +double* h_y_t = new double[num_cells * num_species]; +int* h_size = new int[1]; + +memcpy(h_T, &T[0], num_cells * sizeof(double)); +memcpy(h_p, &p[0], num_cells * sizeof(double)); + +forAll(Y, speciesI) { + volScalarField& Yi = Y[speciesI]; + memcpy(h_y + speciesI * num_cells, &Yi[0], num_cells * sizeof(double)); +} + +h_size[0] = num_cells; + +int sp_num = num_species; + +string mechanismFile = CanteraTorchProperties.lookupOrDefault("CanteraMechanismFile", string("")); +char target_mechanismFile[100]; +std::strcpy(target_mechanismFile, mechanismFile.c_str()); + +opencc_ode_init(target_mechanismFile, num_cells, h_T, h_p, h_y); + +double* Ynew = new double[num_cells * num_species]; diff --git a/applications/solvers/dfLowMachFoam/dfLowMachFoam.C b/applications/solvers/dfLowMachFoam/dfLowMachFoam.C index 6191067e4..7217798d6 100644 --- a/applications/solvers/dfLowMachFoam/dfLowMachFoam.C +++ b/applications/solvers/dfLowMachFoam/dfLowMachFoam.C @@ -49,6 +49,10 @@ Description #include "DNNInferencer.H" #endif +#ifdef ODE_GPU_SOLVER +#include "opencc.h" +#endif + #include "fvCFD.H" #include "fluidThermo.H" #include "turbulentFluidThermoModel.H" @@ -137,6 +141,10 @@ int main(int argc, char *argv[]) #include "createFields.H" #include "createRhoUfIfPresent.H" + #ifdef ODE_GPU_SOLVER + #include "createFields_GPU.H" + #endif + double time_monitor_init = 0; double time_monitor_other = 0; diff --git a/bashrc.in b/bashrc.in index ac4e7d398..972424872 100644 --- a/bashrc.in +++ b/bashrc.in @@ -9,6 +9,7 @@ export CANTERA_DATA=$CANTERA_ROOT/share/cantera/data export LD_LIBRARY_PATH=$LIBTORCH_ROOT/lib:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=$CANTERA_ROOT/lib:$LD_LIBRARY_PATH export AMGX_DIR=@AMGX_DIR@ +export ODE_GPU_SOLVER=@ODE_GPU_SOLVER@ export DF_APPBIN=pwd/platforms/$WM_OPTIONS/bin export DF_LIBBIN=pwd/platforms/$WM_OPTIONS/lib @@ -16,4 +17,4 @@ export PATH=$DF_APPBIN:$PATH export LD_LIBRARY_PATH=$DF_LIBBIN:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=$DF_ROOT/src_gpu/build:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=$AMGX_DIR/build:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=$DF_ROOT/src/dfChemistryModel/DNNInferencer/build:$LD_LIBRARY_PATH \ No newline at end of file +export LD_LIBRARY_PATH=$DF_ROOT/src/dfChemistryModel/DNNInferencer/build:$LD_LIBRARY_PATH diff --git a/configure.sh b/configure.sh index 7cfbac845..b611545a5 100644 --- a/configure.sh +++ b/configure.sh @@ -9,10 +9,11 @@ unset PYTORCH_INC unset PYTORCH_LIB unset USE_GPUSOLVER unset AMGX_DIR +unset ODE_GPU_SOLVER print_usage() { echo "Usage: . install.sh --libtorch_no (default) | --libtorch_dir _path_to_libtorch | --libtorch_autodownload | --use_pytorch | --libcantera_dir _path_to_libcantera - | --amgx_dir _path_to_amgx" + | --amgx_dir _path_to_amgx | --use_ode_gpu_solver" } # default @@ -20,6 +21,7 @@ LIBTORCH_AUTO=false USE_LIBTORCH=false USE_PYTORCH=false USE_GPUSOLVER=false +USE_ODE_GPU_SOLVER=false while test $# -gt 0; do case "$1" in @@ -75,6 +77,10 @@ while test $# -gt 0; do fi shift ;; + --use_ode_gpu_solver) + shift + USE_ODE_GPU_SOLVER=true + ;; -h|--help) shift print_usage @@ -163,6 +169,9 @@ fi if [ $USE_GPUSOLVER = true ]; then echo AMGX_DIR=$AMGX_DIR fi +if [ $USE_ODE_GPU_SOLVER = true ]; then + echo ODE_GPU_SOLVER=$OPENCC_PATH +fi cp bashrc.in bashrc sed -i "s#pwd#$PWD#g" ./bashrc @@ -171,6 +180,7 @@ sed -i "s#PYTORCH_INC#$PYTORCH_INC#g" ./bashrc sed -i "s#PYTORCH_LIB#$PYTORCH_LIB#g" ./bashrc sed -i "s#LIBCANTERA_DIR#$LIBCANTERA_DIR#g" ./bashrc sed -i "s#@AMGX_DIR@#$AMGX_DIR#g" ./bashrc +sed -i "s#@ODE_GPU_SOLVER@#$OPENCC_PATH#g" ./bashrc @@ -190,4 +200,4 @@ else cp -r $FOAM_SRC/lagrangian/turbulence src_orig/lagrangian cp -r $FOAM_SRC/regionModels/surfaceFilmModels src_orig/regionModels cp -r $FOAM_SRC/functionObjects/field src_orig/functionObjects -fi \ No newline at end of file +fi