From 4355f4d677aff3529762d5baee68e6a9f5ed9f37 Mon Sep 17 00:00:00 2001 From: Wang Yingrui Date: Tue, 31 Oct 2023 17:42:46 +0800 Subject: [PATCH 1/2] reset amgx solver to avoid memleak --- applications/solvers/dfLowMachFoam/dfLowMachFoam.C | 3 ++- src_gpu/AmgXSolver.cu | 2 +- src_gpu/dfMatrixDataBase.H | 1 + src_gpu/dfMatrixDataBase.cu | 11 +++++++++++ 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/applications/solvers/dfLowMachFoam/dfLowMachFoam.C b/applications/solvers/dfLowMachFoam/dfLowMachFoam.C index 5f3c7aea1..7e6dfa2a8 100644 --- a/applications/solvers/dfLowMachFoam/dfLowMachFoam.C +++ b/applications/solvers/dfLowMachFoam/dfLowMachFoam.C @@ -62,7 +62,7 @@ Description #define GPUSolverNew_ #define TIME -// #define DEBUG_ +// #define DEBUG_ #include "dfMatrixDataBase.H" @@ -622,6 +622,7 @@ int main(int argc, char *argv[]) UEqn_GPU.cleanCudaResources(); rhoEqn_GPU.cleanCudaResources(); thermo_GPU.cleanCudaResources(); + dfDataBase.resetAmgxSolvers(); dfDataBase.cleanCudaResources(); #endif diff --git a/src_gpu/AmgXSolver.cu b/src_gpu/AmgXSolver.cu index 8658a268b..c891a5980 100644 --- a/src_gpu/AmgXSolver.cu +++ b/src_gpu/AmgXSolver.cu @@ -335,7 +335,7 @@ void AmgXSolver::solve( getIters(nIters); getResidual(nIters, rnorm); if (!isMPIEnabled || myRank == 0) - printf("Initial residual = %.10lf, Final residual = %.5e, No Iterations %d\n", irnorm, rnorm, nIters); + fprintf(stderr, "Initial residual = %.10lf, Final residual = %.5e, No Iterations %d\n", irnorm, rnorm, nIters); } diff --git a/src_gpu/dfMatrixDataBase.H b/src_gpu/dfMatrixDataBase.H index 40cf32bae..2c2e7d301 100644 --- a/src_gpu/dfMatrixDataBase.H +++ b/src_gpu/dfMatrixDataBase.H @@ -294,6 +294,7 @@ struct dfMatrixDataBase void setConstantIndexes(const int *owner, const int *neighbor, const int *procRows, const int *procCols, int globalOffset); void setAmgxSolvers(const std::string &mode_string, const std::string &u_setting_path, const std::string &p_setting_path); + void resetAmgxSolvers(); void solve(int num_iteration, AMGXSetting setting, double *d_A, double *d_x, double *d_b); void setCyclicInfo(std::vector &cyclicNeighbor); // when use cyclic boundary diff --git a/src_gpu/dfMatrixDataBase.cu b/src_gpu/dfMatrixDataBase.cu index fd87cb46a..0623dbf7e 100644 --- a/src_gpu/dfMatrixDataBase.cu +++ b/src_gpu/dfMatrixDataBase.cu @@ -151,6 +151,17 @@ void dfMatrixDataBase::setAmgxSolvers(const std::string &mode_string, const std: u_setting_solver = new AmgXSolver(mode_string, u_setting_path, localRank); p_setting_solver = new AmgXSolver(mode_string, p_setting_path, localRank); } + +void dfMatrixDataBase::resetAmgxSolvers() { + if (u_setting_solver) { + delete u_setting_solver; + u_setting_solver = nullptr; + } + if (p_setting_solver) { + delete p_setting_solver; + p_setting_solver = nullptr; + } +} void dfMatrixDataBase::solve(int num_iteration, AMGXSetting setting, double *d_A, double *d_x, double *d_b) { AmgXSolver *solver = (setting == AMGXSetting::u_setting) ? u_setting_solver : p_setting_solver; From ed15f81188b16e5e90c433f89d1357ef959812fa Mon Sep 17 00:00:00 2001 From: Wang Yingrui Date: Tue, 31 Oct 2023 17:43:36 +0800 Subject: [PATCH 2/2] checking memory usage info after each timestep --- .../solvers/dfLowMachFoam/dfLowMachFoam.C | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/applications/solvers/dfLowMachFoam/dfLowMachFoam.C b/applications/solvers/dfLowMachFoam/dfLowMachFoam.C index 7e6dfa2a8..c8a910dda 100644 --- a/applications/solvers/dfLowMachFoam/dfLowMachFoam.C +++ b/applications/solvers/dfLowMachFoam/dfLowMachFoam.C @@ -32,7 +32,7 @@ Description pseudo-transient simulations. \*---------------------------------------------------------------------------*/ - +#include "stdlib.h" #include "dfChemistryModel.H" #include "CanteraMixture.H" // #include "hePsiThermo.H" @@ -63,6 +63,7 @@ Description #define GPUSolverNew_ #define TIME // #define DEBUG_ +#define SHOW_MEMINFO #include "dfMatrixDataBase.H" @@ -452,8 +453,6 @@ int main(int argc, char *argv[]) } end = std::clock(); time_monitor_turbulence_correct += double(end - start) / double(CLOCKS_PER_SEC); - //fprintf(stderr, "sleep for 5s...\n"); - //usleep(5 * 1000 * 1000); } clock_t loop_end = std::clock(); double loop_time = double(loop_end - loop_start) / double(CLOCKS_PER_SEC); @@ -536,6 +535,19 @@ int main(int argc, char *argv[]) Info<< "ExecutionTime = " << runTime.elapsedCpuTime() << " s" << " ClockTime = " << runTime.elapsedClockTime() << " s" << endl; +#ifdef GPUSolverNew_ +#ifdef SHOW_MEMINFO + int rank = -1; + if (mpi_init_flag) { + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + } + if (!mpi_init_flag || rank == 0) { + fprintf(stderr, "show memory info...\n"); + //usleep(1 * 1000 * 1000); + system("nvidia-smi"); + } +#endif +#endif time_monitor_other = 0; time_monitor_rho = 0; time_monitor_U = 0;