Yr/see amgx

applications/solvers/dfLowMachFoam/dfLowMachFoam.C

@@ -32,7 +32,7 @@ Description
     pseudo-transient simulations.
 
 \*---------------------------------------------------------------------------*/
-
+#include "stdlib.h"
 #include "dfChemistryModel.H"
 #include "CanteraMixture.H"
 // #include "hePsiThermo.H"
@@ -62,7 +62,8 @@ Description
 
 #define GPUSolverNew_
 #define TIME
-// #define DEBUG_ 
+// #define DEBUG_
+#define SHOW_MEMINFO
 
 #include "dfMatrixDataBase.H"
 
@@ -452,8 +453,6 @@ int main(int argc, char *argv[])
             }
             end = std::clock();
             time_monitor_turbulence_correct += double(end - start) / double(CLOCKS_PER_SEC);
-            //fprintf(stderr, "sleep for 5s...\n");
-            //usleep(5 * 1000 * 1000);
         }
         clock_t loop_end = std::clock();
         double loop_time = double(loop_end - loop_start) / double(CLOCKS_PER_SEC);
@@ -536,6 +535,19 @@ int main(int argc, char *argv[])
         Info<< "ExecutionTime = " << runTime.elapsedCpuTime() << " s"
             << "  ClockTime = " << runTime.elapsedClockTime() << " s" << endl;
 
+#ifdef GPUSolverNew_
+#ifdef SHOW_MEMINFO
+	int rank = -1;
+	if (mpi_init_flag) {
+    	    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+	}
+	if (!mpi_init_flag || rank == 0) {
+            fprintf(stderr, "show memory info...\n");
+            //usleep(1 * 1000 * 1000);
+	    system("nvidia-smi");
+	}
+#endif
+#endif
         time_monitor_other = 0;
         time_monitor_rho = 0;
         time_monitor_U = 0;
@@ -622,6 +634,7 @@ int main(int argc, char *argv[])
     UEqn_GPU.cleanCudaResources();
     rhoEqn_GPU.cleanCudaResources();
     thermo_GPU.cleanCudaResources();
+    dfDataBase.resetAmgxSolvers();
     dfDataBase.cleanCudaResources();
 #endif
 

src_gpu/AmgXSolver.cu

@@ -335,7 +335,7 @@ void AmgXSolver::solve(
     getIters(nIters);
     getResidual(nIters, rnorm);
     if (!isMPIEnabled || myRank == 0)
-    printf("Initial residual = %.10lf, Final residual = %.5e, No Iterations %d\n", irnorm, rnorm, nIters);
+        fprintf(stderr, "Initial residual = %.10lf, Final residual = %.5e, No Iterations %d\n", irnorm, rnorm, nIters);
 
 }
 

src_gpu/dfMatrixDataBase.H

@@ -294,6 +294,7 @@ struct dfMatrixDataBase
     void setConstantIndexes(const int *owner, const int *neighbor, const int *procRows, 
             const int *procCols, int globalOffset);
     void setAmgxSolvers(const std::string &mode_string, const std::string &u_setting_path, const std::string &p_setting_path);
+    void resetAmgxSolvers();
     void solve(int num_iteration, AMGXSetting setting, double *d_A, double *d_x, double *d_b);
     void setCyclicInfo(std::vector<int> &cyclicNeighbor); // when use cyclic boundary
 

src_gpu/dfMatrixDataBase.cu

@@ -151,6 +151,17 @@ void dfMatrixDataBase::setAmgxSolvers(const std::string &mode_string, const std:
     u_setting_solver = new AmgXSolver(mode_string, u_setting_path, localRank);
     p_setting_solver = new AmgXSolver(mode_string, p_setting_path, localRank);
 }
+
+void dfMatrixDataBase::resetAmgxSolvers() {
+    if (u_setting_solver) {
+        delete u_setting_solver;
+        u_setting_solver = nullptr;
+    }
+    if (p_setting_solver) {
+        delete p_setting_solver;
+        p_setting_solver = nullptr;
+    }
+}
 
 void dfMatrixDataBase::solve(int num_iteration, AMGXSetting setting, double *d_A, double *d_x, double *d_b) {
     AmgXSolver *solver = (setting == AMGXSetting::u_setting) ? u_setting_solver : p_setting_solver;